jsonpb.go 31 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082
  1. // Go support for Protocol Buffers - Google's data interchange format
  2. //
  3. // Copyright 2015 The Go Authors. All rights reserved.
  4. // https://github.com/golang/protobuf
  5. //
  6. // Redistribution and use in source and binary forms, with or without
  7. // modification, are permitted provided that the following conditions are
  8. // met:
  9. //
  10. // * Redistributions of source code must retain the above copyright
  11. // notice, this list of conditions and the following disclaimer.
  12. // * Redistributions in binary form must reproduce the above
  13. // copyright notice, this list of conditions and the following disclaimer
  14. // in the documentation and/or other materials provided with the
  15. // distribution.
  16. // * Neither the name of Google Inc. nor the names of its
  17. // contributors may be used to endorse or promote products derived from
  18. // this software without specific prior written permission.
  19. //
  20. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31. /*
  32. Package jsonpb provides marshaling and unmarshaling between protocol buffers and JSON.
  33. It follows the specification at https://developers.google.com/protocol-buffers/docs/proto3#json.
  34. This package produces a different output than the standard "encoding/json" package,
  35. which does not operate correctly on protocol buffers.
  36. */
  37. package jsonpb
  38. import (
  39. "bytes"
  40. "encoding/json"
  41. "errors"
  42. "fmt"
  43. "io"
  44. "math"
  45. "reflect"
  46. "sort"
  47. "strconv"
  48. "strings"
  49. "time"
  50. "github.com/golang/protobuf/proto"
  51. stpb "github.com/golang/protobuf/ptypes/struct"
  52. )
  53. // Marshaler is a configurable object for converting between
  54. // protocol buffer objects and a JSON representation for them.
  55. type Marshaler struct {
  56. // Whether to render enum values as integers, as opposed to string values.
  57. EnumsAsInts bool
  58. // Whether to render fields with zero values.
  59. EmitDefaults bool
  60. // A string to indent each level by. The presence of this field will
  61. // also cause a space to appear between the field separator and
  62. // value, and for newlines to be appear between fields and array
  63. // elements.
  64. Indent string
  65. // Whether to use the original (.proto) name for fields.
  66. OrigName bool
  67. // A custom URL resolver to use when marshaling Any messages to JSON.
  68. // If unset, the default resolution strategy is to extract the
  69. // fully-qualified type name from the type URL and pass that to
  70. // proto.MessageType(string).
  71. AnyResolver AnyResolver
  72. }
  73. // AnyResolver takes a type URL, present in an Any message, and resolves it into
  74. // an instance of the associated message.
  75. type AnyResolver interface {
  76. Resolve(typeUrl string) (proto.Message, error)
  77. }
  78. func defaultResolveAny(typeUrl string) (proto.Message, error) {
  79. // Only the part of typeUrl after the last slash is relevant.
  80. mname := typeUrl
  81. if slash := strings.LastIndex(mname, "/"); slash >= 0 {
  82. mname = mname[slash+1:]
  83. }
  84. mt := proto.MessageType(mname)
  85. if mt == nil {
  86. return nil, fmt.Errorf("unknown message type %q", mname)
  87. }
  88. return reflect.New(mt.Elem()).Interface().(proto.Message), nil
  89. }
  90. // JSONPBMarshaler is implemented by protobuf messages that customize the
  91. // way they are marshaled to JSON. Messages that implement this should
  92. // also implement JSONPBUnmarshaler so that the custom format can be
  93. // parsed.
  94. type JSONPBMarshaler interface {
  95. MarshalJSONPB(*Marshaler) ([]byte, error)
  96. }
  97. // JSONPBUnmarshaler is implemented by protobuf messages that customize
  98. // the way they are unmarshaled from JSON. Messages that implement this
  99. // should also implement JSONPBMarshaler so that the custom format can be
  100. // produced.
  101. type JSONPBUnmarshaler interface {
  102. UnmarshalJSONPB(*Unmarshaler, []byte) error
  103. }
  104. // Marshal marshals a protocol buffer into JSON.
  105. func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error {
  106. writer := &errWriter{writer: out}
  107. return m.marshalObject(writer, pb, "", "")
  108. }
  109. // MarshalToString converts a protocol buffer object to JSON string.
  110. func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) {
  111. var buf bytes.Buffer
  112. if err := m.Marshal(&buf, pb); err != nil {
  113. return "", err
  114. }
  115. return buf.String(), nil
  116. }
  117. type int32Slice []int32
  118. var nonFinite = map[string]float64{
  119. `"NaN"`: math.NaN(),
  120. `"Infinity"`: math.Inf(1),
  121. `"-Infinity"`: math.Inf(-1),
  122. }
  123. // For sorting extensions ids to ensure stable output.
  124. func (s int32Slice) Len() int { return len(s) }
  125. func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] }
  126. func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  127. type wkt interface {
  128. XXX_WellKnownType() string
  129. }
  130. // marshalObject writes a struct to the Writer.
  131. func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error {
  132. if jsm, ok := v.(JSONPBMarshaler); ok {
  133. b, err := jsm.MarshalJSONPB(m)
  134. if err != nil {
  135. return err
  136. }
  137. if typeURL != "" {
  138. // we are marshaling this object to an Any type
  139. var js map[string]*json.RawMessage
  140. if err = json.Unmarshal(b, &js); err != nil {
  141. return fmt.Errorf("type %T produced invalid JSON: %v", v, err)
  142. }
  143. turl, err := json.Marshal(typeURL)
  144. if err != nil {
  145. return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err)
  146. }
  147. js["@type"] = (*json.RawMessage)(&turl)
  148. if b, err = json.Marshal(js); err != nil {
  149. return err
  150. }
  151. }
  152. out.write(string(b))
  153. return out.err
  154. }
  155. s := reflect.ValueOf(v).Elem()
  156. // Handle well-known types.
  157. if wkt, ok := v.(wkt); ok {
  158. switch wkt.XXX_WellKnownType() {
  159. case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
  160. "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
  161. // "Wrappers use the same representation in JSON
  162. // as the wrapped primitive type, ..."
  163. sprop := proto.GetProperties(s.Type())
  164. return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent)
  165. case "Any":
  166. // Any is a bit more involved.
  167. return m.marshalAny(out, v, indent)
  168. case "Duration":
  169. // "Generated output always contains 3, 6, or 9 fractional digits,
  170. // depending on required precision."
  171. s, ns := s.Field(0).Int(), s.Field(1).Int()
  172. x := fmt.Sprintf("%d.%09d", s, ns)
  173. x = strings.TrimSuffix(x, "000")
  174. x = strings.TrimSuffix(x, "000")
  175. out.write(`"`)
  176. out.write(x)
  177. out.write(`s"`)
  178. return out.err
  179. case "Struct", "ListValue":
  180. // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice.
  181. // TODO: pass the correct Properties if needed.
  182. return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent)
  183. case "Timestamp":
  184. // "RFC 3339, where generated output will always be Z-normalized
  185. // and uses 3, 6 or 9 fractional digits."
  186. s, ns := s.Field(0).Int(), s.Field(1).Int()
  187. t := time.Unix(s, ns).UTC()
  188. // time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits).
  189. x := t.Format("2006-01-02T15:04:05.000000000")
  190. x = strings.TrimSuffix(x, "000")
  191. x = strings.TrimSuffix(x, "000")
  192. out.write(`"`)
  193. out.write(x)
  194. out.write(`Z"`)
  195. return out.err
  196. case "Value":
  197. // Value has a single oneof.
  198. kind := s.Field(0)
  199. if kind.IsNil() {
  200. // "absence of any variant indicates an error"
  201. return errors.New("nil Value")
  202. }
  203. // oneof -> *T -> T -> T.F
  204. x := kind.Elem().Elem().Field(0)
  205. // TODO: pass the correct Properties if needed.
  206. return m.marshalValue(out, &proto.Properties{}, x, indent)
  207. }
  208. }
  209. out.write("{")
  210. if m.Indent != "" {
  211. out.write("\n")
  212. }
  213. firstField := true
  214. if typeURL != "" {
  215. if err := m.marshalTypeURL(out, indent, typeURL); err != nil {
  216. return err
  217. }
  218. firstField = false
  219. }
  220. for i := 0; i < s.NumField(); i++ {
  221. value := s.Field(i)
  222. valueField := s.Type().Field(i)
  223. if strings.HasPrefix(valueField.Name, "XXX_") {
  224. continue
  225. }
  226. // IsNil will panic on most value kinds.
  227. switch value.Kind() {
  228. case reflect.Chan, reflect.Func, reflect.Interface:
  229. if value.IsNil() {
  230. continue
  231. }
  232. }
  233. if !m.EmitDefaults {
  234. switch value.Kind() {
  235. case reflect.Bool:
  236. if !value.Bool() {
  237. continue
  238. }
  239. case reflect.Int32, reflect.Int64:
  240. if value.Int() == 0 {
  241. continue
  242. }
  243. case reflect.Uint32, reflect.Uint64:
  244. if value.Uint() == 0 {
  245. continue
  246. }
  247. case reflect.Float32, reflect.Float64:
  248. if value.Float() == 0 {
  249. continue
  250. }
  251. case reflect.String:
  252. if value.Len() == 0 {
  253. continue
  254. }
  255. case reflect.Map, reflect.Ptr, reflect.Slice:
  256. if value.IsNil() {
  257. continue
  258. }
  259. }
  260. }
  261. // Oneof fields need special handling.
  262. if valueField.Tag.Get("protobuf_oneof") != "" {
  263. // value is an interface containing &T{real_value}.
  264. sv := value.Elem().Elem() // interface -> *T -> T
  265. value = sv.Field(0)
  266. valueField = sv.Type().Field(0)
  267. }
  268. prop := jsonProperties(valueField, m.OrigName)
  269. if !firstField {
  270. m.writeSep(out)
  271. }
  272. if err := m.marshalField(out, prop, value, indent); err != nil {
  273. return err
  274. }
  275. firstField = false
  276. }
  277. // Handle proto2 extensions.
  278. if ep, ok := v.(proto.Message); ok {
  279. extensions := proto.RegisteredExtensions(v)
  280. // Sort extensions for stable output.
  281. ids := make([]int32, 0, len(extensions))
  282. for id, desc := range extensions {
  283. if !proto.HasExtension(ep, desc) {
  284. continue
  285. }
  286. ids = append(ids, id)
  287. }
  288. sort.Sort(int32Slice(ids))
  289. for _, id := range ids {
  290. desc := extensions[id]
  291. if desc == nil {
  292. // unknown extension
  293. continue
  294. }
  295. ext, extErr := proto.GetExtension(ep, desc)
  296. if extErr != nil {
  297. return extErr
  298. }
  299. value := reflect.ValueOf(ext)
  300. var prop proto.Properties
  301. prop.Parse(desc.Tag)
  302. prop.JSONName = fmt.Sprintf("[%s]", desc.Name)
  303. if !firstField {
  304. m.writeSep(out)
  305. }
  306. if err := m.marshalField(out, &prop, value, indent); err != nil {
  307. return err
  308. }
  309. firstField = false
  310. }
  311. }
  312. if m.Indent != "" {
  313. out.write("\n")
  314. out.write(indent)
  315. }
  316. out.write("}")
  317. return out.err
  318. }
  319. func (m *Marshaler) writeSep(out *errWriter) {
  320. if m.Indent != "" {
  321. out.write(",\n")
  322. } else {
  323. out.write(",")
  324. }
  325. }
  326. func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error {
  327. // "If the Any contains a value that has a special JSON mapping,
  328. // it will be converted as follows: {"@type": xxx, "value": yyy}.
  329. // Otherwise, the value will be converted into a JSON object,
  330. // and the "@type" field will be inserted to indicate the actual data type."
  331. v := reflect.ValueOf(any).Elem()
  332. turl := v.Field(0).String()
  333. val := v.Field(1).Bytes()
  334. var msg proto.Message
  335. var err error
  336. if m.AnyResolver != nil {
  337. msg, err = m.AnyResolver.Resolve(turl)
  338. } else {
  339. msg, err = defaultResolveAny(turl)
  340. }
  341. if err != nil {
  342. return err
  343. }
  344. if err := proto.Unmarshal(val, msg); err != nil {
  345. return err
  346. }
  347. if _, ok := msg.(wkt); ok {
  348. out.write("{")
  349. if m.Indent != "" {
  350. out.write("\n")
  351. }
  352. if err := m.marshalTypeURL(out, indent, turl); err != nil {
  353. return err
  354. }
  355. m.writeSep(out)
  356. if m.Indent != "" {
  357. out.write(indent)
  358. out.write(m.Indent)
  359. out.write(`"value": `)
  360. } else {
  361. out.write(`"value":`)
  362. }
  363. if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil {
  364. return err
  365. }
  366. if m.Indent != "" {
  367. out.write("\n")
  368. out.write(indent)
  369. }
  370. out.write("}")
  371. return out.err
  372. }
  373. return m.marshalObject(out, msg, indent, turl)
  374. }
  375. func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error {
  376. if m.Indent != "" {
  377. out.write(indent)
  378. out.write(m.Indent)
  379. }
  380. out.write(`"@type":`)
  381. if m.Indent != "" {
  382. out.write(" ")
  383. }
  384. b, err := json.Marshal(typeURL)
  385. if err != nil {
  386. return err
  387. }
  388. out.write(string(b))
  389. return out.err
  390. }
  391. // marshalField writes field description and value to the Writer.
  392. func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
  393. if m.Indent != "" {
  394. out.write(indent)
  395. out.write(m.Indent)
  396. }
  397. out.write(`"`)
  398. out.write(prop.JSONName)
  399. out.write(`":`)
  400. if m.Indent != "" {
  401. out.write(" ")
  402. }
  403. if err := m.marshalValue(out, prop, v, indent); err != nil {
  404. return err
  405. }
  406. return nil
  407. }
  408. // marshalValue writes the value to the Writer.
  409. func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
  410. var err error
  411. v = reflect.Indirect(v)
  412. // Handle nil pointer
  413. if v.Kind() == reflect.Invalid {
  414. out.write("null")
  415. return out.err
  416. }
  417. // Handle repeated elements.
  418. if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
  419. out.write("[")
  420. comma := ""
  421. for i := 0; i < v.Len(); i++ {
  422. sliceVal := v.Index(i)
  423. out.write(comma)
  424. if m.Indent != "" {
  425. out.write("\n")
  426. out.write(indent)
  427. out.write(m.Indent)
  428. out.write(m.Indent)
  429. }
  430. if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil {
  431. return err
  432. }
  433. comma = ","
  434. }
  435. if m.Indent != "" {
  436. out.write("\n")
  437. out.write(indent)
  438. out.write(m.Indent)
  439. }
  440. out.write("]")
  441. return out.err
  442. }
  443. // Handle well-known types.
  444. // Most are handled up in marshalObject (because 99% are messages).
  445. if wkt, ok := v.Interface().(wkt); ok {
  446. switch wkt.XXX_WellKnownType() {
  447. case "NullValue":
  448. out.write("null")
  449. return out.err
  450. }
  451. }
  452. // Handle enumerations.
  453. if !m.EnumsAsInts && prop.Enum != "" {
  454. // Unknown enum values will are stringified by the proto library as their
  455. // value. Such values should _not_ be quoted or they will be interpreted
  456. // as an enum string instead of their value.
  457. enumStr := v.Interface().(fmt.Stringer).String()
  458. var valStr string
  459. if v.Kind() == reflect.Ptr {
  460. valStr = strconv.Itoa(int(v.Elem().Int()))
  461. } else {
  462. valStr = strconv.Itoa(int(v.Int()))
  463. }
  464. isKnownEnum := enumStr != valStr
  465. if isKnownEnum {
  466. out.write(`"`)
  467. }
  468. out.write(enumStr)
  469. if isKnownEnum {
  470. out.write(`"`)
  471. }
  472. return out.err
  473. }
  474. // Handle nested messages.
  475. if v.Kind() == reflect.Struct {
  476. return m.marshalObject(out, v.Addr().Interface().(proto.Message), indent+m.Indent, "")
  477. }
  478. // Handle maps.
  479. // Since Go randomizes map iteration, we sort keys for stable output.
  480. if v.Kind() == reflect.Map {
  481. out.write(`{`)
  482. keys := v.MapKeys()
  483. sort.Sort(mapKeys(keys))
  484. for i, k := range keys {
  485. if i > 0 {
  486. out.write(`,`)
  487. }
  488. if m.Indent != "" {
  489. out.write("\n")
  490. out.write(indent)
  491. out.write(m.Indent)
  492. out.write(m.Indent)
  493. }
  494. b, err := json.Marshal(k.Interface())
  495. if err != nil {
  496. return err
  497. }
  498. s := string(b)
  499. // If the JSON is not a string value, encode it again to make it one.
  500. if !strings.HasPrefix(s, `"`) {
  501. b, err := json.Marshal(s)
  502. if err != nil {
  503. return err
  504. }
  505. s = string(b)
  506. }
  507. out.write(s)
  508. out.write(`:`)
  509. if m.Indent != "" {
  510. out.write(` `)
  511. }
  512. if err := m.marshalValue(out, prop, v.MapIndex(k), indent+m.Indent); err != nil {
  513. return err
  514. }
  515. }
  516. if m.Indent != "" {
  517. out.write("\n")
  518. out.write(indent)
  519. out.write(m.Indent)
  520. }
  521. out.write(`}`)
  522. return out.err
  523. }
  524. // Handle non-finite floats, e.g. NaN, Infinity and -Infinity.
  525. if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 {
  526. f := v.Float()
  527. var sval string
  528. switch {
  529. case math.IsInf(f, 1):
  530. sval = `"Infinity"`
  531. case math.IsInf(f, -1):
  532. sval = `"-Infinity"`
  533. case math.IsNaN(f):
  534. sval = `"NaN"`
  535. }
  536. if sval != "" {
  537. out.write(sval)
  538. return out.err
  539. }
  540. }
  541. // Default handling defers to the encoding/json library.
  542. b, err := json.Marshal(v.Interface())
  543. if err != nil {
  544. return err
  545. }
  546. needToQuote := string(b[0]) != `"` && (v.Kind() == reflect.Int64 || v.Kind() == reflect.Uint64)
  547. if needToQuote {
  548. out.write(`"`)
  549. }
  550. out.write(string(b))
  551. if needToQuote {
  552. out.write(`"`)
  553. }
  554. return out.err
  555. }
  556. // Unmarshaler is a configurable object for converting from a JSON
  557. // representation to a protocol buffer object.
  558. type Unmarshaler struct {
  559. // Whether to allow messages to contain unknown fields, as opposed to
  560. // failing to unmarshal.
  561. AllowUnknownFields bool
  562. // A custom URL resolver to use when unmarshaling Any messages from JSON.
  563. // If unset, the default resolution strategy is to extract the
  564. // fully-qualified type name from the type URL and pass that to
  565. // proto.MessageType(string).
  566. AnyResolver AnyResolver
  567. }
  568. // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
  569. // This function is lenient and will decode any options permutations of the
  570. // related Marshaler.
  571. func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
  572. inputValue := json.RawMessage{}
  573. if err := dec.Decode(&inputValue); err != nil {
  574. return err
  575. }
  576. return u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil)
  577. }
  578. // Unmarshal unmarshals a JSON object stream into a protocol
  579. // buffer. This function is lenient and will decode any options
  580. // permutations of the related Marshaler.
  581. func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error {
  582. dec := json.NewDecoder(r)
  583. return u.UnmarshalNext(dec, pb)
  584. }
  585. // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
  586. // This function is lenient and will decode any options permutations of the
  587. // related Marshaler.
  588. func UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
  589. return new(Unmarshaler).UnmarshalNext(dec, pb)
  590. }
  591. // Unmarshal unmarshals a JSON object stream into a protocol
  592. // buffer. This function is lenient and will decode any options
  593. // permutations of the related Marshaler.
  594. func Unmarshal(r io.Reader, pb proto.Message) error {
  595. return new(Unmarshaler).Unmarshal(r, pb)
  596. }
  597. // UnmarshalString will populate the fields of a protocol buffer based
  598. // on a JSON string. This function is lenient and will decode any options
  599. // permutations of the related Marshaler.
  600. func UnmarshalString(str string, pb proto.Message) error {
  601. return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb)
  602. }
  603. // unmarshalValue converts/copies a value into the target.
  604. // prop may be nil.
  605. func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error {
  606. targetType := target.Type()
  607. // Allocate memory for pointer fields.
  608. if targetType.Kind() == reflect.Ptr {
  609. // If input value is "null" and target is a pointer type, then the field should be treated as not set
  610. // UNLESS the target is structpb.Value, in which case it should be set to structpb.NullValue.
  611. _, isJSONPBUnmarshaler := target.Interface().(JSONPBUnmarshaler)
  612. if string(inputValue) == "null" && targetType != reflect.TypeOf(&stpb.Value{}) && !isJSONPBUnmarshaler {
  613. return nil
  614. }
  615. target.Set(reflect.New(targetType.Elem()))
  616. return u.unmarshalValue(target.Elem(), inputValue, prop)
  617. }
  618. if jsu, ok := target.Addr().Interface().(JSONPBUnmarshaler); ok {
  619. return jsu.UnmarshalJSONPB(u, []byte(inputValue))
  620. }
  621. // Handle well-known types that are not pointers.
  622. if w, ok := target.Addr().Interface().(wkt); ok {
  623. switch w.XXX_WellKnownType() {
  624. case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
  625. "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
  626. return u.unmarshalValue(target.Field(0), inputValue, prop)
  627. case "Any":
  628. // Use json.RawMessage pointer type instead of value to support pre-1.8 version.
  629. // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see
  630. // https://github.com/golang/go/issues/14493
  631. var jsonFields map[string]*json.RawMessage
  632. if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
  633. return err
  634. }
  635. val, ok := jsonFields["@type"]
  636. if !ok || val == nil {
  637. return errors.New("Any JSON doesn't have '@type'")
  638. }
  639. var turl string
  640. if err := json.Unmarshal([]byte(*val), &turl); err != nil {
  641. return fmt.Errorf("can't unmarshal Any's '@type': %q", *val)
  642. }
  643. target.Field(0).SetString(turl)
  644. var m proto.Message
  645. var err error
  646. if u.AnyResolver != nil {
  647. m, err = u.AnyResolver.Resolve(turl)
  648. } else {
  649. m, err = defaultResolveAny(turl)
  650. }
  651. if err != nil {
  652. return err
  653. }
  654. if _, ok := m.(wkt); ok {
  655. val, ok := jsonFields["value"]
  656. if !ok {
  657. return errors.New("Any JSON doesn't have 'value'")
  658. }
  659. if err := u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil {
  660. return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err)
  661. }
  662. } else {
  663. delete(jsonFields, "@type")
  664. nestedProto, err := json.Marshal(jsonFields)
  665. if err != nil {
  666. return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err)
  667. }
  668. if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil {
  669. return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err)
  670. }
  671. }
  672. b, err := proto.Marshal(m)
  673. if err != nil {
  674. return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err)
  675. }
  676. target.Field(1).SetBytes(b)
  677. return nil
  678. case "Duration":
  679. unq, err := strconv.Unquote(string(inputValue))
  680. if err != nil {
  681. return err
  682. }
  683. d, err := time.ParseDuration(unq)
  684. if err != nil {
  685. return fmt.Errorf("bad Duration: %v", err)
  686. }
  687. ns := d.Nanoseconds()
  688. s := ns / 1e9
  689. ns %= 1e9
  690. target.Field(0).SetInt(s)
  691. target.Field(1).SetInt(ns)
  692. return nil
  693. case "Timestamp":
  694. unq, err := strconv.Unquote(string(inputValue))
  695. if err != nil {
  696. return err
  697. }
  698. t, err := time.Parse(time.RFC3339Nano, unq)
  699. if err != nil {
  700. return fmt.Errorf("bad Timestamp: %v", err)
  701. }
  702. target.Field(0).SetInt(t.Unix())
  703. target.Field(1).SetInt(int64(t.Nanosecond()))
  704. return nil
  705. case "Struct":
  706. var m map[string]json.RawMessage
  707. if err := json.Unmarshal(inputValue, &m); err != nil {
  708. return fmt.Errorf("bad StructValue: %v", err)
  709. }
  710. target.Field(0).Set(reflect.ValueOf(map[string]*stpb.Value{}))
  711. for k, jv := range m {
  712. pv := &stpb.Value{}
  713. if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil {
  714. return fmt.Errorf("bad value in StructValue for key %q: %v", k, err)
  715. }
  716. target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv))
  717. }
  718. return nil
  719. case "ListValue":
  720. var s []json.RawMessage
  721. if err := json.Unmarshal(inputValue, &s); err != nil {
  722. return fmt.Errorf("bad ListValue: %v", err)
  723. }
  724. target.Field(0).Set(reflect.ValueOf(make([]*stpb.Value, len(s), len(s))))
  725. for i, sv := range s {
  726. if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil {
  727. return err
  728. }
  729. }
  730. return nil
  731. case "Value":
  732. ivStr := string(inputValue)
  733. if ivStr == "null" {
  734. target.Field(0).Set(reflect.ValueOf(&stpb.Value_NullValue{}))
  735. } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil {
  736. target.Field(0).Set(reflect.ValueOf(&stpb.Value_NumberValue{v}))
  737. } else if v, err := strconv.Unquote(ivStr); err == nil {
  738. target.Field(0).Set(reflect.ValueOf(&stpb.Value_StringValue{v}))
  739. } else if v, err := strconv.ParseBool(ivStr); err == nil {
  740. target.Field(0).Set(reflect.ValueOf(&stpb.Value_BoolValue{v}))
  741. } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil {
  742. lv := &stpb.ListValue{}
  743. target.Field(0).Set(reflect.ValueOf(&stpb.Value_ListValue{lv}))
  744. return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop)
  745. } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil {
  746. sv := &stpb.Struct{}
  747. target.Field(0).Set(reflect.ValueOf(&stpb.Value_StructValue{sv}))
  748. return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop)
  749. } else {
  750. return fmt.Errorf("unrecognized type for Value %q", ivStr)
  751. }
  752. return nil
  753. }
  754. }
  755. // Handle enums, which have an underlying type of int32,
  756. // and may appear as strings.
  757. // The case of an enum appearing as a number is handled
  758. // at the bottom of this function.
  759. if inputValue[0] == '"' && prop != nil && prop.Enum != "" {
  760. vmap := proto.EnumValueMap(prop.Enum)
  761. // Don't need to do unquoting; valid enum names
  762. // are from a limited character set.
  763. s := inputValue[1 : len(inputValue)-1]
  764. n, ok := vmap[string(s)]
  765. if !ok {
  766. return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum)
  767. }
  768. if target.Kind() == reflect.Ptr { // proto2
  769. target.Set(reflect.New(targetType.Elem()))
  770. target = target.Elem()
  771. }
  772. target.SetInt(int64(n))
  773. return nil
  774. }
  775. // Handle nested messages.
  776. if targetType.Kind() == reflect.Struct {
  777. var jsonFields map[string]json.RawMessage
  778. if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
  779. return err
  780. }
  781. consumeField := func(prop *proto.Properties) (json.RawMessage, bool) {
  782. // Be liberal in what names we accept; both orig_name and camelName are okay.
  783. fieldNames := acceptedJSONFieldNames(prop)
  784. vOrig, okOrig := jsonFields[fieldNames.orig]
  785. vCamel, okCamel := jsonFields[fieldNames.camel]
  786. if !okOrig && !okCamel {
  787. return nil, false
  788. }
  789. // If, for some reason, both are present in the data, favour the camelName.
  790. var raw json.RawMessage
  791. if okOrig {
  792. raw = vOrig
  793. delete(jsonFields, fieldNames.orig)
  794. }
  795. if okCamel {
  796. raw = vCamel
  797. delete(jsonFields, fieldNames.camel)
  798. }
  799. return raw, true
  800. }
  801. sprops := proto.GetProperties(targetType)
  802. for i := 0; i < target.NumField(); i++ {
  803. ft := target.Type().Field(i)
  804. if strings.HasPrefix(ft.Name, "XXX_") {
  805. continue
  806. }
  807. valueForField, ok := consumeField(sprops.Prop[i])
  808. if !ok {
  809. continue
  810. }
  811. if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil {
  812. return err
  813. }
  814. }
  815. // Check for any oneof fields.
  816. if len(jsonFields) > 0 {
  817. for _, oop := range sprops.OneofTypes {
  818. raw, ok := consumeField(oop.Prop)
  819. if !ok {
  820. continue
  821. }
  822. nv := reflect.New(oop.Type.Elem())
  823. target.Field(oop.Field).Set(nv)
  824. if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil {
  825. return err
  826. }
  827. }
  828. }
  829. // Handle proto2 extensions.
  830. if len(jsonFields) > 0 {
  831. if ep, ok := target.Addr().Interface().(proto.Message); ok {
  832. for _, ext := range proto.RegisteredExtensions(ep) {
  833. name := fmt.Sprintf("[%s]", ext.Name)
  834. raw, ok := jsonFields[name]
  835. if !ok {
  836. continue
  837. }
  838. delete(jsonFields, name)
  839. nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem())
  840. if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil {
  841. return err
  842. }
  843. if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil {
  844. return err
  845. }
  846. }
  847. }
  848. }
  849. if !u.AllowUnknownFields && len(jsonFields) > 0 {
  850. // Pick any field to be the scapegoat.
  851. var f string
  852. for fname := range jsonFields {
  853. f = fname
  854. break
  855. }
  856. return fmt.Errorf("unknown field %q in %v", f, targetType)
  857. }
  858. return nil
  859. }
  860. // Handle arrays (which aren't encoded bytes)
  861. if targetType.Kind() == reflect.Slice && targetType.Elem().Kind() != reflect.Uint8 {
  862. var slc []json.RawMessage
  863. if err := json.Unmarshal(inputValue, &slc); err != nil {
  864. return err
  865. }
  866. if slc != nil {
  867. l := len(slc)
  868. target.Set(reflect.MakeSlice(targetType, l, l))
  869. for i := 0; i < l; i++ {
  870. if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil {
  871. return err
  872. }
  873. }
  874. }
  875. return nil
  876. }
  877. // Handle maps (whose keys are always strings)
  878. if targetType.Kind() == reflect.Map {
  879. var mp map[string]json.RawMessage
  880. if err := json.Unmarshal(inputValue, &mp); err != nil {
  881. return err
  882. }
  883. if mp != nil {
  884. target.Set(reflect.MakeMap(targetType))
  885. var keyprop, valprop *proto.Properties
  886. if prop != nil {
  887. // These could still be nil if the protobuf metadata is broken somehow.
  888. // TODO: This won't work because the fields are unexported.
  889. // We should probably just reparse them.
  890. //keyprop, valprop = prop.mkeyprop, prop.mvalprop
  891. }
  892. for ks, raw := range mp {
  893. // Unmarshal map key. The core json library already decoded the key into a
  894. // string, so we handle that specially. Other types were quoted post-serialization.
  895. var k reflect.Value
  896. if targetType.Key().Kind() == reflect.String {
  897. k = reflect.ValueOf(ks)
  898. } else {
  899. k = reflect.New(targetType.Key()).Elem()
  900. if err := u.unmarshalValue(k, json.RawMessage(ks), keyprop); err != nil {
  901. return err
  902. }
  903. }
  904. // Unmarshal map value.
  905. v := reflect.New(targetType.Elem()).Elem()
  906. if err := u.unmarshalValue(v, raw, valprop); err != nil {
  907. return err
  908. }
  909. target.SetMapIndex(k, v)
  910. }
  911. }
  912. return nil
  913. }
  914. // 64-bit integers can be encoded as strings. In this case we drop
  915. // the quotes and proceed as normal.
  916. isNum := targetType.Kind() == reflect.Int64 || targetType.Kind() == reflect.Uint64
  917. if isNum && strings.HasPrefix(string(inputValue), `"`) {
  918. inputValue = inputValue[1 : len(inputValue)-1]
  919. }
  920. // Non-finite numbers can be encoded as strings.
  921. isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64
  922. if isFloat {
  923. if num, ok := nonFinite[string(inputValue)]; ok {
  924. target.SetFloat(num)
  925. return nil
  926. }
  927. }
  928. // Use the encoding/json for parsing other value types.
  929. return json.Unmarshal(inputValue, target.Addr().Interface())
  930. }
  931. // jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute.
  932. func jsonProperties(f reflect.StructField, origName bool) *proto.Properties {
  933. var prop proto.Properties
  934. prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f)
  935. if origName || prop.JSONName == "" {
  936. prop.JSONName = prop.OrigName
  937. }
  938. return &prop
  939. }
  940. type fieldNames struct {
  941. orig, camel string
  942. }
  943. func acceptedJSONFieldNames(prop *proto.Properties) fieldNames {
  944. opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName}
  945. if prop.JSONName != "" {
  946. opts.camel = prop.JSONName
  947. }
  948. return opts
  949. }
  950. // Writer wrapper inspired by https://blog.golang.org/errors-are-values
  951. type errWriter struct {
  952. writer io.Writer
  953. err error
  954. }
  955. func (w *errWriter) write(str string) {
  956. if w.err != nil {
  957. return
  958. }
  959. _, w.err = w.writer.Write([]byte(str))
  960. }
  961. // Map fields may have key types of non-float scalars, strings and enums.
  962. // The easiest way to sort them in some deterministic order is to use fmt.
  963. // If this turns out to be inefficient we can always consider other options,
  964. // such as doing a Schwartzian transform.
  965. //
  966. // Numeric keys are sorted in numeric order per
  967. // https://developers.google.com/protocol-buffers/docs/proto#maps.
  968. type mapKeys []reflect.Value
  969. func (s mapKeys) Len() int { return len(s) }
  970. func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  971. func (s mapKeys) Less(i, j int) bool {
  972. if k := s[i].Kind(); k == s[j].Kind() {
  973. switch k {
  974. case reflect.Int32, reflect.Int64:
  975. return s[i].Int() < s[j].Int()
  976. case reflect.Uint32, reflect.Uint64:
  977. return s[i].Uint() < s[j].Uint()
  978. }
  979. }
  980. return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface())
  981. }