jsonpb.go 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059
  1. // Go support for Protocol Buffers - Google's data interchange format
  2. //
  3. // Copyright 2015 The Go Authors. All rights reserved.
  4. // https://github.com/golang/protobuf
  5. //
  6. // Redistribution and use in source and binary forms, with or without
  7. // modification, are permitted provided that the following conditions are
  8. // met:
  9. //
  10. // * Redistributions of source code must retain the above copyright
  11. // notice, this list of conditions and the following disclaimer.
  12. // * Redistributions in binary form must reproduce the above
  13. // copyright notice, this list of conditions and the following disclaimer
  14. // in the documentation and/or other materials provided with the
  15. // distribution.
  16. // * Neither the name of Google Inc. nor the names of its
  17. // contributors may be used to endorse or promote products derived from
  18. // this software without specific prior written permission.
  19. //
  20. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31. /*
  32. Package jsonpb provides marshaling and unmarshaling between protocol buffers and JSON.
  33. It follows the specification at https://developers.google.com/protocol-buffers/docs/proto3#json.
  34. This package produces a different output than the standard "encoding/json" package,
  35. which does not operate correctly on protocol buffers.
  36. */
  37. package jsonpb
  38. import (
  39. "bytes"
  40. "encoding/json"
  41. "errors"
  42. "fmt"
  43. "io"
  44. "math"
  45. "reflect"
  46. "sort"
  47. "strconv"
  48. "strings"
  49. "time"
  50. "github.com/golang/protobuf/proto"
  51. stpb "github.com/golang/protobuf/ptypes/struct"
  52. )
  53. // Marshaler is a configurable object for converting between
  54. // protocol buffer objects and a JSON representation for them.
  55. type Marshaler struct {
  56. // Whether to render enum values as integers, as opposed to string values.
  57. EnumsAsInts bool
  58. // Whether to render fields with zero values.
  59. EmitDefaults bool
  60. // A string to indent each level by. The presence of this field will
  61. // also cause a space to appear between the field separator and
  62. // value, and for newlines to be appear between fields and array
  63. // elements.
  64. Indent string
  65. // Whether to use the original (.proto) name for fields.
  66. OrigName bool
  67. }
  68. // JSONPBMarshaler is implemented by protobuf messages that customize the
  69. // way they are marshaled to JSON. Messages that implement this should
  70. // also implement JSONPBUnmarshaler so that the custom format can be
  71. // parsed.
  72. type JSONPBMarshaler interface {
  73. MarshalJSONPB(*Marshaler) ([]byte, error)
  74. }
  75. // JSONPBUnmarshaler is implemented by protobuf messages that customize
  76. // the way they are unmarshaled from JSON. Messages that implement this
  77. // should also implement JSONPBMarshaler so that the custom format can be
  78. // produced.
  79. type JSONPBUnmarshaler interface {
  80. UnmarshalJSONPB(*Unmarshaler, []byte) error
  81. }
  82. // Marshal marshals a protocol buffer into JSON.
  83. func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error {
  84. writer := &errWriter{writer: out}
  85. return m.marshalObject(writer, pb, "", "")
  86. }
  87. // MarshalToString converts a protocol buffer object to JSON string.
  88. func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) {
  89. var buf bytes.Buffer
  90. if err := m.Marshal(&buf, pb); err != nil {
  91. return "", err
  92. }
  93. return buf.String(), nil
  94. }
  95. type int32Slice []int32
  96. var nonFinite = map[string]float64{
  97. `"NaN"`: math.NaN(),
  98. `"Infinity"`: math.Inf(1),
  99. `"-Infinity"`: math.Inf(-1),
  100. }
  101. // For sorting extensions ids to ensure stable output.
  102. func (s int32Slice) Len() int { return len(s) }
  103. func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] }
  104. func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  105. type wkt interface {
  106. XXX_WellKnownType() string
  107. }
  108. // marshalObject writes a struct to the Writer.
  109. func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error {
  110. if jsm, ok := v.(JSONPBMarshaler); ok {
  111. b, err := jsm.MarshalJSONPB(m)
  112. if err != nil {
  113. return err
  114. }
  115. if typeURL != "" {
  116. // we are marshaling this object to an Any type
  117. var js map[string]*json.RawMessage
  118. if err = json.Unmarshal(b, &js); err != nil {
  119. return fmt.Errorf("type %T produced invalid JSON: %v", v, err)
  120. }
  121. turl, err := json.Marshal(typeURL)
  122. if err != nil {
  123. return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err)
  124. }
  125. js["@type"] = (*json.RawMessage)(&turl)
  126. if b, err = json.Marshal(js); err != nil {
  127. return err
  128. }
  129. }
  130. out.write(string(b))
  131. return out.err
  132. }
  133. s := reflect.ValueOf(v).Elem()
  134. // Handle well-known types.
  135. if wkt, ok := v.(wkt); ok {
  136. switch wkt.XXX_WellKnownType() {
  137. case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
  138. "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
  139. // "Wrappers use the same representation in JSON
  140. // as the wrapped primitive type, ..."
  141. sprop := proto.GetProperties(s.Type())
  142. return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent)
  143. case "Any":
  144. // Any is a bit more involved.
  145. return m.marshalAny(out, v, indent)
  146. case "Duration":
  147. // "Generated output always contains 3, 6, or 9 fractional digits,
  148. // depending on required precision."
  149. s, ns := s.Field(0).Int(), s.Field(1).Int()
  150. d := time.Duration(s)*time.Second + time.Duration(ns)*time.Nanosecond
  151. x := fmt.Sprintf("%.9f", d.Seconds())
  152. x = strings.TrimSuffix(x, "000")
  153. x = strings.TrimSuffix(x, "000")
  154. out.write(`"`)
  155. out.write(x)
  156. out.write(`s"`)
  157. return out.err
  158. case "Struct", "ListValue":
  159. // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice.
  160. // TODO: pass the correct Properties if needed.
  161. return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent)
  162. case "Timestamp":
  163. // "RFC 3339, where generated output will always be Z-normalized
  164. // and uses 3, 6 or 9 fractional digits."
  165. s, ns := s.Field(0).Int(), s.Field(1).Int()
  166. t := time.Unix(s, ns).UTC()
  167. // time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits).
  168. x := t.Format("2006-01-02T15:04:05.000000000")
  169. x = strings.TrimSuffix(x, "000")
  170. x = strings.TrimSuffix(x, "000")
  171. out.write(`"`)
  172. out.write(x)
  173. out.write(`Z"`)
  174. return out.err
  175. case "Value":
  176. // Value has a single oneof.
  177. kind := s.Field(0)
  178. if kind.IsNil() {
  179. // "absence of any variant indicates an error"
  180. return errors.New("nil Value")
  181. }
  182. // oneof -> *T -> T -> T.F
  183. x := kind.Elem().Elem().Field(0)
  184. // TODO: pass the correct Properties if needed.
  185. return m.marshalValue(out, &proto.Properties{}, x, indent)
  186. }
  187. }
  188. out.write("{")
  189. if m.Indent != "" {
  190. out.write("\n")
  191. }
  192. firstField := true
  193. if typeURL != "" {
  194. if err := m.marshalTypeURL(out, indent, typeURL); err != nil {
  195. return err
  196. }
  197. firstField = false
  198. }
  199. for i := 0; i < s.NumField(); i++ {
  200. value := s.Field(i)
  201. valueField := s.Type().Field(i)
  202. if strings.HasPrefix(valueField.Name, "XXX_") {
  203. continue
  204. }
  205. // IsNil will panic on most value kinds.
  206. switch value.Kind() {
  207. case reflect.Chan, reflect.Func, reflect.Interface:
  208. if value.IsNil() {
  209. continue
  210. }
  211. }
  212. if !m.EmitDefaults {
  213. switch value.Kind() {
  214. case reflect.Bool:
  215. if !value.Bool() {
  216. continue
  217. }
  218. case reflect.Int32, reflect.Int64:
  219. if value.Int() == 0 {
  220. continue
  221. }
  222. case reflect.Uint32, reflect.Uint64:
  223. if value.Uint() == 0 {
  224. continue
  225. }
  226. case reflect.Float32, reflect.Float64:
  227. if value.Float() == 0 {
  228. continue
  229. }
  230. case reflect.String:
  231. if value.Len() == 0 {
  232. continue
  233. }
  234. case reflect.Map, reflect.Ptr, reflect.Slice:
  235. if value.IsNil() {
  236. continue
  237. }
  238. }
  239. }
  240. // Oneof fields need special handling.
  241. if valueField.Tag.Get("protobuf_oneof") != "" {
  242. // value is an interface containing &T{real_value}.
  243. sv := value.Elem().Elem() // interface -> *T -> T
  244. value = sv.Field(0)
  245. valueField = sv.Type().Field(0)
  246. }
  247. prop := jsonProperties(valueField, m.OrigName)
  248. if !firstField {
  249. m.writeSep(out)
  250. }
  251. if err := m.marshalField(out, prop, value, indent); err != nil {
  252. return err
  253. }
  254. firstField = false
  255. }
  256. // Handle proto2 extensions.
  257. if ep, ok := v.(proto.Message); ok {
  258. extensions := proto.RegisteredExtensions(v)
  259. // Sort extensions for stable output.
  260. ids := make([]int32, 0, len(extensions))
  261. for id, desc := range extensions {
  262. if !proto.HasExtension(ep, desc) {
  263. continue
  264. }
  265. ids = append(ids, id)
  266. }
  267. sort.Sort(int32Slice(ids))
  268. for _, id := range ids {
  269. desc := extensions[id]
  270. if desc == nil {
  271. // unknown extension
  272. continue
  273. }
  274. ext, extErr := proto.GetExtension(ep, desc)
  275. if extErr != nil {
  276. return extErr
  277. }
  278. value := reflect.ValueOf(ext)
  279. var prop proto.Properties
  280. prop.Parse(desc.Tag)
  281. prop.JSONName = fmt.Sprintf("[%s]", desc.Name)
  282. if !firstField {
  283. m.writeSep(out)
  284. }
  285. if err := m.marshalField(out, &prop, value, indent); err != nil {
  286. return err
  287. }
  288. firstField = false
  289. }
  290. }
  291. if m.Indent != "" {
  292. out.write("\n")
  293. out.write(indent)
  294. }
  295. out.write("}")
  296. return out.err
  297. }
  298. func (m *Marshaler) writeSep(out *errWriter) {
  299. if m.Indent != "" {
  300. out.write(",\n")
  301. } else {
  302. out.write(",")
  303. }
  304. }
  305. func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error {
  306. // "If the Any contains a value that has a special JSON mapping,
  307. // it will be converted as follows: {"@type": xxx, "value": yyy}.
  308. // Otherwise, the value will be converted into a JSON object,
  309. // and the "@type" field will be inserted to indicate the actual data type."
  310. v := reflect.ValueOf(any).Elem()
  311. turl := v.Field(0).String()
  312. val := v.Field(1).Bytes()
  313. // Only the part of type_url after the last slash is relevant.
  314. mname := turl
  315. if slash := strings.LastIndex(mname, "/"); slash >= 0 {
  316. mname = mname[slash+1:]
  317. }
  318. mt := proto.MessageType(mname)
  319. if mt == nil {
  320. return fmt.Errorf("unknown message type %q", mname)
  321. }
  322. msg := reflect.New(mt.Elem()).Interface().(proto.Message)
  323. if err := proto.Unmarshal(val, msg); err != nil {
  324. return err
  325. }
  326. if _, ok := msg.(wkt); ok {
  327. out.write("{")
  328. if m.Indent != "" {
  329. out.write("\n")
  330. }
  331. if err := m.marshalTypeURL(out, indent, turl); err != nil {
  332. return err
  333. }
  334. m.writeSep(out)
  335. if m.Indent != "" {
  336. out.write(indent)
  337. out.write(m.Indent)
  338. out.write(`"value": `)
  339. } else {
  340. out.write(`"value":`)
  341. }
  342. if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil {
  343. return err
  344. }
  345. if m.Indent != "" {
  346. out.write("\n")
  347. out.write(indent)
  348. }
  349. out.write("}")
  350. return out.err
  351. }
  352. return m.marshalObject(out, msg, indent, turl)
  353. }
  354. func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error {
  355. if m.Indent != "" {
  356. out.write(indent)
  357. out.write(m.Indent)
  358. }
  359. out.write(`"@type":`)
  360. if m.Indent != "" {
  361. out.write(" ")
  362. }
  363. b, err := json.Marshal(typeURL)
  364. if err != nil {
  365. return err
  366. }
  367. out.write(string(b))
  368. return out.err
  369. }
  370. // marshalField writes field description and value to the Writer.
  371. func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
  372. if m.Indent != "" {
  373. out.write(indent)
  374. out.write(m.Indent)
  375. }
  376. out.write(`"`)
  377. out.write(prop.JSONName)
  378. out.write(`":`)
  379. if m.Indent != "" {
  380. out.write(" ")
  381. }
  382. if err := m.marshalValue(out, prop, v, indent); err != nil {
  383. return err
  384. }
  385. return nil
  386. }
  387. // marshalValue writes the value to the Writer.
  388. func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
  389. var err error
  390. v = reflect.Indirect(v)
  391. // Handle nil pointer
  392. if v.Kind() == reflect.Invalid {
  393. out.write("null")
  394. return out.err
  395. }
  396. // Handle repeated elements.
  397. if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
  398. out.write("[")
  399. comma := ""
  400. for i := 0; i < v.Len(); i++ {
  401. sliceVal := v.Index(i)
  402. out.write(comma)
  403. if m.Indent != "" {
  404. out.write("\n")
  405. out.write(indent)
  406. out.write(m.Indent)
  407. out.write(m.Indent)
  408. }
  409. if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil {
  410. return err
  411. }
  412. comma = ","
  413. }
  414. if m.Indent != "" {
  415. out.write("\n")
  416. out.write(indent)
  417. out.write(m.Indent)
  418. }
  419. out.write("]")
  420. return out.err
  421. }
  422. // Handle well-known types.
  423. // Most are handled up in marshalObject (because 99% are messages).
  424. if wkt, ok := v.Interface().(wkt); ok {
  425. switch wkt.XXX_WellKnownType() {
  426. case "NullValue":
  427. out.write("null")
  428. return out.err
  429. }
  430. }
  431. // Handle enumerations.
  432. if !m.EnumsAsInts && prop.Enum != "" {
  433. // Unknown enum values will are stringified by the proto library as their
  434. // value. Such values should _not_ be quoted or they will be interpreted
  435. // as an enum string instead of their value.
  436. enumStr := v.Interface().(fmt.Stringer).String()
  437. var valStr string
  438. if v.Kind() == reflect.Ptr {
  439. valStr = strconv.Itoa(int(v.Elem().Int()))
  440. } else {
  441. valStr = strconv.Itoa(int(v.Int()))
  442. }
  443. isKnownEnum := enumStr != valStr
  444. if isKnownEnum {
  445. out.write(`"`)
  446. }
  447. out.write(enumStr)
  448. if isKnownEnum {
  449. out.write(`"`)
  450. }
  451. return out.err
  452. }
  453. // Handle nested messages.
  454. if v.Kind() == reflect.Struct {
  455. return m.marshalObject(out, v.Addr().Interface().(proto.Message), indent+m.Indent, "")
  456. }
  457. // Handle maps.
  458. // Since Go randomizes map iteration, we sort keys for stable output.
  459. if v.Kind() == reflect.Map {
  460. out.write(`{`)
  461. keys := v.MapKeys()
  462. sort.Sort(mapKeys(keys))
  463. for i, k := range keys {
  464. if i > 0 {
  465. out.write(`,`)
  466. }
  467. if m.Indent != "" {
  468. out.write("\n")
  469. out.write(indent)
  470. out.write(m.Indent)
  471. out.write(m.Indent)
  472. }
  473. b, err := json.Marshal(k.Interface())
  474. if err != nil {
  475. return err
  476. }
  477. s := string(b)
  478. // If the JSON is not a string value, encode it again to make it one.
  479. if !strings.HasPrefix(s, `"`) {
  480. b, err := json.Marshal(s)
  481. if err != nil {
  482. return err
  483. }
  484. s = string(b)
  485. }
  486. out.write(s)
  487. out.write(`:`)
  488. if m.Indent != "" {
  489. out.write(` `)
  490. }
  491. if err := m.marshalValue(out, prop, v.MapIndex(k), indent+m.Indent); err != nil {
  492. return err
  493. }
  494. }
  495. if m.Indent != "" {
  496. out.write("\n")
  497. out.write(indent)
  498. out.write(m.Indent)
  499. }
  500. out.write(`}`)
  501. return out.err
  502. }
  503. // Handle non-finite floats, e.g. NaN, Infinity and -Infinity.
  504. if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 {
  505. f := v.Float()
  506. var sval string
  507. switch {
  508. case math.IsInf(f, 1):
  509. sval = `"Infinity"`
  510. case math.IsInf(f, -1):
  511. sval = `"-Infinity"`
  512. case math.IsNaN(f):
  513. sval = `"NaN"`
  514. }
  515. if sval != "" {
  516. out.write(sval)
  517. return out.err
  518. }
  519. }
  520. // Default handling defers to the encoding/json library.
  521. b, err := json.Marshal(v.Interface())
  522. if err != nil {
  523. return err
  524. }
  525. needToQuote := string(b[0]) != `"` && (v.Kind() == reflect.Int64 || v.Kind() == reflect.Uint64)
  526. if needToQuote {
  527. out.write(`"`)
  528. }
  529. out.write(string(b))
  530. if needToQuote {
  531. out.write(`"`)
  532. }
  533. return out.err
  534. }
  535. // Unmarshaler is a configurable object for converting from a JSON
  536. // representation to a protocol buffer object.
  537. type Unmarshaler struct {
  538. // Whether to allow messages to contain unknown fields, as opposed to
  539. // failing to unmarshal.
  540. AllowUnknownFields bool
  541. }
  542. // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
  543. // This function is lenient and will decode any options permutations of the
  544. // related Marshaler.
  545. func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
  546. inputValue := json.RawMessage{}
  547. if err := dec.Decode(&inputValue); err != nil {
  548. return err
  549. }
  550. return u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil)
  551. }
  552. // Unmarshal unmarshals a JSON object stream into a protocol
  553. // buffer. This function is lenient and will decode any options
  554. // permutations of the related Marshaler.
  555. func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error {
  556. dec := json.NewDecoder(r)
  557. return u.UnmarshalNext(dec, pb)
  558. }
  559. // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
  560. // This function is lenient and will decode any options permutations of the
  561. // related Marshaler.
  562. func UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
  563. return new(Unmarshaler).UnmarshalNext(dec, pb)
  564. }
  565. // Unmarshal unmarshals a JSON object stream into a protocol
  566. // buffer. This function is lenient and will decode any options
  567. // permutations of the related Marshaler.
  568. func Unmarshal(r io.Reader, pb proto.Message) error {
  569. return new(Unmarshaler).Unmarshal(r, pb)
  570. }
  571. // UnmarshalString will populate the fields of a protocol buffer based
  572. // on a JSON string. This function is lenient and will decode any options
  573. // permutations of the related Marshaler.
  574. func UnmarshalString(str string, pb proto.Message) error {
  575. return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb)
  576. }
  577. // unmarshalValue converts/copies a value into the target.
  578. // prop may be nil.
  579. func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error {
  580. targetType := target.Type()
  581. // Allocate memory for pointer fields.
  582. if targetType.Kind() == reflect.Ptr {
  583. target.Set(reflect.New(targetType.Elem()))
  584. return u.unmarshalValue(target.Elem(), inputValue, prop)
  585. }
  586. if jsu, ok := target.Addr().Interface().(JSONPBUnmarshaler); ok {
  587. return jsu.UnmarshalJSONPB(u, []byte(inputValue))
  588. }
  589. // Handle well-known types.
  590. if w, ok := target.Addr().Interface().(wkt); ok {
  591. switch w.XXX_WellKnownType() {
  592. case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
  593. "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
  594. // "Wrappers use the same representation in JSON
  595. // as the wrapped primitive type, except that null is allowed."
  596. // encoding/json will turn JSON `null` into Go `nil`,
  597. // so we don't have to do any extra work.
  598. return u.unmarshalValue(target.Field(0), inputValue, prop)
  599. case "Any":
  600. // Use json.RawMessage pointer type instead of value to support pre-1.8 version.
  601. // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see
  602. // https://github.com/golang/go/issues/14493
  603. var jsonFields map[string]*json.RawMessage
  604. if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
  605. return err
  606. }
  607. val, ok := jsonFields["@type"]
  608. if !ok || val == nil {
  609. return errors.New("Any JSON doesn't have '@type'")
  610. }
  611. var turl string
  612. if err := json.Unmarshal([]byte(*val), &turl); err != nil {
  613. return fmt.Errorf("can't unmarshal Any's '@type': %q", *val)
  614. }
  615. target.Field(0).SetString(turl)
  616. mname := turl
  617. if slash := strings.LastIndex(mname, "/"); slash >= 0 {
  618. mname = mname[slash+1:]
  619. }
  620. mt := proto.MessageType(mname)
  621. if mt == nil {
  622. return fmt.Errorf("unknown message type %q", mname)
  623. }
  624. m := reflect.New(mt.Elem()).Interface().(proto.Message)
  625. if _, ok := m.(wkt); ok {
  626. val, ok := jsonFields["value"]
  627. if !ok {
  628. return errors.New("Any JSON doesn't have 'value'")
  629. }
  630. if err := u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil {
  631. return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err)
  632. }
  633. } else {
  634. delete(jsonFields, "@type")
  635. nestedProto, err := json.Marshal(jsonFields)
  636. if err != nil {
  637. return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err)
  638. }
  639. if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil {
  640. return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err)
  641. }
  642. }
  643. b, err := proto.Marshal(m)
  644. if err != nil {
  645. return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err)
  646. }
  647. target.Field(1).SetBytes(b)
  648. return nil
  649. case "Duration":
  650. ivStr := string(inputValue)
  651. if ivStr == "null" {
  652. target.Field(0).SetInt(0)
  653. target.Field(1).SetInt(0)
  654. return nil
  655. }
  656. unq, err := strconv.Unquote(ivStr)
  657. if err != nil {
  658. return err
  659. }
  660. d, err := time.ParseDuration(unq)
  661. if err != nil {
  662. return fmt.Errorf("bad Duration: %v", err)
  663. }
  664. ns := d.Nanoseconds()
  665. s := ns / 1e9
  666. ns %= 1e9
  667. target.Field(0).SetInt(s)
  668. target.Field(1).SetInt(ns)
  669. return nil
  670. case "Timestamp":
  671. ivStr := string(inputValue)
  672. if ivStr == "null" {
  673. target.Field(0).SetInt(0)
  674. target.Field(1).SetInt(0)
  675. return nil
  676. }
  677. unq, err := strconv.Unquote(ivStr)
  678. if err != nil {
  679. return err
  680. }
  681. t, err := time.Parse(time.RFC3339Nano, unq)
  682. if err != nil {
  683. return fmt.Errorf("bad Timestamp: %v", err)
  684. }
  685. target.Field(0).SetInt(int64(t.Unix()))
  686. target.Field(1).SetInt(int64(t.Nanosecond()))
  687. return nil
  688. case "Struct":
  689. if string(inputValue) == "null" {
  690. // Interpret a null struct as empty.
  691. return nil
  692. }
  693. var m map[string]json.RawMessage
  694. if err := json.Unmarshal(inputValue, &m); err != nil {
  695. return fmt.Errorf("bad StructValue: %v", err)
  696. }
  697. target.Field(0).Set(reflect.ValueOf(map[string]*stpb.Value{}))
  698. for k, jv := range m {
  699. pv := &stpb.Value{}
  700. if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil {
  701. return fmt.Errorf("bad value in StructValue for key %q: %v", k, err)
  702. }
  703. target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv))
  704. }
  705. return nil
  706. case "ListValue":
  707. if string(inputValue) == "null" {
  708. // Interpret a null ListValue as empty.
  709. return nil
  710. }
  711. var s []json.RawMessage
  712. if err := json.Unmarshal(inputValue, &s); err != nil {
  713. return fmt.Errorf("bad ListValue: %v", err)
  714. }
  715. target.Field(0).Set(reflect.ValueOf(make([]*stpb.Value, len(s), len(s))))
  716. for i, sv := range s {
  717. if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil {
  718. return err
  719. }
  720. }
  721. return nil
  722. case "Value":
  723. ivStr := string(inputValue)
  724. if ivStr == "null" {
  725. target.Field(0).Set(reflect.ValueOf(&stpb.Value_NullValue{}))
  726. } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil {
  727. target.Field(0).Set(reflect.ValueOf(&stpb.Value_NumberValue{v}))
  728. } else if v, err := strconv.Unquote(ivStr); err == nil {
  729. target.Field(0).Set(reflect.ValueOf(&stpb.Value_StringValue{v}))
  730. } else if v, err := strconv.ParseBool(ivStr); err == nil {
  731. target.Field(0).Set(reflect.ValueOf(&stpb.Value_BoolValue{v}))
  732. } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil {
  733. lv := &stpb.ListValue{}
  734. target.Field(0).Set(reflect.ValueOf(&stpb.Value_ListValue{lv}))
  735. return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop)
  736. } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil {
  737. sv := &stpb.Struct{}
  738. target.Field(0).Set(reflect.ValueOf(&stpb.Value_StructValue{sv}))
  739. return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop)
  740. } else {
  741. return fmt.Errorf("unrecognized type for Value %q", ivStr)
  742. }
  743. return nil
  744. }
  745. }
  746. // Handle enums, which have an underlying type of int32,
  747. // and may appear as strings.
  748. // The case of an enum appearing as a number is handled
  749. // at the bottom of this function.
  750. if inputValue[0] == '"' && prop != nil && prop.Enum != "" {
  751. vmap := proto.EnumValueMap(prop.Enum)
  752. // Don't need to do unquoting; valid enum names
  753. // are from a limited character set.
  754. s := inputValue[1 : len(inputValue)-1]
  755. n, ok := vmap[string(s)]
  756. if !ok {
  757. return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum)
  758. }
  759. if target.Kind() == reflect.Ptr { // proto2
  760. target.Set(reflect.New(targetType.Elem()))
  761. target = target.Elem()
  762. }
  763. target.SetInt(int64(n))
  764. return nil
  765. }
  766. // Handle nested messages.
  767. if targetType.Kind() == reflect.Struct {
  768. var jsonFields map[string]json.RawMessage
  769. if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
  770. return err
  771. }
  772. consumeField := func(prop *proto.Properties) (json.RawMessage, bool) {
  773. // Be liberal in what names we accept; both orig_name and camelName are okay.
  774. fieldNames := acceptedJSONFieldNames(prop)
  775. vOrig, okOrig := jsonFields[fieldNames.orig]
  776. vCamel, okCamel := jsonFields[fieldNames.camel]
  777. if !okOrig && !okCamel {
  778. return nil, false
  779. }
  780. // If, for some reason, both are present in the data, favour the camelName.
  781. var raw json.RawMessage
  782. if okOrig {
  783. raw = vOrig
  784. delete(jsonFields, fieldNames.orig)
  785. }
  786. if okCamel {
  787. raw = vCamel
  788. delete(jsonFields, fieldNames.camel)
  789. }
  790. return raw, true
  791. }
  792. sprops := proto.GetProperties(targetType)
  793. for i := 0; i < target.NumField(); i++ {
  794. ft := target.Type().Field(i)
  795. if strings.HasPrefix(ft.Name, "XXX_") {
  796. continue
  797. }
  798. valueForField, ok := consumeField(sprops.Prop[i])
  799. if !ok {
  800. continue
  801. }
  802. if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil {
  803. return err
  804. }
  805. }
  806. // Check for any oneof fields.
  807. if len(jsonFields) > 0 {
  808. for _, oop := range sprops.OneofTypes {
  809. raw, ok := consumeField(oop.Prop)
  810. if !ok {
  811. continue
  812. }
  813. nv := reflect.New(oop.Type.Elem())
  814. target.Field(oop.Field).Set(nv)
  815. if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil {
  816. return err
  817. }
  818. }
  819. }
  820. // Handle proto2 extensions.
  821. if len(jsonFields) > 0 {
  822. if ep, ok := target.Addr().Interface().(proto.Message); ok {
  823. for _, ext := range proto.RegisteredExtensions(ep) {
  824. name := fmt.Sprintf("[%s]", ext.Name)
  825. raw, ok := jsonFields[name]
  826. if !ok {
  827. continue
  828. }
  829. delete(jsonFields, name)
  830. nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem())
  831. if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil {
  832. return err
  833. }
  834. if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil {
  835. return err
  836. }
  837. }
  838. }
  839. }
  840. if !u.AllowUnknownFields && len(jsonFields) > 0 {
  841. // Pick any field to be the scapegoat.
  842. var f string
  843. for fname := range jsonFields {
  844. f = fname
  845. break
  846. }
  847. return fmt.Errorf("unknown field %q in %v", f, targetType)
  848. }
  849. return nil
  850. }
  851. // Handle arrays (which aren't encoded bytes)
  852. if targetType.Kind() == reflect.Slice && targetType.Elem().Kind() != reflect.Uint8 {
  853. var slc []json.RawMessage
  854. if err := json.Unmarshal(inputValue, &slc); err != nil {
  855. return err
  856. }
  857. len := len(slc)
  858. target.Set(reflect.MakeSlice(targetType, len, len))
  859. for i := 0; i < len; i++ {
  860. if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil {
  861. return err
  862. }
  863. }
  864. return nil
  865. }
  866. // Handle maps (whose keys are always strings)
  867. if targetType.Kind() == reflect.Map {
  868. var mp map[string]json.RawMessage
  869. if err := json.Unmarshal(inputValue, &mp); err != nil {
  870. return err
  871. }
  872. target.Set(reflect.MakeMap(targetType))
  873. var keyprop, valprop *proto.Properties
  874. if prop != nil {
  875. // These could still be nil if the protobuf metadata is broken somehow.
  876. // TODO: This won't work because the fields are unexported.
  877. // We should probably just reparse them.
  878. //keyprop, valprop = prop.mkeyprop, prop.mvalprop
  879. }
  880. for ks, raw := range mp {
  881. // Unmarshal map key. The core json library already decoded the key into a
  882. // string, so we handle that specially. Other types were quoted post-serialization.
  883. var k reflect.Value
  884. if targetType.Key().Kind() == reflect.String {
  885. k = reflect.ValueOf(ks)
  886. } else {
  887. k = reflect.New(targetType.Key()).Elem()
  888. if err := u.unmarshalValue(k, json.RawMessage(ks), keyprop); err != nil {
  889. return err
  890. }
  891. }
  892. // Unmarshal map value.
  893. v := reflect.New(targetType.Elem()).Elem()
  894. if err := u.unmarshalValue(v, raw, valprop); err != nil {
  895. return err
  896. }
  897. target.SetMapIndex(k, v)
  898. }
  899. return nil
  900. }
  901. // 64-bit integers can be encoded as strings. In this case we drop
  902. // the quotes and proceed as normal.
  903. isNum := targetType.Kind() == reflect.Int64 || targetType.Kind() == reflect.Uint64
  904. if isNum && strings.HasPrefix(string(inputValue), `"`) {
  905. inputValue = inputValue[1 : len(inputValue)-1]
  906. }
  907. // Non-finite numbers can be encoded as strings.
  908. isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64
  909. if isFloat {
  910. if num, ok := nonFinite[string(inputValue)]; ok {
  911. target.SetFloat(num)
  912. return nil
  913. }
  914. }
  915. // Use the encoding/json for parsing other value types.
  916. return json.Unmarshal(inputValue, target.Addr().Interface())
  917. }
  918. // jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute.
  919. func jsonProperties(f reflect.StructField, origName bool) *proto.Properties {
  920. var prop proto.Properties
  921. prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f)
  922. if origName || prop.JSONName == "" {
  923. prop.JSONName = prop.OrigName
  924. }
  925. return &prop
  926. }
  927. type fieldNames struct {
  928. orig, camel string
  929. }
  930. func acceptedJSONFieldNames(prop *proto.Properties) fieldNames {
  931. opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName}
  932. if prop.JSONName != "" {
  933. opts.camel = prop.JSONName
  934. }
  935. return opts
  936. }
  937. // Writer wrapper inspired by https://blog.golang.org/errors-are-values
  938. type errWriter struct {
  939. writer io.Writer
  940. err error
  941. }
  942. func (w *errWriter) write(str string) {
  943. if w.err != nil {
  944. return
  945. }
  946. _, w.err = w.writer.Write([]byte(str))
  947. }
  948. // Map fields may have key types of non-float scalars, strings and enums.
  949. // The easiest way to sort them in some deterministic order is to use fmt.
  950. // If this turns out to be inefficient we can always consider other options,
  951. // such as doing a Schwartzian transform.
  952. //
  953. // Numeric keys are sorted in numeric order per
  954. // https://developers.google.com/protocol-buffers/docs/proto#maps.
  955. type mapKeys []reflect.Value
  956. func (s mapKeys) Len() int { return len(s) }
  957. func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  958. func (s mapKeys) Less(i, j int) bool {
  959. if k := s[i].Kind(); k == s[j].Kind() {
  960. switch k {
  961. case reflect.Int32, reflect.Int64:
  962. return s[i].Int() < s[j].Int()
  963. case reflect.Uint32, reflect.Uint64:
  964. return s[i].Uint() < s[j].Uint()
  965. }
  966. }
  967. return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface())
  968. }