jsonpb.go 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241
  1. // Go support for Protocol Buffers - Google's data interchange format
  2. //
  3. // Copyright 2015 The Go Authors. All rights reserved.
  4. // https://github.com/golang/protobuf
  5. //
  6. // Redistribution and use in source and binary forms, with or without
  7. // modification, are permitted provided that the following conditions are
  8. // met:
  9. //
  10. // * Redistributions of source code must retain the above copyright
  11. // notice, this list of conditions and the following disclaimer.
  12. // * Redistributions in binary form must reproduce the above
  13. // copyright notice, this list of conditions and the following disclaimer
  14. // in the documentation and/or other materials provided with the
  15. // distribution.
  16. // * Neither the name of Google Inc. nor the names of its
  17. // contributors may be used to endorse or promote products derived from
  18. // this software without specific prior written permission.
  19. //
  20. // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21. // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22. // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23. // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24. // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25. // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26. // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27. // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28. // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29. // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31. /*
  32. Package jsonpb provides marshaling and unmarshaling between protocol buffers and JSON.
  33. It follows the specification at https://developers.google.com/protocol-buffers/docs/proto3#json.
  34. This package produces a different output than the standard "encoding/json" package,
  35. which does not operate correctly on protocol buffers.
  36. */
  37. package jsonpb
  38. import (
  39. "bytes"
  40. "encoding/json"
  41. "errors"
  42. "fmt"
  43. "io"
  44. "math"
  45. "reflect"
  46. "sort"
  47. "strconv"
  48. "strings"
  49. "time"
  50. "github.com/golang/protobuf/proto"
  51. stpb "github.com/golang/protobuf/ptypes/struct"
  52. )
  53. const secondInNanos = int64(time.Second / time.Nanosecond)
  54. // Marshaler is a configurable object for converting between
  55. // protocol buffer objects and a JSON representation for them.
  56. type Marshaler struct {
  57. // Whether to render enum values as integers, as opposed to string values.
  58. EnumsAsInts bool
  59. // Whether to render fields with zero values.
  60. EmitDefaults bool
  61. // A string to indent each level by. The presence of this field will
  62. // also cause a space to appear between the field separator and
  63. // value, and for newlines to be appear between fields and array
  64. // elements.
  65. Indent string
  66. // Whether to use the original (.proto) name for fields.
  67. OrigName bool
  68. // A custom URL resolver to use when marshaling Any messages to JSON.
  69. // If unset, the default resolution strategy is to extract the
  70. // fully-qualified type name from the type URL and pass that to
  71. // proto.MessageType(string).
  72. AnyResolver AnyResolver
  73. }
  74. // AnyResolver takes a type URL, present in an Any message, and resolves it into
  75. // an instance of the associated message.
  76. type AnyResolver interface {
  77. Resolve(typeUrl string) (proto.Message, error)
  78. }
  79. func defaultResolveAny(typeUrl string) (proto.Message, error) {
  80. // Only the part of typeUrl after the last slash is relevant.
  81. mname := typeUrl
  82. if slash := strings.LastIndex(mname, "/"); slash >= 0 {
  83. mname = mname[slash+1:]
  84. }
  85. mt := proto.MessageType(mname)
  86. if mt == nil {
  87. return nil, fmt.Errorf("unknown message type %q", mname)
  88. }
  89. return reflect.New(mt.Elem()).Interface().(proto.Message), nil
  90. }
  91. // JSONPBMarshaler is implemented by protobuf messages that customize the
  92. // way they are marshaled to JSON. Messages that implement this should
  93. // also implement JSONPBUnmarshaler so that the custom format can be
  94. // parsed.
  95. type JSONPBMarshaler interface {
  96. MarshalJSONPB(*Marshaler) ([]byte, error)
  97. }
  98. // JSONPBUnmarshaler is implemented by protobuf messages that customize
  99. // the way they are unmarshaled from JSON. Messages that implement this
  100. // should also implement JSONPBMarshaler so that the custom format can be
  101. // produced.
  102. type JSONPBUnmarshaler interface {
  103. UnmarshalJSONPB(*Unmarshaler, []byte) error
  104. }
  105. // Marshal marshals a protocol buffer into JSON.
  106. func (m *Marshaler) Marshal(out io.Writer, pb proto.Message) error {
  107. v := reflect.ValueOf(pb)
  108. if pb == nil || (v.Kind() == reflect.Ptr && v.IsNil()) {
  109. return errors.New("Marshal called with nil")
  110. }
  111. // Check for unset required fields first.
  112. if err := checkRequiredFields(pb); err != nil {
  113. return err
  114. }
  115. writer := &errWriter{writer: out}
  116. return m.marshalObject(writer, pb, "", "")
  117. }
  118. // MarshalToString converts a protocol buffer object to JSON string.
  119. func (m *Marshaler) MarshalToString(pb proto.Message) (string, error) {
  120. var buf bytes.Buffer
  121. if err := m.Marshal(&buf, pb); err != nil {
  122. return "", err
  123. }
  124. return buf.String(), nil
  125. }
  126. type int32Slice []int32
  127. var nonFinite = map[string]float64{
  128. `"NaN"`: math.NaN(),
  129. `"Infinity"`: math.Inf(1),
  130. `"-Infinity"`: math.Inf(-1),
  131. }
  132. // For sorting extensions ids to ensure stable output.
  133. func (s int32Slice) Len() int { return len(s) }
  134. func (s int32Slice) Less(i, j int) bool { return s[i] < s[j] }
  135. func (s int32Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  136. type wkt interface {
  137. XXX_WellKnownType() string
  138. }
  139. // marshalObject writes a struct to the Writer.
  140. func (m *Marshaler) marshalObject(out *errWriter, v proto.Message, indent, typeURL string) error {
  141. if jsm, ok := v.(JSONPBMarshaler); ok {
  142. b, err := jsm.MarshalJSONPB(m)
  143. if err != nil {
  144. return err
  145. }
  146. if typeURL != "" {
  147. // we are marshaling this object to an Any type
  148. var js map[string]*json.RawMessage
  149. if err = json.Unmarshal(b, &js); err != nil {
  150. return fmt.Errorf("type %T produced invalid JSON: %v", v, err)
  151. }
  152. turl, err := json.Marshal(typeURL)
  153. if err != nil {
  154. return fmt.Errorf("failed to marshal type URL %q to JSON: %v", typeURL, err)
  155. }
  156. js["@type"] = (*json.RawMessage)(&turl)
  157. if b, err = json.Marshal(js); err != nil {
  158. return err
  159. }
  160. }
  161. out.write(string(b))
  162. return out.err
  163. }
  164. s := reflect.ValueOf(v).Elem()
  165. // Handle well-known types.
  166. if wkt, ok := v.(wkt); ok {
  167. switch wkt.XXX_WellKnownType() {
  168. case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
  169. "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
  170. // "Wrappers use the same representation in JSON
  171. // as the wrapped primitive type, ..."
  172. sprop := proto.GetProperties(s.Type())
  173. return m.marshalValue(out, sprop.Prop[0], s.Field(0), indent)
  174. case "Any":
  175. // Any is a bit more involved.
  176. return m.marshalAny(out, v, indent)
  177. case "Duration":
  178. // "Generated output always contains 0, 3, 6, or 9 fractional digits,
  179. // depending on required precision."
  180. s, ns := s.Field(0).Int(), s.Field(1).Int()
  181. if ns <= -secondInNanos || ns >= secondInNanos {
  182. return fmt.Errorf("ns out of range (%v, %v)", -secondInNanos, secondInNanos)
  183. }
  184. if (s > 0 && ns < 0) || (s < 0 && ns > 0) {
  185. return errors.New("signs of seconds and nanos do not match")
  186. }
  187. if s < 0 {
  188. ns = -ns
  189. }
  190. x := fmt.Sprintf("%d.%09d", s, ns)
  191. x = strings.TrimSuffix(x, "000")
  192. x = strings.TrimSuffix(x, "000")
  193. x = strings.TrimSuffix(x, ".000")
  194. out.write(`"`)
  195. out.write(x)
  196. out.write(`s"`)
  197. return out.err
  198. case "Struct", "ListValue":
  199. // Let marshalValue handle the `Struct.fields` map or the `ListValue.values` slice.
  200. // TODO: pass the correct Properties if needed.
  201. return m.marshalValue(out, &proto.Properties{}, s.Field(0), indent)
  202. case "Timestamp":
  203. // "RFC 3339, where generated output will always be Z-normalized
  204. // and uses 0, 3, 6 or 9 fractional digits."
  205. s, ns := s.Field(0).Int(), s.Field(1).Int()
  206. if ns < 0 || ns >= secondInNanos {
  207. return fmt.Errorf("ns out of range [0, %v)", secondInNanos)
  208. }
  209. t := time.Unix(s, ns).UTC()
  210. // time.RFC3339Nano isn't exactly right (we need to get 3/6/9 fractional digits).
  211. x := t.Format("2006-01-02T15:04:05.000000000")
  212. x = strings.TrimSuffix(x, "000")
  213. x = strings.TrimSuffix(x, "000")
  214. x = strings.TrimSuffix(x, ".000")
  215. out.write(`"`)
  216. out.write(x)
  217. out.write(`Z"`)
  218. return out.err
  219. case "Value":
  220. // Value has a single oneof.
  221. kind := s.Field(0)
  222. if kind.IsNil() {
  223. // "absence of any variant indicates an error"
  224. return errors.New("nil Value")
  225. }
  226. // oneof -> *T -> T -> T.F
  227. x := kind.Elem().Elem().Field(0)
  228. // TODO: pass the correct Properties if needed.
  229. return m.marshalValue(out, &proto.Properties{}, x, indent)
  230. }
  231. }
  232. out.write("{")
  233. if m.Indent != "" {
  234. out.write("\n")
  235. }
  236. firstField := true
  237. if typeURL != "" {
  238. if err := m.marshalTypeURL(out, indent, typeURL); err != nil {
  239. return err
  240. }
  241. firstField = false
  242. }
  243. for i := 0; i < s.NumField(); i++ {
  244. value := s.Field(i)
  245. valueField := s.Type().Field(i)
  246. if strings.HasPrefix(valueField.Name, "XXX_") {
  247. continue
  248. }
  249. // IsNil will panic on most value kinds.
  250. switch value.Kind() {
  251. case reflect.Chan, reflect.Func, reflect.Interface:
  252. if value.IsNil() {
  253. continue
  254. }
  255. }
  256. if !m.EmitDefaults {
  257. switch value.Kind() {
  258. case reflect.Bool:
  259. if !value.Bool() {
  260. continue
  261. }
  262. case reflect.Int32, reflect.Int64:
  263. if value.Int() == 0 {
  264. continue
  265. }
  266. case reflect.Uint32, reflect.Uint64:
  267. if value.Uint() == 0 {
  268. continue
  269. }
  270. case reflect.Float32, reflect.Float64:
  271. if value.Float() == 0 {
  272. continue
  273. }
  274. case reflect.String:
  275. if value.Len() == 0 {
  276. continue
  277. }
  278. case reflect.Map, reflect.Ptr, reflect.Slice:
  279. if value.IsNil() {
  280. continue
  281. }
  282. }
  283. }
  284. // Oneof fields need special handling.
  285. if valueField.Tag.Get("protobuf_oneof") != "" {
  286. // value is an interface containing &T{real_value}.
  287. sv := value.Elem().Elem() // interface -> *T -> T
  288. value = sv.Field(0)
  289. valueField = sv.Type().Field(0)
  290. }
  291. prop := jsonProperties(valueField, m.OrigName)
  292. if !firstField {
  293. m.writeSep(out)
  294. }
  295. if err := m.marshalField(out, prop, value, indent); err != nil {
  296. return err
  297. }
  298. firstField = false
  299. }
  300. // Handle proto2 extensions.
  301. if ep, ok := v.(proto.Message); ok {
  302. extensions := proto.RegisteredExtensions(v)
  303. // Sort extensions for stable output.
  304. ids := make([]int32, 0, len(extensions))
  305. for id, desc := range extensions {
  306. if !proto.HasExtension(ep, desc) {
  307. continue
  308. }
  309. ids = append(ids, id)
  310. }
  311. sort.Sort(int32Slice(ids))
  312. for _, id := range ids {
  313. desc := extensions[id]
  314. if desc == nil {
  315. // unknown extension
  316. continue
  317. }
  318. ext, extErr := proto.GetExtension(ep, desc)
  319. if extErr != nil {
  320. return extErr
  321. }
  322. value := reflect.ValueOf(ext)
  323. var prop proto.Properties
  324. prop.Parse(desc.Tag)
  325. prop.JSONName = fmt.Sprintf("[%s]", desc.Name)
  326. if !firstField {
  327. m.writeSep(out)
  328. }
  329. if err := m.marshalField(out, &prop, value, indent); err != nil {
  330. return err
  331. }
  332. firstField = false
  333. }
  334. }
  335. if m.Indent != "" {
  336. out.write("\n")
  337. out.write(indent)
  338. }
  339. out.write("}")
  340. return out.err
  341. }
  342. func (m *Marshaler) writeSep(out *errWriter) {
  343. if m.Indent != "" {
  344. out.write(",\n")
  345. } else {
  346. out.write(",")
  347. }
  348. }
  349. func (m *Marshaler) marshalAny(out *errWriter, any proto.Message, indent string) error {
  350. // "If the Any contains a value that has a special JSON mapping,
  351. // it will be converted as follows: {"@type": xxx, "value": yyy}.
  352. // Otherwise, the value will be converted into a JSON object,
  353. // and the "@type" field will be inserted to indicate the actual data type."
  354. v := reflect.ValueOf(any).Elem()
  355. turl := v.Field(0).String()
  356. val := v.Field(1).Bytes()
  357. var msg proto.Message
  358. var err error
  359. if m.AnyResolver != nil {
  360. msg, err = m.AnyResolver.Resolve(turl)
  361. } else {
  362. msg, err = defaultResolveAny(turl)
  363. }
  364. if err != nil {
  365. return err
  366. }
  367. if err := proto.Unmarshal(val, msg); err != nil {
  368. return err
  369. }
  370. if _, ok := msg.(wkt); ok {
  371. out.write("{")
  372. if m.Indent != "" {
  373. out.write("\n")
  374. }
  375. if err := m.marshalTypeURL(out, indent, turl); err != nil {
  376. return err
  377. }
  378. m.writeSep(out)
  379. if m.Indent != "" {
  380. out.write(indent)
  381. out.write(m.Indent)
  382. out.write(`"value": `)
  383. } else {
  384. out.write(`"value":`)
  385. }
  386. if err := m.marshalObject(out, msg, indent+m.Indent, ""); err != nil {
  387. return err
  388. }
  389. if m.Indent != "" {
  390. out.write("\n")
  391. out.write(indent)
  392. }
  393. out.write("}")
  394. return out.err
  395. }
  396. return m.marshalObject(out, msg, indent, turl)
  397. }
  398. func (m *Marshaler) marshalTypeURL(out *errWriter, indent, typeURL string) error {
  399. if m.Indent != "" {
  400. out.write(indent)
  401. out.write(m.Indent)
  402. }
  403. out.write(`"@type":`)
  404. if m.Indent != "" {
  405. out.write(" ")
  406. }
  407. b, err := json.Marshal(typeURL)
  408. if err != nil {
  409. return err
  410. }
  411. out.write(string(b))
  412. return out.err
  413. }
  414. // marshalField writes field description and value to the Writer.
  415. func (m *Marshaler) marshalField(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
  416. if m.Indent != "" {
  417. out.write(indent)
  418. out.write(m.Indent)
  419. }
  420. out.write(`"`)
  421. out.write(prop.JSONName)
  422. out.write(`":`)
  423. if m.Indent != "" {
  424. out.write(" ")
  425. }
  426. if err := m.marshalValue(out, prop, v, indent); err != nil {
  427. return err
  428. }
  429. return nil
  430. }
  431. // marshalValue writes the value to the Writer.
  432. func (m *Marshaler) marshalValue(out *errWriter, prop *proto.Properties, v reflect.Value, indent string) error {
  433. var err error
  434. v = reflect.Indirect(v)
  435. // Handle nil pointer
  436. if v.Kind() == reflect.Invalid {
  437. out.write("null")
  438. return out.err
  439. }
  440. // Handle repeated elements.
  441. if v.Kind() == reflect.Slice && v.Type().Elem().Kind() != reflect.Uint8 {
  442. out.write("[")
  443. comma := ""
  444. for i := 0; i < v.Len(); i++ {
  445. sliceVal := v.Index(i)
  446. out.write(comma)
  447. if m.Indent != "" {
  448. out.write("\n")
  449. out.write(indent)
  450. out.write(m.Indent)
  451. out.write(m.Indent)
  452. }
  453. if err := m.marshalValue(out, prop, sliceVal, indent+m.Indent); err != nil {
  454. return err
  455. }
  456. comma = ","
  457. }
  458. if m.Indent != "" {
  459. out.write("\n")
  460. out.write(indent)
  461. out.write(m.Indent)
  462. }
  463. out.write("]")
  464. return out.err
  465. }
  466. // Handle well-known types.
  467. // Most are handled up in marshalObject (because 99% are messages).
  468. if wkt, ok := v.Interface().(wkt); ok {
  469. switch wkt.XXX_WellKnownType() {
  470. case "NullValue":
  471. out.write("null")
  472. return out.err
  473. }
  474. }
  475. // Handle enumerations.
  476. if !m.EnumsAsInts && prop.Enum != "" {
  477. // Unknown enum values will are stringified by the proto library as their
  478. // value. Such values should _not_ be quoted or they will be interpreted
  479. // as an enum string instead of their value.
  480. enumStr := v.Interface().(fmt.Stringer).String()
  481. var valStr string
  482. if v.Kind() == reflect.Ptr {
  483. valStr = strconv.Itoa(int(v.Elem().Int()))
  484. } else {
  485. valStr = strconv.Itoa(int(v.Int()))
  486. }
  487. isKnownEnum := enumStr != valStr
  488. if isKnownEnum {
  489. out.write(`"`)
  490. }
  491. out.write(enumStr)
  492. if isKnownEnum {
  493. out.write(`"`)
  494. }
  495. return out.err
  496. }
  497. // Handle nested messages.
  498. if v.Kind() == reflect.Struct {
  499. return m.marshalObject(out, v.Addr().Interface().(proto.Message), indent+m.Indent, "")
  500. }
  501. // Handle maps.
  502. // Since Go randomizes map iteration, we sort keys for stable output.
  503. if v.Kind() == reflect.Map {
  504. out.write(`{`)
  505. keys := v.MapKeys()
  506. sort.Sort(mapKeys(keys))
  507. for i, k := range keys {
  508. if i > 0 {
  509. out.write(`,`)
  510. }
  511. if m.Indent != "" {
  512. out.write("\n")
  513. out.write(indent)
  514. out.write(m.Indent)
  515. out.write(m.Indent)
  516. }
  517. b, err := json.Marshal(k.Interface())
  518. if err != nil {
  519. return err
  520. }
  521. s := string(b)
  522. // If the JSON is not a string value, encode it again to make it one.
  523. if !strings.HasPrefix(s, `"`) {
  524. b, err := json.Marshal(s)
  525. if err != nil {
  526. return err
  527. }
  528. s = string(b)
  529. }
  530. out.write(s)
  531. out.write(`:`)
  532. if m.Indent != "" {
  533. out.write(` `)
  534. }
  535. if err := m.marshalValue(out, prop, v.MapIndex(k), indent+m.Indent); err != nil {
  536. return err
  537. }
  538. }
  539. if m.Indent != "" {
  540. out.write("\n")
  541. out.write(indent)
  542. out.write(m.Indent)
  543. }
  544. out.write(`}`)
  545. return out.err
  546. }
  547. // Handle non-finite floats, e.g. NaN, Infinity and -Infinity.
  548. if v.Kind() == reflect.Float32 || v.Kind() == reflect.Float64 {
  549. f := v.Float()
  550. var sval string
  551. switch {
  552. case math.IsInf(f, 1):
  553. sval = `"Infinity"`
  554. case math.IsInf(f, -1):
  555. sval = `"-Infinity"`
  556. case math.IsNaN(f):
  557. sval = `"NaN"`
  558. }
  559. if sval != "" {
  560. out.write(sval)
  561. return out.err
  562. }
  563. }
  564. // Default handling defers to the encoding/json library.
  565. b, err := json.Marshal(v.Interface())
  566. if err != nil {
  567. return err
  568. }
  569. needToQuote := string(b[0]) != `"` && (v.Kind() == reflect.Int64 || v.Kind() == reflect.Uint64)
  570. if needToQuote {
  571. out.write(`"`)
  572. }
  573. out.write(string(b))
  574. if needToQuote {
  575. out.write(`"`)
  576. }
  577. return out.err
  578. }
  579. // Unmarshaler is a configurable object for converting from a JSON
  580. // representation to a protocol buffer object.
  581. type Unmarshaler struct {
  582. // Whether to allow messages to contain unknown fields, as opposed to
  583. // failing to unmarshal.
  584. AllowUnknownFields bool
  585. // A custom URL resolver to use when unmarshaling Any messages from JSON.
  586. // If unset, the default resolution strategy is to extract the
  587. // fully-qualified type name from the type URL and pass that to
  588. // proto.MessageType(string).
  589. AnyResolver AnyResolver
  590. }
  591. // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
  592. // This function is lenient and will decode any options permutations of the
  593. // related Marshaler.
  594. func (u *Unmarshaler) UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
  595. inputValue := json.RawMessage{}
  596. if err := dec.Decode(&inputValue); err != nil {
  597. return err
  598. }
  599. if err := u.unmarshalValue(reflect.ValueOf(pb).Elem(), inputValue, nil); err != nil {
  600. return err
  601. }
  602. return checkRequiredFields(pb)
  603. }
  604. // Unmarshal unmarshals a JSON object stream into a protocol
  605. // buffer. This function is lenient and will decode any options
  606. // permutations of the related Marshaler.
  607. func (u *Unmarshaler) Unmarshal(r io.Reader, pb proto.Message) error {
  608. dec := json.NewDecoder(r)
  609. return u.UnmarshalNext(dec, pb)
  610. }
  611. // UnmarshalNext unmarshals the next protocol buffer from a JSON object stream.
  612. // This function is lenient and will decode any options permutations of the
  613. // related Marshaler.
  614. func UnmarshalNext(dec *json.Decoder, pb proto.Message) error {
  615. return new(Unmarshaler).UnmarshalNext(dec, pb)
  616. }
  617. // Unmarshal unmarshals a JSON object stream into a protocol
  618. // buffer. This function is lenient and will decode any options
  619. // permutations of the related Marshaler.
  620. func Unmarshal(r io.Reader, pb proto.Message) error {
  621. return new(Unmarshaler).Unmarshal(r, pb)
  622. }
  623. // UnmarshalString will populate the fields of a protocol buffer based
  624. // on a JSON string. This function is lenient and will decode any options
  625. // permutations of the related Marshaler.
  626. func UnmarshalString(str string, pb proto.Message) error {
  627. return new(Unmarshaler).Unmarshal(strings.NewReader(str), pb)
  628. }
  629. // unmarshalValue converts/copies a value into the target.
  630. // prop may be nil.
  631. func (u *Unmarshaler) unmarshalValue(target reflect.Value, inputValue json.RawMessage, prop *proto.Properties) error {
  632. targetType := target.Type()
  633. // Allocate memory for pointer fields.
  634. if targetType.Kind() == reflect.Ptr {
  635. // If input value is "null" and target is a pointer type, then the field should be treated as not set
  636. // UNLESS the target is structpb.Value, in which case it should be set to structpb.NullValue.
  637. _, isJSONPBUnmarshaler := target.Interface().(JSONPBUnmarshaler)
  638. if string(inputValue) == "null" && targetType != reflect.TypeOf(&stpb.Value{}) && !isJSONPBUnmarshaler {
  639. return nil
  640. }
  641. target.Set(reflect.New(targetType.Elem()))
  642. return u.unmarshalValue(target.Elem(), inputValue, prop)
  643. }
  644. if jsu, ok := target.Addr().Interface().(JSONPBUnmarshaler); ok {
  645. return jsu.UnmarshalJSONPB(u, []byte(inputValue))
  646. }
  647. // Handle well-known types that are not pointers.
  648. if w, ok := target.Addr().Interface().(wkt); ok {
  649. switch w.XXX_WellKnownType() {
  650. case "DoubleValue", "FloatValue", "Int64Value", "UInt64Value",
  651. "Int32Value", "UInt32Value", "BoolValue", "StringValue", "BytesValue":
  652. return u.unmarshalValue(target.Field(0), inputValue, prop)
  653. case "Any":
  654. // Use json.RawMessage pointer type instead of value to support pre-1.8 version.
  655. // 1.8 changed RawMessage.MarshalJSON from pointer type to value type, see
  656. // https://github.com/golang/go/issues/14493
  657. var jsonFields map[string]*json.RawMessage
  658. if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
  659. return err
  660. }
  661. val, ok := jsonFields["@type"]
  662. if !ok || val == nil {
  663. return errors.New("Any JSON doesn't have '@type'")
  664. }
  665. var turl string
  666. if err := json.Unmarshal([]byte(*val), &turl); err != nil {
  667. return fmt.Errorf("can't unmarshal Any's '@type': %q", *val)
  668. }
  669. target.Field(0).SetString(turl)
  670. var m proto.Message
  671. var err error
  672. if u.AnyResolver != nil {
  673. m, err = u.AnyResolver.Resolve(turl)
  674. } else {
  675. m, err = defaultResolveAny(turl)
  676. }
  677. if err != nil {
  678. return err
  679. }
  680. if _, ok := m.(wkt); ok {
  681. val, ok := jsonFields["value"]
  682. if !ok {
  683. return errors.New("Any JSON doesn't have 'value'")
  684. }
  685. if err := u.unmarshalValue(reflect.ValueOf(m).Elem(), *val, nil); err != nil {
  686. return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err)
  687. }
  688. } else {
  689. delete(jsonFields, "@type")
  690. nestedProto, err := json.Marshal(jsonFields)
  691. if err != nil {
  692. return fmt.Errorf("can't generate JSON for Any's nested proto to be unmarshaled: %v", err)
  693. }
  694. if err = u.unmarshalValue(reflect.ValueOf(m).Elem(), nestedProto, nil); err != nil {
  695. return fmt.Errorf("can't unmarshal Any nested proto %T: %v", m, err)
  696. }
  697. }
  698. b, err := proto.Marshal(m)
  699. if err != nil {
  700. return fmt.Errorf("can't marshal proto %T into Any.Value: %v", m, err)
  701. }
  702. target.Field(1).SetBytes(b)
  703. return nil
  704. case "Duration":
  705. unq, err := strconv.Unquote(string(inputValue))
  706. if err != nil {
  707. return err
  708. }
  709. d, err := time.ParseDuration(unq)
  710. if err != nil {
  711. return fmt.Errorf("bad Duration: %v", err)
  712. }
  713. ns := d.Nanoseconds()
  714. s := ns / 1e9
  715. ns %= 1e9
  716. target.Field(0).SetInt(s)
  717. target.Field(1).SetInt(ns)
  718. return nil
  719. case "Timestamp":
  720. unq, err := strconv.Unquote(string(inputValue))
  721. if err != nil {
  722. return err
  723. }
  724. t, err := time.Parse(time.RFC3339Nano, unq)
  725. if err != nil {
  726. return fmt.Errorf("bad Timestamp: %v", err)
  727. }
  728. target.Field(0).SetInt(t.Unix())
  729. target.Field(1).SetInt(int64(t.Nanosecond()))
  730. return nil
  731. case "Struct":
  732. var m map[string]json.RawMessage
  733. if err := json.Unmarshal(inputValue, &m); err != nil {
  734. return fmt.Errorf("bad StructValue: %v", err)
  735. }
  736. target.Field(0).Set(reflect.ValueOf(map[string]*stpb.Value{}))
  737. for k, jv := range m {
  738. pv := &stpb.Value{}
  739. if err := u.unmarshalValue(reflect.ValueOf(pv).Elem(), jv, prop); err != nil {
  740. return fmt.Errorf("bad value in StructValue for key %q: %v", k, err)
  741. }
  742. target.Field(0).SetMapIndex(reflect.ValueOf(k), reflect.ValueOf(pv))
  743. }
  744. return nil
  745. case "ListValue":
  746. var s []json.RawMessage
  747. if err := json.Unmarshal(inputValue, &s); err != nil {
  748. return fmt.Errorf("bad ListValue: %v", err)
  749. }
  750. target.Field(0).Set(reflect.ValueOf(make([]*stpb.Value, len(s))))
  751. for i, sv := range s {
  752. if err := u.unmarshalValue(target.Field(0).Index(i), sv, prop); err != nil {
  753. return err
  754. }
  755. }
  756. return nil
  757. case "Value":
  758. ivStr := string(inputValue)
  759. if ivStr == "null" {
  760. target.Field(0).Set(reflect.ValueOf(&stpb.Value_NullValue{}))
  761. } else if v, err := strconv.ParseFloat(ivStr, 0); err == nil {
  762. target.Field(0).Set(reflect.ValueOf(&stpb.Value_NumberValue{v}))
  763. } else if v, err := strconv.Unquote(ivStr); err == nil {
  764. target.Field(0).Set(reflect.ValueOf(&stpb.Value_StringValue{v}))
  765. } else if v, err := strconv.ParseBool(ivStr); err == nil {
  766. target.Field(0).Set(reflect.ValueOf(&stpb.Value_BoolValue{v}))
  767. } else if err := json.Unmarshal(inputValue, &[]json.RawMessage{}); err == nil {
  768. lv := &stpb.ListValue{}
  769. target.Field(0).Set(reflect.ValueOf(&stpb.Value_ListValue{lv}))
  770. return u.unmarshalValue(reflect.ValueOf(lv).Elem(), inputValue, prop)
  771. } else if err := json.Unmarshal(inputValue, &map[string]json.RawMessage{}); err == nil {
  772. sv := &stpb.Struct{}
  773. target.Field(0).Set(reflect.ValueOf(&stpb.Value_StructValue{sv}))
  774. return u.unmarshalValue(reflect.ValueOf(sv).Elem(), inputValue, prop)
  775. } else {
  776. return fmt.Errorf("unrecognized type for Value %q", ivStr)
  777. }
  778. return nil
  779. }
  780. }
  781. // Handle enums, which have an underlying type of int32,
  782. // and may appear as strings.
  783. // The case of an enum appearing as a number is handled
  784. // at the bottom of this function.
  785. if inputValue[0] == '"' && prop != nil && prop.Enum != "" {
  786. vmap := proto.EnumValueMap(prop.Enum)
  787. // Don't need to do unquoting; valid enum names
  788. // are from a limited character set.
  789. s := inputValue[1 : len(inputValue)-1]
  790. n, ok := vmap[string(s)]
  791. if !ok {
  792. return fmt.Errorf("unknown value %q for enum %s", s, prop.Enum)
  793. }
  794. if target.Kind() == reflect.Ptr { // proto2
  795. target.Set(reflect.New(targetType.Elem()))
  796. target = target.Elem()
  797. }
  798. target.SetInt(int64(n))
  799. return nil
  800. }
  801. // Handle nested messages.
  802. if targetType.Kind() == reflect.Struct {
  803. var jsonFields map[string]json.RawMessage
  804. if err := json.Unmarshal(inputValue, &jsonFields); err != nil {
  805. return err
  806. }
  807. consumeField := func(prop *proto.Properties) (json.RawMessage, bool) {
  808. // Be liberal in what names we accept; both orig_name and camelName are okay.
  809. fieldNames := acceptedJSONFieldNames(prop)
  810. vOrig, okOrig := jsonFields[fieldNames.orig]
  811. vCamel, okCamel := jsonFields[fieldNames.camel]
  812. if !okOrig && !okCamel {
  813. return nil, false
  814. }
  815. // If, for some reason, both are present in the data, favour the camelName.
  816. var raw json.RawMessage
  817. if okOrig {
  818. raw = vOrig
  819. delete(jsonFields, fieldNames.orig)
  820. }
  821. if okCamel {
  822. raw = vCamel
  823. delete(jsonFields, fieldNames.camel)
  824. }
  825. return raw, true
  826. }
  827. sprops := proto.GetProperties(targetType)
  828. for i := 0; i < target.NumField(); i++ {
  829. ft := target.Type().Field(i)
  830. if strings.HasPrefix(ft.Name, "XXX_") {
  831. continue
  832. }
  833. valueForField, ok := consumeField(sprops.Prop[i])
  834. if !ok {
  835. continue
  836. }
  837. if err := u.unmarshalValue(target.Field(i), valueForField, sprops.Prop[i]); err != nil {
  838. return err
  839. }
  840. }
  841. // Check for any oneof fields.
  842. if len(jsonFields) > 0 {
  843. for _, oop := range sprops.OneofTypes {
  844. raw, ok := consumeField(oop.Prop)
  845. if !ok {
  846. continue
  847. }
  848. nv := reflect.New(oop.Type.Elem())
  849. target.Field(oop.Field).Set(nv)
  850. if err := u.unmarshalValue(nv.Elem().Field(0), raw, oop.Prop); err != nil {
  851. return err
  852. }
  853. }
  854. }
  855. // Handle proto2 extensions.
  856. if len(jsonFields) > 0 {
  857. if ep, ok := target.Addr().Interface().(proto.Message); ok {
  858. for _, ext := range proto.RegisteredExtensions(ep) {
  859. name := fmt.Sprintf("[%s]", ext.Name)
  860. raw, ok := jsonFields[name]
  861. if !ok {
  862. continue
  863. }
  864. delete(jsonFields, name)
  865. nv := reflect.New(reflect.TypeOf(ext.ExtensionType).Elem())
  866. if err := u.unmarshalValue(nv.Elem(), raw, nil); err != nil {
  867. return err
  868. }
  869. if err := proto.SetExtension(ep, ext, nv.Interface()); err != nil {
  870. return err
  871. }
  872. }
  873. }
  874. }
  875. if !u.AllowUnknownFields && len(jsonFields) > 0 {
  876. // Pick any field to be the scapegoat.
  877. var f string
  878. for fname := range jsonFields {
  879. f = fname
  880. break
  881. }
  882. return fmt.Errorf("unknown field %q in %v", f, targetType)
  883. }
  884. return nil
  885. }
  886. // Handle arrays (which aren't encoded bytes)
  887. if targetType.Kind() == reflect.Slice && targetType.Elem().Kind() != reflect.Uint8 {
  888. var slc []json.RawMessage
  889. if err := json.Unmarshal(inputValue, &slc); err != nil {
  890. return err
  891. }
  892. if slc != nil {
  893. l := len(slc)
  894. target.Set(reflect.MakeSlice(targetType, l, l))
  895. for i := 0; i < l; i++ {
  896. if err := u.unmarshalValue(target.Index(i), slc[i], prop); err != nil {
  897. return err
  898. }
  899. }
  900. }
  901. return nil
  902. }
  903. // Handle maps (whose keys are always strings)
  904. if targetType.Kind() == reflect.Map {
  905. var mp map[string]json.RawMessage
  906. if err := json.Unmarshal(inputValue, &mp); err != nil {
  907. return err
  908. }
  909. if mp != nil {
  910. target.Set(reflect.MakeMap(targetType))
  911. for ks, raw := range mp {
  912. // Unmarshal map key. The core json library already decoded the key into a
  913. // string, so we handle that specially. Other types were quoted post-serialization.
  914. var k reflect.Value
  915. if targetType.Key().Kind() == reflect.String {
  916. k = reflect.ValueOf(ks)
  917. } else {
  918. k = reflect.New(targetType.Key()).Elem()
  919. // TODO: pass the correct Properties if needed.
  920. if err := u.unmarshalValue(k, json.RawMessage(ks), nil); err != nil {
  921. return err
  922. }
  923. }
  924. // Unmarshal map value.
  925. v := reflect.New(targetType.Elem()).Elem()
  926. // TODO: pass the correct Properties if needed.
  927. if err := u.unmarshalValue(v, raw, nil); err != nil {
  928. return err
  929. }
  930. target.SetMapIndex(k, v)
  931. }
  932. }
  933. return nil
  934. }
  935. // 64-bit integers can be encoded as strings. In this case we drop
  936. // the quotes and proceed as normal.
  937. isNum := targetType.Kind() == reflect.Int64 || targetType.Kind() == reflect.Uint64
  938. if isNum && strings.HasPrefix(string(inputValue), `"`) {
  939. inputValue = inputValue[1 : len(inputValue)-1]
  940. }
  941. // Non-finite numbers can be encoded as strings.
  942. isFloat := targetType.Kind() == reflect.Float32 || targetType.Kind() == reflect.Float64
  943. if isFloat {
  944. if num, ok := nonFinite[string(inputValue)]; ok {
  945. target.SetFloat(num)
  946. return nil
  947. }
  948. }
  949. // Use the encoding/json for parsing other value types.
  950. return json.Unmarshal(inputValue, target.Addr().Interface())
  951. }
  952. // jsonProperties returns parsed proto.Properties for the field and corrects JSONName attribute.
  953. func jsonProperties(f reflect.StructField, origName bool) *proto.Properties {
  954. var prop proto.Properties
  955. prop.Init(f.Type, f.Name, f.Tag.Get("protobuf"), &f)
  956. if origName || prop.JSONName == "" {
  957. prop.JSONName = prop.OrigName
  958. }
  959. return &prop
  960. }
  961. type fieldNames struct {
  962. orig, camel string
  963. }
  964. func acceptedJSONFieldNames(prop *proto.Properties) fieldNames {
  965. opts := fieldNames{orig: prop.OrigName, camel: prop.OrigName}
  966. if prop.JSONName != "" {
  967. opts.camel = prop.JSONName
  968. }
  969. return opts
  970. }
  971. // Writer wrapper inspired by https://blog.golang.org/errors-are-values
  972. type errWriter struct {
  973. writer io.Writer
  974. err error
  975. }
  976. func (w *errWriter) write(str string) {
  977. if w.err != nil {
  978. return
  979. }
  980. _, w.err = w.writer.Write([]byte(str))
  981. }
  982. // Map fields may have key types of non-float scalars, strings and enums.
  983. // The easiest way to sort them in some deterministic order is to use fmt.
  984. // If this turns out to be inefficient we can always consider other options,
  985. // such as doing a Schwartzian transform.
  986. //
  987. // Numeric keys are sorted in numeric order per
  988. // https://developers.google.com/protocol-buffers/docs/proto#maps.
  989. type mapKeys []reflect.Value
  990. func (s mapKeys) Len() int { return len(s) }
  991. func (s mapKeys) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
  992. func (s mapKeys) Less(i, j int) bool {
  993. if k := s[i].Kind(); k == s[j].Kind() {
  994. switch k {
  995. case reflect.Int32, reflect.Int64:
  996. return s[i].Int() < s[j].Int()
  997. case reflect.Uint32, reflect.Uint64:
  998. return s[i].Uint() < s[j].Uint()
  999. }
  1000. }
  1001. return fmt.Sprint(s[i].Interface()) < fmt.Sprint(s[j].Interface())
  1002. }
  1003. // checkRequiredFields returns an error if any required field in the given proto message is not set.
  1004. // This function is used by both Marshal and Unmarshal. While required fields only exist in a
  1005. // proto2 message, a proto3 message can contain proto2 message(s).
  1006. func checkRequiredFields(pb proto.Message) error {
  1007. // Most well-known type messages do not contain required fields. The "Any" type may contain
  1008. // a message that has required fields.
  1009. //
  1010. // When an Any message is being marshaled, the code will invoked proto.Unmarshal on Any.Value
  1011. // field in order to transform that into JSON, and that should have returned an error if a
  1012. // required field is not set in the embedded message.
  1013. //
  1014. // When an Any message is being unmarshaled, the code will have invoked proto.Marshal on the
  1015. // embedded message to store the serialized message in Any.Value field, and that should have
  1016. // returned an error if a required field is not set.
  1017. if _, ok := pb.(wkt); ok {
  1018. return nil
  1019. }
  1020. v := reflect.ValueOf(pb)
  1021. // Skip message if it is not a struct pointer.
  1022. if v.Kind() != reflect.Ptr {
  1023. return nil
  1024. }
  1025. v = v.Elem()
  1026. if v.Kind() != reflect.Struct {
  1027. return nil
  1028. }
  1029. for i := 0; i < v.NumField(); i++ {
  1030. field := v.Field(i)
  1031. sfield := v.Type().Field(i)
  1032. if sfield.PkgPath != "" {
  1033. // blank PkgPath means the field is exported; skip if not exported
  1034. continue
  1035. }
  1036. if strings.HasPrefix(sfield.Name, "XXX_") {
  1037. continue
  1038. }
  1039. // Oneof field is an interface implemented by wrapper structs containing the actual oneof
  1040. // field, i.e. an interface containing &T{real_value}.
  1041. if sfield.Tag.Get("protobuf_oneof") != "" {
  1042. if field.Kind() != reflect.Interface {
  1043. continue
  1044. }
  1045. v := field.Elem()
  1046. if v.Kind() != reflect.Ptr || v.IsNil() {
  1047. continue
  1048. }
  1049. v = v.Elem()
  1050. if v.Kind() != reflect.Struct || v.NumField() < 1 {
  1051. continue
  1052. }
  1053. field = v.Field(0)
  1054. sfield = v.Type().Field(0)
  1055. }
  1056. protoTag := sfield.Tag.Get("protobuf")
  1057. if protoTag == "" {
  1058. continue
  1059. }
  1060. var prop proto.Properties
  1061. prop.Init(sfield.Type, sfield.Name, protoTag, &sfield)
  1062. switch field.Kind() {
  1063. case reflect.Map:
  1064. if field.IsNil() {
  1065. continue
  1066. }
  1067. // Check each map value.
  1068. keys := field.MapKeys()
  1069. for _, k := range keys {
  1070. v := field.MapIndex(k)
  1071. if err := checkRequiredFieldsInValue(v); err != nil {
  1072. return err
  1073. }
  1074. }
  1075. case reflect.Slice:
  1076. // Handle non-repeated type, e.g. bytes.
  1077. if !prop.Repeated {
  1078. if prop.Required && field.IsNil() {
  1079. return fmt.Errorf("required field %q is not set", prop.Name)
  1080. }
  1081. continue
  1082. }
  1083. // Handle repeated type.
  1084. if field.IsNil() {
  1085. continue
  1086. }
  1087. // Check each slice item.
  1088. for i := 0; i < field.Len(); i++ {
  1089. v := field.Index(i)
  1090. if err := checkRequiredFieldsInValue(v); err != nil {
  1091. return err
  1092. }
  1093. }
  1094. case reflect.Ptr:
  1095. if field.IsNil() {
  1096. if prop.Required {
  1097. return fmt.Errorf("required field %q is not set", prop.Name)
  1098. }
  1099. continue
  1100. }
  1101. if err := checkRequiredFieldsInValue(field); err != nil {
  1102. return err
  1103. }
  1104. }
  1105. }
  1106. // Handle proto2 extensions.
  1107. for _, ext := range proto.RegisteredExtensions(pb) {
  1108. if !proto.HasExtension(pb, ext) {
  1109. continue
  1110. }
  1111. ep, err := proto.GetExtension(pb, ext)
  1112. if err != nil {
  1113. return err
  1114. }
  1115. err = checkRequiredFieldsInValue(reflect.ValueOf(ep))
  1116. if err != nil {
  1117. return err
  1118. }
  1119. }
  1120. return nil
  1121. }
  1122. func checkRequiredFieldsInValue(v reflect.Value) error {
  1123. if pm, ok := v.Interface().(proto.Message); ok {
  1124. return checkRequiredFields(pm)
  1125. }
  1126. return nil
  1127. }