encoding/jsonpb: add support for unmarshaling Any

Also added json.Decoder.Clone API for unmarshaling Any to look
ahead remaining bytes for @type field.

Change-Id: I2f803743534dfb64f9092d716805b115faa5975a
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/170102
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>

encoding/jsonpb/decode.go

@@ -59,7 +59,7 @@ func (o UnmarshalOptions) Unmarshal(m proto.Message, b []byte) error {
 	o.decoder = json.NewDecoder(b)
 
 	var nerr errors.NonFatal
-	if err := o.unmarshalMessage(mr); !nerr.Merge(err) {
+	if err := o.unmarshalMessage(mr, false); !nerr.Merge(err) {
 		return err
 	}
 
@@ -126,7 +126,7 @@ func newError(f string, x ...interface{}) error {
 }
 
 // unmarshalMessage unmarshals a message into the given protoreflect.Message.
-func (o UnmarshalOptions) unmarshalMessage(m pref.Message) error {
+func (o UnmarshalOptions) unmarshalMessage(m pref.Message, skipTypeURL bool) error {
 	var nerr errors.NonFatal
 
 	if isCustomType(m.Type().FullName()) {
@@ -141,7 +141,7 @@ func (o UnmarshalOptions) unmarshalMessage(m pref.Message) error {
 		return unexpectedJSONError{jval}
 	}
 
-	if err := o.unmarshalFields(m); !nerr.Merge(err) {
+	if err := o.unmarshalFields(m, skipTypeURL); !nerr.Merge(err) {
 		return err
 	}
 
@@ -149,7 +149,7 @@ func (o UnmarshalOptions) unmarshalMessage(m pref.Message) error {
 }
 
 // unmarshalFields unmarshals the fields into the given protoreflect.Message.
-func (o UnmarshalOptions) unmarshalFields(m pref.Message) error {
+func (o UnmarshalOptions) unmarshalFields(m pref.Message, skipTypeURL bool) error {
 	var nerr errors.NonFatal
 	var reqNums set.Ints
 	var seenNums set.Ints
@@ -179,6 +179,13 @@ Loop:
 		if !nerr.Merge(err) {
 			return err
 		}
+		// Unmarshaling a non-custom embedded message in Any will contain the
+		// JSON field "@type" which should be skipped because it is not a field
+		// of the embedded message, but simply an artifact of the Any format.
+		if skipTypeURL && name == "@type" {
+			o.decoder.Read()
+			continue
+		}
 
 		// Get the FieldDescriptor.
 		var fd pref.FieldDescriptor
@@ -280,7 +287,7 @@ func (o UnmarshalOptions) unmarshalSingular(knownFields pref.KnownFields, fd pre
 	switch fd.Kind() {
 	case pref.MessageKind, pref.GroupKind:
 		m := knownFields.NewMessage(num)
-		err = o.unmarshalMessage(m)
+		err = o.unmarshalMessage(m, false)
 		val = pref.ValueOf(m)
 	default:
 		val, err = o.unmarshalScalar(fd)
@@ -539,7 +546,7 @@ func (o UnmarshalOptions) unmarshalList(list pref.List, fd pref.FieldDescriptor)
 	case pref.MessageKind, pref.GroupKind:
 		for {
 			m := list.NewMessage()
-			err := o.unmarshalMessage(m)
+			err := o.unmarshalMessage(m, false)
 			if !nerr.Merge(err) {
 				if e, ok := err.(unexpectedJSONError); ok {
 					if e.value.Type() == json.EndArray {
@@ -596,7 +603,7 @@ func (o UnmarshalOptions) unmarshalMap(mmap pref.Map, fd pref.FieldDescriptor) e
 		unmarshalMapValue = func() (pref.Value, error) {
 			var nerr errors.NonFatal
 			m := mmap.NewMessage()
-			if err := o.unmarshalMessage(m); !nerr.Merge(err) {
+			if err := o.unmarshalMessage(m, false); !nerr.Merge(err) {
 				return pref.Value{}, err
 			}
 			return pref.ValueOf(m), nerr.E

encoding/jsonpb/decode_test.go

@@ -1866,6 +1866,535 @@ func TestUnmarshal(t *testing.T) {
 				},
 			},
 		},
+	}, {
+		desc:         "Any empty",
+		inputMessage: &knownpb.Any{},
+		inputText:    `{}`,
+		wantMessage:  &knownpb.Any{},
+	}, {
+		desc: "Any with non-custom message",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "foo/pb2.Nested",
+  "optString": "embedded inside Any",
+  "optNested": {
+    "optString": "inception"
+  }
+}`,
+		wantMessage: func() proto.Message {
+			m := &pb2.Nested{
+				OptString: scalar.String("embedded inside Any"),
+				OptNested: &pb2.Nested{
+					OptString: scalar.String("inception"),
+				},
+			}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "foo/pb2.Nested",
+				Value:   b,
+			}
+		}(),
+	}, {
+		desc: "Any with empty embedded message",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "foo/pb2.Nested"
+}`,
+		wantMessage: &knownpb.Any{TypeUrl: "foo/pb2.Nested"},
+	}, {
+		desc:         "Any without registered type",
+		umo:          jsonpb.UnmarshalOptions{Resolver: preg.NewTypes()},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "foo/pb2.Nested"
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with missing required error",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&pb2.PartialRequired{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "pb2.PartialRequired",
+  "optString": "embedded inside Any"
+}`,
+		wantMessage: func() proto.Message {
+			m := &pb2.PartialRequired{
+				OptString: scalar.String("embedded inside Any"),
+			}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			// TODO: Marshal may fail due to required field not set at some
+			// point. Need to ignore required not set error here.
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: string(m.ProtoReflect().Type().FullName()),
+				Value:   b,
+			}
+		}(),
+		wantErr: true,
+	}, {
+		desc: "Any with partial required and AllowPartial",
+		umo: jsonpb.UnmarshalOptions{
+			AllowPartial: true,
+			Resolver:     preg.NewTypes((&pb2.PartialRequired{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "pb2.PartialRequired",
+  "optString": "embedded inside Any"
+}`,
+		wantMessage: func() proto.Message {
+			m := &pb2.PartialRequired{
+				OptString: scalar.String("embedded inside Any"),
+			}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			// TODO: Marshal may fail due to required field not set at some
+			// point. Need to ignore required not set error here.
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: string(m.ProtoReflect().Type().FullName()),
+				Value:   b,
+			}
+		}(),
+	}, {
+		desc: "Any with invalid UTF8",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "optString": "` + "abc\xff" + `",
+  "@type": "foo/pb2.Nested"
+}`,
+		wantMessage: func() proto.Message {
+			m := &pb2.Nested{
+				OptString: scalar.String("abc\xff"),
+			}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "foo/pb2.Nested",
+				Value:   b,
+			}
+		}(),
+		wantErr: true,
+	}, {
+		desc: "Any with BoolValue",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.BoolValue{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "type.googleapis.com/google.protobuf.BoolValue",
+  "value": true
+}`,
+		wantMessage: func() proto.Message {
+			m := &knownpb.BoolValue{Value: true}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "type.googleapis.com/google.protobuf.BoolValue",
+				Value:   b,
+			}
+		}(),
+	}, {
+		desc: "Any with Empty",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Empty{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "value": {},
+  "@type": "type.googleapis.com/google.protobuf.Empty"
+}`,
+		wantMessage: func() proto.Message {
+			m := &knownpb.Empty{}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "type.googleapis.com/google.protobuf.Empty",
+				Value:   b,
+			}
+		}(),
+	}, {
+		desc: "Any with missing Empty",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Empty{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "type.googleapis.com/google.protobuf.Empty"
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with StringValue containing invalid UTF8",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.StringValue{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.StringValue",
+  "value": "` + "abc\xff" + `"
+}`,
+		wantMessage: func() proto.Message {
+			m := &knownpb.StringValue{Value: "abc\xff"}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "google.protobuf.StringValue",
+				Value:   b,
+			}
+		}(),
+		wantErr: true,
+	}, {
+		desc: "Any with Int64Value",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Int64Value{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.Int64Value",
+  "value": "42"
+}`,
+		wantMessage: func() proto.Message {
+			m := &knownpb.Int64Value{Value: 42}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "google.protobuf.Int64Value",
+				Value:   b,
+			}
+		}(),
+	}, {
+		desc: "Any with invalid Int64Value",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Int64Value{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.Int64Value",
+  "value": "forty-two"
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with invalid UInt64Value",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.UInt64Value{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.UInt64Value",
+  "value": -42
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with Duration",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Duration{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "type.googleapis.com/google.protobuf.Duration",
+  "value": "0s"
+}`,
+		wantMessage: func() proto.Message {
+			m := &knownpb.Duration{}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "type.googleapis.com/google.protobuf.Duration",
+				Value:   b,
+			}
+		}(),
+	}, {
+		desc: "Any with Value of StringValue",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.Value",
+  "value": "` + "abc\xff" + `"
+}`,
+		wantMessage: func() proto.Message {
+			m := &knownpb.Value{Kind: &knownpb.Value_StringValue{"abc\xff"}}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "google.protobuf.Value",
+				Value:   b,
+			}
+		}(),
+		wantErr: true,
+	}, {
+		desc: "Any with Value of NullValue",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.Value",
+  "value": null
+}`,
+		wantMessage: func() proto.Message {
+			m := &knownpb.Value{Kind: &knownpb.Value_NullValue{}}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "google.protobuf.Value",
+				Value:   b,
+			}
+		}(),
+	}, {
+		desc: "Any with Struct",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes(
+				(&knownpb.Struct{}).ProtoReflect().Type(),
+				(&knownpb.Value{}).ProtoReflect().Type(),
+				(&knownpb.BoolValue{}).ProtoReflect().Type(),
+				knownpb.NullValue_NULL_VALUE.Type(),
+				(&knownpb.StringValue{}).ProtoReflect().Type(),
+			),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.Struct",
+  "value": {
+    "bool": true,
+    "null": null,
+    "string": "hello",
+    "struct": {
+      "string": "world"
+    }
+  }
+}`,
+		wantMessage: func() proto.Message {
+			m := &knownpb.Struct{
+				Fields: map[string]*knownpb.Value{
+					"bool":   {Kind: &knownpb.Value_BoolValue{true}},
+					"null":   {Kind: &knownpb.Value_NullValue{}},
+					"string": {Kind: &knownpb.Value_StringValue{"hello"}},
+					"struct": {
+						Kind: &knownpb.Value_StructValue{
+							&knownpb.Struct{
+								Fields: map[string]*knownpb.Value{
+									"string": {Kind: &knownpb.Value_StringValue{"world"}},
+								},
+							},
+						},
+					},
+				},
+			}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "google.protobuf.Struct",
+				Value:   b,
+			}
+		}(),
+	}, {
+		desc:         "Any with missing @type",
+		umo:          jsonpb.UnmarshalOptions{},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "value": {}
+}`,
+		wantErr: true,
+	}, {
+		desc:         "Any with empty @type",
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": ""
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with duplicate @type",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes(
+				(&pb2.Nested{}).ProtoReflect().Type(),
+				(&knownpb.StringValue{}).ProtoReflect().Type(),
+			),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.StringValue",
+  "value": "hello",
+  "@type": "pb2.Nested"
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with duplicate value",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.StringValue{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "google.protobuf.StringValue",
+  "value": "hello",
+  "value": "world"
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with unknown field",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "pb2.Nested",
+  "optString": "hello",
+  "unknown": "world"
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with embedded type containing Any",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes(
+				(&pb2.KnownTypes{}).ProtoReflect().Type(),
+				(&knownpb.Any{}).ProtoReflect().Type(),
+				(&knownpb.StringValue{}).ProtoReflect().Type(),
+			),
+		},
+		inputMessage: &knownpb.Any{},
+		inputText: `{
+  "@type": "pb2.KnownTypes",
+  "optAny": {
+    "@type": "google.protobuf.StringValue",
+	"value": "` + "abc\xff" + `"
+  }
+}`,
+		wantMessage: func() proto.Message {
+			m1 := &knownpb.StringValue{Value: "abc\xff"}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m1)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			m2 := &knownpb.Any{
+				TypeUrl: "google.protobuf.StringValue",
+				Value:   b,
+			}
+			m3 := &pb2.KnownTypes{OptAny: m2}
+			b, err = proto.MarshalOptions{Deterministic: true}.Marshal(m3)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "pb2.KnownTypes",
+				Value:   b,
+			}
+		}(),
+		wantErr: true,
+	}, {
+		desc: "well known types as field values",
+		umo: jsonpb.UnmarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Empty{}).ProtoReflect().Type()),
+		},
+		inputMessage: &pb2.KnownTypes{},
+		inputText: `{
+  "optBool": false,
+  "optInt32": 42,
+  "optInt64": "42",
+  "optUint32": 42,
+  "optUint64": "42",
+  "optFloat": 1.23,
+  "optDouble": 3.1415,
+  "optString": "hello",
+  "optBytes": "aGVsbG8=",
+  "optDuration": "123s",
+  "optTimestamp": "2019-03-19T23:03:21Z",
+  "optStruct": {
+    "string": "hello"
+  },
+  "optList": [
+    null,
+    "",
+    {},
+    []
+  ],
+  "optValue": "world",
+  "optEmpty": {},
+  "optAny": {
+    "@type": "google.protobuf.Empty",
+    "value": {}
+  },
+  "optFieldmask": "fooBar,barFoo"
+}`,
+		wantMessage: &pb2.KnownTypes{
+			OptBool:      &knownpb.BoolValue{Value: false},
+			OptInt32:     &knownpb.Int32Value{Value: 42},
+			OptInt64:     &knownpb.Int64Value{Value: 42},
+			OptUint32:    &knownpb.UInt32Value{Value: 42},
+			OptUint64:    &knownpb.UInt64Value{Value: 42},
+			OptFloat:     &knownpb.FloatValue{Value: 1.23},
+			OptDouble:    &knownpb.DoubleValue{Value: 3.1415},
+			OptString:    &knownpb.StringValue{Value: "hello"},
+			OptBytes:     &knownpb.BytesValue{Value: []byte("hello")},
+			OptDuration:  &knownpb.Duration{Seconds: 123},
+			OptTimestamp: &knownpb.Timestamp{Seconds: 1553036601},
+			OptStruct: &knownpb.Struct{
+				Fields: map[string]*knownpb.Value{
+					"string": {Kind: &knownpb.Value_StringValue{"hello"}},
+				},
+			},
+			OptList: &knownpb.ListValue{
+				Values: []*knownpb.Value{
+					{Kind: &knownpb.Value_NullValue{}},
+					{Kind: &knownpb.Value_StringValue{}},
+					{
+						Kind: &knownpb.Value_StructValue{
+							&knownpb.Struct{Fields: map[string]*knownpb.Value{}},
+						},
+					},
+					{
+						Kind: &knownpb.Value_ListValue{
+							&knownpb.ListValue{Values: []*knownpb.Value{}},
+						},
+					},
+				},
+			},
+			OptValue: &knownpb.Value{
+				Kind: &knownpb.Value_StringValue{"world"},
+			},
+			OptEmpty: &knownpb.Empty{},
+			OptAny: &knownpb.Any{
+				TypeUrl: "google.protobuf.Empty",
+			},
+			OptFieldmask: &knownpb.FieldMask{
+				Paths: []string{"foo_bar", "bar_foo"},
+			},
+		},
 	}}
 
 	for _, tt := range tests {

encoding/jsonpb/encode_test.go

@@ -1509,7 +1509,7 @@ func TestMarshal(t *testing.T) {
 		input: &knownpb.Any{},
 		want:  `{}`,
 	}, {
-		desc: "Any",
+		desc: "Any with non-custom message",
 		mo: jsonpb.MarshalOptions{
 			Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
 		},
@@ -1537,7 +1537,7 @@ func TestMarshal(t *testing.T) {
   }
 }`,
 	}, {
-		desc: "Any without value",
+		desc: "Any with empty embedded message",
 		mo: jsonpb.MarshalOptions{
 			Resolver: preg.NewTypes((&pb2.Nested{}).ProtoReflect().Type()),
 		},
@@ -1546,11 +1546,9 @@ func TestMarshal(t *testing.T) {
   "@type": "foo/pb2.Nested"
 }`,
 	}, {
-		desc: "Any without registered type",
-		mo:   jsonpb.MarshalOptions{Resolver: preg.NewTypes()},
-		input: func() proto.Message {
-			return &knownpb.Any{TypeUrl: "foo/pb2.Nested"}
-		}(),
+		desc:    "Any without registered type",
+		mo:      jsonpb.MarshalOptions{Resolver: preg.NewTypes()},
+		input:   &knownpb.Any{TypeUrl: "foo/pb2.Nested"},
 		wantErr: true,
 	}, {
 		desc: "Any with missing required error",
@@ -1577,6 +1575,31 @@ func TestMarshal(t *testing.T) {
   "optString": "embedded inside Any"
 }`,
 		wantErr: true,
+	}, {
+		desc: "Any with partial required and AllowPartial",
+		mo: jsonpb.MarshalOptions{
+			AllowPartial: true,
+			Resolver:     preg.NewTypes((&pb2.PartialRequired{}).ProtoReflect().Type()),
+		},
+		input: func() proto.Message {
+			m := &pb2.PartialRequired{
+				OptString: scalar.String("embedded inside Any"),
+			}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			// TODO: Marshal may fail due to required field not set at some
+			// point. Need to ignore required not set error here.
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: string(m.ProtoReflect().Type().FullName()),
+				Value:   b,
+			}
+		}(),
+		want: `{
+  "@type": "pb2.PartialRequired",
+  "optString": "embedded inside Any"
+}`,
 	}, {
 		desc: "Any with invalid UTF8",
 		mo: jsonpb.MarshalOptions{
@@ -1672,26 +1695,45 @@ func TestMarshal(t *testing.T) {
 }`,
 		wantErr: true,
 	}, {
-		desc: "Any with Value of StringValue",
+		desc: "Any with Int64Value",
 		mo: jsonpb.MarshalOptions{
-			Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
+			Resolver: preg.NewTypes((&knownpb.Int64Value{}).ProtoReflect().Type()),
 		},
 		input: func() proto.Message {
-			m := &knownpb.Value{Kind: &knownpb.Value_StringValue{"abc\xff"}}
+			m := &knownpb.Int64Value{Value: 42}
 			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
 			if err != nil {
 				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
 			}
 			return &knownpb.Any{
-				TypeUrl: "type.googleapis.com/google.protobuf.Value",
+				TypeUrl: "google.protobuf.Int64Value",
 				Value:   b,
 			}
 		}(),
 		want: `{
-  "@type": "type.googleapis.com/google.protobuf.Value",
-  "value": "` + "abc\xff" + `"
+  "@type": "google.protobuf.Int64Value",
+  "value": "42"
+}`,
+	}, {
+		desc: "Any with Duration",
+		mo: jsonpb.MarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Duration{}).ProtoReflect().Type()),
+		},
+		input: func() proto.Message {
+			m := &knownpb.Duration{}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "type.googleapis.com/google.protobuf.Duration",
+				Value:   b,
+			}
+		}(),
+		want: `{
+  "@type": "type.googleapis.com/google.protobuf.Duration",
+  "value": "0s"
 }`,
-		wantErr: true,
 	}, {
 		desc: "Any with empty Value",
 		mo: jsonpb.MarshalOptions{
@@ -1710,24 +1752,45 @@ func TestMarshal(t *testing.T) {
 		}(),
 		wantErr: true,
 	}, {
-		desc: "Any with Duration",
+		desc: "Any with Value of StringValue",
 		mo: jsonpb.MarshalOptions{
-			Resolver: preg.NewTypes((&knownpb.Duration{}).ProtoReflect().Type()),
+			Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
 		},
 		input: func() proto.Message {
-			m := &knownpb.Duration{}
+			m := &knownpb.Value{Kind: &knownpb.Value_StringValue{"abc\xff"}}
 			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
 			if err != nil {
 				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
 			}
 			return &knownpb.Any{
-				TypeUrl: "type.googleapis.com/google.protobuf.Duration",
+				TypeUrl: "type.googleapis.com/google.protobuf.Value",
 				Value:   b,
 			}
 		}(),
 		want: `{
-  "@type": "type.googleapis.com/google.protobuf.Duration",
-  "value": "0s"
+  "@type": "type.googleapis.com/google.protobuf.Value",
+  "value": "` + "abc\xff" + `"
+}`,
+		wantErr: true,
+	}, {
+		desc: "Any with Value of NullValue",
+		mo: jsonpb.MarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.Value{}).ProtoReflect().Type()),
+		},
+		input: func() proto.Message {
+			m := &knownpb.Value{Kind: &knownpb.Value_NullValue{}}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				TypeUrl: "type.googleapis.com/google.protobuf.Value",
+				Value:   b,
+			}
+		}(),
+		want: `{
+  "@type": "type.googleapis.com/google.protobuf.Value",
+  "value": null
 }`,
 	}, {
 		desc: "Any with Struct",
@@ -1777,6 +1840,22 @@ func TestMarshal(t *testing.T) {
     }
   }
 }`,
+	}, {
+		desc: "Any with missing type_url",
+		mo: jsonpb.MarshalOptions{
+			Resolver: preg.NewTypes((&knownpb.BoolValue{}).ProtoReflect().Type()),
+		},
+		input: func() proto.Message {
+			m := &knownpb.BoolValue{Value: true}
+			b, err := proto.MarshalOptions{Deterministic: true}.Marshal(m)
+			if err != nil {
+				t.Fatalf("error in binary marshaling message for Any.value: %v", err)
+			}
+			return &knownpb.Any{
+				Value: b,
+			}
+		}(),
+		wantErr: true,
 	}, {
 		desc: "well known types as field values",
 		mo: jsonpb.MarshalOptions{

encoding/jsonpb/well_known_types.go

@@ -98,7 +98,7 @@ func (o UnmarshalOptions) unmarshalCustomType(m pref.Message) error {
 	name := m.Type().FullName()
 	switch name {
 	case "google.protobuf.Any":
-		panic("unmarshaling of google.protobuf.Any is not implemented yet")
+		return o.unmarshalAny(m)
 
 	case "google.protobuf.BoolValue",
 		"google.protobuf.DoubleValue",
@@ -138,12 +138,11 @@ func (o UnmarshalOptions) unmarshalCustomType(m pref.Message) error {
 
 // The JSON representation of an Any message uses the regular representation of
 // the deserialized, embedded message, with an additional field `@type` which
-// contains the type URL.  If the embedded message type is well-known and has a
+// contains the type URL. If the embedded message type is well-known and has a
 // custom JSON representation, that representation will be embedded adding a
 // field `value` which holds the custom JSON in addition to the `@type` field.
 
 func (o MarshalOptions) marshalAny(m pref.Message) error {
-	var nerr errors.NonFatal
 	msgType := m.Type()
 	knownFields := m.KnownFields()
 
@@ -167,6 +166,7 @@ func (o MarshalOptions) marshalAny(m pref.Message) error {
 	// Marshal out @type field.
 	typeURL := typeVal.String()
 	o.encoder.WriteName("@type")
+	var nerr errors.NonFatal
 	if err := o.encoder.WriteString(typeURL); !nerr.Merge(err) {
 		return err
 	}
@@ -200,6 +200,225 @@ func (o MarshalOptions) marshalAny(m pref.Message) error {
 	return nerr.E
 }
 
+func (o UnmarshalOptions) unmarshalAny(m pref.Message) error {
+	// Use Peek to check for json.StartObject to avoid advancing a read.
+	if o.decoder.Peek() != json.StartObject {
+		jval, _ := o.decoder.Read()
+		return unexpectedJSONError{jval}
+	}
+
+	// Use another json.Decoder to parse the unread bytes from o.decoder for
+	// @type field. This avoids advancing a read from o.decoder because the
+	// current JSON object may contain the fields of the embedded type.
+	dec := o.decoder.Clone()
+	typeURL, err := findTypeURL(dec)
+	if err == errEmptyObject {
+		// An empty JSON object translates to an empty Any message.
+		o.decoder.Read() // Read json.StartObject.
+		o.decoder.Read() // Read json.EndObject.
+		return nil
+	}
+	var nerr errors.NonFatal
+	if !nerr.Merge(err) {
+		return errors.New("google.protobuf.Any: %v", err)
+	}
+
+	emt, err := o.Resolver.FindMessageByURL(typeURL)
+	if err != nil {
+		return errors.New("google.protobuf.Any: unable to resolve type %q: %v", typeURL, err)
+	}
+
+	// Create new message for the embedded message type and unmarshal into it.
+	em := emt.New()
+	if isCustomType(emt.FullName()) {
+		// If embedded message is a custom type, unmarshal the JSON "value" field
+		// into it.
+		if err := o.unmarshalAnyValue(em); !nerr.Merge(err) {
+			return errors.New("google.protobuf.Any: %v", err)
+		}
+	} else {
+		// Else unmarshal the current JSON object into it.
+		if err := o.unmarshalMessage(em, true); !nerr.Merge(err) {
+			return errors.New("google.protobuf.Any: %v", err)
+		}
+	}
+	// Serialize the embedded message and assign the resulting bytes to the
+	// proto value field.
+	b, err := proto.MarshalOptions{Deterministic: true}.Marshal(em.Interface())
+	if !nerr.Merge(err) {
+		return errors.New("google.protobuf.Any: %v", err)
+	}
+
+	knownFields := m.KnownFields()
+	knownFields.Set(fieldnum.Any_TypeUrl, pref.ValueOf(typeURL))
+	knownFields.Set(fieldnum.Any_Value, pref.ValueOf(b))
+	return nerr.E
+}
+
+var errEmptyObject = errors.New(`empty object`)
+
+// findTypeURL returns the "@type" field value from the given JSON bytes. It is
+// expected that the given bytes start with json.StartObject. It returns
+// errEmptyObject if the JSON object is empty. It returns error if the object
+// does not contain the field or other decoding problems.
+func findTypeURL(dec *json.Decoder) (string, error) {
+	var typeURL string
+	var nerr errors.NonFatal
+	numFields := 0
+	// Skip start object.
+	dec.Read()
+
+Loop:
+	for {
+		jval, err := dec.Read()
+		if !nerr.Merge(err) {
+			return "", err
+		}
+
+		switch jval.Type() {
+		case json.EndObject:
+			if typeURL == "" {
+				// Did not find @type field.
+				if numFields > 0 {
+					return "", errors.New(`missing "@type" field`)
+				}
+				return "", errEmptyObject
+			}
+			break Loop
+
+		case json.Name:
+			numFields++
+			name, err := jval.Name()
+			if !nerr.Merge(err) {
+				return "", err
+			}
+			if name != "@type" {
+				// Skip value.
+				if err := skipJSONValue(dec); err != nil {
+					return "", err
+				}
+				continue
+			}
+
+			// Return error if this was previously set already.
+			if typeURL != "" {
+				return "", errors.New(`duplicate "@type" field`)
+			}
+			// Read field value.
+			jval, err := dec.Read()
+			if !nerr.Merge(err) {
+				return "", err
+			}
+			if jval.Type() != json.String {
+				return "", unexpectedJSONError{jval}
+			}
+			typeURL = jval.String()
+			if typeURL == "" {
+				return "", errors.New(`"@type" field contains empty value`)
+			}
+		}
+	}
+
+	return typeURL, nerr.E
+}
+
+// skipJSONValue makes the given decoder parse a JSON value (null, boolean,
+// string, number, object and array) in order to advance the read to the next
+// JSON value. It relies on Decoder.Read returning an error if the types are
+// not in valid sequence.
+func skipJSONValue(dec *json.Decoder) error {
+	// Ignore non-fatal errors, do not return nerr.E.
+	var nerr errors.NonFatal
+	jval, err := dec.Read()
+	if !nerr.Merge(err) {
+		return err
+	}
+	// Only need to continue reading for objects and arrays.
+	switch jval.Type() {
+	case json.StartObject:
+		for {
+			jval, err := dec.Read()
+			if !nerr.Merge(err) {
+				return err
+			}
+			switch jval.Type() {
+			case json.EndObject:
+				return nil
+			case json.Name:
+				// Skip object field value.
+				if err := skipJSONValue(dec); err != nil {
+					return err
+				}
+			}
+		}
+
+	case json.StartArray:
+		for {
+			switch dec.Peek() {
+			case json.EndArray:
+				dec.Read()
+				return nil
+			case json.Invalid:
+				_, err := dec.Read()
+				return err
+			default:
+				// Skip array item.
+				if err := skipJSONValue(dec); err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// unmarshalAnyValue unmarshals the given custom-type message from the JSON
+// object's "value" field.
+func (o UnmarshalOptions) unmarshalAnyValue(m pref.Message) error {
+	var nerr errors.NonFatal
+	// Skip StartObject, and start reading the fields.
+	o.decoder.Read()
+
+	var found bool // Used for detecting duplicate "value".
+	for {
+		jval, err := o.decoder.Read()
+		if !nerr.Merge(err) {
+			return err
+		}
+		switch jval.Type() {
+		case json.EndObject:
+			if !found {
+				return errors.New(`missing "value" field`)
+			}
+			return nerr.E
+
+		case json.Name:
+			name, err := jval.Name()
+			if !nerr.Merge(err) {
+				return err
+			}
+			switch name {
+			default:
+				return errors.New("unknown field %q", name)
+
+			case "@type":
+				// Skip the value as this was previously parsed already.
+				o.decoder.Read()
+
+			case "value":
+				if found {
+					return errors.New(`duplicate "value" field`)
+				}
+				// Unmarshal the field value into the given message.
+				if err := o.unmarshalCustomType(m); !nerr.Merge(err) {
+					return err
+				}
+				found = true
+			}
+		}
+	}
+}
+
 // Wrapper types are encoded as JSON primitives like string, number or boolean.
 
 func (o MarshalOptions) marshalWrapperType(m pref.Message) error {

internal/encoding/json/decode.go

@@ -25,7 +25,8 @@ const (
 
 // Decoder is a token-based JSON decoder.
 type Decoder struct {
-	// lastCall is last method called, eiterh readCall or peekCall.
+	// lastCall is last method called, either readCall or peekCall.
+	// Initial value is readCall.
 	lastCall call
 
 	// value contains the last read value.
@@ -88,7 +89,7 @@ func (d *Decoder) Read() (Value, error) {
 
 	case Bool, Number:
 		if !d.isValueNext() {
-			return Value{}, d.newSyntaxError("unexpected value %v", value)
+			return Value{}, d.newSyntaxError("unexpected value %v", value.Raw())
 		}
 
 	case String:
@@ -97,7 +98,7 @@ func (d *Decoder) Read() (Value, error) {
 		}
 		// Check if this is for an object name.
 		if d.value.typ&(StartObject|comma) == 0 {
-			return Value{}, d.newSyntaxError("unexpected value %q", value)
+			return Value{}, d.newSyntaxError("unexpected value %v", value.Raw())
 		}
 		d.in = d.in[n:]
 		d.consume(0)
@@ -109,7 +110,7 @@ func (d *Decoder) Read() (Value, error) {
 
 	case StartObject, StartArray:
 		if !d.isValueNext() {
-			return Value{}, d.newSyntaxError("unexpected character %v", value)
+			return Value{}, d.newSyntaxError("unexpected character %v", value.Raw())
 		}
 		d.startStack = append(d.startStack, value.typ)
 
@@ -323,6 +324,14 @@ func (d *Decoder) newStringValue(input []byte, s string) Value {
 	}
 }
 
+// Clone returns a copy of the Decoder for use in reading ahead the next JSON
+// object, array or other values without affecting current Decoder.
+func (d *Decoder) Clone() *Decoder {
+	ret := *d
+	ret.startStack = append([]Type(nil), ret.startStack...)
+	return &ret
+}
+
 // Value contains a JSON type and value parsed from calling Decoder.Read.
 // For JSON boolean and string, it holds the converted value in boo and str
 // fields respectively. For JSON number, input field holds a valid number which
@@ -377,6 +386,11 @@ func (v Value) Name() (string, error) {
 	return v.str, nil
 }
 
+// Raw returns the read value in string.
+func (v Value) Raw() string {
+	return string(v.input)
+}
+
 // Float returns the floating-point number if token is Number, else it will
 // return an error.
 //

internal/encoding/json/decode_test.go

@@ -1087,3 +1087,37 @@ func checkValue(t *testing.T, value json.Value, wantIdx int, want R) {
 		t.Errorf("want#%d: %v got %v, want %v", wantIdx, value, got, want.V)
 	}
 }
+
+func TestClone(t *testing.T) {
+	input := `{"outer":{"str":"hello", "number": 123}}`
+	dec := json.NewDecoder([]byte(input))
+
+	// Clone at the start should produce the same reads as the original.
+	clone := dec.Clone()
+	compareDecoders(t, dec, clone)
+
+	// Advance to inner object, clone and compare again.
+	dec.Read() // Read StartObject.
+	dec.Read() // Read Name.
+	clone = dec.Clone()
+	compareDecoders(t, dec, clone)
+}
+
+func compareDecoders(t *testing.T, d1 *json.Decoder, d2 *json.Decoder) {
+	for {
+		v1, err1 := d1.Read()
+		v2, err2 := d2.Read()
+		if v1.Type() != v2.Type() {
+			t.Errorf("cloned decoder: got Type %v, want %v", v2.Type(), v1.Type())
+		}
+		if v1.Raw() != v2.Raw() {
+			t.Errorf("cloned decoder: got Raw %v, want %v", v2.Raw(), v1.Raw())
+		}
+		if err1 != err2 {
+			t.Errorf("cloned decoder: got error %v, want %v", err2, err1)
+		}
+		if v1.Type() == json.EOF {
+			break
+		}
+	}
+}