encoding/protojson: add MarshalOptions.EmitUnpopulated

Add option to marshal out all fields including unset ones except for
unset oneof fields and extension fields.

This is to make V2 compatible with V1's EmitDefaults option.

Change-Id: Ifa7bae48e82740b623c74f936bcbe9e66b11344a
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/193759
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>

encoding/protojson/encode.go

@@ -27,12 +27,29 @@ func Marshal(m proto.Message) ([]byte, error) {
 // MarshalOptions is a configurable JSON format marshaler.
 type MarshalOptions struct {
 	pragma.NoUnkeyedLiterals
+	encoder *json.Encoder
 
 	// AllowPartial allows messages that have missing required fields to marshal
 	// without returning an error. If AllowPartial is false (the default),
 	// Marshal will return error if there are any missing required fields.
 	AllowPartial bool
 
+	// EmitUnpopulated specifies whether to emit unpopulated fields. It does not
+	// emit unpopulated oneof fields or unpopulated extension fields.
+	// The JSON value emitted for unpopulated fields are as follows:
+	// ╔═══════╤════════════════════════════╗
+	// ║ JSON  │ Protobuf field             ║
+	// ╠═══════╪════════════════════════════╣
+	// ║ false │ proto3 boolean fields      ║
+	// ║ 0     │ proto3 numeric fields      ║
+	// ║ ""    │ proto3 string/bytes fields ║
+	// ║ null  │ proto2 scalar fields       ║
+	// ║ null  │ message fields             ║
+	// ║ []    │ list fields                ║
+	// ║ {}    │ map fields                 ║
+	// ╚═══════╧════════════════════════════╝
+	EmitUnpopulated bool
+
 	// If Indent is a non-empty string, it causes entries for an Array or Object
 	// to be preceded by the indent and trailed by a newline. Indent can only be
 	// composed of space or tab characters.
@@ -44,8 +61,6 @@ type MarshalOptions struct {
 		protoregistry.ExtensionTypeResolver
 		protoregistry.MessageTypeResolver
 	}
-
-	encoder *json.Encoder
 }
 
 // Marshal marshals the given proto.Message in the JSON format using options in
@@ -96,12 +111,20 @@ func (o MarshalOptions) marshalFields(m pref.Message) error {
 	fieldDescs := messageDesc.Fields()
 	for i := 0; i < fieldDescs.Len(); i++ {
 		fd := fieldDescs.Get(i)
+		val := m.Get(fd)
 		if !m.Has(fd) {
-			continue
+			if !o.EmitUnpopulated || fd.ContainingOneof() != nil {
+				continue
+			}
+			isProto2Scalar := fd.Syntax() == pref.Proto2 && fd.Default().IsValid()
+			isSingularMessage := fd.Cardinality() != pref.Repeated && fd.Message() != nil
+			if isProto2Scalar || isSingularMessage {
+				// Use invalid value to emit null.
+				val = pref.Value{}
+			}
 		}
 
 		name := fd.JSONName()
-		val := m.Get(fd)
 		if err := o.encoder.WriteName(name); err != nil {
 			return err
 		}
@@ -132,6 +155,11 @@ func (o MarshalOptions) marshalValue(val pref.Value, fd pref.FieldDescriptor) er
 // marshalSingular marshals the given non-repeated field value. This includes
 // all scalar types, enums, messages, and groups.
 func (o MarshalOptions) marshalSingular(val pref.Value, fd pref.FieldDescriptor) error {
+	if !val.IsValid() {
+		o.encoder.WriteNull()
+		return nil
+	}
+
 	switch kind := fd.Kind(); kind {
 	case pref.BoolKind:
 		o.encoder.WriteBool(val.Bool())

encoding/protojson/encode_test.go

@@ -1892,6 +1892,238 @@ func TestMarshal(t *testing.T) {
     "value": {}
   },
   "optFieldmask": "fooBar,barFoo"
+}`,
+	}, {
+		desc:  "EmitUnpopulated: proto2 optional scalars",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb2.Scalars{},
+		want: `{
+  "optBool": null,
+  "optInt32": null,
+  "optInt64": null,
+  "optUint32": null,
+  "optUint64": null,
+  "optSint32": null,
+  "optSint64": null,
+  "optFixed32": null,
+  "optFixed64": null,
+  "optSfixed32": null,
+  "optSfixed64": null,
+  "optFloat": null,
+  "optDouble": null,
+  "optBytes": null,
+  "optString": null
+}`,
+	}, {
+		desc:  "EmitUnpopulated: proto3 scalars",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb3.Scalars{},
+		want: `{
+  "sBool": false,
+  "sInt32": 0,
+  "sInt64": "0",
+  "sUint32": 0,
+  "sUint64": "0",
+  "sSint32": 0,
+  "sSint64": "0",
+  "sFixed32": 0,
+  "sFixed64": "0",
+  "sSfixed32": 0,
+  "sSfixed64": "0",
+  "sFloat": 0,
+  "sDouble": 0,
+  "sBytes": "",
+  "sString": ""
+}`,
+	}, {
+		desc:  "EmitUnpopulated: proto2 enum",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb2.Enums{},
+		want: `{
+  "optEnum": null,
+  "rptEnum": [],
+  "optNestedEnum": null,
+  "rptNestedEnum": []
+}`,
+	}, {
+		desc:  "EmitUnpopulated: proto3 enum",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb3.Enums{},
+		want: `{
+  "sEnum": "ZERO",
+  "sNestedEnum": "CERO"
+}`,
+	}, {
+		desc:  "EmitUnpopulated: proto2 message and group fields",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb2.Nests{},
+		want: `{
+  "optNested": null,
+  "optgroup": null,
+  "rptNested": [],
+  "rptgroup": []
+}`,
+	}, {
+		desc:  "EmitUnpopulated: proto3 message field",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb3.Nests{},
+		want: `{
+  "sNested": null
+}`,
+	}, {
+		desc: "EmitUnpopulated: proto2 empty message and group fields",
+		mo:   protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb2.Nests{
+			OptNested: &pb2.Nested{},
+			Optgroup:  &pb2.Nests_OptGroup{},
+		},
+		want: `{
+  "optNested": {
+    "optString": null,
+    "optNested": null
+  },
+  "optgroup": {
+    "optString": null,
+    "optNested": null,
+    "optnestedgroup": null
+  },
+  "rptNested": [],
+  "rptgroup": []
+}`,
+	}, {
+		desc: "EmitUnpopulated: proto3 empty message field",
+		mo:   protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb3.Nests{
+			SNested: &pb3.Nested{},
+		},
+		want: `{
+  "sNested": {
+    "sString": "",
+    "sNested": null
+  }
+}`,
+	}, {
+		desc: "EmitUnpopulated: proto2 required fields",
+		mo: protojson.MarshalOptions{
+			AllowPartial:    true,
+			EmitUnpopulated: true,
+		},
+		input: &pb2.Requireds{},
+		want: `{
+  "reqBool": null,
+  "reqSfixed64": null,
+  "reqDouble": null,
+  "reqString": null,
+  "reqEnum": null,
+  "reqNested": null
+}`,
+	}, {
+		desc:  "EmitUnpopulated: repeated fields",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb2.Repeats{},
+		want: `{
+  "rptBool": [],
+  "rptInt32": [],
+  "rptInt64": [],
+  "rptUint32": [],
+  "rptUint64": [],
+  "rptFloat": [],
+  "rptDouble": [],
+  "rptString": [],
+  "rptBytes": []
+}`,
+	}, {
+		desc: "EmitUnpopulated: repeated containing empty message",
+		mo:   protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb2.Nests{
+			RptNested: []*pb2.Nested{nil, {}},
+		},
+		want: `{
+  "optNested": null,
+  "optgroup": null,
+  "rptNested": [
+    {
+      "optString": null,
+      "optNested": null
+    },
+    {
+      "optString": null,
+      "optNested": null
+    }
+  ],
+  "rptgroup": []
+}`,
+	}, {
+		desc:  "EmitUnpopulated: map fields",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb3.Maps{},
+		want: `{
+  "int32ToStr": {},
+  "boolToUint32": {},
+  "uint64ToEnum": {},
+  "strToNested": {},
+  "strToOneofs": {}
+}`,
+	}, {
+		desc: "EmitUnpopulated: map containing empty message",
+		mo:   protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb3.Maps{
+			StrToNested: map[string]*pb3.Nested{
+				"nested": &pb3.Nested{},
+			},
+			StrToOneofs: map[string]*pb3.Oneofs{
+				"nested": &pb3.Oneofs{},
+			},
+		},
+		want: `{
+  "int32ToStr": {},
+  "boolToUint32": {},
+  "uint64ToEnum": {},
+  "strToNested": {
+    "nested": {
+      "sString": "",
+      "sNested": null
+    }
+  },
+  "strToOneofs": {
+    "nested": {}
+  }
+}`,
+	}, {
+		desc:  "EmitUnpopulated: oneof fields",
+		mo:    protojson.MarshalOptions{EmitUnpopulated: true},
+		input: &pb3.Oneofs{},
+		want:  `{}`,
+	}, {
+		desc: "EmitUnpopulated: extensions",
+		mo:   protojson.MarshalOptions{EmitUnpopulated: true},
+		input: func() proto.Message {
+			m := &pb2.Extensions{}
+			proto.SetExtension(m, pb2.E_OptExtNested, &pb2.Nested{})
+			proto.SetExtension(m, pb2.E_RptExtNested, []*pb2.Nested{
+				nil,
+				{},
+			})
+			return m
+		}(),
+		want: `{
+  "optString": null,
+  "optBool": null,
+  "optInt32": null,
+  "[pb2.opt_ext_nested]": {
+    "optString": null,
+    "optNested": null
+  },
+  "[pb2.rpt_ext_nested]": [
+    {
+      "optString": null,
+      "optNested": null
+    },
+    {
+      "optString": null,
+      "optNested": null
+    }
+  ]
 }`,
 	}}