goprotobuf: push out internal changes.

  - group text parsing and formatting
  - foreign extension types

R=adg
CC=golang-dev
http://codereview.appspot.com/4437079

compiler/generator/generator.go

@@ -49,7 +49,7 @@ import (
 	"strings"
 
 	"goprotobuf.googlecode.com/hg/proto"
-	plugin     "goprotobuf.googlecode.com/hg/compiler/plugin"
+	plugin "goprotobuf.googlecode.com/hg/compiler/plugin"
 	descriptor "goprotobuf.googlecode.com/hg/compiler/descriptor"
 )
 
@@ -890,7 +890,17 @@ func (g *Generator) goTag(field *descriptor.FieldDescriptorProto, wiretype strin
 		packed = ",packed"
 	}
 	name := proto.GetString(field.Name)
-	if name == CamelCase(name) {
+	if *field.Type == descriptor.FieldDescriptorProto_TYPE_GROUP {
+		// We must use the type name for groups instead of
+		// the field name to preserve capitalization.
+		// type_name in FieldDescriptorProto is fully-qualified,
+		// but we only want the local part.
+		name = *field.TypeName
+		if i := strings.LastIndex(name, "."); i >= 0 {
+			name = name[i+1:]
+		}
+		name = ",name=" + name
+	} else if name == CamelCase(name) {
 		name = ""
 	} else {
 		name = ",name=" + name
@@ -1140,6 +1150,10 @@ func (g *Generator) generateExtension(ext *ExtensionDescriptor) {
 	fieldType, wireType := g.GoType(ext.parent, field)
 	tag := g.goTag(field, wireType)
 	g.RecordTypeUse(*ext.Extendee)
+	if n := ext.FieldDescriptorProto.TypeName; n != nil {
+		// foreign extension type
+		g.RecordTypeUse(*n)
+	}
 
 	g.P("var ", ccTypeName, " = &", g.ProtoPkg, ".ExtensionDesc{")
 	g.In()

compiler/testdata/Makefile

@@ -65,4 +65,4 @@ multi.a: multi3.pb.$O multi2.pb.$O multi1.pb.$O
 test.pb.go:	imp.pb.go
 multi1.pb.go:	multi2.pb.go multi3.pb.go
 main.$O: imp.pb.$O test.pb.$O multi.a
-extension_test.$O: extension_base.pb.$O extension_user.pb.$O
+extension_test.$O: extension_base.pb.$O extension_extra.pb.$O extension_user.pb.$O

compiler/testdata/extension_extra.proto

@@ -0,0 +1,38 @@
+// Go support for Protocol Buffers - Google's data interchange format
+//
+// Copyright 2011 Google Inc.  All rights reserved.
+// http://code.google.com/p/goprotobuf/
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto2";
+
+package extension_extra;
+
+message ExtraMessage {
+  optional int32 width = 1;
+}

compiler/testdata/extension_user.proto

@@ -30,6 +30,7 @@
 // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 import "extension_base.proto";
+import "extension_extra.proto";
 
 package extension_user;
 
@@ -43,6 +44,11 @@ extend extension_base.BaseMessage {
   optional UserMessage user_message = 5;
 }
 
+// Extend with a foreign message
+extend extension_base.BaseMessage {
+  optional extension_extra.ExtraMessage extra_message = 9;
+}
+
 // Extend with some primitive types
 extend extension_base.BaseMessage {
   optional int32 width = 6;

compiler/testdata/test.pb.go.golden

@@ -123,6 +123,7 @@ type Request struct {
 	Hat			*HatType			"PB(varint,4,opt,name=hat,enum=my_test.HatType,def=1)"
 	Owner			*imp.ImportedMessage_Owner	"PB(varint,6,opt,name=owner,enum=imp.ImportedMessage_Owner)"
 	Deadline		*float32			"PB(fixed32,7,opt,name=deadline,def=inf)"
+	Somegroup		*Request_SomeGroup		"PB(group,8,opt,name=SomeGroup)"
 	XXX_unrecognized	[]byte
 }
 
@@ -134,6 +135,15 @@ const Default_Request_Hat HatType = HatType_FEDORA
 
 var Default_Request_Deadline float32 = float32(math.Inf(1))
 
+type Request_SomeGroup struct {
+	GroupField		*int32	"PB(varint,9,opt,name=group_field)"
+	XXX_unrecognized	[]byte
+}
+
+func (this *Request_SomeGroup) Reset() {
+	*this = Request_SomeGroup{}
+}
+
 type Reply struct {
 	Found			[]*Reply_Entry	"PB(bytes,1,rep,name=found)"
 	CompactKeys		[]int32		"PB(varint,2,rep,packed,name=compact_keys)"

compiler/testdata/test.proto

@@ -58,6 +58,9 @@ message Request {
   optional HatType hat = 4 [default=FEDORA];
   optional imp.ImportedMessage.Owner owner = 6;
   optional float deadline = 7 [default=inf];
+  optional group SomeGroup = 8 {
+    optional int32 group_field = 9;
+  }
 }
 
 message Reply {

proto/testdata/test.pb.go

@@ -207,9 +207,9 @@ type GoTest struct {
 	F_DoubleRepeatedPacked	[]float64		"PB(fixed64,58,rep,packed,name=F_Double_repeated_packed)"
 	F_Sint32RepeatedPacked	[]int32			"PB(zigzag32,502,rep,packed,name=F_Sint32_repeated_packed)"
 	F_Sint64RepeatedPacked	[]int64			"PB(zigzag64,503,rep,packed,name=F_Sint64_repeated_packed)"
-	Requiredgroup		*GoTest_RequiredGroup	"PB(group,70,req,name=requiredgroup)"
-	Repeatedgroup		[]*GoTest_RepeatedGroup	"PB(group,80,rep,name=repeatedgroup)"
-	Optionalgroup		*GoTest_OptionalGroup	"PB(group,90,opt,name=optionalgroup)"
+	Requiredgroup		*GoTest_RequiredGroup	"PB(group,70,req,name=RequiredGroup)"
+	Repeatedgroup		[]*GoTest_RepeatedGroup	"PB(group,80,rep,name=RepeatedGroup)"
+	Optionalgroup		*GoTest_OptionalGroup	"PB(group,90,opt,name=OptionalGroup)"
 	XXX_unrecognized	[]byte
 }
 
@@ -265,7 +265,7 @@ type GoSkipTest struct {
 	SkipFixed32		*uint32			"PB(fixed32,12,req,name=skip_fixed32)"
 	SkipFixed64		*uint64			"PB(fixed64,13,req,name=skip_fixed64)"
 	SkipString		*string			"PB(bytes,14,req,name=skip_string)"
-	Skipgroup		*GoSkipTest_SkipGroup	"PB(group,15,req,name=skipgroup)"
+	Skipgroup		*GoSkipTest_SkipGroup	"PB(group,15,req,name=SkipGroup)"
 	XXX_unrecognized	[]byte
 }
 
@@ -334,6 +334,7 @@ type MyMessage struct {
 	Inner			*InnerMessage		"PB(bytes,5,opt,name=inner)"
 	Others			[]*OtherMessage		"PB(bytes,6,rep,name=others)"
 	Bikeshed		*MyMessage_Color	"PB(varint,7,opt,name=bikeshed,enum=test_proto.MyMessage_Color)"
+	Somegroup		*MyMessage_SomeGroup	"PB(group,8,opt,name=SomeGroup)"
 	XXX_unrecognized	[]byte
 }
 
@@ -341,8 +342,17 @@ func (this *MyMessage) Reset() {
 	*this = MyMessage{}
 }
 
+type MyMessage_SomeGroup struct {
+	GroupField		*int32	"PB(varint,9,opt,name=group_field)"
+	XXX_unrecognized	[]byte
+}
+
+func (this *MyMessage_SomeGroup) Reset() {
+	*this = MyMessage_SomeGroup{}
+}
+
 type MessageList struct {
-	Message			[]*MessageList_Message	"PB(group,1,rep,name=message)"
+	Message			[]*MessageList_Message	"PB(group,1,rep,name=Message)"
 	XXX_unrecognized	[]byte
 }
 

proto/testdata/test.proto

@@ -222,6 +222,10 @@ message MyMessage {
     BLUE = 2;
   };
   optional Color bikeshed = 7;
+
+  optional group SomeGroup = 8 {
+    optional int32 group_field = 9;
+  }
 }
 
 message MessageList {

proto/text.go

@@ -32,8 +32,7 @@
 package proto
 
 // Functions for writing the Text protocol buffer format.
-// TODO:
-//	- Message sets, groups.
+// TODO: message sets, extensions.
 
 import (
 	"bytes"
@@ -51,6 +50,8 @@ type textWriter struct {
 	complete     bool // if the current position is a complete line
 	compact      bool // whether to write out as a one-liner
 	writer       io.Writer
+
+	c [1]byte // scratch
 }
 
 func (w *textWriter) Write(p []byte) (n int, err os.Error) {
@@ -80,6 +81,12 @@ func (w *textWriter) Write(p []byte) (n int, err os.Error) {
 	return
 }
 
+func (w *textWriter) WriteByte(c byte) os.Error {
+	w.c[0] = c
+	_, err := w.Write(w.c[:])
+	return err
+}
+
 func (w *textWriter) indent() { w.indent_level++ }
 
 func (w *textWriter) unindent() {
@@ -90,6 +97,13 @@ func (w *textWriter) unindent() {
 	}
 }
 
+func writeName(w *textWriter, props *Properties) {
+	io.WriteString(w, props.OrigName)
+	if props.Wire != "group" {
+		w.WriteByte(':')
+	}
+}
+
 func writeStruct(w *textWriter, sv reflect.Value) {
 	st := sv.Type()
 	sprops := GetProperties(st)
@@ -114,23 +128,23 @@ func writeStruct(w *textWriter, sv reflect.Value) {
 			if av := fv; av.Kind() == reflect.Slice {
 				// Repeated field.
 				for j := 0; j < av.Len(); j++ {
-					fmt.Fprintf(w, "%v:", props.OrigName)
+					writeName(w, props)
 					if !w.compact {
-						w.Write([]byte{' '})
+						w.WriteByte(' ')
 					}
-					writeAny(w, av.Index(j))
+					writeAny(w, av.Index(j), props)
 					fmt.Fprint(w, "\n")
 				}
 				continue
 			}
 		}
 
-		fmt.Fprintf(w, "%v:", props.OrigName)
+		writeName(w, props)
 		if !w.compact {
-			w.Write([]byte{' '})
+			w.WriteByte(' ')
 		}
 		if len(props.Enum) == 0 || !tryWriteEnum(w, props.Enum, fv) {
-			writeAny(w, fv)
+			writeAny(w, fv, props)
 		}
 		fmt.Fprint(w, "\n")
 	}
@@ -153,7 +167,7 @@ func tryWriteEnum(w *textWriter, enum string, v reflect.Value) bool {
 	return true
 }
 
-func writeAny(w *textWriter, v reflect.Value) {
+func writeAny(w *textWriter, v reflect.Value, props *Properties) {
 	v = reflect.Indirect(v)
 
 	// We don't attempt to serialise every possible value type; only those
@@ -173,16 +187,18 @@ func writeAny(w *textWriter, v reflect.Value) {
 		fmt.Fprint(w, strconv.Quote(val.String()))
 	case reflect.Struct:
 		// Required/optional group/message.
-		// TODO: groups use { } instead of < >, and no colon.
+		var bra, ket byte = '<', '>'
+		if props != nil && props.Wire == "group" {
+			bra, ket = '{', '}'
+		}
+		w.WriteByte(bra)
 		if !w.compact {
-			fmt.Fprint(w, "<\n")
-		} else {
-			fmt.Fprint(w, "<")
+			w.WriteByte('\n')
 		}
 		w.indent()
 		writeStruct(w, val)
 		w.unindent()
-		fmt.Fprint(w, ">")
+		w.WriteByte(ket)
 	default:
 		fmt.Fprint(w, val.Interface())
 	}
@@ -205,7 +221,7 @@ func marshalText(w io.Writer, pb interface{}, compact bool) {
 	if v.Kind() == reflect.Struct {
 		writeStruct(aw, v)
 	} else {
-		writeAny(aw, v)
+		writeAny(aw, v, nil)
 	}
 }
 

proto/text_parser.go

@@ -32,8 +32,7 @@
 package proto
 
 // Functions for parsing the Text protocol buffer format.
-// TODO:
-//     - message sets, groups.
+// TODO: message sets, extensions.
 
 import (
 	"fmt"

proto/text_parser_test.go

@@ -46,7 +46,7 @@ type UnmarshalTextTest struct {
 
 var unMarshalTextTests = []UnmarshalTextTest{
 	// Basic
-	UnmarshalTextTest{
+	{
 		in: " count:42\n  name:\"Dave\" ",
 		out: &MyMessage{
 			Count: Int32(42),
@@ -55,7 +55,7 @@ var unMarshalTextTests = []UnmarshalTextTest{
 	},
 
 	// Empty quoted string
-	UnmarshalTextTest{
+	{
 		in: `count:42 name:""`,
 		out: &MyMessage{
 			Count: Int32(42),
@@ -64,7 +64,7 @@ var unMarshalTextTests = []UnmarshalTextTest{
 	},
 
 	// Quoted string concatenation
-	UnmarshalTextTest{
+	{
 		in: `count:42 name: "My name is "` + "\n" + `"elsewhere"`,
 		out: &MyMessage{
 			Count: Int32(42),
@@ -73,49 +73,49 @@ var unMarshalTextTests = []UnmarshalTextTest{
 	},
 
 	// Bad quoted string
-	UnmarshalTextTest{
+	{
 		in:    `inner: < host: "\0" >` + "\n",
 		error: `line 1.15: invalid quoted string "\0"`,
 	},
 
 	// Number too large for int64
-	UnmarshalTextTest{
+	{
 		in:    "count: 123456789012345678901",
 		error: "line 1.7: invalid int32: 123456789012345678901",
 	},
 
 	// Number too large for int32
-	UnmarshalTextTest{
+	{
 		in:    "count: 1234567890123",
 		error: "line 1.7: invalid int32: 1234567890123",
 	},
 
 	// Number too large for float32
-	UnmarshalTextTest{
+	{
 		in:    "others:< weight: 12345678901234567890123456789012345678901234567890 >",
 		error: "line 1.17: invalid float32: 12345678901234567890123456789012345678901234567890",
 	},
 
 	// Number posing as a quoted string
-	UnmarshalTextTest{
+	{
 		in:    `inner: < host: 12 >` + "\n",
 		error: `line 1.15: invalid string: 12`,
 	},
 
 	// Quoted string posing as int32
-	UnmarshalTextTest{
+	{
 		in:    `count: "12"`,
 		error: `line 1.7: invalid int32: "12"`,
 	},
 
 	// Quoted string posing a float32
-	UnmarshalTextTest{
+	{
 		in:    `others:< weight: "17.4" >`,
 		error: `line 1.17: invalid float32: "17.4"`,
 	},
 
 	// Enum
-	UnmarshalTextTest{
+	{
 		in: `count:42 bikeshed: BLUE`,
 		out: &MyMessage{
 			Count:    Int32(42),
@@ -124,7 +124,7 @@ var unMarshalTextTests = []UnmarshalTextTest{
 	},
 
 	// Repeated field
-	UnmarshalTextTest{
+	{
 		in: `count:42 pet: "horsey" pet:"bunny"`,
 		out: &MyMessage{
 			Count: Int32(42),
@@ -133,7 +133,7 @@ var unMarshalTextTests = []UnmarshalTextTest{
 	},
 
 	// Repeated message with/without colon and <>/{}
-	UnmarshalTextTest{
+	{
 		in: `count:42 others:{} others{} others:<> others:{}`,
 		out: &MyMessage{
 			Count: Int32(42),
@@ -147,7 +147,7 @@ var unMarshalTextTests = []UnmarshalTextTest{
 	},
 
 	// Missing colon for inner message
-	UnmarshalTextTest{
+	{
 		in: `count:42 inner < host: "cauchy.syd" >`,
 		out: &MyMessage{
 			Count: Int32(42),
@@ -158,31 +158,42 @@ var unMarshalTextTests = []UnmarshalTextTest{
 	},
 
 	// Missing colon for string field
-	UnmarshalTextTest{
+	{
 		in:    `name "Dave"`,
 		error: `line 1.5: expected ':', found "\"Dave\""`,
 	},
 
 	// Missing colon for int32 field
-	UnmarshalTextTest{
+	{
 		in:    `count 42`,
 		error: `line 1.6: expected ':', found "42"`,
 	},
 
 	// Missing required field
-	UnmarshalTextTest{
+	{
 		in:    ``,
 		error: `line 1.0: message test_proto.MyMessage missing required field "count"`,
 	},
 
 	// Repeated non-repeated field
-	UnmarshalTextTest{
+	{
 		in:    `name: "Rob" name: "Russ"`,
 		error: `line 1.12: non-repeated field "name" was repeated`,
 	},
 
+	// Group
+	{
+		in: `count: 17 SomeGroup { group_field: 12 }`,
+		out: &MyMessage{
+			Count: Int32(17),
+			Somegroup: &MyMessage_SomeGroup{
+				GroupField: Int32(12),
+			},
+		},
+	},
+
 	// Big all-in-one
-	UnmarshalTextTest{
+	{
 		in: "count:42  # Meaning\n" +
 			`name:"Dave" ` +
 			`quote:"\"I didn't want to go.\"" ` +

proto/text_test.go

@@ -63,6 +63,9 @@ func newTestMessage() *MyMessage {
 			},
 		},
 		Bikeshed: NewMyMessage_Color(MyMessage_BLUE),
+		Somegroup: &MyMessage_SomeGroup{
+			GroupField: Int32(8),
+		},
 	}
 }
 
@@ -89,6 +92,9 @@ others: <
   >
 >
 bikeshed: BLUE
+SomeGroup {
+  group_field: 8
+}
 `
 
 func TestMarshalTextFull(t *testing.T) {
@@ -111,7 +117,10 @@ func compact(src string) string {
 			space = true
 			continue
 		}
-		if j > 0 && (dst[j-1] == ':' || dst[j-1] == '<') {
+		if j > 0 && (dst[j-1] == ':' || dst[j-1] == '<' || dst[j-1] == '{') {
+			space = false
+		}
+		if c == '{' {
 			space = false
 		}
 		if space {