reflect/protoreflect: add FileDescriptor.SourceLocations

This adds minimal support for preserving the source context information.

Change-Id: I4b3cac9690b7469ecb4e5434251a809be4d7894c
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/183157
Reviewed-by: Damien Neil <dneil@google.com>

internal/descfmt/desc_test.go

@@ -18,18 +18,15 @@ func TestDescriptorAccessors(t *testing.T) {
 		"Name":          true,
 		"FullName":      true,
 		"IsPlaceholder": true,
+		"Options":       true,
 		"ProtoInternal": true,
 		"ProtoType":     true,
 
-		"DescriptorByName": true, // specific to FileDescriptor
-		"DefaultEnumValue": true, // specific to FieldDescriptor
-		"MapKey":           true, // specific to FieldDescriptor
-		"MapValue":         true, // specific to FieldDescriptor
-
-		// TODO: These should be removed or handled.
-		"DescriptorProto":       true,
-		"ExtensionRangeOptions": true,
-		"Options":               true,
+		"SourceLocations":       true, // specific to FileDescriptor
+		"ExtensionRangeOptions": true, // specific to MessageDescriptor
+		"DefaultEnumValue":      true, // specific to FieldDescriptor
+		"MapKey":                true, // specific to FieldDescriptor
+		"MapValue":              true, // specific to FieldDescriptor
 	}
 
 	for rt, m := range descriptorAccessors {

internal/filedesc/desc.go

@@ -50,8 +50,9 @@ type (
 		Services   Services
 	}
 	FileL2 struct {
-		Options func() pref.ProtoMessage
-		Imports FileImports
+		Options   func() pref.ProtoMessage
+		Imports   FileImports
+		Locations SourceLocations
 	}
 )
 
@@ -75,6 +76,7 @@ func (fd *File) Enums() pref.EnumDescriptors           { return &fd.L1.Enums }
 func (fd *File) Messages() pref.MessageDescriptors     { return &fd.L1.Messages }
 func (fd *File) Extensions() pref.ExtensionDescriptors { return &fd.L1.Extensions }
 func (fd *File) Services() pref.ServiceDescriptors     { return &fd.L1.Services }
+func (fd *File) SourceLocations() pref.SourceLocations { return &fd.L2.Locations }
 func (fd *File) Format(s fmt.State, r rune)            { descfmt.FormatDesc(s, r, fd) }
 func (fd *File) ProtoType(pref.FileDescriptor)         {}
 func (fd *File) ProtoInternal(pragma.DoNotImplement)   {}

internal/filedesc/desc_list.go

@@ -276,3 +276,11 @@ func (p *OneofFields) lazyInit() *OneofFields {
 	})
 	return p
 }
+
+type SourceLocations struct {
+	List []pref.SourceLocation
+}
+
+func (p *SourceLocations) Len() int                            { return len(p.List) }
+func (p *SourceLocations) Get(i int) pref.SourceLocation       { return p.List[i] }
+func (p *SourceLocations) ProtoInternal(pragma.DoNotImplement) {}

internal/filedesc/placeholder.go

@@ -11,10 +11,11 @@ import (
 )
 
 var (
-	emptyNames        = new(Names)
-	emptyEnumRanges   = new(EnumRanges)
-	emptyFieldRanges  = new(FieldRanges)
-	emptyFieldNumbers = new(FieldNumbers)
+	emptyNames           = new(Names)
+	emptyEnumRanges      = new(EnumRanges)
+	emptyFieldRanges     = new(FieldRanges)
+	emptyFieldNumbers    = new(FieldNumbers)
+	emptySourceLocations = new(SourceLocations)
 
 	emptyFiles      = new(FileImports)
 	emptyMessages   = new(Messages)
@@ -44,6 +45,7 @@ func (f PlaceholderFile) Messages() pref.MessageDescriptors     { return emptyMe
 func (f PlaceholderFile) Enums() pref.EnumDescriptors           { return emptyEnums }
 func (f PlaceholderFile) Extensions() pref.ExtensionDescriptors { return emptyExtensions }
 func (f PlaceholderFile) Services() pref.ServiceDescriptors     { return emptyServices }
+func (f PlaceholderFile) SourceLocations() pref.SourceLocations { return emptySourceLocations }
 func (f PlaceholderFile) ProtoType(pref.FileDescriptor)         { return }
 func (f PlaceholderFile) ProtoInternal(pragma.DoNotImplement)   { return }
 

reflect/protodesc/desc.go

@@ -136,6 +136,32 @@ func newFile(fd *descriptorpb.FileDescriptorProto, r Resolver, opts ...option) (
 		imps.importPublic(imp.Imports())
 	}
 
+	// Handle source locations.
+	for _, loc := range fd.GetSourceCodeInfo().GetLocation() {
+		var l protoreflect.SourceLocation
+		// TODO: Validate that the path points to an actual declaration?
+		l.Path = protoreflect.SourcePath(loc.GetPath())
+		s := loc.GetSpan()
+		switch len(s) {
+		case 3:
+			l.StartLine, l.StartColumn, l.EndLine, l.EndColumn = int(s[0]), int(s[1]), int(s[0]), int(s[2])
+		case 4:
+			l.StartLine, l.StartColumn, l.EndLine, l.EndColumn = int(s[0]), int(s[1]), int(s[2]), int(s[3])
+		default:
+			return nil, errors.New("invalid span: %v", s)
+		}
+		// TODO: Validate that the span information is sensible?
+		// See https://github.com/protocolbuffers/protobuf/issues/6378.
+		if false && (l.EndLine < l.StartLine || l.StartLine < 0 || l.StartColumn < 0 || l.EndColumn < 0 ||
+			(l.StartLine == l.EndLine && l.EndColumn <= l.StartColumn)) {
+			return nil, errors.New("invalid span: %v", s)
+		}
+		l.LeadingDetachedComments = loc.GetLeadingDetachedComments()
+		l.LeadingComments = loc.GetLeadingComments()
+		l.TrailingComments = loc.GetTrailingComments()
+		f.L2.Locations.List = append(f.L2.Locations.List, l)
+	}
+
 	// Step 1: Allocate and derive the names for all declarations.
 	// This copies all fields from the descriptor proto except:
 	//	google.protobuf.FieldDescriptorProto.type_name

reflect/protodesc/file_test.go

@@ -180,6 +180,33 @@ func TestNewFile(t *testing.T) {
 			dependency: ["import_public4.proto"],
 		`),
 		// TODO: Test import public
+	}, {
+		label: "preserve source code locations",
+		inDesc: mustParseFile(`
+			name: "test.proto"
+			package: "fizz.buzz"
+			source_code_info: {location: [{
+				span: [39,0,882,1]
+			}, {
+				path: [12]
+				span: [39,0,18]
+				leading_detached_comments: [" foo\n"," bar\n"]
+			}, {
+				path: [8,9]
+				span: [51,0,28]
+				leading_comments: " Comment\n"
+			}]}
+		`),
+	}, {
+		label: "invalid source code span",
+		inDesc: mustParseFile(`
+			name: "test.proto"
+			package: "fizz.buzz"
+			source_code_info: {location: [{
+				span: [39]
+			}]}
+		`),
+		wantErr: `invalid span: [39]`,
 	}, {
 		label: "resolve relative reference",
 		inDesc: mustParseFile(`

reflect/protodesc/proto.go

@@ -34,6 +34,28 @@ func ToFileDescriptorProto(file protoreflect.FileDescriptor) *descriptorpb.FileD
 			p.WeakDependency = append(p.WeakDependency, int32(i))
 		}
 	}
+	for i, locs := 0, file.SourceLocations(); i < locs.Len(); i++ {
+		loc := locs.Get(i)
+		l := &descriptorpb.SourceCodeInfo_Location{}
+		l.Path = append(l.Path, loc.Path...)
+		if loc.StartLine == loc.EndLine {
+			l.Span = []int32{int32(loc.StartLine), int32(loc.StartColumn), int32(loc.EndColumn)}
+		} else {
+			l.Span = []int32{int32(loc.StartLine), int32(loc.StartColumn), int32(loc.EndLine), int32(loc.EndColumn)}
+		}
+		l.LeadingDetachedComments = append([]string(nil), loc.LeadingDetachedComments...)
+		if loc.LeadingComments != "" {
+			l.LeadingComments = proto.String(loc.LeadingComments)
+		}
+		if loc.TrailingComments != "" {
+			l.TrailingComments = proto.String(loc.TrailingComments)
+		}
+		if p.SourceCodeInfo == nil {
+			p.SourceCodeInfo = &descriptorpb.SourceCodeInfo{}
+		}
+		p.SourceCodeInfo.Location = append(p.SourceCodeInfo.Location, l)
+
+	}
 	for i, messages := 0, file.Messages(); i < messages.Len(); i++ {
 		p.MessageType = append(p.MessageType, ToDescriptorProto(messages.Get(i)))
 	}

reflect/protoreflect/source.go

@@ -0,0 +1,52 @@
+// Copyright 2019 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package protoreflect
+
+// SourceLocations is a list of source locations.
+type SourceLocations interface {
+	// Len reports the number of source locations in the proto file.
+	Len() int
+	// Get returns the ith SourceLocation. It panics if out of bounds.
+	Get(int) SourceLocation
+
+	doNotImplement
+
+	// TODO: Add ByPath and ByDescriptor helper methods.
+}
+
+// SourceLocation describes a source location and
+// corresponds with the google.protobuf.SourceCodeInfo.Location message.
+type SourceLocation struct {
+	// Path is the path to the declaration from the root file descriptor.
+	// The contents of this slice must not be mutated.
+	Path SourcePath
+
+	// StartLine and StartColumn are the zero-indexed starting location
+	// in the source file for the declaration.
+	StartLine, StartColumn int
+	// EndLine and EndColumn are the zero-indexed ending location
+	// in the source file for the declaration.
+	// In the descriptor.proto, the end line may be omitted if it is identical
+	// to the start line. Here, it is always populated.
+	EndLine, EndColumn int
+
+	// LeadingDetachedComments are the leading detached comments
+	// for the declaration. The contents of this slice must not be mutated.
+	LeadingDetachedComments []string
+	// LeadingComments is the leading attached comment for the declaration.
+	LeadingComments string
+	// TrailingComments is the trailing attached comment for the declaration.
+	TrailingComments string
+}
+
+// SourcePath identifies a part of a file descriptor for a source location.
+// The SourcePath is a sequence of either field numbers or indexes into
+// a repeated field that form a path starting from the root file descriptor.
+//
+// See google.protobuf.SourceCodeInfo.Location.path.
+type SourcePath []int32
+
+// TODO: Add SourcePath.String method to pretty-print the path. For example:
+//	".message_type[6].nested_type[15].field[3]"

reflect/protoreflect/type.go

@@ -133,6 +133,9 @@ type FileDescriptor interface {
 	// Services is a list of the top-level service declarations.
 	Services() ServiceDescriptors
 
+	// SourceLocations is a list of source locations.
+	SourceLocations() SourceLocations
+
 	isFileDescriptor
 }
 type isFileDescriptor interface{ ProtoType(FileDescriptor) }