codec: json: inline skipWhitespace, handle exported anonymous members, docs

- json: inline skipWhitespace:
  this may give some slight performance improvement by eliding the function call
  on the hot-path.
- handle exported members of unexported anonymous fields
  See https://github.com/golang/go/issues/12367 for more info
- document that WriteExt, ConvertExt may take a pointer value.
  To avoid potentially expensive copying, we may pass in the address to a struct or array

codec/helper.go

@@ -112,8 +112,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"unicode"
-	"unicode/utf8"
 )
 
 const (
@@ -357,6 +355,8 @@ type RawExt struct {
 // It is used by codecs (e.g. binc, msgpack, simple) which do custom serialization of the types.
 type BytesExt interface {
 	// WriteExt converts a value to a []byte.
+	//
+	// Note: v *may* be a pointer to the extension type, if the extension type was a struct or array.
 	WriteExt(v interface{}) []byte
 
 	// ReadExt updates a value from a []byte.
@@ -369,6 +369,8 @@ type BytesExt interface {
 // It is used by codecs (e.g. cbor, json) which use the format to do custom serialization of the types.
 type InterfaceExt interface {
 	// ConvertExt converts a value into a simpler interface for easy encoding e.g. convert time.Time to int64.
+	//
+	// Note: v *may* be a pointer to the extension type, if the extension type was a struct or array.
 	ConvertExt(v interface{}) interface{}
 
 	// UpdateExt updates a value from a simpler interface for easy decoding e.g. convert int64 to time.Time.
@@ -388,7 +390,6 @@ type addExtWrapper struct {
 }
 
 func (x addExtWrapper) WriteExt(v interface{}) []byte {
-	// fmt.Printf(">>>>>>>>>> WriteExt: %T, %v\n", v, v)
 	bs, err := x.encFn(reflect.ValueOf(v))
 	if err != nil {
 		panic(err)
@@ -397,7 +398,6 @@ func (x addExtWrapper) WriteExt(v interface{}) []byte {
 }
 
 func (x addExtWrapper) ReadExt(v interface{}, bs []byte) {
-	// fmt.Printf(">>>>>>>>>> ReadExt: %T, %v\n", v, v)
 	if err := x.decFn(reflect.ValueOf(v), bs); err != nil {
 		panic(err)
 	}
@@ -858,45 +858,49 @@ func (x *TypeInfos) rget(rt reflect.Type, indexstack []int, fnameToHastag map[st
 ) {
 	for j := 0; j < rt.NumField(); j++ {
 		f := rt.Field(j)
-		// func types are skipped.
-		if tk := f.Type.Kind(); tk == reflect.Func {
+		fkind := f.Type.Kind()
+		// skip if a func type, or is unexported, or structTag value == "-"
+		if fkind == reflect.Func {
 			continue
 		}
-		stag := x.structTag(f.Tag)
-		if stag == "-" {
+		// if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
+		if f.PkgPath != "" && !f.Anonymous { // unexported, not embedded
 			continue
 		}
-		if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
+		stag := x.structTag(f.Tag)
+		if stag == "-" {
 			continue
 		}
 		var si *structFieldInfo
-		// if anonymous and there is no struct tag (or it's blank)
-		// and its a struct (or pointer to struct), inline it.
-		var doInline bool
-		if f.Anonymous && f.Type.Kind() != reflect.Interface {
-			doInline = stag == ""
+		// if anonymous and no struct tag (or it's blank), and a struct (or pointer to struct), inline it.
+		if f.Anonymous && fkind != reflect.Interface {
+			doInline := stag == ""
 			if !doInline {
 				si = parseStructFieldInfo("", stag)
 				doInline = si.encName == ""
 				// doInline = si.isZero()
-				// fmt.Printf(">>>> doInline for si.isZero: %s: %v\n", f.Name, doInline)
+			}
+			if doInline {
+				ft := f.Type
+				for ft.Kind() == reflect.Ptr {
+					ft = ft.Elem()
+				}
+				if ft.Kind() == reflect.Struct {
+					indexstack2 := make([]int, len(indexstack)+1, len(indexstack)+4)
+					copy(indexstack2, indexstack)
+					indexstack2[len(indexstack)] = j
+					// indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
+					x.rget(ft, indexstack2, fnameToHastag, sfi, siInfo)
+					continue
+				}
 			}
 		}
 
-		if doInline {
-			ft := f.Type
-			for ft.Kind() == reflect.Ptr {
-				ft = ft.Elem()
-			}
-			if ft.Kind() == reflect.Struct {
-				indexstack2 := make([]int, len(indexstack)+1, len(indexstack)+4)
-				copy(indexstack2, indexstack)
-				indexstack2[len(indexstack)] = j
-				// indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
-				x.rget(ft, indexstack2, fnameToHastag, sfi, siInfo)
-				continue
-			}
+		// after the anonymous dance: if an unexported field, skip
+		if f.PkgPath != "" { // unexported
+			continue
 		}
+
 		// do not let fields with same name in embedded structs override field at higher level.
 		// this must be done after anonymous check, to allow anonymous field
 		// still include their child fields

codec/json.go

@@ -385,41 +385,26 @@ func jsonIsWS(b byte) bool {
 	return b == ' ' || b == '\t' || b == '\r' || b == '\n'
 }
 
-// This will skip whitespace characters and return the next byte to read.
-// The next byte determines what the value will be one of.
-func (d *jsonDecDriver) skipWhitespace() {
-	// fast-path: do not enter loop. Just check first (in case no whitespace).
-	b := d.r.readn1()
-	if jsonIsWS(b) {
-		r := d.r
-		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
-		}
-	}
-	d.tok = b
-}
-
-// // To inline  if d.tok == 0 { d.skipWhitespace() }: USE:
-// func (d *jsonDecDriver) XXX() {
-// 	if d.tok == 0 {
-// 		b := d.r.readn1()
-// 		if jsonIsWS(b) {
-// 			r := d.r
-// 			for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
-// 			}
-// 		}
-// 		d.tok = b
-// 	}
-// 	// OR
-// 	if b, r := d.tok, d.r; b == 0 {
+// // This will skip whitespace characters and return the next byte to read.
+// // The next byte determines what the value will be one of.
+// func (d *jsonDecDriver) skipWhitespace() {
+// 	// fast-path: do not enter loop. Just check first (in case no whitespace).
+// 	b := d.r.readn1()
+// 	if jsonIsWS(b) {
+// 		r := d.r
 // 		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
 // 		}
-// 		d.tok = b
 // 	}
+// 	d.tok = b
 // }
 
 func (d *jsonDecDriver) sendContainerState(c containerState) {
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	var xc uint8 // char expected
 	if c == containerMapKey {
@@ -448,7 +433,11 @@ func (d *jsonDecDriver) sendContainerState(c containerState) {
 
 func (d *jsonDecDriver) CheckBreak() bool {
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	if d.tok == '}' || d.tok == ']' {
 		// d.tok = 0 // only checking, not consuming
@@ -470,7 +459,11 @@ func (d *jsonDecDriver) readStrIdx(fromIdx, toIdx uint8) {
 
 func (d *jsonDecDriver) TryDecodeAsNil() bool {
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	if d.tok == 'n' {
 		d.readStrIdx(10, 13) // ull
@@ -481,7 +474,11 @@ func (d *jsonDecDriver) TryDecodeAsNil() bool {
 
 func (d *jsonDecDriver) DecodeBool() bool {
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	if d.tok == 'f' {
 		d.readStrIdx(5, 9) // alse
@@ -497,7 +494,11 @@ func (d *jsonDecDriver) DecodeBool() bool {
 
 func (d *jsonDecDriver) ReadMapStart() int {
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	if d.tok != '{' {
 		d.d.errorf("json: expect char '%c' but got char '%c'", '{', d.tok)
@@ -509,7 +510,11 @@ func (d *jsonDecDriver) ReadMapStart() int {
 
 func (d *jsonDecDriver) ReadArrayStart() int {
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	if d.tok != '[' {
 		d.d.errorf("json: expect char '%c' but got char '%c'", '[', d.tok)
@@ -522,7 +527,11 @@ func (d *jsonDecDriver) ReadArrayStart() int {
 func (d *jsonDecDriver) ContainerType() (vt valueType) {
 	// check container type by checking the first char
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	if b := d.tok; b == '{' {
 		return valueTypeMap
@@ -541,7 +550,11 @@ func (d *jsonDecDriver) ContainerType() (vt valueType) {
 func (d *jsonDecDriver) decNum(storeBytes bool) {
 	// If it is has a . or an e|E, decode as a float; else decode as an int.
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	b := d.tok
 	if !(b == '+' || b == '-' || b == '.' || (b >= '0' && b <= '9')) {
@@ -833,7 +846,11 @@ func (d *jsonDecDriver) DecodeString() (s string) {
 
 func (d *jsonDecDriver) appendStringAsBytes() {
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	if d.tok != '"' {
 		d.d.errorf("json: expect char '%c' but got char '%c'", '"', d.tok)
@@ -910,7 +927,11 @@ func (d *jsonDecDriver) DecodeNaked() {
 	// var decodeFurther bool
 
 	if d.tok == 0 {
-		d.skipWhitespace()
+		var b byte
+		r := d.r
+		for b = r.readn1(); jsonIsWS(b); b = r.readn1() {
+		}
+		d.tok = b
 	}
 	switch d.tok {
 	case 'n':

codec/time.go

@@ -34,32 +34,10 @@ func (x timeExt) ReadExt(v interface{}, bs []byte) {
 }
 
 func (x timeExt) ConvertExt(v interface{}) interface{} {
-	var bs []byte
-	switch v2 := v.(type) {
-	case time.Time:
-		bs = encodeTime(v2)
-	case *time.Time:
-		bs = encodeTime(*v2)
-	default:
-		panic(fmt.Errorf("unsupported format for time conversion: expecting time.Time; got %T", v2))
-	}
-	return bs
+	return x.WriteExt(v)
 }
 func (x timeExt) UpdateExt(v interface{}, src interface{}) {
-	var bs []byte
-	switch s2 := src.(type) {
-	case []byte:
-		bs = s2
-	case *[]byte:
-		bs = *s2
-	default:
-		panic(fmt.Errorf("unsupported format for time conversion: expecting []byte; got %T", s2))
-	}
-	tt, err := decodeTime(bs)
-	if err != nil {
-		panic(err)
-	}
-	*(v.(*time.Time)) = tt
+	x.ReadExt(v, src.([]byte))
 }
 
 // EncodeTime encodes a time.Time as a []byte, including