codec: streamline go's rules for anonymous field conflict, detect circular references, json indent

anonymous fields:
we use go's rules to determine how conflicts in encNames of fieldNames are resolved.
- Anonymous fields are now processed at the end of the execution
- if a field has been seen while traversing, skip it
- if an encName has been seen while traversing, skip it
- if an embedded type has been seen, skip it

performance improvements using sync.Pool and slices (not maps) when getting typeInfo
- use sync.Pool to reduce amount of recreation of buffers during rget calls.
- use slices not maps to track a set of values.
  For small number typical when caching a typeInfo, the number of fields typically <16.
- move encoder.cirRef{Add,Rm} to its own type called "set", for potential re-use.

CheckCircularRef encode option: error on circular reference
- A circular reference comes when we see a pointer to a struct,
  and see it again before we are done encoding it.
- If configured, we will report an error on detecting a circular reference,
  instead of allowing a possible stack-overflow which crashes the process.
- Tests are included.
- *This is not yet supported in codecgen*

json: indent support during encode
- we can encode json as either compact or a prettified indented multi-line text.
- we support tabs or spaces

Misc:
- move timeExt(Enc|Dec)Fn into time.go
- add tests for std encoding support
- implement reset of (end|dec)Driver's completely
- add more tests for issues found in github

Update #115

codec/0doc.go

@@ -64,6 +64,7 @@ Rich Feature Set includes:
   - Never silently skip data when decoding.
     User decides whether to return an error or silently skip data when keys or indexes
     in the data stream do not map to fields in the struct.
+  - Detect and error when encoding a cyclic reference (instead of stack overflow shutdown)
   - Encode/Decode from/to chan types (for iterative streaming support)
   - Drop-in replacement for encoding/json. `json:` key in struct tag supported.
   - Provides a RPC Server and Client Codec for net/rpc communication protocol.
@@ -171,6 +172,8 @@ package codec
 
 // TODO:
 //
+//   - optimization for codecgen:
+//     if len of entity is <= 3 words, then support a value receiver for encode.
 //   - (En|De)coder should store an error when it occurs.
 //     Until reset, subsequent calls return that error that was stored.
 //     This means that free panics must go away.
@@ -190,4 +193,7 @@ package codec
 //    - Consider making Handle used AS-IS within the encoding/decoding session.
 //      This means that we don't cache Handle information within the (En|De)coder,
 //      except we really need it at Reset(...)
-//    - Handle recursive types during encoding/decoding?
+//    - Consider adding math/big support
+//    - Consider reducing the size of the generated functions:
+//      Maybe use one loop, and put the conditionals in the loop.
+//      for ... { if cLen > 0 { if j == cLen { break } } else if dd.CheckBreak() { break } }

codec/binc.go

@@ -908,10 +908,14 @@ func (h *BincHandle) newDecDriver(d *Decoder) decDriver {
 
 func (e *bincEncDriver) reset() {
 	e.w = e.e.w
+	e.s = 0
+	e.m = nil
 }
 
 func (d *bincDecDriver) reset() {
 	d.r = d.d.r
+	d.s = nil
+	d.bd, d.bdRead, d.vd, d.vs = 0, false, 0, 0
 }
 
 var _ decDriver = (*bincDecDriver)(nil)

codec/cbor.go

@@ -578,6 +578,7 @@ func (e *cborEncDriver) reset() {
 
 func (d *cborDecDriver) reset() {
 	d.r = d.d.r
+	d.bd, d.bdRead = 0, false
 }
 
 var _ decDriver = (*cborDecDriver)(nil)

codec/codec_test.go

@@ -37,6 +37,7 @@ import (
 	"reflect"
 	"runtime"
 	"strconv"
+	"strings"
 	"sync/atomic"
 	"testing"
 	"time"
@@ -70,6 +71,8 @@ var (
 	testSkipIntf       bool
 	testInternStr      bool
 	testUseMust        bool
+	testCheckCircRef   bool
+	testJsonIndent     int
 
 	skipVerifyVal interface{} = &(struct{}{})
 
@@ -104,7 +107,9 @@ func testInitFlags() {
 	flag.BoolVar(&testInternStr, "te", false, "Set InternStr option")
 	flag.BoolVar(&testSkipIntf, "tf", false, "Skip Interfaces")
 	flag.BoolVar(&testUseReset, "tr", false, "Use Reset")
+	flag.IntVar(&testJsonIndent, "td", 0, "Use JSON Indent")
 	flag.BoolVar(&testUseMust, "tm", true, "Use Must(En|De)code")
+	flag.BoolVar(&testCheckCircRef, "tl", false, "Use Check Circular Ref")
 }
 
 func testByteBuf(in []byte) *bytes.Buffer {
@@ -115,6 +120,46 @@ type TestABC struct {
 	A, B, C string
 }
 
+func (x *TestABC) MarshalBinary() ([]byte, error) {
+	return []byte(fmt.Sprintf("%s %s %s", x.A, x.B, x.C)), nil
+}
+func (x *TestABC) MarshalText() ([]byte, error) {
+	return []byte(fmt.Sprintf("%s %s %s", x.A, x.B, x.C)), nil
+}
+func (x *TestABC) MarshalJSON() ([]byte, error) {
+	return []byte(fmt.Sprintf(`"%s %s %s"`, x.A, x.B, x.C)), nil
+}
+
+func (x *TestABC) UnmarshalBinary(data []byte) (err error) {
+	ss := strings.Split(string(data), " ")
+	x.A, x.B, x.C = ss[0], ss[1], ss[2]
+	return
+}
+func (x *TestABC) UnmarshalText(data []byte) (err error) {
+	return x.UnmarshalBinary(data)
+}
+func (x *TestABC) UnmarshalJSON(data []byte) (err error) {
+	return x.UnmarshalBinary(data[1 : len(data)-1])
+}
+
+type TestABC2 struct {
+	A, B, C string
+}
+
+func (x TestABC2) MarshalText() ([]byte, error) {
+	return []byte(fmt.Sprintf("%s %s %s", x.A, x.B, x.C)), nil
+}
+func (x *TestABC2) UnmarshalText(data []byte) (err error) {
+	ss := strings.Split(string(data), " ")
+	x.A, x.B, x.C = ss[0], ss[1], ss[2]
+	return
+	// _, err = fmt.Sscanf(string(data), "%s %s %s", &x.A, &x.B, &x.C)
+}
+
+type TestRpcABC struct {
+	A, B, C string
+}
+
 type TestRpcInt struct {
 	i int
 }
@@ -122,7 +167,7 @@ type TestRpcInt struct {
 func (r *TestRpcInt) Update(n int, res *int) error      { r.i = n; *res = r.i; return nil }
 func (r *TestRpcInt) Square(ignore int, res *int) error { *res = r.i * r.i; return nil }
 func (r *TestRpcInt) Mult(n int, res *int) error        { *res = r.i * n; return nil }
-func (r *TestRpcInt) EchoStruct(arg TestABC, res *string) error {
+func (r *TestRpcInt) EchoStruct(arg TestRpcABC, res *string) error {
 	*res = fmt.Sprintf("%#v", arg)
 	return nil
 }
@@ -292,6 +337,7 @@ func testInit() {
 		bh := v.getBasicHandle()
 		bh.InternString = testInternStr
 		bh.Canonical = testCanonical
+		bh.CheckCircularRef = testCheckCircRef
 		bh.StructToArray = testStructToArray
 		// mostly doing this for binc
 		if testWriteNoSymbols {
@@ -301,27 +347,20 @@ func testInit() {
 		}
 	}
 
+	testJsonH.Indent = int8(testJsonIndent)
 	testMsgpackH.RawToString = true
 
 	// testMsgpackH.AddExt(byteSliceTyp, 0, testMsgpackH.BinaryEncodeExt, testMsgpackH.BinaryDecodeExt)
 	// testMsgpackH.AddExt(timeTyp, 1, testMsgpackH.TimeEncodeExt, testMsgpackH.TimeDecodeExt)
-	timeEncExt := func(rv reflect.Value) (bs []byte, err error) {
-		defer panicToErr(&err)
-		bs = timeExt{}.WriteExt(rv.Interface())
-		return
-	}
-	timeDecExt := func(rv reflect.Value, bs []byte) (err error) {
-		defer panicToErr(&err)
-		timeExt{}.ReadExt(rv.Interface(), bs)
-		return
-	}
 
 	// add extensions for msgpack, simple for time.Time, so we can encode/decode same way.
 	// use different flavors of XXXExt calls, including deprecated ones.
-	testSimpleH.AddExt(timeTyp, 1, timeEncExt, timeDecExt)
+	// NOTE:
+	// DO NOT set extensions for JsonH, so we can test json(M|Unm)arshal support.
+	testSimpleH.AddExt(timeTyp, 1, timeExtEncFn, timeExtDecFn)
 	testMsgpackH.SetBytesExt(timeTyp, 1, timeExt{})
 	testCborH.SetInterfaceExt(timeTyp, 1, &testUnixNanoTimeExt{})
-	testJsonH.SetInterfaceExt(timeTyp, 1, &testUnixNanoTimeExt{})
+	// testJsonH.SetInterfaceExt(timeTyp, 1, &testUnixNanoTimeExt{})
 
 	primitives := []interface{}{
 		int8(-8),
@@ -818,8 +857,8 @@ func testCodecRpcOne(t *testing.T, rr Rpc, h Handle, doRequest bool, exitSleepMs
 		checkEqualT(t, sq, 25, "sq=25")
 		checkErrT(t, cl.Call("TestRpcInt.Mult", 20, &mult))
 		checkEqualT(t, mult, 100, "mult=100")
-		checkErrT(t, cl.Call("TestRpcInt.EchoStruct", TestABC{"Aa", "Bb", "Cc"}, &rstr))
-		checkEqualT(t, rstr, fmt.Sprintf("%#v", TestABC{"Aa", "Bb", "Cc"}), "rstr=")
+		checkErrT(t, cl.Call("TestRpcInt.EchoStruct", TestRpcABC{"Aa", "Bb", "Cc"}, &rstr))
+		checkEqualT(t, rstr, fmt.Sprintf("%#v", TestRpcABC{"Aa", "Bb", "Cc"}), "rstr=")
 		checkErrT(t, cl.Call("TestRpcInt.Echo123", []string{"A1", "B2", "C3"}, &rstr))
 		checkEqualT(t, rstr, fmt.Sprintf("%#v", []string{"A1", "B2", "C3"}), "rstr=")
 	}
@@ -917,6 +956,95 @@ func doTestMapEncodeForCanonical(t *testing.T, name string, h Handle) {
 	}
 }
 
+func doTestStdEncIntf(t *testing.T, name string, h Handle) {
+	args := [][2]interface{}{
+		{&TestABC{"A", "BB", "CCC"}, new(TestABC)},
+		{&TestABC2{"AAA", "BB", "C"}, new(TestABC2)},
+	}
+	for _, a := range args {
+		var b []byte
+		e := NewEncoderBytes(&b, h)
+		e.MustEncode(a[0])
+		d := NewDecoderBytes(b, h)
+		d.MustDecode(a[1])
+		if err := deepEqual(a[0], a[1]); err == nil {
+			logT(t, "++++ Objects match")
+		} else {
+			logT(t, "---- Objects do not match: y1: %v, err: %v", a[1], err)
+			failT(t)
+		}
+	}
+}
+
+func doTestEncCircularRef(t *testing.T, name string, h Handle) {
+	type T1 struct {
+		S string
+		B bool
+		T interface{}
+	}
+	type T2 struct {
+		S string
+		T *T1
+	}
+	type T3 struct {
+		S string
+		T *T2
+	}
+	t1 := T1{"t1", true, nil}
+	t2 := T2{"t2", &t1}
+	t3 := T3{"t3", &t2}
+	t1.T = &t3
+
+	var bs []byte
+	var err error
+
+	bh := h.getBasicHandle()
+	if !bh.CheckCircularRef {
+		bh.CheckCircularRef = true
+		defer func() { bh.CheckCircularRef = false }()
+	}
+	err = NewEncoderBytes(&bs, h).Encode(&t3)
+	if err == nil {
+		logT(t, "expecting error due to circular reference. found none")
+		t.FailNow()
+	}
+	if x := err.Error(); strings.Contains(x, "circular") || strings.Contains(x, "cyclic") {
+		logT(t, "error detected as expected: %v", x)
+	} else {
+		logT(t, "error detected was not as expected: %v", x)
+		t.FailNow()
+	}
+}
+
+// TestAnonCycleT{1,2,3} types are used to test anonymous cycles.
+// They are top-level, so that they can have circular references.
+type (
+	TestAnonCycleT1 struct {
+		S string
+		TestAnonCycleT2
+	}
+	TestAnonCycleT2 struct {
+		S2 string
+		TestAnonCycleT3
+	}
+	TestAnonCycleT3 struct {
+		*TestAnonCycleT1
+	}
+)
+
+func doTestAnonCycle(t *testing.T, name string, h Handle) {
+	var x TestAnonCycleT1
+	x.S = "hello"
+	x.TestAnonCycleT2.S2 = "hello.2"
+	x.TestAnonCycleT2.TestAnonCycleT3.TestAnonCycleT1 = &x
+
+	// just check that you can get typeInfo for T1
+	rt := reflect.TypeOf((*TestAnonCycleT1)(nil)).Elem()
+	rtid := reflect.ValueOf(rt).Pointer()
+	pti := h.getBasicHandle().getTypeInfo(rtid, rt)
+	logT(t, "pti: %v", pti)
+}
+
 // Comprehensive testing that generates data encoded from python handle (cbor, msgpack),
 // and validates that our code can read and write it out accordingly.
 // We keep this unexported here, and put actual test in ext_dep_test.go.
@@ -1038,7 +1166,7 @@ func doTestMsgpackRpcSpecGoClientToPythonSvc(t *testing.T) {
 	cl := rpc.NewClientWithCodec(cc)
 	defer cl.Close()
 	var rstr string
-	checkErrT(t, cl.Call("EchoStruct", TestABC{"Aa", "Bb", "Cc"}, &rstr))
+	checkErrT(t, cl.Call("EchoStruct", TestRpcABC{"Aa", "Bb", "Cc"}, &rstr))
 	//checkEqualT(t, rstr, "{'A': 'Aa', 'B': 'Bb', 'C': 'Cc'}")
 	var mArgs MsgpackSpecRpcMultiArgs = []interface{}{"A1", "B2", "C3"}
 	checkErrT(t, cl.Call("Echo123", mArgs, &rstr))
@@ -1061,7 +1189,7 @@ func doTestMsgpackRpcSpecPythonClientToGoSvc(t *testing.T) {
 		t.FailNow()
 	}
 	checkEqualT(t, string(cmdout),
-		fmt.Sprintf("%#v\n%#v\n", []string{"A1", "B2", "C3"}, TestABC{"Aa", "Bb", "Cc"}), "cmdout=")
+		fmt.Sprintf("%#v\n%#v\n", []string{"A1", "B2", "C3"}, TestRpcABC{"Aa", "Bb", "Cc"}), "cmdout=")
 }
 
 func TestBincCodecsTable(t *testing.T) {
@@ -1076,6 +1204,10 @@ func TestBincCodecsEmbeddedPointer(t *testing.T) {
 	testCodecEmbeddedPointer(t, testBincH)
 }
 
+func TestBincStdEncIntf(t *testing.T) {
+	doTestStdEncIntf(t, "binc", testBincH)
+}
+
 func TestSimpleCodecsTable(t *testing.T) {
 	testCodecTableOne(t, testSimpleH)
 }
@@ -1088,6 +1220,10 @@ func TestSimpleCodecsEmbeddedPointer(t *testing.T) {
 	testCodecEmbeddedPointer(t, testSimpleH)
 }
 
+func TestSimpleStdEncIntf(t *testing.T) {
+	doTestStdEncIntf(t, "simple", testSimpleH)
+}
+
 func TestMsgpackCodecsTable(t *testing.T) {
 	testCodecTableOne(t, testMsgpackH)
 }
@@ -1100,6 +1236,10 @@ func TestMsgpackCodecsEmbeddedPointer(t *testing.T) {
 	testCodecEmbeddedPointer(t, testMsgpackH)
 }
 
+func TestMsgpackStdEncIntf(t *testing.T) {
+	doTestStdEncIntf(t, "msgpack", testMsgpackH)
+}
+
 func TestCborCodecsTable(t *testing.T) {
 	testCodecTableOne(t, testCborH)
 }
@@ -1116,6 +1256,14 @@ func TestCborMapEncodeForCanonical(t *testing.T) {
 	doTestMapEncodeForCanonical(t, "cbor", testCborH)
 }
 
+func TestCborCodecChan(t *testing.T) {
+	testCodecChan(t, testCborH)
+}
+
+func TestCborStdEncIntf(t *testing.T) {
+	doTestStdEncIntf(t, "cbor", testCborH)
+}
+
 func TestJsonCodecsTable(t *testing.T) {
 	testCodecTableOne(t, testJsonH)
 }
@@ -1132,8 +1280,18 @@ func TestJsonCodecChan(t *testing.T) {
 	testCodecChan(t, testJsonH)
 }
 
-func TestCborCodecChan(t *testing.T) {
-	testCodecChan(t, testCborH)
+func TestJsonStdEncIntf(t *testing.T) {
+	doTestStdEncIntf(t, "json", testJsonH)
+}
+
+// ----- ALL (framework based) -----
+
+func TestAllEncCircularRef(t *testing.T) {
+	doTestEncCircularRef(t, "cbor", testCborH)
+}
+
+func TestAllAnonCycle(t *testing.T) {
+	doTestAnonCycle(t, "cbor", testCborH)
 }
 
 // ----- RPC -----
@@ -1180,6 +1338,7 @@ func TestBincUnderlyingType(t *testing.T) {
 //   - struct tags:
 //     on anonymous fields, _struct (all fields), etc
 //   - codecgen of struct containing channels.
+//   - bad input with large array length prefix
 //
 //   Cleanup tests:
 //   - The are brittle in their handling of validation and skipping

codec/decode.go

@@ -1862,6 +1862,7 @@ func (d *Decoder) intern(s string) {
 	}
 }
 
+// nextValueBytes returns the next value in the stream as a set of bytes.
 func (d *Decoder) nextValueBytes() []byte {
 	d.d.uncacheRead()
 	d.r.track()

codec/encode.go

@@ -110,6 +110,15 @@ type EncodeOptions struct {
 	//
 	Canonical bool
 
+	// CheckCircularRef controls whether we check for circular references
+	// and error fast during an encode.
+	//
+	// If enabled, an error is received if a pointer to a struct
+	// references itself either directly or through one of its fields (iteratively).
+	//
+	// This is opt-in, as there may be a performance hit to checking circular references.
+	CheckCircularRef bool
+
 	// AsSymbols defines what should be encoded as symbols.
 	//
 	// Encoding as symbols can reduce the encoded size significantly.
@@ -503,7 +512,7 @@ func (f *encFnInfo) kStruct(rv reflect.Value) {
 	newlen := len(fti.sfi)
 
 	// Use sync.Pool to reduce allocating slices unnecessarily.
-	// The cost of the occasional locking is less than the cost of new allocation.
+	// The cost of sync.Pool is less than the cost of new allocation.
 	pool, poolv, fkvs := encStructPoolGet(newlen)
 
 	// if toMap, use the sorted array. If toArray, use unsorted array (to match sequence in struct)
@@ -514,11 +523,6 @@ func (f *encFnInfo) kStruct(rv reflect.Value) {
 	var kv stringRv
 	for _, si := range tisfi {
 		kv.r = si.field(rv, false)
-		// if si.i != -1 {
-		// 	rvals[newlen] = rv.Field(int(si.i))
-		// } else {
-		// 	rvals[newlen] = rv.FieldByIndex(si.is)
-		// }
 		if toMap {
 			if si.omitEmpty && isEmptyValue(kv.r) {
 				continue
@@ -596,13 +600,15 @@ func (f *encFnInfo) kStruct(rv reflect.Value) {
 // 	f.e.encodeValue(rv.Elem())
 // }
 
-func (f *encFnInfo) kInterface(rv reflect.Value) {
-	if rv.IsNil() {
-		f.e.e.EncodeNil()
-		return
-	}
-	f.e.encodeValue(rv.Elem(), nil)
-}
+// func (f *encFnInfo) kInterface(rv reflect.Value) {
+// 	println("kInterface called")
+// 	debug.PrintStack()
+// 	if rv.IsNil() {
+// 		f.e.e.EncodeNil()
+// 		return
+// 	}
+// 	f.e.encodeValue(rv.Elem(), nil)
+// }
 
 func (f *encFnInfo) kMap(rv reflect.Value) {
 	ee := f.e.e
@@ -877,6 +883,7 @@ type Encoder struct {
 	// as the handler MAY need to do some coordination.
 	w  encWriter
 	s  []encRtidFn
+	ci set
 	be bool // is binary encoding
 	js bool // is json handle
 
@@ -1133,20 +1140,23 @@ func (e *Encoder) encode(iv interface{}) {
 	}
 }
 
-func (e *Encoder) encodeI(iv interface{}, checkFastpath, checkCodecSelfer bool) {
-	if rv, proceed := e.preEncodeValue(reflect.ValueOf(iv)); proceed {
-		rt := rv.Type()
-		rtid := reflect.ValueOf(rt).Pointer()
-		fn := e.getEncFn(rtid, rt, checkFastpath, checkCodecSelfer)
-		fn.f(&fn.i, rv)
-	}
-}
-
-func (e *Encoder) preEncodeValue(rv reflect.Value) (rv2 reflect.Value, proceed bool) {
+func (e *Encoder) preEncodeValue(rv reflect.Value) (rv2 reflect.Value, sptr uintptr, proceed bool) {
 	// use a goto statement instead of a recursive function for ptr/interface.
 TOP:
 	switch rv.Kind() {
-	case reflect.Ptr, reflect.Interface:
+	case reflect.Ptr:
+		if rv.IsNil() {
+			e.e.EncodeNil()
+			return
+		}
+		rv = rv.Elem()
+		if e.h.CheckCircularRef && rv.Kind() == reflect.Struct {
+			// TODO: Movable pointers will be an issue here. Future problem.
+			sptr = rv.UnsafeAddr()
+			break TOP
+		}
+		goto TOP
+	case reflect.Interface:
 		if rv.IsNil() {
 			e.e.EncodeNil()
 			return
@@ -1163,18 +1173,39 @@ TOP:
 		return
 	}
 
-	return rv, true
+	proceed = true
+	rv2 = rv
+	return
+}
+
+func (e *Encoder) doEncodeValue(rv reflect.Value, fn *encFn, sptr uintptr,
+	checkFastpath, checkCodecSelfer bool) {
+	if sptr != 0 {
+		if (&e.ci).add(sptr) {
+			e.errorf("circular reference found: # %d", sptr)
+		}
+	}
+	if fn == nil {
+		rt := rv.Type()
+		rtid := reflect.ValueOf(rt).Pointer()
+		fn = e.getEncFn(rtid, rt, true, true)
+	}
+	fn.f(&fn.i, rv)
+	if sptr != 0 {
+		(&e.ci).remove(sptr)
+	}
+}
+
+func (e *Encoder) encodeI(iv interface{}, checkFastpath, checkCodecSelfer bool) {
+	if rv, sptr, proceed := e.preEncodeValue(reflect.ValueOf(iv)); proceed {
+		e.doEncodeValue(rv, nil, sptr, checkFastpath, checkCodecSelfer)
+	}
 }
 
 func (e *Encoder) encodeValue(rv reflect.Value, fn *encFn) {
 	// if a valid fn is passed, it MUST BE for the dereferenced type of rv
-	if rv, proceed := e.preEncodeValue(rv); proceed {
-		if fn == nil {
-			rt := rv.Type()
-			rtid := reflect.ValueOf(rt).Pointer()
-			fn = e.getEncFn(rtid, rt, true, true)
-		}
-		fn.f(&fn.i, rv)
+	if rv, sptr, proceed := e.preEncodeValue(rv); proceed {
+		e.doEncodeValue(rv, fn, sptr, true, true)
 	}
 }
 
@@ -1284,10 +1315,11 @@ func (e *Encoder) getEncFn(rtid uintptr, rt reflect.Type, checkFastpath, checkCo
 				fn.f = (*encFnInfo).kSlice
 			case reflect.Struct:
 				fn.f = (*encFnInfo).kStruct
+				// reflect.Ptr and reflect.Interface are handled already by preEncodeValue
 				// case reflect.Ptr:
 				// 	fn.f = (*encFnInfo).kPtr
-			case reflect.Interface:
-				fn.f = (*encFnInfo).kInterface
+				// case reflect.Interface:
+				// 	fn.f = (*encFnInfo).kInterface
 			case reflect.Map:
 				fn.f = (*encFnInfo).kMap
 			default:
@@ -1353,25 +1385,6 @@ func encStructPoolGet(newlen int) (p *sync.Pool, v interface{}, s []stringRv) {
 	// 	panic(errors.New("encStructPoolLen must be equal to 4")) // defensive, in case it is changed
 	// }
 	// idxpool := newlen / 8
-
-	// if pool == nil {
-	// 	fkvs = make([]stringRv, newlen)
-	// } else {
-	// 	poolv = pool.Get()
-	// 	switch vv := poolv.(type) {
-	// 	case *[8]stringRv:
-	// 		fkvs = vv[:newlen]
-	// 	case *[16]stringRv:
-	// 		fkvs = vv[:newlen]
-	// 	case *[32]stringRv:
-	// 		fkvs = vv[:newlen]
-	// 	case *[64]stringRv:
-	// 		fkvs = vv[:newlen]
-	// 	case *[128]stringRv:
-	// 		fkvs = vv[:newlen]
-	// 	}
-	// }
-
 	if newlen <= 8 {
 		p = &encStructPool[0]
 		v = p.Get()

codec/helper.go

@@ -155,8 +155,10 @@ const (
 	resetSliceElemToZeroValue bool = false
 )
 
-var oneByteArr = [1]byte{0}
-var zeroByteSlice = oneByteArr[:0:0]
+var (
+	oneByteArr    = [1]byte{0}
+	zeroByteSlice = oneByteArr[:0:0]
+)
 
 type charEncoding uint8
 
@@ -215,6 +217,24 @@ const (
 	containerArrayEnd
 )
 
+type rgetPoolT struct {
+	encNames [8]string
+	fNames   [8]string
+	etypes   [8]uintptr
+	sfis     [8]*structFieldInfo
+}
+
+var rgetPool = sync.Pool{
+	New: func() interface{} { return new(rgetPoolT) },
+}
+
+type rgetT struct {
+	fNames   []string
+	encNames []string
+	etypes   []uintptr
+	sfis     []*structFieldInfo
+}
+
 type containerStateRecv interface {
 	sendContainerState(containerState)
 }
@@ -833,14 +853,17 @@ func (x *TypeInfos) get(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
 			siInfo = parseStructFieldInfo(structInfoFieldName, x.structTag(f.Tag))
 			ti.toArray = siInfo.toArray
 		}
-		sfip := make([]*structFieldInfo, 0, rt.NumField())
-		x.rget(rt, nil, make(map[string]bool, 16), &sfip, siInfo)
-
-		ti.sfip = make([]*structFieldInfo, len(sfip))
-		ti.sfi = make([]*structFieldInfo, len(sfip))
-		copy(ti.sfip, sfip)
-		sort.Sort(sfiSortedByEncName(sfip))
-		copy(ti.sfi, sfip)
+		pi := rgetPool.Get()
+		pv := pi.(*rgetPoolT)
+		pv.etypes[0] = ti.baseId
+		vv := rgetT{pv.fNames[:0], pv.encNames[:0], pv.etypes[:1], pv.sfis[:0]}
+		x.rget(rt, rtid, nil, &vv, siInfo)
+		ti.sfip = make([]*structFieldInfo, len(vv.sfis))
+		ti.sfi = make([]*structFieldInfo, len(vv.sfis))
+		copy(ti.sfip, vv.sfis)
+		sort.Sort(sfiSortedByEncName(vv.sfis))
+		copy(ti.sfi, vv.sfis)
+		rgetPool.Put(pi)
 	}
 	// sfi = sfip
 
@@ -853,16 +876,37 @@ func (x *TypeInfos) get(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
 	return
 }
 
-func (x *TypeInfos) rget(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool,
-	sfi *[]*structFieldInfo, siInfo *structFieldInfo,
+func (x *TypeInfos) rget(rt reflect.Type, rtid uintptr,
+	indexstack []int, pv *rgetT, siInfo *structFieldInfo,
 ) {
-	for j := 0; j < rt.NumField(); j++ {
+	// This will read up the fields and store how to access the value.
+	// It uses the go language's rules for embedding, as below:
+	//   - if a field has been seen while traversing, skip it
+	//   - if an encName has been seen while traversing, skip it
+	//   - if an embedded type has been seen, skip it
+	//
+	// Also, per Go's rules, embedded fields must be analyzed AFTER all top-level fields.
+	//
+	// Note: we consciously use slices, not a map, to simulate a set.
+	//       Typically, types have < 16 fields, and iteration using equals is faster than maps there
+
+	type anonField struct {
+		ft  reflect.Type
+		idx int
+	}
+
+	var anonFields []anonField
+
+LOOP:
+	for j, jlen := 0, rt.NumField(); j < jlen; j++ {
 		f := rt.Field(j)
 		fkind := f.Type.Kind()
 		// skip if a func type, or is unexported, or structTag value == "-"
-		if fkind == reflect.Func {
-			continue
+		switch fkind {
+		case reflect.Func, reflect.Complex64, reflect.Complex128, reflect.UnsafePointer:
+			continue LOOP
 		}
+
 		// if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
 		if f.PkgPath != "" && !f.Anonymous { // unexported, not embedded
 			continue
@@ -886,11 +930,8 @@ func (x *TypeInfos) rget(rt reflect.Type, indexstack []int, fnameToHastag map[st
 					ft = ft.Elem()
 				}
 				if ft.Kind() == reflect.Struct {
-					indexstack2 := make([]int, len(indexstack)+1, len(indexstack)+4)
-					copy(indexstack2, indexstack)
-					indexstack2[len(indexstack)] = j
-					// indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
-					x.rget(ft, indexstack2, fnameToHastag, sfi, siInfo)
+					// handle anonymous fields after handling all the non-anon fields
+					anonFields = append(anonFields, anonField{ft, j})
 					continue
 				}
 			}
@@ -901,26 +942,39 @@ func (x *TypeInfos) rget(rt reflect.Type, indexstack []int, fnameToHastag map[st
 			continue
 		}
 
-		// do not let fields with same name in embedded structs override field at higher level.
-		// this must be done after anonymous check, to allow anonymous field
-		// still include their child fields
-		if _, ok := fnameToHastag[f.Name]; ok {
-			continue
-		}
 		if f.Name == "" {
 			panic(noFieldNameToStructFieldInfoErr)
 		}
+
+		for _, k := range pv.fNames {
+			if k == f.Name {
+				continue LOOP
+			}
+		}
+		pv.fNames = append(pv.fNames, f.Name)
+
 		if si == nil {
 			si = parseStructFieldInfo(f.Name, stag)
 		} else if si.encName == "" {
 			si.encName = f.Name
 		}
+
+		for _, k := range pv.encNames {
+			if k == si.encName {
+				continue LOOP
+			}
+		}
+		pv.encNames = append(pv.encNames, si.encName)
+
 		// si.ikind = int(f.Type.Kind())
 		if len(indexstack) == 0 {
 			si.i = int16(j)
 		} else {
 			si.i = -1
-			si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
+			si.is = make([]int, len(indexstack)+1)
+			copy(si.is, indexstack)
+			si.is[len(indexstack)] = j
+			// si.is = append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
 		}
 
 		if siInfo != nil {
@@ -928,8 +982,26 @@ func (x *TypeInfos) rget(rt reflect.Type, indexstack []int, fnameToHastag map[st
 				si.omitEmpty = true
 			}
 		}
-		*sfi = append(*sfi, si)
-		fnameToHastag[f.Name] = stag != ""
+		pv.sfis = append(pv.sfis, si)
+	}
+
+	// now handle anonymous fields
+LOOP2:
+	for _, af := range anonFields {
+		// if etypes contains this, then do not call rget again (as the fields are already seen here)
+		ftid := reflect.ValueOf(af.ft).Pointer()
+		for _, k := range pv.etypes {
+			if k == ftid {
+				continue LOOP2
+			}
+		}
+		pv.etypes = append(pv.etypes, ftid)
+
+		indexstack2 := make([]int, len(indexstack)+1)
+		copy(indexstack2, indexstack)
+		indexstack2[len(indexstack)] = af.idx
+		// indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
+		x.rget(af.ft, ftid, indexstack2, pv, siInfo)
 	}
 }
 
@@ -1127,3 +1199,73 @@ type bytesISlice []bytesI
 func (p bytesISlice) Len() int           { return len(p) }
 func (p bytesISlice) Less(i, j int) bool { return bytes.Compare(p[i].v, p[j].v) == -1 }
 func (p bytesISlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+// -----------------
+
+type set []uintptr
+
+func (s *set) add(v uintptr) (exists bool) {
+	// e.ci is always nil, or len >= 1
+	// defer func() { fmt.Printf("$$$$$$$$$$$ cirRef Add: %v, exists: %v\n", v, exists) }()
+	x := *s
+	if x == nil {
+		x = make([]uintptr, 1, 8)
+		x[0] = v
+		*s = x
+		return
+	}
+	// typically, length will be 1. make this perform.
+	if len(x) == 1 {
+		if j := x[0]; j == 0 {
+			x[0] = v
+		} else if j == v {
+			exists = true
+		} else {
+			x = append(x, v)
+			*s = x
+		}
+		return
+	}
+	// check if it exists
+	for _, j := range x {
+		if j == v {
+			exists = true
+			return
+		}
+	}
+	// try to replace a "deleted" slot
+	for i, j := range x {
+		if j == 0 {
+			x[i] = v
+			return
+		}
+	}
+	// if unable to replace deleted slot, just append it.
+	x = append(x, v)
+	*s = x
+	return
+}
+
+func (s *set) remove(v uintptr) (exists bool) {
+	// defer func() { fmt.Printf("$$$$$$$$$$$ cirRef Rm: %v, exists: %v\n", v, exists) }()
+	x := *s
+	if len(x) == 0 {
+		return
+	}
+	if len(x) == 1 {
+		if x[0] == v {
+			x[0] = 0
+		}
+		return
+	}
+	for i, j := range x {
+		if j == v {
+			exists = true
+			x[i] = 0 // set it to 0, as way to delete it.
+			// copy(x[i:], x[i+1:])
+			// x = x[:len(x)-1]
+			return
+		}
+	}
+	return
+}

codec/json.go

@@ -43,18 +43,23 @@ import (
 
 //--------------------------------
 
-var jsonLiterals = [...]byte{'t', 'r', 'u', 'e', 'f', 'a', 'l', 's', 'e', 'n', 'u', 'l', 'l'}
+var (
+	jsonLiterals = [...]byte{'t', 'r', 'u', 'e', 'f', 'a', 'l', 's', 'e', 'n', 'u', 'l', 'l'}
 
-var jsonFloat64Pow10 = [...]float64{
-	1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
-	1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
-	1e20, 1e21, 1e22,
-}
+	jsonFloat64Pow10 = [...]float64{
+		1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
+		1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
+		1e20, 1e21, 1e22,
+	}
 
-var jsonUint64Pow10 = [...]uint64{
-	1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
-	1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
-}
+	jsonUint64Pow10 = [...]uint64{
+		1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
+		1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
+	}
+
+	// jsonTabs and jsonSpaces are used as caches for indents
+	jsonTabs, jsonSpaces string
+)
 
 const (
 	// jsonUnreadAfterDecNum controls whether we unread after decoding a number.
@@ -85,8 +90,23 @@ const (
 	jsonNumUintMaxVal = 1<<uint64(64) - 1
 
 	// jsonNumDigitsUint64Largest = 19
+
+	jsonSpacesOrTabsLen = 128
 )
 
+func init() {
+	var bs [jsonSpacesOrTabsLen]byte
+	for i := 0; i < jsonSpacesOrTabsLen; i++ {
+		bs[i] = ' '
+	}
+	jsonSpaces = string(bs[:])
+
+	for i := 0; i < jsonSpacesOrTabsLen; i++ {
+		bs[i] = '\t'
+	}
+	jsonTabs = string(bs[:])
+}
+
 type jsonEncDriver struct {
 	e  *Encoder
 	w  encWriter
@@ -94,30 +114,76 @@ type jsonEncDriver struct {
 	b  [64]byte // scratch
 	bs []byte   // scratch
 	se setExtWrapper
+	ds string // indent string
+	dl uint16 // indent level
+	dt bool   // indent using tabs
+	d  bool   // indent
 	c  containerState
 	noBuiltInTypes
 }
 
+// indent is done as below:
+//   - newline and indent are added before each mapKey or arrayElem
+//   - newline and indent are added before each ending,
+//     except there was no entry (so we can have {} or [])
+
 func (e *jsonEncDriver) sendContainerState(c containerState) {
 	// determine whether to output separators
 	if c == containerMapKey {
 		if e.c != containerMapStart {
 			e.w.writen1(',')
 		}
+		if e.d {
+			e.writeIndent()
+		}
 	} else if c == containerMapValue {
-		e.w.writen1(':')
+		if e.d {
+			e.w.writen2(':', ' ')
+		} else {
+			e.w.writen1(':')
+		}
 	} else if c == containerMapEnd {
+		if e.d {
+			e.dl--
+			if e.c != containerMapStart {
+				e.writeIndent()
+			}
+		}
 		e.w.writen1('}')
 	} else if c == containerArrayElem {
 		if e.c != containerArrayStart {
 			e.w.writen1(',')
 		}
+		if e.d {
+			e.writeIndent()
+		}
 	} else if c == containerArrayEnd {
+		if e.d {
+			e.dl--
+			if e.c != containerArrayStart {
+				e.writeIndent()
+			}
+		}
 		e.w.writen1(']')
 	}
 	e.c = c
 }
 
+func (e *jsonEncDriver) writeIndent() {
+	e.w.writen1('\n')
+	if x := len(e.ds) * int(e.dl); x <= jsonSpacesOrTabsLen {
+		if e.dt {
+			e.w.writestr(jsonTabs[:x])
+		} else {
+			e.w.writestr(jsonSpaces[:x])
+		}
+	} else {
+		for i := uint16(0); i < e.dl; i++ {
+			e.w.writestr(e.ds)
+		}
+	}
+}
+
 func (e *jsonEncDriver) EncodeNil() {
 	e.w.writeb(jsonLiterals[9:13]) // null
 }
@@ -165,11 +231,17 @@ func (e *jsonEncDriver) EncodeRawExt(re *RawExt, en *Encoder) {
 }
 
 func (e *jsonEncDriver) EncodeArrayStart(length int) {
+	if e.d {
+		e.dl++
+	}
 	e.w.writen1('[')
 	e.c = containerArrayStart
 }
 
 func (e *jsonEncDriver) EncodeMapStart(length int) {
+	if e.d {
+		e.dl++
+	}
 	e.w.writen1('{')
 	e.c = containerMapStart
 }
@@ -1033,6 +1105,11 @@ type JsonHandle struct {
 	// RawBytesExt, if configured, is used to encode and decode raw bytes in a custom way.
 	// If not configured, raw bytes are encoded to/from base64 text.
 	RawBytesExt InterfaceExt
+
+	// Indent indicates how a value is encoded.
+	//   - If positive, indent by that number of spaces.
+	//   - If negative, indent by that number of tabs.
+	Indent int8
 }
 
 func (h *JsonHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) {
@@ -1040,26 +1117,48 @@ func (h *JsonHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceE
 }
 
 func (h *JsonHandle) newEncDriver(e *Encoder) encDriver {
-	hd := jsonEncDriver{e: e, w: e.w, h: h}
+	hd := jsonEncDriver{e: e, h: h}
 	hd.bs = hd.b[:0]
-	hd.se.i = h.RawBytesExt
+
+	hd.reset()
+
 	return &hd
 }
 
 func (h *JsonHandle) newDecDriver(d *Decoder) decDriver {
 	// d := jsonDecDriver{r: r.(*bytesDecReader), h: h}
-	hd := jsonDecDriver{d: d, r: d.r, h: h}
+	hd := jsonDecDriver{d: d, h: h}
 	hd.bs = hd.b[:0]
-	hd.se.i = h.RawBytesExt
+	hd.reset()
 	return &hd
 }
 
 func (e *jsonEncDriver) reset() {
 	e.w = e.e.w
+	e.se.i = e.h.RawBytesExt
+	if e.bs != nil {
+		e.bs = e.bs[:0]
+	}
+	e.d, e.dt, e.dl, e.ds = false, false, 0, ""
+	e.c = 0
+	if e.h.Indent > 0 {
+		e.d = true
+		e.ds = jsonSpaces[:e.h.Indent]
+	} else if e.h.Indent < 0 {
+		e.d = true
+		e.dt = true
+		e.ds = jsonTabs[:-(e.h.Indent)]
+	}
 }
 
 func (d *jsonDecDriver) reset() {
 	d.r = d.d.r
+	d.se.i = d.h.RawBytesExt
+	if d.bs != nil {
+		d.bs = d.bs[:0]
+	}
+	d.c, d.tok = 0, 0
+	d.n.reset()
 }
 
 var jsonEncodeTerminate = []byte{' '}

codec/msgpack.go

@@ -729,6 +729,7 @@ func (e *msgpackEncDriver) reset() {
 
 func (d *msgpackDecDriver) reset() {
 	d.r = d.d.r
+	d.bd, d.bdRead = 0, false
 }
 
 //--------------------------------------------------

codec/simple.go

@@ -512,6 +512,7 @@ func (e *simpleEncDriver) reset() {
 
 func (d *simpleDecDriver) reset() {
 	d.r = d.d.r
+	d.bd, d.bdRead = 0, false
 }
 
 var _ decDriver = (*simpleDecDriver)(nil)

codec/tests.sh

@@ -6,6 +6,7 @@
 _run() {
     # 1. VARIATIONS: regular (t), canonical (c), IO R/W (i),
     #                binc-nosymbols (n), struct2array (s), intern string (e),
+    #                json-indent (d), circular (l)
     # 2. MODE: reflection (r), external (x), codecgen (g), unsafe (u), notfastpath (f)
     # 3. OPTIONS: verbose (v), reset (z), must (m),
     # 
@@ -16,7 +17,7 @@ _run() {
     zargs=""
     local OPTIND 
     OPTIND=1
-    while getopts "xurtcinsvgzmef" flag
+    while getopts "_xurtcinsvgzmefdl" flag
     do
         case "x$flag" in 
             'xr')  ;;
@@ -27,6 +28,7 @@ _run() {
             'xv') zargs="$zargs -tv" ;;
             'xz') zargs="$zargs -tr" ;;
             'xm') zargs="$zargs -tm" ;;
+            'xl') zargs="$zargs -tl" ;;
             *) ;;
         esac
     done
@@ -35,15 +37,19 @@ _run() {
     # echo ">>>>>>> TAGS: $ztags"
     
     OPTIND=1
-    while getopts "xurtcinsvgzmef" flag
+    while getopts "_xurtcinsvgzmefdl" flag
     do
         case "x$flag" in 
             'xt') printf ">>>>>>> REGULAR    : "; go test "-tags=$ztags" $zargs ; sleep 2 ;;
             'xc') printf ">>>>>>> CANONICAL  : "; go test "-tags=$ztags" $zargs -tc; sleep 2 ;;
             'xi') printf ">>>>>>> I/O        : "; go test "-tags=$ztags" $zargs -ti; sleep 2 ;;
-            'xn') printf ">>>>>>> NO_SYMBOLS : "; go test "-tags=$ztags" $zargs -tn; sleep 2 ;;
+            'xn') printf ">>>>>>> NO_SYMBOLS : "; go test "-tags=$ztags" -run=Binc $zargs -tn; sleep 2 ;;
             'xs') printf ">>>>>>> TO_ARRAY   : "; go test "-tags=$ztags" $zargs -ts; sleep 2 ;;
             'xe') printf ">>>>>>> INTERN     : "; go test "-tags=$ztags" $zargs -te; sleep 2 ;;
+            'xd') printf ">>>>>>> INDENT     : ";
+                  go test "-tags=$ztags" -run=JsonCodecsTable -td=-1 $zargs;
+                  go test "-tags=$ztags" -run=JsonCodecsTable -td=8 $zargs;
+                  sleep 2 ;;
             *) ;;
         esac
     done
@@ -55,20 +61,20 @@ _run() {
 # echo ">>>>>>> RUNNING VARIATIONS OF TESTS"    
 if [[ "x$@" = "x" ]]; then
     # All: r, x, g, gu
-    _run "-rtcinsm"  # regular
-    _run "-rtcinsmz" # regular with reset
-    _run "-rtcinsmf" # regular with no fastpath (notfastpath)
-    _run "-xtcinsm" # external
-    _run "-gxtcinsm" # codecgen: requires external
-    _run "-gxutcinsm" # codecgen + unsafe
+    _run "-_tcinsed_ml"  # regular
+    _run "-_tcinsed_ml_z" # regular with reset
+    _run "-_tcinsed_ml_f" # regular with no fastpath (notfastpath)
+    _run "-x_tcinsed_ml" # external
+    _run "-gx_tcinsed_ml" # codecgen: requires external
+    _run "-gxu_tcinsed_ml" # codecgen + unsafe
 elif [[ "x$@" = "x-Z" ]]; then
     # Regular
-    _run "-rtcinsm"  # regular
-    _run "-rtcinsmz" # regular with reset
+    _run "-_tcinsed_ml"  # regular
+    _run "-_tcinsed_ml_z" # regular with reset
 elif [[ "x$@" = "x-F" ]]; then
     # regular with notfastpath
-    _run "-rtcinsmf"  # regular
-    _run "-rtcinsmzf" # regular with reset
+    _run "-_tcinsed_ml_f"  # regular
+    _run "-_tcinsed_ml_zf" # regular with reset
 else
     _run "$@"
 fi

codec/time.go

@@ -5,11 +5,22 @@ package codec
 
 import (
 	"fmt"
+	"reflect"
 	"time"
 )
 
 var (
-	timeDigits = [...]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
+	timeDigits   = [...]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
+	timeExtEncFn = func(rv reflect.Value) (bs []byte, err error) {
+		defer panicToErr(&err)
+		bs = timeExt{}.WriteExt(rv.Interface())
+		return
+	}
+	timeExtDecFn = func(rv reflect.Value, bs []byte) (err error) {
+		defer panicToErr(&err)
+		timeExt{}.ReadExt(rv.Interface(), bs)
+		return
+	}
 )
 
 type timeExt struct{}