codec: clean up recognizedRtid support, add more tests and update testing TODO

test:
- implement test for binc time and msgpack writeext=true|false
- add test for time.Time and update test TODOs
- expand integer values for testing
- add field of type map[bool]something to test canonical for maps with bool keys
- updated TODO

codec/codec_test.go

@@ -1091,6 +1091,10 @@ func doTestMapEncodeForCanonical(t *testing.T, name string, h Handle) {
 				"4444":  4444,
 				"55555": 55555,
 			},
+			"c/g": map[bool]int{
+				false: 0,
+				true:  1,
+			},
 		},
 	}
 	var v2 map[stringUint64T]interface{}
@@ -1807,6 +1811,22 @@ func testMammoth(t *testing.T, name string, h Handle) {
 	testDeepEqualErr(&m, &m2, t, "mammoth-"+name)
 }
 
+func testTime(t *testing.T, name string, h Handle) {
+	// test time which uses the time.go implementation (ie Binc)
+	var tt, tt2 time.Time
+	// time in 1990
+	tt = time.Unix(20*366*24*60*60, 1000*900).In(time.FixedZone("UGO", -5*60*60))
+	// fmt.Printf("time tt: %v\n", tt)
+	b, _ := testMarshalErr(tt, h, t, "time-"+name)
+	testUnmarshalErr(&tt2, b, h, t, "time-"+name)
+	// per go documentation, test time with .Equal not ==
+	if !tt2.Equal(tt) {
+		logT(t, "%s: values not equal: 1: %v, 2: %v", name, tt2, tt)
+		failT(t)
+	}
+	// testDeepEqualErr(tt.UTC(), tt2, t, "time-"+name)
+}
+
 // -----------------
 
 func TestJsonDecodeNonStringScalarInStringContext(t *testing.T) {
@@ -2354,43 +2374,64 @@ func TestSimpleMammothMapsAndSlices(t *testing.T) {
 	doTestMammothMapsAndSlices(t, testSimpleH)
 }
 
+func TestJsonTime(t *testing.T) {
+	testTime(t, "json", testJsonH)
+}
+
+func TestCborTime(t *testing.T) {
+	testTime(t, "cbor", testCborH)
+}
+
+func TestMsgpackTime(t *testing.T) {
+	testTime(t, "msgpack", testMsgpackH)
+}
+
+func TestBincTime(t *testing.T) {
+	testTime(t, "binc", testBincH)
+}
+
+func TestSimpleTime(t *testing.T) {
+	testTime(t, "simple", testSimpleH)
+}
+
 // TODO:
+//
 //   Add Tests for:
-//   - decoding empty list/map in stream into a nil slice/map
-//   - binary(M|Unm)arsher support for time.Time (e.g. cbor encoding)
-//   - text(M|Unm)arshaler support for time.Time (e.g. json encoding)
-//   - non fast-path scenarios e.g. map[string]uint16, []customStruct.
-//     Expand cbor to include indefinite length stuff for this non-fast-path types.
-//     This may not be necessary, since we have the manual tests (fastpathEnabled=false) to test/validate with.
-//   - CodecSelfer
-//     Ensure it is called when (en|de)coding interface{} or reflect.Value (2 different codepaths).
-//   - interfaces: textMarshaler, binaryMarshaler, codecSelfer
 //   - struct tags:
 //     on anonymous fields, _struct (all fields), etc
 //   - codecgen of struct containing channels.
+//   - (encode extensions: ext, raw ext, etc)
+//   - extension that isn't builtin e.g. type uint64Ext uint64.
+//     it encodes as a uint64.
 //
 //  Add negative tests for failure conditions:
 //   - bad input with large array length prefix
 //
-//  More tests:
-//  - large integers x: int64 and uint64
-//    x = 0, -1, 1, maxUint8, maxUint16, maxuint32
-//    maxUint8 < x < maxUint16
-//    maxUint16 < x < maxuint32
-//    x > maxuint32
-//  - standard floats x: float32 and float64
-//    0, -1, 1, +inf, -inf, etc
-//  - length of containers (array, string, bytes, map):
-//    container == nil
-//    len = 0
-//  - length of containers (array, string, bytes, map):
-//    len = maxUint8, maxUint16, maxuint32
-//    maxUint8 < len < maxUint16
-//    maxUint16 < len < maxuint32
-//    len > maxuint32
-//  - (encode extensions: ext, raw ext, etc)
-//  - tracking:
+// msgpack
+// - add reading uint from encoded int
+// - support time as built-in extension:
+//   see https://github.com/msgpack/msgpack/blob/master/spec.md#timestamp-extension-type
+//
+// decode.go
+// - UnreadByte: only 2 states (z.ls = 2 and z.ls = 1) (0 --> 2 --> 1)
+// - track
 //    z.trb: track, stop track, check
-//  - test with diff: mapType and sliceType, and decodeNaked
-//  - extension that isn't builtin e.g. type uint64Ext uint64.
-//    it encodes as a uint64.
+// - maptype, slicetype: diff from map[string]intf, map[intf]intf or []intf,
+// - PreferArrayOverSlice??? (standalone test)
+// - InterfaceReset (standalone test)
+// - (chan byte) to decode []byte (with mapbyslice track)
+// - decode slice of len 6, 16 into slice of (len 4, cap 8) and (len ) with maxinitlen=6, 8, 16
+// - ktypeisintf
+// - DeleteOnNilMapValue
+// - decnaked: n.l == nil
+// - setZero: all types: *bool, *intXXX, *uintXXX, *floatXXX, *Raw, *[]byte, etc
+// - codec.Selfer implementation
+//     Ensure it is called when (en|de)coding interface{} or reflect.Value (2 different codepaths).
+// - ensureDecodeable (try to decode into a non-decodeable thing e.g. a nil interface{},
+//
+// encode.go
+// - mapbyslice (for non-fastpath things)
+// - nil and 0-len slices and maps for non-fastpath things
+//
+// cbor:
+// - halffloat

codec/encode.go

@@ -508,8 +508,11 @@ func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 
 	if l > 0 {
 		var fn *codecFn
-		var recognizedVtyp = useLookupRecognizedTypes && isRecognizedRtidOrPtr(rt2id(rtelem))
-		if !recognizedVtyp {
+		var recognizedVtyp bool
+		if useLookupRecognizedTypes {
+			recognizedVtyp = isRecognizedRtidOrPtr(rt2id(rtelem))
+		}
+		if !(useLookupRecognizedTypes && recognizedVtyp) {
 			for rtelem.Kind() == reflect.Ptr {
 				rtelem = rtelem.Elem()
 			}
@@ -786,7 +789,8 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
 		asSymbols = e.h.AsSymbols&AsSymbolMapStringKeysFlag != 0
 	} else {
 		if useLookupRecognizedTypes {
-			if recognizedKtyp = isRecognizedRtidOrPtr(rtkeyid); recognizedKtyp {
+			recognizedKtyp = isRecognizedRtidOrPtr(rtkeyid)
+			if recognizedKtyp {
 				goto LABEL1
 			}
 		}
@@ -801,7 +805,9 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
 
 	// for j, lmks := 0, len(mks); j < lmks; j++ {
 LABEL1:
-	recognizedVtyp = useLookupRecognizedTypes && isRecognizedRtidOrPtr(rtvalid)
+	if useLookupRecognizedTypes {
+		recognizedVtyp = isRecognizedRtidOrPtr(rtvalid)
+	}
 	for j := range mks {
 		if elemsep {
 			ee.WriteMapElemKey()

codec/fast-path.generated.go

@@ -80,6 +80,9 @@ var fastpathAV fastpathA
 
 // due to possible initialization loop error, make fastpath in an init()
 func init() {
+	if useLookupRecognizedTypes && recognizedRtidsLoaded {
+		panic("recognizedRtidsLoaded = true - cannot happen")
+	}
 	i := 0
 	fn := func(v interface{},
 		fe func(*Encoder, *codecFnInfo, reflect.Value),

codec/fast-path.go.tmpl

@@ -80,6 +80,9 @@ var fastpathAV fastpathA
 
 // due to possible initialization loop error, make fastpath in an init()
 func init() {
+	if useLookupRecognizedTypes && recognizedRtidsLoaded {
+		panic("recognizedRtidsLoaded = true - cannot happen")
+	}
 	i := 0
 	fn := func(v interface{},
 		fe func(*Encoder, *codecFnInfo, reflect.Value),

codec/helper.go

@@ -408,14 +408,20 @@ var immutableKindsSet = [32]bool{
 	// reflect.UnsafePointer
 }
 
-var recognizedRtids []uintptr
-var recognizedRtidPtrs []uintptr
-var recognizedRtidOrPtrs []uintptr
+var (
+	recognizedRtids       []uintptr
+	recognizedRtidPtrs    []uintptr
+	recognizedRtidOrPtrs  []uintptr
+	recognizedRtidsLoaded bool
+)
 
 func init() {
 	if !useLookupRecognizedTypes {
 		return
 	}
+	if recognizedRtidsLoaded {
+		panic("recognizedRtidsLoaded = true - cannot happen")
+	}
 	for _, v := range [...]interface{}{
 		float32(0),
 		float64(0),
@@ -440,6 +446,16 @@ func init() {
 		recognizedRtids = append(recognizedRtids, rt2id(rt))
 		recognizedRtidPtrs = append(recognizedRtidPtrs, rt2id(reflect.PtrTo(rt)))
 	}
+
+	// now sort it.
+	sort.Sort(uintptrSlice(recognizedRtids))
+	sort.Sort(uintptrSlice(recognizedRtidPtrs))
+	recognizedRtidOrPtrs = make([]uintptr, len(recognizedRtids)+len(recognizedRtidPtrs))
+	copy(recognizedRtidOrPtrs, recognizedRtids)
+	copy(recognizedRtidOrPtrs[len(recognizedRtids):], recognizedRtidPtrs)
+	sort.Sort(uintptrSlice(recognizedRtidOrPtrs))
+
+	recognizedRtidsLoaded = true
 }
 
 func containsU(s []uintptr, v uintptr) bool {
@@ -966,7 +982,7 @@ type typeInfo struct {
 
 	mbs bool // base type (T or *T) is a MapBySlice
 
-	// [btj][mu]p? OR csp?
+	// format of marshal type fields below: [btj][mu]p? OR csp?
 
 	bm  bool // T is a binaryMarshaler
 	bmp bool // *T is a binaryMarshaler
@@ -982,26 +998,6 @@ type typeInfo struct {
 	jup bool // *T is a jsonUnmarshaler
 	cs  bool // T is a Selfer
 	csp bool // *T is a Selfer
-	// su  bool // T is a
-	// sup  bool // *T is a
-
-	// bm        bool // base type (T or *T) is a binaryMarshaler
-	// bunm      bool // base type (T or *T) is a binaryUnmarshaler
-	// bmIndir   bool // is *T binaryMarshaler type
-	// bunmIndir bool // number of indirections to get to binaryUnmarshaler type
-
-	// tm        bool // base type (T or *T) is a textMarshaler
-	// tunm      bool // base type (T or *T) is a textUnmarshaler
-	// tmIndir   bool // number of indirections to get to textMarshaler type
-	// tunmIndir bool // number of indirections to get to textUnmarshaler type
-
-	// jm        bool // base type (T or *T) is a jsonMarshaler
-	// junm      bool // base type (T or *T) is a jsonUnmarshaler
-	// jmIndir   bool // number of indirections to get to jsonMarshaler type
-	// junmIndir bool // number of indirections to get to jsonUnmarshaler type
-
-	// cs      bool // base type (T or *T) is a Selfer
-	// csIndir bool // number of indirections to get to Selfer type
 
 	toArray bool // whether this (struct) type should be encoded as an array
 }

codec/values_flex_test.go

@@ -12,6 +12,7 @@ type TestStrucFlex struct {
 	testStrucCommon
 
 	Mis     map[int]string
+	Mbu64   map[bool]struct{}
 	Miwu64s map[int]wrapUint64Slice
 	Mfwss   map[float64]wrapStringSlice
 	Mf32wss map[float32]wrapStringSlice
@@ -51,6 +52,7 @@ func newTestStrucFlex(depth, n int, bench, useInterface, useStringKeyOnly bool)
 			22:  "twenty two",
 			-44: "minus forty four",
 		},
+		Mbu64: map[bool]struct{}{false: struct{}{}, true: struct{}{}},
 	}
 	populateTestStrucCommon(&ts.testStrucCommon, n, bench, useInterface, useStringKeyOnly)
 	if depth > 0 {

codec/values_test.go

@@ -236,12 +236,18 @@ func populateTestStrucCommon(ts *testStrucCommon, n int, bench, useInterface, us
 			strRpt(n, "Afive.Gclef.\U0001d11E\"ugorji\"done.")},
 		AI64slice: []int64{
 			0, 1, -1, -22, 333, -4444, 55555, -666666,
+			// msgpack ones
+			-48, -32, -24,
 			// standard ones
 			0, -1, 1,
 			math.MaxInt8, math.MaxInt8 + 4, math.MaxInt8 - 4,
 			math.MaxInt16, math.MaxInt16 + 4, math.MaxInt16 - 4,
 			math.MaxInt32, math.MaxInt32 + 4, math.MaxInt32 - 4,
 			math.MaxInt64, math.MaxInt64 - 4,
+			math.MinInt8, math.MinInt8 + 4, math.MinInt8 - 4,
+			math.MinInt16, math.MinInt16 + 4, math.MinInt16 - 4,
+			math.MinInt32, math.MinInt32 + 4, math.MinInt32 - 4,
+			math.MinInt64, math.MinInt64 + 4,
 		},
 		AUi64slice: []uint64{
 			0, 1, 22, 333, 4444, 55555, 666666,

codec/z_all_test.go

@@ -201,8 +201,14 @@ func testCodecGroup(t *testing.T) {
 	t.Run("TestMsgpackMammothMapsAndSlices", TestMsgpackMammothMapsAndSlices)
 	t.Run("TestBincMammothMapsAndSlices", TestBincMammothMapsAndSlices)
 	t.Run("TestSimpleMammothMapsAndSlices", TestSimpleMammothMapsAndSlices)
+	t.Run("TestJsonTime", TestJsonTime)
+	t.Run("TestCborTime", TestCborTime)
+	t.Run("TestMsgpackTime", TestMsgpackTime)
+	t.Run("TestBincTime", TestBincTime)
+	t.Run("TestSimpleTime", TestSimpleTime)
 
 	t.Run("TestJsonInvalidUnicode", TestJsonInvalidUnicode)
+	t.Run("TestBincTime", TestBincTime)
 	// <tear-down code>
 }
 
@@ -227,6 +233,7 @@ func testJsonGroup(t *testing.T) {
 	t.Run("TestJsonLargeContainerLen", TestJsonLargeContainerLen)
 	t.Run("TestJsonMammothMapsAndSlices", TestJsonMammothMapsAndSlices)
 	t.Run("TestJsonInvalidUnicode", TestJsonInvalidUnicode)
+	t.Run("TestJsonTime", TestJsonTime)
 }
 
 func testBincGroup(t *testing.T) {
@@ -246,6 +253,7 @@ func testBincGroup(t *testing.T) {
 	t.Run("TestBincEmbeddedFieldPrecedence", TestBincEmbeddedFieldPrecedence)
 	t.Run("TestBincLargeContainerLen", TestBincLargeContainerLen)
 	t.Run("TestBincMammothMapsAndSlices", TestBincMammothMapsAndSlices)
+	t.Run("TestBincTime", TestBincTime)
 }
 
 func testCborGroup(t *testing.T) {
@@ -266,6 +274,26 @@ func testCborGroup(t *testing.T) {
 	t.Run("TestCborEmbeddedFieldPrecedence", TestCborEmbeddedFieldPrecedence)
 	t.Run("TestCborLargeContainerLen", TestCborLargeContainerLen)
 	t.Run("TestCborMammothMapsAndSlices", TestCborMammothMapsAndSlices)
+	t.Run("TestCborTime", TestCborTime)
+}
+
+func testMsgpackGroup(t *testing.T) {
+	t.Run("TestMsgpackCodecsTable", TestMsgpackCodecsTable)
+	t.Run("TestMsgpackCodecsMisc", TestMsgpackCodecsMisc)
+	t.Run("TestMsgpackCodecsEmbeddedPointer", TestMsgpackCodecsEmbeddedPointer)
+	t.Run("TestMsgpackStdEncIntf", TestMsgpackStdEncIntf)
+	t.Run("TestMsgpackMammoth", TestMsgpackMammoth)
+	t.Run("TestMsgpackRaw", TestMsgpackRaw)
+	t.Run("TestMsgpackRpcGo", TestMsgpackRpcGo)
+	t.Run("TestMsgpackRpcSpec", TestMsgpackRpcSpec)
+	t.Run("TestMsgpackSwallowAndZero", TestMsgpackSwallowAndZero)
+	t.Run("TestMsgpackRawExt", TestMsgpackRawExt)
+	t.Run("TestMsgpackMapStructKey", TestMsgpackMapStructKey)
+	t.Run("TestMsgpackDecodeNilMapValue", TestMsgpackDecodeNilMapValue)
+	t.Run("TestMsgpackEmbeddedFieldPrecedence", TestMsgpackEmbeddedFieldPrecedence)
+	t.Run("TestMsgpackLargeContainerLen", TestMsgpackLargeContainerLen)
+	t.Run("TestMsgpackMammothMapsAndSlices", TestMsgpackMammothMapsAndSlices)
+	t.Run("TestMsgpackTime", TestMsgpackTime)
 }
 
 func TestCodecSuite(t *testing.T) {
@@ -315,6 +343,18 @@ func TestCodecSuite(t *testing.T) {
 
 	testBincH.getBasicHandle().AsSymbols = oldSymbols
 
+	oldWriteExt := testMsgpackH.WriteExt
+
+	testMsgpackH.WriteExt = true
+	testReinit()
+	t.Run("msgpack-writeext", testMsgpackGroup)
+
+	testMsgpackH.WriteExt = false
+	testReinit()
+	t.Run("msgpack-no-writeext", testMsgpackGroup)
+
+	testMsgpackH.WriteExt = oldWriteExt
+
 	testGroupResetFlags()
 }