codec: add (unsafe) alloc-free support for map iteration 1.12+

We do this by introducing safe and unsafe 1.12+ variants for
map iteration and map indexing.

This is necessary due to https://github.com/golang/go/issues/32424
- Map Iteration using reflection always causes allocation
  because every key and value might have to be converted to an
  interface{} first, meaning scalars may be allocated on the heap.
- Map Indexing also causes allocation

We workaround both by implementing a copy-free allocation
where a holder value is passed into the indexing or iterating operation,
and we "copy" the value into that pointer.

Also, improve heuristics of decInferLen

Also, fix lint, staticcheck and other issues raised via static analysis.

Clean out xdebugf comments and remove much old commented code

Finally, make more changes to assist the bounds check elimination part of compiler.

codec/binc.go

@@ -221,7 +221,6 @@ func (e *bincEncDriver) EncodeExt(v interface{}, xtag uint64, ext Ext) {
 	if ext == SelfExt {
 		bs = bufp.get(1024)[:0]
 		e.e.sideEncode(v, &bs)
-		// xdebugf("binc EncodeExt: xbs: len: %d, %v", len(bs), bs)
 	} else {
 		bs = ext.WriteExt(v)
 	}
@@ -275,7 +274,6 @@ func (e *bincEncDriver) EncodeSymbol(v string) {
 	}
 	if e.m == nil {
 		e.m = pool.mapStrU16.Get().(map[string]uint16)
-		// xdebug2f("creating e.m: %v, isnil: %v", e.m, e.m == nil)
 	}
 	ui, ok := e.m[v]
 	if ok {
@@ -404,7 +402,7 @@ type bincDecDriver struct {
 	// noStreamingCodec
 	// decNoSeparator
 
-	b [(8 + 1) * 8]byte // scratch
+	b [3 * 8]byte // scratch
 }
 
 func (d *bincDecDriver) readNextBd() {
@@ -699,7 +697,7 @@ func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
 			if d.br {
 				bs2 = d.r.readx(uint(slen))
 			} else if len(bs) == 0 {
-				bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, d.b[:])
+				bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, d.d.b[:])
 			} else {
 				bs2 = decByteSlice(d.r, slen, d.d.h.MaxInitLen, bs)
 			}
@@ -754,7 +752,7 @@ func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
 }
 
 func (d *bincDecDriver) DecodeStringAsBytes() (s []byte) {
-	return d.decStringBytes(d.b[:], true)
+	return d.decStringBytes(d.d.b[:], true)
 }
 
 func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
@@ -790,7 +788,7 @@ func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 		if d.br {
 			return d.r.readx(uint(clen))
 		} else if len(bs) == 0 {
-			bs = d.b[:]
+			bs = d.d.b[:]
 		}
 	}
 	return decByteSlice(d.r, clen, d.d.h.MaxInitLen, bs)
@@ -803,7 +801,6 @@ func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 	}
 	realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag))
 	realxtag = uint64(realxtag1)
-	// xdebugf("binc DecodeExt: xbs: len: %d, %v", len(xbs), xbs)
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = realxtag
@@ -978,7 +975,7 @@ type BincHandle struct {
 	// - n: none
 	// - a: all: same as m, s, ...
 
-	_ [1]uint64 // padding (cache-aligned)
+	_ [7]uint64 // padding (cache-aligned)
 }
 
 // Name returns the name of the handle: binc
@@ -998,7 +995,6 @@ func (h *BincHandle) newDecDriver(d *Decoder) decDriver {
 }
 
 func (e *bincEncDriver) reset() {
-	// xdebugf("bincEncDriver: reset: e.m: %v, isnil: %v", e.m, e.m == nil)
 	e.w = e.e.w()
 	e.s = 0
 	e.m = nil
@@ -1006,7 +1002,6 @@ func (e *bincEncDriver) reset() {
 
 func (e *bincEncDriver) atEndOfEncode() {
 	if e.m != nil {
-		// xdebug2f("bincEncDriver: reset: len: %d", len(e.m))
 		for k := range e.m {
 			delete(e.m, k)
 		}
@@ -1016,7 +1011,6 @@ func (e *bincEncDriver) atEndOfEncode() {
 }
 
 func (d *bincDecDriver) reset() {
-	// xdebugf("bincDecDriver: reset")
 	d.r, d.br = d.d.r(), d.d.bytes
 	d.s = nil
 	d.bd, d.bdRead, d.vd, d.vs = 0, false, 0, 0
@@ -1024,7 +1018,6 @@ func (d *bincDecDriver) reset() {
 
 func (d *bincDecDriver) atEndOfDecode() {
 	if d.s != nil {
-		// xdebug2f("bincDecDriver: reset: len: %d", len(d.s))
 		for k := range d.s {
 			delete(d.s, k)
 		}

codec/cbor.go

@@ -197,7 +197,6 @@ func (e *cborEncDriver) EncodeTime(t time.Time) {
 }
 
 func (e *cborEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext) {
-	// xdebugf("cbor EncodeExt: v: %T, %v, ext: %T, %v, ==Self: %v", rv, rv, ext, ext, ext == SelfExt)
 	e.encUint(uint64(xtag), cborBaseTag)
 	if ext == SelfExt {
 		rv2 := baseRV(rv)
@@ -279,7 +278,7 @@ func (e *cborEncDriver) encStringBytesS(bb byte, v string) {
 		for i := uint(0); i < vlen; {
 			var v2 string
 			i2 := i + blen
-			if i2 < vlen {
+			if i2 >= i && i2 < vlen {
 				v2 = v[i:i2]
 			} else {
 				v2 = v[i:]
@@ -495,7 +494,8 @@ func (d *cborDecDriver) DecodeFloat64() (f float64) {
 		} else if major == cborMajorNegInt {
 			f = float64(cborDecInt64(d.decUint(), true))
 		} else {
-			d.d.errorf("float only valid from float16/32/64 - invalid descriptor %x/%s", d.bd, cbordesc(d.bd))
+			d.d.errorf("invalid float descriptor; got %d/%s, expected float16/32/64 or (-)int",
+				d.bd, cbordesc(d.bd))
 		}
 	}
 	d.bdRead = false
@@ -533,7 +533,8 @@ func (d *cborDecDriver) ReadMapStart() (length int) {
 		return -1
 	}
 	if d.bd>>5 != cborMajorMap {
-		d.d.errorf("error reading map; got major type: %x, expected: %x/%s", d.bd>>5, cborMajorMap, cbordesc(d.bd))
+		d.d.errorf("error reading map; got major type: %x, expected %x/%s",
+			d.bd>>5, cborMajorMap, cbordesc(d.bd))
 	}
 	return d.decLen()
 }
@@ -550,7 +551,8 @@ func (d *cborDecDriver) ReadArrayStart() (length int) {
 		return -1
 	}
 	if d.bd>>5 != cborMajorArray {
-		d.d.errorf("error reading array; got major type: %x, expect: %x/%s", d.bd>>5, cborMajorArray, cbordesc(d.bd))
+		d.d.errorf("invalid array; got major type: %x, expect: %x/%s",
+			d.bd>>5, cborMajorArray, cbordesc(d.bd))
 	}
 	return d.decLen()
 }
@@ -563,14 +565,14 @@ func (d *cborDecDriver) decAppendIndefiniteBytes(bs []byte) []byte {
 	d.bdRead = false
 	for !d.CheckBreak() {
 		if major := d.bd >> 5; major != cborMajorBytes && major != cborMajorString {
-			d.d.errorf("error reading indefinite string/bytes;"+
-				" got major %v, expected %x/%s", major, d.bd, cbordesc(d.bd))
+			d.d.errorf("invalid indefinite string/bytes; got major %v, expected %x/%s",
+				major, d.bd, cbordesc(d.bd))
 		}
-		n := d.decLen()
-		oldLen := len(bs)
+		n := uint(d.decLen())
+		oldLen := uint(len(bs))
 		newLen := oldLen + n
-		if newLen > cap(bs) {
-			bs2 := make([]byte, newLen, 2*cap(bs)+n)
+		if newLen > uint(cap(bs)) {
+			bs2 := make([]byte, newLen, 2*uint(cap(bs))+n)
 			copy(bs2, bs)
 			bs = bs2
 		} else {
@@ -632,7 +634,7 @@ func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 		}
 		slen := d.decLen()
 		bs = usableByteSlice(bs, slen)
-		for i := 0; i < slen; i++ {
+		for i := 0; i < len(bs); i++ {
 			bs[i] = uint8(chkOvf.UintV(d.DecodeUint64(), 8))
 		}
 		return bs
@@ -704,7 +706,6 @@ func (d *cborDecDriver) decodeTime(xtag uint64) (t time.Time) {
 }
 
 func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
-	// xdebugf("cbor DecodeExt: v: %T, %v", rv, rv)
 	if !d.bdRead {
 		d.readNextBd()
 	}
@@ -739,8 +740,6 @@ func (d *cborDecDriver) DecodeNaked() {
 	n := d.d.naked()
 	var decodeFurther bool
 
-	// xdebug2f("DecodeNaked: bd: %x, bd >> 5: %x", d.bd, d.bd>>5)
-
 	switch d.bd >> 5 {
 	case cborMajorUint:
 		if d.h.SignedInteger {
@@ -851,7 +850,7 @@ type CborHandle struct {
 	// Furthermore, this allows the skipping over of the Self Describing Tag 0xd9d9f7.
 	SkipUnexpectedTags bool
 
-	_ [1]uint64 // padding (cache-aligned)
+	_ [7]uint64 // padding (cache-aligned)
 }
 
 // Name returns the name of the handle: cbor

codec/codec_test.go

@@ -358,7 +358,7 @@ func checkEqualT(t *testing.T, v1 interface{}, v2 interface{}, desc string) {
 func testInit() {
 	gob.Register(new(TestStrucFlex))
 	if testInitDebug {
-		ts0 := newTestStrucFlex(2, testNumRepeatString, false, !testSkipIntf, false)
+		ts0 := newTestStrucFlex(2, testNumRepeatString, false, !testSkipIntf, testMapStringKeyOnly)
 		logTv(nil, "====> depth: %v, ts: %#v\n", 2, ts0)
 	}
 
@@ -529,7 +529,7 @@ ugorji
 	table = append(table, primitives)
 	table = append(table, testMbsT(primitives))
 	table = append(table, maps...)
-	table = append(table, newTestStrucFlex(0, testNumRepeatString, false, !testSkipIntf, false))
+	table = append(table, newTestStrucFlex(0, testNumRepeatString, false, !testSkipIntf, testMapStringKeyOnly))
 
 }
 
@@ -899,7 +899,7 @@ func testCodecMiscOne(t *testing.T, h Handle) {
 	}
 
 	// func TestMsgpackDecodePtr(t *testing.T) {
-	ts := newTestStrucFlex(testDepth, testNumRepeatString, false, !testSkipIntf, false)
+	ts := newTestStrucFlex(testDepth, testNumRepeatString, false, !testSkipIntf, testMapStringKeyOnly)
 	b = testMarshalErr(ts, h, t, "pointer-to-struct")
 	if len(b) < 40 {
 		failT(t, "------- Size must be > 40. Size: %d", len(b))
@@ -1692,7 +1692,7 @@ func doTestMsgpackRpcSpecPythonClientToGoSvc(t *testing.T) {
 
 func doTestSwallowAndZero(t *testing.T, h Handle) {
 	testOnce.Do(testInitAll)
-	v1 := newTestStrucFlex(testDepth, testNumRepeatString, false, false, false)
+	v1 := newTestStrucFlex(testDepth, testNumRepeatString, false, false, testMapStringKeyOnly)
 	var b1 []byte
 
 	e1 := NewEncoderBytes(&b1, h)
@@ -2463,20 +2463,6 @@ func doTestSelfExt(t *testing.T, name string, h Handle) {
 	bs := testMarshalErr(&ts, h, t, name)
 	testUnmarshalErr(&ts2, bs, h, t, name)
 	testDeepEqualErr(&ts, &ts2, t, name)
-
-	var ts3 TestSelfExtImpl2
-	ts3.M = "moo"
-	ts3.O = true
-
-	s := TestTwoNakedInterfaces{
-		A: ts,
-		B: ts3,
-	}
-	var s2 TestTwoNakedInterfaces
-
-	bs = testMarshalErr(&s, h, t, name)
-	testUnmarshalErr(&s2, bs, h, t, name)
-	testDeepEqualErr(&s, &s2, t, name)
 }
 
 func doTestBytesEncodedAsArray(t *testing.T, name string, h Handle) {
@@ -2808,6 +2794,99 @@ func TestMsgpackDecodeMapAndExtSizeMismatch(t *testing.T) {
 	// fn(t, b, &s)
 }
 
+func TestMapRangeIndex(t *testing.T) {
+	// t.Skip()
+	type T struct {
+		I int
+		S string
+		B bool
+		M map[int]T
+	}
+
+	t1 := T{I: 1, B: true, S: "11", M: map[int]T{11: T{I: 11}}}
+	t2 := T{I: 1, B: true, S: "12", M: map[int]T{12: T{I: 12}}}
+
+	// ------
+
+	var m1 = map[string]*T{
+		"11": &t1,
+		"12": &t2,
+	}
+	var m1c = make(map[string]T)
+	for k, v := range m1 {
+		m1c[k] = *v
+	}
+
+	mt := reflect.TypeOf(m1)
+	it := mapRange(reflect.ValueOf(m1), mapAddressableRV(mt.Key()), mapAddressableRV(mt.Elem()), true)
+	for it.Next() {
+		k := it.Key().Interface().(string)
+		v := it.Value().Interface().(*T)
+		testDeepEqualErr(m1[k], v, t, "map-key-eq-it-key")
+		if _, ok := m1c[k]; ok {
+			delete(m1c, k)
+		} else {
+			failT(t, "unexpected key in map: %v", k)
+		}
+	}
+	testDeepEqualErr(len(m1c), 0, t, "all-keys-not-consumed")
+
+	// ------
+
+	var m2 = map[*T]T{
+		&t1: t1,
+		&t2: t2,
+	}
+	var m2c = make(map[*T]*T)
+	for k, _ := range m2 {
+		m2c[k] = k
+	}
+
+	mt = reflect.TypeOf(m2)
+	it = mapRange(reflect.ValueOf(m2), mapAddressableRV(mt.Key()), mapAddressableRV(mt.Elem()), true)
+	for it.Next() {
+		k := it.Key().Interface().(*T)
+		v := it.Value().Interface().(T)
+		testDeepEqualErr(m2[k], v, t, "map-key-eq-it-key")
+		if _, ok := m2c[k]; ok {
+			delete(m2c, k)
+		} else {
+			failT(t, "unexpected key in map: %v", k)
+		}
+	}
+	testDeepEqualErr(len(m2c), 0, t, "all-keys-not-consumed")
+
+	// ---- test mapIndex
+
+	fnTestMapIndex := func(mi ...interface{}) {
+		for _, m0 := range mi {
+			m := reflect.ValueOf(m0)
+			rvv := mapAddressableRV(m.Type().Elem())
+			for _, k := range m.MapKeys() {
+				testDeepEqualErr(m.MapIndex(k).Interface(), mapIndex(m, k, rvv).Interface(), t, "map-index-eq")
+			}
+		}
+	}
+
+	fnTestMapIndex(m1, m1c, m2, m2c)
+
+	// var s string = "hello"
+	// var tt = &T{I: 3}
+	// ttTyp := reflect.TypeOf(tt)
+	// _, _ = tt, ttTyp
+	// mv := reflect.ValueOf(m)
+	// it := mapRange(mv, reflect.ValueOf(&s).Elem(), reflect.ValueOf(&tt).Elem(), true) //ok
+	// it := mapRange(mv, reflect.New(reflect.TypeOf(s)).Elem(), reflect.New(reflect.TypeOf(T{})).Elem(), true) // ok
+	// it := mapRange(mv, reflect.New(reflect.TypeOf(s)).Elem(), reflect.New(ttTyp.Elem()), true) // !ok
+	// it := mapRange(mv, reflect.New(reflect.TypeOf(s)).Elem(), reflect.New(reflect.TypeOf(T{})), true) !ok
+	// it := mapRange(mv, reflect.New(reflect.TypeOf(s)).Elem(), reflect.New(reflect.TypeOf(T{})).Elem(), true) // ok
+
+	// fmt.Printf("key: %#v\n", it.Key())
+	// fmt.Printf("exp: %#v\n", mv.MapIndex(it.Key()))
+	// fmt.Printf("val: %#v\n", it.Value())
+	// testDeepEqualErr(mv.MapIndex(it.Key()), it.Value().Interface()
+}
+
 // ----------
 
 func TestBincCodecsTable(t *testing.T) {

codec/decode.go

@@ -698,7 +698,6 @@ func (z *bufioDecReader) readx(n uint) (bs []byte) {
 	return
 }
 
-//go:noinline - track called by Decoder.nextValueBytes() (called by jsonUnmarshal,rawBytes)
 func (z *bufioDecReader) doTrack(y uint) {
 	z.tr = append(z.tr, z.buf[z.c:y]...) // cost=14???
 }
@@ -1299,8 +1298,8 @@ func (d *Decoder) kInterfaceNaked(f *codecFnInfo) (rvn reflect.Value) {
 		bfn := d.h.getExtForTag(tag)
 		var re = RawExt{Tag: tag}
 		if bytes == nil {
-			// it is one of the InterfaceExt ones: json and cbor
-			// almost definitely cbor, as json naked decoding never reveals valueTypeExt (no tagging support).
+			// it is one of the InterfaceExt ones: json and cbor.
+			// most likely cbor, as json decoding never reveals valueTypeExt (no tagging support)
 			if bfn == nil {
 				d.decode(&re.Value)
 				rvn = reflect.ValueOf(&re).Elem()
@@ -1457,9 +1456,7 @@ func (d *Decoder) kStruct(f *codecFnInfo, rv reflect.Value) {
 				copy(rvkencname, name2)
 
 				var f interface{}
-				// xdebugf("kStruct: mf != nil: before decode: rvkencname: %s", rvkencname)
 				d.decode(&f)
-				// xdebugf("kStruct: mf != nil: after decode: rvkencname: %s", rvkencname)
 				if !mf.CodecMissingField(rvkencname, f) && d.h.ErrorIfNoField {
 					d.errorf("no matching struct field found when decoding stream map with key: %s ",
 						stringView(rvkencname))
@@ -1762,7 +1759,7 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 	ktype, vtype := ti.key, ti.elem
 	ktypeId := rt2id(ktype)
 	vtypeKind := vtype.Kind()
-	ktypeKind := ktype.Kind()
+	// ktypeKind := ktype.Kind()
 	var keyFn, valFn *codecFn
 	var ktypeLo, vtypeLo reflect.Type
 
@@ -1774,6 +1771,7 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 
 	rvvMut := !isImmutableKind(vtypeKind)
 
+	// we do a mapGet if kind is mutable, and InterfaceReset=true if interface
 	var mapGet, mapSet bool
 	if !d.h.MapValueReset {
 		if rvvMut {
@@ -1787,34 +1785,28 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 		}
 	}
 
-	var rvk, rvkn, rvv, rvvn, rvvz reflect.Value
+	var rvk, rvkn, rvv, rvvn, rvva, rvvz reflect.Value
+	var rvvaSet bool
 	rvkMut := !isImmutableKind(ktype.Kind()) // if ktype is immutable, then re-use the same rvk.
 	ktypeIsString := ktypeId == stringTypId
 	ktypeIsIntf := ktypeId == intfTypId
 	hasLen := containerLen > 0
 	var kstrbs []byte
 
-	vNew := func(t reflect.Type, k reflect.Kind) reflect.Value {
-		if k == reflect.Ptr {
-			return reflect.New(t.Elem())
-		}
-		return reflect.New(t).Elem()
-	}
-
 	for j := 0; (hasLen && j < containerLen) || !(hasLen || dd.CheckBreak()); j++ {
 		if j == 0 {
 			// rvvz = reflect.Zero(vtype)
 			// rvkz = reflect.Zero(ktype)
 			if !rvkMut {
-				rvkn = vNew(ktype, ktypeKind)
+				rvkn = reflect.New(ktype).Elem() //, ktypeKind)
 			}
 			if !rvvMut {
-				rvvn = vNew(vtype, vtypeKind)
+				rvvn = reflect.New(vtype).Elem() //, vtypeKind)
 			}
 		}
 
 		if rvkMut {
-			rvk = vNew(ktype, ktypeKind)
+			rvk = reflect.New(ktype).Elem() //, ktypeKind)
 		} else {
 			rvk = rvkn
 		}
@@ -1860,7 +1852,11 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 
 		mapSet = true // set to false if u do a get, and its a non-nil pointer
 		if mapGet {
-			rvv = rv.MapIndex(rvk)
+			if !rvvaSet {
+				rvva = mapAddressableRV(vtype)
+				rvvaSet = true
+			}
+			rvv = mapIndex(rv, rvk, rvva) // reflect.Value{})
 			if vtypeKind == reflect.Ptr {
 				if rvv.IsValid() && !rvv.IsNil() {
 					mapSet = false
@@ -1877,7 +1873,7 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 				rvv = rvvn
 			}
 		} else if rvvMut {
-			rvv = vNew(vtype, vtypeKind)
+			rvv = reflect.New(vtype).Elem() //, vtypeKind)
 		} else {
 			rvv = rvvn
 		}
@@ -1886,8 +1882,8 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 			valFn = d.h.fn(vtypeLo)
 		}
 
-		// We MUST be done with the stringview of the key, before decoding the value
-		// so that we don't bastardize the reused byte array.
+		// We MUST be done with the stringview of the key, BEFORE decoding the value (rvv)
+		// so that we don't unknowingly reuse the rvk backing buffer during rvv decode.
 		if mapSet && ktypeIsString { // set to a real string (not string view)
 			rvk.SetString(d.string(kstrbs))
 		}
@@ -2313,7 +2309,7 @@ type Decoder struct {
 	depth    int16
 	c        containerState
 	_        [3]byte                      // padding
-	b        [decScratchByteArrayLen]byte // scratch buffer, used by Decoder and xxxEncDrivers
+	b        [decScratchByteArrayLen]byte // scratch buffer, used by Decoder and xxxDecDrivers
 
 	// padding - false sharing help // modify 232 if Decoder struct changes.
 	// _ [cacheLineSize - 232%cacheLineSize]byte
@@ -2344,7 +2340,6 @@ func newDecoder(h Handle) *Decoder {
 	d.bytes = true
 	if useFinalizers {
 		runtime.SetFinalizer(d, (*Decoder).finalize)
-		// xdebugf(">>>> new(Decoder) with finalizer")
 	}
 	// d.r = &d.decReaderSwitch
 	d.hh = h
@@ -2578,7 +2573,6 @@ func (d *Decoder) mustDecode(v interface{}) {
 
 //go:noinline -- as it is run by finalizer
 func (d *Decoder) finalize() {
-	// xdebugf("finalizing Decoder")
 	d.Release()
 }
 
@@ -3125,10 +3119,21 @@ func detachZeroCopyBytes(isBytesReader bool, dest []byte, in []byte) (out []byte
 //      if <= 0, it is unset, and we infer it based on the unit size
 //    - unit: number of bytes for each element of the collection
 func decInferLen(clen, maxlen, unit int) (rvlen int) {
+	const maxLenIfUnset = 8 // 64
 	// handle when maxlen is not set i.e. <= 0
-	if clen <= 0 {
+
+	// clen==0:           use 0
+	// maxlen<=0, clen<0: use default
+	// maxlen> 0, clen<0: use default
+	// maxlen<=0, clen>0: infer maxlen, and cap on it
+	// maxlen> 0, clen>0: cap at maxlen
+
+	if clen == 0 {
 		return
 	}
+	if clen < 0 {
+		return maxLenIfUnset
+	}
 	if unit == 0 {
 		return clen
 	}
@@ -3143,6 +3148,9 @@ func decInferLen(clen, maxlen, unit int) (rvlen int) {
 		} else {
 			maxlen = 4 * 1024
 		}
+		// if maxlen > maxLenIfUnset {
+		// 	maxlen = maxLenIfUnset
+		// }
 	}
 	if clen > maxlen {
 		rvlen = maxlen
@@ -3195,7 +3203,6 @@ func decReadFull(r io.Reader, bs []byte) (n uint, err error) {
 			n += uint(nn)
 		}
 	}
-	// xdebugf("decReadFull: len(bs): %v, n: %v, err: %v", len(bs), n, err)
 	// do not do this - it serves no purpose
 	// if n != len(bs) && err == io.EOF { err = io.ErrUnexpectedEOF }
 	return
@@ -3210,3 +3217,7 @@ func decNakedReadRawBytes(dr decDriver, d *Decoder, n *decNaked, rawToString boo
 		n.l = dr.DecodeBytes(nil, false)
 	}
 }
+
+// register these here, so that staticcheck stops barfing
+var _ = (*bytesDecReader).readTo
+var _ = (*bytesDecReader).readUntil

codec/encode.go

@@ -452,7 +452,9 @@ func (z *bytesEncAppender) writeqstr(s string) {
 	// z.writen1('"')
 	// z.writestr(s)
 	// z.writen1('"')
+
 	z.b = append(append(append(z.b, '"'), s...), '"')
+
 	// z.b = append(z.b, '"')
 	// z.b = append(z.b, s...)
 	// z.b = append(z.b, '"')
@@ -479,7 +481,6 @@ func (e *Encoder) rawExt(f *codecFnInfo, rv reflect.Value) {
 }
 
 func (e *Encoder) ext(f *codecFnInfo, rv reflect.Value) {
-	// xdebugf("*Encoder.ext: rv: %v, rv2i: %T, %v", rv, rv2i(rv), rv2i(rv))
 	e.e.EncodeExt(rv2i(rv), f.xfTag, f.xfFn)
 }
 
@@ -619,7 +620,6 @@ func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 }
 
 func (e *Encoder) kSliceBytes(rv reflect.Value, seq seqType) {
-	// xdebugf("kSliceBytes: seq: %d, rvType: %v", seq, rv.Type())
 	switch seq {
 	case seqTypeSlice:
 		e.e.EncodeStringBytesRaw(rv.Bytes())
@@ -857,7 +857,7 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
 	}
 
 	if e.h.Canonical {
-		e.kMapCanonical(f.ti.key, rv, valFn)
+		e.kMapCanonical(f.ti.key, f.ti.elem, rv, valFn)
 		e.mapEnd()
 		return
 	}
@@ -874,7 +874,11 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
 		}
 	}
 
-	it := mapRange(rv)
+	var rvk = mapAddressableRV(f.ti.key)  //, f.ti.key.Kind())
+	var rvv = mapAddressableRV(f.ti.elem) //, f.ti.elem.Kind())
+
+	it := mapRange(rv, rvk, rvv, true)
+
 	for it.Next() {
 		e.mapElemKey()
 		if keyTypeIsString {
@@ -884,21 +888,23 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
 				e.e.EncodeStringEnc(cUTF8, it.Key().String())
 			}
 		} else {
-			e.encodeValue(it.Key(), keyFn)
+			e.encodeValue(it.Key(), keyFn) //
 		}
 		e.mapElemValue()
-		e.encodeValue(it.Value(), valFn)
+		iv := it.Value()
+		e.encodeValue(iv, valFn)
 	}
 
 	e.mapEnd()
 }
 
-func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *codecFn) {
+func (e *Encoder) kMapCanonical(rtkey, rtval reflect.Type, rv reflect.Value, valFn *codecFn) {
 	// we previously did out-of-band if an extension was registered.
 	// This is not necessary, as the natural kind is sufficient for ordering.
 
-	mks := rv.MapKeys()
+	rvv := mapAddressableRV(rtval)
 
+	mks := rv.MapKeys()
 	switch rtkey.Kind() {
 	case reflect.Bool:
 		mksv := make([]boolRv, len(mks))
@@ -912,7 +918,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *cod
 			e.mapElemKey()
 			e.e.EncodeBool(mksv[i].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			e.encodeValue(mapIndex(rv, mksv[i].r, rvv), valFn) // e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.String:
 		mksv := make([]stringRv, len(mks))
@@ -930,7 +936,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *cod
 				e.e.EncodeStringEnc(cUTF8, mksv[i].v)
 			}
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			e.encodeValue(mapIndex(rv, mksv[i].r, rvv), valFn) // e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint, reflect.Uintptr:
 		mksv := make([]uint64Rv, len(mks))
@@ -944,7 +950,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *cod
 			e.mapElemKey()
 			e.e.EncodeUint(mksv[i].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			e.encodeValue(mapIndex(rv, mksv[i].r, rvv), valFn) // e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
 		mksv := make([]int64Rv, len(mks))
@@ -958,7 +964,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *cod
 			e.mapElemKey()
 			e.e.EncodeInt(mksv[i].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			e.encodeValue(mapIndex(rv, mksv[i].r, rvv), valFn) // e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Float32:
 		mksv := make([]float64Rv, len(mks))
@@ -972,7 +978,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *cod
 			e.mapElemKey()
 			e.e.EncodeFloat32(float32(mksv[i].v))
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			e.encodeValue(mapIndex(rv, mksv[i].r, rvv), valFn) // e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Float64:
 		mksv := make([]float64Rv, len(mks))
@@ -986,7 +992,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *cod
 			e.mapElemKey()
 			e.e.EncodeFloat64(mksv[i].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			e.encodeValue(mapIndex(rv, mksv[i].r, rvv), valFn) // e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Struct:
 		if rv.Type() == timeTyp {
@@ -1001,7 +1007,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *cod
 				e.mapElemKey()
 				e.e.EncodeTime(mksv[i].v)
 				e.mapElemValue()
-				e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+				e.encodeValue(mapIndex(rv, mksv[i].r, rvv), valFn) // e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 			}
 			break
 		}
@@ -1025,7 +1031,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, valFn *cod
 			e.mapElemKey()
 			e.asis(mksbv[j].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksbv[j].r), valFn)
+			e.encodeValue(mapIndex(rv, mksbv[j].r, rvv), valFn) // e.encodeValue(rv.MapIndex(mksbv[j].r), valFn)
 		}
 		bufp.end()
 	}
@@ -1259,7 +1265,6 @@ func newEncoder(h Handle) *Encoder {
 	e.bytes = true
 	if useFinalizers {
 		runtime.SetFinalizer(e, (*Encoder).finalize)
-		// xdebugf(">>>> new(Encoder) with finalizer")
 	}
 	// e.w = &e.encWriterSwitch
 	e.hh = h
@@ -1512,7 +1517,6 @@ func (e *Encoder) mustEncode(v interface{}) {
 
 //go:noinline -- as it is run by finalizer
 func (e *Encoder) finalize() {
-	// xdebugf("finalizing Encoder")
 	e.Release()
 }
 
@@ -1826,7 +1830,6 @@ func (e *Encoder) sideEncode(v interface{}, bs *[]byte) {
 	e2.encodeValue(rv, e.h.fnNoExt(rv.Type()))
 	e2.e.atEndOfEncode()
 	e2.w().end()
-	// xdebugf("sideEncode: xbs: len: %d, %v, v: %v", len(*bs), *bs, v)
 }
 
 func encStructFieldKey(encName string, ee encDriver, w *encWriterSwitch,

codec/float.go

@@ -304,10 +304,10 @@ L:
 
 // fMul10ShiftU64
 
-func parseFloatDebug(b []byte, bitsize int, strconv bool, exp int8, trunc, ok bool) {
-	if strconv {
-		xdebugf("parseFloat%d: delegating: %s, exp: %d, trunc: %v, ok: %v", bitsize, b, exp, trunc, ok)
-	} else {
-		xdebug2f("parseFloat%d: attempting: %s, exp: %d, trunc: %v, ok: %v", bitsize, b, exp, trunc, ok)
-	}
-}
+// func parseFloatDebug(b []byte, bitsize int, strconv bool, exp int8, trunc, ok bool) {
+// 	if strconv {
+// 		xdebugf("parseFloat%d: delegating: %s, exp: %d, trunc: %v, ok: %v", bitsize, b, exp, trunc, ok)
+// 	} else {
+// 		xdebug2f("parseFloat%d: attempting: %s, exp: %d, trunc: %v, ok: %v", bitsize, b, exp, trunc, ok)
+// 	}
+// }

codec/gen.go

@@ -696,7 +696,6 @@ func (x *genRunner) registerXtraT(t reflect.Type) {
 // encVar will encode a variable.
 // The parameter, t, is the reflect.Type of the variable itself
 func (x *genRunner) encVar(varname string, t reflect.Type) {
-	// xdebugf("varname: %s, t: %v", varname, t)
 	var checkNil bool
 	switch t.Kind() {
 	case reflect.Ptr, reflect.Interface, reflect.Slice, reflect.Map, reflect.Chan:
@@ -979,7 +978,6 @@ func (x *genRunner) encOmitEmptyLine(t2 reflect.StructField, varname string, buf
 }
 
 func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
-	// xdebug2f("encStruct, varname: %s, t: %s", varname, t)
 	// Use knowledge from structfieldinfo (mbs, encodable fields. Ignore omitempty. )
 	// replicate code in kStruct i.e. for each field, deref type to non-pointer, and call x.enc on it
 
@@ -1112,7 +1110,6 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 		if si.omitEmpty() {
 			x.linef("if %s[%v] {", numfieldsvar, j)
 		}
-		// xdebug2f("varname: %s, t2.Type: %s", varname3, t2.Type)
 		x.encVarChkNil(q.fqname, q.sf.Type, false)
 		if si.omitEmpty() {
 			x.linef("} else {")

codec/goversion_maprange_gte_go112.go

@@ -2,13 +2,70 @@
 // Use of this source code is governed by a MIT license found in the LICENSE file.
 
 // +build go1.12
-
-// add support for reflect.Value.MapRange.
+// +build safe
 
 package codec
 
 import "reflect"
 
-func mapRange(rv reflect.Value) *reflect.MapIter {
-	return rv.MapRange()
+type mapIter struct {
+	t        *reflect.MapIter
+	m, k, v  reflect.Value
+	kOk, vOk bool
+	values   bool
+}
+
+func (t *mapIter) Next() (r bool) {
+	r = t.t.Next()
+	if r {
+		if t.kOk {
+			t.k.Set(t.t.Key())
+		}
+		if t.vOk {
+			t.v.Set(t.t.Value())
+		}
+	}
+	return
+}
+
+func (t *mapIter) Key() reflect.Value {
+	if t.kOk {
+		return t.k
+	}
+	return t.t.Key()
+}
+
+func (t *mapIter) Value() (r reflect.Value) {
+	if !t.values {
+		return
+	}
+	if t.vOk {
+		return t.v
+	}
+	return t.t.Value()
+}
+
+func mapRange(m, k, v reflect.Value, values bool) *mapIter {
+	return &mapIter{
+		m:      m,
+		k:      k,
+		v:      v,
+		kOk:    k.CanSet(),
+		vOk:    values && v.CanSet(),
+		t:      m.MapRange(),
+		values: values,
+	}
+}
+
+func mapIndex(m, k, v reflect.Value) (vv reflect.Value) {
+	vv = m.MapIndex(k)
+	if vv.IsValid() && v.CanSet() {
+		v.Set(vv)
+	}
+	return
+}
+
+// return an addressable reflect value that can be used in mapRange and mapIndex operations.
+func mapAddressableRV(t reflect.Type) (r reflect.Value) {
+	return // reflect.New(t).Elem()
 }

codec/goversion_maprange_lt_go112.go

@@ -7,23 +7,61 @@ package codec
 
 import "reflect"
 
-type mapRanger struct {
-	rv  reflect.Value
-	mks []reflect.Value
-	j   int
+type mapIter struct {
+	m, k, v  reflect.Value
+	keys     []reflect.Value
+	j        int
+	kOk, vOk bool
+	values   bool
 }
 
-func (x *mapRanger) Next() bool {
-	x.j++
-	return x.j < len(x.mks)
+func (t *mapIter) Next() (r bool) {
+	t.j++
+	r = t.j < len(t.keys)
+	if r {
+		if t.kOk {
+			t.k.Set(t.keys[t.j])
+		}
+		if t.vOk {
+			t.v.Set(t.rv.MapIndex(t.keys[t.j]))
+		}
+	}
 }
-func (x *mapRanger) Key() reflect.Value {
-	return x.mks[x.j]
+
+func (t *mapIter) Key() reflect.Value {
+	if t.kOk {
+		return t.k
+	}
+	return t.keys[t.j]
+}
+
+func (t *mapIter) Value() (r reflect.Value) {
+	if !t.values {
+		return
+	}
+	if t.vOk {
+		return t.v
+	}
+	return t.rv.MapIndex(t.keys[t.j])
 }
-func (x *mapRanger) Value() reflect.Value {
-	return x.rv.MapIndex(x.mks[x.j])
+
+func mapRange(m, k, v reflect.Value, values bool) *mapIter {
+	return &mapIter{m: m, k: k, v: v,
+		kOk: k.CanSet(), vOk: values && v.CanSet(),
+		keys: rv.MapKeys(), j: -1,
+		values: values,
+	}
+}
+
+func mapIndex(m, k, v reflect.Value) (vv reflect.Value) {
+	vv = m.MapIndex(k)
+	if vv.IsValid() && v.CanSet() {
+		v.Set(vv)
+	}
+	return
 }
 
-func mapRange(rv reflect.Value) (g *mapRanger) {
-	return &mapRanger{rv: rv, mks: rv.MapKeys(), j: -1}
+// return an addressable reflect value that can be used in mapRange and mapIndex operations.
+func mapAddressableRV(t reflect.Type) (r reflect.Value) {
+	return // reflect.New(t).Elem()
 }

codec/helper.go

@@ -150,6 +150,9 @@ const (
 	//    runtime.SetFinalizer(e, (*Encoder).Release)
 	//    runtime.SetFinalizer(d, (*Decoder).Release)
 	useFinalizers = false
+
+	// xdebug controls whether xdebugf prints any output
+	xdebug = true
 )
 
 var oneByteArr [1]byte
@@ -168,6 +171,7 @@ func init() {
 	refBitset.set(byte(reflect.Ptr))
 	refBitset.set(byte(reflect.Func))
 	refBitset.set(byte(reflect.Chan))
+	refBitset.set(byte(reflect.UnsafePointer))
 }
 
 type handleFlag uint8
@@ -546,12 +550,22 @@ type BasicHandle struct {
 
 	extHandle
 
-	intf2impls
+	rtidFns      atomicRtidFnSlice
+	rtidFnsNoExt atomicRtidFnSlice
 
-	EncodeOptions
+	// ---- cache line
 
 	DecodeOptions
 
+	// ---- cache line
+
+	EncodeOptions
+
+	intf2impls
+
+	mu     sync.Mutex
+	inited uint32 // holds if inited, and also handle flags (binary encoding, json handler, etc)
+
 	RPCOptions
 
 	// TimeNotBuiltin configures whether time.Time should be treated as a builtin type.
@@ -594,11 +608,7 @@ type BasicHandle struct {
 
 	// noBuiltInTypeChecker
 
-	inited uint32 // holds if inited, and also handle flags (binary encoding, json handler, etc)
-	mu     sync.Mutex
 	// _      uint32 // padding
-	rtidFns      atomicRtidFnSlice
-	rtidFnsNoExt atomicRtidFnSlice
 	// r []uintptr     // rtids mapped to s above
 }
 
@@ -692,12 +702,10 @@ func (x *BasicHandle) fnNoExt(rt reflect.Type) (fn *codecFn) {
 }
 
 func (x *BasicHandle) fnVia(rt reflect.Type, fs *atomicRtidFnSlice, checkExt bool) (fn *codecFn) {
-	// xdebug2f("fnVia: rt: %v, checkExt: %v", rt, checkExt)
 	rtid := rt2id(rt)
 	sp := fs.load()
 	if sp != nil {
 		if _, fn = findFn(sp, rtid); fn != nil {
-			// xdebugf("<<<< %c: found fn for %v in rtidfns of size: %v", x.n, rt, len(sp))
 			return
 		}
 	}
@@ -708,8 +716,6 @@ func (x *BasicHandle) fnVia(rt reflect.Type, fs *atomicRtidFnSlice, checkExt boo
 	if sp == nil {
 		sp2 = []codecRtidFn{{rtid, fn}}
 		fs.store(sp2)
-		// xdebugf(">>>> adding rt: %v to rtidfns of size: %v", rt, len(sp2))
-		// xdebugf(">>>> loading stored rtidfns of size: %v", len(fs.load()))
 	} else {
 		idx, fn2 := findFn(sp, rtid)
 		if fn2 == nil {
@@ -718,8 +724,6 @@ func (x *BasicHandle) fnVia(rt reflect.Type, fs *atomicRtidFnSlice, checkExt boo
 			copy(sp2[idx+1:], sp[idx:])
 			sp2[idx] = codecRtidFn{rtid, fn}
 			fs.store(sp2)
-			// xdebugf(">>>> adding rt: %v to rtidfns of size: %v", rt, len(sp2))
-
 		}
 	}
 	x.mu.Unlock()
@@ -727,7 +731,6 @@ func (x *BasicHandle) fnVia(rt reflect.Type, fs *atomicRtidFnSlice, checkExt boo
 }
 
 func (x *BasicHandle) fnLoad(rt reflect.Type, rtid uintptr, checkExt bool) (fn *codecFn) {
-	// xdebugf("#### for %c: load fn for %v in rtidfns of size: %v", x.n, rt, len(sp))
 	fn = new(codecFn)
 	fi := &(fn.i)
 	ti := x.getTypeInfo(rtid, rt)
@@ -2148,6 +2151,17 @@ func baseRV(v interface{}) (rv reflect.Value) {
 	return
 }
 
+// func newAddressableRV(t reflect.Type, k reflect.Kind) reflect.Value {
+// 	if k == reflect.Ptr {
+// 		return reflect.New(t.Elem()) // this is not addressable???
+// 	}
+// 	return reflect.New(t).Elem()
+// }
+
+// func newAddressableRV(t reflect.Type) reflect.Value {
+// 	return reflect.New(t).Elem()
+// }
+
 // ----
 
 // these "checkOverflow" functions must be inlinable, and not call anybody.
@@ -2305,7 +2319,6 @@ type set []interface{}
 func (s *set) add(v interface{}) (exists bool) {
 	// e.ci is always nil, or len >= 1
 	x := *s
-	// defer func() { xdebugf("set.add: len: %d", len(x)) }()
 
 	if x == nil {
 		x = make([]interface{}, 1, 8)
@@ -2417,11 +2430,11 @@ func (x *bitset256) set(pos byte) {
 
 // ------------
 
-type strBytes struct {
-	s string
-	b []byte
-	// i uint16
-}
+// type strBytes struct {
+// 	s string
+// 	b []byte
+// 	// i uint16
+// }
 
 // ------------
 
@@ -2434,7 +2447,8 @@ type pooler struct {
 	// dn                                 sync.Pool // for decNaked
 	buf1k, buf2k, buf4k, buf8k, buf16k, buf32k, buf64k sync.Pool // for [N]byte
 
-	mapStrU16, mapU16Str, mapU16Bytes, mapU16StrBytes sync.Pool // for Binc
+	mapStrU16, mapU16Str, mapU16Bytes sync.Pool // for Binc
+	// mapU16StrBytes sync.Pool // for Binc
 }
 
 func (p *pooler) init() {
@@ -2459,7 +2473,7 @@ func (p *pooler) init() {
 	p.mapStrU16.New = func() interface{} { return make(map[string]uint16, 16) }
 	p.mapU16Str.New = func() interface{} { return make(map[uint16]string, 16) }
 	p.mapU16Bytes.New = func() interface{} { return make(map[uint16][]byte, 16) }
-	p.mapU16StrBytes.New = func() interface{} { return make(map[uint16]strBytes, 16) }
+	// p.mapU16StrBytes.New = func() interface{} { return make(map[uint16]strBytes, 16) }
 }
 
 // func (p *pooler) sfiRv8() (sp *sync.Pool, v interface{}) {
@@ -2712,23 +2726,32 @@ func (z *sfiRvPooler) get(newlen int) (fkvs []sfiRv) {
 // xdebugf printf. the message in red on the terminal.
 // Use it in place of fmt.Printf (which it calls internally)
 func xdebugf(pattern string, args ...interface{}) {
-	var delim string
-	if len(pattern) > 0 && pattern[len(pattern)-1] != '\n' {
-		delim = "\n"
-	}
-	fmt.Printf("\033[1;31m"+pattern+delim+"\033[0m", args...)
+	xdebugAnyf("31", pattern, args...)
 }
 
 // xdebug2f printf. the message in blue on the terminal.
 // Use it in place of fmt.Printf (which it calls internally)
 func xdebug2f(pattern string, args ...interface{}) {
+	xdebugAnyf("34", pattern, args...)
+}
+
+func xdebugAnyf(colorcode, pattern string, args ...interface{}) {
+	if !xdebug {
+		return
+	}
 	var delim string
 	if len(pattern) > 0 && pattern[len(pattern)-1] != '\n' {
 		delim = "\n"
 	}
-	fmt.Printf("\033[1;34m"+pattern+delim+"\033[0m", args...)
+	fmt.Printf("\033[1;"+colorcode+"m"+pattern+delim+"\033[0m", args...)
+	// os.Stderr.Flush()
 }
 
+// register these here, so that staticcheck stops barfing
+var _ = xdebug2f
+var _ = xdebugf
+var _ = isNaN32
+
 // func isImmutableKind(k reflect.Kind) (v bool) {
 // 	return false ||
 // 		k == reflect.Int ||

codec/helper_unsafe.go

@@ -20,7 +20,13 @@ import (
 // var zeroRTv [4]uintptr
 
 const safeMode = false
-const unsafeFlagIndir = 1 << 7 // keep in sync with GO_ROOT/src/reflect/value.go
+
+// keep in sync with GO_ROOT/src/reflect/value.go
+const (
+	unsafeFlagIndir    = 1 << 7
+	unsafeFlagKindMask = (1 << 5) - 1 // 5 bits for 27 kinds (up to 31)
+	// unsafeTypeKindDirectIface = 1 << 5
+)
 
 type unsafeString struct {
 	Data unsafe.Pointer
@@ -76,6 +82,17 @@ func definitelyNil(v interface{}) bool {
 	return ((*unsafeIntf)(unsafe.Pointer(&v))).word == nil
 }
 
+func rv2ptr(urv *unsafeReflectValue) (ptr unsafe.Pointer) {
+	// true references (map, func, chan, ptr - NOT slice) may be double-referenced? as flagIndir
+	// rv := *((*reflect.Value)(unsafe.Pointer(urv)))
+	if refBitset.isset(byte(urv.flag&unsafeFlagKindMask)) && urv.flag&unsafeFlagIndir != 0 {
+		ptr = *(*unsafe.Pointer)(urv.ptr)
+	} else {
+		ptr = urv.ptr
+	}
+	return
+}
+
 func rv2i(rv reflect.Value) interface{} {
 	// TODO: consider a more generally-known optimization for reflect.Value ==> Interface
 	//
@@ -83,14 +100,7 @@ func rv2i(rv reflect.Value) interface{} {
 	// the source go stdlib reflect/value.go, and trims the implementation.
 
 	urv := (*unsafeReflectValue)(unsafe.Pointer(&rv))
-	// true references (map, func, chan, ptr - NOT slice) may be double-referenced as flagIndir
-	var ptr unsafe.Pointer
-	if refBitset.isset(byte(urv.flag&(1<<5-1))) && urv.flag&unsafeFlagIndir != 0 {
-		ptr = *(*unsafe.Pointer)(urv.ptr)
-	} else {
-		ptr = urv.ptr
-	}
-	return *(*interface{})(unsafe.Pointer(&unsafeIntf{typ: urv.typ, word: ptr}))
+	return *(*interface{})(unsafe.Pointer(&unsafeIntf{typ: urv.typ, word: rv2ptr(urv)}))
 }
 
 func rt2id(rt reflect.Type) uintptr {
@@ -204,7 +214,7 @@ func (x *atomicTypeInfoSlice) store(p []rtid2ti) {
 // --------------------------
 type atomicRtidFnSlice struct {
 	v unsafe.Pointer // *[]codecRtidFn
-	// _ uint64         // padding (atomicXXX expected to be 2 words) (make 1 word so JsonHandle fits)
+	_ uint64         // padding (atomicXXX expected to be 2 words) (make 1 word so JsonHandle fits)
 }
 
 func (x *atomicRtidFnSlice) load() (s []codecRtidFn) {

codec/msgpack.go

@@ -699,7 +699,7 @@ func (d *msgpackDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte)
 		// bsOut, _ = fastpathTV.DecSliceUint8V(bs, true, d.d)
 		slen := d.ReadArrayStart()
 		bs = usableByteSlice(bs, slen)
-		for i := 0; i < slen; i++ {
+		for i := 0; i < len(bs); i++ {
 			bs[i] = uint8(chkOvf.UintV(d.DecodeUint64(), 8))
 		}
 		return bs
@@ -949,6 +949,8 @@ func (d *msgpackDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs
 
 //MsgpackHandle is a Handle for the Msgpack Schema-Free Encoding Format.
 type MsgpackHandle struct {
+	binaryEncodingType
+	noElemSeparators
 	BasicHandle
 
 	// NoFixedNum says to output all signed integers as 2-bytes, never as 1-byte fixednum.
@@ -971,10 +973,7 @@ type MsgpackHandle struct {
 	// PositiveIntUnsigned says to encode positive integers as unsigned.
 	PositiveIntUnsigned bool
 
-	binaryEncodingType
-	noElemSeparators
-
-	_ [1]uint64 // padding (cache-aligned)
+	_ [7]uint64 // padding (cache-aligned)
 }
 
 // Name returns the name of the handle: msgpack

codec/simple.go

@@ -428,7 +428,7 @@ func (d *simpleDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 		// bsOut, _ = fastpathTV.DecSliceUint8V(bs, true, d.d)
 		slen := d.ReadArrayStart()
 		bs = usableByteSlice(bs, slen)
-		for i := 0; i < slen; i++ {
+		for i := 0; i < len(bs); i++ {
 			bs[i] = uint8(chkOvf.UintV(d.DecodeUint64(), 8))
 		}
 		return bs
@@ -605,13 +605,13 @@ func (d *simpleDecDriver) DecodeNaked() {
 //
 // The full spec will be published soon.
 type SimpleHandle struct {
-	BasicHandle
 	binaryEncodingType
 	noElemSeparators
+	BasicHandle
 	// EncZeroValuesAsNil says to encode zero values for numbers, bool, string, etc as nil
 	EncZeroValuesAsNil bool
 
-	_ [1]uint64 // padding (cache-aligned)
+	_ [7]uint64 // padding (cache-aligned)
 }
 
 // Name returns the name of the handle: simple

codec/values_codecgen_generated_test.go

@@ -8748,6 +8748,330 @@ func (x *TestSelfExtImpl) codecDecodeSelfFromArray(l int, d *Decoder) {
 	z.DecReadArrayEnd()
 }
 
+func (x *TestSelfExtImpl2) CodecEncodeSelf(e *Encoder) {
+	var h codecSelfer19780
+	z, r := GenHelperEncoder(e)
+	_, _, _ = h, z, r
+	if x == nil {
+		r.EncodeNil()
+	} else {
+		yysep2 := !z.EncBinary()
+		yy2arr2 := z.EncBasicHandle().StructToArray
+		_, _ = yysep2, yy2arr2
+		const yyr2 bool = false // struct tag has 'toArray'
+		if yyr2 || yy2arr2 {
+			z.EncWriteArrayStart(2)
+			z.EncWriteArrayElem()
+			if z.EncBasicHandle().StringToRaw {
+				r.EncodeStringBytesRaw(z.BytesView(string(x.M)))
+			} else {
+				r.EncodeStringEnc(codecSelferCcUTF819780, string(x.M))
+			}
+			z.EncWriteArrayElem()
+			r.EncodeBool(bool(x.O))
+			z.EncWriteArrayEnd()
+		} else {
+			z.EncWriteMapStart(2)
+			z.EncWriteMapElemKey()
+			if z.IsJSONHandle() {
+				z.WriteStr("\"M\"")
+			} else {
+				r.EncodeStringEnc(codecSelferCcUTF819780, `M`)
+			}
+			z.EncWriteMapElemValue()
+			if z.EncBasicHandle().StringToRaw {
+				r.EncodeStringBytesRaw(z.BytesView(string(x.M)))
+			} else {
+				r.EncodeStringEnc(codecSelferCcUTF819780, string(x.M))
+			}
+			z.EncWriteMapElemKey()
+			if z.IsJSONHandle() {
+				z.WriteStr("\"O\"")
+			} else {
+				r.EncodeStringEnc(codecSelferCcUTF819780, `O`)
+			}
+			z.EncWriteMapElemValue()
+			r.EncodeBool(bool(x.O))
+			z.EncWriteMapEnd()
+		}
+	}
+}
+
+func (x *TestSelfExtImpl2) CodecDecodeSelf(d *Decoder) {
+	var h codecSelfer19780
+	z, r := GenHelperDecoder(d)
+	_, _, _ = h, z, r
+	yyct2 := r.ContainerType()
+	if yyct2 == codecSelferValueTypeMap19780 {
+		yyl2 := z.DecReadMapStart()
+		if yyl2 == 0 {
+			z.DecReadMapEnd()
+		} else {
+			x.codecDecodeSelfFromMap(yyl2, d)
+		}
+	} else if yyct2 == codecSelferValueTypeArray19780 {
+		yyl2 := z.DecReadArrayStart()
+		if yyl2 == 0 {
+			z.DecReadArrayEnd()
+		} else {
+			x.codecDecodeSelfFromArray(yyl2, d)
+		}
+	} else {
+		panic(errCodecSelferOnlyMapOrArrayEncodeToStruct19780)
+	}
+}
+
+func (x *TestSelfExtImpl2) codecDecodeSelfFromMap(l int, d *Decoder) {
+	var h codecSelfer19780
+	z, r := GenHelperDecoder(d)
+	_, _, _ = h, z, r
+	var yyhl3 bool = l >= 0
+	for yyj3 := 0; ; yyj3++ {
+		if yyhl3 {
+			if yyj3 >= l {
+				break
+			}
+		} else {
+			if r.CheckBreak() {
+				break
+			}
+		}
+		z.DecReadMapElemKey()
+		yys3 := z.StringView(r.DecodeStringAsBytes())
+		z.DecReadMapElemValue()
+		switch yys3 {
+		case "M":
+			if r.TryDecodeAsNil() {
+				x.M = ""
+			} else {
+				x.M = (string)(string(r.DecodeStringAsBytes()))
+			}
+		case "O":
+			if r.TryDecodeAsNil() {
+				x.O = false
+			} else {
+				x.O = (bool)(r.DecodeBool())
+			}
+		default:
+			z.DecStructFieldNotFound(-1, yys3)
+		} // end switch yys3
+	} // end for yyj3
+	z.DecReadMapEnd()
+}
+
+func (x *TestSelfExtImpl2) codecDecodeSelfFromArray(l int, d *Decoder) {
+	var h codecSelfer19780
+	z, r := GenHelperDecoder(d)
+	_, _, _ = h, z, r
+	var yyj6 int
+	var yyb6 bool
+	var yyhl6 bool = l >= 0
+	yyj6++
+	if yyhl6 {
+		yyb6 = yyj6 > l
+	} else {
+		yyb6 = r.CheckBreak()
+	}
+	if yyb6 {
+		z.DecReadArrayEnd()
+		return
+	}
+	z.DecReadArrayElem()
+	if r.TryDecodeAsNil() {
+		x.M = ""
+	} else {
+		x.M = (string)(string(r.DecodeStringAsBytes()))
+	}
+	yyj6++
+	if yyhl6 {
+		yyb6 = yyj6 > l
+	} else {
+		yyb6 = r.CheckBreak()
+	}
+	if yyb6 {
+		z.DecReadArrayEnd()
+		return
+	}
+	z.DecReadArrayElem()
+	if r.TryDecodeAsNil() {
+		x.O = false
+	} else {
+		x.O = (bool)(r.DecodeBool())
+	}
+	for {
+		yyj6++
+		if yyhl6 {
+			yyb6 = yyj6 > l
+		} else {
+			yyb6 = r.CheckBreak()
+		}
+		if yyb6 {
+			break
+		}
+		z.DecReadArrayElem()
+		z.DecStructFieldNotFound(yyj6-1, "")
+	}
+	z.DecReadArrayEnd()
+}
+
+func (x *TestTwoNakedInterfaces) CodecEncodeSelf(e *Encoder) {
+	var h codecSelfer19780
+	z, r := GenHelperEncoder(e)
+	_, _, _ = h, z, r
+	if x == nil {
+		r.EncodeNil()
+	} else {
+		yysep2 := !z.EncBinary()
+		yy2arr2 := z.EncBasicHandle().StructToArray
+		_, _ = yysep2, yy2arr2
+		const yyr2 bool = false // struct tag has 'toArray'
+		if yyr2 || yy2arr2 {
+			z.EncWriteArrayStart(2)
+			z.EncWriteArrayElem()
+			z.EncFallback(x.A)
+			z.EncWriteArrayElem()
+			z.EncFallback(x.B)
+			z.EncWriteArrayEnd()
+		} else {
+			z.EncWriteMapStart(2)
+			z.EncWriteMapElemKey()
+			if z.IsJSONHandle() {
+				z.WriteStr("\"A\"")
+			} else {
+				r.EncodeStringEnc(codecSelferCcUTF819780, `A`)
+			}
+			z.EncWriteMapElemValue()
+			z.EncFallback(x.A)
+			z.EncWriteMapElemKey()
+			if z.IsJSONHandle() {
+				z.WriteStr("\"B\"")
+			} else {
+				r.EncodeStringEnc(codecSelferCcUTF819780, `B`)
+			}
+			z.EncWriteMapElemValue()
+			z.EncFallback(x.B)
+			z.EncWriteMapEnd()
+		}
+	}
+}
+
+func (x *TestTwoNakedInterfaces) CodecDecodeSelf(d *Decoder) {
+	var h codecSelfer19780
+	z, r := GenHelperDecoder(d)
+	_, _, _ = h, z, r
+	yyct2 := r.ContainerType()
+	if yyct2 == codecSelferValueTypeMap19780 {
+		yyl2 := z.DecReadMapStart()
+		if yyl2 == 0 {
+			z.DecReadMapEnd()
+		} else {
+			x.codecDecodeSelfFromMap(yyl2, d)
+		}
+	} else if yyct2 == codecSelferValueTypeArray19780 {
+		yyl2 := z.DecReadArrayStart()
+		if yyl2 == 0 {
+			z.DecReadArrayEnd()
+		} else {
+			x.codecDecodeSelfFromArray(yyl2, d)
+		}
+	} else {
+		panic(errCodecSelferOnlyMapOrArrayEncodeToStruct19780)
+	}
+}
+
+func (x *TestTwoNakedInterfaces) codecDecodeSelfFromMap(l int, d *Decoder) {
+	var h codecSelfer19780
+	z, r := GenHelperDecoder(d)
+	_, _, _ = h, z, r
+	var yyhl3 bool = l >= 0
+	for yyj3 := 0; ; yyj3++ {
+		if yyhl3 {
+			if yyj3 >= l {
+				break
+			}
+		} else {
+			if r.CheckBreak() {
+				break
+			}
+		}
+		z.DecReadMapElemKey()
+		yys3 := z.StringView(r.DecodeStringAsBytes())
+		z.DecReadMapElemValue()
+		switch yys3 {
+		case "A":
+			if r.TryDecodeAsNil() {
+				x.A = nil
+			} else {
+				z.DecFallback(&x.A, true)
+			}
+		case "B":
+			if r.TryDecodeAsNil() {
+				x.B = nil
+			} else {
+				z.DecFallback(&x.B, true)
+			}
+		default:
+			z.DecStructFieldNotFound(-1, yys3)
+		} // end switch yys3
+	} // end for yyj3
+	z.DecReadMapEnd()
+}
+
+func (x *TestTwoNakedInterfaces) codecDecodeSelfFromArray(l int, d *Decoder) {
+	var h codecSelfer19780
+	z, r := GenHelperDecoder(d)
+	_, _, _ = h, z, r
+	var yyj8 int
+	var yyb8 bool
+	var yyhl8 bool = l >= 0
+	yyj8++
+	if yyhl8 {
+		yyb8 = yyj8 > l
+	} else {
+		yyb8 = r.CheckBreak()
+	}
+	if yyb8 {
+		z.DecReadArrayEnd()
+		return
+	}
+	z.DecReadArrayElem()
+	if r.TryDecodeAsNil() {
+		x.A = nil
+	} else {
+		z.DecFallback(&x.A, true)
+	}
+	yyj8++
+	if yyhl8 {
+		yyb8 = yyj8 > l
+	} else {
+		yyb8 = r.CheckBreak()
+	}
+	if yyb8 {
+		z.DecReadArrayEnd()
+		return
+	}
+	z.DecReadArrayElem()
+	if r.TryDecodeAsNil() {
+		x.B = nil
+	} else {
+		z.DecFallback(&x.B, true)
+	}
+	for {
+		yyj8++
+		if yyhl8 {
+			yyb8 = yyj8 > l
+		} else {
+			yyb8 = r.CheckBreak()
+		}
+		if yyb8 {
+			break
+		}
+		z.DecReadArrayElem()
+		z.DecStructFieldNotFound(yyj8-1, "")
+	}
+	z.DecReadArrayEnd()
+}
+
 func (x *TestStrucFlex) CodecEncodeSelf(e *Encoder) {
 	var h codecSelfer19780
 	z, r := GenHelperEncoder(e)

codec/values_flex_test.go

@@ -113,7 +113,6 @@ type missingFielderT1 struct {
 }
 
 func (t *missingFielderT1) CodecMissingField(field []byte, value interface{}) bool {
-	// xdebugf(">> calling CodecMissingField with field: %s, value: %v", field, value)
 	switch string(field) {
 	case "F":
 		t.f = value.(float64)

codec/values_test.go

@@ -27,7 +27,7 @@ import (
 // 	defTypeInfos.get(rt2id(rt), rt)
 // }
 
-const numStrUi64T = 4 // TODO: prefer 32
+const numStrUi64T = 32 // use 8, prefer 32, test with 1024
 
 type wrapSliceUint64 []uint64
 type wrapSliceString []string
@@ -346,7 +346,7 @@ func populateTestStrucCommon(ts *TestStrucCommon, n int, bench, useInterface, us
 	}
 
 	for i := uint64(0); i < numStrUi64T; i++ {
-		ss := strings.Repeat(strconv.FormatUint(i, 10), 4)
+		ss := strings.Repeat(strconv.FormatUint(i, 10), int(i)) // 4)
 		ts.SstrUi64T[i] = stringUint64T{S: ss, U: i}
 		ts.MstrUi64T[ss] = &ts.SstrUi64T[i]
 	}

codec/z_all_test.go

@@ -58,9 +58,9 @@ func testSuite(t *testing.T, f func(t *testing.T)) {
 
 	testUseMust = true
 	testUseIoEncDec = 0
-	// xdebugf("setting StructToArray=true")
 	testUseReset = true
 
+	// xdebugf("with StructToArray=true")
 	testDecodeOptions.InternString = true
 	testDecodeOptions.MapValueReset = true
 	// testDecodeOptions.SignedInteger = true
@@ -320,6 +320,7 @@ func testNonHandlesGroup(t *testing.T) {
 	t.Run("TestAllAnonCycle", TestAllAnonCycle)
 	t.Run("TestMultipleEncDec", TestMultipleEncDec)
 	t.Run("TestAllErrWriter", TestAllErrWriter)
+	t.Run("TestMapRangeIndex", TestMapRangeIndex)
 }
 
 func TestCodecSuite(t *testing.T) {