codec: streamline nil decoding: nil means zero value consistently

Previously, we had a TryDecodeAsNil that all formats implemented.

However, the same formats did double duty by still checking if nil in stream
when decoding a value.

We introduce TryNil (replacing TryDecodeAsNil) which is used iff
the type is a pointer and we need to set it to null if null was seen in the stream.

Now, nil means zero value consistently, regardless of the context.
In other words, a 'nil' in a stream means to reset the value to its zero state.

A side effect of this is that the decode option 'DeleteOnNilMapValue`
is a no-op. It is kept for compiling compatibility but has no effect
during decoding.

The main benefits are in the architecture
- TryNil is only called if updating a pointer value
- Nil is only handled at container boundaries: map, slice, null
  i.e. only when calling decDriver.ContainerType or decDriver.Read(Map|Array)Len

----

Also, reduce size of typeInfo by using a bitset flag instead of multiple bools.

This gives us space (within 3*8 words) to store a reflect.Zero value
in each typeInfo. This saves some allocation done when calling reflect.Zero(Type).

----

Add some code cleanup and re-organization.

README.md

@@ -262,7 +262,7 @@ some caveats. See Encode documentation.
 
 ```go
 const CborStreamBytes byte = 0x5f ...
-const GenVersion = 13
+const GenVersion = 14
 var SelfExt = &extFailWrapper{}
 var GoRpc goRpc
 var MsgpackSpecRpc msgpackSpecRpc

codec/bench/shared_test.go

@@ -276,12 +276,13 @@ func sTestCodecDecode(bs []byte, ts interface{}, h Handle, bh *BasicHandle) (err
 // These are for intormational messages that do not necessarily
 // help with diagnosing a failure, or which are too large.
 func logTv(x interface{}, format string, args ...interface{}) {
-	if testVerbose {
-		if t, ok := x.(testing.TB); ok { // only available from go 1.9
-			t.Helper()
-		}
-		logT(x, format, args...)
+	if !testVerbose {
+		return
+	}
+	if t, ok := x.(testing.TB); ok { // only available from go 1.9
+		t.Helper()
 	}
+	logT(x, format, args...)
 }
 
 // logT logs messages when running as go test -v
@@ -304,12 +305,22 @@ func logT(x interface{}, format string, args ...interface{}) {
 	}
 }
 
-func failT(x interface{}, args ...interface{}) {
-	t, ok := x.(testing.TB) // only available from go 1.9
-	if ok {
-		t.Helper()
+func failTv(x testing.TB, args ...interface{}) {
+	x.Helper()
+	if testVerbose {
+		failTMsg(x, args...)
 	}
+	x.FailNow()
+}
 
+func failT(x testing.TB, args ...interface{}) {
+	x.Helper()
+	failTMsg(x, args...)
+	x.FailNow()
+}
+
+func failTMsg(x testing.TB, args ...interface{}) {
+	x.Helper()
 	if len(args) > 0 {
 		if format, ok := args[0].(string); ok {
 			logT(x, format, args[1:]...)
@@ -319,9 +330,6 @@ func failT(x interface{}, args ...interface{}) {
 			logT(x, "%v", args)
 		}
 	}
-	if ok {
-		t.FailNow()
-	}
 }
 
 // --- functions below are used only by benchmarks alone

codec/binc.go

@@ -394,6 +394,8 @@ type bincDecDriver struct {
 	bd     byte
 	vd     byte
 	vs     byte
+
+	fnil bool
 	// _      [3]byte // padding
 	// linear searching on this slice is ok,
 	// because we typically expect < 32 symbols in each stream.
@@ -419,11 +421,36 @@ func (d *bincDecDriver) uncacheRead() {
 	}
 }
 
+func (d *bincDecDriver) advanceNil() (null bool) {
+	d.fnil = false
+	if !d.bdRead {
+		d.readNextBd()
+	}
+	if d.bd == bincVdSpecial<<4|bincSpNil {
+		d.bdRead = false
+		d.fnil = true
+		null = true
+	}
+	return
+}
+
+func (d *bincDecDriver) Nil() bool {
+	return d.fnil
+}
+
+func (d *bincDecDriver) TryNil() bool {
+	return d.advanceNil()
+}
+
 func (d *bincDecDriver) ContainerType() (vt valueType) {
 	if !d.bdRead {
 		d.readNextBd()
 	}
-	if d.vd == bincVdSpecial && d.vs == bincSpNil {
+	d.fnil = false
+	// if d.vd == bincVdSpecial && d.vs == bincSpNil {
+	if d.bd == bincVdSpecial<<4|bincSpNil {
+		d.bdRead = false
+		d.fnil = true
 		return valueTypeNil
 	} else if d.vd == bincVdByteArray {
 		return valueTypeBytes
@@ -440,23 +467,8 @@ func (d *bincDecDriver) ContainerType() (vt valueType) {
 	return valueTypeUnset
 }
 
-func (d *bincDecDriver) TryDecodeAsNil() bool {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == bincVdSpecial<<4|bincSpNil {
-		d.bdRead = false
-		return true
-	}
-	return false
-}
-
 func (d *bincDecDriver) DecodeTime() (t time.Time) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == bincVdSpecial<<4|bincSpNil {
-		d.bdRead = false
+	if d.advanceNil() {
 		return
 	}
 	if d.vd != bincVdTimestamp {
@@ -536,9 +548,6 @@ func (d *bincDecDriver) decUint() (v uint64) {
 }
 
 func (d *bincDecDriver) decCheckInteger() (ui uint64, neg bool) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
 	vd, vs := d.vd, d.vs
 	if vd == bincVdPosInt {
 		ui = d.decUint()
@@ -566,6 +575,9 @@ func (d *bincDecDriver) decCheckInteger() (ui uint64, neg bool) {
 }
 
 func (d *bincDecDriver) DecodeInt64() (i int64) {
+	if d.advanceNil() {
+		return
+	}
 	ui, neg := d.decCheckInteger()
 	i = chkOvf.SignedIntV(ui)
 	if neg {
@@ -576,6 +588,9 @@ func (d *bincDecDriver) DecodeInt64() (i int64) {
 }
 
 func (d *bincDecDriver) DecodeUint64() (ui uint64) {
+	if d.advanceNil() {
+		return
+	}
 	ui, neg := d.decCheckInteger()
 	if neg {
 		d.d.errorf("assigning negative signed value to unsigned integer type")
@@ -586,8 +601,8 @@ func (d *bincDecDriver) DecodeUint64() (ui uint64) {
 }
 
 func (d *bincDecDriver) DecodeFloat64() (f float64) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	vd, vs := d.vd, d.vs
 	if vd == bincVdSpecial {
@@ -616,12 +631,12 @@ func (d *bincDecDriver) DecodeFloat64() (f float64) {
 
 // bool can be decoded from bool only (single byte).
 func (d *bincDecDriver) DecodeBool() (b bool) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
-	if bd := d.bd; bd == (bincVdSpecial | bincSpFalse) {
+	if d.bd == (bincVdSpecial | bincSpFalse) {
 		// b = false
-	} else if bd == (bincVdSpecial | bincSpTrue) {
+	} else if d.bd == (bincVdSpecial | bincSpTrue) {
 		b = true
 	} else {
 		d.d.errorf("bool - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs))
@@ -632,8 +647,8 @@ func (d *bincDecDriver) DecodeBool() (b bool) {
 }
 
 func (d *bincDecDriver) ReadMapStart() (length int) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return decContainerLenNil
 	}
 	if d.vd != bincVdMap {
 		d.d.errorf("map - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs))
@@ -645,8 +660,8 @@ func (d *bincDecDriver) ReadMapStart() (length int) {
 }
 
 func (d *bincDecDriver) ReadArrayStart() (length int) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return decContainerLenNil
 	}
 	if d.vd != bincVdArray {
 		d.d.errorf("array - %s %x-%x/%s", msgBadDesc, d.vd, d.vs, bincdesc(d.vd, d.vs))
@@ -681,11 +696,7 @@ func (d *bincDecDriver) decLenNumber() (v uint64) {
 }
 
 func (d *bincDecDriver) decStringBytes(bs []byte, zerocopy bool) (bs2 []byte) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == bincVdSpecial<<4|bincSpNil {
-		d.bdRead = false
+	if d.advanceNil() {
 		return
 	}
 	var slen = -1
@@ -756,12 +767,8 @@ func (d *bincDecDriver) DecodeStringAsBytes() (s []byte) {
 }
 
 func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == bincVdSpecial<<4|bincSpNil {
-		d.bdRead = false
-		return nil
+	if d.advanceNil() {
+		return
 	}
 	// check if an "array" of uint8's (see ContainerType for how to infer if an array)
 	if d.vd == bincVdArray {
@@ -794,13 +801,16 @@ func (d *bincDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 	return decByteSlice(d.r, clen, d.d.h.MaxInitLen, bs)
 }
 
-func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
+func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) {
 	if xtag > 0xff {
 		d.d.errorf("ext: tag must be <= 0xff; got: %v", xtag)
 		return
 	}
+	if d.advanceNil() {
+		return
+	}
 	realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag))
-	realxtag = uint64(realxtag1)
+	realxtag := uint64(realxtag1)
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = realxtag
@@ -810,13 +820,9 @@ func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 	} else {
 		ext.ReadExt(rv, xbs)
 	}
-	return
 }
 
 func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
 	if d.vd == bincVdCustomExt {
 		l := d.decLen()
 		xtag = d.r.readn1()
@@ -845,6 +851,7 @@ func (d *bincDecDriver) DecodeNaked() {
 		d.readNextBd()
 	}
 
+	d.fnil = false
 	n := d.d.naked()
 	var decodeFurther bool
 
@@ -853,6 +860,7 @@ func (d *bincDecDriver) DecodeNaked() {
 		switch d.vs {
 		case bincSpNil:
 			n.v = valueTypeNil
+			d.fnil = true
 		case bincSpFalse:
 			n.v = valueTypeBool
 			n.b = false
@@ -1014,6 +1022,7 @@ func (d *bincDecDriver) reset() {
 	d.r, d.br = d.d.r(), d.d.bytes
 	d.s = nil
 	d.bd, d.bdRead, d.vd, d.vs = 0, false, 0, 0
+	d.fnil = false
 }
 
 func (d *bincDecDriver) atEndOfDecode() {

codec/cbor.go

@@ -304,6 +304,7 @@ type cborDecDriver struct {
 	bdRead bool
 	bd     byte
 	st     bool // skip tags
+	fnil   bool // found nil
 	noBuiltInTypes
 	// decNoSeparator
 	decDriverNoopContainerReader
@@ -340,6 +341,19 @@ func (d *cborDecDriver) readNextBd() {
 	d.bdRead = true
 }
 
+func (d *cborDecDriver) advanceNil() (null bool) {
+	d.fnil = false
+	if !d.bdRead {
+		d.readNextBd()
+	}
+	if d.bd == cborBdNil || d.bd == cborBdUndefined {
+		d.bdRead = false
+		d.fnil = true
+		null = true
+	}
+	return
+}
+
 // skipTags is called to skip any tags in the stream.
 //
 // Since any value can be tagged, then we should call skipTags
@@ -362,6 +376,7 @@ func (d *cborDecDriver) uncacheRead() {
 }
 
 func (d *cborDecDriver) ContainerType() (vt valueType) {
+	d.fnil = false
 	if !d.bdRead {
 		d.readNextBd()
 	}
@@ -369,6 +384,8 @@ func (d *cborDecDriver) ContainerType() (vt valueType) {
 		d.skipTags()
 	}
 	if d.bd == cborBdNil {
+		d.bdRead = false // always consume nil after seeing it in container type
+		d.fnil = true
 		return valueTypeNil
 	} else if d.bd == cborBdIndefiniteBytes || (d.bd>>5 == cborMajorBytes) {
 		return valueTypeBytes
@@ -385,16 +402,12 @@ func (d *cborDecDriver) ContainerType() (vt valueType) {
 	return valueTypeUnset
 }
 
-func (d *cborDecDriver) TryDecodeAsNil() bool {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	// treat Nil and Undefined as nil values
-	if d.bd == cborBdNil || d.bd == cborBdUndefined {
-		d.bdRead = false
-		return true
-	}
-	return false
+func (d *cborDecDriver) Nil() bool {
+	return d.fnil
+}
+
+func (d *cborDecDriver) TryNil() bool {
+	return d.advanceNil()
 }
 
 func (d *cborDecDriver) CheckBreak() (v bool) {
@@ -430,9 +443,6 @@ func (d *cborDecDriver) decUint() (ui uint64) {
 }
 
 func (d *cborDecDriver) decCheckInteger() (neg bool) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
 	if d.st {
 		d.skipTags()
 	}
@@ -447,13 +457,6 @@ func (d *cborDecDriver) decCheckInteger() (neg bool) {
 	return
 }
 
-func (d *cborDecDriver) DecodeInt64() (i int64) {
-	neg := d.decCheckInteger()
-	ui := d.decUint()
-	d.bdRead = false
-	return cborDecInt64(ui, neg)
-}
-
 func cborDecInt64(ui uint64, neg bool) (i int64) {
 	// check if this number can be converted to an int without overflow
 	if neg {
@@ -464,7 +467,49 @@ func cborDecInt64(ui uint64, neg bool) (i int64) {
 	return
 }
 
+func (d *cborDecDriver) decLen() int {
+	return int(d.decUint())
+}
+
+func (d *cborDecDriver) decAppendIndefiniteBytes(bs []byte) []byte {
+	d.bdRead = false
+	for !d.CheckBreak() {
+		if major := d.bd >> 5; major != cborMajorBytes && major != cborMajorString {
+			d.d.errorf("invalid indefinite string/bytes; got major %v, expected %x/%s",
+				major, d.bd, cbordesc(d.bd))
+		}
+		n := uint(d.decLen())
+		oldLen := uint(len(bs))
+		newLen := oldLen + n
+		if newLen > uint(cap(bs)) {
+			bs2 := make([]byte, newLen, 2*uint(cap(bs))+n)
+			copy(bs2, bs)
+			bs = bs2
+		} else {
+			bs = bs[:newLen]
+		}
+		d.r.readb(bs[oldLen:newLen])
+		// bs = append(bs, d.r.readn()...)
+		d.bdRead = false
+	}
+	d.bdRead = false
+	return bs
+}
+
+func (d *cborDecDriver) DecodeInt64() (i int64) {
+	if d.advanceNil() {
+		return
+	}
+	neg := d.decCheckInteger()
+	ui := d.decUint()
+	d.bdRead = false
+	return cborDecInt64(ui, neg)
+}
+
 func (d *cborDecDriver) DecodeUint64() (ui uint64) {
+	if d.advanceNil() {
+		return
+	}
 	if d.decCheckInteger() {
 		d.d.errorf("cannot assign negative signed value to unsigned type")
 	}
@@ -474,8 +519,8 @@ func (d *cborDecDriver) DecodeUint64() (ui uint64) {
 }
 
 func (d *cborDecDriver) DecodeFloat64() (f float64) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	if d.st {
 		d.skipTags()
@@ -504,8 +549,8 @@ func (d *cborDecDriver) DecodeFloat64() (f float64) {
 
 // bool can be decoded from bool only (single byte).
 func (d *cborDecDriver) DecodeBool() (b bool) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	if d.st {
 		d.skipTags()
@@ -522,15 +567,15 @@ func (d *cborDecDriver) DecodeBool() (b bool) {
 }
 
 func (d *cborDecDriver) ReadMapStart() (length int) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return decContainerLenNil
 	}
 	if d.st {
 		d.skipTags()
 	}
 	d.bdRead = false
 	if d.bd == cborBdIndefiniteMap {
-		return -1
+		return decContainerLenUnknown
 	}
 	if d.bd>>5 != cborMajorMap {
 		d.d.errorf("error reading map; got major type: %x, expected %x/%s",
@@ -540,15 +585,15 @@ func (d *cborDecDriver) ReadMapStart() (length int) {
 }
 
 func (d *cborDecDriver) ReadArrayStart() (length int) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return decContainerLenNil
 	}
 	if d.st {
 		d.skipTags()
 	}
 	d.bdRead = false
 	if d.bd == cborBdIndefiniteArray {
-		return -1
+		return decContainerLenUnknown
 	}
 	if d.bd>>5 != cborMajorArray {
 		d.d.errorf("invalid array; got major type: %x, expect: %x/%s",
@@ -557,46 +602,13 @@ func (d *cborDecDriver) ReadArrayStart() (length int) {
 	return d.decLen()
 }
 
-func (d *cborDecDriver) decLen() int {
-	return int(d.decUint())
-}
-
-func (d *cborDecDriver) decAppendIndefiniteBytes(bs []byte) []byte {
-	d.bdRead = false
-	for !d.CheckBreak() {
-		if major := d.bd >> 5; major != cborMajorBytes && major != cborMajorString {
-			d.d.errorf("invalid indefinite string/bytes; got major %v, expected %x/%s",
-				major, d.bd, cbordesc(d.bd))
-		}
-		n := uint(d.decLen())
-		oldLen := uint(len(bs))
-		newLen := oldLen + n
-		if newLen > uint(cap(bs)) {
-			bs2 := make([]byte, newLen, 2*uint(cap(bs))+n)
-			copy(bs2, bs)
-			bs = bs2
-		} else {
-			bs = bs[:newLen]
-		}
-		d.r.readb(bs[oldLen:newLen])
-		// bs = append(bs, d.r.readn()...)
-		d.bdRead = false
-	}
-	d.bdRead = false
-	return bs
-}
-
 func (d *cborDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	if d.st {
 		d.skipTags()
 	}
-	if d.bd == cborBdNil || d.bd == cborBdUndefined {
-		d.bdRead = false
-		return nil
-	}
 	if d.bd == cborBdIndefiniteBytes || d.bd == cborBdIndefiniteString {
 		d.bdRead = false
 		if bs == nil {
@@ -656,11 +668,7 @@ func (d *cborDecDriver) DecodeStringAsBytes() (s []byte) {
 }
 
 func (d *cborDecDriver) DecodeTime() (t time.Time) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == cborBdNil || d.bd == cborBdUndefined {
-		d.bdRead = false
+	if d.advanceNil() {
 		return
 	}
 	if d.bd>>5 != cborMajorTag {
@@ -705,16 +713,15 @@ func (d *cborDecDriver) decodeTime(xtag uint64) (t time.Time) {
 	return
 }
 
-func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
-	if !d.bdRead {
-		d.readNextBd()
+func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) {
+	if d.advanceNil() {
+		return
 	}
 	if d.bd>>5 != cborMajorTag {
 		d.d.errorf("error reading tag; expected major type: %x, got: %x", cborMajorTag, d.bd>>5)
 	}
-	u := d.decUint()
+	realxtag := d.decUint()
 	d.bdRead = false
-	realxtag = u
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = realxtag
@@ -729,7 +736,6 @@ func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 		d.d.interfaceExtConvertAndDecode(rv, ext)
 	}
 	d.bdRead = false
-	return
 }
 
 func (d *cborDecDriver) DecodeNaked() {
@@ -737,6 +743,7 @@ func (d *cborDecDriver) DecodeNaked() {
 		d.readNextBd()
 	}
 
+	d.fnil = false
 	n := d.d.naked()
 	var decodeFurther bool
 
@@ -782,8 +789,9 @@ func (d *cborDecDriver) DecodeNaked() {
 		// decodeFurther = true
 	case cborMajorSimpleOrFloat:
 		switch d.bd {
-		case cborBdNil:
+		case cborBdNil, cborBdUndefined:
 			n.v = valueTypeNil
+			d.fnil = true
 		case cborBdFalse:
 			n.v = valueTypeBool
 			n.b = false
@@ -875,7 +883,9 @@ func (e *cborEncDriver) reset() {
 
 func (d *cborDecDriver) reset() {
 	d.r, d.br = d.d.r(), d.d.bytes
-	d.bd, d.bdRead = 0, false
+	d.bd = 0
+	d.bdRead = false
+	d.fnil = false
 	d.st = d.h.SkipUnexpectedTags
 }
 

codec/codec_test.go

@@ -717,24 +717,30 @@ func testMarshal(v interface{}, h Handle) (bs []byte, err error) {
 }
 
 func testMarshalErr(v interface{}, h Handle, t *testing.T, name string) (bs []byte) {
+	t.Helper()
 	bs, err := testMarshal(v, h)
 	if err != nil {
+		logT("%s: marshal failed: %v", name, err)
 		failT(t, "Error encoding %s: %v, Err: %v", name, v, err)
 	}
 	return
 }
 
 func testUnmarshalErr(v interface{}, data []byte, h Handle, t *testing.T, name string) {
+	t.Helper()
 	if err := testUnmarshal(v, data, h); err != nil {
-		failT(t, "Error Decoding into %s: %v, Err: %v", name, v, err)
+		logT("%s: unmarshal failed: %v", name, err)
+		failTv(t, "Error Decoding into %s: %v, Err: %v", name, v, err)
 	}
 }
 
 func testDeepEqualErr(v1, v2 interface{}, t *testing.T, name string) {
+	t.Helper()
 	if err := deepEqual(v1, v2); err == nil {
 		logTv(t, "%s: values equal", name)
 	} else {
-		failT(t, "%s: values not equal: %v. 1: %#v, 2: %#v", name, err, v1, v2)
+		logT(t, "%s: values not equal: %v", name, err)
+		failTv(t, "%s: values not equal: %v. 1: %#v, 2: %#v", name, err, v1, v2)
 	}
 }
 
@@ -778,7 +784,7 @@ func doTestCodecTableOne(t *testing.T, testNil bool, h Handle,
 		if h.isBinary() {
 			logTv(t, "         Encoded bytes: len: %v, %v\n", len(b0), b1)
 		} else {
-			logTv(t, "         Encoded string: len: %v, %v\n", len(b0), string(b1))
+			logTv(t, "         Encoded string: len: %v, %s\n", len(b0), b0)
 			// println("########### encoded string: " + string(b0))
 		}
 		var v1 interface{}

codec/encode.go

@@ -216,11 +216,43 @@ func (e *Encoder) kErr(f *codecFnInfo, rv reflect.Value) {
 	e.errorf("unsupported kind %s, for %#v", rv.Kind(), rv)
 }
 
+func chanToSlice(rv reflect.Value, rtelem reflect.Type, timeout time.Duration) (rvcs reflect.Value) {
+	// TODO: ensure this doesn't mess up anywhere that rv of kind chan is expected
+	rvcs = reflect.Zero(reflect.SliceOf(rtelem))
+	if timeout < 0 { // consume until close
+		for {
+			recv, recvOk := rv.Recv()
+			if !recvOk {
+				break
+			}
+			rvcs = reflect.Append(rvcs, recv)
+		}
+	} else {
+		cases := make([]reflect.SelectCase, 2)
+		cases[0] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: rv}
+		if timeout == 0 {
+			cases[1] = reflect.SelectCase{Dir: reflect.SelectDefault}
+		} else {
+			tt := time.NewTimer(timeout)
+			cases[1] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(tt.C)}
+		}
+		for {
+			chosen, recv, recvOk := reflect.Select(cases)
+			if chosen == 1 || !recvOk {
+				break
+			}
+			rvcs = reflect.Append(rvcs, recv)
+		}
+	}
+	return
+}
+
 func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 	// array may be non-addressable, so we have to manage with care
 	//   (don't call rv.Bytes, rv.Slice, etc).
 	// E.g. type struct S{B [2]byte};
 	//   Encode(S{}) will bomb on "panic: slice of unaddressable array".
+	mbs := f.ti.mbs
 	if f.seq != seqTypeArray {
 		if rvisnil(rv) {
 			e.e.EncodeNil()
@@ -228,7 +260,7 @@ func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 		}
 		// If in this method, then there was no extension function defined.
 		// So it's okay to treat as []byte.
-		if f.ti.rtid == uint8SliceTypId {
+		if !mbs && f.ti.rtid == uint8SliceTypId {
 			e.e.EncodeStringBytesRaw(rv.Bytes())
 			return
 		}
@@ -236,48 +268,40 @@ func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 	if f.seq == seqTypeChan && f.ti.chandir&uint8(reflect.RecvDir) == 0 {
 		e.errorf("send-only channel cannot be encoded")
 	}
-	mbs := f.ti.mbs
 	rtelem := f.ti.elem
 
+	var l int
 	// if a slice, array or chan of bytes, treat specially
 	if !mbs && uint8TypId == rt2id(rtelem) { // NOT rtelem.Kind() == reflect.Uint8
-		e.kSliceBytes(rv, f.seq)
+		switch f.seq {
+		case seqTypeSlice:
+			e.e.EncodeStringBytesRaw(rv.Bytes())
+		case seqTypeArray:
+			l = rv.Len()
+			if rv.CanAddr() {
+				e.e.EncodeStringBytesRaw(rv.Slice(0, l).Bytes())
+			} else {
+				var bs []byte
+				if l <= cap(e.b) {
+					bs = e.b[:l]
+				} else {
+					bs = make([]byte, l)
+				}
+				reflect.Copy(reflect.ValueOf(bs), rv)
+				e.e.EncodeStringBytesRaw(bs)
+			}
+		case seqTypeChan:
+			e.kSliceBytesChan(rv)
+		}
 		return
 	}
 
 	// if chan, consume chan into a slice, and work off that slice.
 	if f.seq == seqTypeChan {
-		rvcs := reflect.Zero(reflect.SliceOf(rtelem))
-		timeout := e.h.ChanRecvTimeout
-		if timeout < 0 { // consume until close
-			for {
-				recv, recvOk := rv.Recv()
-				if !recvOk {
-					break
-				}
-				rvcs = reflect.Append(rvcs, recv)
-			}
-		} else {
-			cases := make([]reflect.SelectCase, 2)
-			cases[0] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: rv}
-			if timeout == 0 {
-				cases[1] = reflect.SelectCase{Dir: reflect.SelectDefault}
-			} else {
-				tt := time.NewTimer(timeout)
-				cases[1] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(tt.C)}
-			}
-			for {
-				chosen, recv, recvOk := reflect.Select(cases)
-				if chosen == 1 || !recvOk {
-					break
-				}
-				rvcs = reflect.Append(rvcs, recv)
-			}
-		}
-		rv = rvcs // TODO: ensure this doesn't mess up anywhere that rv of kind chan is expected
+		rv = chanToSlice(rv, rtelem, e.h.ChanRecvTimeout)
 	}
 
-	var l = rv.Len() // rv may be slice or array
+	l = rv.Len() // rv may be slice or array
 	if mbs {
 		if l%2 == 1 {
 			e.errorf("mapBySlice requires even slice length, but got %v", l)
@@ -320,71 +344,53 @@ func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 	}
 }
 
-func (e *Encoder) kSliceBytes(rv reflect.Value, seq seqType) {
-	switch seq {
-	case seqTypeSlice:
-		e.e.EncodeStringBytesRaw(rv.Bytes())
-	case seqTypeArray:
-		var l = rv.Len()
-		if rv.CanAddr() {
-			e.e.EncodeStringBytesRaw(rv.Slice(0, l).Bytes())
-		} else {
-			var bs []byte
-			if l <= cap(e.b) {
-				bs = e.b[:l]
-			} else {
-				bs = make([]byte, l)
-			}
-			reflect.Copy(reflect.ValueOf(bs), rv)
-			e.e.EncodeStringBytesRaw(bs)
-		}
-	case seqTypeChan:
-		// do not use range, so that the number of elements encoded
-		// does not change, and encoding does not hang waiting on someone to close chan.
-		// for b := range rv2i(rv).(<-chan byte) { bs = append(bs, b) }
-		// ch := rv2i(rv).(<-chan byte) // fix error - that this is a chan byte, not a <-chan byte.
+func (e *Encoder) kSliceBytesChan(rv reflect.Value) {
+	// do not use range, so that the number of elements encoded
+	// does not change, and encoding does not hang waiting on someone to close chan.
+	// for b := range rv2i(rv).(<-chan byte) { bs = append(bs, b) }
+	// ch := rv2i(rv).(<-chan byte) // fix error - that this is a chan byte, not a <-chan byte.
 
-		if rvisnil(rv) {
-			e.e.EncodeNil()
-			break
-		}
-		bs := e.b[:0]
-		irv := rv2i(rv)
-		ch, ok := irv.(<-chan byte)
-		if !ok {
-			ch = irv.(chan byte)
-		}
+	// if rvisnil(rv) {
+	// 	e.e.EncodeNil()
+	// 	return
+	// }
 
-	L1:
-		switch timeout := e.h.ChanRecvTimeout; {
-		case timeout == 0: // only consume available
-			for {
-				select {
-				case b := <-ch:
-					bs = append(bs, b)
-				default:
-					break L1
-				}
-			}
-		case timeout > 0: // consume until timeout
-			tt := time.NewTimer(timeout)
-			for {
-				select {
-				case b := <-ch:
-					bs = append(bs, b)
-				case <-tt.C:
-					// close(tt.C)
-					break L1
-				}
+	bs := e.b[:0]
+	irv := rv2i(rv)
+	ch, ok := irv.(<-chan byte)
+	if !ok {
+		ch = irv.(chan byte)
+	}
+
+L1:
+	switch timeout := e.h.ChanRecvTimeout; {
+	case timeout == 0: // only consume available
+		for {
+			select {
+			case b := <-ch:
+				bs = append(bs, b)
+			default:
+				break L1
 			}
-		default: // consume until close
-			for b := range ch {
+		}
+	case timeout > 0: // consume until timeout
+		tt := time.NewTimer(timeout)
+		for {
+			select {
+			case b := <-ch:
 				bs = append(bs, b)
+			case <-tt.C:
+				// close(tt.C)
+				break L1
 			}
 		}
-
-		e.e.EncodeStringBytesRaw(bs)
+	default: // consume until close
+		for b := range ch {
+			bs = append(bs, b)
+		}
 	}
+
+	e.e.EncodeStringBytesRaw(bs)
 }
 
 func (e *Encoder) kStructNoOmitempty(f *codecFnInfo, rv reflect.Value) {
@@ -416,11 +422,11 @@ func (e *Encoder) kStruct(f *codecFnInfo, rv reflect.Value) {
 	var newlen int
 	toMap := !(f.ti.toArray || e.h.StructToArray)
 	var mf map[string]interface{}
-	if f.ti.mf {
+	if f.ti.isFlag(tiflagMissingFielder) {
 		mf = rv2i(rv).(MissingFielder).CodecMissingFields()
 		toMap = true
 		newlen += len(mf)
-	} else if f.ti.mfp {
+	} else if f.ti.isFlag(tiflagMissingFielderPtr) {
 		if rv.CanAddr() {
 			mf = rv2i(rv.Addr()).(MissingFielder).CodecMissingFields()
 		} else {
@@ -1080,7 +1086,7 @@ func (e *Encoder) encode(iv interface{}) {
 		return
 	}
 
-	rv, ok := isNilRef(iv)
+	rv, ok := isNil(iv)
 	if ok {
 		e.e.EncodeNil()
 		return
@@ -1133,11 +1139,7 @@ func (e *Encoder) encode(iv interface{}) {
 	case time.Time:
 		e.e.EncodeTime(v)
 	case []uint8:
-		if v == nil {
-			e.e.EncodeNil()
-		} else {
-			e.e.EncodeStringBytesRaw(v)
-		}
+		e.e.EncodeStringBytesRaw(v)
 	case *Raw:
 		e.rawBytes(*v)
 	case *string:

codec/fast-path.go.tmpl

@@ -31,6 +31,26 @@ package codec
 //	 // decoding into p2 will bomb if fast track functions do not treat like unaddressable.
 // 
 
+{{/*
+fastpathEncMapStringUint64R  (called by fastpath...switch)
+EncMapStringUint64V (called by codecgen)
+
+fastpathEncSliceBoolR: (called by fastpath...switch) (checks f.ti.mbs and calls one of them below)
+EncSliceBoolV  (also called by codecgen)
+EncAsMapSliceBoolV (delegate when mapbyslice=true)
+
+fastpathDecSliceIntfR (called by fastpath...switch) (calls Y or N below depending on if it can be updated)
+DecSliceIntfX  (called by codecgen) (calls Y below)
+DecSliceIntfY  (delegate when slice CAN be updated)
+DecSliceIntfN  (delegate when slice CANNOT be updated e.g. from array or non-addressable slice)
+
+fastpathDecMap...R (called by fastpath...switch) (calls L below)
+DecMap...X  (called by codecgen)
+DecMap...L  (delegated to by both above)
+    (update: let both handle when containerLen == 0 or decContainerLenNil)
+    (L doesn't do mapStart or mapEnd - just the meat)
+*/ -}}
+
 import (
 	"reflect"
 	"sort"
@@ -113,11 +133,7 @@ func fastpathEncodeTypeSwitch(iv interface{}, e *Encoder) bool {
 	switch v := iv.(type) {
 {{range .Values}}{{if not .Primitive}}{{if not .MapKey }}{{if ne .Elem "uint8" -}}
 	case []{{ .Elem }}:
-		if v == nil {
-			e.e.EncodeNil()
-		} else {
-			fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, e)
-		}
+		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, e)
 	case *[]{{ .Elem }}:
 		if *v == nil {
 			e.e.EncodeNil()
@@ -257,9 +273,15 @@ func fastpathDecodeTypeSwitch(iv interface{}, d *Decoder) bool {
 // maps only change if nil, and in that case, there's no point copying
 */ -}}
 	case map[{{ .MapKey }}]{{ .Elem }}:
-		fastpathTV.{{ .MethodNamePfx "Dec" false }}L(v, d.mapStart(), d)
+		if containerLen = d.mapStart(); containerLen != decContainerLenNil {
+			fastpathTV.{{ .MethodNamePfx "Dec" false }}L(v, containerLen, d)
+		}
 	case *map[{{ .MapKey }}]{{ .Elem }}:
 		containerLen = d.mapStart()
+		if containerLen == decContainerLenNil {
+			*v = nil
+			break
+		}
 		if *v == nil {
 			*v = make(map[{{ .MapKey }}]{{ .Elem }}, decInferLen(containerLen, d.h.MaxInitLen, {{ .Size }}))
 		}
@@ -310,6 +332,10 @@ func (f fastpathT) {{ .MethodNamePfx "Dec" false }}X(vp *[]{{ .Elem }}, d *Decod
 }
 func (fastpathT) {{ .MethodNamePfx "Dec" false }}Y(v []{{ .Elem }}, d *Decoder) (_ []{{ .Elem }}, changed bool) {
 	slh, containerLenS := d.decSliceHelperStart()
+	if slh.IsNil {
+		if v == nil { return }
+		return nil, true
+	}
 	if containerLenS == 0 {
 		if v == nil { v = []{{ .Elem }}{} } else if len(v) != 0 { v = v[:0] }
 		slh.End()
@@ -348,11 +374,13 @@ func (fastpathT) {{ .MethodNamePfx "Dec" false }}Y(v []{{ .Elem }}, d *Decoder)
 			changed = true
 		} 
 		slh.ElemContainerState(j)
+		{{/*
 		if d.d.TryDecodeAsNil() {
 			v[uint(j)] = {{ zerocmd .Elem }}
 		} else {
 			{{ if eq .Elem "interface{}" }}d.decode(&v[uint(j)]){{ else }}v[uint(j)] = {{ decmd .Elem }}{{ end }}
-		}
+		} */ -}}
+		{{ if eq .Elem "interface{}" }}d.decode(&v[uint(j)]){{ else }}v[uint(j)] = {{ decmd .Elem }}{{ end }}
 	}
 	if j < len(v) {
 		v = v[:uint(j)]
@@ -366,6 +394,9 @@ func (fastpathT) {{ .MethodNamePfx "Dec" false }}Y(v []{{ .Elem }}, d *Decoder)
 }
 func (fastpathT) {{ .MethodNamePfx "Dec" false }}N(v []{{ .Elem }}, d *Decoder) {
 	slh, containerLenS := d.decSliceHelperStart()
+	if slh.IsNil {
+		return
+	}
 	if containerLenS == 0 {
 		slh.End()
 		return
@@ -384,11 +415,13 @@ func (fastpathT) {{ .MethodNamePfx "Dec" false }}N(v []{{ .Elem }}, d *Decoder)
 			break
 		} 
 		slh.ElemContainerState(j)
+		{{/*
 		if d.d.TryDecodeAsNil() {
 			v[uint(j)] = {{ zerocmd .Elem }}
 		} else {
 			{{ if eq .Elem "interface{}" }}d.decode(&v[uint(j)]){{ else }}v[uint(j)] = {{ decmd .Elem }}{{ end }}
-		}
+		} */ -}}
+		{{ if eq .Elem "interface{}" }}d.decode(&v[uint(j)]){{ else }}v[uint(j)] = {{ decmd .Elem }}{{ end }}
 	}
 	slh.End()
 }
@@ -404,16 +437,24 @@ func (d *Decoder) {{ .MethodNamePfx "fastpathDec" false }}R(f *codecFnInfo, rv r
 	containerLen := d.mapStart()
 	if rv.Kind() == reflect.Ptr {
 		vp := rv2i(rv).(*map[{{ .MapKey }}]{{ .Elem }})
+		if containerLen == decContainerLenNil {
+			*vp = nil
+			return
+		}
 		if *vp == nil {
 			*vp = make(map[{{ .MapKey }}]{{ .Elem }}, decInferLen(containerLen, d.h.MaxInitLen, {{ .Size }}))
 		}
 		fastpathTV.{{ .MethodNamePfx "Dec" false }}L(*vp, containerLen, d)
-	} else {
+	} else if containerLen != decContainerLenNil {
 		fastpathTV.{{ .MethodNamePfx "Dec" false }}L(rv2i(rv).(map[{{ .MapKey }}]{{ .Elem }}), containerLen, d)
 	}
 }
 func (f fastpathT) {{ .MethodNamePfx "Dec" false }}X(vp *map[{{ .MapKey }}]{{ .Elem }}, d *Decoder) {
 	containerLen := d.mapStart()
+	if containerLen == decContainerLenNil {
+		*vp = nil
+		return
+	}
 	if *vp == nil {
 		*vp = make(map[{{ .MapKey }}]{{ .Elem }}, decInferLen(containerLen, d.h.MaxInitLen, {{ .Size }}))
 	}
@@ -434,6 +475,11 @@ func (f fastpathT) {{ .MethodNamePfx "Dec" false }}N(v map[{{ .MapKey }}]{{ .Ele
 }
 */ -}}
 func (fastpathT) {{ .MethodNamePfx "Dec" false }}L(v map[{{ .MapKey }}]{{ .Elem }}, containerLen int, d *Decoder) {
+	{{/* No need to check if containerLen == decContainerLenNil, as that is checked by R and L above
+	if containerLen == decContainerLenNil {
+		return
+	}
+	*/ -}}
 	if containerLen == 0 {
 		d.mapEnd()
 		return
@@ -452,10 +498,12 @@ func (fastpathT) {{ .MethodNamePfx "Dec" false }}L(v map[{{ .MapKey }}]{{ .Elem
 			mk = d.string(bv) {{/* // maps cannot have []byte as key. switch to string. */}}
 		}{{ else }}mk = {{ decmd .MapKey }}{{ end }}
 		d.mapElemValue()
+		{{/*
 		if d.d.TryDecodeAsNil() {
 			if v == nil {} else if d.h.DeleteOnNilMapValue { delete(v, mk) } else { v[mk] = {{ zerocmd .Elem }} }
 			continue 
 		}
+		*/ -}}
 		{{ if eq .Elem "interface{}" "[]byte" "bytes" -}}
 		if mapGet { mv = v[mk] } else { mv = nil }
 		{{ end -}}

codec/gen-dec-array.go.tmpl

@@ -1,9 +1,17 @@
 {{var "v"}} := {{if not isArray}}*{{end}}{{ .Varname }}
-{{var "h"}}, {{var "l"}} := z.DecSliceHelperStart() {{/* // helper, containerLenS */}}{{if not isArray}}
+{{var "h"}}, {{var "l"}} := z.DecSliceHelperStart() {{/* // helper, containerLenS */}}
+{{if not isArray -}}
 var {{var "c"}} bool {{/* // changed */}}
-_ = {{var "c"}}{{end}}
+_ = {{var "c"}}
+if {{var "h"}}.IsNil {
+	if {{var "v"}} != nil {
+		{{var "v"}} = nil
+		{{var "c"}} = true
+	}
+} else {{end -}}
 if {{var "l"}} == 0 {
-	{{if isSlice }}if {{var "v"}} == nil {
+	{{if isSlice -}}
+	if {{var "v"}} == nil {
 		{{var "v"}} = []{{ .Typ }}{}
 		{{var "c"}} = true
 	} else if len({{var "v"}}) != 0 {

codec/gen-dec-map.go.tmpl

@@ -1,21 +1,23 @@
 {{var "v"}} := *{{ .Varname }}
 {{var "l"}} := z.DecReadMapStart()
-{{var "bh"}} := z.DecBasicHandle()
+if {{var "l"}} == codecSelferDecContainerLenNil{{xs}} {
+	*{{ .Varname }} = nil
+} else {
 if {{var "v"}} == nil {
-	{{var "rl"}} := z.DecInferLen({{var "l"}}, {{var "bh"}}.MaxInitLen, {{ .Size }})
+	{{var "rl"}} := z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
 	{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}, {{var "rl"}})
 	*{{ .Varname }} = {{var "v"}}
 }
 var {{var "mk"}} {{ .KTyp }}
 var {{var "mv"}} {{ .Typ }}
 var {{var "mg"}}, {{var "mdn"}} {{if decElemKindPtr}}, {{var "ms"}}, {{var "mok"}}{{end}} bool
-if {{var "bh"}}.MapValueReset {
+if z.DecBasicHandle().MapValueReset {
 	{{if decElemKindPtr}}{{var "mg"}} = true
-	{{else if decElemKindIntf}}if !{{var "bh"}}.InterfaceReset { {{var "mg"}} = true }
+	{{else if decElemKindIntf}}if !z.DecBasicHandle().InterfaceReset { {{var "mg"}} = true }
 	{{else if not decElemKindImmutable}}{{var "mg"}} = true
 	{{end}} }
 if {{var "l"}} != 0 {
-{{var "hl"}} := {{var "l"}} > 0 
+	{{var "hl"}} := {{var "l"}} > 0 
 	for {{var "j"}} := 0; ({{var "hl"}} && {{var "j"}} < {{var "l"}}) || !({{var "hl"}} || r.CheckBreak()); {{var "j"}}++ {
 	z.DecReadMapElemKey() {{/* z.DecSendContainerState(codecSelfer_containerMapKey{{ .Sfx }}) */}}
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x -}}
@@ -41,10 +43,11 @@ if {{var "l"}} != 0 {
 	{{var "mdn"}} = false
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ $y := printf "%vmdn%v" .TempVar .Rand }}{{ decLineVar $x $y -}}
 	if {{var "mdn"}} {
-		if {{ var "bh" }}.DeleteOnNilMapValue { delete({{var "v"}}, {{var "mk"}}) } else { {{var "v"}}[{{var "mk"}}] = {{decElemZero}} }
+		if z.DecBasicHandle().DeleteOnNilMapValue { delete({{var "v"}}, {{var "mk"}}) } else { {{var "v"}}[{{var "mk"}}] = {{decElemZero}} }
 	} else if {{if decElemKindPtr}} {{var "ms"}} && {{end}} {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
 } // else len==0: TODO: Should we clear map entries?
 z.DecReadMapEnd() {{/* z.DecSendContainerState(codecSelfer_containerMapEnd{{ .Sfx }}) */}}
+}

codec/gen-helper.generated.go

@@ -13,7 +13,7 @@ import (
 )
 
 // GenVersion is the current version of codecgen.
-const GenVersion = 13
+const GenVersion = 14
 
 // This file is used to generate helper code for codecgen.
 // The values here i.e. genHelper(En|De)coder are not to be used directly by
@@ -174,7 +174,7 @@ func (f genHelperDecoder) DecScratchArrayBuffer() *[decScratchByteArrayLen]byte
 func (f genHelperDecoder) DecFallback(iv interface{}, chkPtr bool) {
 	rv := reflect.ValueOf(iv)
 	if chkPtr {
-		rv = f.d.ensureDecodeable(rv)
+		f.d.ensureDecodeable(rv)
 	}
 	f.d.decodeValue(rv, nil)
 }

codec/gen-helper.go.tmpl

@@ -172,7 +172,7 @@ func (f genHelperDecoder) DecScratchArrayBuffer() *[decScratchByteArrayLen]byte
 func (f genHelperDecoder) DecFallback(iv interface{}, chkPtr bool) {
 	rv := reflect.ValueOf(iv)
 	if chkPtr {
-		rv = f.d.ensureDecodeable(rv)
+		f.d.ensureDecodeable(rv)
 	}
 	f.d.decodeValue(rv, nil)
 }

codec/gen.generated.go

@@ -10,22 +10,24 @@ package codec
 const genDecMapTmpl = `
 {{var "v"}} := *{{ .Varname }}
 {{var "l"}} := z.DecReadMapStart()
-{{var "bh"}} := z.DecBasicHandle()
+if {{var "l"}} == codecSelferDecContainerLenNil{{xs}} {
+	*{{ .Varname }} = nil
+} else {
 if {{var "v"}} == nil {
-	{{var "rl"}} := z.DecInferLen({{var "l"}}, {{var "bh"}}.MaxInitLen, {{ .Size }})
+	{{var "rl"}} := z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
 	{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}, {{var "rl"}})
 	*{{ .Varname }} = {{var "v"}}
 }
 var {{var "mk"}} {{ .KTyp }}
 var {{var "mv"}} {{ .Typ }}
 var {{var "mg"}}, {{var "mdn"}} {{if decElemKindPtr}}, {{var "ms"}}, {{var "mok"}}{{end}} bool
-if {{var "bh"}}.MapValueReset {
+if z.DecBasicHandle().MapValueReset {
 	{{if decElemKindPtr}}{{var "mg"}} = true
-	{{else if decElemKindIntf}}if !{{var "bh"}}.InterfaceReset { {{var "mg"}} = true }
+	{{else if decElemKindIntf}}if !z.DecBasicHandle().InterfaceReset { {{var "mg"}} = true }
 	{{else if not decElemKindImmutable}}{{var "mg"}} = true
 	{{end}} }
 if {{var "l"}} != 0 {
-{{var "hl"}} := {{var "l"}} > 0 
+	{{var "hl"}} := {{var "l"}} > 0 
 	for {{var "j"}} := 0; ({{var "hl"}} && {{var "j"}} < {{var "l"}}) || !({{var "hl"}} || r.CheckBreak()); {{var "j"}}++ {
 	z.DecReadMapElemKey() {{/* z.DecSendContainerState(codecSelfer_containerMapKey{{ .Sfx }}) */}}
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x -}}
@@ -51,22 +53,31 @@ if {{var "l"}} != 0 {
 	{{var "mdn"}} = false
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ $y := printf "%vmdn%v" .TempVar .Rand }}{{ decLineVar $x $y -}}
 	if {{var "mdn"}} {
-		if {{ var "bh" }}.DeleteOnNilMapValue { delete({{var "v"}}, {{var "mk"}}) } else { {{var "v"}}[{{var "mk"}}] = {{decElemZero}} }
+		if z.DecBasicHandle().DeleteOnNilMapValue { delete({{var "v"}}, {{var "mk"}}) } else { {{var "v"}}[{{var "mk"}}] = {{decElemZero}} }
 	} else if {{if decElemKindPtr}} {{var "ms"}} && {{end}} {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
 } // else len==0: TODO: Should we clear map entries?
 z.DecReadMapEnd() {{/* z.DecSendContainerState(codecSelfer_containerMapEnd{{ .Sfx }}) */}}
+}
 `
 
 const genDecListTmpl = `
 {{var "v"}} := {{if not isArray}}*{{end}}{{ .Varname }}
-{{var "h"}}, {{var "l"}} := z.DecSliceHelperStart() {{/* // helper, containerLenS */}}{{if not isArray}}
+{{var "h"}}, {{var "l"}} := z.DecSliceHelperStart() {{/* // helper, containerLenS */}}
+{{if not isArray -}}
 var {{var "c"}} bool {{/* // changed */}}
-_ = {{var "c"}}{{end}}
+_ = {{var "c"}}
+if {{var "h"}}.IsNil {
+	if {{var "v"}} != nil {
+		{{var "v"}} = nil
+		{{var "c"}} = true
+	}
+} else {{end -}}
 if {{var "l"}} == 0 {
-	{{if isSlice }}if {{var "v"}} == nil {
+	{{if isSlice -}}
+	if {{var "v"}} == nil {
 		{{var "v"}} = []{{ .Typ }}{}
 		{{var "c"}} = true
 	} else if len({{var "v"}}) != 0 {

codec/gen.go

@@ -109,7 +109,8 @@ import (
 // v11: remove deprecated methods of encDriver and decDriver.
 // v12: removed deprecated methods from genHelper and changed container tracking logic
 // v13: 20190603 removed DecodeString - use DecodeStringAsBytes instead
-const genVersion = 13
+// v14: 20190611 refactored nil handling: TryDecodeAsNil -> selective TryNil, etc
+const genVersion = 14
 
 const (
 	genCodecPkg        = "codec1978"
@@ -135,6 +136,12 @@ const (
 
 	// genFastpathTrimTypes configures whether we trim uncommon fastpath types.
 	genFastpathTrimTypes = true
+
+	// genDecStructArrayInlineLoopCheck configures whether we create a next function
+	// for each iteration in the loop and call it, or just inline it.
+	//
+	// with inlining, we get better performance but about 10% larger files.
+	genDecStructArrayInlineLoopCheck = true
 )
 
 type genStructMapStyle uint8
@@ -315,11 +322,14 @@ func Gen(w io.Writer, buildTags, pkgName, uid string, noExtensions bool,
 	x.linef("// ----- value types used ----")
 	for _, vt := range [...]valueType{
 		valueTypeArray, valueTypeMap, valueTypeString,
-		valueTypeInt, valueTypeUint, valueTypeFloat} {
+		valueTypeInt, valueTypeUint, valueTypeFloat,
+		valueTypeNil,
+	} {
 		x.linef("codecSelferValueType%s%s = %v", vt.String(), x.xs, int64(vt))
 	}
 
 	x.linef("codecSelferBitsize%s = uint8(32 << (^uint(0) >> 63))", x.xs)
+	x.linef("codecSelferDecContainerLenNil%s = %d", x.xs, int64(decContainerLenNil))
 	x.line(")")
 	x.line("var (")
 	x.line("errCodecSelferOnlyMapOrArrayEncodeToStruct" + x.xs + " = " + "\nerrors.New(`only encoded map or array can be decoded into a struct`)")
@@ -342,7 +352,7 @@ func Gen(w io.Writer, buildTags, pkgName, uid string, noExtensions bool,
 	x.linef("}")
 	if len(imKeys) > 0 {
 		x.line("if false { // reference the types, but skip this branch at build/run time")
-		x.line("var _ byte")
+		// x.line("var _ byte")
 		// x.line("_ = strconv.ParseInt")
 		// var n int
 		// for k, t := range x.im {
@@ -490,6 +500,13 @@ func (x *genRunner) out(s string) {
 	}
 }
 
+// func (x *genRunner) outb(s []byte) {
+// 	_, err := x.w.Write(s)
+// 	if err != nil {
+// 		panic(err)
+// 	}
+// }
+
 func (x *genRunner) outf(s string, params ...interface{}) {
 	_, err := fmt.Fprintf(x.w, s, params...)
 	if err != nil {
@@ -497,6 +514,13 @@ func (x *genRunner) outf(s string, params ...interface{}) {
 	}
 }
 
+// func (x *genRunner) lineb(s []byte) {
+// 	x.outb(s)
+// 	if len(s) == 0 || s[len(s)-1] != '\n' {
+// 		x.out("\n")
+// 	}
+// }
+
 func (x *genRunner) line(s string) {
 	x.out(s)
 	if len(s) == 0 || s[len(s)-1] != '\n' {
@@ -518,7 +542,7 @@ func (x *genRunner) linef(s string, params ...interface{}) {
 }
 
 func (x *genRunner) genTypeName(t reflect.Type) (n string) {
-	// defer func() { fmt.Printf(">>>> ####: genTypeName: t: %v, name: '%s'\n", t, n) }()
+	// defer func() { xdebugf(">>>> ####: genTypeName: t: %v, name: '%s'\n", t, n) }()
 
 	// if the type has a PkgPath, which doesn't match the current package,
 	// then include it.
@@ -755,17 +779,18 @@ func (x *genRunner) enc(varname string, t reflect.Type) {
 	// tptr := reflect.PtrTo(t)
 	tk := t.Kind()
 	if x.checkForSelfer(t, varname) {
-		if tk == reflect.Array || (tk == reflect.Struct && rtid != timeTypId) { // varname is of type *T
+		if tk == reflect.Array ||
+			(tk == reflect.Struct && rtid != timeTypId) { // varname is of type *T
 			// if tptr.Implements(selferTyp) || t.Implements(selferTyp) {
-			if ti2.isFlag(typeInfoFlagIsZeroerPtr) || ti2.isFlag(typeInfoFlagIsZeroer) {
+			if ti2.isFlag(tiflagSelfer) || ti2.isFlag(tiflagSelferPtr) {
 				x.line(varname + ".CodecEncodeSelf(e)")
 				return
 			}
 		} else { // varname is of type T
-			if ti2.cs { // t.Implements(selferTyp) {
+			if ti2.isFlag(tiflagSelfer) {
 				x.line(varname + ".CodecEncodeSelf(e)")
 				return
-			} else if ti2.csp { // tptr.Implements(selferTyp) {
+			} else if ti2.isFlag(tiflagSelferPtr) {
 				x.linef("%ssf%s := &%s", genTempVarPfx, mi, varname)
 				x.linef("%ssf%s.CodecEncodeSelf(e)", genTempVarPfx, mi)
 				return
@@ -825,27 +850,27 @@ func (x *genRunner) enc(varname string, t reflect.Type) {
 			hasIf.c(false), yy, varname, yy, varname, yy)
 	}
 	if arrayOrStruct { // varname is of type *T
-		if ti2.bm || ti2.bmp { // t.Implements(binaryMarshalerTyp) || tptr.Implements(binaryMarshalerTyp) {
+		if ti2.isFlag(tiflagBinaryMarshaler) || ti2.isFlag(tiflagBinaryMarshalerPtr) {
 			x.linef("%s z.EncBinary() { z.EncBinaryMarshal(%v) ", hasIf.c(false), varname)
 		}
-		if ti2.jm || ti2.jmp { // t.Implements(jsonMarshalerTyp) || tptr.Implements(jsonMarshalerTyp) {
+		if ti2.isFlag(tiflagJsonMarshaler) || ti2.isFlag(tiflagJsonMarshalerPtr) {
 			x.linef("%s !z.EncBinary() && z.IsJSONHandle() { z.EncJSONMarshal(%v) ", hasIf.c(false), varname)
-		} else if ti2.tm || ti2.tmp { // t.Implements(textMarshalerTyp) || tptr.Implements(textMarshalerTyp) {
+		} else if ti2.isFlag(tiflagTextUnmarshaler) || ti2.isFlag(tiflagTextUnmarshalerPtr) {
 			x.linef("%s !z.EncBinary() { z.EncTextMarshal(%v) ", hasIf.c(false), varname)
 		}
 	} else { // varname is of type T
-		if ti2.bm { // t.Implements(binaryMarshalerTyp) {
+		if ti2.isFlag(tiflagBinaryMarshaler) {
 			x.linef("%s z.EncBinary() { z.EncBinaryMarshal(%v) ", hasIf.c(false), varname)
-		} else if ti2.bmp { // tptr.Implements(binaryMarshalerTyp) {
+		} else if ti2.isFlag(tiflagBinaryMarshalerPtr) {
 			x.linef("%s z.EncBinary() { z.EncBinaryMarshal(&%v) ", hasIf.c(false), varname)
 		}
-		if ti2.jm { // t.Implements(jsonMarshalerTyp) {
+		if ti2.isFlag(tiflagJsonMarshaler) {
 			x.linef("%s !z.EncBinary() && z.IsJSONHandle() { z.EncJSONMarshal(%v) ", hasIf.c(false), varname)
-		} else if ti2.jmp { // tptr.Implements(jsonMarshalerTyp) {
+		} else if ti2.isFlag(tiflagJsonMarshalerPtr) {
 			x.linef("%s !z.EncBinary() && z.IsJSONHandle() { z.EncJSONMarshal(&%v) ", hasIf.c(false), varname)
-		} else if ti2.tm { // t.Implements(textMarshalerTyp) {
+		} else if ti2.isFlag(tiflagTextMarshaler) {
 			x.linef("%s !z.EncBinary() { z.EncTextMarshal(%v) ", hasIf.c(false), varname)
-		} else if ti2.tmp { // tptr.Implements(textMarshalerTyp) {
+		} else if ti2.isFlag(tiflagTextMarshalerPtr) {
 			x.linef("%s !z.EncBinary() { z.EncTextMarshal(&%v) ", hasIf.c(false), varname)
 		}
 	}
@@ -954,11 +979,11 @@ func (x *genRunner) encOmitEmptyLine(t2 reflect.StructField, varname string, buf
 			buf.s("!(").s(varname2).s(".IsZero())")
 			break
 		}
-		if ti2.isFlag(typeInfoFlagIsZeroerPtr) || ti2.isFlag(typeInfoFlagIsZeroer) {
+		if ti2.isFlag(tiflagIsZeroerPtr) || ti2.isFlag(tiflagIsZeroer) {
 			buf.s("!(").s(varname2).s(".IsZero())")
 			break
 		}
-		if ti2.isFlag(typeInfoFlagComparable) {
+		if ti2.isFlag(tiflagComparable) {
 			buf.s(varname2).s(" != ").s(x.genZeroValueR(t2.Type))
 			break
 		}
@@ -1256,7 +1281,8 @@ func (x *genRunner) decVarInitPtr(varname, nilvar string, t reflect.Type, si *st
 			if uint8(ij) == si.nis {
 				break
 			}
-			for t2typ.Kind() == reflect.Ptr {
+			// only one-level pointers can be seen in a type
+			if t2typ.Kind() == reflect.Ptr {
 				t2typ = t2typ.Elem()
 			}
 			t2 = t2typ.Field(int(ix))
@@ -1267,7 +1293,10 @@ func (x *genRunner) decVarInitPtr(varname, nilvar string, t reflect.Type, si *st
 				continue
 			}
 			if newbuf != nil {
-				newbuf.f("if %s == nil { %s = new(%s) }\n", varname3, varname3, x.genTypeName(t2typ.Elem()))
+				if len(newbuf.buf) > 0 {
+					newbuf.s("\n")
+				}
+				newbuf.f("if %s == nil { %s = new(%s) }", varname3, varname3, x.genTypeName(t2typ.Elem()))
 			}
 			if nilbuf != nil {
 				if !nilbufed {
@@ -1327,8 +1356,7 @@ func (x *genRunner) decVarMain(varname, rand string, t reflect.Type, checkNotNil
 		for t = t.Elem(); t.Kind() == reflect.Ptr; t = t.Elem() {
 			ptrPfx += "*"
 			if checkNotNil {
-				x.linef("if %s%s == nil { %s%s = new(%s)}",
-					ptrPfx, varname, ptrPfx, varname, x.genTypeName(t))
+				x.linef("if %s%s == nil { %s%s = new(%s)}", ptrPfx, varname, ptrPfx, varname, x.genTypeName(t))
 			}
 		}
 		// Should we create temp var if a slice/map indexing? No. dec(...) can now handle it.
@@ -1345,7 +1373,6 @@ func (x *genRunner) decVarMain(varname, rand string, t reflect.Type, checkNotNil
 
 // decVar takes a variable called varname, of type t
 func (x *genRunner) decVar(varname, nilvar string, t reflect.Type, canBeNil, checkNotNil bool) {
-	i := x.varsfx()
 
 	// We only encode as nil if a nillable value.
 	// This removes some of the wasted checks for TryDecodeAsNil.
@@ -1354,22 +1381,34 @@ func (x *genRunner) decVar(varname, nilvar string, t reflect.Type, canBeNil, che
 	// This could happen when decoding from a struct encoded as an array.
 	// For that, decVar should be called with canNil=true, to force true as its value.
 
-	if !canBeNil {
-		canBeNil = genAnythingCanBeNil || !genIsImmutable(t)
-	}
+	// i := x.varsfx()
+	// if !canBeNil {
+	// 	canBeNil = genAnythingCanBeNil || !genIsImmutable(t)
+	// }
+	//
+	// if canBeNil {
+	// 	var buf genBuf
+	// 	x.decVarInitPtr(varname, nilvar, t, nil, nil, &buf)
+	// 	x.linef("if r.TryDecodeAsNil() { %s } else {", buf.buf)
+	// } else {
+	// 	x.line("// cannot be nil")
+	// }
+	// x.decVarMain(varname, i, t, checkNotNil)
+	// if canBeNil {
+	// 	x.line("} ")
+	// }
 
-	if canBeNil {
+	// x.decVarMain(varname, i, t, checkNotNil)
+
+	i := x.varsfx()
+	if t.Kind() == reflect.Ptr {
 		var buf genBuf
 		x.decVarInitPtr(varname, nilvar, t, nil, nil, &buf)
-		x.linef("if r.TryDecodeAsNil() { %s } else {", buf.buf)
-	} else {
-		x.line("// cannot be nil")
-	}
-
-	x.decVarMain(varname, i, t, checkNotNil)
-
-	if canBeNil {
+		x.linef("if r.TryNil() { %s } else {", buf.buf)
+		x.decVarMain(varname, i, t, checkNotNil)
 		x.line("} ")
+	} else {
+		x.decVarMain(varname, i, t, checkNotNil)
 	}
 }
 
@@ -1383,7 +1422,7 @@ func (x *genRunner) dec(varname string, t reflect.Type, isptr bool) {
 	ti2 := x.ti.get(rtid, t)
 	// tptr := reflect.PtrTo(t)
 	if x.checkForSelfer(t, varname) {
-		if ti2.cs || ti2.csp { // t.Implements(selferTyp) || tptr.Implements(selferTyp) {
+		if ti2.isFlag(tiflagSelfer) || ti2.isFlag(tiflagSelferPtr) {
 			x.line(varname + ".CodecDecodeSelf(d)")
 			return
 		}
@@ -1452,12 +1491,12 @@ func (x *genRunner) dec(varname string, t reflect.Type, isptr bool) {
 		x.linef("%s %s := z.Extension(z.I2Rtid(%s)); %s != nil { z.DecExtension(%s, %s) ", hasIf.c(false), yy, varname, yy, varname, yy)
 	}
 
-	if ti2.bu || ti2.bup { // t.Implements(binaryUnmarshalerTyp) || tptr.Implements(binaryUnmarshalerTyp) {
+	if ti2.isFlag(tiflagBinaryUnmarshaler) || ti2.isFlag(tiflagBinaryUnmarshalerPtr) {
 		x.linef("%s z.DecBinary() { z.DecBinaryUnmarshal(%s%v) ", hasIf.c(false), addrPfx, varname)
 	}
-	if ti2.ju || ti2.jup { // t.Implements(jsonUnmarshalerTyp) || tptr.Implements(jsonUnmarshalerTyp) {
+	if ti2.isFlag(tiflagJsonUnmarshaler) || ti2.isFlag(tiflagJsonUnmarshalerPtr) {
 		x.linef("%s !z.DecBinary() && z.IsJSONHandle() { z.DecJSONUnmarshal(%s%v)", hasIf.c(false), addrPfx, varname)
-	} else if ti2.tu || ti2.tup { // t.Implements(textUnmarshalerTyp) || tptr.Implements(textUnmarshalerTyp) {
+	} else if ti2.isFlag(tiflagTextUnmarshaler) || ti2.isFlag(tiflagTextUnmarshalerPtr) {
 		x.linef("%s !z.DecBinary() { z.DecTextUnmarshal(%s%v)", hasIf.c(false), addrPfx, varname)
 	}
 
@@ -1580,6 +1619,7 @@ func (x *genRunner) decListFallback(varname string, rtid uintptr, t reflect.Type
 	}
 	type tstruc struct {
 		TempVar   string
+		Sfx       string
 		Rand      string
 		Varname   string
 		CTyp      string
@@ -1588,7 +1628,7 @@ func (x *genRunner) decListFallback(varname string, rtid uintptr, t reflect.Type
 		Size      int
 	}
 	telem := t.Elem()
-	ts := tstruc{genTempVarPfx, x.varsfx(), varname, x.genTypeName(t), x.genTypeName(telem), genIsImmutable(telem), int(telem.Size())}
+	ts := tstruc{genTempVarPfx, x.xs, x.varsfx(), varname, x.genTypeName(t), x.genTypeName(telem), genIsImmutable(telem), int(telem.Size())}
 
 	funcs := make(template.FuncMap)
 
@@ -1599,6 +1639,9 @@ func (x *genRunner) decListFallback(varname string, rtid uintptr, t reflect.Type
 	funcs["var"] = func(s string) string {
 		return ts.TempVar + s + ts.Rand
 	}
+	funcs["xs"] = func() string {
+		return ts.Sfx
+	}
 	funcs["zero"] = func() string {
 		return x.genZeroValueR(telem)
 	}
@@ -1661,6 +1704,9 @@ func (x *genRunner) decMapFallback(varname string, rtid uintptr, t reflect.Type)
 	funcs["var"] = func(s string) string {
 		return ts.TempVar + s + ts.Rand
 	}
+	funcs["xs"] = func() string {
+		return ts.Sfx
+	}
 
 	tm, err := template.New("").Funcs(funcs).Parse(genDecMapTmpl)
 	if err != nil {
@@ -1681,9 +1727,13 @@ func (x *genRunner) decStructMapSwitch(kName string, varname string, rtid uintpt
 		newbuf.reset()
 		nilbuf.reset()
 		varname3, t2 := x.decVarInitPtr(varname, "", t, si, &newbuf, &nilbuf)
-		x.linef("if r.TryDecodeAsNil() { %s } else { %s", nilbuf.buf, newbuf.buf)
+		if len(newbuf.buf) > 0 {
+			x.linef("if r.TryNil() { %s } else { %s", nilbuf.buf, newbuf.buf)
+		}
 		x.decVarMain(varname3, x.varsfx(), t2.Type, false)
-		x.line("}")
+		if len(newbuf.buf) > 0 {
+			x.line("}")
+		}
 	}
 	x.line("default:")
 	// pass the slice here, so that the string will not escape, and maybe save allocation
@@ -1738,19 +1788,35 @@ func (x *genRunner) decStructArray(varname, lenvarname, breakString string, rtid
 	x.linef("var %sj%s int", tpfx, i)
 	x.linef("var %sb%s bool", tpfx, i)                        // break
 	x.linef("var %shl%s bool = %s >= 0", tpfx, i, lenvarname) // has length
-	var newbuf, nilbuf genBuf
-	for _, si := range tisfi {
-		x.linef("%sj%s++; if %shl%s { %sb%s = %sj%s > %s } else { %sb%s = r.CheckBreak() }",
+	if !genDecStructArrayInlineLoopCheck {
+		x.linef("var %sfn%s = func() bool { ", tpfx, i)
+		x.linef("%sj%s++; if %shl%s { %sb%s = %sj%s > %s } else { %sb%s = r.CheckBreak() };",
 			tpfx, i, tpfx, i, tpfx, i,
 			tpfx, i, lenvarname, tpfx, i)
-		x.linef("if %sb%s { z.DecReadArrayEnd(); %s }", tpfx, i, breakString)
+		x.linef("if %sb%s { z.DecReadArrayEnd(); return true }; return false", tpfx, i)
+		x.linef("} // end func %sfn%s", tpfx, i)
+	}
+	var newbuf, nilbuf genBuf
+	for _, si := range tisfi {
+		if genDecStructArrayInlineLoopCheck {
+			x.linef("%sj%s++; if %shl%s { %sb%s = %sj%s > %s } else { %sb%s = r.CheckBreak() }",
+				tpfx, i, tpfx, i, tpfx, i,
+				tpfx, i, lenvarname, tpfx, i)
+			x.linef("if %sb%s { z.DecReadArrayEnd(); %s }", tpfx, i, breakString)
+		} else {
+			x.linef("if %sfn%s() { %s }", tpfx, i, breakString)
+		}
 		x.line("z.DecReadArrayElem()")
 		newbuf.reset()
 		nilbuf.reset()
 		varname3, t2 := x.decVarInitPtr(varname, "", t, si, &newbuf, &nilbuf)
-		x.linef("if r.TryDecodeAsNil() { %s } else { %s", nilbuf.buf, newbuf.buf)
+		if len(newbuf.buf) > 0 {
+			x.linef("if r.TryNil() { %s } else { %s", nilbuf.buf, newbuf.buf)
+		}
 		x.decVarMain(varname3, x.varsfx(), t2.Type, false)
-		x.line("}")
+		if len(newbuf.buf) > 0 {
+			x.line("}")
+		}
 	}
 	// read remaining values and throw away.
 	x.line("for {")
@@ -1765,10 +1831,13 @@ func (x *genRunner) decStructArray(varname, lenvarname, breakString string, rtid
 }
 
 func (x *genRunner) decStruct(varname string, rtid uintptr, t reflect.Type) {
+	// xdebugf("decStruct: t: %v", t)
 	// varname MUST be a ptr, or a struct field or a slice element.
 	i := x.varsfx()
 	x.linef("%sct%s := r.ContainerType()", genTempVarPfx, i)
-	x.linef("if %sct%s == codecSelferValueTypeMap%s {", genTempVarPfx, i, x.xs)
+	x.linef("if %sct%s == codecSelferValueTypeNil%s {", genTempVarPfx, i, x.xs)
+	x.linef("*(%s) = %s{}", varname, x.genTypeName(t))
+	x.linef("} else if %sct%s == codecSelferValueTypeMap%s {", genTempVarPfx, i, x.xs)
 	x.line(genTempVarPfx + "l" + i + " := z.DecReadMapStart()")
 	x.linef("if %sl%s == 0 {", genTempVarPfx, i)
 	if genUseOneFunctionForDecStructMap {

codec/helper.go

@@ -152,7 +152,7 @@ const (
 	useFinalizers = false
 
 	// xdebug controls whether xdebugf prints any output
-	xdebug = false
+	xdebug = true
 )
 
 var oneByteArr [1]byte
@@ -162,6 +162,7 @@ var codecgen bool
 
 var refBitset bitset256
 var isnilBitset bitset256
+var scalarBitset bitset256
 var pool pooler
 var panicv panicHdl
 
@@ -181,6 +182,25 @@ func init() {
 	isnilBitset.set(byte(reflect.UnsafePointer))
 	isnilBitset.set(byte(reflect.Interface))
 	isnilBitset.set(byte(reflect.Slice))
+
+	scalarBitset.set(byte(reflect.Bool))
+	scalarBitset.set(byte(reflect.Int))
+	scalarBitset.set(byte(reflect.Int8))
+	scalarBitset.set(byte(reflect.Int16))
+	scalarBitset.set(byte(reflect.Int32))
+	scalarBitset.set(byte(reflect.Int64))
+	scalarBitset.set(byte(reflect.Uint))
+	scalarBitset.set(byte(reflect.Uint8))
+	scalarBitset.set(byte(reflect.Uint16))
+	scalarBitset.set(byte(reflect.Uint32))
+	scalarBitset.set(byte(reflect.Uint64))
+	scalarBitset.set(byte(reflect.Uintptr))
+	scalarBitset.set(byte(reflect.Float32))
+	scalarBitset.set(byte(reflect.Float64))
+	scalarBitset.set(byte(reflect.Complex64))
+	scalarBitset.set(byte(reflect.Complex128))
+	scalarBitset.set(byte(reflect.String))
+
 }
 
 type handleFlag uint8
@@ -770,32 +790,37 @@ func (x *BasicHandle) fnLoad(rt reflect.Type, rtid uintptr, checkExt bool) (fn *
 		if rk == reflect.Struct || rk == reflect.Array {
 			fi.addrE = true
 		}
-	} else if ti.cs || ti.csp {
+	} else if ti.isFlag(tiflagSelfer) || ti.isFlag(tiflagSelferPtr) {
 		fn.fe = (*Encoder).selferMarshal
 		fn.fd = (*Decoder).selferUnmarshal
 		fi.addrF = true
-		fi.addrD = ti.csp
-		fi.addrE = ti.csp
-	} else if supportMarshalInterfaces && x.isBe() && (ti.bm || ti.bmp) && (ti.bu || ti.bup) {
+		fi.addrD = ti.isFlag(tiflagSelferPtr)
+		fi.addrE = ti.isFlag(tiflagSelferPtr)
+	} else if supportMarshalInterfaces && x.isBe() &&
+		(ti.isFlag(tiflagBinaryMarshaler) || ti.isFlag(tiflagBinaryMarshalerPtr)) &&
+		(ti.isFlag(tiflagBinaryUnmarshaler) || ti.isFlag(tiflagBinaryUnmarshalerPtr)) {
 		fn.fe = (*Encoder).binaryMarshal
 		fn.fd = (*Decoder).binaryUnmarshal
 		fi.addrF = true
-		fi.addrD = ti.bup
-		fi.addrE = ti.bmp
+		fi.addrD = ti.isFlag(tiflagBinaryUnmarshalerPtr)
+		fi.addrE = ti.isFlag(tiflagBinaryMarshalerPtr)
 	} else if supportMarshalInterfaces && !x.isBe() && x.isJs() &&
-		(ti.jm || ti.jmp) && (ti.ju || ti.jup) {
+		(ti.isFlag(tiflagJsonMarshaler) || ti.isFlag(tiflagJsonMarshalerPtr)) &&
+		(ti.isFlag(tiflagJsonUnmarshaler) || ti.isFlag(tiflagJsonUnmarshalerPtr)) {
 		//If JSON, we should check JSONMarshal before textMarshal
 		fn.fe = (*Encoder).jsonMarshal
 		fn.fd = (*Decoder).jsonUnmarshal
 		fi.addrF = true
-		fi.addrD = ti.jup
-		fi.addrE = ti.jmp
-	} else if supportMarshalInterfaces && !x.isBe() && (ti.tm || ti.tmp) && (ti.tu || ti.tup) {
+		fi.addrD = ti.isFlag(tiflagJsonUnmarshalerPtr)
+		fi.addrE = ti.isFlag(tiflagJsonMarshalerPtr)
+	} else if supportMarshalInterfaces && !x.isBe() &&
+		(ti.isFlag(tiflagTextMarshaler) || ti.isFlag(tiflagTextMarshalerPtr)) &&
+		(ti.isFlag(tiflagTextUnmarshaler) || ti.isFlag(tiflagTextUnmarshalerPtr)) {
 		fn.fe = (*Encoder).textMarshal
 		fn.fd = (*Decoder).textUnmarshal
 		fi.addrF = true
-		fi.addrD = ti.tup
-		fi.addrE = ti.tmp
+		fi.addrD = ti.isFlag(tiflagTextUnmarshalerPtr)
+		fi.addrE = ti.isFlag(tiflagTextMarshalerPtr)
 	} else {
 		if fastpathEnabled && (rk == reflect.Map || rk == reflect.Slice) {
 			if ti.pkgpath == "" { // un-named slice or map
@@ -890,7 +915,7 @@ func (x *BasicHandle) fnLoad(rt reflect.Type, rtid uintptr, checkExt bool) (fn *
 			case reflect.Chan:
 				fi.seq = seqTypeChan
 				fn.fe = (*Encoder).kSlice
-				fn.fd = (*Decoder).kSlice
+				fn.fd = (*Decoder).kSliceForChan
 			case reflect.Slice:
 				fi.seq = seqTypeSlice
 				fn.fe = (*Encoder).kSlice
@@ -906,7 +931,9 @@ func (x *BasicHandle) fnLoad(rt reflect.Type, rtid uintptr, checkExt bool) (fn *
 				}
 				// fn.fd = (*Decoder).kArray
 			case reflect.Struct:
-				if ti.anyOmitEmpty || ti.mf || ti.mfp {
+				if ti.anyOmitEmpty ||
+					ti.isFlag(tiflagMissingFielder) ||
+					ti.isFlag(tiflagMissingFielderPtr) {
 					fn.fe = (*Encoder).kStruct
 				} else {
 					fn.fe = (*Encoder).kStructNoOmitempty
@@ -1489,12 +1516,46 @@ func baseStructRv(v reflect.Value, update bool) (v2 reflect.Value, valid bool) {
 	return v, true
 }
 
-type typeInfoFlag uint8
+type tiflag uint32
 
 const (
-	typeInfoFlagComparable = 1 << iota
-	typeInfoFlagIsZeroer
-	typeInfoFlagIsZeroerPtr
+	_ tiflag = 1 << iota
+
+	tiflagComparable
+
+	tiflagIsZeroer
+	tiflagIsZeroerPtr
+
+	tiflagBinaryMarshaler
+	tiflagBinaryMarshalerPtr
+
+	tiflagBinaryUnmarshaler
+	tiflagBinaryUnmarshalerPtr
+
+	tiflagTextMarshaler
+	tiflagTextMarshalerPtr
+
+	tiflagTextUnmarshaler
+	tiflagTextUnmarshalerPtr
+
+	tiflagJsonMarshaler
+	tiflagJsonMarshalerPtr
+
+	tiflagJsonUnmarshaler
+	tiflagJsonUnmarshalerPtr
+
+	tiflagSelfer
+	tiflagSelferPtr
+
+	tiflagMissingFielder
+	tiflagMissingFielderPtr
+
+	// tiflag
+	// tiflag
+	// tiflag
+	// tiflag
+	// tiflag
+	// tiflag
 )
 
 // typeInfo keeps static (non-changing readonly)information
@@ -1533,38 +1594,52 @@ type typeInfo struct {
 	// sfis         []structFieldInfo // all sfi, in src order, as created.
 	sfiNamesSort []byte // all names, with indexes into the sfiSort
 
+	// rv0 is the zero value for the type.
+	// It is mostly beneficial for all non-reference kinds
+	// i.e. all but map/chan/func/ptr/unsafe.pointer
+	// so beneficial for intXX, bool, slices, structs, etc
+	rv0 reflect.Value
+
 	// format of marshal type fields below: [btj][mu]p? OR csp?
 
-	bm  bool // T is a binaryMarshaler
-	bmp bool // *T is a binaryMarshaler
-	bu  bool // T is a binaryUnmarshaler
-	bup bool // *T is a binaryUnmarshaler
-	tm  bool // T is a textMarshaler
-	tmp bool // *T is a textMarshaler
-	tu  bool // T is a textUnmarshaler
-	tup bool // *T is a textUnmarshaler
-
-	jm  bool // T is a jsonMarshaler
-	jmp bool // *T is a jsonMarshaler
-	ju  bool // T is a jsonUnmarshaler
-	jup bool // *T is a jsonUnmarshaler
-	cs  bool // T is a Selfer
-	csp bool // *T is a Selfer
-	mf  bool // T is a MissingFielder
-	mfp bool // *T is a MissingFielder
+	// bm  bool // T is a binaryMarshaler
+	// bmp bool // *T is a binaryMarshaler
+	// bu  bool // T is a binaryUnmarshaler
+	// bup bool // *T is a binaryUnmarshaler
+	// tm  bool // T is a textMarshaler
+	// tmp bool // *T is a textMarshaler
+	// tu  bool // T is a textUnmarshaler
+	// tup bool // *T is a textUnmarshaler
+
+	// jm  bool // T is a jsonMarshaler
+	// jmp bool // *T is a jsonMarshaler
+	// ju  bool // T is a jsonUnmarshaler
+	// jup bool // *T is a jsonUnmarshaler
+	// cs  bool // T is a Selfer
+	// csp bool // *T is a Selfer
+	// mf  bool // T is a MissingFielder
+	// mfp bool // *T is a MissingFielder
 
 	// other flags, with individual bits representing if set.
-	flags              typeInfoFlag
+	flags tiflag
+
 	infoFieldOmitempty bool
 
-	// _ [6]byte   // padding
-	// _ [2]uint64 // padding
+	_ [3]byte   // padding
+	_ [1]uint64 // padding
 }
 
-func (ti *typeInfo) isFlag(f typeInfoFlag) bool {
+func (ti *typeInfo) isFlag(f tiflag) bool {
 	return ti.flags&f != 0
 }
 
+func (ti *typeInfo) flag(when bool, f tiflag) *typeInfo {
+	if when {
+		ti.flags |= f
+	}
+	return ti
+}
+
 func (ti *typeInfo) indexForEncName(name []byte) (index int16) {
 	var sn []byte
 	if len(name)+2 <= 32 {
@@ -1673,30 +1748,32 @@ func (x *TypeInfos) get(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
 		pkgpath: rt.PkgPath(),
 		keyType: valueTypeString, // default it - so it's never 0
 	}
-	// ti.rv0 = reflect.Zero(rt)
+	ti.rv0 = reflect.Zero(rt)
 
 	// ti.comparable = rt.Comparable()
 	ti.numMeth = uint16(rt.NumMethod())
 
-	ti.bm, ti.bmp = implIntf(rt, binaryMarshalerTyp)
-	ti.bu, ti.bup = implIntf(rt, binaryUnmarshalerTyp)
-	ti.tm, ti.tmp = implIntf(rt, textMarshalerTyp)
-	ti.tu, ti.tup = implIntf(rt, textUnmarshalerTyp)
-	ti.jm, ti.jmp = implIntf(rt, jsonMarshalerTyp)
-	ti.ju, ti.jup = implIntf(rt, jsonUnmarshalerTyp)
-	ti.cs, ti.csp = implIntf(rt, selferTyp)
-	ti.mf, ti.mfp = implIntf(rt, missingFielderTyp)
-
-	b1, b2 := implIntf(rt, iszeroTyp)
-	if b1 {
-		ti.flags |= typeInfoFlagIsZeroer
-	}
-	if b2 {
-		ti.flags |= typeInfoFlagIsZeroerPtr
-	}
-	if rt.Comparable() {
-		ti.flags |= typeInfoFlagComparable
-	}
+	var b1, b2 bool
+	b1, b2 = implIntf(rt, binaryMarshalerTyp)
+	ti.flag(b1, tiflagBinaryMarshaler).flag(b2, tiflagBinaryMarshalerPtr)
+	b1, b2 = implIntf(rt, binaryUnmarshalerTyp)
+	ti.flag(b1, tiflagBinaryUnmarshaler).flag(b2, tiflagBinaryUnmarshalerPtr)
+	b1, b2 = implIntf(rt, textMarshalerTyp)
+	ti.flag(b1, tiflagTextMarshaler).flag(b2, tiflagTextMarshalerPtr)
+	b1, b2 = implIntf(rt, textUnmarshalerTyp)
+	ti.flag(b1, tiflagTextUnmarshaler).flag(b2, tiflagTextUnmarshalerPtr)
+	b1, b2 = implIntf(rt, jsonMarshalerTyp)
+	ti.flag(b1, tiflagJsonMarshaler).flag(b2, tiflagJsonMarshalerPtr)
+	b1, b2 = implIntf(rt, jsonUnmarshalerTyp)
+	ti.flag(b1, tiflagJsonUnmarshaler).flag(b2, tiflagJsonUnmarshalerPtr)
+	b1, b2 = implIntf(rt, selferTyp)
+	ti.flag(b1, tiflagSelfer).flag(b2, tiflagSelferPtr)
+	b1, b2 = implIntf(rt, missingFielderTyp)
+	ti.flag(b1, tiflagMissingFielder).flag(b2, tiflagMissingFielderPtr)
+	b1, b2 = implIntf(rt, iszeroTyp)
+	ti.flag(b1, tiflagIsZeroer).flag(b2, tiflagIsZeroerPtr)
+	b1 = rt.Comparable()
+	ti.flag(b1, tiflagComparable)
 
 	switch rk {
 	case reflect.Struct:
@@ -2029,13 +2106,13 @@ func isEmptyStruct(v reflect.Value, tinfos *TypeInfos, deref, checkStruct bool)
 	if ti.rtid == timeTypId {
 		return rv2i(v).(time.Time).IsZero()
 	}
-	if ti.isFlag(typeInfoFlagIsZeroerPtr) && v.CanAddr() {
+	if ti.isFlag(tiflagIsZeroerPtr) && v.CanAddr() {
 		return rv2i(v.Addr()).(isZeroer).IsZero()
 	}
-	if ti.isFlag(typeInfoFlagIsZeroer) {
+	if ti.isFlag(tiflagIsZeroer) {
 		return rv2i(v).(isZeroer).IsZero()
 	}
-	if ti.isFlag(typeInfoFlagComparable) {
+	if ti.isFlag(tiflagComparable) {
 		return rv2i(v) == rv2i(reflect.Zero(vt))
 	}
 	if !checkStruct {

codec/helper_not_unsafe.go

@@ -33,25 +33,15 @@ func bytesView(v string) []byte {
 	return []byte(v)
 }
 
-// isNilRef says whether the interface is a nil reference or not.
-//
-// A reference here is a pointer-sized reference i.e. map, ptr, chan, func, unsafepointer.
-// It is optional to extend this to also check if slices or interfaces are nil also.
-func isNilRef(v interface{}) (rv reflect.Value, isnil bool) {
-	// this is a best-effort option - you should not pay any cost for this.
-	// It is ok to return false, so we don't unnecessarily pay for reflection this early.
-
+// isNil says whether the value v is nil.
+// This applies to references like map/ptr/unsafepointer/chan/func,
+// and non-reference values like interface/slice.
+func isNil(v interface{}) (rv reflect.Value, isnil bool) {
 	rv = reflect.ValueOf(v)
 	if isnilBitset.isset(byte(rv.Kind())) {
 		isnil = rv.IsNil()
 	}
 	return
-
-	// switch rv.Kind() {
-	// case reflect.Invalid, reflect.Ptr, reflect.Interface, reflect.Slice, reflect.Map, reflect.Func, reflect.Chan:
-	// 	return rv.IsNil()
-	// }
-	// return false
 }
 
 func rv2i(rv reflect.Value) interface{} {

codec/helper_unsafe.go

@@ -76,19 +76,33 @@ func bytesView(v string) []byte {
 	return *(*[]byte)(unsafe.Pointer(&unsafeSlice{sx.Data, sx.Len, sx.Len}))
 }
 
-// isNilRef says whether the interface is a nil reference or not.
-//
-// A reference here is a pointer-sized reference i.e. map, ptr, chan, func, unsafepointer.
-// It is optional to extend this to also check if slices or interfaces are nil also.
-func isNilRef(v interface{}) (rv reflect.Value, isnil bool) {
-	// There is no global way of checking if an interface is nil.
-	// For true references (map, ptr, func, chan), you can just look
-	// at the word of the interface.
-	// However, for slices, you have to dereference
-	// the word, and get a pointer to the 3-word interface value.
-
-	isnil = ((*unsafeIntf)(unsafe.Pointer(&v))).word == nil
+// // isNilRef says whether the interface is a nil reference or not.
+// //
+// // A reference here is a pointer-sized reference i.e. map, ptr, chan, func, unsafepointer.
+// // It is optional to extend this to also check if slices or interfaces are nil also.
+// //
+// // NOTE: There is no global way of checking if an interface is nil.
+// // For true references (map, ptr, func, chan), you can just look
+// // at the word of the interface.
+// // However, for slices, you have to dereference
+// // the word, and get a pointer to the 3-word interface value.
+// func isNilRef(v interface{}) (rv reflect.Value, isnil bool) {
+// 	isnil = ((*unsafeIntf)(unsafe.Pointer(&v))).word == nil
+// 	return
+// }
+
+func isNil(v interface{}) (rv reflect.Value, isnil bool) {
+	var ui *unsafeIntf = (*unsafeIntf)(unsafe.Pointer(&v))
+	if ui.word == nil {
+		isnil = true
+		return
+	}
+	rv = reflect.ValueOf(v) // reflect.value is cheap and inline'able
+	tk := rv.Kind()
+	isnil = (tk == reflect.Interface || tk == reflect.Slice) && *(*unsafe.Pointer)(ui.word) == nil
 	return
+	// fmt.Printf(">>>> definitely nil: isnil: %v, TYPE: \t%T, word: %v, *word: %v, type: %v, nil: %v\n",
+	// 	v == nil, v, word, *((*unsafe.Pointer)(word)), ui.typ, nil)
 }
 
 func rv2ptr(urv *unsafeReflectValue) (ptr unsafe.Pointer) {
@@ -694,14 +708,3 @@ func mapdelete(typ unsafe.Pointer, m unsafe.Pointer, key unsafe.Pointer)
 //go:linkname typedmemmove reflect.typedmemmove
 //go:noescape
 func typedmemmove(typ unsafe.Pointer, dst, src unsafe.Pointer)
-
-// func definitelyNil(v interface{}) bool {
-// 	var ui *unsafeIntf = (*unsafeIntf)(unsafe.Pointer(&v))
-// 	if ui.word == nil {
-// 		return true
-// 	}
-// 	var tk = reflect.TypeOf(v).Kind()
-// 	return (tk == reflect.Interface || tk == reflect.Slice) && *(*unsafe.Pointer)(ui.word) == nil
-// 	fmt.Printf(">>>> definitely nil: isnil: %v, TYPE: \t%T, word: %v, *word: %v, type: %v, nil: %v\n",
-// 	v == nil, v, word, *((*unsafe.Pointer)(word)), ui.typ, nil)
-// }

codec/json.go

@@ -602,11 +602,11 @@ type jsonDecDriver struct {
 	b [jsonScratchArrayLen]byte // scratch 1, used for parsing strings or numbers or time.Time
 	// ---- cpu cache line boundary?
 	// c     containerState
-	tok   uint8                         // used to store the token read right after skipWhiteSpace
-	fnull bool                          // found null from appendStringAsBytes
-	_     [2]byte                       // padding
-	bstr  [4]byte                       // scratch used for string \UXXX parsing
-	b2    [jsonScratchArrayLen - 8]byte // scratch 2, used only for readUntil, decNumBytes
+	tok  uint8                         // used to store the token read right after skipWhiteSpace
+	fnil bool                          // found null
+	_    [2]byte                       // padding
+	bstr [4]byte                       // scratch used for string \UXXX parsing
+	b2   [jsonScratchArrayLen - 8]byte // scratch 2, used only for readUntil, decNumBytes
 
 	// _ [3]uint64 // padding
 	// n jsonNum
@@ -625,33 +625,33 @@ func (d *jsonDecDriver) uncacheRead() {
 }
 
 func (d *jsonDecDriver) ReadMapStart() int {
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
+	d.advance()
+	if d.tok == 'n' {
+		d.readLit4Null()
+		return decContainerLenNil
 	}
-	const xc uint8 = '{'
-	if d.tok != xc {
-		d.d.errorf("read map - expect char '%c' but got char '%c'", xc, d.tok)
+	if d.tok != '{' {
+		d.d.errorf("read map - expect char '%c' but got char '%c'", '{', d.tok)
 	}
 	d.tok = 0
-	return -1
+	return decContainerLenUnknown
 }
 
 func (d *jsonDecDriver) ReadArrayStart() int {
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
+	d.advance()
+	if d.tok == 'n' {
+		d.readLit4Null()
+		return decContainerLenNil
 	}
-	const xc uint8 = '['
-	if d.tok != xc {
-		d.d.errorf("read array - expect char '%c' but got char '%c'", xc, d.tok)
+	if d.tok != '[' {
+		d.d.errorf("read array - expect char '%c' but got char '%c'", '[', d.tok)
 	}
 	d.tok = 0
-	return -1
+	return decContainerLenUnknown
 }
 
 func (d *jsonDecDriver) CheckBreak() bool {
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	return d.tok == '}' || d.tok == ']'
 }
 
@@ -667,9 +667,7 @@ func (d *jsonDecDriver) CheckBreak() bool {
 
 func (d *jsonDecDriver) ReadArrayElem() {
 	const xc uint8 = ','
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	if d.d.c != containerArrayStart {
 		if d.tok != xc {
 			d.d.errorf("read array element - expect char '%c' but got char '%c'", xc, d.tok)
@@ -680,9 +678,7 @@ func (d *jsonDecDriver) ReadArrayElem() {
 
 func (d *jsonDecDriver) ReadArrayEnd() {
 	const xc uint8 = ']'
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	if d.tok != xc {
 		d.d.errorf("read array end - expect char '%c' but got char '%c'", xc, d.tok)
 	}
@@ -691,9 +687,7 @@ func (d *jsonDecDriver) ReadArrayEnd() {
 
 func (d *jsonDecDriver) ReadMapElemKey() {
 	const xc uint8 = ','
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	if d.d.c != containerMapStart {
 		if d.tok != xc {
 			d.d.errorf("read map key - expect char '%c' but got char '%c'", xc, d.tok)
@@ -704,9 +698,7 @@ func (d *jsonDecDriver) ReadMapElemKey() {
 
 func (d *jsonDecDriver) ReadMapElemValue() {
 	const xc uint8 = ':'
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	if d.tok != xc {
 		d.d.errorf("read map value - expect char '%c' but got char '%c'", xc, d.tok)
 	}
@@ -715,9 +707,7 @@ func (d *jsonDecDriver) ReadMapElemValue() {
 
 func (d *jsonDecDriver) ReadMapEnd() {
 	const xc uint8 = '}'
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	if d.tok != xc {
 		d.d.errorf("read map end - expect char '%c' but got char '%c'", xc, d.tok)
 	}
@@ -755,13 +745,19 @@ func (d *jsonDecDriver) readLit4Null() {
 	if jsonValidateSymbols && !bytes.Equal(bs, jsonLiteral4Null) {
 		d.d.errorf("expecting %s: got %s", jsonLiteral4Null, bs)
 	}
+	d.fnil = true
 }
 
-func (d *jsonDecDriver) TryDecodeAsNil() bool {
+func (d *jsonDecDriver) advance() {
 	if d.tok == 0 {
+		d.fnil = false
 		d.tok = d.r.skip(&jsonCharWhitespaceSet)
 	}
-	// we shouldn't try to see if "null" was here, right?
+}
+
+func (d *jsonDecDriver) TryNil() bool {
+	d.advance()
+	// we shouldn't try to see if quoted "null" was here, right?
 	// only the plain string: `null` denotes a nil (ie not quotes)
 	if d.tok == 'n' {
 		d.readLit4Null()
@@ -770,9 +766,15 @@ func (d *jsonDecDriver) TryDecodeAsNil() bool {
 	return false
 }
 
+func (d *jsonDecDriver) Nil() bool {
+	return d.fnil
+}
+
 func (d *jsonDecDriver) DecodeBool() (v bool) {
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
+	d.advance()
+	if d.tok == 'n' {
+		d.readLit4Null()
+		return
 	}
 	fquot := d.d.c == containerMapKey && d.tok == '"'
 	if fquot {
@@ -798,7 +800,7 @@ func (d *jsonDecDriver) DecodeBool() (v bool) {
 func (d *jsonDecDriver) DecodeTime() (t time.Time) {
 	// read string, and pass the string into json.unmarshal
 	d.appendStringAsBytes()
-	if d.fnull {
+	if d.fnil {
 		return
 	}
 	t, err := time.Parse(time.RFC3339, stringView(d.bs))
@@ -810,22 +812,21 @@ func (d *jsonDecDriver) DecodeTime() (t time.Time) {
 
 func (d *jsonDecDriver) ContainerType() (vt valueType) {
 	// check container type by checking the first char
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 
 	// optimize this, so we don't do 4 checks but do one computation.
 	// return jsonContainerSet[d.tok]
 
 	// ContainerType is mostly called for Map and Array,
 	// so this conditional is good enough (max 2 checks typically)
-	if b := d.tok; b == '{' {
+	if d.tok == '{' {
 		return valueTypeMap
-	} else if b == '[' {
+	} else if d.tok == '[' {
 		return valueTypeArray
-	} else if b == 'n' {
+	} else if d.tok == 'n' {
+		d.readLit4Null()
 		return valueTypeNil
-	} else if b == '"' {
+	} else if d.tok == '"' {
 		return valueTypeString
 	}
 	return valueTypeUnset
@@ -833,18 +834,18 @@ func (d *jsonDecDriver) ContainerType() (vt valueType) {
 
 func (d *jsonDecDriver) decNumBytes() (bs []byte) {
 	// stores num bytes in d.bs
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	if d.tok == '"' {
 		bs = d.r.readUntil(d.b2[:0], '"')
 		bs = bs[:len(bs)-1]
+	} else if d.tok == 'n' {
+		d.readLit4Null()
 	} else {
 		d.r.unreadn1()
 		bs = d.r.readTo(d.bs[:0], &jsonNumSet)
 	}
 	d.tok = 0
-	return bs
+	return
 }
 
 func (d *jsonDecDriver) DecodeUint64() (u uint64) {
@@ -947,7 +948,12 @@ func (d *jsonDecDriver) DecodeFloat32() (f float32) {
 	return
 }
 
-func (d *jsonDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
+func (d *jsonDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) {
+	d.advance()
+	if d.tok == 'n' {
+		d.readLit4Null()
+		return
+	}
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = xtag
@@ -958,7 +964,6 @@ func (d *jsonDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 	} else {
 		d.d.interfaceExtConvertAndDecode(rv, ext)
 	}
-	return
 }
 
 func (d *jsonDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
@@ -968,9 +973,7 @@ func (d *jsonDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 		d.DecodeExt(&bsOut, 0, &d.se)
 		return
 	}
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	// check if an "array" of uint8's (see ContainerType for how to infer if an array)
 	if d.tok == '[' {
 		// bsOut, _ = fastpathTV.DecSliceUint8V(bs, true, d.d)
@@ -1000,11 +1003,11 @@ func (d *jsonDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 	// base64 encodes []byte{} as "", and we encode nil []byte as null.
 	// Consequently, base64 should decode null as a nil []byte, and "" as an empty []byte{}.
 	// appendStringAsBytes returns a zero-len slice for both, so as not to reset d.bs.
-	// However, it sets a fnull field to true, so we can check if a null was found.
+	// However, it sets a fnil field to true, so we can check if a null was found.
+	if d.fnil {
+		return nil
+	}
 	if len(d.bs) == 0 {
-		if d.fnull {
-			return nil
-		}
 		return []byte{}
 	}
 	bs0 := d.bs
@@ -1033,16 +1036,18 @@ func (d *jsonDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 // }
 
 func (d *jsonDecDriver) DecodeStringAsBytes() (s []byte) {
+	// defer func() { xdebug2f("DecodeStringAsBytes: %s", s) }()
 	d.appendStringAsBytes()
+	if d.fnil {
+		return nil
+	}
 	return d.bs
 }
 
 func (d *jsonDecDriver) appendStringAsBytes() {
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 
-	d.fnull = false
+	// xdebug2f("appendStringAsBytes: found: '%c'", d.tok)
 	if d.tok != '"' {
 		// d.d.errorf("expect char '%c' but got char '%c'", '"', d.tok)
 		// handle non-string scalar: null, true, false or a number
@@ -1050,7 +1055,6 @@ func (d *jsonDecDriver) appendStringAsBytes() {
 		case 'n':
 			d.readLit4Null()
 			d.bs = d.bs[:0]
-			d.fnull = true
 		case 'f':
 			d.readLit4False()
 			d.bs = d.bs[:5]
@@ -1241,9 +1245,7 @@ func (d *jsonDecDriver) DecodeNaked() {
 	z := d.d.naked()
 	// var decodeFurther bool
 
-	if d.tok == 0 {
-		d.tok = d.r.skip(&jsonCharWhitespaceSet)
-	}
+	d.advance()
 	switch d.tok {
 	case 'n':
 		d.readLit4Null()
@@ -1444,6 +1446,7 @@ func (d *jsonDecDriver) reset() {
 		d.bs = d.bs[:0]
 	}
 	d.tok = 0
+	d.fnil = false
 	// d.n.reset()
 }
 

codec/mammoth-test.go.tmpl

@@ -18,37 +18,37 @@ func init() { _ = fmt.Printf } // so we can include fmt as needed
 
 type TestMammoth struct {
 
-{{range .Values }}{{if .Primitive }}{{/*
-*/}}{{ .MethodNamePfx "F" true }} {{ .Primitive }}
+{{range .Values }}{{if .Primitive -}}
+{{ .MethodNamePfx "F" true }} {{ .Primitive }}
 {{ .MethodNamePfx "Fptr" true }} *{{ .Primitive }}
 {{end}}{{end}}
 
-{{range .Values }}{{if not .Primitive }}{{if not .MapKey }}{{/*
-*/}}{{ .MethodNamePfx "F" false }} []{{ .Elem }}
+{{range .Values }}{{if not .Primitive }}{{if not .MapKey -}}
+{{ .MethodNamePfx "F" false }} []{{ .Elem }}
 {{ .MethodNamePfx "Fptr" false }} *[]{{ .Elem }}
 {{end}}{{end}}{{end}}
 
-{{range .Values }}{{if not .Primitive }}{{if .MapKey }}{{/*
-*/}}{{ .MethodNamePfx "F" false }} map[{{ .MapKey }}]{{ .Elem }}
+{{range .Values }}{{if not .Primitive }}{{if .MapKey -}}
+{{ .MethodNamePfx "F" false }} map[{{ .MapKey }}]{{ .Elem }}
 {{ .MethodNamePfx "Fptr" false }} *map[{{ .MapKey }}]{{ .Elem }}
 {{end}}{{end}}{{end}}
 
 }
 
-{{range .Values }}{{if not .Primitive }}{{if not .MapKey }}{{/*
-*/}} type {{ .MethodNamePfx "typMbs" false }} []{{ .Elem }}
+{{range .Values }}{{if not .Primitive }}{{if not .MapKey -}}
+type {{ .MethodNamePfx "typMbs" false }} []{{ .Elem }}
 func (_ {{ .MethodNamePfx "typMbs" false }}) MapBySlice() { }
 {{end}}{{end}}{{end}}
 
-{{range .Values }}{{if not .Primitive }}{{if .MapKey }}{{/*
-*/}} type {{ .MethodNamePfx "typMap" false }} map[{{ .MapKey }}]{{ .Elem }}
+{{range .Values }}{{if not .Primitive }}{{if .MapKey -}}
+type {{ .MethodNamePfx "typMap" false }} map[{{ .MapKey }}]{{ .Elem }}
 {{end}}{{end}}{{end}}
 
 func doTestMammothSlices(t *testing.T, h Handle) {
-{{range $i, $e := .Values }}{{if not .Primitive }}{{if not .MapKey }}{{/*
-*/}}
+{{range $i, $e := .Values }}{{if not .Primitive }}{{if not .MapKey -}}
     var v{{$i}}va [8]{{ .Elem }}
-    for _, v := range [][]{{ .Elem }}{ nil, {}, { {{ nonzerocmd .Elem }}, {{ zerocmd .Elem }}, {{ zerocmd .Elem }}, {{ nonzerocmd .Elem }} } } { {{/*
+    for _, v := range [][]{{ .Elem }}{ nil, {}, { {{ nonzerocmd .Elem }}, {{ zerocmd .Elem }}, {{ zerocmd .Elem }}, {{ nonzerocmd .Elem }} } } {
+	{{/*
     // fmt.Printf(">>>> running mammoth slice v{{$i}}: %v\n", v)
     //   - encode value to some []byte
     //   - decode into a length-wise-equal []byte
@@ -61,16 +61,19 @@ func doTestMammothSlices(t *testing.T, h Handle) {
     //   - 
     //   - rinse and repeat for a MapBySlice version
     //   - 
-    */}}
+    */ -}}
     var v{{$i}}v1, v{{$i}}v2 []{{ .Elem }}
+    var bs{{$i}} []byte
 	v{{$i}}v1 = v
-	bs{{$i}} := testMarshalErr(v{{$i}}v1, h, t, "enc-slice-v{{$i}}")
+	bs{{$i}} = testMarshalErr(v{{$i}}v1, h, t, "enc-slice-v{{$i}}")
+	if v != nil {	
 	if v == nil { v{{$i}}v2 = nil } else { v{{$i}}v2 = make([]{{ .Elem }}, len(v)) }
 	testUnmarshalErr(v{{$i}}v2, bs{{$i}}, h, t, "dec-slice-v{{$i}}")
 	testDeepEqualErr(v{{$i}}v1, v{{$i}}v2, t, "equal-slice-v{{$i}}")
 	if v == nil { v{{$i}}v2 = nil } else { v{{$i}}v2 = make([]{{ .Elem }}, len(v)) }
 	testUnmarshalErr(reflect.ValueOf(v{{$i}}v2), bs{{$i}}, h, t, "dec-slice-v{{$i}}-noaddr") // non-addressable value
 	testDeepEqualErr(v{{$i}}v1, v{{$i}}v2, t, "equal-slice-v{{$i}}-noaddr")
+	}	
 	// ...
 	bs{{$i}} = testMarshalErr(&v{{$i}}v1, h, t, "enc-slice-v{{$i}}-p")
 	v{{$i}}v2 = nil
@@ -102,9 +105,11 @@ func doTestMammothSlices(t *testing.T, h Handle) {
     if v != nil { v{{$i}}v2 = make([]{{ .Elem }}, len(v)) }
     v{{$i}}v3 = {{ .MethodNamePfx "typMbs" false }}(v{{$i}}v1)
     v{{$i}}v4 = {{ .MethodNamePfx "typMbs" false }}(v{{$i}}v2)
+    if v != nil {
     bs{{$i}} = testMarshalErr(v{{$i}}v3, h, t, "enc-slice-v{{$i}}-custom")
     testUnmarshalErr(v{{$i}}v4, bs{{$i}}, h, t, "dec-slice-v{{$i}}-custom")
     testDeepEqualErr(v{{$i}}v3, v{{$i}}v4, t, "equal-slice-v{{$i}}-custom")
+    }
     bs{{$i}} = testMarshalErr(&v{{$i}}v3, h, t, "enc-slice-v{{$i}}-custom-p")
     v{{$i}}v2 = nil
     v{{$i}}v4 = {{ .MethodNamePfx "typMbs" false }}(v{{$i}}v2)
@@ -115,19 +120,21 @@ func doTestMammothSlices(t *testing.T, h Handle) {
 }
 
 func doTestMammothMaps(t *testing.T, h Handle) {
-{{range $i, $e := .Values }}{{if not .Primitive }}{{if .MapKey }}{{/*
-*/}}
+{{range $i, $e := .Values }}{{if not .Primitive }}{{if .MapKey -}}
     for _, v := range []map[{{ .MapKey }}]{{ .Elem }}{ nil, {}, { {{ nonzerocmd .MapKey }}:{{ zerocmd .Elem }} {{if ne "bool" .MapKey}}, {{ nonzerocmd .MapKey }}:{{ nonzerocmd .Elem }} {{end}} } } {
     // fmt.Printf(">>>> running mammoth map v{{$i}}: %v\n", v)
     var v{{$i}}v1, v{{$i}}v2 map[{{ .MapKey }}]{{ .Elem }}
+    var bs{{$i}} []byte
 	v{{$i}}v1 = v
-	bs{{$i}} := testMarshalErr(v{{$i}}v1, h, t, "enc-map-v{{$i}}")
+	bs{{$i}} = testMarshalErr(v{{$i}}v1, h, t, "enc-map-v{{$i}}")
+    if v != nil {
 	if v == nil { v{{$i}}v2 = nil } else { v{{$i}}v2 = make(map[{{ .MapKey }}]{{ .Elem }}, len(v)) } // reset map
 	testUnmarshalErr(v{{$i}}v2, bs{{$i}}, h, t, "dec-map-v{{$i}}")
 	testDeepEqualErr(v{{$i}}v1, v{{$i}}v2, t, "equal-map-v{{$i}}")
 	if v == nil { v{{$i}}v2 = nil } else { v{{$i}}v2 = make(map[{{ .MapKey }}]{{ .Elem }}, len(v)) } // reset map
 	testUnmarshalErr(reflect.ValueOf(v{{$i}}v2), bs{{$i}}, h, t, "dec-map-v{{$i}}-noaddr") // decode into non-addressable map value
 	testDeepEqualErr(v{{$i}}v1, v{{$i}}v2, t, "equal-map-v{{$i}}-noaddr")
+    }
 	if v == nil { v{{$i}}v2 = nil } else { v{{$i}}v2 = make(map[{{ .MapKey }}]{{ .Elem }}, len(v)) } // reset map
 	testUnmarshalErr(&v{{$i}}v2, bs{{$i}}, h, t, "dec-map-v{{$i}}-p-len")
 	testDeepEqualErr(v{{$i}}v1, v{{$i}}v2, t, "equal-map-v{{$i}}-p-len")
@@ -140,10 +147,12 @@ func doTestMammothMaps(t *testing.T, h Handle) {
     var v{{$i}}v3, v{{$i}}v4 {{ .MethodNamePfx "typMap" false }}
 	v{{$i}}v3 = {{ .MethodNamePfx "typMap" false }}(v{{$i}}v1)
 	v{{$i}}v4 = {{ .MethodNamePfx "typMap" false }}(v{{$i}}v2)
+    if v != nil {
     bs{{$i}} = testMarshalErr(v{{$i}}v3, h, t, "enc-map-v{{$i}}-custom")
 	testUnmarshalErr(v{{$i}}v4, bs{{$i}}, h, t, "dec-map-v{{$i}}-p-len")
 	testDeepEqualErr(v{{$i}}v3, v{{$i}}v4, t, "equal-map-v{{$i}}-p-len")
     }
+    }
 {{end}}{{end}}{{end}}
 
 }

codec/msgpack.go

@@ -432,6 +432,7 @@ type msgpackDecDriver struct {
 	bd     byte
 	bdRead bool
 	br     bool // bytes reader
+	fnil   bool
 	noBuiltInTypes
 	// noStreamingCodec
 	// decNoSeparator
@@ -448,6 +449,7 @@ func (d *msgpackDecDriver) DecodeNaked() {
 	if !d.bdRead {
 		d.readNextBd()
 	}
+	d.fnil = false
 	bd := d.bd
 	n := d.d.naked()
 	var decodeFurther bool
@@ -456,6 +458,7 @@ func (d *msgpackDecDriver) DecodeNaked() {
 	case mpNil:
 		n.v = valueTypeNil
 		d.bdRead = false
+		d.fnil = true
 	case mpFalse:
 		n.v = valueTypeBool
 		n.b = false
@@ -549,8 +552,8 @@ func (d *msgpackDecDriver) DecodeNaked() {
 
 // int can be decoded from msgpack type: intXXX or uintXXX
 func (d *msgpackDecDriver) DecodeInt64() (i int64) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	switch d.bd {
 	case mpUint8:
@@ -586,8 +589,8 @@ func (d *msgpackDecDriver) DecodeInt64() (i int64) {
 
 // uint can be decoded from msgpack type: intXXX or uintXXX
 func (d *msgpackDecDriver) DecodeUint64() (ui uint64) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	switch d.bd {
 	case mpUint8:
@@ -644,8 +647,8 @@ func (d *msgpackDecDriver) DecodeUint64() (ui uint64) {
 
 // float can either be decoded from msgpack type: float, double or intX
 func (d *msgpackDecDriver) DecodeFloat64() (f float64) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	if d.bd == mpFloat {
 		f = float64(math.Float32frombits(bigen.Uint32(d.r.readx(4))))
@@ -660,8 +663,8 @@ func (d *msgpackDecDriver) DecodeFloat64() (f float64) {
 
 // bool can be decoded from bool, fixnum 0 or 1.
 func (d *msgpackDecDriver) DecodeBool() (b bool) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	if d.bd == mpFalse || d.bd == 0 {
 		// b = false
@@ -676,16 +679,13 @@ func (d *msgpackDecDriver) DecodeBool() (b bool) {
 }
 
 func (d *msgpackDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 
 	bd := d.bd
 	var clen int
-	if bd == mpNil {
-		d.bdRead = false
-		return
-	} else if bd == mpBin8 || bd == mpBin16 || bd == mpBin32 {
+	if bd == mpBin8 || bd == mpBin16 || bd == mpBin32 {
 		clen = d.readContainerLen(msgpackContainerBin) // binary
 	} else if bd == mpStr8 || bd == mpStr16 || bd == mpStr32 ||
 		(bd >= mpFixStrMin && bd <= mpFixStrMax) {
@@ -735,25 +735,32 @@ func (d *msgpackDecDriver) uncacheRead() {
 	}
 }
 
+func (d *msgpackDecDriver) advanceNil() (null bool) {
+	d.fnil = false
+	if !d.bdRead {
+		d.readNextBd()
+	}
+	if d.bd == mpNil {
+		d.bdRead = false
+		d.fnil = true
+		null = true
+	}
+	return
+}
+
+func (d *msgpackDecDriver) Nil() bool {
+	return d.fnil
+}
+
 func (d *msgpackDecDriver) ContainerType() (vt valueType) {
 	if !d.bdRead {
 		d.readNextBd()
 	}
 	bd := d.bd
-	// if bd == mpNil {
-	// 	// nil
-	// } else if bd == mpBin8 || bd == mpBin16 || bd == mpBin32 {
-	// 	// binary
-	// } else if bd == mpStr8 || bd == mpStr16 || bd == mpStr32 ||
-	// (bd >= mpFixStrMin && bd <= mpFixStrMax) {
-	// 	// string/raw
-	// } else if bd == mpArray16 || bd == mpArray32 ||
-	// (bd >= mpFixArrayMin && bd <= mpFixArrayMax) {
-	// 	// array
-	// } else if bd == mpMap16 || bd == mpMap32 || (bd >= mpFixMapMin && bd <= mpFixMapMax) {
-	// 	// map
-	// }
+	d.fnil = false
 	if bd == mpNil {
+		d.bdRead = false
+		d.fnil = true
 		return valueTypeNil
 	} else if bd == mpBin8 || bd == mpBin16 || bd == mpBin32 {
 		return valueTypeBytes
@@ -774,22 +781,13 @@ func (d *msgpackDecDriver) ContainerType() (vt valueType) {
 	return valueTypeUnset
 }
 
-func (d *msgpackDecDriver) TryDecodeAsNil() (v bool) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == mpNil {
-		d.bdRead = false
-		return true
-	}
-	return
+func (d *msgpackDecDriver) TryNil() (v bool) {
+	return d.advanceNil()
 }
 
 func (d *msgpackDecDriver) readContainerLen(ct msgpackContainerType) (clen int) {
 	bd := d.bd
-	if bd == mpNil {
-		clen = -1 // to represent nil
-	} else if bd == ct.b8 {
+	if bd == ct.b8 {
 		clen = int(d.r.readn1())
 	} else if bd == ct.b16 {
 		clen = int(bigen.Uint16(d.r.readx(2)))
@@ -806,23 +804,21 @@ func (d *msgpackDecDriver) readContainerLen(ct msgpackContainerType) (clen int)
 }
 
 func (d *msgpackDecDriver) ReadMapStart() int {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return decContainerLenNil
 	}
 	return d.readContainerLen(msgpackContainerMap)
 }
 
 func (d *msgpackDecDriver) ReadArrayStart() int {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return decContainerLenNil
 	}
 	return d.readContainerLen(msgpackContainerList)
 }
 
 func (d *msgpackDecDriver) readExtLen() (clen int) {
 	switch d.bd {
-	case mpNil:
-		clen = -1 // to represent nil
 	case mpFixExt1:
 		clen = 1
 	case mpFixExt2:
@@ -848,15 +844,12 @@ func (d *msgpackDecDriver) readExtLen() (clen int) {
 
 func (d *msgpackDecDriver) DecodeTime() (t time.Time) {
 	// decode time from string bytes or ext
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	bd := d.bd
 	var clen int
-	if bd == mpNil {
-		d.bdRead = false
-		return
-	} else if bd == mpBin8 || bd == mpBin16 || bd == mpBin32 {
+	if bd == mpBin8 || bd == mpBin16 || bd == mpBin32 {
 		clen = d.readContainerLen(msgpackContainerBin) // binary
 	} else if bd == mpStr8 || bd == mpStr16 || bd == mpStr32 ||
 		(bd >= mpFixStrMin && bd <= mpFixStrMax) {
@@ -899,13 +892,16 @@ func (d *msgpackDecDriver) decodeTime(clen int) (t time.Time) {
 	return
 }
 
-func (d *msgpackDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
+func (d *msgpackDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) {
 	if xtag > 0xff {
 		d.d.errorf("ext: tag must be <= 0xff; got: %v", xtag)
 		return
 	}
+	if d.advanceNil() {
+		return
+	}
 	realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag))
-	realxtag = uint64(realxtag1)
+	realxtag := uint64(realxtag1)
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = realxtag
@@ -915,13 +911,9 @@ func (d *msgpackDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (real
 	} else {
 		ext.ReadExt(rv, xbs)
 	}
-	return
 }
 
 func (d *msgpackDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
 	xbd := d.bd
 	if xbd == mpBin8 || xbd == mpBin16 || xbd == mpBin32 {
 		xbs = d.DecodeBytes(nil, true)
@@ -999,6 +991,7 @@ func (e *msgpackEncDriver) reset() {
 func (d *msgpackDecDriver) reset() {
 	d.r, d.br = d.d.r(), d.d.bytes
 	d.bd, d.bdRead = 0, false
+	d.fnil = false
 }
 
 //--------------------------------------------------

codec/shared_test.go

@@ -276,12 +276,13 @@ func sTestCodecDecode(bs []byte, ts interface{}, h Handle, bh *BasicHandle) (err
 // These are for intormational messages that do not necessarily
 // help with diagnosing a failure, or which are too large.
 func logTv(x interface{}, format string, args ...interface{}) {
-	if testVerbose {
-		if t, ok := x.(testing.TB); ok { // only available from go 1.9
-			t.Helper()
-		}
-		logT(x, format, args...)
+	if !testVerbose {
+		return
+	}
+	if t, ok := x.(testing.TB); ok { // only available from go 1.9
+		t.Helper()
 	}
+	logT(x, format, args...)
 }
 
 // logT logs messages when running as go test -v
@@ -304,12 +305,22 @@ func logT(x interface{}, format string, args ...interface{}) {
 	}
 }
 
-func failT(x interface{}, args ...interface{}) {
-	t, ok := x.(testing.TB) // only available from go 1.9
-	if ok {
-		t.Helper()
+func failTv(x testing.TB, args ...interface{}) {
+	x.Helper()
+	if testVerbose {
+		failTMsg(x, args...)
 	}
+	x.FailNow()
+}
 
+func failT(x testing.TB, args ...interface{}) {
+	x.Helper()
+	failTMsg(x, args...)
+	x.FailNow()
+}
+
+func failTMsg(x testing.TB, args ...interface{}) {
+	x.Helper()
 	if len(args) > 0 {
 		if format, ok := args[0].(string); ok {
 			logT(x, format, args[1:]...)
@@ -319,9 +330,6 @@ func failT(x interface{}, args ...interface{}) {
 			logT(x, "%v", args)
 		}
 	}
-	if ok {
-		t.FailNow()
-	}
 }
 
 // --- functions below are used only by benchmarks alone

codec/simple.go

@@ -213,6 +213,7 @@ type simpleDecDriver struct {
 	bdRead bool
 	bd     byte
 	br     bool // a bytes reader?
+	fnil   bool
 	// c      containerState
 	// b      [scratchByteArrayLen]byte
 	noBuiltInTypes
@@ -233,12 +234,32 @@ func (d *simpleDecDriver) uncacheRead() {
 	}
 }
 
+func (d *simpleDecDriver) advanceNil() (null bool) {
+	d.fnil = false
+	if !d.bdRead {
+		d.readNextBd()
+	}
+	if d.bd == simpleVdNil {
+		d.bdRead = false
+		d.fnil = true
+		null = true
+	}
+	return
+}
+
+func (d *simpleDecDriver) Nil() bool {
+	return d.fnil
+}
+
 func (d *simpleDecDriver) ContainerType() (vt valueType) {
 	if !d.bdRead {
 		d.readNextBd()
 	}
+	d.fnil = false
 	switch d.bd {
 	case simpleVdNil:
+		d.bdRead = false
+		d.fnil = true
 		return valueTypeNil
 	case simpleVdByteArray, simpleVdByteArray + 1,
 		simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4:
@@ -261,21 +282,11 @@ func (d *simpleDecDriver) ContainerType() (vt valueType) {
 	return valueTypeUnset
 }
 
-func (d *simpleDecDriver) TryDecodeAsNil() bool {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == simpleVdNil {
-		d.bdRead = false
-		return true
-	}
-	return false
+func (d *simpleDecDriver) TryNil() bool {
+	return d.advanceNil()
 }
 
 func (d *simpleDecDriver) decCheckInteger() (ui uint64, neg bool) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
 	switch d.bd {
 	case simpleVdPosInt:
 		ui = uint64(d.r.readn1())
@@ -310,6 +321,9 @@ func (d *simpleDecDriver) decCheckInteger() (ui uint64, neg bool) {
 }
 
 func (d *simpleDecDriver) DecodeInt64() (i int64) {
+	if d.advanceNil() {
+		return
+	}
 	ui, neg := d.decCheckInteger()
 	i = chkOvf.SignedIntV(ui)
 	if neg {
@@ -320,6 +334,9 @@ func (d *simpleDecDriver) DecodeInt64() (i int64) {
 }
 
 func (d *simpleDecDriver) DecodeUint64() (ui uint64) {
+	if d.advanceNil() {
+		return
+	}
 	ui, neg := d.decCheckInteger()
 	if neg {
 		d.d.errorf("assigning negative signed value to unsigned type")
@@ -330,8 +347,8 @@ func (d *simpleDecDriver) DecodeUint64() (ui uint64) {
 }
 
 func (d *simpleDecDriver) DecodeFloat64() (f float64) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
 	if d.bd == simpleVdFloat32 {
 		f = float64(math.Float32frombits(bigen.Uint32(d.r.readx(4))))
@@ -351,12 +368,12 @@ func (d *simpleDecDriver) DecodeFloat64() (f float64) {
 
 // bool can be decoded from bool only (single byte).
 func (d *simpleDecDriver) DecodeBool() (b bool) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return
 	}
-	if d.bd == simpleVdTrue {
+	if d.bd == simpleVdFalse {
+	} else if d.bd == simpleVdTrue {
 		b = true
-	} else if d.bd == simpleVdFalse {
 	} else {
 		d.d.errorf("cannot decode bool - %s: %x", msgBadDesc, d.bd)
 		return
@@ -366,16 +383,16 @@ func (d *simpleDecDriver) DecodeBool() (b bool) {
 }
 
 func (d *simpleDecDriver) ReadMapStart() (length int) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return decContainerLenNil
 	}
 	d.bdRead = false
 	return d.decLen()
 }
 
 func (d *simpleDecDriver) ReadArrayStart() (length int) {
-	if !d.bdRead {
-		d.readNextBd()
+	if d.advanceNil() {
+		return decContainerLenNil
 	}
 	d.bdRead = false
 	return d.decLen()
@@ -413,11 +430,7 @@ func (d *simpleDecDriver) DecodeStringAsBytes() (s []byte) {
 }
 
 func (d *simpleDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == simpleVdNil {
-		d.bdRead = false
+	if d.advanceNil() {
 		return
 	}
 	// check if an "array" of uint8's (see ContainerType for how to infer if an array)
@@ -447,11 +460,7 @@ func (d *simpleDecDriver) DecodeBytes(bs []byte, zerocopy bool) (bsOut []byte) {
 }
 
 func (d *simpleDecDriver) DecodeTime() (t time.Time) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
-	if d.bd == simpleVdNil {
-		d.bdRead = false
+	if d.advanceNil() {
 		return
 	}
 	if d.bd != simpleVdTime {
@@ -467,13 +476,16 @@ func (d *simpleDecDriver) DecodeTime() (t time.Time) {
 	return
 }
 
-func (d *simpleDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
+func (d *simpleDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) {
 	if xtag > 0xff {
 		d.d.errorf("ext: tag must be <= 0xff; got: %v", xtag)
 		return
 	}
+	if d.advanceNil() {
+		return
+	}
 	realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag))
-	realxtag = uint64(realxtag1)
+	realxtag := uint64(realxtag1)
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = realxtag
@@ -483,13 +495,9 @@ func (d *simpleDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realx
 	} else {
 		ext.ReadExt(rv, xbs)
 	}
-	return
 }
 
 func (d *simpleDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []byte) {
-	if !d.bdRead {
-		d.readNextBd()
-	}
 	switch d.bd {
 	case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4:
 		l := d.decLen()
@@ -519,12 +527,14 @@ func (d *simpleDecDriver) DecodeNaked() {
 		d.readNextBd()
 	}
 
+	d.fnil = false
 	n := d.d.naked()
 	var decodeFurther bool
 
 	switch d.bd {
 	case simpleVdNil:
 		n.v = valueTypeNil
+		d.fnil = true
 	case simpleVdFalse:
 		n.v = valueTypeBool
 		n.b = false
@@ -639,6 +649,7 @@ func (e *simpleEncDriver) reset() {
 func (d *simpleDecDriver) reset() {
 	d.r, d.br = d.d.r(), d.d.bytes
 	d.bd, d.bdRead = 0, false
+	d.fnil = false
 }
 
 var _ decDriver = (*simpleDecDriver)(nil)