codec(gen): code clean-up to remove separator concerns from framework and major re-architecture.

Up until now, the framework that handles encode and decode did all the work to manage
formats that used separators and those that didn't.

This led to the following issues:

- "framework" handling a lot of functionality that only applies to json
  e.g. handling separators [:,] appropriately.
- challenging to support other formats e.g. XML,
  which do not use separators but are text-based. It also made it hard to put changes in,
  since all work done by the framework in reflection-mode MUST be replicated in generated-mode.
- Generated code was long to handle all the ideosynchracies of the different formats.

We have now simplified the code to push handling of different separators and end tags to JSON
(who cares about it). This created a lot of cleanup by:
- removing separator handling code and its complexities from reflection and codecgen code
- reduced the size of generated files

JSONHandle will now handle reading and emitting separators for arrays, maps and closing tags
itself. It keeps a stack to track where it is in the "tree" and handles everything.

All these changes have been tested in all permutations, and they work.

In addition, we now keep track of number of bytes read (during decoding) and annotate
any error message with it, so we can track whee the decoding fails (if/when it does).

--------------

We also created a tests.sh script that runs all permutations of our functional
tests, running all tests agains reflection, codecgen and codecgen+unsafe,
and testing canonical, read/write from []byte or io.Reader|Writer, struct2array, etc.

All tests pass.

-----------------

In addition, we did the following:

- Binc encoding builtin for time.Time should handle *time.Time also.
- doc clear requirements for running python-based msgpack/cbor tests
- clearly document requirements for python-based msgpack/cbor tests.

-----------------

Finally, we updated GenVersion to 4, to reflect all these changes, and so
users have an quick fail and know to re-generate their static code.

-----------------

ENJOY.

codec/binc.go

@@ -69,7 +69,15 @@ func (e *bincEncDriver) IsBuiltinType(rt uintptr) bool {
 
 func (e *bincEncDriver) EncodeBuiltin(rt uintptr, v interface{}) {
 	if rt == timeTypId {
-		bs := encodeTime(v.(time.Time))
+		var bs []byte
+		switch x := v.(type) {
+		case time.Time:
+			bs = encodeTime(x)
+		case *time.Time:
+			bs = encodeTime(*x)
+		default:
+			e.e.errorf("binc error encoding builtin: expect time.Time, received %T", v)
+		}
 		e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs)))
 		e.w.writeb(bs)
 	}

codec/decode.go

@@ -25,10 +25,6 @@ var (
 // decReader abstracts the reading source, allowing implementations that can
 // read from an io.Reader or directly off a byte slice with zero-copying.
 type decReader interface {
-	// TODO:
-	//   Add method to get num bytes read.
-	//   This will be used to annotate errors, so user knows at what point the error occurred.
-
 	unreadn1()
 
 	// readx will use the implementation scratch buffer if possible i.e. n < len(scratchbuf), OR
@@ -38,7 +34,7 @@ type decReader interface {
 	readb([]byte)
 	readn1() uint8
 	readn1eof() (v uint8, eof bool)
-
+	numread() int // number of bytes read
 	track()
 	stopTrack() []byte
 }
@@ -82,20 +78,13 @@ type decDriver interface {
 	// decodeExt(verifyTag bool, tag byte) (xtag byte, xbs []byte)
 	ReadMapStart() int
 	ReadArrayStart() int
-	ReadMapEnd()
-	ReadArrayEnd()
-	ReadArrayEntrySeparator()
-	ReadMapEntrySeparator()
-	ReadMapKVSeparator()
+	// ReadEnd registers the end of a map or array.
+	ReadEnd()
 }
 
 type decNoSeparator struct{}
 
-func (_ decNoSeparator) ReadMapEnd()              {}
-func (_ decNoSeparator) ReadArrayEnd()            {}
-func (_ decNoSeparator) ReadArrayEntrySeparator() {}
-func (_ decNoSeparator) ReadMapEntrySeparator()   {}
-func (_ decNoSeparator) ReadMapKVSeparator()      {}
+func (_ decNoSeparator) ReadEnd() {}
 
 type DecodeOptions struct {
 	// MapType specifies type to use during schema-less decoding of a map in the stream.
@@ -184,13 +173,17 @@ type ioDecReader struct {
 	br decReaderByteScanner
 	// temp byte array re-used internally for efficiency during read.
 	// shares buffer with Decoder, so we keep size of struct within 8 words.
-	x  *[scratchByteArrayLen]byte
-	bs ioDecByteScanner
-
+	x   *[scratchByteArrayLen]byte
+	bs  ioDecByteScanner
+	n   int    // num read
 	tr  []byte // tracking bytes read
 	trb bool
 }
 
+func (z *ioDecReader) numread() int {
+	return z.n
+}
+
 func (z *ioDecReader) readx(n int) (bs []byte) {
 	if n <= 0 {
 		return
@@ -203,6 +196,7 @@ func (z *ioDecReader) readx(n int) (bs []byte) {
 	if _, err := io.ReadAtLeast(z.br, bs, n); err != nil {
 		panic(err)
 	}
+	z.n += len(bs)
 	if z.trb {
 		z.tr = append(z.tr, bs...)
 	}
@@ -213,7 +207,9 @@ func (z *ioDecReader) readb(bs []byte) {
 	if len(bs) == 0 {
 		return
 	}
-	if _, err := io.ReadAtLeast(z.br, bs, len(bs)); err != nil {
+	n, err := io.ReadAtLeast(z.br, bs, len(bs))
+	z.n += n
+	if err != nil {
 		panic(err)
 	}
 	if z.trb {
@@ -226,6 +222,7 @@ func (z *ioDecReader) readn1() (b uint8) {
 	if err != nil {
 		panic(err)
 	}
+	z.n++
 	if z.trb {
 		z.tr = append(z.tr, b)
 	}
@@ -235,21 +232,24 @@ func (z *ioDecReader) readn1() (b uint8) {
 func (z *ioDecReader) readn1eof() (b uint8, eof bool) {
 	b, err := z.br.ReadByte()
 	if err == nil {
+		z.n++
+		if z.trb {
+			z.tr = append(z.tr, b)
+		}
 	} else if err == io.EOF {
 		eof = true
 	} else {
 		panic(err)
 	}
-	if z.trb {
-		z.tr = append(z.tr, b)
-	}
 	return
 }
 
 func (z *ioDecReader) unreadn1() {
-	if err := z.br.UnreadByte(); err != nil {
+	err := z.br.UnreadByte()
+	if err != nil {
 		panic(err)
 	}
+	z.n--
 	if z.trb {
 		if l := len(z.tr) - 1; l >= 0 {
 			z.tr = z.tr[:l]
@@ -281,6 +281,10 @@ type bytesDecReader struct {
 	t int    // track start
 }
 
+func (z *bytesDecReader) numread() int {
+	return z.c
+}
+
 func (z *bytesDecReader) unreadn1() {
 	if z.c == 0 || len(z.b) == 0 {
 		panic(bytesDecReaderCannotUnreadErr)
@@ -417,14 +421,6 @@ func (f decFnInfo) binaryUnmarshal(rv reflect.Value) {
 func (f decFnInfo) textUnmarshal(rv reflect.Value) {
 	tm := f.getValueForUnmarshalInterface(rv, f.ti.tunmIndir).(encoding.TextUnmarshaler)
 	fnerr := tm.UnmarshalText(f.dd.DecodeBytes(f.d.b[:], true, true))
-	// fnerr := tm.UnmarshalText(f.dd.DecodeStringAsBytes(f.d.b[:]))
-
-	// var fnerr error
-	// if sb, sbok := f.dd.(decDriverStringAsBytes); sbok {
-	// 	fnerr = tm.UnmarshalText(sb.decStringAsBytes(f.d.b[:0]))
-	// } else {
-	// 	fnerr = tm.UnmarshalText([]byte(f.dd.decodeString()))
-	// }
 	if fnerr != nil {
 		panic(fnerr)
 	}
@@ -433,7 +429,7 @@ func (f decFnInfo) textUnmarshal(rv reflect.Value) {
 func (f decFnInfo) jsonUnmarshal(rv reflect.Value) {
 	tm := f.getValueForUnmarshalInterface(rv, f.ti.junmIndir).(jsonUnmarshaler)
 	// bs := f.dd.DecodeBytes(f.d.b[:], true, true)
-	// grab the bytes to be read
+	// grab the bytes to be read, as UnmarshalJSON wants the full JSON to unmarshal it itself.
 	f.d.r.track()
 	f.d.swallow()
 	bs := f.d.r.stopTrack()
@@ -621,7 +617,7 @@ func (f decFnInfo) kStruct(rv reflect.Value) {
 	if f.dd.IsContainerType(valueTypeMap) {
 		containerLen := f.dd.ReadMapStart()
 		if containerLen == 0 {
-			f.dd.ReadMapEnd()
+			f.dd.ReadEnd()
 			return
 		}
 		tisfi := fti.sfi
@@ -644,12 +640,8 @@ func (f decFnInfo) kStruct(rv reflect.Value) {
 			}
 		} else {
 			for j := 0; !f.dd.CheckBreak(); j++ {
-				if j > 0 {
-					f.dd.ReadMapEntrySeparator()
-				}
 				// rvkencname := f.dd.DecodeString()
 				rvkencname := stringView(f.dd.DecodeBytes(f.d.b[:], true, true))
-				f.dd.ReadMapKVSeparator()
 				// rvksi := ti.getForEncName(rvkencname)
 				if k := fti.indexForEncName(rvkencname); k > -1 {
 					si := tisfi[k]
@@ -662,12 +654,12 @@ func (f decFnInfo) kStruct(rv reflect.Value) {
 					d.structFieldNotFound(-1, rvkencname)
 				}
 			}
-			f.dd.ReadMapEnd()
+			f.dd.ReadEnd()
 		}
 	} else if f.dd.IsContainerType(valueTypeArray) {
 		containerLen := f.dd.ReadArrayStart()
 		if containerLen == 0 {
-			f.dd.ReadArrayEnd()
+			f.dd.ReadEnd()
 			return
 		}
 		// Not much gain from doing it two ways for array.
@@ -681,30 +673,19 @@ func (f decFnInfo) kStruct(rv reflect.Value) {
 			} else if f.dd.CheckBreak() {
 				break
 			}
-			if j > 0 {
-				f.dd.ReadArrayEntrySeparator()
-			}
 			if f.dd.TryDecodeAsNil() {
 				si.setToZeroValue(rv)
 			} else {
 				d.decodeValue(si.field(rv, true), decFn{})
 			}
-			// if si.i != -1 {
-			// 	d.decodeValue(rv.Field(int(si.i)), decFn{})
-			// } else {
-			// 	d.decEmbeddedField(rv, si.is)
-			// }
 		}
 		if containerLen > len(fti.sfip) {
 			// read remaining values and throw away
 			for j := len(fti.sfip); j < containerLen; j++ {
-				if j > 0 {
-					f.dd.ReadArrayEntrySeparator()
-				}
 				d.structFieldNotFound(j, "")
 			}
 		}
-		f.dd.ReadArrayEnd()
+		f.dd.ReadEnd()
 	} else {
 		f.d.error(onlyMapOrArrayCanDecodeIntoStructErr)
 		return
@@ -766,7 +747,7 @@ func (f decFnInfo) kSlice(rv reflect.Value) {
 		if f.seq == seqTypeSlice && rvlen != 0 {
 			rv.SetLen(0)
 		}
-		// slh.End() // f.dd.ReadArrayEnd()
+		// f.dd.ReadEnd()
 		return
 	}
 
@@ -839,9 +820,6 @@ func (f decFnInfo) kSlice(rv reflect.Value) {
 					rvChanged = true
 				}
 			}
-			if j > 0 {
-				slh.Sep(j)
-			}
 			if f.seq == seqTypeChan {
 				rv0 := reflect.New(rtelem0).Elem()
 				d.decodeValue(rv0, fn)
@@ -874,6 +852,7 @@ func (f decFnInfo) kMap(rv reflect.Value) {
 	}
 
 	if containerLen == 0 {
+		// It is not length-prefix style container. They have no End marker.
 		// f.dd.ReadMapEnd()
 		return
 	}
@@ -913,9 +892,6 @@ func (f decFnInfo) kMap(rv reflect.Value) {
 		}
 	} else {
 		for j := 0; !f.dd.CheckBreak(); j++ {
-			if j > 0 {
-				f.dd.ReadMapEntrySeparator()
-			}
 			rvk := reflect.New(ktype).Elem()
 			d.decodeValue(rvk, keyFn)
 
@@ -930,11 +906,10 @@ func (f decFnInfo) kMap(rv reflect.Value) {
 			if !rvv.IsValid() {
 				rvv = reflect.New(vtype).Elem()
 			}
-			f.dd.ReadMapKVSeparator()
 			d.decodeValue(rvv, valFn)
 			rv.SetMapIndex(rvk, rvv)
 		}
-		f.dd.ReadMapEnd()
+		f.dd.ReadEnd()
 	}
 }
 
@@ -949,6 +924,8 @@ type Decoder struct {
 	// Try to put things that go together to fit within a cache line (8 words).
 
 	d decDriver
+	// NOTE: Decoder shouldn't call it's read methods,
+	// as the handler MAY need to do some coordination.
 	r decReader
 	//sa [32]rtidDecFn
 	s []rtidDecFn
@@ -1079,14 +1056,10 @@ func (d *Decoder) swallow() {
 			} else if dd.CheckBreak() {
 				break
 			}
-			if j > 0 {
-				dd.ReadMapEntrySeparator()
-			}
 			d.swallow()
-			dd.ReadMapKVSeparator()
 			d.swallow()
 		}
-		dd.ReadMapEnd()
+		dd.ReadEnd()
 	case dd.IsContainerType(valueTypeArray):
 		containerLenS := dd.ReadArrayStart()
 		clenGtEqualZero := containerLenS >= 0
@@ -1098,12 +1071,9 @@ func (d *Decoder) swallow() {
 			} else if dd.CheckBreak() {
 				break
 			}
-			if j > 0 {
-				dd.ReadArrayEntrySeparator()
-			}
 			d.swallow()
 		}
-		dd.ReadArrayEnd()
+		dd.ReadEnd()
 	case dd.IsContainerType(valueTypeBytes):
 		dd.DecodeBytes(d.b[:], false, true)
 	case dd.IsContainerType(valueTypeString):
@@ -1488,7 +1458,10 @@ func (d *Decoder) error(err error) {
 }
 
 func (d *Decoder) errorf(format string, params ...interface{}) {
-	err := fmt.Errorf(format, params...)
+	params2 := make([]interface{}, len(params)+1)
+	params2[0] = d.r.numread()
+	copy(params2[1:], params)
+	err := fmt.Errorf("[pos %d]: "+format, params2...)
 	panic(err)
 }
 
@@ -1515,30 +1488,10 @@ func (d *Decoder) decSliceHelperStart() (x decSliceHelper, clen int) {
 	return
 }
 
-func (x decSliceHelper) Sep(index int) {
-	if x.ct == valueTypeArray {
-		x.dd.ReadArrayEntrySeparator()
-	} else {
-		if index%2 == 0 {
-			x.dd.ReadMapEntrySeparator()
-		} else {
-			x.dd.ReadMapKVSeparator()
-		}
-	}
-}
-
 func (x decSliceHelper) End() {
-	if x.ct == valueTypeArray {
-		x.dd.ReadArrayEnd()
-	} else {
-		x.dd.ReadMapEnd()
-	}
+	x.dd.ReadEnd()
 }
 
-// func decErr(format string, params ...interface{}) {
-// 	doPanic(msgTagDec, format, params...)
-// }
-
 func decByteSlice(r decReader, clen int, bs []byte) (bsOut []byte) {
 	if clen == 0 {
 		return zeroByteSlice
@@ -1587,37 +1540,5 @@ func detachZeroCopyBytes(isBytesReader bool, dest []byte, in []byte) (out []byte
 // 		d.ri.unreadn1()
 // 	}
 // }
-
-// func (d *Decoder) readb(b []byte) {
-// 	if d.bytes {
-// 		d.rb.readb(b)
-// 	} else {
-// 		d.ri.readb(b)
-// 	}
-// }
-
-// func (d *Decoder) readx(n int) []byte {
-// 	if d.bytes {
-// 		return d.rb.readx(n)
-// 	} else {
-// 		return d.ri.readx(n)
-// 	}
-// }
-
-// func (d *Decoder) readn1() uint8 {
-// 	if d.bytes {
-// 		return d.rb.readn1()
-// 	} else {
-// 		return d.ri.readn1()
-// 	}
-// }
-
-// func (d *Decoder) readn1eof() (v uint8, eof bool) {
-// 	if d.bytes {
-// 		return d.rb.readn1eof()
-// 	} else {
-// 		return d.ri.readn1eof()
-// 	}
-// }
-
-// var _ decReader = (*Decoder)(nil) // decReaderT{} //
+// ... for other methods of decReader.
+// Testing showed that performance improvement was negligible.

codec/encode.go

@@ -63,12 +63,8 @@ type encDriver interface {
 	EncodeRawExt(re *RawExt, e *Encoder)
 	EncodeExt(v interface{}, xtag uint64, ext Ext, e *Encoder)
 	EncodeArrayStart(length int)
-	EncodeArrayEnd()
-	EncodeArrayEntrySeparator()
 	EncodeMapStart(length int)
-	EncodeMapEnd()
-	EncodeMapEntrySeparator()
-	EncodeMapKVSeparator()
+	EncodeEnd()
 	EncodeString(c charEncoding, v string)
 	EncodeSymbol(v string)
 	EncodeStringBytes(c charEncoding, v []byte)
@@ -77,13 +73,13 @@ type encDriver interface {
 	//encStringRunes(c charEncoding, v []rune)
 }
 
+type encDriverAsis interface {
+	EncodeAsis(v []byte)
+}
+
 type encNoSeparator struct{}
 
-func (_ encNoSeparator) EncodeMapEnd()              {}
-func (_ encNoSeparator) EncodeArrayEnd()            {}
-func (_ encNoSeparator) EncodeArrayEntrySeparator() {}
-func (_ encNoSeparator) EncodeMapEntrySeparator()   {}
-func (_ encNoSeparator) EncodeMapKVSeparator()      {}
+func (_ encNoSeparator) EncodeEnd() {}
 
 type encStructFieldBytesV struct {
 	b []byte
@@ -466,7 +462,6 @@ func (f encFnInfo) kSlice(rv reflect.Value) {
 	}
 
 	e := f.e
-	sep := !e.be
 	if l > 0 {
 		for rtelem.Kind() == reflect.Ptr {
 			rtelem = rtelem.Elem()
@@ -480,47 +475,19 @@ func (f encFnInfo) kSlice(rv reflect.Value) {
 			fn = e.getEncFn(rtelemid, rtelem, true, true)
 		}
 		// TODO: Consider perf implication of encoding odd index values as symbols if type is string
-		if sep {
-			for j := 0; j < l; j++ {
-				if j > 0 {
-					if ti.mbs {
-						if j%2 == 0 {
-							f.ee.EncodeMapEntrySeparator()
-						} else {
-							f.ee.EncodeMapKVSeparator()
-						}
-					} else {
-						f.ee.EncodeArrayEntrySeparator()
-					}
-				}
-				if f.seq == seqTypeChan {
-					if rv2, ok2 := rv.Recv(); ok2 {
-						e.encodeValue(rv2, fn)
-					}
-				} else {
-					e.encodeValue(rv.Index(j), fn)
-				}
-			}
-		} else {
-			for j := 0; j < l; j++ {
-				if f.seq == seqTypeChan {
-					if rv2, ok2 := rv.Recv(); ok2 {
-						e.encodeValue(rv2, fn)
-					}
-				} else {
-					e.encodeValue(rv.Index(j), fn)
+		for j := 0; j < l; j++ {
+			if f.seq == seqTypeChan {
+				if rv2, ok2 := rv.Recv(); ok2 {
+					e.encodeValue(rv2, fn)
 				}
+			} else {
+				e.encodeValue(rv.Index(j), fn)
 			}
 		}
-	}
 
-	if sep {
-		if ti.mbs {
-			f.ee.EncodeMapEnd()
-		} else {
-			f.ee.EncodeArrayEnd()
-		}
 	}
+
+	f.ee.EncodeEnd()
 }
 
 func (f encFnInfo) kStruct(rv reflect.Value) {
@@ -590,60 +557,30 @@ func (f encFnInfo) kStruct(rv reflect.Value) {
 	}
 
 	// debugf(">>>> kStruct: newlen: %v", newlen)
-	sep := !e.be
+	// sep := !e.be
 	ee := f.ee //don't dereference everytime
-	if sep {
-		if toMap {
-			ee.EncodeMapStart(newlen)
-			// asSymbols := e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
-			asSymbols := e.h.AsSymbols == AsSymbolDefault || e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
-			for j := 0; j < newlen; j++ {
-				kv = fkvs[j]
-				if j > 0 {
-					ee.EncodeMapEntrySeparator()
-				}
-				if asSymbols {
-					ee.EncodeSymbol(kv.k)
-				} else {
-					ee.EncodeString(c_UTF8, kv.k)
-				}
-				ee.EncodeMapKVSeparator()
-				e.encodeValue(kv.v, encFn{})
-			}
-			ee.EncodeMapEnd()
-		} else {
-			ee.EncodeArrayStart(newlen)
-			for j := 0; j < newlen; j++ {
-				kv = fkvs[j]
-				if j > 0 {
-					ee.EncodeArrayEntrySeparator()
-				}
-				e.encodeValue(kv.v, encFn{})
+
+	if toMap {
+		ee.EncodeMapStart(newlen)
+		// asSymbols := e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
+		asSymbols := e.h.AsSymbols == AsSymbolDefault || e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
+		for j := 0; j < newlen; j++ {
+			kv = fkvs[j]
+			if asSymbols {
+				ee.EncodeSymbol(kv.k)
+			} else {
+				ee.EncodeString(c_UTF8, kv.k)
 			}
-			ee.EncodeArrayEnd()
+			e.encodeValue(kv.v, encFn{})
 		}
 	} else {
-		if toMap {
-			ee.EncodeMapStart(newlen)
-			// asSymbols := e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
-			asSymbols := e.h.AsSymbols == AsSymbolDefault || e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
-			for j := 0; j < newlen; j++ {
-				kv = fkvs[j]
-				if asSymbols {
-					ee.EncodeSymbol(kv.k)
-				} else {
-					ee.EncodeString(c_UTF8, kv.k)
-				}
-				e.encodeValue(kv.v, encFn{})
-			}
-		} else {
-			ee.EncodeArrayStart(newlen)
-			for j := 0; j < newlen; j++ {
-				kv = fkvs[j]
-				e.encodeValue(kv.v, encFn{})
-			}
+		ee.EncodeArrayStart(newlen)
+		for j := 0; j < newlen; j++ {
+			kv = fkvs[j]
+			e.encodeValue(kv.v, encFn{})
 		}
 	}
+	ee.EncodeEnd()
 
 	// do not use defer. Instead, use explicit pool return at end of function.
 	// defer has a cost we are trying to avoid.
@@ -679,11 +616,8 @@ func (f encFnInfo) kMap(rv reflect.Value) {
 	l := rv.Len()
 	f.ee.EncodeMapStart(l)
 	e := f.e
-	sep := !e.be
 	if l == 0 {
-		if sep {
-			f.ee.EncodeMapEnd()
-		}
+		f.ee.EncodeEnd()
 		return
 	}
 	var asSymbols bool
@@ -733,35 +667,13 @@ func (f encFnInfo) kMap(rv reflect.Value) {
 			e2.MustEncode(k)
 			mksbv[i].v = k
 			mksbv[i].b = mksv[l:]
+			// fmt.Printf(">>>>> %s\n", mksv[l:])
 		}
 		sort.Sort(encStructFieldBytesVslice(mksbv))
 		for j := range mksbv {
-			if j > 0 {
-				ee.EncodeMapEntrySeparator()
-			}
-			e.w.writeb(mksbv[j].b)
-			ee.EncodeMapKVSeparator()
+			e.asis(mksbv[j].b)
 			e.encodeValue(rv.MapIndex(mksbv[j].v), valFn)
 		}
-		ee.EncodeMapEnd()
-	} else if sep {
-		for j := range mks {
-			if j > 0 {
-				ee.EncodeMapEntrySeparator()
-			}
-			if keyTypeIsString {
-				if asSymbols {
-					ee.EncodeSymbol(mks[j].String())
-				} else {
-					ee.EncodeString(c_UTF8, mks[j].String())
-				}
-			} else {
-				e.encodeValue(mks[j], keyFn)
-			}
-			ee.EncodeMapKVSeparator()
-			e.encodeValue(rv.MapIndex(mks[j]), valFn)
-		}
-		ee.EncodeMapEnd()
 	} else {
 		for j := range mks {
 			if keyTypeIsString {
@@ -776,6 +688,7 @@ func (f encFnInfo) kMap(rv reflect.Value) {
 			e.encodeValue(rv.MapIndex(mks[j]), valFn)
 		}
 	}
+	ee.EncodeEnd()
 }
 
 // --------------------------------------------------
@@ -799,7 +712,9 @@ type rtidEncFn struct {
 // An Encoder writes an object to an output stream in the codec format.
 type Encoder struct {
 	// hopefully, reduce derefencing cost by laying the encWriter inside the Encoder
-	e  encDriver
+	e encDriver
+	// NOTE: Encoder shouldn't call it's write methods,
+	// as the handler MAY need to do some coordination.
 	w  encWriter
 	s  []rtidEncFn
 	be bool // is binary encoding
@@ -809,6 +724,7 @@ type Encoder struct {
 	wb bytesEncWriter
 	h  *BasicHandle
 
+	as encDriverAsis
 	hh Handle
 	f  map[uintptr]encFn
 	b  [scratchByteArrayLen]byte
@@ -832,6 +748,7 @@ func NewEncoder(w io.Writer, h Handle) *Encoder {
 	e.w = &e.wi
 	_, e.js = h.(*JsonHandle)
 	e.e = h.newEncDriver(e)
+	e.as, _ = e.e.(encDriverAsis)
 	return e
 }
 
@@ -850,6 +767,7 @@ func NewEncoderBytes(out *[]byte, h Handle) *Encoder {
 	e.w = &e.wb
 	_, e.js = h.(*JsonHandle)
 	e.e = h.newEncDriver(e)
+	e.as, _ = e.e.(encDriverAsis)
 	return e
 }
 
@@ -1219,12 +1137,20 @@ func (e *Encoder) marshal(bs []byte, fnerr error, asis bool, c charEncoding) {
 	if bs == nil {
 		e.e.EncodeNil()
 	} else if asis {
-		e.w.writeb(bs)
+		e.asis(bs)
 	} else {
 		e.e.EncodeStringBytes(c, bs)
 	}
 }
 
+func (e *Encoder) asis(v []byte) {
+	if e.as == nil {
+		e.w.writeb(v)
+	} else {
+		e.as.EncodeAsis(v)
+	}
+}
+
 func (e *Encoder) errorf(format string, params ...interface{}) {
 	err := fmt.Errorf(format, params...)
 	panic(err)

codec/fast-path.go.tmpl

@@ -162,19 +162,10 @@ func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v []{{ .Elem }}, checkNil b
 		return
 	}
 	ee.EncodeArrayStart(len(v))
-	if e.be {
-		for _, v2 := range v {
-			{{ encmd .Elem "v2"}}
-		}
-	} else {
-		for j, v2 := range v {
-			if j > 0 {
-				ee.EncodeArrayEntrySeparator()
-			}
-			{{ encmd .Elem "v2"}}
-		}
-		ee.EncodeArrayEnd()
+	for _, v2 := range v {
+		{{ encmd .Elem "v2"}}
 	}
+	ee.EncodeEnd()
 }
 
 {{end}}{{end}}{{end}}
@@ -192,32 +183,15 @@ func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v map[{{ .MapKey }}]{{ .Ele
 	}
 	ee.EncodeMapStart(len(v))
 	{{if eq .MapKey "string"}}asSymbols := e.h.AsSymbols&AsSymbolMapStringKeysFlag != 0{{end}}
-	if e.be {
-		for k2, v2 := range v {
-			{{if eq .MapKey "string"}}if asSymbols {
-				ee.EncodeSymbol(k2)
-			} else {
-				ee.EncodeString(c_UTF8, k2)
-			}{{else}}{{ encmd .MapKey "k2"}}{{end}}
-			{{ encmd .Elem "v2"}}
-		}
-	} else {
-		j := 0
-		for k2, v2 := range v {
-			if j > 0 {
-				ee.EncodeMapEntrySeparator()
-			}
-			{{if eq .MapKey "string"}}if asSymbols {
-				ee.EncodeSymbol(k2)
-			} else {
-				ee.EncodeString(c_UTF8, k2)
-			}{{else}}{{ encmd .MapKey "k2"}}{{end}}
-			ee.EncodeMapKVSeparator()
-			{{ encmd .Elem "v2"}}
-			j++
-		}
-		ee.EncodeMapEnd()
+	for k2, v2 := range v {
+		{{if eq .MapKey "string"}}if asSymbols {
+			ee.EncodeSymbol(k2)
+		} else {
+			ee.EncodeString(c_UTF8, k2)
+		}{{else}}{{ encmd .MapKey "k2"}}{{end}}
+		{{ encmd .Elem "v2"}}
 	}
+	ee.EncodeEnd()
 }
 
 {{end}}{{end}}{{end}}
@@ -340,9 +314,6 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil b
 					d.arrayCannotExpand(len(v), j+1)
 				}
 			} 
-			if j > 0 {
-				slh.Sep(j)
-			}
 			if j < len(v) { // all checks done. cannot go past len.
 				{{ if eq .Elem "interface{}" }}d.decode(&v[j])
 				{{ else }}v[j] = {{ decmd .Elem }}{{ end }}
@@ -418,15 +389,11 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Ele
 		}
 	} else if containerLen < 0 {
 		for j := 0; !dd.CheckBreak(); j++ {
-			if j > 0 {
-				dd.ReadMapEntrySeparator()
-			}
 			{{ if eq .MapKey "interface{}" }}var mk interface{}
 			d.decode(&mk)
 			if bv, bok := mk.([]byte); bok {
 				mk = string(bv) // maps cannot have []byte as key. switch to string.
 			}{{ else }}mk := {{ decmd .MapKey }}{{ end }}
-			dd.ReadMapKVSeparator()
 			mv := v[mk]
 			{{ if eq .Elem "interface{}" }}d.decode(&mv)
 			{{ else }}mv = {{ decmd .Elem }}{{ end }}
@@ -434,7 +401,7 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Ele
 				v[mk] = mv
 			}
 		}
-		dd.ReadMapEnd()
+		dd.ReadEnd()
 	}
 	return v, changed
 }

codec/gen-dec-array.go.tmpl

@@ -57,9 +57,6 @@ if {{var "l"}} == 0 { {{ if isSlice }}
 			{{ else if isSlice}}{{var "v"}} = append({{var "v"}}, {{zero}})// var {{var "z"}} {{ .Typ }}
 			{{var "c"}} = true {{ end }}
 		}
-		if {{var "j"}} > 0 {
-			{{var "h"}}.Sep({{var "j"}})
-		}
 		{{ if isChan}}
 		var {{var "t"}} {{ .Typ }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}

codec/gen-dec-map.go.tmpl

@@ -25,9 +25,6 @@ for {{var "j"}} := 0; {{var "j"}} < {{var "l"}}; {{var "j"}}++ {
 }
 } else if {{var "l"}} < 0  {
 for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
-	if {{var "j"}} > 0 {
-		r.ReadMapEntrySeparator()
-	}
 	var {{var "mk"}} {{ .KTyp }} 
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x }}
 {{ if eq .KTyp "interface{}" }}// special case if a byte array.
@@ -35,12 +32,11 @@ for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
 		{{var "mk"}} = string({{var "bv"}})
 	}
 {{ end }}
-	r.ReadMapKVSeparator()
 	{{var "mv"}} := {{var "v"}}[{{var "mk"}}]
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ decLineVar $x }}
 	if {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
-r.ReadMapEnd()
+r.ReadEnd()
 } // else len==0: TODO: Should we clear map entries?

codec/gen.generated.go

@@ -33,9 +33,6 @@ for {{var "j"}} := 0; {{var "j"}} < {{var "l"}}; {{var "j"}}++ {
 }
 } else if {{var "l"}} < 0  {
 for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
-	if {{var "j"}} > 0 {
-		r.ReadMapEntrySeparator()
-	}
 	var {{var "mk"}} {{ .KTyp }} 
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x }}
 {{ if eq .KTyp "interface{}" }}// special case if a byte array.
@@ -43,14 +40,13 @@ for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
 		{{var "mk"}} = string({{var "bv"}})
 	}
 {{ end }}
-	r.ReadMapKVSeparator()
 	{{var "mv"}} := {{var "v"}}[{{var "mk"}}]
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ decLineVar $x }}
 	if {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
-r.ReadMapEnd()
+r.ReadEnd()
 } // else len==0: TODO: Should we clear map entries?
 `
 
@@ -114,9 +110,6 @@ if {{var "l"}} == 0 { {{ if isSlice }}
 			{{ else if isSlice}}{{var "v"}} = append({{var "v"}}, {{zero}})// var {{var "z"}} {{ .Typ }}
 			{{var "c"}} = true {{ end }}
 		}
-		if {{var "j"}} > 0 {
-			{{var "h"}}.Sep({{var "j"}})
-		}
 		{{ if isChan}}
 		var {{var "t"}} {{ .Typ }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}

codec/gen.go

@@ -84,7 +84,12 @@ import (
 //   - helper methods change (signature change, new ones added, some removed, etc)
 //   - codecgen command line changes
 //
-const GenVersion = 3
+// v1: Initial Version
+// v2:
+// v3: Changes for Kubernetes:
+//     changes in signature of some unpublished helper methods and codecgen cmdline arguments.
+// v4: Removed separator support from (en|de)cDriver, and refactored codec(gen)
+const GenVersion = 4
 
 const (
 	genCodecPkg        = "codec1978"
@@ -746,21 +751,19 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 	ti := getTypeInfo(rtid, t)
 	i := x.varsfx()
 	sepVarname := genTempVarPfx + "sep" + i
-	firstVarname := genTempVarPfx + "first" + i
 	numfieldsvar := genTempVarPfx + "q" + i
 	ti2arrayvar := genTempVarPfx + "r" + i
 	struct2arrvar := genTempVarPfx + "2arr" + i
 
 	x.line(sepVarname + " := !z.EncBinary()")
 	x.linef("%s := z.EncBasicHandle().StructToArray", struct2arrvar)
-	x.line("var " + firstVarname + " bool")
 	tisfi := ti.sfip // always use sequence from file. decStruct expects same thing.
 	// due to omitEmpty, we need to calculate the
 	// number of non-empty things we write out first.
 	// This is required as we need to pre-determine the size of the container,
 	// to support length-prefixing.
 	x.linef("var %s [%v]bool", numfieldsvar, len(tisfi))
-	x.linef("_, _, _, _ = %s, %s, %s, %s", sepVarname, firstVarname, numfieldsvar, struct2arrvar)
+	x.linef("_, _, _ = %s, %s, %s", sepVarname, numfieldsvar, struct2arrvar)
 	x.linef("const %s bool = %v", ti2arrayvar, ti.toArray)
 	nn := 0
 	for j, si := range tisfi {
@@ -845,11 +848,6 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 		// if the type of the field is a Selfer, or one of the ones
 
 		x.linef("if %s || %s {", ti2arrayvar, struct2arrvar) // if ti.toArray
-		if j > 0 {
-			x.line("if " + sepVarname + " {")
-			x.line("r.EncodeArrayEntrySeparator()")
-			x.line("}")
-		}
 		if labelUsed {
 			x.line("if " + isNilVarName + " { r.EncodeNil() } else { ")
 		}
@@ -881,17 +879,8 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 			// }
 			// x.line(varname + "." + t2.Name + " != " + genZeroValueR(t2.Type, x.tc) + " {")
 		}
-		if j == 0 {
-			x.linef("%s = true", firstVarname)
-		} else {
-			x.linef("if %s { r.EncodeMapEntrySeparator() } else { %s = true }", firstVarname, firstVarname)
-		}
-
 		// x.line("r.EncodeString(codecSelferC_UTF8" + x.xs + ", string(\"" + t2.Name + "\"))")
 		x.line("r.EncodeString(codecSelferC_UTF8" + x.xs + ", string(\"" + si.encName + "\"))")
-		x.line("if " + sepVarname + " {")
-		x.line("r.EncodeMapKVSeparator()")
-		x.line("}")
 		if labelUsed {
 			x.line("if " + isNilVarName + " { r.EncodeNil() } else { ")
 			x.encVar(varname+"."+t2.Name, t2.Type)
@@ -905,11 +894,7 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 		x.linef("} ") // end if/else ti.toArray
 	}
 	x.line("if " + sepVarname + " {")
-	x.linef("if %s || %s {", ti2arrayvar, struct2arrvar) // if ti.toArray {
-	x.line("r.EncodeArrayEnd()")
-	x.linef("} else {") // if not ti.toArray
-	x.line("r.EncodeMapEnd()")
-	x.linef("} ") // end if/else ti.toArray
+	x.line("r.EncodeEnd()")
 	x.line("}")
 }
 
@@ -917,56 +902,27 @@ func (x *genRunner) encListFallback(varname string, t reflect.Type) {
 	i := x.varsfx()
 	g := genTempVarPfx
 	x.line("r.EncodeArrayStart(len(" + varname + "))")
-	x.line(genTempVarPfx + "s" + i + " := !z.EncBinary()")
-	x.line("if " + genTempVarPfx + "s" + i + " {")
-	if t.Kind() == reflect.Chan {
-		x.linef("for %si%s, %si2%s := 0, len(%s); %si%s < %si2%s; %si%s++ {", g, i, g, i, varname, g, i, g, i, g, i)
-		x.linef("%sv%s := <-%s", g, i, varname)
-	} else {
-		x.linef("for %si%s, %sv%s := range %s {", genTempVarPfx, i, genTempVarPfx, i, varname)
-	}
-	x.linef("if %si%s > 0 { r.EncodeArrayEntrySeparator() }", genTempVarPfx, i)
-	x.encVar(genTempVarPfx+"v"+i, t.Elem())
-	x.line("}")
-	x.line("r.EncodeArrayEnd()")
-	x.line("} else {")
 	if t.Kind() == reflect.Chan {
 		x.linef("for %si%s, %si2%s := 0, len(%s); %si%s < %si2%s; %si%s++ {", g, i, g, i, varname, g, i, g, i, g, i)
 		x.linef("%sv%s := <-%s", g, i, varname)
 	} else {
-		x.line("for _, " + genTempVarPfx + "v" + i + " := range " + varname + " {")
+		// x.linef("for %si%s, %sv%s := range %s {", genTempVarPfx, i, genTempVarPfx, i, varname)
+		x.linef("for _, %sv%s := range %s {", genTempVarPfx, i, varname)
 	}
 	x.encVar(genTempVarPfx+"v"+i, t.Elem())
 	x.line("}")
-	x.line("}")
+	x.line("r.EncodeEnd()")
 }
 
 func (x *genRunner) encMapFallback(varname string, t reflect.Type) {
 	i := x.varsfx()
 	x.line("r.EncodeMapStart(len(" + varname + "))")
-	x.line(genTempVarPfx + "s" + i + " := !z.EncBinary()")
-
-	x.line(genTempVarPfx + "j" + i + " := 0")
-
-	x.line("if " + genTempVarPfx + "s" + i + " {")
-
-	x.line("for " + genTempVarPfx + "k" + i + ", " +
-		genTempVarPfx + "v" + i + " := range " + varname + " {")
-	x.line("if " + genTempVarPfx + "j" + i + " > 0 { r.EncodeMapEntrySeparator() }")
-	x.encVar(genTempVarPfx+"k"+i, t.Key())
-	x.line("r.EncodeMapKVSeparator()")
-	x.encVar(genTempVarPfx+"v"+i, t.Elem())
-	x.line(genTempVarPfx + "j" + i + "++")
-	x.line("}")
-	x.line("r.EncodeMapEnd()")
-
-	x.line("} else {")
 	x.linef("for %sk%s, %sv%s := range %s {", genTempVarPfx, i, genTempVarPfx, i, varname)
+	// x.line("for " + genTempVarPfx + "k" + i + ", " + genTempVarPfx + "v" + i + " := range " + varname + " {")
 	x.encVar(genTempVarPfx+"k"+i, t.Key())
 	x.encVar(genTempVarPfx+"v"+i, t.Elem())
 	x.line("}")
-
-	x.line("}")
+	x.line("r.EncodeEnd()")
 }
 
 func (x *genRunner) decVar(varname string, t reflect.Type, canBeNil bool) {
@@ -1426,13 +1382,11 @@ func (x *genRunner) decStructMap(varname, lenvarname string, rtid uintptr, t ref
 		x.linef("for %sj%s := 0; %sj%s < %s; %sj%s++ {", tpfx, i, tpfx, i, lenvarname, tpfx, i)
 	case 2:
 		x.linef("for %sj%s := 0; !r.CheckBreak(); %sj%s++ {", tpfx, i, tpfx, i)
-		x.linef("if %sj%s > 0 { r.ReadMapEntrySeparator() }", tpfx, i)
 	default: // 0, otherwise.
 		x.linef("var %shl%s bool = %s >= 0", tpfx, i, lenvarname) // has length
 		x.linef("for %sj%s := 0; ; %sj%s++ {", tpfx, i, tpfx, i)
 		x.linef("if %shl%s { if %sj%s >= %s { break }", tpfx, i, tpfx, i, lenvarname)
-		x.linef("} else { if r.CheckBreak() { break }; if %sj%s > 0 { r.ReadMapEntrySeparator() } }",
-			tpfx, i)
+		x.line("} else { if r.CheckBreak() { break }; }")
 	}
 	// x.line(kName + " = z.ReadStringAsBytes(" + kName + ")")
 	// x.line(kName + " = z.ReadString()")
@@ -1446,22 +1400,15 @@ func (x *genRunner) decStructMap(varname, lenvarname string, rtid uintptr, t ref
 	} else {
 		x.line(kName + " := string(" + kName + "Slc)")
 	}
-	switch style {
-	case 1:
-	case 2:
-		x.line("r.ReadMapKVSeparator()")
-	default:
-		x.linef("if !%shl%s { r.ReadMapKVSeparator() }", tpfx, i)
-	}
 	x.decStructMapSwitch(kName, varname, rtid, t)
 
 	x.line("} // end for " + tpfx + "j" + i)
 	switch style {
 	case 1:
 	case 2:
-		x.line("r.ReadMapEnd()")
+		x.line("r.ReadEnd()")
 	default:
-		x.linef("if !%shl%s { r.ReadMapEnd() }", tpfx, i)
+		x.linef("if !%shl%s { r.ReadEnd() }", tpfx, i)
 	}
 }
 
@@ -1474,7 +1421,7 @@ func (x *genRunner) decStructArray(varname, lenvarname, breakString string, rtid
 	x.linef("var %sb%s bool", tpfx, i) // break
 	// x.linef("var %sl%s := r.ReadArrayStart()", tpfx, i)
 	x.linef("var %shl%s bool = %s >= 0", tpfx, i, lenvarname) // has length
-	for j, si := range tisfi {
+	for _, si := range tisfi {
 		var t2 reflect.StructField
 		if si.i != -1 {
 			t2 = t.Field(int(si.i))
@@ -1487,10 +1434,7 @@ func (x *genRunner) decStructArray(varname, lenvarname, breakString string, rtid
 			tpfx, i, lenvarname, tpfx, i)
 		// x.line("if " + tpfx + "j" + i + "++; " + tpfx + "j" +
 		// i + " <=  " + tpfx + "l" + i + " {")
-		x.linef("if %sb%s { r.ReadArrayEnd(); %s }", tpfx, i, breakString)
-		if j > 0 {
-			x.line("r.ReadArrayEntrySeparator()")
-		}
+		x.linef("if %sb%s { r.ReadEnd(); %s }", tpfx, i, breakString)
 		x.decVar(varname+"."+t2.Name, t2.Type, true)
 		// x.line("} // end if " + tpfx + "j" + i + " <=  " + tpfx + "l" + i)
 	}
@@ -1500,10 +1444,9 @@ func (x *genRunner) decStructArray(varname, lenvarname, breakString string, rtid
 		tpfx, i, tpfx, i, tpfx, i,
 		tpfx, i, lenvarname, tpfx, i)
 	x.linef("if %sb%s { break }", tpfx, i)
-	x.linef("if %sj%s > 1 { r.ReadArrayEntrySeparator() }", tpfx, i)
 	x.linef(`z.DecStructFieldNotFound(%sj%s - 1, "")`, tpfx, i)
 	x.line("}")
-	x.line("r.ReadArrayEnd()")
+	x.line("r.ReadEnd()")
 }
 
 func (x *genRunner) decStruct(varname string, rtid uintptr, t reflect.Type) {
@@ -1513,7 +1456,7 @@ func (x *genRunner) decStruct(varname string, rtid uintptr, t reflect.Type) {
 	x.line("if r.IsContainerType(codecSelverValueTypeMap" + x.xs + ") {")
 	x.line(genTempVarPfx + "l" + i + " := r.ReadMapStart()")
 	x.linef("if %sl%s == 0 {", genTempVarPfx, i)
-	x.line("r.ReadMapEnd()")
+	x.line("r.ReadEnd()")
 	if genUseOneFunctionForDecStructMap {
 		x.line("} else { ")
 		x.linef("x.codecDecodeSelfFromMap(%sl%s, d)", genTempVarPfx, i)
@@ -1530,7 +1473,7 @@ func (x *genRunner) decStruct(varname string, rtid uintptr, t reflect.Type) {
 	x.line("} else if r.IsContainerType(codecSelverValueTypeArray" + x.xs + ") {")
 	x.line(genTempVarPfx + "l" + i + " := r.ReadArrayStart()")
 	x.linef("if %sl%s == 0 {", genTempVarPfx, i)
-	x.line("r.ReadArrayEnd()")
+	x.line("r.ReadEnd()")
 	x.line("} else { ")
 	x.linef("x.codecDecodeSelfFromArray(%sl%s, d)", genTempVarPfx, i)
 	x.line("}")

codec/json.go

@@ -27,6 +27,11 @@ package codec
 //   - encode does not beautify. There is no whitespace when encoding.
 //   - rpc calls which take single integer arguments or write single numeric arguments will need care.
 
+// Top-level methods of json(End|Dec)Driver (which are implementations of (en|de)cDriver
+// MUST not call one-another.
+// They all must call sep(), and sep() MUST NOT be called more than once for each read.
+// If sep() is called and read is not done, you MUST call retryRead so separator wouldn't be read/written twice.
+
 import (
 	"bytes"
 	"encoding/base64"
@@ -81,20 +86,102 @@ const (
 	// jsonNumDigitsUint64Largest = 19
 )
 
+// A stack is used to keep track of where we are in the tree.
+// This is necessary, as the Handle must know whether to consume or emit a separator.
+
+type jsonStackElem struct {
+	st byte // top of stack (either '}' or ']' or 0 for map, array or neither).
+	sf bool // NOT first time in that container at top of stack
+	so bool // stack ctr odd
+	sr bool // value has NOT been read, so do not re-send separator
+}
+
+func (x *jsonStackElem) retryRead() {
+	if x != nil && !x.sr {
+		x.sr = true
+	}
+}
+
+func (x *jsonStackElem) sep() (c byte) {
+	// do not use switch, so it's a candidate for inlining.
+	// to inline effectively, this must not be called from within another method.
+	// v := j.st
+	if x == nil || x.st == 0 {
+		return
+	}
+	if x.sr {
+		x.sr = false
+		return
+	}
+	// v == '}' OR ']'
+	if x.st == '}' {
+		// put , or : depending on if even or odd respectively
+		if x.so {
+			c = ':'
+			if !x.sf {
+				x.sf = true
+			}
+		} else if x.sf {
+			c = ','
+		}
+	} else {
+		if x.sf {
+			c = ','
+		} else {
+			x.sf = true
+		}
+	}
+	x.so = !x.so
+	if x.sr {
+		x.sr = false
+	}
+	return
+}
+
+// jsonStack contains the stack for tracking the state of the container (branch).
+// The same data structure is used during encode and decode, as it is similar functionality.
+type jsonStack struct {
+	s  []jsonStackElem // stack for map or array end tag. map=}, array=]
+	sc *jsonStackElem  // pointer to current (top) element on the stack.
+}
+
+func (j *jsonStack) start(c byte) {
+	j.s = append(j.s, jsonStackElem{st: c})
+	j.sc = &(j.s[len(j.s)-1])
+}
+
+func (j *jsonStack) end() {
+	l := len(j.s) - 1 // length of new stack after pop'ing
+	j.s = j.s[:l]
+	if l == 0 {
+		j.sc = nil
+	} else {
+		j.sc = &(j.s[l-1])
+	}
+	//j.sc = &(j.s[len(j.s)-1])
+}
+
 type jsonEncDriver struct {
 	e  *Encoder
 	w  encWriter
 	h  *JsonHandle
 	b  [64]byte // scratch
 	bs []byte   // scratch
+	s  jsonStack
 	noBuiltInTypes
 }
 
 func (e *jsonEncDriver) EncodeNil() {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.w.writeb(jsonLiterals[9:13]) // null
 }
 
 func (e *jsonEncDriver) EncodeBool(b bool) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	if b {
 		e.w.writeb(jsonLiterals[0:4]) // true
 	} else {
@@ -103,78 +190,101 @@ func (e *jsonEncDriver) EncodeBool(b bool) {
 }
 
 func (e *jsonEncDriver) EncodeFloat32(f float32) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.w.writeb(strconv.AppendFloat(e.b[:0], float64(f), 'E', -1, 32))
 }
 
 func (e *jsonEncDriver) EncodeFloat64(f float64) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	// e.w.writestr(strconv.FormatFloat(f, 'E', -1, 64))
 	e.w.writeb(strconv.AppendFloat(e.b[:0], f, 'E', -1, 64))
 }
 
 func (e *jsonEncDriver) EncodeInt(v int64) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.w.writeb(strconv.AppendInt(e.b[:0], v, 10))
 }
 
 func (e *jsonEncDriver) EncodeUint(v uint64) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.w.writeb(strconv.AppendUint(e.b[:0], v, 10))
 }
 
 func (e *jsonEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, en *Encoder) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	if v := ext.ConvertExt(rv); v == nil {
-		e.EncodeNil()
+		e.w.writeb(jsonLiterals[9:13]) // null // e.EncodeNil()
 	} else {
+		e.s.sc.retryRead()
 		en.encode(v)
 	}
 }
 
 func (e *jsonEncDriver) EncodeRawExt(re *RawExt, en *Encoder) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	// only encodes re.Value (never re.Data)
 	if re.Value == nil {
-		e.EncodeNil()
+		e.w.writeb(jsonLiterals[9:13]) // null // e.EncodeNil()
 	} else {
+		e.s.sc.retryRead()
 		en.encode(re.Value)
 	}
 }
 
 func (e *jsonEncDriver) EncodeArrayStart(length int) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
+	e.s.start(']')
 	e.w.writen1('[')
 }
 
-func (e *jsonEncDriver) EncodeArrayEntrySeparator() {
-	e.w.writen1(',')
-}
-
-func (e *jsonEncDriver) EncodeArrayEnd() {
-	e.w.writen1(']')
-}
-
 func (e *jsonEncDriver) EncodeMapStart(length int) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
+	e.s.start('}')
 	e.w.writen1('{')
 }
 
-func (e *jsonEncDriver) EncodeMapEntrySeparator() {
-	e.w.writen1(',')
-}
-
-func (e *jsonEncDriver) EncodeMapKVSeparator() {
-	e.w.writen1(':')
-}
-
-func (e *jsonEncDriver) EncodeMapEnd() {
-	e.w.writen1('}')
+func (e *jsonEncDriver) EncodeEnd() {
+	b := e.s.sc.st
+	e.s.end()
+	e.w.writen1(b)
 }
 
 func (e *jsonEncDriver) EncodeString(c charEncoding, v string) {
 	// e.w.writestr(strconv.Quote(v))
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.quoteStr(v)
 }
 
 func (e *jsonEncDriver) EncodeSymbol(v string) {
 	// e.EncodeString(c_UTF8, v)
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.quoteStr(v)
 }
 
 func (e *jsonEncDriver) EncodeStringBytes(c charEncoding, v []byte) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	if c == c_RAW {
 		slen := base64.StdEncoding.EncodedLen(len(v))
 		if e.bs == nil {
@@ -195,6 +305,13 @@ func (e *jsonEncDriver) EncodeStringBytes(c charEncoding, v []byte) {
 	}
 }
 
+func (e *jsonEncDriver) EncodeAsis(v []byte) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
+	e.w.writeb(v)
+}
+
 func (e *jsonEncDriver) quoteStr(s string) {
 	// adapted from std pkg encoding/json
 	const hex = "0123456789abcdef"
@@ -359,6 +476,8 @@ type jsonDecDriver struct {
 
 	wsSkipped bool // whitespace skipped
 
+	s jsonStack
+
 	n jsonNum
 	noBuiltInTypes
 }
@@ -402,16 +521,27 @@ func (d *jsonDecDriver) readStrIdx(fromIdx, toIdx uint8) {
 }
 
 func (d *jsonDecDriver) TryDecodeAsNil() bool {
-	b := d.skipWhitespace(true)
+	// we mustn't consume the state here, and end up trying to read separator twice.
+	// Instead, we keep track of the state and restore it if we couldn't decode as nil.
+
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
+	b := d.skipWhitespace(false)
 	if b == 'n' {
-		d.readStrIdx(9, 13) // null
+		d.readStrIdx(10, 13) // ull
 		d.ct = valueTypeNil
 		return true
 	}
+	d.r.unreadn1()
+	d.s.sc.retryRead()
 	return false
 }
 
 func (d *jsonDecDriver) DecodeBool() bool {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	b := d.skipWhitespace(false)
 	if b == 'f' {
 		d.readStrIdx(5, 9) // alse
@@ -426,35 +556,35 @@ func (d *jsonDecDriver) DecodeBool() bool {
 }
 
 func (d *jsonDecDriver) ReadMapStart() int {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
+	d.s.start('}')
 	d.expectChar('{')
 	d.ct = valueTypeMap
 	return -1
 }
 
 func (d *jsonDecDriver) ReadArrayStart() int {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
+	d.s.start(']')
 	d.expectChar('[')
 	d.ct = valueTypeArray
 	return -1
 }
-func (d *jsonDecDriver) ReadMapEnd() {
-	d.expectChar('}')
-}
-func (d *jsonDecDriver) ReadArrayEnd() {
-	d.expectChar(']')
-}
-func (d *jsonDecDriver) ReadArrayEntrySeparator() {
-	d.expectChar(',')
-}
-func (d *jsonDecDriver) ReadMapEntrySeparator() {
-	d.expectChar(',')
-}
-func (d *jsonDecDriver) ReadMapKVSeparator() {
-	d.expectChar(':')
+
+func (d *jsonDecDriver) ReadEnd() {
+	b := d.s.sc.st
+	d.s.end()
+	d.expectChar(b)
 }
+
 func (d *jsonDecDriver) expectChar(c uint8) {
 	b := d.skipWhitespace(false)
 	if b != c {
-		d.d.errorf("json: expect char %c but got char %c", c, b)
+		d.d.errorf("json: expect char '%c' but got char '%c'", c, b)
 		return
 	}
 	if jsonTrackSkipWhitespace {
@@ -462,6 +592,17 @@ func (d *jsonDecDriver) expectChar(c uint8) {
 	}
 }
 
+// func (d *jsonDecDriver) maybeChar(c uint8) {
+// 	b := d.skipWhitespace(false)
+// 	if b != c {
+// 		d.r.unreadn1()
+// 		return
+// 	}
+// 	if jsonTrackSkipWhitespace {
+// 		d.wsSkipped = false
+// 	}
+// }
+
 func (d *jsonDecDriver) IsContainerType(vt valueType) bool {
 	// check container type by checking the first char
 	if d.ct == valueTypeUnset {
@@ -635,6 +776,9 @@ LOOP:
 }
 
 func (d *jsonDecDriver) DecodeInt(bitsize uint8) (i int64) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	d.decNum(false)
 	n := &d.n
 	if n.manOverflow {
@@ -667,6 +811,9 @@ func (d *jsonDecDriver) DecodeInt(bitsize uint8) (i int64) {
 }
 
 func (d *jsonDecDriver) DecodeUint(bitsize uint8) (u uint64) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	d.decNum(false)
 	n := &d.n
 	if n.neg {
@@ -698,6 +845,9 @@ func (d *jsonDecDriver) DecodeUint(bitsize uint8) (u uint64) {
 }
 
 func (d *jsonDecDriver) DecodeFloat(chkOverflow32 bool) (f float64) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	d.decNum(true)
 	n := &d.n
 	f = n.floatVal()
@@ -709,6 +859,10 @@ func (d *jsonDecDriver) DecodeFloat(chkOverflow32 bool) (f float64) {
 }
 
 func (d *jsonDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
+	// No need to call sep here, as d.d.decode() handles it
+	// if c := d.s.sc.sep(); c != 0 {
+	// 	d.expectChar(c)
+	// }
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = xtag
@@ -722,6 +876,9 @@ func (d *jsonDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 }
 
 func (d *jsonDecDriver) DecodeBytes(bs []byte, isstring, zerocopy bool) (bsOut []byte) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	// zerocopy doesn't matter for json, as the bytes must be parsed.
 	bs0 := d.appendStringAsBytes(d.b[:0])
 	if isstring {
@@ -745,6 +902,9 @@ func (d *jsonDecDriver) DecodeBytes(bs []byte, isstring, zerocopy bool) (bsOut [
 }
 
 func (d *jsonDecDriver) DecodeString() (s string) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	return string(d.appendStringAsBytes(d.b[:0]))
 }
 
@@ -816,6 +976,9 @@ func (d *jsonDecDriver) jsonU4(checkSlashU bool) rune {
 }
 
 func (d *jsonDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	n := d.skipWhitespace(true)
 	switch n {
 	case 'n':
@@ -837,7 +1000,7 @@ func (d *jsonDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurthe
 		decodeFurther = true
 	case '"':
 		vt = valueTypeString
-		v = d.DecodeString()
+		v = string(d.appendStringAsBytes(d.b[:0])) // same as d.DecodeString(), but skipping sep() call.
 	default: // number
 		d.decNum(true)
 		n := &d.n
@@ -878,6 +1041,9 @@ func (d *jsonDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurthe
 		}
 		// fmt.Printf("DecodeNaked: Number: %T, %v\n", v, v)
 	}
+	if decodeFurther {
+		d.s.sc.retryRead()
+	}
 	return
 }
 

codec/noop.go

@@ -11,6 +11,7 @@ import (
 // NoopHandle returns a no-op handle. It basically does nothing.
 // It is only useful for benchmarking, as it gives an idea of the
 // overhead from the codec framework.
+//
 // LIBRARY USERS: *** DO NOT USE ***
 func NoopHandle(slen int) *noopHandle {
 	h := noopHandle{}
@@ -40,7 +41,8 @@ type noopDrv struct {
 	i    int
 	S    []string
 	B    [][]byte
-	mk   bool      // are we about to read a map key?
+	mks  []bool    // stack. if map (true), else if array (false)
+	mk   bool      // top of stack. what container are we on? map or array?
 	ct   valueType // last request for IsContainerType.
 	cb   bool      // last response for IsContainerType.
 	rand *rand.Rand
@@ -54,21 +56,22 @@ func (h *noopDrv) newDecDriver(_ *Decoder) decDriver { return h }
 
 // --- encDriver
 
-func (h *noopDrv) EncodeBuiltin(rt uintptr, v interface{})    {}
-func (h *noopDrv) EncodeNil()                                 {}
-func (h *noopDrv) EncodeInt(i int64)                          {}
-func (h *noopDrv) EncodeUint(i uint64)                        {}
-func (h *noopDrv) EncodeBool(b bool)                          {}
-func (h *noopDrv) EncodeFloat32(f float32)                    {}
-func (h *noopDrv) EncodeFloat64(f float64)                    {}
-func (h *noopDrv) EncodeRawExt(re *RawExt, e *Encoder)        {}
-func (h *noopDrv) EncodeArrayStart(length int)                {}
-func (h *noopDrv) EncodeArrayEnd()                            {}
-func (h *noopDrv) EncodeArrayEntrySeparator()                 {}
-func (h *noopDrv) EncodeMapStart(length int)                  {}
-func (h *noopDrv) EncodeMapEnd()                              {}
-func (h *noopDrv) EncodeMapEntrySeparator()                   {}
-func (h *noopDrv) EncodeMapKVSeparator()                      {}
+// stack functions (for map and array)
+func (h *noopDrv) start(b bool) { h.mks = append(h.mks, b); h.mk = b }
+func (h *noopDrv) end()         { h.mks = h.mks[:len(h.mks)-1]; h.mk = h.mks[len(h.mks)-1] }
+
+func (h *noopDrv) EncodeBuiltin(rt uintptr, v interface{}) {}
+func (h *noopDrv) EncodeNil()                              {}
+func (h *noopDrv) EncodeInt(i int64)                       {}
+func (h *noopDrv) EncodeUint(i uint64)                     {}
+func (h *noopDrv) EncodeBool(b bool)                       {}
+func (h *noopDrv) EncodeFloat32(f float32)                 {}
+func (h *noopDrv) EncodeFloat64(f float64)                 {}
+func (h *noopDrv) EncodeRawExt(re *RawExt, e *Encoder)     {}
+func (h *noopDrv) EncodeArrayStart(length int)             { h.start(true) }
+func (h *noopDrv) EncodeMapStart(length int)               { h.start(false) }
+func (h *noopDrv) EncodeEnd()                              { h.end() }
+
 func (h *noopDrv) EncodeString(c charEncoding, v string)      {}
 func (h *noopDrv) EncodeSymbol(v string)                      {}
 func (h *noopDrv) EncodeStringBytes(c charEncoding, v []byte) {}
@@ -90,15 +93,11 @@ func (h *noopDrv) DecodeString() (s string)                   { return h.S[h.m(8
 
 func (h *noopDrv) DecodeBytes(bs []byte, isstring, zerocopy bool) []byte { return h.B[h.m(len(h.B))] }
 
-func (h *noopDrv) ReadMapEnd()              { h.mk = false }
-func (h *noopDrv) ReadArrayEnd()            {}
-func (h *noopDrv) ReadArrayEntrySeparator() {}
-func (h *noopDrv) ReadMapEntrySeparator()   { h.mk = true }
-func (h *noopDrv) ReadMapKVSeparator()      { h.mk = false }
+func (h *noopDrv) ReadEnd() { h.start(true) }
 
 // toggle map/slice
-func (h *noopDrv) ReadMapStart() int   { h.mk = true; return h.m(10) }
-func (h *noopDrv) ReadArrayStart() int { return h.m(10) }
+func (h *noopDrv) ReadMapStart() int   { h.start(true); return h.m(10) }
+func (h *noopDrv) ReadArrayStart() int { h.start(false); return h.m(10) }
 
 func (h *noopDrv) IsContainerType(vt valueType) bool {
 	// return h.m(2) == 0

codec/py_test.go

@@ -6,7 +6,8 @@
 package codec
 
 // These tests are used to verify msgpack and cbor implementations against their python libraries.
-// If you have the library installed, you can enable the tests back by removing the //+build ignore.
+// If you have the library installed, you can enable the tests back by running: go test -tags=x .
+// Look at test.py for how to setup your environment.
 
 import (
 	"testing"

codec/test.py

@@ -5,8 +5,9 @@
 # So it can process them (so we don't have to checkin the files).
 
 # Ensure msgpack-python and cbor are installed first, using:
-#   pip install --user msgpack-python
-#   pip install --user cbor
+#   sudo apt-get install python-dev
+#   sudo apt-get install python-pip
+#   pip install --user msgpack-python msgpack-rpc-python cbor
 
 import cbor, msgpack, msgpackrpc, sys, os, threading
 

codec/tests.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Run all the different permutations of all the tests.
+# This helps ensure that nothing gets broken.
+
+_run() {
+    # 1. VARIATIONS: regular (t), canonical (c), IO R/W (i), binc-nosymbols (n), struct2array (s)
+    # 2. MODE: reflection (r), codecgen (x), codecgen+unsafe (u)
+    # 
+    # Typically, you would run a combination of one value from a and b.
+
+    ztags=""
+    local OPTIND 
+    OPTIND=1
+    while getopts "xurtcinsvg" flag
+    do
+        case "x$flag" in 
+            'xr')  ;;
+            'xg') ztags="$ztags codecgen" ;;
+            'xx') ztags="$ztags x" ;;
+            'xu') ztags="$ztags unsafe" ;;
+            'xv') zverbose="-tv" ;; 
+            *) ;;
+        esac
+    done
+    # shift $((OPTIND-1))
+    echo ">>>>>>> tags: $ztags"
+    
+    OPTIND=1
+    while getopts "xurtcinsvg" flag
+    do
+        case "x$flag" in 
+            'xt') echo ">>>>>>> REGULAR    "; go test "-tags=$ztags" "$zverbose" ; sleep 2 ;;
+            'xc') echo ">>>>>>> CANONICAL  "; go test "-tags=$ztags" "$zverbose" -tc; sleep 2 ;;
+            'xi') echo ">>>>>>> I/O        "; go test "-tags=$ztags" "$zverbose" -ti; sleep 2 ;;
+            'xn') echo ">>>>>>> NO_SYMBOLS "; go test "-tags=$ztags" "$zverbose" -tn; sleep 2 ;;
+            'xs') echo ">>>>>>> TO_ARRAY   "; go test "-tags=$ztags" "$zverbose" -ts; sleep 2 ;;
+            *) ;;
+        esac
+    done
+    shift $((OPTIND-1))
+
+    OPTIND=1
+}
+
+echo ">>>>>>> RUNNING VARIATIONS OF TESTS"    
+if [[ "x$@" = x ]]; then
+    # r, x, g, gu
+    _run "-rtcins"
+    _run "-xtcins"
+    _run "-gtcins"
+    _run "-gutcins"
+else
+    _run "$@"
+fi