Merge pull request #3792 from wojtek-t/update_ugorji

Update dependency on ugorji/go/codec

Godeps/Godeps.json

@@ -111,7 +111,7 @@
 		},
 		{
 			"ImportPath": "github.com/ugorji/go/codec",
-			"Rev": "45ce7596ace4534e47b69051a92aef7b64ec7b3f"
+			"Rev": "f1f1a805ed361a0e078bb537e4ea78cd37dcf065"
 		},
 		{
 			"ImportPath": "github.com/xiang90/probing",

Godeps/_workspace/src/github.com/ugorji/go/codec/0doc.go

@@ -98,7 +98,21 @@ with the standard net/rpc package.
 
 Usage
 
-Typical usage model:
+The Handle is SAFE for concurrent READ, but NOT SAFE for concurrent modification.
+
+The Encoder and Decoder are NOT safe for concurrent use.
+
+Consequently, the usage model is basically:
+
+    - Create and initialize the Handle before any use.
+      Once created, DO NOT modify it.
+    - Multiple Encoders or Decoders can now use the Handle concurrently.
+      They only read information off the Handle (never write).
+    - However, each Encoder or Decoder MUST not be used concurrently
+    - To re-use an Encoder/Decoder, call Reset(...) on it first.
+      This allows you use state maintained on the Encoder/Decoder.
+
+Sample usage model:
 
     // create and configure Handle
     var (
@@ -148,3 +162,32 @@ Typical usage model:
 */
 package codec
 
+// Benefits of go-codec:
+//
+//    - encoding/json always reads whole file into memory first.
+//      This makes it unsuitable for parsing very large files.
+//    - encoding/xml cannot parse into a map[string]interface{}
+//      I found this out on reading https://github.com/clbanning/mxj
+
+// TODO:
+//
+//   - (En|De)coder should store an error when it occurs.
+//     Until reset, subsequent calls return that error that was stored.
+//     This means that free panics must go away.
+//     All errors must be raised through errorf method.
+//   - Decoding using a chan is good, but incurs concurrency costs.
+//     This is because there's no fast way to use a channel without it
+//     having to switch goroutines constantly.
+//     Callback pattern is still the best. Maybe cnsider supporting something like:
+//        type X struct {
+//             Name string
+//             Ys []Y
+//             Ys chan <- Y
+//             Ys func(interface{}) -> call this interface for each entry in there.
+//        }
+//    - Consider adding a isZeroer interface { isZero() bool }
+//      It is used within isEmpty, for omitEmpty support.
+//    - Consider making Handle used AS-IS within the encoding/decoding session.
+//      This means that we don't cache Handle information within the (En|De)coder,
+//      except we really need it at Reset(...)
+//    - Handle recursive types during encoding/decoding?

Godeps/_workspace/src/github.com/ugorji/go/codec/binc.go

@@ -59,8 +59,8 @@ type bincEncDriver struct {
 	e *Encoder
 	w encWriter
 	m map[string]uint16 // symbols
-	s uint16            // symbols sequencer
 	b [scratchByteArrayLen]byte
+	s uint16 // symbols sequencer
 	encNoSeparator
 }
 
@@ -318,9 +318,9 @@ func (e *bincEncDriver) encLenNumber(bd byte, v uint64) {
 //------------------------------------
 
 type bincDecSymbol struct {
-	i uint16
 	s string
 	b []byte
+	i uint16
 }
 
 type bincDecDriver struct {
@@ -329,7 +329,6 @@ type bincDecDriver struct {
 	r      decReader
 	br     bool // bytes reader
 	bdRead bool
-	bdType valueType
 	bd     byte
 	vd     byte
 	vs     byte
@@ -347,24 +346,23 @@ func (d *bincDecDriver) readNextBd() {
 	d.vd = d.bd >> 4
 	d.vs = d.bd & 0x0f
 	d.bdRead = true
-	d.bdType = valueTypeUnset
 }
 
-func (d *bincDecDriver) IsContainerType(vt valueType) (b bool) {
-	switch vt {
-	case valueTypeNil:
-		return d.vd == bincVdSpecial && d.vs == bincSpNil
-	case valueTypeBytes:
-		return d.vd == bincVdByteArray
-	case valueTypeString:
-		return d.vd == bincVdString
-	case valueTypeArray:
-		return d.vd == bincVdArray
-	case valueTypeMap:
-		return d.vd == bincVdMap
+func (d *bincDecDriver) ContainerType() (vt valueType) {
+	if d.vd == bincVdSpecial && d.vs == bincSpNil {
+		return valueTypeNil
+	} else if d.vd == bincVdByteArray {
+		return valueTypeBytes
+	} else if d.vd == bincVdString {
+		return valueTypeString
+	} else if d.vd == bincVdArray {
+		return valueTypeArray
+	} else if d.vd == bincVdMap {
+		return valueTypeMap
+	} else {
+		// d.d.errorf("isContainerType: unsupported parameter: %v", vt)
 	}
-	d.d.errorf("isContainerType: unsupported parameter: %v", vt)
-	return // "unreachable"
+	return valueTypeUnset
 }
 
 func (d *bincDecDriver) TryDecodeAsNil() bool {
@@ -695,7 +693,7 @@ func (d *bincDecDriver) decStringAndBytes(bs []byte, withString, zerocopy bool)
 			if withString {
 				s = string(bs2)
 			}
-			d.s = append(d.s, bincDecSymbol{symbol, s, bs2})
+			d.s = append(d.s, bincDecSymbol{i: symbol, s: s, b: bs2})
 		}
 	default:
 		d.d.errorf("Invalid d.vd. Expecting string:0x%x, bytearray:0x%x or symbol: 0x%x. Got: 0x%x",
@@ -784,97 +782,95 @@ func (d *bincDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs []b
 	return
 }
 
-func (d *bincDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+func (d *bincDecDriver) DecodeNaked() {
 	if !d.bdRead {
 		d.readNextBd()
 	}
 
+	n := &d.d.n
+	var decodeFurther bool
+
 	switch d.vd {
 	case bincVdSpecial:
 		switch d.vs {
 		case bincSpNil:
-			vt = valueTypeNil
+			n.v = valueTypeNil
 		case bincSpFalse:
-			vt = valueTypeBool
-			v = false
+			n.v = valueTypeBool
+			n.b = false
 		case bincSpTrue:
-			vt = valueTypeBool
-			v = true
+			n.v = valueTypeBool
+			n.b = true
 		case bincSpNan:
-			vt = valueTypeFloat
-			v = math.NaN()
+			n.v = valueTypeFloat
+			n.f = math.NaN()
 		case bincSpPosInf:
-			vt = valueTypeFloat
-			v = math.Inf(1)
+			n.v = valueTypeFloat
+			n.f = math.Inf(1)
 		case bincSpNegInf:
-			vt = valueTypeFloat
-			v = math.Inf(-1)
+			n.v = valueTypeFloat
+			n.f = math.Inf(-1)
 		case bincSpZeroFloat:
-			vt = valueTypeFloat
-			v = float64(0)
+			n.v = valueTypeFloat
+			n.f = float64(0)
 		case bincSpZero:
-			vt = valueTypeUint
-			v = uint64(0) // int8(0)
+			n.v = valueTypeUint
+			n.u = uint64(0) // int8(0)
 		case bincSpNegOne:
-			vt = valueTypeInt
-			v = int64(-1) // int8(-1)
+			n.v = valueTypeInt
+			n.i = int64(-1) // int8(-1)
 		default:
 			d.d.errorf("decodeNaked: Unrecognized special value 0x%x", d.vs)
-			return
 		}
 	case bincVdSmallInt:
-		vt = valueTypeUint
-		v = uint64(int8(d.vs)) + 1 // int8(d.vs) + 1
+		n.v = valueTypeUint
+		n.u = uint64(int8(d.vs)) + 1 // int8(d.vs) + 1
 	case bincVdPosInt:
-		vt = valueTypeUint
-		v = d.decUint()
+		n.v = valueTypeUint
+		n.u = d.decUint()
 	case bincVdNegInt:
-		vt = valueTypeInt
-		v = -(int64(d.decUint()))
+		n.v = valueTypeInt
+		n.i = -(int64(d.decUint()))
 	case bincVdFloat:
-		vt = valueTypeFloat
-		v = d.decFloat()
+		n.v = valueTypeFloat
+		n.f = d.decFloat()
 	case bincVdSymbol:
-		vt = valueTypeSymbol
-		v = d.DecodeString()
+		n.v = valueTypeSymbol
+		n.s = d.DecodeString()
 	case bincVdString:
-		vt = valueTypeString
-		v = d.DecodeString()
+		n.v = valueTypeString
+		n.s = d.DecodeString()
 	case bincVdByteArray:
-		vt = valueTypeBytes
-		v = d.DecodeBytes(nil, false, false)
+		n.v = valueTypeBytes
+		n.l = d.DecodeBytes(nil, false, false)
 	case bincVdTimestamp:
-		vt = valueTypeTimestamp
+		n.v = valueTypeTimestamp
 		tt, err := decodeTime(d.r.readx(int(d.vs)))
 		if err != nil {
 			panic(err)
 		}
-		v = tt
+		n.t = tt
 	case bincVdCustomExt:
-		vt = valueTypeExt
+		n.v = valueTypeExt
 		l := d.decLen()
-		var re RawExt
-		re.Tag = uint64(d.r.readn1())
-		re.Data = d.r.readx(l)
-		v = &re
-		vt = valueTypeExt
+		n.u = uint64(d.r.readn1())
+		n.l = d.r.readx(l)
 	case bincVdArray:
-		vt = valueTypeArray
+		n.v = valueTypeArray
 		decodeFurther = true
 	case bincVdMap:
-		vt = valueTypeMap
+		n.v = valueTypeMap
 		decodeFurther = true
 	default:
 		d.d.errorf("decodeNaked: Unrecognized d.vd: 0x%x", d.vd)
-		return
 	}
 
 	if !decodeFurther {
 		d.bdRead = false
 	}
-	if vt == valueTypeUint && d.h.SignedInteger {
-		d.bdType = valueTypeInt
-		v = int64(v.(uint64))
+	if n.v == valueTypeUint && d.h.SignedInteger {
+		n.v = valueTypeInt
+		n.i = int64(n.u)
 	}
 	return
 }
@@ -898,6 +894,10 @@ type BincHandle struct {
 	binaryEncodingType
 }
 
+func (h *BincHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
+}
+
 func (h *BincHandle) newEncDriver(e *Encoder) encDriver {
 	return &bincEncDriver{e: e, w: e.w}
 }
@@ -906,8 +906,12 @@ func (h *BincHandle) newDecDriver(d *Decoder) decDriver {
 	return &bincDecDriver{d: d, r: d.r, h: h, br: d.bytes}
 }
 
-func (h *BincHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
-	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
+func (e *bincEncDriver) reset() {
+	e.w = e.e.w
+}
+
+func (d *bincDecDriver) reset() {
+	d.r = d.d.r
 }
 
 var _ decDriver = (*bincDecDriver)(nil)

Godeps/_workspace/src/github.com/ugorji/go/codec/cbor.go

@@ -60,11 +60,11 @@ const (
 // -------------------
 
 type cborEncDriver struct {
+	noBuiltInTypes
+	encNoSeparator
 	e *Encoder
 	w encWriter
 	h *CborHandle
-	noBuiltInTypes
-	encNoSeparator
 	x [8]byte
 }
 
@@ -175,11 +175,10 @@ type cborDecDriver struct {
 	d      *Decoder
 	h      *CborHandle
 	r      decReader
+	b      [scratchByteArrayLen]byte
 	br     bool // bytes reader
 	bdRead bool
-	bdType valueType
 	bd     byte
-	b      [scratchByteArrayLen]byte
 	noBuiltInTypes
 	decNoSeparator
 }
@@ -187,24 +186,23 @@ type cborDecDriver struct {
 func (d *cborDecDriver) readNextBd() {
 	d.bd = d.r.readn1()
 	d.bdRead = true
-	d.bdType = valueTypeUnset
 }
 
-func (d *cborDecDriver) IsContainerType(vt valueType) (bv bool) {
-	switch vt {
-	case valueTypeNil:
-		return d.bd == cborBdNil
-	case valueTypeBytes:
-		return d.bd == cborBdIndefiniteBytes || (d.bd >= cborBaseBytes && d.bd < cborBaseString)
-	case valueTypeString:
-		return d.bd == cborBdIndefiniteString || (d.bd >= cborBaseString && d.bd < cborBaseArray)
-	case valueTypeArray:
-		return d.bd == cborBdIndefiniteArray || (d.bd >= cborBaseArray && d.bd < cborBaseMap)
-	case valueTypeMap:
-		return d.bd == cborBdIndefiniteMap || (d.bd >= cborBaseMap && d.bd < cborBaseTag)
+func (d *cborDecDriver) ContainerType() (vt valueType) {
+	if d.bd == cborBdNil {
+		return valueTypeNil
+	} else if d.bd == cborBdIndefiniteBytes || (d.bd >= cborBaseBytes && d.bd < cborBaseString) {
+		return valueTypeBytes
+	} else if d.bd == cborBdIndefiniteString || (d.bd >= cborBaseString && d.bd < cborBaseArray) {
+		return valueTypeString
+	} else if d.bd == cborBdIndefiniteArray || (d.bd >= cborBaseArray && d.bd < cborBaseMap) {
+		return valueTypeArray
+	} else if d.bd == cborBdIndefiniteMap || (d.bd >= cborBaseMap && d.bd < cborBaseTag) {
+		return valueTypeMap
+	} else {
+		// d.d.errorf("isContainerType: unsupported parameter: %v", vt)
 	}
-	d.d.errorf("isContainerType: unsupported parameter: %v", vt)
-	return // "unreachable"
+	return valueTypeUnset
 }
 
 func (d *cborDecDriver) TryDecodeAsNil() bool {
@@ -446,71 +444,72 @@ func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 	return
 }
 
-func (d *cborDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+func (d *cborDecDriver) DecodeNaked() {
 	if !d.bdRead {
 		d.readNextBd()
 	}
 
+	n := &d.d.n
+	var decodeFurther bool
+
 	switch d.bd {
 	case cborBdNil:
-		vt = valueTypeNil
+		n.v = valueTypeNil
 	case cborBdFalse:
-		vt = valueTypeBool
-		v = false
+		n.v = valueTypeBool
+		n.b = false
 	case cborBdTrue:
-		vt = valueTypeBool
-		v = true
+		n.v = valueTypeBool
+		n.b = true
 	case cborBdFloat16, cborBdFloat32:
-		vt = valueTypeFloat
-		v = d.DecodeFloat(true)
+		n.v = valueTypeFloat
+		n.f = d.DecodeFloat(true)
 	case cborBdFloat64:
-		vt = valueTypeFloat
-		v = d.DecodeFloat(false)
+		n.v = valueTypeFloat
+		n.f = d.DecodeFloat(false)
 	case cborBdIndefiniteBytes:
-		vt = valueTypeBytes
-		v = d.DecodeBytes(nil, false, false)
+		n.v = valueTypeBytes
+		n.l = d.DecodeBytes(nil, false, false)
 	case cborBdIndefiniteString:
-		vt = valueTypeString
-		v = d.DecodeString()
+		n.v = valueTypeString
+		n.s = d.DecodeString()
 	case cborBdIndefiniteArray:
-		vt = valueTypeArray
+		n.v = valueTypeArray
 		decodeFurther = true
 	case cborBdIndefiniteMap:
-		vt = valueTypeMap
+		n.v = valueTypeMap
 		decodeFurther = true
 	default:
 		switch {
 		case d.bd >= cborBaseUint && d.bd < cborBaseNegInt:
 			if d.h.SignedInteger {
-				vt = valueTypeInt
-				v = d.DecodeInt(64)
+				n.v = valueTypeInt
+				n.i = d.DecodeInt(64)
 			} else {
-				vt = valueTypeUint
-				v = d.DecodeUint(64)
+				n.v = valueTypeUint
+				n.u = d.DecodeUint(64)
 			}
 		case d.bd >= cborBaseNegInt && d.bd < cborBaseBytes:
-			vt = valueTypeInt
-			v = d.DecodeInt(64)
+			n.v = valueTypeInt
+			n.i = d.DecodeInt(64)
 		case d.bd >= cborBaseBytes && d.bd < cborBaseString:
-			vt = valueTypeBytes
-			v = d.DecodeBytes(nil, false, false)
+			n.v = valueTypeBytes
+			n.l = d.DecodeBytes(nil, false, false)
 		case d.bd >= cborBaseString && d.bd < cborBaseArray:
-			vt = valueTypeString
-			v = d.DecodeString()
+			n.v = valueTypeString
+			n.s = d.DecodeString()
 		case d.bd >= cborBaseArray && d.bd < cborBaseMap:
-			vt = valueTypeArray
+			n.v = valueTypeArray
 			decodeFurther = true
 		case d.bd >= cborBaseMap && d.bd < cborBaseTag:
-			vt = valueTypeMap
+			n.v = valueTypeMap
 			decodeFurther = true
 		case d.bd >= cborBaseTag && d.bd < cborBaseSimple:
-			vt = valueTypeExt
-			var re RawExt
-			ui := d.decUint()
+			n.v = valueTypeExt
+			n.u = d.decUint()
+			n.l = nil
 			d.bdRead = false
-			re.Tag = ui
-			d.d.decode(&re.Value)
-			v = &re
+			// d.d.decode(&re.Value) // handled by decode itself.
 			// decodeFurther = true
 		default:
 			d.d.errorf("decodeNaked: Unrecognized d.bd: 0x%x", d.bd)
@@ -557,8 +556,12 @@ func (d *cborDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurthe
 //     // Now, vv contains the same string "one-byte"
 //
 type CborHandle struct {
-	BasicHandle
 	binaryEncodingType
+	BasicHandle
+}
+
+func (h *CborHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{i: ext})
 }
 
 func (h *CborHandle) newEncDriver(e *Encoder) encDriver {
@@ -569,8 +572,12 @@ func (h *CborHandle) newDecDriver(d *Decoder) decDriver {
 	return &cborDecDriver{d: d, r: d.r, h: h, br: d.bytes}
 }
 
-func (h *CborHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) {
-	return h.SetExt(rt, tag, &setExtWrapper{i: ext})
+func (e *cborEncDriver) reset() {
+	e.w = e.e.w
+}
+
+func (d *cborDecDriver) reset() {
+	d.r = d.d.r
 }
 
 var _ decDriver = (*cborDecDriver)(nil)

Godeps/_workspace/src/github.com/ugorji/go/codec/codec_test.go

@@ -28,6 +28,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math"
+	"math/rand"
 	"net"
 	"net/rpc"
 	"os"
@@ -64,8 +65,11 @@ var (
 	testUseIoEncDec    bool
 	testStructToArray  bool
 	testCanonical      bool
+	testUseReset       bool
 	testWriteNoSymbols bool
 	testSkipIntf       bool
+	testInternStr      bool
+	testUseMust        bool
 
 	skipVerifyVal interface{} = &(struct{}{})
 
@@ -97,7 +101,14 @@ func testInitFlags() {
 	flag.BoolVar(&testStructToArray, "ts", false, "Set StructToArray option")
 	flag.BoolVar(&testWriteNoSymbols, "tn", false, "Set NoSymbols option")
 	flag.BoolVar(&testCanonical, "tc", false, "Set Canonical option")
+	flag.BoolVar(&testInternStr, "te", false, "Set InternStr option")
 	flag.BoolVar(&testSkipIntf, "tf", false, "Skip Interfaces")
+	flag.BoolVar(&testUseReset, "tr", false, "Use Reset")
+	flag.BoolVar(&testUseMust, "tm", true, "Use Must(En|De)code")
+}
+
+func testByteBuf(in []byte) *bytes.Buffer {
+	return bytes.NewBuffer(in)
 }
 
 type TestABC struct {
@@ -120,28 +131,36 @@ func (r *TestRpcInt) Echo123(args []string, res *string) error {
 	return nil
 }
 
-type testUnixNanoTimeExt struct{}
+type testUnixNanoTimeExt struct {
+	// keep timestamp here, so that do not incur interface-conversion costs
+	ts int64
+}
 
-func (x testUnixNanoTimeExt) WriteExt(interface{}) []byte { panic("unsupported") }
-func (x testUnixNanoTimeExt) ReadExt(interface{}, []byte) { panic("unsupported") }
-func (x testUnixNanoTimeExt) ConvertExt(v interface{}) interface{} {
+// func (x *testUnixNanoTimeExt) WriteExt(interface{}) []byte { panic("unsupported") }
+// func (x *testUnixNanoTimeExt) ReadExt(interface{}, []byte) { panic("unsupported") }
+func (x *testUnixNanoTimeExt) ConvertExt(v interface{}) interface{} {
 	switch v2 := v.(type) {
 	case time.Time:
-		return v2.UTC().UnixNano()
+		x.ts = v2.UTC().UnixNano()
 	case *time.Time:
-		return v2.UTC().UnixNano()
+		x.ts = v2.UTC().UnixNano()
 	default:
 		panic(fmt.Sprintf("unsupported format for time conversion: expecting time.Time; got %T", v))
 	}
+	return &x.ts
 }
-func (x testUnixNanoTimeExt) UpdateExt(dest interface{}, v interface{}) {
+func (x *testUnixNanoTimeExt) UpdateExt(dest interface{}, v interface{}) {
 	// fmt.Printf("testUnixNanoTimeExt.UpdateExt: v: %v\n", v)
 	tt := dest.(*time.Time)
 	switch v2 := v.(type) {
 	case int64:
 		*tt = time.Unix(0, v2).UTC()
+	case *int64:
+		*tt = time.Unix(0, *v2).UTC()
 	case uint64:
 		*tt = time.Unix(0, int64(v2)).UTC()
+	case *uint64:
+		*tt = time.Unix(0, int64(*v2)).UTC()
 	//case float64:
 	//case string:
 	default:
@@ -269,52 +288,40 @@ func testInit() {
 		fmt.Printf("====> depth: %v, ts: %#v\n", 2, ts0)
 	}
 
-	testJsonH.Canonical = testCanonical
-	testCborH.Canonical = testCanonical
-	testSimpleH.Canonical = testCanonical
-	testBincH.Canonical = testCanonical
-	testMsgpackH.Canonical = testCanonical
-
-	testJsonH.StructToArray = testStructToArray
-	testCborH.StructToArray = testStructToArray
-	testSimpleH.StructToArray = testStructToArray
-	testBincH.StructToArray = testStructToArray
-	testMsgpackH.StructToArray = testStructToArray
+	for _, v := range testHandles {
+		bh := v.getBasicHandle()
+		bh.InternString = testInternStr
+		bh.Canonical = testCanonical
+		bh.StructToArray = testStructToArray
+		// mostly doing this for binc
+		if testWriteNoSymbols {
+			bh.AsSymbols = AsSymbolNone
+		} else {
+			bh.AsSymbols = AsSymbolAll
+		}
+	}
 
 	testMsgpackH.RawToString = true
 
-	if testWriteNoSymbols {
-		testBincH.AsSymbols = AsSymbolNone
-	} else {
-		testBincH.AsSymbols = AsSymbolAll
-	}
-
 	// testMsgpackH.AddExt(byteSliceTyp, 0, testMsgpackH.BinaryEncodeExt, testMsgpackH.BinaryDecodeExt)
 	// testMsgpackH.AddExt(timeTyp, 1, testMsgpackH.TimeEncodeExt, testMsgpackH.TimeDecodeExt)
 	timeEncExt := func(rv reflect.Value) (bs []byte, err error) {
-		switch v2 := rv.Interface().(type) {
-		case time.Time:
-			bs = encodeTime(v2)
-		case *time.Time:
-			bs = encodeTime(*v2)
-		default:
-			err = fmt.Errorf("unsupported format for time conversion: expecting time.Time; got %T", v2)
-		}
+		defer panicToErr(&err)
+		bs = timeExt{}.WriteExt(rv.Interface())
 		return
 	}
 	timeDecExt := func(rv reflect.Value, bs []byte) (err error) {
-		tt, err := decodeTime(bs)
-		if err == nil {
-			*(rv.Interface().(*time.Time)) = tt
-		}
+		defer panicToErr(&err)
+		timeExt{}.ReadExt(rv.Interface(), bs)
 		return
 	}
 
 	// add extensions for msgpack, simple for time.Time, so we can encode/decode same way.
-	testMsgpackH.AddExt(timeTyp, 1, timeEncExt, timeDecExt)
+	// use different flavors of XXXExt calls, including deprecated ones.
 	testSimpleH.AddExt(timeTyp, 1, timeEncExt, timeDecExt)
-	testCborH.SetExt(timeTyp, 1, &testUnixNanoTimeExt{})
-	testJsonH.SetExt(timeTyp, 1, &testUnixNanoTimeExt{})
+	testMsgpackH.SetBytesExt(timeTyp, 1, timeExt{})
+	testCborH.SetInterfaceExt(timeTyp, 1, &testUnixNanoTimeExt{})
+	testJsonH.SetInterfaceExt(timeTyp, 1, &testUnixNanoTimeExt{})
 
 	primitives := []interface{}{
 		int8(-8),
@@ -429,23 +436,11 @@ func testInit() {
 }
 
 func testUnmarshal(v interface{}, data []byte, h Handle) (err error) {
-	if testUseIoEncDec {
-		NewDecoder(bytes.NewBuffer(data), h).MustDecode(v)
-	} else {
-		NewDecoderBytes(data, h).MustDecode(v)
-	}
-	return
+	return testCodecDecode(data, v, h)
 }
 
 func testMarshal(v interface{}, h Handle) (bs []byte, err error) {
-	if testUseIoEncDec {
-		var buf bytes.Buffer
-		NewEncoder(&buf, h).MustEncode(v)
-		bs = buf.Bytes()
-		return
-	}
-	NewEncoderBytes(&bs, h).MustEncode(v)
-	return
+	return testCodecEncode(v, nil, testByteBuf, h)
 }
 
 func testMarshalErr(v interface{}, h Handle, t *testing.T, name string) (bs []byte, err error) {
@@ -663,6 +658,13 @@ func testCodecMiscOne(t *testing.T, h Handle) {
 	// test both pointer and non-pointer (value)
 	for _, tarr1 := range []interface{}{tarr0, &tarr0} {
 		bs, err = testMarshalErr(tarr1, h, t, "tarr1")
+		if err != nil {
+			logT(t, "Error marshalling: %v", err)
+			t.FailNow()
+		}
+		if _, ok := h.(*JsonHandle); ok {
+			logT(t, "Marshal as: %s", bs)
+		}
 		var tarr2 tarr
 		testUnmarshalErr(&tarr2, bs, h, t, "tarr2")
 		checkEqualT(t, tarr0, tarr2, "tarr0=tarr2")
@@ -898,9 +900,10 @@ func doTestMapEncodeForCanonical(t *testing.T, name string, h Handle) {
 	// encode v1 into b1, decode b1 into v2, encode v2 into b2, compare b1 and b2
 
 	bh := h.getBasicHandle()
-	canonical0 := bh.Canonical
-	bh.Canonical = true
-	defer func() { bh.Canonical = canonical0 }()
+	if !bh.Canonical {
+		bh.Canonical = true
+		defer func() { bh.Canonical = false }()
+	}
 
 	e1 := NewEncoderBytes(&b1, h)
 	e1.MustEncode(v1)
@@ -1019,11 +1022,17 @@ func doTestMsgpackRpcSpecGoClientToPythonSvc(t *testing.T) {
 	if testSkipRPCTests {
 		return
 	}
-	openPort := "6789"
-	cmd := exec.Command("python", "test.py", "rpc-server", openPort, "2")
+	// openPorts are between 6700 and 6800
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	openPort := strconv.FormatInt(6700+r.Int63n(99), 10)
+	// openPort := "6792"
+	cmd := exec.Command("python", "test.py", "rpc-server", openPort, "4")
 	checkErrT(t, cmd.Start())
-	time.Sleep(100 * time.Millisecond) // time for python rpc server to start
 	bs, err2 := net.Dial("tcp", ":"+openPort)
+	for i := 0; i < 10 && err2 != nil; i++ {
+		time.Sleep(50 * time.Millisecond) // time for python rpc server to start
+		bs, err2 = net.Dial("tcp", ":"+openPort)
+	}
 	checkErrT(t, err2)
 	cc := MsgpackSpecRpc.ClientCodec(bs, testMsgpackH)
 	cl := rpc.NewClientWithCodec(cc)
@@ -1034,6 +1043,7 @@ func doTestMsgpackRpcSpecGoClientToPythonSvc(t *testing.T) {
 	var mArgs MsgpackSpecRpcMultiArgs = []interface{}{"A1", "B2", "C3"}
 	checkErrT(t, cl.Call("Echo123", mArgs, &rstr))
 	checkEqualT(t, rstr, "1:A1 2:B2 3:C3", "rstr=")
+	cmd.Process.Kill()
 }
 
 func doTestMsgpackRpcSpecPythonClientToGoSvc(t *testing.T) {

Godeps/_workspace/src/github.com/ugorji/go/codec/codecgen/gen.go

@@ -41,24 +41,13 @@ package {{ $.PackageName }}
 
 import (
 	{{ if not .CodecPkgFiles }}{{ .CodecPkgName }} "{{ .CodecImportPath }}"{{ end }}
-{{/*
-	{{ if .Types }}"{{ .ImportPath }}"{{ end }}
-	"io"
-*/}}
 	"os"
 	"reflect"
 	"bytes"
+	"strings"
 	"go/format"
 )
 
-{{/* This is not used anymore. Remove it.
-func write(w io.Writer, s string) {
-	if _, err := io.WriteString(w, s); err != nil {
-		panic(err)
-	}
-}
-*/}}
-
 func CodecGenTempWrite{{ .RandString }}() {
 	fout, err := os.Create("{{ .OutFile }}")
 	if err != nil {
@@ -72,7 +61,7 @@ func CodecGenTempWrite{{ .RandString }}() {
 	var t{{ $index }} {{ . }}
 	typs = append(typs, reflect.TypeOf(t{{ $index }}))
 {{ end }}
-	{{ if not .CodecPkgFiles }}{{ .CodecPkgName }}.{{ end }}Gen(&out, "{{ .BuildTag }}", "{{ .PackageName }}", "{{ .RandString }}", {{ .UseUnsafe }}, typs...)
+	{{ if not .CodecPkgFiles }}{{ .CodecPkgName }}.{{ end }}Gen(&out, "{{ .BuildTag }}", "{{ .PackageName }}", "{{ .RandString }}", {{ .UseUnsafe }}, {{ if not .CodecPkgFiles }}{{ .CodecPkgName }}.{{ end }}NewTypeInfos(strings.Split("{{ .StructTags }}", ",")), typs...)
 	bout, err := format.Source(out.Bytes())
 	if err != nil {
 		fout.Write(out.Bytes())
@@ -93,7 +82,7 @@ func CodecGenTempWrite{{ .RandString }}() {
 // fout contains Codec(En|De)codeSelf implementations for every type T.
 //
 func Generate(outfile, buildTag, codecPkgPath string, uid int64, useUnsafe bool, goRunTag string,
-	regexName *regexp.Regexp, deleteTempFile bool, infiles ...string) (err error) {
+	st string, regexName *regexp.Regexp, deleteTempFile bool, infiles ...string) (err error) {
 	// For each file, grab AST, find each type, and write a call to it.
 	if len(infiles) == 0 {
 		return
@@ -128,6 +117,7 @@ func Generate(outfile, buildTag, codecPkgPath string, uid int64, useUnsafe bool,
 		PackageName     string
 		RandString      string
 		BuildTag        string
+		StructTags      string
 		Types           []string
 		CodecPkgFiles   bool
 		UseUnsafe       bool
@@ -139,6 +129,7 @@ func Generate(outfile, buildTag, codecPkgPath string, uid int64, useUnsafe bool,
 		BuildTag:        buildTag,
 		UseUnsafe:       useUnsafe,
 		RandString:      strconv.FormatInt(uid, 10),
+		StructTags:      st,
 	}
 	tv.ImportPath = pkg.ImportPath
 	if tv.ImportPath == tv.CodecImportPath {
@@ -269,11 +260,12 @@ func main() {
 	t := flag.String("t", "", "build tag to put in file")
 	r := flag.String("r", ".*", "regex for type name to match")
 	rt := flag.String("rt", "", "tags for go run")
+	st := flag.String("st", "codec,json", "struct tag keys to introspect")
 	x := flag.Bool("x", false, "keep temp file")
 	u := flag.Bool("u", false, "Use unsafe, e.g. to avoid unnecessary allocation on []byte->string")
 	d := flag.Int64("d", 0, "random identifier for use in generated code")
 	flag.Parse()
-	if err := Generate(*o, *t, *c, *d, *u, *rt,
+	if err := Generate(*o, *t, *c, *d, *u, *rt, *st,
 		regexp.MustCompile(*r), !*x, flag.Args()...); err != nil {
 		fmt.Fprintf(os.Stderr, "codecgen error: %v\n", err)
 		os.Exit(1)

Godeps/_workspace/src/github.com/ugorji/go/codec/codecgen_test.go

@@ -7,7 +7,9 @@ import (
 	"testing"
 )
 
-func TestCodecgenJson1(t *testing.T) {
+func _TestCodecgenJson1(t *testing.T) {
+	// This is just a simplistic test for codecgen.
+	// It is typically disabled. We only enable it for debugging purposes.
 	const callCodecgenDirect bool = true
 	v := newTestStruc(2, false, !testSkipIntf, false)
 	var bs []byte

Godeps/_workspace/src/github.com/ugorji/go/codec/encode.go

@@ -4,7 +4,6 @@
 package codec
 
 import (
-	"bytes"
 	"encoding"
 	"fmt"
 	"io"
@@ -63,13 +62,14 @@ type encDriver interface {
 	EncodeExt(v interface{}, xtag uint64, ext Ext, e *Encoder)
 	EncodeArrayStart(length int)
 	EncodeMapStart(length int)
-	EncodeEnd()
 	EncodeString(c charEncoding, v string)
 	EncodeSymbol(v string)
 	EncodeStringBytes(c charEncoding, v []byte)
 	//TODO
 	//encBignum(f *big.Int)
 	//encStringRunes(c charEncoding, v []rune)
+
+	reset()
 }
 
 type encDriverAsis interface {
@@ -80,17 +80,6 @@ type encNoSeparator struct{}
 
 func (_ encNoSeparator) EncodeEnd() {}
 
-type encStructFieldBytesV struct {
-	b []byte
-	v reflect.Value
-}
-
-type encStructFieldBytesVslice []encStructFieldBytesV
-
-func (p encStructFieldBytesVslice) Len() int           { return len(p) }
-func (p encStructFieldBytesVslice) Less(i, j int) bool { return bytes.Compare(p[i].b, p[j].b) == -1 }
-func (p encStructFieldBytesVslice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
-
 type ioEncWriterWriter interface {
 	WriteByte(c byte) error
 	WriteString(s string) (n int, err error)
@@ -109,8 +98,16 @@ type EncodeOptions struct {
 	// sequence of bytes.
 	//
 	// This only affects maps, as the iteration order for maps is random.
-	// In this case, the map keys will first be encoded into []byte, and then sorted,
-	// before writing the sorted keys and the corresponding map values to the stream.
+	//
+	// The implementation MAY use the natural sort order for the map keys if possible:
+	//
+	//     - If there is a natural sort order (ie for number, bool, string or []byte keys),
+	//       then the map keys are first sorted in natural order and then written
+	//       with corresponding map values to the strema.
+	//     - If there is no natural sort order, then the map keys will first be
+	//       encoded into []byte, and then sorted,
+	//       before writing the sorted keys and the corresponding map values to the stream.
+	//
 	Canonical bool
 
 	// AsSymbols defines what should be encoded as symbols.
@@ -162,6 +159,7 @@ func (o *simpleIoEncWriterWriter) Write(p []byte) (n int, err error) {
 // ioEncWriter implements encWriter and can write to an io.Writer implementation
 type ioEncWriter struct {
 	w ioEncWriterWriter
+	s simpleIoEncWriterWriter
 	// x [8]byte // temp byte array re-used internally for efficiency
 }
 
@@ -386,30 +384,32 @@ func (f *encFnInfo) kSlice(rv reflect.Value) {
 	//   (don't call rv.Bytes, rv.Slice, etc).
 	// E.g. type struct S{B [2]byte};
 	//   Encode(S{}) will bomb on "panic: slice of unaddressable array".
+	e := f.e
 	if f.seq != seqTypeArray {
 		if rv.IsNil() {
-			f.e.e.EncodeNil()
+			e.e.EncodeNil()
 			return
 		}
 		// If in this method, then there was no extension function defined.
 		// So it's okay to treat as []byte.
 		if ti.rtid == uint8SliceTypId {
-			f.e.e.EncodeStringBytes(c_RAW, rv.Bytes())
+			e.e.EncodeStringBytes(c_RAW, rv.Bytes())
 			return
 		}
 	}
+	cr := e.cr
 	rtelem := ti.rt.Elem()
 	l := rv.Len()
-	if rtelem.Kind() == reflect.Uint8 {
+	if ti.rtid == uint8SliceTypId || rtelem.Kind() == reflect.Uint8 {
 		switch f.seq {
 		case seqTypeArray:
-			// if l == 0 { f.e.e.encodeStringBytes(c_RAW, nil) } else
+			// if l == 0 { e.e.encodeStringBytes(c_RAW, nil) } else
 			if rv.CanAddr() {
-				f.e.e.EncodeStringBytes(c_RAW, rv.Slice(0, l).Bytes())
+				e.e.EncodeStringBytes(c_RAW, rv.Slice(0, l).Bytes())
 			} else {
 				var bs []byte
-				if l <= cap(f.e.b) {
-					bs = f.e.b[:l]
+				if l <= cap(e.b) {
+					bs = e.b[:l]
 				} else {
 					bs = make([]byte, l)
 				}
@@ -418,12 +418,12 @@ func (f *encFnInfo) kSlice(rv reflect.Value) {
 				// for i := 0; i < l; i++ {
 				// 	bs[i] = byte(rv.Index(i).Uint())
 				// }
-				f.e.e.EncodeStringBytes(c_RAW, bs)
+				e.e.EncodeStringBytes(c_RAW, bs)
 			}
 		case seqTypeSlice:
-			f.e.e.EncodeStringBytes(c_RAW, rv.Bytes())
+			e.e.EncodeStringBytes(c_RAW, rv.Bytes())
 		case seqTypeChan:
-			bs := f.e.b[:0]
+			bs := e.b[:0]
 			// do not use range, so that the number of elements encoded
 			// does not change, and encoding does not hang waiting on someone to close chan.
 			// for b := range rv.Interface().(<-chan byte) {
@@ -433,22 +433,21 @@ func (f *encFnInfo) kSlice(rv reflect.Value) {
 			for i := 0; i < l; i++ {
 				bs = append(bs, <-ch)
 			}
-			f.e.e.EncodeStringBytes(c_RAW, bs)
+			e.e.EncodeStringBytes(c_RAW, bs)
 		}
 		return
 	}
 
 	if ti.mbs {
 		if l%2 == 1 {
-			f.e.errorf("mapBySlice requires even slice length, but got %v", l)
+			e.errorf("mapBySlice requires even slice length, but got %v", l)
 			return
 		}
-		f.e.e.EncodeMapStart(l / 2)
+		e.e.EncodeMapStart(l / 2)
 	} else {
-		f.e.e.EncodeArrayStart(l)
+		e.e.EncodeArrayStart(l)
 	}
 
-	e := f.e
 	if l > 0 {
 		for rtelem.Kind() == reflect.Ptr {
 			rtelem = rtelem.Elem()
@@ -463,29 +462,48 @@ func (f *encFnInfo) kSlice(rv reflect.Value) {
 		}
 		// TODO: Consider perf implication of encoding odd index values as symbols if type is string
 		for j := 0; j < l; j++ {
+			if cr != nil {
+				if ti.mbs {
+					if l%2 == 0 {
+						cr.sendContainerState(containerMapKey)
+					} else {
+						cr.sendContainerState(containerMapValue)
+					}
+				} else {
+					cr.sendContainerState(containerArrayElem)
+				}
+			}
 			if f.seq == seqTypeChan {
 				if rv2, ok2 := rv.Recv(); ok2 {
 					e.encodeValue(rv2, fn)
+				} else {
+					e.encode(nil) // WE HAVE TO DO SOMETHING, so nil if nothing received.
 				}
 			} else {
 				e.encodeValue(rv.Index(j), fn)
 			}
 		}
-
 	}
 
-	f.e.e.EncodeEnd()
+	if cr != nil {
+		if ti.mbs {
+			cr.sendContainerState(containerMapEnd)
+		} else {
+			cr.sendContainerState(containerArrayEnd)
+		}
+	}
 }
 
 func (f *encFnInfo) kStruct(rv reflect.Value) {
 	fti := f.ti
 	e := f.e
+	cr := e.cr
 	tisfi := fti.sfip
 	toMap := !(fti.toArray || e.h.StructToArray)
 	newlen := len(fti.sfi)
 
 	// Use sync.Pool to reduce allocating slices unnecessarily.
-	// The cost of the occasional locking is less than the cost of locking.
+	// The cost of the occasional locking is less than the cost of new allocation.
 	pool, poolv, fkvs := encStructPoolGet(newlen)
 
 	// if toMap, use the sorted array. If toArray, use unsorted array (to match sequence in struct)
@@ -493,27 +511,27 @@ func (f *encFnInfo) kStruct(rv reflect.Value) {
 		tisfi = fti.sfi
 	}
 	newlen = 0
-	var kv encStructFieldKV
+	var kv stringRv
 	for _, si := range tisfi {
-		kv.v = si.field(rv, false)
+		kv.r = si.field(rv, false)
 		// if si.i != -1 {
 		// 	rvals[newlen] = rv.Field(int(si.i))
 		// } else {
 		// 	rvals[newlen] = rv.FieldByIndex(si.is)
 		// }
 		if toMap {
-			if si.omitEmpty && isEmptyValue(kv.v) {
+			if si.omitEmpty && isEmptyValue(kv.r) {
 				continue
 			}
-			kv.k = si.encName
+			kv.v = si.encName
 		} else {
 			// use the zero value.
 			// if a reference or struct, set to nil (so you do not output too much)
-			if si.omitEmpty && isEmptyValue(kv.v) {
-				switch kv.v.Kind() {
+			if si.omitEmpty && isEmptyValue(kv.r) {
+				switch kv.r.Kind() {
 				case reflect.Struct, reflect.Interface, reflect.Ptr, reflect.Array,
 					reflect.Map, reflect.Slice:
-					kv.v = reflect.Value{} //encode as nil
+					kv.r = reflect.Value{} //encode as nil
 				}
 			}
 		}
@@ -523,7 +541,7 @@ func (f *encFnInfo) kStruct(rv reflect.Value) {
 
 	// debugf(">>>> kStruct: newlen: %v", newlen)
 	// sep := !e.be
-	ee := f.e.e //don't dereference everytime
+	ee := e.e //don't dereference everytime
 
 	if toMap {
 		ee.EncodeMapStart(newlen)
@@ -531,21 +549,35 @@ func (f *encFnInfo) kStruct(rv reflect.Value) {
 		asSymbols := e.h.AsSymbols == AsSymbolDefault || e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
 		for j := 0; j < newlen; j++ {
 			kv = fkvs[j]
+			if cr != nil {
+				cr.sendContainerState(containerMapKey)
+			}
 			if asSymbols {
-				ee.EncodeSymbol(kv.k)
+				ee.EncodeSymbol(kv.v)
 			} else {
-				ee.EncodeString(c_UTF8, kv.k)
+				ee.EncodeString(c_UTF8, kv.v)
+			}
+			if cr != nil {
+				cr.sendContainerState(containerMapValue)
 			}
-			e.encodeValue(kv.v, nil)
+			e.encodeValue(kv.r, nil)
+		}
+		if cr != nil {
+			cr.sendContainerState(containerMapEnd)
 		}
 	} else {
 		ee.EncodeArrayStart(newlen)
 		for j := 0; j < newlen; j++ {
 			kv = fkvs[j]
-			e.encodeValue(kv.v, nil)
+			if cr != nil {
+				cr.sendContainerState(containerArrayElem)
+			}
+			e.encodeValue(kv.r, nil)
+		}
+		if cr != nil {
+			cr.sendContainerState(containerArrayEnd)
 		}
 	}
-	ee.EncodeEnd()
 
 	// do not use defer. Instead, use explicit pool return at end of function.
 	// defer has a cost we are trying to avoid.
@@ -582,8 +614,11 @@ func (f *encFnInfo) kMap(rv reflect.Value) {
 	l := rv.Len()
 	ee.EncodeMapStart(l)
 	e := f.e
+	cr := e.cr
 	if l == 0 {
-		ee.EncodeEnd()
+		if cr != nil {
+			cr.sendContainerState(containerMapEnd)
+		}
 		return
 	}
 	var asSymbols bool
@@ -621,26 +656,14 @@ func (f *encFnInfo) kMap(rv reflect.Value) {
 	}
 	mks := rv.MapKeys()
 	// for j, lmks := 0, len(mks); j < lmks; j++ {
+
 	if e.h.Canonical {
-		// first encode each key to a []byte first, then sort them, then record
-		// println(">>>>>>>> CANONICAL <<<<<<<<")
-		var mksv []byte = make([]byte, 0, len(mks)*16) // temporary byte slice for the encoding
-		e2 := NewEncoderBytes(&mksv, e.hh)
-		mksbv := make([]encStructFieldBytesV, len(mks))
-		for i, k := range mks {
-			l := len(mksv)
-			e2.MustEncode(k)
-			mksbv[i].v = k
-			mksbv[i].b = mksv[l:]
-			// fmt.Printf(">>>>> %s\n", mksv[l:])
-		}
-		sort.Sort(encStructFieldBytesVslice(mksbv))
-		for j := range mksbv {
-			e.asis(mksbv[j].b)
-			e.encodeValue(rv.MapIndex(mksbv[j].v), valFn)
-		}
+		e.kMapCanonical(rtkeyid, rtkey, rv, mks, valFn, asSymbols)
 	} else {
 		for j := range mks {
+			if cr != nil {
+				cr.sendContainerState(containerMapKey)
+			}
 			if keyTypeIsString {
 				if asSymbols {
 					ee.EncodeSymbol(mks[j].String())
@@ -650,10 +673,182 @@ func (f *encFnInfo) kMap(rv reflect.Value) {
 			} else {
 				e.encodeValue(mks[j], keyFn)
 			}
+			if cr != nil {
+				cr.sendContainerState(containerMapValue)
+			}
 			e.encodeValue(rv.MapIndex(mks[j]), valFn)
 		}
 	}
-	ee.EncodeEnd()
+	if cr != nil {
+		cr.sendContainerState(containerMapEnd)
+	}
+}
+
+func (e *Encoder) kMapCanonical(rtkeyid uintptr, rtkey reflect.Type, rv reflect.Value, mks []reflect.Value, valFn *encFn, asSymbols bool) {
+	ee := e.e
+	cr := e.cr
+	// we previously did out-of-band if an extension was registered.
+	// This is not necessary, as the natural kind is sufficient for ordering.
+
+	if rtkeyid == uint8SliceTypId {
+		mksv := make([]bytesRv, len(mks))
+		for i, k := range mks {
+			v := &mksv[i]
+			v.r = k
+			v.v = k.Bytes()
+		}
+		sort.Sort(bytesRvSlice(mksv))
+		for i := range mksv {
+			if cr != nil {
+				cr.sendContainerState(containerMapKey)
+			}
+			ee.EncodeStringBytes(c_RAW, mksv[i].v)
+			if cr != nil {
+				cr.sendContainerState(containerMapValue)
+			}
+			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+		}
+	} else {
+		switch rtkey.Kind() {
+		case reflect.Bool:
+			mksv := make([]boolRv, len(mks))
+			for i, k := range mks {
+				v := &mksv[i]
+				v.r = k
+				v.v = k.Bool()
+			}
+			sort.Sort(boolRvSlice(mksv))
+			for i := range mksv {
+				if cr != nil {
+					cr.sendContainerState(containerMapKey)
+				}
+				ee.EncodeBool(mksv[i].v)
+				if cr != nil {
+					cr.sendContainerState(containerMapValue)
+				}
+				e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			}
+		case reflect.String:
+			mksv := make([]stringRv, len(mks))
+			for i, k := range mks {
+				v := &mksv[i]
+				v.r = k
+				v.v = k.String()
+			}
+			sort.Sort(stringRvSlice(mksv))
+			for i := range mksv {
+				if cr != nil {
+					cr.sendContainerState(containerMapKey)
+				}
+				if asSymbols {
+					ee.EncodeSymbol(mksv[i].v)
+				} else {
+					ee.EncodeString(c_UTF8, mksv[i].v)
+				}
+				if cr != nil {
+					cr.sendContainerState(containerMapValue)
+				}
+				e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			}
+		case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint, reflect.Uintptr:
+			mksv := make([]uintRv, len(mks))
+			for i, k := range mks {
+				v := &mksv[i]
+				v.r = k
+				v.v = k.Uint()
+			}
+			sort.Sort(uintRvSlice(mksv))
+			for i := range mksv {
+				if cr != nil {
+					cr.sendContainerState(containerMapKey)
+				}
+				ee.EncodeUint(mksv[i].v)
+				if cr != nil {
+					cr.sendContainerState(containerMapValue)
+				}
+				e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			}
+		case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
+			mksv := make([]intRv, len(mks))
+			for i, k := range mks {
+				v := &mksv[i]
+				v.r = k
+				v.v = k.Int()
+			}
+			sort.Sort(intRvSlice(mksv))
+			for i := range mksv {
+				if cr != nil {
+					cr.sendContainerState(containerMapKey)
+				}
+				ee.EncodeInt(mksv[i].v)
+				if cr != nil {
+					cr.sendContainerState(containerMapValue)
+				}
+				e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			}
+		case reflect.Float32:
+			mksv := make([]floatRv, len(mks))
+			for i, k := range mks {
+				v := &mksv[i]
+				v.r = k
+				v.v = k.Float()
+			}
+			sort.Sort(floatRvSlice(mksv))
+			for i := range mksv {
+				if cr != nil {
+					cr.sendContainerState(containerMapKey)
+				}
+				ee.EncodeFloat32(float32(mksv[i].v))
+				if cr != nil {
+					cr.sendContainerState(containerMapValue)
+				}
+				e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			}
+		case reflect.Float64:
+			mksv := make([]floatRv, len(mks))
+			for i, k := range mks {
+				v := &mksv[i]
+				v.r = k
+				v.v = k.Float()
+			}
+			sort.Sort(floatRvSlice(mksv))
+			for i := range mksv {
+				if cr != nil {
+					cr.sendContainerState(containerMapKey)
+				}
+				ee.EncodeFloat64(mksv[i].v)
+				if cr != nil {
+					cr.sendContainerState(containerMapValue)
+				}
+				e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
+			}
+		default:
+			// out-of-band
+			// first encode each key to a []byte first, then sort them, then record
+			var mksv []byte = make([]byte, 0, len(mks)*16) // temporary byte slice for the encoding
+			e2 := NewEncoderBytes(&mksv, e.hh)
+			mksbv := make([]bytesRv, len(mks))
+			for i, k := range mks {
+				v := &mksbv[i]
+				l := len(mksv)
+				e2.MustEncode(k)
+				v.r = k
+				v.v = mksv[l:]
+				// fmt.Printf(">>>>> %s\n", mksv[l:])
+			}
+			sort.Sort(bytesRvSlice(mksbv))
+			for j := range mksbv {
+				if cr != nil {
+					cr.sendContainerState(containerMapKey)
+				}
+				e.asis(mksbv[j].v)
+				if cr != nil {
+					cr.sendContainerState(containerMapValue)
+				}
+				e.encodeValue(rv.MapIndex(mksbv[j].r), valFn)
+			}
+		}
+	}
 }
 
 // --------------------------------------------------
@@ -687,12 +882,15 @@ type Encoder struct {
 
 	wi ioEncWriter
 	wb bytesEncWriter
+
 	h  *BasicHandle
+	hh Handle
 
+	cr containerStateRecv
 	as encDriverAsis
-	hh Handle
-	f  map[uintptr]*encFn
-	b  [scratchByteArrayLen]byte
+
+	f map[uintptr]*encFn
+	b [scratchByteArrayLen]byte
 }
 
 // NewEncoder returns an Encoder for encoding into an io.Writer.
@@ -700,20 +898,8 @@ type Encoder struct {
 // For efficiency, Users are encouraged to pass in a memory buffered writer
 // (eg bufio.Writer, bytes.Buffer).
 func NewEncoder(w io.Writer, h Handle) *Encoder {
-	e := &Encoder{hh: h, h: h.getBasicHandle(), be: h.isBinary()}
-	ww, ok := w.(ioEncWriterWriter)
-	if !ok {
-		sww := simpleIoEncWriterWriter{w: w}
-		sww.bw, _ = w.(io.ByteWriter)
-		sww.sw, _ = w.(ioEncStringWriter)
-		ww = &sww
-		//ww = bufio.NewWriterSize(w, defEncByteBufSize)
-	}
-	e.wi.w = ww
-	e.w = &e.wi
-	_, e.js = h.(*JsonHandle)
-	e.e = h.newEncDriver(e)
-	e.as, _ = e.e.(encDriverAsis)
+	e := newEncoder(h)
+	e.Reset(w)
 	return e
 }
 
@@ -723,19 +909,56 @@ func NewEncoder(w io.Writer, h Handle) *Encoder {
 // It will potentially replace the output byte slice pointed to.
 // After encoding, the out parameter contains the encoded contents.
 func NewEncoderBytes(out *[]byte, h Handle) *Encoder {
+	e := newEncoder(h)
+	e.ResetBytes(out)
+	return e
+}
+
+func newEncoder(h Handle) *Encoder {
 	e := &Encoder{hh: h, h: h.getBasicHandle(), be: h.isBinary()}
+	_, e.js = h.(*JsonHandle)
+	e.e = h.newEncDriver(e)
+	e.as, _ = e.e.(encDriverAsis)
+	e.cr, _ = e.e.(containerStateRecv)
+	return e
+}
+
+// Reset the Encoder with a new output stream.
+//
+// This accomodates using the state of the Encoder,
+// where it has "cached" information about sub-engines.
+func (e *Encoder) Reset(w io.Writer) {
+	ww, ok := w.(ioEncWriterWriter)
+	if ok {
+		e.wi.w = ww
+	} else {
+		sww := &e.wi.s
+		sww.w = w
+		sww.bw, _ = w.(io.ByteWriter)
+		sww.sw, _ = w.(ioEncStringWriter)
+		e.wi.w = sww
+		//ww = bufio.NewWriterSize(w, defEncByteBufSize)
+	}
+	e.w = &e.wi
+	e.e.reset()
+}
+
+func (e *Encoder) ResetBytes(out *[]byte) {
 	in := *out
 	if in == nil {
 		in = make([]byte, defEncByteBufSize)
 	}
-	e.wb.b, e.wb.out = in, out
+	e.wb.b, e.wb.out, e.wb.c = in, out, 0
 	e.w = &e.wb
-	_, e.js = h.(*JsonHandle)
-	e.e = h.newEncDriver(e)
-	e.as, _ = e.e.(encDriverAsis)
-	return e
+	e.e.reset()
 }
 
+// func (e *Encoder) sendContainerState(c containerState) {
+// 	if e.cr != nil {
+// 		e.cr.sendContainerState(c)
+// 	}
+// }
+
 // Encode writes an object into a stream.
 //
 // Encoding can be configured via the struct tag for the fields.
@@ -903,16 +1126,9 @@ func (e *Encoder) encode(iv interface{}) {
 		e.e.EncodeStringBytes(c_RAW, *v)
 
 	default:
-		// canonical mode is not supported for fastpath of maps (but is fine for slices)
 		const checkCodecSelfer1 = true // in case T is passed, where *T is a Selfer, still checkCodecSelfer
-		if e.h.Canonical {
-			if !fastpathEncodeTypeSwitchSlice(iv, e) {
-				e.encodeI(iv, false, checkCodecSelfer1)
-			}
-		} else {
-			if !fastpathEncodeTypeSwitch(iv, e) {
-				e.encodeI(iv, false, checkCodecSelfer1)
-			}
+		if !fastpathEncodeTypeSwitch(iv, e) {
+			e.encodeI(iv, false, checkCodecSelfer1)
 		}
 	}
 }
@@ -927,26 +1143,24 @@ func (e *Encoder) encodeI(iv interface{}, checkFastpath, checkCodecSelfer bool)
 }
 
 func (e *Encoder) preEncodeValue(rv reflect.Value) (rv2 reflect.Value, proceed bool) {
-LOOP:
-	for {
-		switch rv.Kind() {
-		case reflect.Ptr, reflect.Interface:
-			if rv.IsNil() {
-				e.e.EncodeNil()
-				return
-			}
-			rv = rv.Elem()
-			continue LOOP
-		case reflect.Slice, reflect.Map:
-			if rv.IsNil() {
-				e.e.EncodeNil()
-				return
-			}
-		case reflect.Invalid, reflect.Func:
+	// use a goto statement instead of a recursive function for ptr/interface.
+TOP:
+	switch rv.Kind() {
+	case reflect.Ptr, reflect.Interface:
+		if rv.IsNil() {
 			e.e.EncodeNil()
 			return
 		}
-		break
+		rv = rv.Elem()
+		goto TOP
+	case reflect.Slice, reflect.Map:
+		if rv.IsNil() {
+			e.e.EncodeNil()
+			return
+		}
+	case reflect.Invalid, reflect.Func:
+		e.e.EncodeNil()
+		return
 	}
 
 	return rv, true
@@ -996,7 +1210,7 @@ func (e *Encoder) getEncFn(rtid uintptr, rt reflect.Type, checkFastpath, checkCo
 		fn = &(e.s[len(e.s)-1]).fn
 	}
 
-	ti := getTypeInfo(rtid, rt)
+	ti := e.h.getTypeInfo(rtid, rt)
 	fi := &(fn.i)
 	fi.e = e
 	fi.ti = ti
@@ -1019,8 +1233,7 @@ func (e *Encoder) getEncFn(rtid uintptr, rt reflect.Type, checkFastpath, checkCo
 		fn.f = (*encFnInfo).textMarshal
 	} else {
 		rk := rt.Kind()
-		// if fastpathEnabled && checkFastpath && (rk == reflect.Map || rk == reflect.Slice) {
-		if fastpathEnabled && checkFastpath && (rk == reflect.Slice || (rk == reflect.Map && !e.h.Canonical)) {
+		if fastpathEnabled && checkFastpath && (rk == reflect.Map || rk == reflect.Slice) {
 			if rt.PkgPath() == "" {
 				if idx := fastpathAV.index(rtid); idx != -1 {
 					fn.f = fastpathAV[idx].encfn
@@ -1114,11 +1327,6 @@ func (e *Encoder) errorf(format string, params ...interface{}) {
 
 // ----------------------------------------
 
-type encStructFieldKV struct {
-	k string
-	v reflect.Value
-}
-
 const encStructPoolLen = 5
 
 // encStructPool is an array of sync.Pool.
@@ -1133,33 +1341,33 @@ const encStructPoolLen = 5
 var encStructPool [encStructPoolLen]sync.Pool
 
 func init() {
-	encStructPool[0].New = func() interface{} { return new([8]encStructFieldKV) }
-	encStructPool[1].New = func() interface{} { return new([16]encStructFieldKV) }
-	encStructPool[2].New = func() interface{} { return new([32]encStructFieldKV) }
-	encStructPool[3].New = func() interface{} { return new([64]encStructFieldKV) }
-	encStructPool[4].New = func() interface{} { return new([128]encStructFieldKV) }
+	encStructPool[0].New = func() interface{} { return new([8]stringRv) }
+	encStructPool[1].New = func() interface{} { return new([16]stringRv) }
+	encStructPool[2].New = func() interface{} { return new([32]stringRv) }
+	encStructPool[3].New = func() interface{} { return new([64]stringRv) }
+	encStructPool[4].New = func() interface{} { return new([128]stringRv) }
 }
 
-func encStructPoolGet(newlen int) (p *sync.Pool, v interface{}, s []encStructFieldKV) {
+func encStructPoolGet(newlen int) (p *sync.Pool, v interface{}, s []stringRv) {
 	// if encStructPoolLen != 5 { // constant chec, so removed at build time.
 	// 	panic(errors.New("encStructPoolLen must be equal to 4")) // defensive, in case it is changed
 	// }
 	// idxpool := newlen / 8
 
 	// if pool == nil {
-	// 	fkvs = make([]encStructFieldKV, newlen)
+	// 	fkvs = make([]stringRv, newlen)
 	// } else {
 	// 	poolv = pool.Get()
 	// 	switch vv := poolv.(type) {
-	// 	case *[8]encStructFieldKV:
+	// 	case *[8]stringRv:
 	// 		fkvs = vv[:newlen]
-	// 	case *[16]encStructFieldKV:
+	// 	case *[16]stringRv:
 	// 		fkvs = vv[:newlen]
-	// 	case *[32]encStructFieldKV:
+	// 	case *[32]stringRv:
 	// 		fkvs = vv[:newlen]
-	// 	case *[64]encStructFieldKV:
+	// 	case *[64]stringRv:
 	// 		fkvs = vv[:newlen]
-	// 	case *[128]encStructFieldKV:
+	// 	case *[128]stringRv:
 	// 		fkvs = vv[:newlen]
 	// 	}
 	// }
@@ -1167,25 +1375,25 @@ func encStructPoolGet(newlen int) (p *sync.Pool, v interface{}, s []encStructFie
 	if newlen <= 8 {
 		p = &encStructPool[0]
 		v = p.Get()
-		s = v.(*[8]encStructFieldKV)[:newlen]
+		s = v.(*[8]stringRv)[:newlen]
 	} else if newlen <= 16 {
 		p = &encStructPool[1]
 		v = p.Get()
-		s = v.(*[16]encStructFieldKV)[:newlen]
+		s = v.(*[16]stringRv)[:newlen]
 	} else if newlen <= 32 {
 		p = &encStructPool[2]
 		v = p.Get()
-		s = v.(*[32]encStructFieldKV)[:newlen]
+		s = v.(*[32]stringRv)[:newlen]
 	} else if newlen <= 64 {
 		p = &encStructPool[3]
 		v = p.Get()
-		s = v.(*[64]encStructFieldKV)[:newlen]
+		s = v.(*[64]stringRv)[:newlen]
 	} else if newlen <= 128 {
 		p = &encStructPool[4]
 		v = p.Get()
-		s = v.(*[128]encStructFieldKV)[:newlen]
+		s = v.(*[128]stringRv)[:newlen]
 	} else {
-		s = make([]encStructFieldKV, newlen)
+		s = make([]stringRv, newlen)
 	}
 	return
 }

Godeps/_workspace/src/github.com/ugorji/go/codec/fast-path.go.tmpl

@@ -1,4 +1,4 @@
-// //+build ignore
+// +build !notfastpath
 
 // Copyright (c) 2012-2015 Ugorji Nwoke. All rights reserved.
 // Use of this source code is governed by a MIT license found in the LICENSE file.
@@ -106,6 +106,9 @@ func init() {
 
 // -- -- fast path type switch
 func fastpathEncodeTypeSwitch(iv interface{}, e *Encoder) bool {
+	if !fastpathEnabled {
+		return false
+	}
 	switch v := iv.(type) {
 {{range .Values}}{{if not .Primitive}}{{if not .MapKey }}
 	case []{{ .Elem }}:{{else}}
@@ -116,12 +119,16 @@ func fastpathEncodeTypeSwitch(iv interface{}, e *Encoder) bool {
 		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(*v, fastpathCheckNilTrue, e)
 {{end}}{{end}}
 	default:
+        _ = v // TODO: workaround https://github.com/golang/go/issues/12927 (remove after go 1.6 release)
 		return false
 	}
 	return true
 }
 
 func fastpathEncodeTypeSwitchSlice(iv interface{}, e *Encoder) bool {
+	if !fastpathEnabled {
+		return false
+	}
 	switch v := iv.(type) {
 {{range .Values}}{{if not .Primitive}}{{if not .MapKey }}
 	case []{{ .Elem }}:
@@ -130,12 +137,16 @@ func fastpathEncodeTypeSwitchSlice(iv interface{}, e *Encoder) bool {
 		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(*v, fastpathCheckNilTrue, e)
 {{end}}{{end}}{{end}}
 	default:
+        _ = v // TODO: workaround https://github.com/golang/go/issues/12927 (remove after go 1.6 release)
 		return false
 	}
 	return true
 }
 
 func fastpathEncodeTypeSwitchMap(iv interface{}, e *Encoder) bool {
+	if !fastpathEnabled {
+		return false
+	}
 	switch v := iv.(type) {
 {{range .Values}}{{if not .Primitive}}{{if .MapKey }}
 	case map[{{ .MapKey }}]{{ .Elem }}:
@@ -144,6 +155,7 @@ func fastpathEncodeTypeSwitchMap(iv interface{}, e *Encoder) bool {
 		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(*v, fastpathCheckNilTrue, e)
 {{end}}{{end}}{{end}}
 	default:
+        _ = v // TODO: workaround https://github.com/golang/go/issues/12927 (remove after go 1.6 release)
 		return false
 	}
 	return true
@@ -156,16 +168,18 @@ func (f *encFnInfo) {{ .MethodNamePfx "fastpathEnc" false }}R(rv reflect.Value)
 	fastpathTV.{{ .MethodNamePfx "Enc" false }}V(rv.Interface().([]{{ .Elem }}), fastpathCheckNilFalse, f.e)
 }
 func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v []{{ .Elem }}, checkNil bool, e *Encoder) {
-	ee := e.e 
+	ee := e.e
+	cr := e.cr
 	if checkNil && v == nil {
 		ee.EncodeNil()
 		return
 	}
 	ee.EncodeArrayStart(len(v))
 	for _, v2 := range v {
+		if cr != nil { cr.sendContainerState(containerArrayElem) }
 		{{ encmd .Elem "v2"}}
 	}
-	ee.EncodeEnd()
+	if cr != nil { cr.sendContainerState(containerArrayEnd) }{{/* ee.EncodeEnd() */}}
 }
 
 {{end}}{{end}}{{end}}
@@ -177,21 +191,64 @@ func (f *encFnInfo) {{ .MethodNamePfx "fastpathEnc" false }}R(rv reflect.Value)
 }
 func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v map[{{ .MapKey }}]{{ .Elem }}, checkNil bool, e *Encoder) {
 	ee := e.e
+	cr := e.cr
 	if checkNil && v == nil {
 		ee.EncodeNil()
 		return
 	}
 	ee.EncodeMapStart(len(v))
-	{{if eq .MapKey "string"}}asSymbols := e.h.AsSymbols&AsSymbolMapStringKeysFlag != 0{{end}}
-	for k2, v2 := range v {
-		{{if eq .MapKey "string"}}if asSymbols {
-			ee.EncodeSymbol(k2)
-		} else {
-			ee.EncodeString(c_UTF8, k2)
-		}{{else}}{{ encmd .MapKey "k2"}}{{end}}
-		{{ encmd .Elem "v2"}}
+	{{if eq .MapKey "string"}}asSymbols := e.h.AsSymbols&AsSymbolMapStringKeysFlag != 0
+	{{end}}if e.h.Canonical {
+		{{if eq .MapKey "interface{}"}}{{/* out of band 
+		*/}}var mksv []byte = make([]byte, 0, len(v)*16) // temporary byte slice for the encoding
+		e2 := NewEncoderBytes(&mksv, e.hh)
+		v2 := make([]bytesI, len(v))
+		var i, l int
+		var vp *bytesI {{/* put loop variables outside. seems currently needed for better perf */}}
+		for k2, _ := range v {
+			l = len(mksv)
+			e2.MustEncode(k2)
+			vp = &v2[i]
+			vp.v = mksv[l:]
+			vp.i = k2 
+			i++
+		}
+		sort.Sort(bytesISlice(v2))
+		for j := range v2 {
+			if cr != nil { cr.sendContainerState(containerMapKey) }
+			e.asis(v2[j].v)
+			if cr != nil { cr.sendContainerState(containerMapValue) }
+			e.encode(v[v2[j].i])
+		} {{else}}{{ $x := sorttype .MapKey true}}v2 := make([]{{ $x }}, len(v))
+		var i int 
+		for k, _ := range v {
+			v2[i] = {{ $x }}(k)
+			i++
+		}
+		sort.Sort({{ sorttype .MapKey false}}(v2))
+		for _, k2 := range v2 {
+			if cr != nil { cr.sendContainerState(containerMapKey) }
+			{{if eq .MapKey "string"}}if asSymbols {
+				ee.EncodeSymbol(k2)
+			} else {
+				ee.EncodeString(c_UTF8, k2)
+			}{{else}}{{ $y := printf "%s(k2)" .MapKey }}{{ encmd .MapKey $y }}{{end}}
+			if cr != nil { cr.sendContainerState(containerMapValue) }
+			{{ $y := printf "v[%s(k2)]" .MapKey }}{{ encmd .Elem $y }}
+		} {{end}}
+	} else {
+		for k2, v2 := range v {
+			if cr != nil { cr.sendContainerState(containerMapKey) }
+			{{if eq .MapKey "string"}}if asSymbols {
+				ee.EncodeSymbol(k2)
+			} else {
+				ee.EncodeString(c_UTF8, k2)
+			}{{else}}{{ encmd .MapKey "k2"}}{{end}}
+			if cr != nil { cr.sendContainerState(containerMapValue) }
+			{{ encmd .Elem "v2"}}
+		}
 	}
-	ee.EncodeEnd()
+	if cr != nil { cr.sendContainerState(containerMapEnd) }{{/* ee.EncodeEnd() */}}
 }
 
 {{end}}{{end}}{{end}}
@@ -200,6 +257,9 @@ func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v map[{{ .MapKey }}]{{ .Ele
 
 // -- -- fast path type switch
 func fastpathDecodeTypeSwitch(iv interface{}, d *Decoder) bool {
+	if !fastpathEnabled {
+		return false
+	}
 	switch v := iv.(type) {
 {{range .Values}}{{if not .Primitive}}{{if not .MapKey }}
 	case []{{ .Elem }}:{{else}}
@@ -213,6 +273,7 @@ func fastpathDecodeTypeSwitch(iv interface{}, d *Decoder) bool {
 		}
 {{end}}{{end}}
 	default:
+        _ = v // TODO: workaround https://github.com/golang/go/issues/12927 (remove after go 1.6 release)
 		return false
 	}
 	return true
@@ -246,8 +307,7 @@ func (f fastpathT) {{ .MethodNamePfx "Dec" false }}X(vp *[]{{ .Elem }}, checkNil
 		*vp = v 
 	}
 }
-func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil bool, canChange bool, 
-	d *Decoder) (_ []{{ .Elem }}, changed bool) {
+func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil bool, canChange bool, d *Decoder) (_ []{{ .Elem }}, changed bool) {
 	dd := d.d
 	{{/* // if dd.isContainerType(valueTypeNil) { dd.TryDecodeAsNil() */}}
 	if checkNil && dd.TryDecodeAsNil() {
@@ -258,28 +318,22 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil b
 	}
 
 	slh, containerLenS := d.decSliceHelperStart()
-    x2read := containerLenS
-    var xtrunc bool 
-	if canChange && v == nil {
-		var xlen int 
-		if xlen, xtrunc = decInferLen(containerLenS, d.h.MaxInitLen, {{ .Size }}); xtrunc {
-			x2read = xlen
-		}
-		v = make([]{{ .Elem }}, xlen)
-		changed = true
-	} 
 	if containerLenS == 0 {
-		if canChange && len(v) != 0 {
-			v = v[:0]
-			changed = true 
-		}{{/*
-			// slh.End() // dd.ReadArrayEnd()
-		*/}}
-		return v, changed 
+		if canChange {
+			if v == nil {
+				v = []{{ .Elem }}{}
+			} else if len(v) != 0 {
+				v = v[:0]
+			}
+			changed = true
+		}
+		slh.End()
+		return
 	}
 	
-	{{/* // for j := 0; j < containerLenS; j++ { */}}
 	if containerLenS > 0 {
+		x2read := containerLenS
+		var xtrunc bool 
 		if containerLenS > cap(v) {
 			if canChange { {{/*
 				// fast-path is for "basic" immutable types, so no need to copy them over
@@ -287,37 +341,64 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil b
 				// copy(s, v[:cap(v)])
 				// v = s */}}
 				var xlen int 
-                if xlen, xtrunc = decInferLen(containerLenS, d.h.MaxInitLen, {{ .Size }}); xtrunc {
-					x2read = xlen
+                xlen, xtrunc = decInferLen(containerLenS, d.h.MaxInitLen, {{ .Size }})
+				if xtrunc {
+					if xlen <= cap(v) {
+						v = v[:xlen]
+					} else {
+						v = make([]{{ .Elem }}, xlen)
+					}
+				} else {
+					v = make([]{{ .Elem }}, xlen)
 				}
-                v = make([]{{ .Elem }}, xlen)
 				changed = true
 			} else {
 				d.arrayCannotExpand(len(v), containerLenS)
-				x2read = len(v)
 			}
+			x2read = len(v)
 		} else if containerLenS != len(v) {
-			v = v[:containerLenS]
-			changed = true
-		}
-		{{/* // all checks done. cannot go past len. */}}
+			if canChange {
+				v = v[:containerLenS]
+				changed = true
+			}
+		} {{/* // all checks done. cannot go past len. */}}
 		j := 0
-		for ; j < x2read; j++ { 
+		for ; j < x2read; j++ {
+			slh.ElemContainerState(j)
 			{{ if eq .Elem "interface{}" }}d.decode(&v[j]){{ else }}v[j] = {{ decmd .Elem }}{{ end }}
 		}
 		if xtrunc { {{/* // means canChange=true, changed=true already. */}}
 			for ; j < containerLenS; j++ {
 				v = append(v, {{ zerocmd .Elem }})
+				slh.ElemContainerState(j)
 				{{ if eq .Elem "interface{}" }}d.decode(&v[j]){{ else }}v[j] = {{ decmd .Elem }}{{ end }}
 			}
 		} else if !canChange {
-			for ; j < containerLenS; j++ { 
+			for ; j < containerLenS; j++ {
+				slh.ElemContainerState(j)
 				d.swallow()
 			}
 		}
 	} else {
-		j := 0
-		for ; !dd.CheckBreak(); j++ {
+		breakFound := dd.CheckBreak() {{/* check break first, so we can initialize v with a capacity of 4 if necessary */}}
+		if breakFound {
+			if canChange {
+				if v == nil {
+					v = []{{ .Elem }}{}
+				} else if len(v) != 0 {
+					v = v[:0]
+				}
+				changed = true
+			}
+			slh.End()
+			return
+		}
+		if cap(v) == 0 {
+			v = make([]{{ .Elem }}, 1, 4)
+			changed = true
+		}
+		j := 0	
+		for ; !breakFound; j++ {
 			if j >= len(v) { 
 				if canChange {
 					v = append(v, {{ zerocmd .Elem }})
@@ -325,16 +406,22 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil b
 				} else {
 					d.arrayCannotExpand(len(v), j+1)
 				}
-			} 
+			}
+			slh.ElemContainerState(j)
 			if j < len(v) { {{/* // all checks done. cannot go past len. */}}
 				{{ if eq .Elem "interface{}" }}d.decode(&v[j])
 				{{ else }}v[j] = {{ decmd .Elem }}{{ end }}
 			} else {
 				d.swallow()
 			}
+			breakFound = dd.CheckBreak()
+		}
+		if canChange && j < len(v) {
+			v = v[:j]
+			changed = true
 		}
-		slh.End() 
 	}
+	slh.End() 
 	return v, changed 
 }
 
@@ -368,6 +455,7 @@ func (f fastpathT) {{ .MethodNamePfx "Dec" false }}X(vp *map[{{ .MapKey }}]{{ .E
 func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Elem }}, checkNil bool, canChange bool, 
 	d *Decoder) (_ map[{{ .MapKey }}]{{ .Elem }}, changed bool) {
 	dd := d.d
+	cr := d.cr
 	{{/* // if dd.isContainerType(valueTypeNil) {dd.TryDecodeAsNil() */}}
 	if checkNil && dd.TryDecodeAsNil() {
 		if v != nil {
@@ -382,36 +470,41 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Ele
 		v = make(map[{{ .MapKey }}]{{ .Elem }}, xlen)
 		changed = true
 	}
+	{{ if eq .Elem "interface{}" }}mapGet := !d.h.MapValueReset && !d.h.InterfaceReset{{end}}
+	var mk {{ .MapKey }}
+	var mv {{ .Elem }}
 	if containerLen > 0 {
 		for j := 0; j < containerLen; j++ {
-			{{ if eq .MapKey "interface{}" }}var mk interface{}
+			if cr != nil { cr.sendContainerState(containerMapKey) }
+			{{ if eq .MapKey "interface{}" }}mk = nil 
 			d.decode(&mk)
 			if bv, bok := mk.([]byte); bok {
-				mk = string(bv) {{/* // maps cannot have []byte as key. switch to string. */}}
-			}{{ else }}mk := {{ decmd .MapKey }}{{ end }}
-			mv := v[mk]
-			{{ if eq .Elem "interface{}" }}d.decode(&mv)
-			{{ else }}mv = {{ decmd .Elem }}{{ end }}
+				mk = d.string(bv) {{/* // maps cannot have []byte as key. switch to string. */}}
+			}{{ else }}mk = {{ decmd .MapKey }}{{ end }}
+			if cr != nil { cr.sendContainerState(containerMapValue) }
+			{{ if eq .Elem "interface{}" }}if mapGet { mv = v[mk] } else { mv = nil }
+			d.decode(&mv){{ else }}mv = {{ decmd .Elem }}{{ end }}
 			if v != nil {
 				v[mk] = mv
 			}
 		}
 	} else if containerLen < 0 {
 		for j := 0; !dd.CheckBreak(); j++ {
-			{{ if eq .MapKey "interface{}" }}var mk interface{}
+			if cr != nil { cr.sendContainerState(containerMapKey) }
+			{{ if eq .MapKey "interface{}" }}mk = nil 
 			d.decode(&mk)
 			if bv, bok := mk.([]byte); bok {
-				mk = string(bv) {{/* // maps cannot have []byte as key. switch to string. */}}
-			}{{ else }}mk := {{ decmd .MapKey }}{{ end }}
-			mv := v[mk]
-			{{ if eq .Elem "interface{}" }}d.decode(&mv)
-			{{ else }}mv = {{ decmd .Elem }}{{ end }}
+				mk = d.string(bv) {{/* // maps cannot have []byte as key. switch to string. */}}
+			}{{ else }}mk = {{ decmd .MapKey }}{{ end }}
+			if cr != nil { cr.sendContainerState(containerMapValue) }
+			{{ if eq .Elem "interface{}" }}if mapGet { mv = v[mk] } else { mv = nil }
+			d.decode(&mv){{ else }}mv = {{ decmd .Elem }}{{ end }}
 			if v != nil {
 				v[mk] = mv
 			}
 		}
-		dd.ReadEnd()
 	}
+	if cr != nil { cr.sendContainerState(containerMapEnd) }
 	return v, changed
 }
 

Godeps/_workspace/src/github.com/ugorji/go/codec/gen-dec-array.go.tmpl

@@ -1,86 +1,101 @@
-{{var "v"}} := {{ if not isArray}}*{{ end }}{{ .Varname }}
+{{var "v"}} := {{if not isArray}}*{{end}}{{ .Varname }}
 {{var "h"}}, {{var "l"}} := z.DecSliceHelperStart() {{/* // helper, containerLenS */}}
-
-var {{var "rr"}}, {{var "rl"}} int {{/* // num2read, length of slice/array/chan */}}
-var {{var "c"}}, {{var "rt"}} bool {{/* // changed, truncated */}}
-_, _, _ = {{var "c"}}, {{var "rt"}}, {{var "rl"}}
-{{var "rr"}} = {{var "l"}}
-{{/* rl is NOT used. Only used for getting DecInferLen. len(r) used directly in code */}}
-
-{{ if not isArray }}if {{var "v"}} == nil {
-	if {{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }}); {{var "rt"}} {
-		{{var "rr"}} = {{var "rl"}}
-	}
-	{{var "v"}} = make({{ .CTyp }}, {{var "rl"}})
-	{{var "c"}} = true 
-} 
-{{ end }}
-if {{var "l"}} == 0 { {{ if isSlice }}
-	if len({{var "v"}}) != 0 { 
-		{{var "v"}} = {{var "v"}}[:0] 
-		{{var "c"}} = true 
-	} {{ end }}
+var {{var "c"}} bool {{/* // changed */}}
+if {{var "l"}} == 0 {
+	{{if isSlice }}if {{var "v"}} == nil {
+		{{var "v"}} = []{{ .Typ }}{}
+		{{var "c"}} = true
+	} else if len({{var "v"}}) != 0 {
+		{{var "v"}} = {{var "v"}}[:0]
+		{{var "c"}} = true
+	} {{end}} {{if isChan }}if {{var "v"}} == nil {
+		{{var "v"}} = make({{ .CTyp }}, 0)
+		{{var "c"}} = true
+	} {{end}}
 } else if {{var "l"}} > 0 {
-	{{ if isChan }}
+	{{if isChan }}if {{var "v"}} == nil {
+		{{var "rl"}}, _ = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+		{{var "v"}} = make({{ .CTyp }}, {{var "rl"}})
+		{{var "c"}} = true
+	}
 	for {{var "r"}} := 0; {{var "r"}} < {{var "l"}}; {{var "r"}}++ {
+		{{var "h"}}.ElemContainerState({{var "r"}})
 		var {{var "t"}} {{ .Typ }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}
-		{{var "v"}} <- {{var "t"}} 
-	{{ else }} 
+		{{var "v"}} <- {{var "t"}}
+	}
+	{{ else }}	var {{var "rr"}}, {{var "rl"}} int {{/* // num2read, length of slice/array/chan */}}
+	var {{var "rt"}} bool {{/* truncated */}}
 	if {{var "l"}} > cap({{var "v"}}) {
-		{{ if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "l"}})
-		{{ else }}{{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
-		{{ if .Immutable }}
-		{{var "v2"}} := {{var "v"}}
-		{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
-		if len({{var "v"}}) > 0 {
-			copy({{var "v"}}, {{var "v2"}}[:cap({{var "v2"}})])
+		{{if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "l"}})
+		{{ else }}{{if not .Immutable }}
+		{{var "rg"}} := len({{var "v"}}) > 0
+		{{var "v2"}} := {{var "v"}} {{end}}
+		{{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+		if {{var "rt"}} {
+			if {{var "rl"}} <= cap({{var "v"}}) {
+				{{var "v"}} = {{var "v"}}[:{{var "rl"}}]
+			} else {
+				{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
+			}
+		} else {
+			{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
 		}
-		{{ else }}{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
-		{{ end }}{{var "c"}} = true 
-		{{ end }}
-		{{var "rr"}} = len({{var "v"}})
-	} else if {{var "l"}} != len({{var "v"}}) {
-		{{ if isSlice }}{{var "v"}} = {{var "v"}}[:{{var "l"}}]
-		{{var "c"}} = true {{ end }}
-	}
+		{{var "c"}} = true
+		{{var "rr"}} = len({{var "v"}}) {{if not .Immutable }}
+			if {{var "rg"}} { copy({{var "v"}}, {{var "v2"}}) } {{end}} {{end}}{{/* end not Immutable, isArray */}}
+	} {{if isSlice }} else if {{var "l"}} != len({{var "v"}}) {
+		{{var "v"}} = {{var "v"}}[:{{var "l"}}]
+		{{var "c"}} = true
+	} {{end}}	{{/* end isSlice:47 */}} 
 	{{var "j"}} := 0
 	for ; {{var "j"}} < {{var "rr"}} ; {{var "j"}}++ {
+		{{var "h"}}.ElemContainerState({{var "j"}})
 		{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
 	}
-	{{ if isArray }}for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
+	{{if isArray }}for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
+		{{var "h"}}.ElemContainerState({{var "j"}})
 		z.DecSwallow()
 	}
-	{{ else }}if {{var "rt"}} { {{/* means that it is mutable and slice */}}
+	{{ else }}if {{var "rt"}} {
 		for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
 			{{var "v"}} = append({{var "v"}}, {{ zero}})
+			{{var "h"}}.ElemContainerState({{var "j"}})
 			{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
 		}
-	}
-	{{ end }}
-	{{ end }}{{/* closing 'if not chan' */}}
-} else {
-	for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
-		if {{var "j"}} >= len({{var "v"}}) {
-			{{ if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "j"}}+1)
-			{{ else if isSlice}}{{var "v"}} = append({{var "v"}}, {{zero}})// var {{var "z"}} {{ .Typ }}
-			{{var "c"}} = true {{ end }}
-		}
-		{{ if isChan}}
+	} {{end}} {{/* end isArray:56 */}}
+	{{end}} {{/* end isChan:16 */}}
+} else { {{/* len < 0 */}}
+	{{var "j"}} := 0
+	for ; !r.CheckBreak(); {{var "j"}}++ {
+		{{if isChan }}
+		{{var "h"}}.ElemContainerState({{var "j"}})
 		var {{var "t"}} {{ .Typ }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}
 		{{var "v"}} <- {{var "t"}} 
 		{{ else }}
+		if {{var "j"}} >= len({{var "v"}}) {
+			{{if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "j"}}+1)
+			{{ else }}{{var "v"}} = append({{var "v"}}, {{zero}})// var {{var "z"}} {{ .Typ }}
+			{{var "c"}} = true {{end}}
+		}
+		{{var "h"}}.ElemContainerState({{var "j"}})
 		if {{var "j"}} < len({{var "v"}}) {
 			{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
 		} else {
 			z.DecSwallow()
 		}
-		{{ end }}
+		{{end}}
 	}
-	{{var "h"}}.End()
+	{{if isSlice }}if {{var "j"}} < len({{var "v"}}) {
+		{{var "v"}} = {{var "v"}}[:{{var "j"}}]
+		{{var "c"}} = true
+	} else if {{var "j"}} == 0 && {{var "v"}} == nil {
+		{{var "v"}} = []{{ .Typ }}{}
+		{{var "c"}} = true
+	}{{end}}
 }
-{{ if not isArray }}if {{var "c"}} { 
+{{var "h"}}.End()
+{{if not isArray }}if {{var "c"}} { 
 	*{{ .Varname }} = {{var "v"}}
-}{{ end }}
-
+}{{end}}

Godeps/_workspace/src/github.com/ugorji/go/codec/gen-dec-map.go.tmpl

@@ -1,39 +1,58 @@
 {{var "v"}} := *{{ .Varname }}
 {{var "l"}} := r.ReadMapStart()
+{{var "bh"}} := z.DecBasicHandle()
 if {{var "v"}} == nil {
-	{{var "rl"}}, _ := z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+	{{var "rl"}}, _ := z.DecInferLen({{var "l"}}, {{var "bh"}}.MaxInitLen, {{ .Size }})
 	{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}, {{var "rl"}})
 	*{{ .Varname }} = {{var "v"}}
 }
+var {{var "mk"}} {{ .KTyp }}
+var {{var "mv"}} {{ .Typ }}
+var {{var "mg"}} {{if decElemKindPtr}}, {{var "ms"}}, {{var "mok"}}{{end}} bool
+if {{var "bh"}}.MapValueReset {
+	{{if decElemKindPtr}}{{var "mg"}} = true
+	{{else if decElemKindIntf}}if !{{var "bh"}}.InterfaceReset { {{var "mg"}} = true }
+	{{else if not decElemKindImmutable}}{{var "mg"}} = true
+	{{end}} }
 if {{var "l"}} > 0  {
 for {{var "j"}} := 0; {{var "j"}} < {{var "l"}}; {{var "j"}}++ {
-	var {{var "mk"}} {{ .KTyp }} 
+	z.DecSendContainerState(codecSelfer_containerMapKey{{ .Sfx }})
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x }}
-{{ if eq .KTyp "interface{}" }}// special case if a byte array.
-	if {{var "bv"}}, {{var "bok"}} := {{var "mk"}}.([]byte); {{var "bok"}} {
+{{ if eq .KTyp "interface{}" }}{{/* // special case if a byte array. */}}if {{var "bv"}}, {{var "bok"}} := {{var "mk"}}.([]byte); {{var "bok"}} {
 		{{var "mk"}} = string({{var "bv"}})
-	}
-{{ end }}
-	{{var "mv"}} := {{var "v"}}[{{var "mk"}}]
+	}{{ end }}{{if decElemKindPtr}}
+	{{var "ms"}} = true{{end}}
+	if {{var "mg"}} {
+		{{if decElemKindPtr}}{{var "mv"}}, {{var "mok"}} = {{var "v"}}[{{var "mk"}}] 
+		if {{var "mok"}} {
+			{{var "ms"}} = false
+		} {{else}}{{var "mv"}} = {{var "v"}}[{{var "mk"}}] {{end}}
+	} {{if not decElemKindImmutable}}else { {{var "mv"}} = {{decElemZero}} }{{end}}
+	z.DecSendContainerState(codecSelfer_containerMapValue{{ .Sfx }})
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ decLineVar $x }}
-	if {{var "v"}} != nil {
+	if {{if decElemKindPtr}} {{var "ms"}} && {{end}} {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
 } else if {{var "l"}} < 0  {
 for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
-	var {{var "mk"}} {{ .KTyp }} 
+	z.DecSendContainerState(codecSelfer_containerMapKey{{ .Sfx }})
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x }}
-{{ if eq .KTyp "interface{}" }}// special case if a byte array.
-	if {{var "bv"}}, {{var "bok"}} := {{var "mk"}}.([]byte); {{var "bok"}} {
+{{ if eq .KTyp "interface{}" }}{{/* // special case if a byte array. */}}if {{var "bv"}}, {{var "bok"}} := {{var "mk"}}.([]byte); {{var "bok"}} {
 		{{var "mk"}} = string({{var "bv"}})
-	}
-{{ end }}
-	{{var "mv"}} := {{var "v"}}[{{var "mk"}}]
+	}{{ end }}{{if decElemKindPtr}}
+	{{var "ms"}} = true {{ end }}
+	if {{var "mg"}} {
+		{{if decElemKindPtr}}{{var "mv"}}, {{var "mok"}} = {{var "v"}}[{{var "mk"}}] 
+		if {{var "mok"}} {
+			{{var "ms"}} = false
+		} {{else}}{{var "mv"}} = {{var "v"}}[{{var "mk"}}] {{end}}
+	} {{if not decElemKindImmutable}}else { {{var "mv"}} = {{decElemZero}} }{{end}}
+	z.DecSendContainerState(codecSelfer_containerMapValue{{ .Sfx }})
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ decLineVar $x }}
-	if {{var "v"}} != nil {
+	if {{if decElemKindPtr}} {{var "ms"}} && {{end}} {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
-r.ReadEnd()
 } // else len==0: TODO: Should we clear map entries?
+z.DecSendContainerState(codecSelfer_containerMapEnd{{ .Sfx }})

Godeps/_workspace/src/github.com/ugorji/go/codec/gen-helper.generated.go

@@ -115,6 +115,15 @@ func (f genHelperEncoder) EncExt(v interface{}) (r bool) {
 	return false
 }
 
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncSendContainerState(c containerState) {
+	if f.e.cr != nil {
+		f.e.cr.sendContainerState(c)
+	}
+}
+
+// ---------------- DECODER FOLLOWS -----------------
+
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) DecBasicHandle() *BasicHandle {
 	return f.d.h
@@ -167,11 +176,8 @@ func (f genHelperDecoder) DecTextUnmarshal(tm encoding.TextUnmarshaler) {
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) DecJSONUnmarshal(tm jsonUnmarshaler) {
 	// bs := f.dd.DecodeBytes(f.d.b[:], true, true)
-	f.d.r.track()
-	f.d.swallow()
-	bs := f.d.r.stopTrack()
-	// fmt.Printf(">>>>>> CODECGEN JSON: %s\n", bs)
-	fnerr := tm.UnmarshalJSON(bs)
+	// grab the bytes to be read, as UnmarshalJSON needs the full JSON so as to unmarshal it itself.
+	fnerr := tm.UnmarshalJSON(f.d.nextValueBytes())
 	if fnerr != nil {
 		panic(fnerr)
 	}
@@ -218,3 +224,10 @@ func (f genHelperDecoder) DecExt(v interface{}) (r bool) {
 func (f genHelperDecoder) DecInferLen(clen, maxlen, unit int) (rvlen int, truncated bool) {
 	return decInferLen(clen, maxlen, unit)
 }
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecSendContainerState(c containerState) {
+	if f.d.cr != nil {
+		f.d.cr.sendContainerState(c)
+	}
+}

Godeps/_workspace/src/github.com/ugorji/go/codec/gen-helper.go.tmpl

@@ -106,6 +106,14 @@ func (f genHelperEncoder) EncExt(v interface{}) (r bool) {
 	}
 	return false 
 }
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncSendContainerState(c containerState) {
+	if f.e.cr != nil {
+		f.e.cr.sendContainerState(c)
+	}
+}
+
+// ---------------- DECODER FOLLOWS -----------------
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) DecBasicHandle() *BasicHandle {
@@ -150,11 +158,8 @@ func (f genHelperDecoder) DecTextUnmarshal(tm encoding.TextUnmarshaler) {
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) DecJSONUnmarshal(tm jsonUnmarshaler) {
 	// bs := f.dd.DecodeBytes(f.d.b[:], true, true)
-	f.d.r.track()
-	f.d.swallow()
-	bs := f.d.r.stopTrack()
-	// fmt.Printf(">>>>>> CODECGEN JSON: %s\n", bs)
-	fnerr := tm.UnmarshalJSON(bs)
+	// grab the bytes to be read, as UnmarshalJSON needs the full JSON so as to unmarshal it itself.
+	fnerr := tm.UnmarshalJSON(f.d.nextValueBytes())
 	if fnerr != nil {
 		panic(fnerr)
 	}
@@ -195,6 +200,12 @@ func (f genHelperDecoder) DecExt(v interface{}) (r bool) {
 func (f genHelperDecoder) DecInferLen(clen, maxlen, unit int) (rvlen int, truncated bool) {
 	return decInferLen(clen, maxlen, unit)
 }
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecSendContainerState(c containerState) {
+	if f.d.cr != nil {
+		f.d.cr.sendContainerState(c)
+	}
+}
 
 {{/*
 

Godeps/_workspace/src/github.com/ugorji/go/codec/gen.generated.go

@@ -8,131 +8,165 @@ package codec
 const genDecMapTmpl = `
 {{var "v"}} := *{{ .Varname }}
 {{var "l"}} := r.ReadMapStart()
+{{var "bh"}} := z.DecBasicHandle()
 if {{var "v"}} == nil {
-	{{var "rl"}}, _ := z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+	{{var "rl"}}, _ := z.DecInferLen({{var "l"}}, {{var "bh"}}.MaxInitLen, {{ .Size }})
 	{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}, {{var "rl"}})
 	*{{ .Varname }} = {{var "v"}}
 }
+var {{var "mk"}} {{ .KTyp }}
+var {{var "mv"}} {{ .Typ }}
+var {{var "mg"}} {{if decElemKindPtr}}, {{var "ms"}}, {{var "mok"}}{{end}} bool
+if {{var "bh"}}.MapValueReset {
+	{{if decElemKindPtr}}{{var "mg"}} = true
+	{{else if decElemKindIntf}}if !{{var "bh"}}.InterfaceReset { {{var "mg"}} = true }
+	{{else if not decElemKindImmutable}}{{var "mg"}} = true
+	{{end}} }
 if {{var "l"}} > 0  {
 for {{var "j"}} := 0; {{var "j"}} < {{var "l"}}; {{var "j"}}++ {
-	var {{var "mk"}} {{ .KTyp }} 
+	z.DecSendContainerState(codecSelfer_containerMapKey{{ .Sfx }})
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x }}
-{{ if eq .KTyp "interface{}" }}// special case if a byte array.
-	if {{var "bv"}}, {{var "bok"}} := {{var "mk"}}.([]byte); {{var "bok"}} {
+{{ if eq .KTyp "interface{}" }}{{/* // special case if a byte array. */}}if {{var "bv"}}, {{var "bok"}} := {{var "mk"}}.([]byte); {{var "bok"}} {
 		{{var "mk"}} = string({{var "bv"}})
-	}
-{{ end }}
-	{{var "mv"}} := {{var "v"}}[{{var "mk"}}]
+	}{{ end }}{{if decElemKindPtr}}
+	{{var "ms"}} = true{{end}}
+	if {{var "mg"}} {
+		{{if decElemKindPtr}}{{var "mv"}}, {{var "mok"}} = {{var "v"}}[{{var "mk"}}] 
+		if {{var "mok"}} {
+			{{var "ms"}} = false
+		} {{else}}{{var "mv"}} = {{var "v"}}[{{var "mk"}}] {{end}}
+	} {{if not decElemKindImmutable}}else { {{var "mv"}} = {{decElemZero}} }{{end}}
+	z.DecSendContainerState(codecSelfer_containerMapValue{{ .Sfx }})
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ decLineVar $x }}
-	if {{var "v"}} != nil {
+	if {{if decElemKindPtr}} {{var "ms"}} && {{end}} {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
 } else if {{var "l"}} < 0  {
 for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
-	var {{var "mk"}} {{ .KTyp }} 
+	z.DecSendContainerState(codecSelfer_containerMapKey{{ .Sfx }})
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x }}
-{{ if eq .KTyp "interface{}" }}// special case if a byte array.
-	if {{var "bv"}}, {{var "bok"}} := {{var "mk"}}.([]byte); {{var "bok"}} {
+{{ if eq .KTyp "interface{}" }}{{/* // special case if a byte array. */}}if {{var "bv"}}, {{var "bok"}} := {{var "mk"}}.([]byte); {{var "bok"}} {
 		{{var "mk"}} = string({{var "bv"}})
-	}
-{{ end }}
-	{{var "mv"}} := {{var "v"}}[{{var "mk"}}]
+	}{{ end }}{{if decElemKindPtr}}
+	{{var "ms"}} = true {{ end }}
+	if {{var "mg"}} {
+		{{if decElemKindPtr}}{{var "mv"}}, {{var "mok"}} = {{var "v"}}[{{var "mk"}}] 
+		if {{var "mok"}} {
+			{{var "ms"}} = false
+		} {{else}}{{var "mv"}} = {{var "v"}}[{{var "mk"}}] {{end}}
+	} {{if not decElemKindImmutable}}else { {{var "mv"}} = {{decElemZero}} }{{end}}
+	z.DecSendContainerState(codecSelfer_containerMapValue{{ .Sfx }})
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ decLineVar $x }}
-	if {{var "v"}} != nil {
+	if {{if decElemKindPtr}} {{var "ms"}} && {{end}} {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
-r.ReadEnd()
 } // else len==0: TODO: Should we clear map entries?
+z.DecSendContainerState(codecSelfer_containerMapEnd{{ .Sfx }})
 `
 
 const genDecListTmpl = `
-{{var "v"}} := {{ if not isArray}}*{{ end }}{{ .Varname }}
+{{var "v"}} := {{if not isArray}}*{{end}}{{ .Varname }}
 {{var "h"}}, {{var "l"}} := z.DecSliceHelperStart() {{/* // helper, containerLenS */}}
-
-var {{var "rr"}}, {{var "rl"}} int {{/* // num2read, length of slice/array/chan */}}
-var {{var "c"}}, {{var "rt"}} bool {{/* // changed, truncated */}}
-_, _, _ = {{var "c"}}, {{var "rt"}}, {{var "rl"}}
-{{var "rr"}} = {{var "l"}}
-{{/* rl is NOT used. Only used for getting DecInferLen. len(r) used directly in code */}}
-
-{{ if not isArray }}if {{var "v"}} == nil {
-	if {{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }}); {{var "rt"}} {
-		{{var "rr"}} = {{var "rl"}}
-	}
-	{{var "v"}} = make({{ .CTyp }}, {{var "rl"}})
-	{{var "c"}} = true 
-} 
-{{ end }}
-if {{var "l"}} == 0 { {{ if isSlice }}
-	if len({{var "v"}}) != 0 { 
-		{{var "v"}} = {{var "v"}}[:0] 
-		{{var "c"}} = true 
-	} {{ end }}
+var {{var "c"}} bool {{/* // changed */}}
+if {{var "l"}} == 0 {
+	{{if isSlice }}if {{var "v"}} == nil {
+		{{var "v"}} = []{{ .Typ }}{}
+		{{var "c"}} = true
+	} else if len({{var "v"}}) != 0 {
+		{{var "v"}} = {{var "v"}}[:0]
+		{{var "c"}} = true
+	} {{end}} {{if isChan }}if {{var "v"}} == nil {
+		{{var "v"}} = make({{ .CTyp }}, 0)
+		{{var "c"}} = true
+	} {{end}}
 } else if {{var "l"}} > 0 {
-	{{ if isChan }}
+	{{if isChan }}if {{var "v"}} == nil {
+		{{var "rl"}}, _ = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+		{{var "v"}} = make({{ .CTyp }}, {{var "rl"}})
+		{{var "c"}} = true
+	}
 	for {{var "r"}} := 0; {{var "r"}} < {{var "l"}}; {{var "r"}}++ {
+		{{var "h"}}.ElemContainerState({{var "r"}})
 		var {{var "t"}} {{ .Typ }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}
-		{{var "v"}} <- {{var "t"}} 
-	{{ else }} 
+		{{var "v"}} <- {{var "t"}}
+	}
+	{{ else }}	var {{var "rr"}}, {{var "rl"}} int {{/* // num2read, length of slice/array/chan */}}
+	var {{var "rt"}} bool {{/* truncated */}}
 	if {{var "l"}} > cap({{var "v"}}) {
-		{{ if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "l"}})
-		{{ else }}{{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
-		{{ if .Immutable }}
-		{{var "v2"}} := {{var "v"}}
-		{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
-		if len({{var "v"}}) > 0 {
-			copy({{var "v"}}, {{var "v2"}}[:cap({{var "v2"}})])
+		{{if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "l"}})
+		{{ else }}{{if not .Immutable }}
+		{{var "rg"}} := len({{var "v"}}) > 0
+		{{var "v2"}} := {{var "v"}} {{end}}
+		{{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+		if {{var "rt"}} {
+			if {{var "rl"}} <= cap({{var "v"}}) {
+				{{var "v"}} = {{var "v"}}[:{{var "rl"}}]
+			} else {
+				{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
+			}
+		} else {
+			{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
 		}
-		{{ else }}{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
-		{{ end }}{{var "c"}} = true 
-		{{ end }}
-		{{var "rr"}} = len({{var "v"}})
-	} else if {{var "l"}} != len({{var "v"}}) {
-		{{ if isSlice }}{{var "v"}} = {{var "v"}}[:{{var "l"}}]
-		{{var "c"}} = true {{ end }}
-	}
+		{{var "c"}} = true
+		{{var "rr"}} = len({{var "v"}}) {{if not .Immutable }}
+			if {{var "rg"}} { copy({{var "v"}}, {{var "v2"}}) } {{end}} {{end}}{{/* end not Immutable, isArray */}}
+	} {{if isSlice }} else if {{var "l"}} != len({{var "v"}}) {
+		{{var "v"}} = {{var "v"}}[:{{var "l"}}]
+		{{var "c"}} = true
+	} {{end}}	{{/* end isSlice:47 */}} 
 	{{var "j"}} := 0
 	for ; {{var "j"}} < {{var "rr"}} ; {{var "j"}}++ {
+		{{var "h"}}.ElemContainerState({{var "j"}})
 		{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
 	}
-	{{ if isArray }}for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
+	{{if isArray }}for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
+		{{var "h"}}.ElemContainerState({{var "j"}})
 		z.DecSwallow()
 	}
-	{{ else }}if {{var "rt"}} { {{/* means that it is mutable and slice */}}
+	{{ else }}if {{var "rt"}} {
 		for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
 			{{var "v"}} = append({{var "v"}}, {{ zero}})
+			{{var "h"}}.ElemContainerState({{var "j"}})
 			{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
 		}
-	}
-	{{ end }}
-	{{ end }}{{/* closing 'if not chan' */}}
-} else {
-	for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
-		if {{var "j"}} >= len({{var "v"}}) {
-			{{ if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "j"}}+1)
-			{{ else if isSlice}}{{var "v"}} = append({{var "v"}}, {{zero}})// var {{var "z"}} {{ .Typ }}
-			{{var "c"}} = true {{ end }}
-		}
-		{{ if isChan}}
+	} {{end}} {{/* end isArray:56 */}}
+	{{end}} {{/* end isChan:16 */}}
+} else { {{/* len < 0 */}}
+	{{var "j"}} := 0
+	for ; !r.CheckBreak(); {{var "j"}}++ {
+		{{if isChan }}
+		{{var "h"}}.ElemContainerState({{var "j"}})
 		var {{var "t"}} {{ .Typ }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}
 		{{var "v"}} <- {{var "t"}} 
 		{{ else }}
+		if {{var "j"}} >= len({{var "v"}}) {
+			{{if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "j"}}+1)
+			{{ else }}{{var "v"}} = append({{var "v"}}, {{zero}})// var {{var "z"}} {{ .Typ }}
+			{{var "c"}} = true {{end}}
+		}
+		{{var "h"}}.ElemContainerState({{var "j"}})
 		if {{var "j"}} < len({{var "v"}}) {
 			{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
 		} else {
 			z.DecSwallow()
 		}
-		{{ end }}
+		{{end}}
 	}
-	{{var "h"}}.End()
+	{{if isSlice }}if {{var "j"}} < len({{var "v"}}) {
+		{{var "v"}} = {{var "v"}}[:{{var "j"}}]
+		{{var "c"}} = true
+	} else if {{var "j"}} == 0 && {{var "v"}} == nil {
+		{{var "v"}} = []{{ .Typ }}{}
+		{{var "c"}} = true
+	}{{end}}
 }
-{{ if not isArray }}if {{var "c"}} { 
+{{var "h"}}.End()
+{{if not isArray }}if {{var "c"}} { 
 	*{{ .Varname }} = {{var "v"}}
-}{{ end }}
-
+}{{end}}
 `
 

Godeps/_workspace/src/github.com/ugorji/go/codec/gen.go

@@ -91,7 +91,8 @@ import (
 // v3: Changes for Kubernetes:
 //     changes in signature of some unpublished helper methods and codecgen cmdline arguments.
 // v4: Removed separator support from (en|de)cDriver, and refactored codec(gen)
-const GenVersion = 4
+// v5: changes to support faster json decoding. Let encoder/decoder maintain state of collections.
+const GenVersion = 5
 
 const (
 	genCodecPkg        = "codec1978"
@@ -110,6 +111,14 @@ const (
 	genUseOneFunctionForDecStructMap = true
 )
 
+type genStructMapStyle uint8
+
+const (
+	genStructMapStyleConsolidated genStructMapStyle = iota
+	genStructMapStyleLenPrefix
+	genStructMapStyleCheckBreak
+)
+
 var (
 	genAllTypesSamePkgErr  = errors.New("All types must be in the same package")
 	genExpectArrayOrMapErr = errors.New("unexpected type. Expecting array/map/slice")
@@ -144,6 +153,7 @@ type genRunner struct {
 	xs string // top level variable/constant suffix
 	hn string // fn helper type name
 
+	ti *TypeInfos
 	// rr *rand.Rand // random generator for file-specific types
 }
 
@@ -151,7 +161,7 @@ type genRunner struct {
 // type passed. All the types must be in the same package.
 //
 // Library users: *DO NOT USE IT DIRECTLY. IT WILL CHANGE CONTINOUSLY WITHOUT NOTICE.*
-func Gen(w io.Writer, buildTags, pkgName, uid string, useUnsafe bool, typ ...reflect.Type) {
+func Gen(w io.Writer, buildTags, pkgName, uid string, useUnsafe bool, ti *TypeInfos, typ ...reflect.Type) {
 	if len(typ) == 0 {
 		return
 	}
@@ -168,6 +178,10 @@ func Gen(w io.Writer, buildTags, pkgName, uid string, useUnsafe bool, typ ...ref
 		ts:     []reflect.Type{},
 		bp:     genImportPath(typ[0]),
 		xs:     uid,
+		ti:     ti,
+	}
+	if x.ti == nil {
+		x.ti = defTypeInfos
 	}
 	if x.xs == "" {
 		rr := rand.New(rand.NewSource(time.Now().UnixNano()))
@@ -225,10 +239,18 @@ func Gen(w io.Writer, buildTags, pkgName, uid string, useUnsafe bool, typ ...ref
 	x.line("")
 
 	x.line("const (")
+	x.linef("// ----- content types ----")
 	x.linef("codecSelferC_UTF8%s = %v", x.xs, int64(c_UTF8))
 	x.linef("codecSelferC_RAW%s = %v", x.xs, int64(c_RAW))
+	x.linef("// ----- value types used ----")
 	x.linef("codecSelferValueTypeArray%s = %v", x.xs, int64(valueTypeArray))
 	x.linef("codecSelferValueTypeMap%s = %v", x.xs, int64(valueTypeMap))
+	x.linef("// ----- containerStateValues ----")
+	x.linef("codecSelfer_containerMapKey%s = %v", x.xs, int64(containerMapKey))
+	x.linef("codecSelfer_containerMapValue%s = %v", x.xs, int64(containerMapValue))
+	x.linef("codecSelfer_containerMapEnd%s = %v", x.xs, int64(containerMapEnd))
+	x.linef("codecSelfer_containerArrayElem%s = %v", x.xs, int64(containerArrayElem))
+	x.linef("codecSelfer_containerArrayEnd%s = %v", x.xs, int64(containerArrayEnd))
 	x.line(")")
 	x.line("var (")
 	x.line("codecSelferBitsize" + x.xs + " = uint8(reflect.TypeOf(uint(0)).Bits())")
@@ -250,8 +272,6 @@ func Gen(w io.Writer, buildTags, pkgName, uid string, useUnsafe bool, typ ...ref
 	x.line(`err := fmt.Errorf("codecgen version mismatch: current: %v, need %v. Re-generate file: %v", `)
 	x.linef(`%v, %sGenVersion, file)`, GenVersion, x.cpfx)
 	x.line("panic(err)")
-	// x.linef(`panic(fmt.Errorf("Re-run codecgen due to version mismatch: `+
-	// 	`current: %%v, need %%v, file: %%v", %v, %sGenVersion, file))`, GenVersion, x.cpfx)
 	x.linef("}")
 	x.line("if false { // reference the types, but skip this branch at build/run time")
 	var n int
@@ -510,21 +530,21 @@ func (x *genRunner) selfer(encode bool) {
 		x.out(fnSigPfx)
 		x.line(") codecDecodeSelfFromMap(l int, d *" + x.cpfx + "Decoder) {")
 		x.genRequiredMethodVars(false)
-		x.decStructMap(genTopLevelVarName, "l", reflect.ValueOf(t0).Pointer(), t0, 0)
+		x.decStructMap(genTopLevelVarName, "l", reflect.ValueOf(t0).Pointer(), t0, genStructMapStyleConsolidated)
 		x.line("}")
 		x.line("")
 	} else {
 		x.out(fnSigPfx)
 		x.line(") codecDecodeSelfFromMapLenPrefix(l int, d *" + x.cpfx + "Decoder) {")
 		x.genRequiredMethodVars(false)
-		x.decStructMap(genTopLevelVarName, "l", reflect.ValueOf(t0).Pointer(), t0, 1)
+		x.decStructMap(genTopLevelVarName, "l", reflect.ValueOf(t0).Pointer(), t0, genStructMapStyleLenPrefix)
 		x.line("}")
 		x.line("")
 
 		x.out(fnSigPfx)
 		x.line(") codecDecodeSelfFromMapCheckBreak(l int, d *" + x.cpfx + "Decoder) {")
 		x.genRequiredMethodVars(false)
-		x.decStructMap(genTopLevelVarName, "l", reflect.ValueOf(t0).Pointer(), t0, 2)
+		x.decStructMap(genTopLevelVarName, "l", reflect.ValueOf(t0).Pointer(), t0, genStructMapStyleCheckBreak)
 		x.line("}")
 		x.line("")
 	}
@@ -543,10 +563,8 @@ func (x *genRunner) selfer(encode bool) {
 func (x *genRunner) xtraSM(varname string, encode bool, t reflect.Type) {
 	if encode {
 		x.linef("h.enc%s((%s%s)(%s), e)", x.genMethodNameT(t), x.arr2str(t, "*"), x.genTypeName(t), varname)
-		// x.line("h.enc" + x.genMethodNameT(t) + "(" + x.genTypeName(t) + "(" + varname + "), e)")
 	} else {
 		x.linef("h.dec%s((*%s)(%s), d)", x.genMethodNameT(t), x.genTypeName(t), varname)
-		// x.line("h.dec" + x.genMethodNameT(t) + "((*" + x.genTypeName(t) + ")(" + varname + "), d)")
 	}
 	if _, ok := x.tm[t]; !ok {
 		x.tm[t] = struct{}{}
@@ -756,7 +774,7 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 	// replicate code in kStruct i.e. for each field, deref type to non-pointer, and call x.enc on it
 
 	// if t === type currently running selfer on, do for all
-	ti := getTypeInfo(rtid, t)
+	ti := x.ti.get(rtid, t)
 	i := x.varsfx()
 	sepVarname := genTempVarPfx + "sep" + i
 	numfieldsvar := genTempVarPfx + "q" + i
@@ -810,12 +828,14 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 		}
 		x.linef("%s[%v] = %s", numfieldsvar, j, omitline)
 	}
+	x.linef("var %snn%s int", genTempVarPfx, i)
 	x.linef("if %s || %s {", ti2arrayvar, struct2arrvar) // if ti.toArray {
 	x.line("r.EncodeArrayStart(" + strconv.FormatInt(int64(len(tisfi)), 10) + ")")
 	x.linef("} else {") // if not ti.toArray
-	x.linef("var %snn%s int = %v", genTempVarPfx, i, nn)
+	x.linef("%snn%s = %v", genTempVarPfx, i, nn)
 	x.linef("for _, b := range %s { if b { %snn%s++ } }", numfieldsvar, genTempVarPfx, i)
 	x.linef("r.EncodeMapStart(%snn%s)", genTempVarPfx, i)
+	x.linef("%snn%s = %v", genTempVarPfx, i, 0)
 	// x.line("r.EncodeMapStart(" + strconv.FormatInt(int64(len(tisfi)), 10) + ")")
 	x.line("}") // close if not StructToArray
 
@@ -859,11 +879,9 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 		if labelUsed {
 			x.line("if " + isNilVarName + " { r.EncodeNil() } else { ")
 		}
+		x.linef("z.EncSendContainerState(codecSelfer_containerArrayElem%s)", x.xs)
 		if si.omitEmpty {
 			x.linef("if %s[%v] {", numfieldsvar, j)
-			// omitEmptyVarNameX := genTempVarPfx + "ov" + i
-			// x.line("var " + omitEmptyVarNameX + " " + x.genTypeName(t2.Type))
-			// x.encVar(omitEmptyVarNameX, t2.Type)
 		}
 		x.encVar(varname+"."+t2.Name, t2.Type)
 		if si.omitEmpty {
@@ -874,21 +892,15 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 		if labelUsed {
 			x.line("}")
 		}
+
 		x.linef("} else {") // if not ti.toArray
-		// omitEmptyVar := genTempVarPfx + "x" + i + t2.Name
-		// x.line("const " + omitEmptyVar + " bool = " + strconv.FormatBool(si.omitEmpty))
-		// doOmitEmpty := si.omitEmpty && t2.Type.Kind() != reflect.Struct
+
 		if si.omitEmpty {
 			x.linef("if %s[%v] {", numfieldsvar, j)
-			// x.linef(`println("Encoding field: %v")`, j)
-			// x.out("if ")
-			// if labelUsed {
-			// 	x.out("!" + isNilVarName + " && ")
-			// }
-			// x.line(varname + "." + t2.Name + " != " + genZeroValueR(t2.Type, x.tc) + " {")
 		}
-		// x.line("r.EncodeString(codecSelferC_UTF8" + x.xs + ", string(\"" + t2.Name + "\"))")
+		x.linef("z.EncSendContainerState(codecSelfer_containerMapKey%s)", x.xs)
 		x.line("r.EncodeString(codecSelferC_UTF8" + x.xs + ", string(\"" + si.encName + "\"))")
+		x.linef("z.EncSendContainerState(codecSelfer_containerMapValue%s)", x.xs)
 		if labelUsed {
 			x.line("if " + isNilVarName + " { r.EncodeNil() } else { ")
 			x.encVar(varname+"."+t2.Name, t2.Type)
@@ -901,9 +913,12 @@ func (x *genRunner) encStruct(varname string, rtid uintptr, t reflect.Type) {
 		}
 		x.linef("} ") // end if/else ti.toArray
 	}
-	x.line("if " + sepVarname + " {")
-	x.line("r.EncodeEnd()")
+	x.linef("if %s || %s {", ti2arrayvar, struct2arrvar) // if ti.toArray {
+	x.linef("z.EncSendContainerState(codecSelfer_containerArrayEnd%s)", x.xs)
+	x.line("} else {")
+	x.linef("z.EncSendContainerState(codecSelfer_containerMapEnd%s)", x.xs)
 	x.line("}")
+
 }
 
 func (x *genRunner) encListFallback(varname string, t reflect.Type) {
@@ -912,25 +927,30 @@ func (x *genRunner) encListFallback(varname string, t reflect.Type) {
 	x.line("r.EncodeArrayStart(len(" + varname + "))")
 	if t.Kind() == reflect.Chan {
 		x.linef("for %si%s, %si2%s := 0, len(%s); %si%s < %si2%s; %si%s++ {", g, i, g, i, varname, g, i, g, i, g, i)
+		x.linef("z.EncSendContainerState(codecSelfer_containerArrayElem%s)", x.xs)
 		x.linef("%sv%s := <-%s", g, i, varname)
 	} else {
 		// x.linef("for %si%s, %sv%s := range %s {", genTempVarPfx, i, genTempVarPfx, i, varname)
 		x.linef("for _, %sv%s := range %s {", genTempVarPfx, i, varname)
+		x.linef("z.EncSendContainerState(codecSelfer_containerArrayElem%s)", x.xs)
 	}
 	x.encVar(genTempVarPfx+"v"+i, t.Elem())
 	x.line("}")
-	x.line("r.EncodeEnd()")
+	x.linef("z.EncSendContainerState(codecSelfer_containerArrayEnd%s)", x.xs)
 }
 
 func (x *genRunner) encMapFallback(varname string, t reflect.Type) {
+	// TODO: expand this to handle canonical.
 	i := x.varsfx()
 	x.line("r.EncodeMapStart(len(" + varname + "))")
 	x.linef("for %sk%s, %sv%s := range %s {", genTempVarPfx, i, genTempVarPfx, i, varname)
 	// x.line("for " + genTempVarPfx + "k" + i + ", " + genTempVarPfx + "v" + i + " := range " + varname + " {")
+	x.linef("z.EncSendContainerState(codecSelfer_containerMapKey%s)", x.xs)
 	x.encVar(genTempVarPfx+"k"+i, t.Key())
+	x.linef("z.EncSendContainerState(codecSelfer_containerMapValue%s)", x.xs)
 	x.encVar(genTempVarPfx+"v"+i, t.Elem())
 	x.line("}")
-	x.line("r.EncodeEnd()")
+	x.linef("z.EncSendContainerState(codecSelfer_containerMapEnd%s)", x.xs)
 }
 
 func (x *genRunner) decVar(varname string, t reflect.Type, canBeNil bool) {
@@ -948,8 +968,6 @@ func (x *genRunner) decVar(varname string, t reflect.Type, canBeNil bool) {
 		x.line("if r.TryDecodeAsNil() {")
 		if t.Kind() == reflect.Ptr {
 			x.line("if " + varname + " != nil { ")
-			// x.line("var " + genTempVarPfx + i + " " + x.genTypeName(t.Elem()))
-			// x.line("*" + varname + " = " + genTempVarPfx + i)
 
 			// if varname is a field of a struct (has a dot in it),
 			// then just set it to nil
@@ -958,12 +976,8 @@ func (x *genRunner) decVar(varname string, t reflect.Type, canBeNil bool) {
 			} else {
 				x.line("*" + varname + " = " + x.genZeroValueR(t.Elem()))
 			}
-			// x.line("*" + varname + " = nil")
 			x.line("}")
-
 		} else {
-			// x.line("var " + genTempVarPfx + i + " " + x.genTypeName(t))
-			// x.line(varname + " = " + genTempVarPfx + i)
 			x.line(varname + " = " + x.genZeroValueR(t))
 		}
 		x.line("} else {")
@@ -1143,8 +1157,6 @@ func (x *genRunner) dec(varname string, t reflect.Type) {
 		} else if fastpathAV.index(rtid) != -1 {
 			g := x.newGenV(t)
 			x.line("z.F." + g.MethodNamePfx("Dec", false) + "X(" + varname + ", false, d)")
-			// x.line("z." + g.MethodNamePfx("Dec", false) + "(" + varname + ")")
-			// x.line(g.FastpathName(false) + "(" + varname + ", d)")
 		} else {
 			x.xtraSM(varname, false, t)
 			// x.decListFallback(varname, rtid, false, t)
@@ -1157,8 +1169,6 @@ func (x *genRunner) dec(varname string, t reflect.Type) {
 		if fastpathAV.index(rtid) != -1 {
 			g := x.newGenV(t)
 			x.line("z.F." + g.MethodNamePfx("Dec", false) + "X(" + varname + ", false, d)")
-			// x.line("z." + g.MethodNamePfx("Dec", false) + "(" + varname + ")")
-			// x.line(g.FastpathName(false) + "(" + varname + ", d)")
 		} else {
 			x.xtraSM(varname, false, t)
 			// x.decMapFallback(varname, rtid, t)
@@ -1288,6 +1298,7 @@ func (x *genRunner) decListFallback(varname string, rtid uintptr, t reflect.Type
 func (x *genRunner) decMapFallback(varname string, rtid uintptr, t reflect.Type) {
 	type tstruc struct {
 		TempVar string
+		Sfx     string
 		Rand    string
 		Varname string
 		KTyp    string
@@ -1296,8 +1307,24 @@ func (x *genRunner) decMapFallback(varname string, rtid uintptr, t reflect.Type)
 	}
 	telem := t.Elem()
 	tkey := t.Key()
-	ts := tstruc{genTempVarPfx, x.varsfx(), varname, x.genTypeName(tkey), x.genTypeName(telem), int(telem.Size() + tkey.Size())}
+	ts := tstruc{
+		genTempVarPfx, x.xs, x.varsfx(), varname, x.genTypeName(tkey),
+		x.genTypeName(telem), int(telem.Size() + tkey.Size()),
+	}
+
 	funcs := make(template.FuncMap)
+	funcs["decElemZero"] = func() string {
+		return x.genZeroValueR(telem)
+	}
+	funcs["decElemKindImmutable"] = func() bool {
+		return genIsImmutable(telem)
+	}
+	funcs["decElemKindPtr"] = func() bool {
+		return telem.Kind() == reflect.Ptr
+	}
+	funcs["decElemKindIntf"] = func() bool {
+		return telem.Kind() == reflect.Interface
+	}
 	funcs["decLineVarK"] = func(varname string) string {
 		x.decVar(varname, tkey, false)
 		return ""
@@ -1328,7 +1355,7 @@ func (x *genRunner) decMapFallback(varname string, rtid uintptr, t reflect.Type)
 }
 
 func (x *genRunner) decStructMapSwitch(kName string, varname string, rtid uintptr, t reflect.Type) {
-	ti := getTypeInfo(rtid, t)
+	ti := x.ti.get(rtid, t)
 	tisfi := ti.sfip // always use sequence from file. decStruct expects same thing.
 	x.line("switch (" + kName + ") {")
 	for _, si := range tisfi {
@@ -1337,6 +1364,7 @@ func (x *genRunner) decStructMapSwitch(kName string, varname string, rtid uintpt
 		if si.i != -1 {
 			t2 = t.Field(int(si.i))
 		} else {
+			//we must accomodate anonymous fields, where the embedded field is a nil pointer in the value.
 			// t2 = t.FieldByIndex(si.is)
 			t2typ := t
 			varname3 := varname
@@ -1348,8 +1376,7 @@ func (x *genRunner) decStructMapSwitch(kName string, varname string, rtid uintpt
 				t2typ = t2.Type
 				varname3 = varname3 + "." + t2.Name
 				if t2typ.Kind() == reflect.Ptr {
-					x.line("if " + varname3 + " == nil {" +
-						varname3 + " = new(" + x.genTypeName(t2typ.Elem()) + ") }")
+					x.linef("if %s == nil { %s = new(%s) }", varname3, varname3, x.genTypeName(t2typ.Elem()))
 				}
 			}
 		}
@@ -1358,11 +1385,10 @@ func (x *genRunner) decStructMapSwitch(kName string, varname string, rtid uintpt
 	x.line("default:")
 	// pass the slice here, so that the string will not escape, and maybe save allocation
 	x.line("z.DecStructFieldNotFound(-1, " + kName + ")")
-	// x.line("z.DecStructFieldNotFoundB(" + kName + "Slc)")
 	x.line("} // end switch " + kName)
 }
 
-func (x *genRunner) decStructMap(varname, lenvarname string, rtid uintptr, t reflect.Type, style uint8) {
+func (x *genRunner) decStructMap(varname, lenvarname string, rtid uintptr, t reflect.Type, style genStructMapStyle) {
 	tpfx := genTempVarPfx
 	i := x.varsfx()
 	kName := tpfx + "s" + i
@@ -1384,14 +1410,11 @@ func (x *genRunner) decStructMap(varname, lenvarname string, rtid uintptr, t ref
 
 	x.line("var " + kName + "Slc = z.DecScratchBuffer() // default slice to decode into")
 
-	// x.line("var " + kName + " string // default string to decode into")
-	// x.line("_ = " + kName)
 	x.line("_ = " + kName + "Slc")
-	// x.linef("var %sb%s bool", tpfx, i)                        // break
 	switch style {
-	case 1:
+	case genStructMapStyleLenPrefix:
 		x.linef("for %sj%s := 0; %sj%s < %s; %sj%s++ {", tpfx, i, tpfx, i, lenvarname, tpfx, i)
-	case 2:
+	case genStructMapStyleCheckBreak:
 		x.linef("for %sj%s := 0; !r.CheckBreak(); %sj%s++ {", tpfx, i, tpfx, i)
 	default: // 0, otherwise.
 		x.linef("var %shl%s bool = %s >= 0", tpfx, i, lenvarname) // has length
@@ -1399,11 +1422,9 @@ func (x *genRunner) decStructMap(varname, lenvarname string, rtid uintptr, t ref
 		x.linef("if %shl%s { if %sj%s >= %s { break }", tpfx, i, tpfx, i, lenvarname)
 		x.line("} else { if r.CheckBreak() { break }; }")
 	}
-	// x.line(kName + " = z.ReadStringAsBytes(" + kName + ")")
-	// x.line(kName + " = z.ReadString()")
+	x.linef("z.DecSendContainerState(codecSelfer_containerMapKey%s)", x.xs)
 	x.line(kName + "Slc = r.DecodeBytes(" + kName + "Slc, true, true)")
 	// let string be scoped to this loop alone, so it doesn't escape.
-	// x.line(kName + " := " + x.cpfx + "GenBytesToStringRO(" + kName + "Slc)")
 	if x.unsafe {
 		x.line(kName + "SlcHdr := codecSelferUnsafeString" + x.xs + "{uintptr(unsafe.Pointer(&" +
 			kName + "Slc[0])), len(" + kName + "Slc)}")
@@ -1411,43 +1432,50 @@ func (x *genRunner) decStructMap(varname, lenvarname string, rtid uintptr, t ref
 	} else {
 		x.line(kName + " := string(" + kName + "Slc)")
 	}
+	x.linef("z.DecSendContainerState(codecSelfer_containerMapValue%s)", x.xs)
 	x.decStructMapSwitch(kName, varname, rtid, t)
 
 	x.line("} // end for " + tpfx + "j" + i)
-	switch style {
-	case 1:
-	case 2:
-		x.line("r.ReadEnd()")
-	default:
-		x.linef("if !%shl%s { r.ReadEnd() }", tpfx, i)
-	}
+	x.linef("z.DecSendContainerState(codecSelfer_containerMapEnd%s)", x.xs)
 }
 
 func (x *genRunner) decStructArray(varname, lenvarname, breakString string, rtid uintptr, t reflect.Type) {
 	tpfx := genTempVarPfx
 	i := x.varsfx()
-	ti := getTypeInfo(rtid, t)
+	ti := x.ti.get(rtid, t)
 	tisfi := ti.sfip // always use sequence from file. decStruct expects same thing.
 	x.linef("var %sj%s int", tpfx, i)
-	x.linef("var %sb%s bool", tpfx, i) // break
-	// x.linef("var %sl%s := r.ReadArrayStart()", tpfx, i)
+	x.linef("var %sb%s bool", tpfx, i)                        // break
 	x.linef("var %shl%s bool = %s >= 0", tpfx, i, lenvarname) // has length
 	for _, si := range tisfi {
 		var t2 reflect.StructField
 		if si.i != -1 {
 			t2 = t.Field(int(si.i))
 		} else {
-			t2 = t.FieldByIndex(si.is)
+			//we must accomodate anonymous fields, where the embedded field is a nil pointer in the value.
+			// t2 = t.FieldByIndex(si.is)
+			t2typ := t
+			varname3 := varname
+			for _, ix := range si.is {
+				for t2typ.Kind() == reflect.Ptr {
+					t2typ = t2typ.Elem()
+				}
+				t2 = t2typ.Field(ix)
+				t2typ = t2.Type
+				varname3 = varname3 + "." + t2.Name
+				if t2typ.Kind() == reflect.Ptr {
+					x.linef("if %s == nil { %s = new(%s) }", varname3, varname3, x.genTypeName(t2typ.Elem()))
+				}
+			}
 		}
 
 		x.linef("%sj%s++; if %shl%s { %sb%s = %sj%s > %s } else { %sb%s = r.CheckBreak() }",
 			tpfx, i, tpfx, i, tpfx, i,
 			tpfx, i, lenvarname, tpfx, i)
-		// x.line("if " + tpfx + "j" + i + "++; " + tpfx + "j" +
-		// i + " <=  " + tpfx + "l" + i + " {")
-		x.linef("if %sb%s { r.ReadEnd(); %s }", tpfx, i, breakString)
+		x.linef("if %sb%s { z.DecSendContainerState(codecSelfer_containerArrayEnd%s); %s }",
+			tpfx, i, x.xs, breakString)
+		x.linef("z.DecSendContainerState(codecSelfer_containerArrayElem%s)", x.xs)
 		x.decVar(varname+"."+t2.Name, t2.Type, true)
-		// x.line("} // end if " + tpfx + "j" + i + " <=  " + tpfx + "l" + i)
 	}
 	// read remaining values and throw away.
 	x.line("for {")
@@ -1455,19 +1483,20 @@ func (x *genRunner) decStructArray(varname, lenvarname, breakString string, rtid
 		tpfx, i, tpfx, i, tpfx, i,
 		tpfx, i, lenvarname, tpfx, i)
 	x.linef("if %sb%s { break }", tpfx, i)
+	x.linef("z.DecSendContainerState(codecSelfer_containerArrayElem%s)", x.xs)
 	x.linef(`z.DecStructFieldNotFound(%sj%s - 1, "")`, tpfx, i)
 	x.line("}")
-	x.line("r.ReadEnd()")
+	x.linef("z.DecSendContainerState(codecSelfer_containerArrayEnd%s)", x.xs)
 }
 
 func (x *genRunner) decStruct(varname string, rtid uintptr, t reflect.Type) {
 	// if container is map
-	// x.line("if z.DecContainerIsMap() { ")
 	i := x.varsfx()
-	x.line("if r.IsContainerType(codecSelferValueTypeMap" + x.xs + ") {")
+	x.linef("%sct%s := r.ContainerType()", genTempVarPfx, i)
+	x.linef("if %sct%s == codecSelferValueTypeMap%s {", genTempVarPfx, i, x.xs)
 	x.line(genTempVarPfx + "l" + i + " := r.ReadMapStart()")
 	x.linef("if %sl%s == 0 {", genTempVarPfx, i)
-	x.line("r.ReadEnd()")
+	x.linef("z.DecSendContainerState(codecSelfer_containerMapEnd%s)", x.xs)
 	if genUseOneFunctionForDecStructMap {
 		x.line("} else { ")
 		x.linef("x.codecDecodeSelfFromMap(%sl%s, d)", genTempVarPfx, i)
@@ -1480,18 +1509,16 @@ func (x *genRunner) decStruct(varname string, rtid uintptr, t reflect.Type) {
 	x.line("}")
 
 	// else if container is array
-	// x.line("} else if z.DecContainerIsArray() { ")
-	x.line("} else if r.IsContainerType(codecSelferValueTypeArray" + x.xs + ") {")
+	x.linef("} else if %sct%s == codecSelferValueTypeArray%s {", genTempVarPfx, i, x.xs)
 	x.line(genTempVarPfx + "l" + i + " := r.ReadArrayStart()")
 	x.linef("if %sl%s == 0 {", genTempVarPfx, i)
-	x.line("r.ReadEnd()")
+	x.linef("z.DecSendContainerState(codecSelfer_containerArrayEnd%s)", x.xs)
 	x.line("} else { ")
 	x.linef("x.codecDecodeSelfFromArray(%sl%s, d)", genTempVarPfx, i)
 	x.line("}")
 	// else panic
 	x.line("} else { ")
 	x.line("panic(codecSelferOnlyMapOrArrayEncodeToStructErr" + x.xs + ")")
-	// x.line("panic(`only encoded map or array can be decoded into a struct`)")
 	x.line("} ")
 }
 
@@ -1721,6 +1748,8 @@ func genInternalDecCommandAsString(s string) string {
 		return "uint32(dd.DecodeUint(32))"
 	case "uint64":
 		return "dd.DecodeUint(64)"
+	case "uintptr":
+		return "uintptr(dd.DecodeUint(uintBitsize))"
 	case "int":
 		return "int(dd.DecodeInt(intBitsize))"
 	case "int8":
@@ -1741,9 +1770,24 @@ func genInternalDecCommandAsString(s string) string {
 	case "bool":
 		return "dd.DecodeBool()"
 	default:
-		panic(errors.New("unknown type for decode: " + s))
+		panic(errors.New("gen internal: unknown type for decode: " + s))
 	}
+}
 
+func genInternalSortType(s string, elem bool) string {
+	for _, v := range [...]string{"int", "uint", "float", "bool", "string"} {
+		if strings.HasPrefix(s, v) {
+			if elem {
+				if v == "int" || v == "uint" || v == "float" {
+					return v + "64"
+				} else {
+					return v
+				}
+			}
+			return v + "Slice"
+		}
+	}
+	panic("sorttype: unexpected type: " + s)
 }
 
 // var genInternalMu sync.Mutex
@@ -1762,6 +1806,7 @@ func genInternalInit() {
 		"uint16",
 		"uint32",
 		"uint64",
+		"uintptr",
 		"int",
 		"int8",
 		"int16",
@@ -1779,6 +1824,7 @@ func genInternalInit() {
 		"uint16",
 		"uint32",
 		"uint64",
+		"uintptr",
 		"int",
 		"int8",
 		"int16",
@@ -1798,6 +1844,7 @@ func genInternalInit() {
 		"uint16":      2,
 		"uint32":      4,
 		"uint64":      8,
+		"uintptr":     1 * wordSizeBytes,
 		"int":         1 * wordSizeBytes,
 		"int8":        1,
 		"int16":       2,
@@ -1807,10 +1854,6 @@ func genInternalInit() {
 		"float64":     8,
 		"bool":        1,
 	}
-	// mapvaltypes2 := make(map[string]bool)
-	// for _, s := range mapvaltypes {
-	// 	mapvaltypes2[s] = true
-	// }
 	var gt genInternal
 
 	// For each slice or map type, there must be a (symetrical) Encode and Decode fast-path function
@@ -1832,16 +1875,18 @@ func genInternalInit() {
 	funcs["encmd"] = genInternalEncCommandAsString
 	funcs["decmd"] = genInternalDecCommandAsString
 	funcs["zerocmd"] = genInternalZeroValue
+	funcs["hasprefix"] = strings.HasPrefix
+	funcs["sorttype"] = genInternalSortType
 
 	genInternalV = gt
 	genInternalTmplFuncs = funcs
 }
 
-// GenInternalGoFile is used to generate source files from templates.
+// genInternalGoFile is used to generate source files from templates.
 // It is run by the program author alone.
 // Unfortunately, it has to be exported so that it can be called from a command line tool.
 // *** DO NOT USE ***
-func GenInternalGoFile(r io.Reader, w io.Writer, safe bool) (err error) {
+func genInternalGoFile(r io.Reader, w io.Writer, safe bool) (err error) {
 	genInternalOnce.Do(genInternalInit)
 
 	gt := genInternalV

Godeps/_workspace/src/github.com/ugorji/go/codec/helper.go

@@ -101,6 +101,7 @@ package codec
 // check for these error conditions.
 
 import (
+	"bytes"
 	"encoding"
 	"encoding/binary"
 	"errors"
@@ -111,8 +112,6 @@ import (
 	"strings"
 	"sync"
 	"time"
-	"unicode"
-	"unicode/utf8"
 )
 
 const (
@@ -193,14 +192,6 @@ const (
 
 type seqType uint8
 
-// mirror json.Marshaler and json.Unmarshaler here, so we don't import the encoding/json package
-type jsonMarshaler interface {
-	MarshalJSON() ([]byte, error)
-}
-type jsonUnmarshaler interface {
-	UnmarshalJSON([]byte) error
-}
-
 const (
 	_ seqType = iota
 	seqTypeArray
@@ -208,16 +199,43 @@ const (
 	seqTypeChan
 )
 
+// note that containerMapStart and containerArraySend are not sent.
+// This is because the ReadXXXStart and EncodeXXXStart already does these.
+type containerState uint8
+
+const (
+	_ containerState = iota
+
+	containerMapStart // slot left open, since Driver method already covers it
+	containerMapKey
+	containerMapValue
+	containerMapEnd
+	containerArrayStart // slot left open, since Driver methods already cover it
+	containerArrayElem
+	containerArrayEnd
+)
+
+type containerStateRecv interface {
+	sendContainerState(containerState)
+}
+
+// mirror json.Marshaler and json.Unmarshaler here,
+// so we don't import the encoding/json package
+type jsonMarshaler interface {
+	MarshalJSON() ([]byte, error)
+}
+type jsonUnmarshaler interface {
+	UnmarshalJSON([]byte) error
+}
+
 var (
 	bigen               = binary.BigEndian
 	structInfoFieldName = "_struct"
 
-	cachedTypeInfo      = make(map[uintptr]*typeInfo, 64)
-	cachedTypeInfoMutex sync.RWMutex
-
-	// mapStrIntfTyp = reflect.TypeOf(map[string]interface{}(nil))
-	intfSliceTyp = reflect.TypeOf([]interface{}(nil))
-	intfTyp      = intfSliceTyp.Elem()
+	mapStrIntfTyp  = reflect.TypeOf(map[string]interface{}(nil))
+	mapIntfIntfTyp = reflect.TypeOf(map[interface{}]interface{}(nil))
+	intfSliceTyp   = reflect.TypeOf([]interface{}(nil))
+	intfTyp        = intfSliceTyp.Elem()
 
 	stringTyp     = reflect.TypeOf("")
 	timeTyp       = reflect.TypeOf(time.Time{})
@@ -243,6 +261,9 @@ var (
 	timeTypId       = reflect.ValueOf(timeTyp).Pointer()
 	stringTypId     = reflect.ValueOf(stringTyp).Pointer()
 
+	mapStrIntfTypId  = reflect.ValueOf(mapStrIntfTyp).Pointer()
+	mapIntfIntfTypId = reflect.ValueOf(mapIntfIntfTyp).Pointer()
+	intfSliceTypId   = reflect.ValueOf(intfSliceTyp).Pointer()
 	// mapBySliceTypId  = reflect.ValueOf(mapBySliceTyp).Pointer()
 
 	intBitsize  uint8 = uint8(reflect.TypeOf(int(0)).Bits())
@@ -256,6 +277,8 @@ var (
 	noFieldNameToStructFieldInfoErr = errors.New("no field name passed to parseStructFieldInfo")
 )
 
+var defTypeInfos = NewTypeInfos([]string{"codec", "json"})
+
 // Selfer defines methods by which a value can encode or decode itself.
 //
 // Any type which implements Selfer will be able to encode or decode itself.
@@ -281,6 +304,11 @@ type MapBySlice interface {
 //
 // BasicHandle encapsulates the common options and extension functions.
 type BasicHandle struct {
+	// TypeInfos is used to get the type info for any type.
+	//
+	// If not configured, the default TypeInfos is used, which uses struct tag keys: codec, json
+	TypeInfos *TypeInfos
+
 	extHandle
 	EncodeOptions
 	DecodeOptions
@@ -290,6 +318,13 @@ func (x *BasicHandle) getBasicHandle() *BasicHandle {
 	return x
 }
 
+func (x *BasicHandle) getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
+	if x.TypeInfos != nil {
+		return x.TypeInfos.get(rtid, rt)
+	}
+	return defTypeInfos.get(rtid, rt)
+}
+
 // Handle is the interface for a specific encoding format.
 //
 // Typically, a Handle is pre-configured before first time use,
@@ -320,6 +355,8 @@ type RawExt struct {
 // It is used by codecs (e.g. binc, msgpack, simple) which do custom serialization of the types.
 type BytesExt interface {
 	// WriteExt converts a value to a []byte.
+	//
+	// Note: v *may* be a pointer to the extension type, if the extension type was a struct or array.
 	WriteExt(v interface{}) []byte
 
 	// ReadExt updates a value from a []byte.
@@ -332,6 +369,8 @@ type BytesExt interface {
 // It is used by codecs (e.g. cbor, json) which use the format to do custom serialization of the types.
 type InterfaceExt interface {
 	// ConvertExt converts a value into a simpler interface for easy encoding e.g. convert time.Time to int64.
+	//
+	// Note: v *may* be a pointer to the extension type, if the extension type was a struct or array.
 	ConvertExt(v interface{}) interface{}
 
 	// UpdateExt updates a value from a simpler interface for easy decoding e.g. convert int64 to time.Time.
@@ -351,7 +390,6 @@ type addExtWrapper struct {
 }
 
 func (x addExtWrapper) WriteExt(v interface{}) []byte {
-	// fmt.Printf(">>>>>>>>>> WriteExt: %T, %v\n", v, v)
 	bs, err := x.encFn(reflect.ValueOf(v))
 	if err != nil {
 		panic(err)
@@ -360,7 +398,6 @@ func (x addExtWrapper) WriteExt(v interface{}) []byte {
 }
 
 func (x addExtWrapper) ReadExt(v interface{}, bs []byte) {
-	// fmt.Printf(">>>>>>>>>> ReadExt: %T, %v\n", v, v)
 	if err := x.decFn(reflect.ValueOf(v), bs); err != nil {
 		panic(err)
 	}
@@ -462,7 +499,7 @@ type extTypeTagFn struct {
 	ext  Ext
 }
 
-type extHandle []*extTypeTagFn
+type extHandle []extTypeTagFn
 
 // DEPRECATED: Use SetBytesExt or SetInterfaceExt on the Handle instead.
 //
@@ -501,12 +538,17 @@ func (o *extHandle) SetExt(rt reflect.Type, tag uint64, ext Ext) (err error) {
 		}
 	}
 
-	*o = append(*o, &extTypeTagFn{rtid, rt, tag, ext})
+	if *o == nil {
+		*o = make([]extTypeTagFn, 0, 4)
+	}
+	*o = append(*o, extTypeTagFn{rtid, rt, tag, ext})
 	return
 }
 
 func (o extHandle) getExt(rtid uintptr) *extTypeTagFn {
-	for _, v := range o {
+	var v *extTypeTagFn
+	for i := range o {
+		v = &o[i]
 		if v.rtid == rtid {
 			return v
 		}
@@ -515,7 +557,9 @@ func (o extHandle) getExt(rtid uintptr) *extTypeTagFn {
 }
 
 func (o extHandle) getExtForTag(tag uint64) *extTypeTagFn {
-	for _, v := range o {
+	var v *extTypeTagFn
+	for i := range o {
+		v = &o[i]
 		if v.tag == tag {
 			return v
 		}
@@ -638,6 +682,8 @@ type typeInfo struct {
 	rt   reflect.Type
 	rtid uintptr
 
+	numMeth uint16 // number of methods
+
 	// baseId gives pointer to the base reflect.Type, after deferencing
 	// the pointers. E.g. base type of ***time.Time is time.Time.
 	base      reflect.Type
@@ -695,33 +741,49 @@ func (ti *typeInfo) indexForEncName(name string) int {
 	return -1
 }
 
-func getStructTag(t reflect.StructTag) (s string) {
+// TypeInfos caches typeInfo for each type on first inspection.
+//
+// It is configured with a set of tag keys, which are used to get
+// configuration for the type.
+type TypeInfos struct {
+	infos map[uintptr]*typeInfo
+	mu    sync.RWMutex
+	tags  []string
+}
+
+// NewTypeInfos creates a TypeInfos given a set of struct tags keys.
+//
+// This allows users customize the struct tag keys which contain configuration
+// of their types.
+func NewTypeInfos(tags []string) *TypeInfos {
+	return &TypeInfos{tags: tags, infos: make(map[uintptr]*typeInfo, 64)}
+}
+
+func (x *TypeInfos) structTag(t reflect.StructTag) (s string) {
 	// check for tags: codec, json, in that order.
 	// this allows seamless support for many configured structs.
-	s = t.Get("codec")
-	if s == "" {
-		s = t.Get("json")
+	for _, x := range x.tags {
+		s = t.Get(x)
+		if s != "" {
+			return s
+		}
 	}
 	return
 }
 
-func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
+func (x *TypeInfos) get(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
 	var ok bool
-	cachedTypeInfoMutex.RLock()
-	pti, ok = cachedTypeInfo[rtid]
-	cachedTypeInfoMutex.RUnlock()
+	x.mu.RLock()
+	pti, ok = x.infos[rtid]
+	x.mu.RUnlock()
 	if ok {
 		return
 	}
 
-	cachedTypeInfoMutex.Lock()
-	defer cachedTypeInfoMutex.Unlock()
-	if pti, ok = cachedTypeInfo[rtid]; ok {
-		return
-	}
-
+	// do not hold lock while computing this.
+	// it may lead to duplication, but that's ok.
 	ti := typeInfo{rt: rt, rtid: rtid}
-	pti = &ti
+	ti.numMeth = uint16(rt.NumMethod())
 
 	var indir int8
 	if ok, indir = implementsIntf(rt, binaryMarshalerTyp); ok {
@@ -768,11 +830,11 @@ func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
 	if rt.Kind() == reflect.Struct {
 		var siInfo *structFieldInfo
 		if f, ok := rt.FieldByName(structInfoFieldName); ok {
-			siInfo = parseStructFieldInfo(structInfoFieldName, getStructTag(f.Tag))
+			siInfo = parseStructFieldInfo(structInfoFieldName, x.structTag(f.Tag))
 			ti.toArray = siInfo.toArray
 		}
 		sfip := make([]*structFieldInfo, 0, rt.NumField())
-		rgetTypeInfo(rt, nil, make(map[string]bool, 16), &sfip, siInfo)
+		x.rget(rt, nil, make(map[string]bool, 16), &sfip, siInfo)
 
 		ti.sfip = make([]*structFieldInfo, len(sfip))
 		ti.sfi = make([]*structFieldInfo, len(sfip))
@@ -781,54 +843,64 @@ func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
 		copy(ti.sfi, sfip)
 	}
 	// sfi = sfip
-	cachedTypeInfo[rtid] = pti
+
+	x.mu.Lock()
+	if pti, ok = x.infos[rtid]; !ok {
+		pti = &ti
+		x.infos[rtid] = pti
+	}
+	x.mu.Unlock()
 	return
 }
 
-func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool,
+func (x *TypeInfos) rget(rt reflect.Type, indexstack []int, fnameToHastag map[string]bool,
 	sfi *[]*structFieldInfo, siInfo *structFieldInfo,
 ) {
 	for j := 0; j < rt.NumField(); j++ {
 		f := rt.Field(j)
-		// func types are skipped.
-		if tk := f.Type.Kind(); tk == reflect.Func {
+		fkind := f.Type.Kind()
+		// skip if a func type, or is unexported, or structTag value == "-"
+		if fkind == reflect.Func {
 			continue
 		}
-		stag := getStructTag(f.Tag)
-		if stag == "-" {
+		// if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
+		if f.PkgPath != "" && !f.Anonymous { // unexported, not embedded
 			continue
 		}
-		if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
+		stag := x.structTag(f.Tag)
+		if stag == "-" {
 			continue
 		}
 		var si *structFieldInfo
-		// if anonymous and there is no struct tag (or it's blank)
-		// and its a struct (or pointer to struct), inline it.
-		var doInline bool
-		if f.Anonymous && f.Type.Kind() != reflect.Interface {
-			doInline = stag == ""
+		// if anonymous and no struct tag (or it's blank), and a struct (or pointer to struct), inline it.
+		if f.Anonymous && fkind != reflect.Interface {
+			doInline := stag == ""
 			if !doInline {
 				si = parseStructFieldInfo("", stag)
 				doInline = si.encName == ""
 				// doInline = si.isZero()
-				// fmt.Printf(">>>> doInline for si.isZero: %s: %v\n", f.Name, doInline)
+			}
+			if doInline {
+				ft := f.Type
+				for ft.Kind() == reflect.Ptr {
+					ft = ft.Elem()
+				}
+				if ft.Kind() == reflect.Struct {
+					indexstack2 := make([]int, len(indexstack)+1, len(indexstack)+4)
+					copy(indexstack2, indexstack)
+					indexstack2[len(indexstack)] = j
+					// indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
+					x.rget(ft, indexstack2, fnameToHastag, sfi, siInfo)
+					continue
+				}
 			}
 		}
 
-		if doInline {
-			ft := f.Type
-			for ft.Kind() == reflect.Ptr {
-				ft = ft.Elem()
-			}
-			if ft.Kind() == reflect.Struct {
-				indexstack2 := make([]int, len(indexstack)+1, len(indexstack)+4)
-				copy(indexstack2, indexstack)
-				indexstack2[len(indexstack)] = j
-				// indexstack2 := append(append(make([]int, 0, len(indexstack)+4), indexstack...), j)
-				rgetTypeInfo(ft, indexstack2, fnameToHastag, sfi, siInfo)
-				continue
-			}
+		// after the anonymous dance: if an unexported field, skip
+		if f.PkgPath != "" { // unexported
+			continue
 		}
+
 		// do not let fields with same name in embedded structs override field at higher level.
 		// this must be done after anonymous check, to allow anonymous field
 		// still include their child fields
@@ -944,3 +1016,114 @@ func (_ checkOverflow) SignedInt(v uint64) (i int64, overflow bool) {
 	i = int64(v)
 	return
 }
+
+// ------------------ SORT -----------------
+
+func isNaN(f float64) bool { return f != f }
+
+// -----------------------
+
+type intSlice []int64
+type uintSlice []uint64
+type floatSlice []float64
+type boolSlice []bool
+type stringSlice []string
+type bytesSlice [][]byte
+
+func (p intSlice) Len() int           { return len(p) }
+func (p intSlice) Less(i, j int) bool { return p[i] < p[j] }
+func (p intSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (p uintSlice) Len() int           { return len(p) }
+func (p uintSlice) Less(i, j int) bool { return p[i] < p[j] }
+func (p uintSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (p floatSlice) Len() int { return len(p) }
+func (p floatSlice) Less(i, j int) bool {
+	return p[i] < p[j] || isNaN(p[i]) && !isNaN(p[j])
+}
+func (p floatSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func (p stringSlice) Len() int           { return len(p) }
+func (p stringSlice) Less(i, j int) bool { return p[i] < p[j] }
+func (p stringSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (p bytesSlice) Len() int           { return len(p) }
+func (p bytesSlice) Less(i, j int) bool { return bytes.Compare(p[i], p[j]) == -1 }
+func (p bytesSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (p boolSlice) Len() int           { return len(p) }
+func (p boolSlice) Less(i, j int) bool { return !p[i] && p[j] }
+func (p boolSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+// ---------------------
+
+type intRv struct {
+	v int64
+	r reflect.Value
+}
+type intRvSlice []intRv
+type uintRv struct {
+	v uint64
+	r reflect.Value
+}
+type uintRvSlice []uintRv
+type floatRv struct {
+	v float64
+	r reflect.Value
+}
+type floatRvSlice []floatRv
+type boolRv struct {
+	v bool
+	r reflect.Value
+}
+type boolRvSlice []boolRv
+type stringRv struct {
+	v string
+	r reflect.Value
+}
+type stringRvSlice []stringRv
+type bytesRv struct {
+	v []byte
+	r reflect.Value
+}
+type bytesRvSlice []bytesRv
+
+func (p intRvSlice) Len() int           { return len(p) }
+func (p intRvSlice) Less(i, j int) bool { return p[i].v < p[j].v }
+func (p intRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (p uintRvSlice) Len() int           { return len(p) }
+func (p uintRvSlice) Less(i, j int) bool { return p[i].v < p[j].v }
+func (p uintRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (p floatRvSlice) Len() int { return len(p) }
+func (p floatRvSlice) Less(i, j int) bool {
+	return p[i].v < p[j].v || isNaN(p[i].v) && !isNaN(p[j].v)
+}
+func (p floatRvSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
+
+func (p stringRvSlice) Len() int           { return len(p) }
+func (p stringRvSlice) Less(i, j int) bool { return p[i].v < p[j].v }
+func (p stringRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (p bytesRvSlice) Len() int           { return len(p) }
+func (p bytesRvSlice) Less(i, j int) bool { return bytes.Compare(p[i].v, p[j].v) == -1 }
+func (p bytesRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+func (p boolRvSlice) Len() int           { return len(p) }
+func (p boolRvSlice) Less(i, j int) bool { return !p[i].v && p[j].v }
+func (p boolRvSlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }
+
+// -----------------
+
+type bytesI struct {
+	v []byte
+	i interface{}
+}
+
+type bytesISlice []bytesI
+
+func (p bytesISlice) Len() int           { return len(p) }
+func (p bytesISlice) Less(i, j int) bool { return bytes.Compare(p[i].v, p[j].v) == -1 }
+func (p bytesISlice) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

Godeps/_workspace/src/github.com/ugorji/go/codec/helper_test.go

@@ -29,7 +29,22 @@ package codec
 // To fully test everything:
 //    go test -tags=x -benchtime=100ms -tv -bg -bi  -brw -bu -v -run=. -bench=.
 
+// Handling flags
+// codec_test.go will define a set of global flags for testing, including:
+//   - Use Reset
+//   - Use IO reader/writer (vs direct bytes)
+//   - Set Canonical
+//   - Set InternStrings
+//   - Use Symbols
+//
+// This way, we can test them all by running same set of tests with a different
+// set of flags.
+//
+// Following this, all the benchmarks will utilize flags set by codec_test.go
+// and will not redefine these "global" flags.
+
 import (
+	"bytes"
 	"errors"
 	"flag"
 	"fmt"
@@ -38,30 +53,45 @@ import (
 	"testing"
 )
 
-const (
-	testLogToT    = true
-	failNowOnFail = true
-)
+type testHED struct {
+	H Handle
+	E *Encoder
+	D *Decoder
+}
 
 var (
-	testNoopH      = NoopHandle(8)
-	testMsgpackH   = &MsgpackHandle{}
-	testBincH      = &BincHandle{}
-	testBincHNoSym = &BincHandle{}
-	testBincHSym   = &BincHandle{}
-	testSimpleH    = &SimpleHandle{}
-	testCborH      = &CborHandle{}
-	testJsonH      = &JsonHandle{}
-
+	testNoopH    = NoopHandle(8)
+	testMsgpackH = &MsgpackHandle{}
+	testBincH    = &BincHandle{}
+	testSimpleH  = &SimpleHandle{}
+	testCborH    = &CborHandle{}
+	testJsonH    = &JsonHandle{}
+
+	testHandles     []Handle
 	testPreInitFns  []func()
 	testPostInitFns []func()
 
 	testOnce sync.Once
+
+	testHEDs []testHED
 )
 
 func init() {
-	testBincHSym.AsSymbols = AsSymbolAll
-	testBincHNoSym.AsSymbols = AsSymbolNone
+	testHEDs = make([]testHED, 0, 32)
+	testHandles = append(testHandles,
+		testNoopH, testMsgpackH, testBincH, testSimpleH,
+		testCborH, testJsonH)
+}
+
+func testHEDGet(h Handle) *testHED {
+	for i := range testHEDs {
+		v := &testHEDs[i]
+		if v.H == h {
+			return v
+		}
+	}
+	testHEDs = append(testHEDs, testHED{h, NewEncoder(nil, h), NewDecoder(nil, h)})
+	return &testHEDs[len(testHEDs)-1]
 }
 
 func testInitAll() {
@@ -74,6 +104,95 @@ func testInitAll() {
 	}
 }
 
+func testCodecEncode(ts interface{}, bsIn []byte,
+	fn func([]byte) *bytes.Buffer, h Handle) (bs []byte, err error) {
+	// bs = make([]byte, 0, approxSize)
+	var e *Encoder
+	var buf *bytes.Buffer
+	if testUseReset {
+		e = testHEDGet(h).E
+	} else {
+		e = NewEncoder(nil, h)
+	}
+	if testUseIoEncDec {
+		buf = fn(bsIn)
+		e.Reset(buf)
+	} else {
+		bs = bsIn
+		e.ResetBytes(&bs)
+	}
+	if testUseMust {
+		e.MustEncode(ts)
+	} else {
+		err = e.Encode(ts)
+	}
+	if testUseIoEncDec {
+		bs = buf.Bytes()
+	}
+	return
+}
+
+func testCodecDecode(bs []byte, ts interface{}, h Handle) (err error) {
+	var d *Decoder
+	var buf *bytes.Reader
+	if testUseReset {
+		d = testHEDGet(h).D
+	} else {
+		d = NewDecoder(nil, h)
+	}
+	if testUseIoEncDec {
+		buf = bytes.NewReader(bs)
+		d.Reset(buf)
+	} else {
+		d.ResetBytes(bs)
+	}
+	if testUseMust {
+		d.MustDecode(ts)
+	} else {
+		err = d.Decode(ts)
+	}
+	return
+}
+
+// ----- functions below are used only by tests (not benchmarks)
+
+const (
+	testLogToT    = true
+	failNowOnFail = true
+)
+
+func checkErrT(t *testing.T, err error) {
+	if err != nil {
+		logT(t, err.Error())
+		failT(t)
+	}
+}
+
+func checkEqualT(t *testing.T, v1 interface{}, v2 interface{}, desc string) (err error) {
+	if err = deepEqual(v1, v2); err != nil {
+		logT(t, "Not Equal: %s: %v. v1: %v, v2: %v", desc, err, v1, v2)
+		failT(t)
+	}
+	return
+}
+
+func failT(t *testing.T) {
+	if failNowOnFail {
+		t.FailNow()
+	} else {
+		t.Fail()
+	}
+}
+
+// --- these functions are used by both benchmarks and tests
+
+func deepEqual(v1, v2 interface{}) (err error) {
+	if !reflect.DeepEqual(v1, v2) {
+		err = errors.New("Not Match")
+	}
+	return
+}
+
 func logT(x interface{}, format string, args ...interface{}) {
 	if t, ok := x.(*testing.T); ok && t != nil && testLogToT {
 		if testVerbose {
@@ -121,35 +240,3 @@ func approxDataSize(rv reflect.Value) (sum int) {
 	}
 	return
 }
-
-// ----- functions below are used only by tests (not benchmarks)
-
-func checkErrT(t *testing.T, err error) {
-	if err != nil {
-		logT(t, err.Error())
-		failT(t)
-	}
-}
-
-func checkEqualT(t *testing.T, v1 interface{}, v2 interface{}, desc string) (err error) {
-	if err = deepEqual(v1, v2); err != nil {
-		logT(t, "Not Equal: %s: %v. v1: %v, v2: %v", desc, err, v1, v2)
-		failT(t)
-	}
-	return
-}
-
-func failT(t *testing.T) {
-	if failNowOnFail {
-		t.FailNow()
-	} else {
-		t.Fail()
-	}
-}
-
-func deepEqual(v1, v2 interface{}) (err error) {
-	if !reflect.DeepEqual(v1, v2) {
-		err = errors.New("Not Match")
-	}
-	return
-}

Godeps/_workspace/src/github.com/ugorji/go/codec/msgpack.go

@@ -103,11 +103,11 @@ var (
 //---------------------------------------------
 
 type msgpackEncDriver struct {
+	noBuiltInTypes
+	encNoSeparator
 	e *Encoder
 	w encWriter
 	h *MsgpackHandle
-	noBuiltInTypes
-	encNoSeparator
 	x [8]byte
 }
 
@@ -271,7 +271,6 @@ type msgpackDecDriver struct {
 	bd     byte
 	bdRead bool
 	br     bool // bytes reader
-	bdType valueType
 	noBuiltInTypes
 	noStreamingCodec
 	decNoSeparator
@@ -282,106 +281,100 @@ type msgpackDecDriver struct {
 // It is called when a nil interface{} is passed, leaving it up to the DecDriver
 // to introspect the stream and decide how best to decode.
 // It deciphers the value by looking at the stream first.
-func (d *msgpackDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+func (d *msgpackDecDriver) DecodeNaked() {
 	if !d.bdRead {
 		d.readNextBd()
 	}
 	bd := d.bd
+	n := &d.d.n
+	var decodeFurther bool
 
 	switch bd {
 	case mpNil:
-		vt = valueTypeNil
+		n.v = valueTypeNil
 		d.bdRead = false
 	case mpFalse:
-		vt = valueTypeBool
-		v = false
+		n.v = valueTypeBool
+		n.b = false
 	case mpTrue:
-		vt = valueTypeBool
-		v = true
+		n.v = valueTypeBool
+		n.b = true
 
 	case mpFloat:
-		vt = valueTypeFloat
-		v = float64(math.Float32frombits(bigen.Uint32(d.r.readx(4))))
+		n.v = valueTypeFloat
+		n.f = float64(math.Float32frombits(bigen.Uint32(d.r.readx(4))))
 	case mpDouble:
-		vt = valueTypeFloat
-		v = math.Float64frombits(bigen.Uint64(d.r.readx(8)))
+		n.v = valueTypeFloat
+		n.f = math.Float64frombits(bigen.Uint64(d.r.readx(8)))
 
 	case mpUint8:
-		vt = valueTypeUint
-		v = uint64(d.r.readn1())
+		n.v = valueTypeUint
+		n.u = uint64(d.r.readn1())
 	case mpUint16:
-		vt = valueTypeUint
-		v = uint64(bigen.Uint16(d.r.readx(2)))
+		n.v = valueTypeUint
+		n.u = uint64(bigen.Uint16(d.r.readx(2)))
 	case mpUint32:
-		vt = valueTypeUint
-		v = uint64(bigen.Uint32(d.r.readx(4)))
+		n.v = valueTypeUint
+		n.u = uint64(bigen.Uint32(d.r.readx(4)))
 	case mpUint64:
-		vt = valueTypeUint
-		v = uint64(bigen.Uint64(d.r.readx(8)))
+		n.v = valueTypeUint
+		n.u = uint64(bigen.Uint64(d.r.readx(8)))
 
 	case mpInt8:
-		vt = valueTypeInt
-		v = int64(int8(d.r.readn1()))
+		n.v = valueTypeInt
+		n.i = int64(int8(d.r.readn1()))
 	case mpInt16:
-		vt = valueTypeInt
-		v = int64(int16(bigen.Uint16(d.r.readx(2))))
+		n.v = valueTypeInt
+		n.i = int64(int16(bigen.Uint16(d.r.readx(2))))
 	case mpInt32:
-		vt = valueTypeInt
-		v = int64(int32(bigen.Uint32(d.r.readx(4))))
+		n.v = valueTypeInt
+		n.i = int64(int32(bigen.Uint32(d.r.readx(4))))
 	case mpInt64:
-		vt = valueTypeInt
-		v = int64(int64(bigen.Uint64(d.r.readx(8))))
+		n.v = valueTypeInt
+		n.i = int64(int64(bigen.Uint64(d.r.readx(8))))
 
 	default:
 		switch {
 		case bd >= mpPosFixNumMin && bd <= mpPosFixNumMax:
 			// positive fixnum (always signed)
-			vt = valueTypeInt
-			v = int64(int8(bd))
+			n.v = valueTypeInt
+			n.i = int64(int8(bd))
 		case bd >= mpNegFixNumMin && bd <= mpNegFixNumMax:
 			// negative fixnum
-			vt = valueTypeInt
-			v = int64(int8(bd))
+			n.v = valueTypeInt
+			n.i = int64(int8(bd))
 		case bd == mpStr8, bd == mpStr16, bd == mpStr32, bd >= mpFixStrMin && bd <= mpFixStrMax:
 			if d.h.RawToString {
-				var rvm string
-				vt = valueTypeString
-				v = &rvm
+				n.v = valueTypeString
+				n.s = d.DecodeString()
 			} else {
-				var rvm = zeroByteSlice
-				vt = valueTypeBytes
-				v = &rvm
+				n.v = valueTypeBytes
+				n.l = d.DecodeBytes(nil, false, false)
 			}
-			decodeFurther = true
 		case bd == mpBin8, bd == mpBin16, bd == mpBin32:
-			var rvm = zeroByteSlice
-			vt = valueTypeBytes
-			v = &rvm
-			decodeFurther = true
+			n.v = valueTypeBytes
+			n.l = d.DecodeBytes(nil, false, false)
 		case bd == mpArray16, bd == mpArray32, bd >= mpFixArrayMin && bd <= mpFixArrayMax:
-			vt = valueTypeArray
+			n.v = valueTypeArray
 			decodeFurther = true
 		case bd == mpMap16, bd == mpMap32, bd >= mpFixMapMin && bd <= mpFixMapMax:
-			vt = valueTypeMap
+			n.v = valueTypeMap
 			decodeFurther = true
 		case bd >= mpFixExt1 && bd <= mpFixExt16, bd >= mpExt8 && bd <= mpExt32:
+			n.v = valueTypeExt
 			clen := d.readExtLen()
-			var re RawExt
-			re.Tag = uint64(d.r.readn1())
-			re.Data = d.r.readx(clen)
-			v = &re
-			vt = valueTypeExt
+			n.u = uint64(d.r.readn1())
+			n.l = d.r.readx(clen)
 		default:
 			d.d.errorf("Nil-Deciphered DecodeValue: %s: hex: %x, dec: %d", msgBadDesc, bd, bd)
-			return
 		}
 	}
 	if !decodeFurther {
 		d.bdRead = false
 	}
-	if vt == valueTypeUint && d.h.SignedInteger {
-		d.bdType = valueTypeInt
-		v = int64(v.(uint64))
+	if n.v == valueTypeUint && d.h.SignedInteger {
+		n.v = valueTypeInt
+		n.i = int64(n.v)
 	}
 	return
 }
@@ -566,28 +559,27 @@ func (d *msgpackDecDriver) DecodeString() (s string) {
 func (d *msgpackDecDriver) readNextBd() {
 	d.bd = d.r.readn1()
 	d.bdRead = true
-	d.bdType = valueTypeUnset
 }
 
-func (d *msgpackDecDriver) IsContainerType(vt valueType) bool {
+func (d *msgpackDecDriver) ContainerType() (vt valueType) {
 	bd := d.bd
-	switch vt {
-	case valueTypeNil:
-		return bd == mpNil
-	case valueTypeBytes:
-		return bd == mpBin8 || bd == mpBin16 || bd == mpBin32 ||
-			(!d.h.RawToString &&
-				(bd == mpStr8 || bd == mpStr16 || bd == mpStr32 || (bd >= mpFixStrMin && bd <= mpFixStrMax)))
-	case valueTypeString:
-		return d.h.RawToString &&
-			(bd == mpStr8 || bd == mpStr16 || bd == mpStr32 || (bd >= mpFixStrMin && bd <= mpFixStrMax))
-	case valueTypeArray:
-		return bd == mpArray16 || bd == mpArray32 || (bd >= mpFixArrayMin && bd <= mpFixArrayMax)
-	case valueTypeMap:
-		return bd == mpMap16 || bd == mpMap32 || (bd >= mpFixMapMin && bd <= mpFixMapMax)
-	}
-	d.d.errorf("isContainerType: unsupported parameter: %v", vt)
-	return false // "unreachable"
+	if bd == mpNil {
+		return valueTypeNil
+	} else if bd == mpBin8 || bd == mpBin16 || bd == mpBin32 ||
+		(!d.h.RawToString &&
+			(bd == mpStr8 || bd == mpStr16 || bd == mpStr32 || (bd >= mpFixStrMin && bd <= mpFixStrMax))) {
+		return valueTypeBytes
+	} else if d.h.RawToString &&
+		(bd == mpStr8 || bd == mpStr16 || bd == mpStr32 || (bd >= mpFixStrMin && bd <= mpFixStrMax)) {
+		return valueTypeString
+	} else if bd == mpArray16 || bd == mpArray32 || (bd >= mpFixArrayMin && bd <= mpFixArrayMax) {
+		return valueTypeArray
+	} else if bd == mpMap16 || bd == mpMap32 || (bd >= mpFixMapMin && bd <= mpFixMapMax) {
+		return valueTypeMap
+	} else {
+		// d.d.errorf("isContainerType: unsupported parameter: %v", vt)
+	}
+	return valueTypeUnset
 }
 
 func (d *msgpackDecDriver) TryDecodeAsNil() (v bool) {
@@ -701,7 +693,6 @@ func (d *msgpackDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs
 //MsgpackHandle is a Handle for the Msgpack Schema-Free Encoding Format.
 type MsgpackHandle struct {
 	BasicHandle
-	binaryEncodingType
 
 	// RawToString controls how raw bytes are decoded into a nil interface{}.
 	RawToString bool
@@ -717,6 +708,11 @@ type MsgpackHandle struct {
 	// type is provided (e.g. decoding into a nil interface{}), you get back
 	// a []byte or string based on the setting of RawToString.
 	WriteExt bool
+	binaryEncodingType
+}
+
+func (h *MsgpackHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
 }
 
 func (h *MsgpackHandle) newEncDriver(e *Encoder) encDriver {
@@ -727,8 +723,12 @@ func (h *MsgpackHandle) newDecDriver(d *Decoder) decDriver {
 	return &msgpackDecDriver{d: d, r: d.r, h: h, br: d.bytes}
 }
 
-func (h *MsgpackHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
-	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
+func (e *msgpackEncDriver) reset() {
+	e.w = e.e.w
+}
+
+func (d *msgpackDecDriver) reset() {
+	d.r = d.d.r
 }
 
 //--------------------------------------------------

Godeps/_workspace/src/github.com/ugorji/go/codec/noop.go

@@ -38,21 +38,26 @@ type noopHandle struct {
 }
 
 type noopDrv struct {
+	d    *Decoder
+	e    *Encoder
 	i    int
 	S    []string
 	B    [][]byte
 	mks  []bool    // stack. if map (true), else if array (false)
 	mk   bool      // top of stack. what container are we on? map or array?
-	ct   valueType // last request for IsContainerType.
-	cb   bool      // last response for IsContainerType.
+	ct   valueType // last response for IsContainerType.
+	cb   int       // counter for ContainerType
 	rand *rand.Rand
 }
 
 func (h *noopDrv) r(v int) int { return h.rand.Intn(v) }
 func (h *noopDrv) m(v int) int { h.i++; return h.i % v }
 
-func (h *noopDrv) newEncDriver(_ *Encoder) encDriver { return h }
-func (h *noopDrv) newDecDriver(_ *Decoder) decDriver { return h }
+func (h *noopDrv) newEncDriver(e *Encoder) encDriver { h.e = e; return h }
+func (h *noopDrv) newDecDriver(d *Decoder) decDriver { h.d = d; return h }
+
+func (h *noopDrv) reset()       {}
+func (h *noopDrv) uncacheRead() {}
 
 // --- encDriver
 
@@ -111,18 +116,48 @@ func (h *noopDrv) ReadEnd() { h.end() }
 func (h *noopDrv) ReadMapStart() int   { h.start(true); return h.m(10) }
 func (h *noopDrv) ReadArrayStart() int { h.start(false); return h.m(10) }
 
-func (h *noopDrv) IsContainerType(vt valueType) bool {
+func (h *noopDrv) ContainerType() (vt valueType) {
 	// return h.m(2) == 0
-	// handle kStruct
-	if h.ct == valueTypeMap && vt == valueTypeArray || h.ct == valueTypeArray && vt == valueTypeMap {
-		h.cb = !h.cb
-		h.ct = vt
-		return h.cb
-	}
-	// go in a loop and check it.
-	h.ct = vt
-	h.cb = h.m(7) == 0
-	return h.cb
+	// handle kStruct, which will bomb is it calls this and doesn't get back a map or array.
+	// consequently, if the return value is not map or array, reset it to one of them based on h.m(7) % 2
+	// for kstruct: at least one out of every 2 times, return one of valueTypeMap or Array (else kstruct bombs)
+	// however, every 10th time it is called, we just return something else.
+	var vals = [...]valueType{valueTypeArray, valueTypeMap}
+	//  ------------ TAKE ------------
+	// if h.cb%2 == 0 {
+	// 	if h.ct == valueTypeMap || h.ct == valueTypeArray {
+	// 	} else {
+	// 		h.ct = vals[h.m(2)]
+	// 	}
+	// } else if h.cb%5 == 0 {
+	// 	h.ct = valueType(h.m(8))
+	// } else {
+	// 	h.ct = vals[h.m(2)]
+	// }
+	//  ------------ TAKE ------------
+	// if h.cb%16 == 0 {
+	// 	h.ct = valueType(h.cb % 8)
+	// } else {
+	// 	h.ct = vals[h.cb%2]
+	// }
+	h.ct = vals[h.cb%2]
+	h.cb++
+	return h.ct
+
+	// if h.ct == valueTypeNil || h.ct == valueTypeString || h.ct == valueTypeBytes {
+	// 	return h.ct
+	// }
+	// return valueTypeUnset
+	// TODO: may need to tweak this so it works.
+	// if h.ct == valueTypeMap && vt == valueTypeArray || h.ct == valueTypeArray && vt == valueTypeMap {
+	// 	h.cb = !h.cb
+	// 	h.ct = vt
+	// 	return h.cb
+	// }
+	// // go in a loop and check it.
+	// h.ct = vt
+	// h.cb = h.m(7) == 0
+	// return h.cb
 }
 func (h *noopDrv) TryDecodeAsNil() bool {
 	if h.mk {
@@ -135,7 +170,7 @@ func (h *noopDrv) DecodeExt(rv interface{}, xtag uint64, ext Ext) uint64 {
 	return 0
 }
 
-func (h *noopDrv) DecodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+func (h *noopDrv) DecodeNaked() {
 	// use h.r (random) not h.m() because h.m() could cause the same value to be given.
 	var sk int
 	if h.mk {
@@ -144,32 +179,35 @@ func (h *noopDrv) DecodeNaked() (v interface{}, vt valueType, decodeFurther bool
 	} else {
 		sk = h.r(12)
 	}
+	n := &h.d.n
 	switch sk {
 	case 0:
-		vt = valueTypeNil
+		n.v = valueTypeNil
 	case 1:
-		vt, v = valueTypeBool, false
+		n.v, n.b = valueTypeBool, false
 	case 2:
-		vt, v = valueTypeBool, true
+		n.v, n.b = valueTypeBool, true
 	case 3:
-		vt, v = valueTypeInt, h.DecodeInt(64)
+		n.v, n.i = valueTypeInt, h.DecodeInt(64)
 	case 4:
-		vt, v = valueTypeUint, h.DecodeUint(64)
+		n.v, n.u = valueTypeUint, h.DecodeUint(64)
 	case 5:
-		vt, v = valueTypeFloat, h.DecodeFloat(true)
+		n.v, n.f = valueTypeFloat, h.DecodeFloat(true)
 	case 6:
-		vt, v = valueTypeFloat, h.DecodeFloat(false)
+		n.v, n.f = valueTypeFloat, h.DecodeFloat(false)
 	case 7:
-		vt, v = valueTypeString, h.DecodeString()
+		n.v, n.s = valueTypeString, h.DecodeString()
 	case 8:
-		vt, v = valueTypeBytes, h.B[h.m(len(h.B))]
+		n.v, n.l = valueTypeBytes, h.B[h.m(len(h.B))]
 	case 9:
-		vt, decodeFurther = valueTypeArray, true
+		n.v = valueTypeArray
 	case 10:
-		vt, decodeFurther = valueTypeMap, true
+		n.v = valueTypeMap
 	default:
-		vt, v = valueTypeExt, &RawExt{Tag: h.DecodeUint(64), Data: h.B[h.m(len(h.B))]}
+		n.v = valueTypeExt
+		n.u = h.DecodeUint(64)
+		n.l = h.B[h.m(len(h.B))]
 	}
-	h.ct = vt
+	h.ct = n.v
 	return
 }

Godeps/_workspace/src/github.com/ugorji/go/codec/prebuild.sh

@@ -49,7 +49,8 @@ _build() {
         # [ -e "safe${_gg}" ] && mv safe${_gg} safe${_gg}__${_zts}.bak
         # [ -e "unsafe${_gg}" ] && mv unsafe${_gg} unsafe${_gg}__${_zts}.bak
     else 
-        rm -f fast-path.generated.go gen.generated.go gen-helper.generated.go *safe.generated.go *_generated_test.go *.generated_ffjson_expose.go
+        rm -f fast-path.generated.go gen.generated.go gen-helper.generated.go \
+           *safe.generated.go *_generated_test.go *.generated_ffjson_expose.go
     fi
 
     cat > gen.generated.go <<EOF
@@ -77,28 +78,15 @@ EOF
 \`
 
 EOF
-    # All functions, variables which must exist are put in this file.
-    # This way, build works before we generate the right things.
-    cat > fast-path.generated.go <<EOF
+
+    cat > gen-from-tmpl.codec.generated.go <<EOF
 package codec 
-import "reflect"
-// func GenBytesToStringRO(b []byte) string { return string(b) }
-func fastpathDecodeTypeSwitch(iv interface{}, d *Decoder) bool { return false }
-func fastpathEncodeTypeSwitch(iv interface{}, e *Encoder) bool { return false }
-func fastpathEncodeTypeSwitchSlice(iv interface{}, e *Encoder) bool { return false }
-func fastpathEncodeTypeSwitchMap(iv interface{}, e *Encoder) bool { return false }
-type fastpathE struct {
-	rtid uintptr
-	rt reflect.Type 
-	encfn func(*encFnInfo, reflect.Value)
-	decfn func(*decFnInfo, reflect.Value)
+import "io"
+func GenInternalGoFile(r io.Reader, w io.Writer, safe bool) error {
+return genInternalGoFile(r, w, safe)
 }
-type fastpathA [0]fastpathE
-func (x fastpathA) index(rtid uintptr) int { return -1 }
-var fastpathAV fastpathA 
-
 EOF
-
+    
     cat > gen-from-tmpl.generated.go <<EOF
 //+build ignore
 
@@ -129,8 +117,8 @@ run("gen-helper.go.tmpl", "gen-helper.generated.go", false)
 }
 
 EOF
-    go run gen-from-tmpl.generated.go && \
-        rm -f gen-from-tmpl.generated.go 
+    go run -tags=notfastpath gen-from-tmpl.generated.go && \
+        rm -f gen-from-tmpl.*generated.go 
 }
 
 _codegenerators() {
@@ -140,18 +128,35 @@ _codegenerators() {
                 "1" == $( _needgen "values_ffjson${zsfx}" ) ||
                 1 == 0 ]] 
     then
-        true && \
-            echo "codecgen - !unsafe ... " && \
-            codecgen  -rt codecgen -t 'x,codecgen,!unsafe' -o values_codecgen${zsfx} -d 1978 $zfin && \
-            echo "codecgen - unsafe ... " && \
-            codecgen  -u -rt codecgen -t 'x,codecgen,unsafe' -o values_codecgen_unsafe${zsfx} -d 1978 $zfin && \
-            echo "msgp ... " && \
-            msgp -tests=false -pkg=codec -o=values_msgp${zsfx} -file=$zfin && \
+        # codecgen creates some temporary files in the directory (main, pkg).
+        # Consequently, we should start msgp and ffjson first, and also put a small time latency before
+        # starting codecgen.
+        # Without this, ffjson chokes on one of the temporary files from codecgen.
+        if [[ $zexternal == "1" ]]
+        then 
             echo "ffjson ... " && \
-            ffjson -w values_ffjson${zsfx} $zfin && \
+                ffjson -w values_ffjson${zsfx} $zfin &
+            zzzIdFF=$!
+            echo "msgp ... " && \
+                msgp -tests=false -o=values_msgp${zsfx} -file=$zfin &
+            zzzIdMsgp=$!
+            
+            sleep 1 # give ffjson and msgp some buffer time. see note above.
+        fi
+        
+        echo "codecgen - !unsafe ... " && \
+            codecgen -rt codecgen -t 'x,codecgen,!unsafe' -o values_codecgen${zsfx} -d 19780 $zfin &
+        zzzIdC=$!
+        echo "codecgen - unsafe ... " && \
+            codecgen  -u -rt codecgen -t 'x,codecgen,unsafe' -o values_codecgen_unsafe${zsfx} -d 19781 $zfin &
+        zzzIdCU=$!
+        wait $zzzIdC $zzzIdCU $zzzIdMsgp $zzzIdFF && \
             # remove (M|Unm)arshalJSON implementations, so they don't conflict with encoding/json bench \
-            sed -i 's+ MarshalJSON(+ _MarshalJSON(+g' values_ffjson${zsfx} && \
-            sed -i 's+ UnmarshalJSON(+ _UnmarshalJSON(+g' values_ffjson${zsfx} && \
+            if [[ $zexternal == "1" ]]
+            then
+                sed -i 's+ MarshalJSON(+ _MarshalJSON(+g' values_ffjson${zsfx} && \
+                    sed -i 's+ UnmarshalJSON(+ _UnmarshalJSON(+g' values_ffjson${zsfx}
+            fi && \
             echo "generators done!" && \
             true
     fi 
@@ -160,11 +165,12 @@ _codegenerators() {
 # _init reads the arguments and sets up the flags
 _init() {
 OPTIND=1
-while getopts "fb" flag
+while getopts "fbx" flag
 do
     case "x$flag" in 
         'xf') zforce=1;;
         'xb') zbak=1;;
+        'xx') zexternal=1;;
         *) echo "prebuild.sh accepts [-fb] only"; return 1;;
     esac
 done

Godeps/_workspace/src/github.com/ugorji/go/codec/simple.go

@@ -29,12 +29,12 @@ const (
 )
 
 type simpleEncDriver struct {
+	noBuiltInTypes
+	encNoSeparator
 	e *Encoder
 	h *SimpleHandle
 	w encWriter
-	noBuiltInTypes
 	b [8]byte
-	encNoSeparator
 }
 
 func (e *simpleEncDriver) EncodeNil() {
@@ -153,7 +153,6 @@ type simpleDecDriver struct {
 	h      *SimpleHandle
 	r      decReader
 	bdRead bool
-	bdType valueType
 	bd     byte
 	br     bool // bytes reader
 	noBuiltInTypes
@@ -165,28 +164,27 @@ type simpleDecDriver struct {
 func (d *simpleDecDriver) readNextBd() {
 	d.bd = d.r.readn1()
 	d.bdRead = true
-	d.bdType = valueTypeUnset
-}
-
-func (d *simpleDecDriver) IsContainerType(vt valueType) bool {
-	switch vt {
-	case valueTypeNil:
-		return d.bd == simpleVdNil
-	case valueTypeBytes:
-		const x uint8 = simpleVdByteArray
-		return d.bd == x || d.bd == x+1 || d.bd == x+2 || d.bd == x+3 || d.bd == x+4
-	case valueTypeString:
-		const x uint8 = simpleVdString
-		return d.bd == x || d.bd == x+1 || d.bd == x+2 || d.bd == x+3 || d.bd == x+4
-	case valueTypeArray:
-		const x uint8 = simpleVdArray
-		return d.bd == x || d.bd == x+1 || d.bd == x+2 || d.bd == x+3 || d.bd == x+4
-	case valueTypeMap:
-		const x uint8 = simpleVdMap
-		return d.bd == x || d.bd == x+1 || d.bd == x+2 || d.bd == x+3 || d.bd == x+4
+}
+
+func (d *simpleDecDriver) ContainerType() (vt valueType) {
+	if d.bd == simpleVdNil {
+		return valueTypeNil
+	} else if d.bd == simpleVdByteArray || d.bd == simpleVdByteArray+1 ||
+		d.bd == simpleVdByteArray+2 || d.bd == simpleVdByteArray+3 || d.bd == simpleVdByteArray+4 {
+		return valueTypeBytes
+	} else if d.bd == simpleVdString || d.bd == simpleVdString+1 ||
+		d.bd == simpleVdString+2 || d.bd == simpleVdString+3 || d.bd == simpleVdString+4 {
+		return valueTypeString
+	} else if d.bd == simpleVdArray || d.bd == simpleVdArray+1 ||
+		d.bd == simpleVdArray+2 || d.bd == simpleVdArray+3 || d.bd == simpleVdArray+4 {
+		return valueTypeArray
+	} else if d.bd == simpleVdMap || d.bd == simpleVdMap+1 ||
+		d.bd == simpleVdMap+2 || d.bd == simpleVdMap+3 || d.bd == simpleVdMap+4 {
+		return valueTypeMap
+	} else {
+		// d.d.errorf("isContainerType: unsupported parameter: %v", vt)
 	}
-	d.d.errorf("isContainerType: unsupported parameter: %v", vt)
-	return false // "unreachable"
+	return valueTypeUnset
 }
 
 func (d *simpleDecDriver) TryDecodeAsNil() bool {
@@ -410,59 +408,59 @@ func (d *simpleDecDriver) decodeExtV(verifyTag bool, tag byte) (xtag byte, xbs [
 	return
 }
 
-func (d *simpleDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+func (d *simpleDecDriver) DecodeNaked() {
 	if !d.bdRead {
 		d.readNextBd()
 	}
 
+	n := &d.d.n
+	var decodeFurther bool
+
 	switch d.bd {
 	case simpleVdNil:
-		vt = valueTypeNil
+		n.v = valueTypeNil
 	case simpleVdFalse:
-		vt = valueTypeBool
-		v = false
+		n.v = valueTypeBool
+		n.b = false
 	case simpleVdTrue:
-		vt = valueTypeBool
-		v = true
+		n.v = valueTypeBool
+		n.b = true
 	case simpleVdPosInt, simpleVdPosInt + 1, simpleVdPosInt + 2, simpleVdPosInt + 3:
 		if d.h.SignedInteger {
-			vt = valueTypeInt
-			v = d.DecodeInt(64)
+			n.v = valueTypeInt
+			n.i = d.DecodeInt(64)
 		} else {
-			vt = valueTypeUint
-			v = d.DecodeUint(64)
+			n.v = valueTypeUint
+			n.u = d.DecodeUint(64)
 		}
 	case simpleVdNegInt, simpleVdNegInt + 1, simpleVdNegInt + 2, simpleVdNegInt + 3:
-		vt = valueTypeInt
-		v = d.DecodeInt(64)
+		n.v = valueTypeInt
+		n.i = d.DecodeInt(64)
 	case simpleVdFloat32:
-		vt = valueTypeFloat
-		v = d.DecodeFloat(true)
+		n.v = valueTypeFloat
+		n.f = d.DecodeFloat(true)
 	case simpleVdFloat64:
-		vt = valueTypeFloat
-		v = d.DecodeFloat(false)
+		n.v = valueTypeFloat
+		n.f = d.DecodeFloat(false)
 	case simpleVdString, simpleVdString + 1, simpleVdString + 2, simpleVdString + 3, simpleVdString + 4:
-		vt = valueTypeString
-		v = d.DecodeString()
+		n.v = valueTypeString
+		n.s = d.DecodeString()
 	case simpleVdByteArray, simpleVdByteArray + 1, simpleVdByteArray + 2, simpleVdByteArray + 3, simpleVdByteArray + 4:
-		vt = valueTypeBytes
-		v = d.DecodeBytes(nil, false, false)
+		n.v = valueTypeBytes
+		n.l = d.DecodeBytes(nil, false, false)
 	case simpleVdExt, simpleVdExt + 1, simpleVdExt + 2, simpleVdExt + 3, simpleVdExt + 4:
-		vt = valueTypeExt
+		n.v = valueTypeExt
 		l := d.decLen()
-		var re RawExt
-		re.Tag = uint64(d.r.readn1())
-		re.Data = d.r.readx(l)
-		v = &re
+		n.u = uint64(d.r.readn1())
+		n.l = d.r.readx(l)
 	case simpleVdArray, simpleVdArray + 1, simpleVdArray + 2, simpleVdArray + 3, simpleVdArray + 4:
-		vt = valueTypeArray
+		n.v = valueTypeArray
 		decodeFurther = true
 	case simpleVdMap, simpleVdMap + 1, simpleVdMap + 2, simpleVdMap + 3, simpleVdMap + 4:
-		vt = valueTypeMap
+		n.v = valueTypeMap
 		decodeFurther = true
 	default:
 		d.d.errorf("decodeNaked: Unrecognized d.bd: 0x%x", d.bd)
-		return
 	}
 
 	if !decodeFurther {
@@ -496,6 +494,10 @@ type SimpleHandle struct {
 	binaryEncodingType
 }
 
+func (h *SimpleHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
+}
+
 func (h *SimpleHandle) newEncDriver(e *Encoder) encDriver {
 	return &simpleEncDriver{e: e, w: e.w, h: h}
 }
@@ -504,8 +506,12 @@ func (h *SimpleHandle) newDecDriver(d *Decoder) decDriver {
 	return &simpleDecDriver{d: d, r: d.r, h: h, br: d.bytes}
 }
 
-func (h *SimpleHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
-	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
+func (e *simpleEncDriver) reset() {
+	e.w = e.e.w
+}
+
+func (d *simpleDecDriver) reset() {
+	d.r = d.d.r
 }
 
 var _ decDriver = (*simpleDecDriver)(nil)

Godeps/_workspace/src/github.com/ugorji/go/codec/time.go

@@ -4,6 +4,7 @@
 package codec
 
 import (
+	"fmt"
 	"time"
 )
 
@@ -11,6 +12,34 @@ var (
 	timeDigits = [...]byte{'0', '1', '2', '3', '4', '5', '6', '7', '8', '9'}
 )
 
+type timeExt struct{}
+
+func (x timeExt) WriteExt(v interface{}) (bs []byte) {
+	switch v2 := v.(type) {
+	case time.Time:
+		bs = encodeTime(v2)
+	case *time.Time:
+		bs = encodeTime(*v2)
+	default:
+		panic(fmt.Errorf("unsupported format for time conversion: expecting time.Time; got %T", v2))
+	}
+	return
+}
+func (x timeExt) ReadExt(v interface{}, bs []byte) {
+	tt, err := decodeTime(bs)
+	if err != nil {
+		panic(err)
+	}
+	*(v.(*time.Time)) = tt
+}
+
+func (x timeExt) ConvertExt(v interface{}) interface{} {
+	return x.WriteExt(v)
+}
+func (x timeExt) UpdateExt(v interface{}, src interface{}) {
+	x.ReadExt(v, src.([]byte))
+}
+
 // EncodeTime encodes a time.Time as a []byte, including
 // information on the instant in time and UTC offset.
 //

client/keys.generated.go

@@ -15,10 +15,18 @@ import (
 )
 
 const (
-	codecSelferC_UTF81819         = 1
-	codecSelferC_RAW1819          = 0
+	// ----- content types ----
+	codecSelferC_UTF81819 = 1
+	codecSelferC_RAW1819  = 0
+	// ----- value types used ----
 	codecSelferValueTypeArray1819 = 10
 	codecSelferValueTypeMap1819   = 9
+	// ----- containerStateValues ----
+	codecSelfer_containerMapKey1819    = 2
+	codecSelfer_containerMapValue1819  = 3
+	codecSelfer_containerMapEnd1819    = 4
+	codecSelfer_containerArrayElem1819 = 6
+	codecSelfer_containerArrayEnd1819  = 7
 )
 
 var (
@@ -29,10 +37,10 @@ var (
 type codecSelfer1819 struct{}
 
 func init() {
-	if codec1978.GenVersion != 4 {
+	if codec1978.GenVersion != 5 {
 		_, file, _, _ := runtime.Caller(0)
 		err := fmt.Errorf("codecgen version mismatch: current: %v, need %v. Re-generate file: %v",
-			4, codec1978.GenVersion, file)
+			5, codec1978.GenVersion, file)
 		panic(err)
 	}
 	if false { // reference the types, but skip this branch at build/run time
@@ -58,18 +66,21 @@ func (x *Response) CodecEncodeSelf(e *codec1978.Encoder) {
 			var yyq2 [3]bool
 			_, _, _ = yysep2, yyq2, yy2arr2
 			const yyr2 bool = false
+			var yynn2 int
 			if yyr2 || yy2arr2 {
 				r.EncodeArrayStart(3)
 			} else {
-				var yynn2 int = 3
+				yynn2 = 3
 				for _, b := range yyq2 {
 					if b {
 						yynn2++
 					}
 				}
 				r.EncodeMapStart(yynn2)
+				yynn2 = 0
 			}
 			if yyr2 || yy2arr2 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				yym4 := z.EncBinary()
 				_ = yym4
 				if false {
@@ -77,7 +88,9 @@ func (x *Response) CodecEncodeSelf(e *codec1978.Encoder) {
 					r.EncodeString(codecSelferC_UTF81819, string(x.Action))
 				}
 			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1819)
 				r.EncodeString(codecSelferC_UTF81819, string("action"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1819)
 				yym5 := z.EncBinary()
 				_ = yym5
 				if false {
@@ -86,13 +99,16 @@ func (x *Response) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr2 || yy2arr2 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				if x.Node == nil {
 					r.EncodeNil()
 				} else {
 					x.Node.CodecEncodeSelf(e)
 				}
 			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1819)
 				r.EncodeString(codecSelferC_UTF81819, string("node"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1819)
 				if x.Node == nil {
 					r.EncodeNil()
 				} else {
@@ -100,21 +116,26 @@ func (x *Response) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr2 || yy2arr2 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				if x.PrevNode == nil {
 					r.EncodeNil()
 				} else {
 					x.PrevNode.CodecEncodeSelf(e)
 				}
 			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1819)
 				r.EncodeString(codecSelferC_UTF81819, string("prevNode"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1819)
 				if x.PrevNode == nil {
 					r.EncodeNil()
 				} else {
 					x.PrevNode.CodecEncodeSelf(e)
 				}
 			}
-			if yysep2 {
-				r.EncodeEnd()
+			if yyr2 || yy2arr2 {
+				z.EncSendContainerState(codecSelfer_containerArrayEnd1819)
+			} else {
+				z.EncSendContainerState(codecSelfer_containerMapEnd1819)
 			}
 		}
 	}
@@ -129,17 +150,18 @@ func (x *Response) CodecDecodeSelf(d *codec1978.Decoder) {
 	if false {
 	} else if z.HasExtensions() && z.DecExt(x) {
 	} else {
-		if r.IsContainerType(codecSelferValueTypeMap1819) {
+		yyct9 := r.ContainerType()
+		if yyct9 == codecSelferValueTypeMap1819 {
 			yyl9 := r.ReadMapStart()
 			if yyl9 == 0 {
-				r.ReadEnd()
+				z.DecSendContainerState(codecSelfer_containerMapEnd1819)
 			} else {
 				x.codecDecodeSelfFromMap(yyl9, d)
 			}
-		} else if r.IsContainerType(codecSelferValueTypeArray1819) {
+		} else if yyct9 == codecSelferValueTypeArray1819 {
 			yyl9 := r.ReadArrayStart()
 			if yyl9 == 0 {
-				r.ReadEnd()
+				z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 			} else {
 				x.codecDecodeSelfFromArray(yyl9, d)
 			}
@@ -166,8 +188,10 @@ func (x *Response) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) {
 				break
 			}
 		}
+		z.DecSendContainerState(codecSelfer_containerMapKey1819)
 		yys10Slc = r.DecodeBytes(yys10Slc, true, true)
 		yys10 := string(yys10Slc)
+		z.DecSendContainerState(codecSelfer_containerMapValue1819)
 		switch yys10 {
 		case "action":
 			if r.TryDecodeAsNil() {
@@ -201,9 +225,7 @@ func (x *Response) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) {
 			z.DecStructFieldNotFound(-1, yys10)
 		} // end switch yys10
 	} // end for yyj10
-	if !yyhl10 {
-		r.ReadEnd()
-	}
+	z.DecSendContainerState(codecSelfer_containerMapEnd1819)
 }
 
 func (x *Response) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
@@ -220,9 +242,10 @@ func (x *Response) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb14 = r.CheckBreak()
 	}
 	if yyb14 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		x.Action = ""
 	} else {
@@ -235,9 +258,10 @@ func (x *Response) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb14 = r.CheckBreak()
 	}
 	if yyb14 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		if x.Node != nil {
 			x.Node = nil
@@ -255,9 +279,10 @@ func (x *Response) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb14 = r.CheckBreak()
 	}
 	if yyb14 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		if x.PrevNode != nil {
 			x.PrevNode = nil
@@ -278,9 +303,10 @@ func (x *Response) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		if yyb14 {
 			break
 		}
+		z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 		z.DecStructFieldNotFound(yyj14-1, "")
 	}
-	r.ReadEnd()
+	z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 }
 
 func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
@@ -303,18 +329,21 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 			yyq19[1] = x.Dir != false
 			yyq19[6] = x.Expiration != nil
 			yyq19[7] = x.TTL != 0
+			var yynn19 int
 			if yyr19 || yy2arr19 {
 				r.EncodeArrayStart(8)
 			} else {
-				var yynn19 int = 5
+				yynn19 = 5
 				for _, b := range yyq19 {
 					if b {
 						yynn19++
 					}
 				}
 				r.EncodeMapStart(yynn19)
+				yynn19 = 0
 			}
 			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				yym21 := z.EncBinary()
 				_ = yym21
 				if false {
@@ -322,7 +351,9 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 					r.EncodeString(codecSelferC_UTF81819, string(x.Key))
 				}
 			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1819)
 				r.EncodeString(codecSelferC_UTF81819, string("key"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1819)
 				yym22 := z.EncBinary()
 				_ = yym22
 				if false {
@@ -331,6 +362,7 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				if yyq19[1] {
 					yym24 := z.EncBinary()
 					_ = yym24
@@ -343,7 +375,9 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			} else {
 				if yyq19[1] {
+					z.EncSendContainerState(codecSelfer_containerMapKey1819)
 					r.EncodeString(codecSelferC_UTF81819, string("dir"))
+					z.EncSendContainerState(codecSelfer_containerMapValue1819)
 					yym25 := z.EncBinary()
 					_ = yym25
 					if false {
@@ -353,6 +387,7 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				yym27 := z.EncBinary()
 				_ = yym27
 				if false {
@@ -360,7 +395,9 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 					r.EncodeString(codecSelferC_UTF81819, string(x.Value))
 				}
 			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1819)
 				r.EncodeString(codecSelferC_UTF81819, string("value"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1819)
 				yym28 := z.EncBinary()
 				_ = yym28
 				if false {
@@ -369,13 +406,16 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				if x.Nodes == nil {
 					r.EncodeNil()
 				} else {
 					x.Nodes.CodecEncodeSelf(e)
 				}
 			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1819)
 				r.EncodeString(codecSelferC_UTF81819, string("nodes"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1819)
 				if x.Nodes == nil {
 					r.EncodeNil()
 				} else {
@@ -383,6 +423,7 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				yym31 := z.EncBinary()
 				_ = yym31
 				if false {
@@ -390,7 +431,9 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 					r.EncodeUint(uint64(x.CreatedIndex))
 				}
 			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1819)
 				r.EncodeString(codecSelferC_UTF81819, string("createdIndex"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1819)
 				yym32 := z.EncBinary()
 				_ = yym32
 				if false {
@@ -399,6 +442,7 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				yym34 := z.EncBinary()
 				_ = yym34
 				if false {
@@ -406,7 +450,9 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 					r.EncodeUint(uint64(x.ModifiedIndex))
 				}
 			} else {
+				z.EncSendContainerState(codecSelfer_containerMapKey1819)
 				r.EncodeString(codecSelferC_UTF81819, string("modifiedIndex"))
+				z.EncSendContainerState(codecSelfer_containerMapValue1819)
 				yym35 := z.EncBinary()
 				_ = yym35
 				if false {
@@ -415,6 +461,7 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				if yyq19[6] {
 					if x.Expiration == nil {
 						r.EncodeNil()
@@ -438,7 +485,9 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			} else {
 				if yyq19[6] {
+					z.EncSendContainerState(codecSelfer_containerMapKey1819)
 					r.EncodeString(codecSelferC_UTF81819, string("expiration"))
+					z.EncSendContainerState(codecSelfer_containerMapValue1819)
 					if x.Expiration == nil {
 						r.EncodeNil()
 					} else {
@@ -459,6 +508,7 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			}
 			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 				if yyq19[7] {
 					yym42 := z.EncBinary()
 					_ = yym42
@@ -471,7 +521,9 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 				}
 			} else {
 				if yyq19[7] {
+					z.EncSendContainerState(codecSelfer_containerMapKey1819)
 					r.EncodeString(codecSelferC_UTF81819, string("ttl"))
+					z.EncSendContainerState(codecSelfer_containerMapValue1819)
 					yym43 := z.EncBinary()
 					_ = yym43
 					if false {
@@ -480,8 +532,10 @@ func (x *Node) CodecEncodeSelf(e *codec1978.Encoder) {
 					}
 				}
 			}
-			if yysep19 {
-				r.EncodeEnd()
+			if yyr19 || yy2arr19 {
+				z.EncSendContainerState(codecSelfer_containerArrayEnd1819)
+			} else {
+				z.EncSendContainerState(codecSelfer_containerMapEnd1819)
 			}
 		}
 	}
@@ -496,17 +550,18 @@ func (x *Node) CodecDecodeSelf(d *codec1978.Decoder) {
 	if false {
 	} else if z.HasExtensions() && z.DecExt(x) {
 	} else {
-		if r.IsContainerType(codecSelferValueTypeMap1819) {
+		yyct45 := r.ContainerType()
+		if yyct45 == codecSelferValueTypeMap1819 {
 			yyl45 := r.ReadMapStart()
 			if yyl45 == 0 {
-				r.ReadEnd()
+				z.DecSendContainerState(codecSelfer_containerMapEnd1819)
 			} else {
 				x.codecDecodeSelfFromMap(yyl45, d)
 			}
-		} else if r.IsContainerType(codecSelferValueTypeArray1819) {
+		} else if yyct45 == codecSelferValueTypeArray1819 {
 			yyl45 := r.ReadArrayStart()
 			if yyl45 == 0 {
-				r.ReadEnd()
+				z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 			} else {
 				x.codecDecodeSelfFromArray(yyl45, d)
 			}
@@ -533,8 +588,10 @@ func (x *Node) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) {
 				break
 			}
 		}
+		z.DecSendContainerState(codecSelfer_containerMapKey1819)
 		yys46Slc = r.DecodeBytes(yys46Slc, true, true)
 		yys46 := string(yys46Slc)
+		z.DecSendContainerState(codecSelfer_containerMapValue1819)
 		switch yys46 {
 		case "key":
 			if r.TryDecodeAsNil() {
@@ -606,9 +663,7 @@ func (x *Node) codecDecodeSelfFromMap(l int, d *codec1978.Decoder) {
 			z.DecStructFieldNotFound(-1, yys46)
 		} // end switch yys46
 	} // end for yyj46
-	if !yyhl46 {
-		r.ReadEnd()
-	}
+	z.DecSendContainerState(codecSelfer_containerMapEnd1819)
 }
 
 func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
@@ -625,9 +680,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb57 = r.CheckBreak()
 	}
 	if yyb57 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		x.Key = ""
 	} else {
@@ -640,9 +696,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb57 = r.CheckBreak()
 	}
 	if yyb57 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		x.Dir = false
 	} else {
@@ -655,9 +712,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb57 = r.CheckBreak()
 	}
 	if yyb57 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		x.Value = ""
 	} else {
@@ -670,9 +728,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb57 = r.CheckBreak()
 	}
 	if yyb57 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		x.Nodes = nil
 	} else {
@@ -686,9 +745,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb57 = r.CheckBreak()
 	}
 	if yyb57 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		x.CreatedIndex = 0
 	} else {
@@ -701,9 +761,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb57 = r.CheckBreak()
 	}
 	if yyb57 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		x.ModifiedIndex = 0
 	} else {
@@ -716,9 +777,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb57 = r.CheckBreak()
 	}
 	if yyb57 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		if x.Expiration != nil {
 			x.Expiration = nil
@@ -748,9 +810,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		yyb57 = r.CheckBreak()
 	}
 	if yyb57 {
-		r.ReadEnd()
+		z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 		return
 	}
+	z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 	if r.TryDecodeAsNil() {
 		x.TTL = 0
 	} else {
@@ -766,9 +829,10 @@ func (x *Node) codecDecodeSelfFromArray(l int, d *codec1978.Decoder) {
 		if yyb57 {
 			break
 		}
+		z.DecSendContainerState(codecSelfer_containerArrayElem1819)
 		z.DecStructFieldNotFound(yyj57-1, "")
 	}
-	r.ReadEnd()
+	z.DecSendContainerState(codecSelfer_containerArrayEnd1819)
 }
 
 func (x Nodes) CodecEncodeSelf(e *codec1978.Encoder) {
@@ -807,13 +871,14 @@ func (x codecSelfer1819) encNodes(v Nodes, e *codec1978.Encoder) {
 	_, _, _ = h, z, r
 	r.EncodeArrayStart(len(v))
 	for _, yyv70 := range v {
+		z.EncSendContainerState(codecSelfer_containerArrayElem1819)
 		if yyv70 == nil {
 			r.EncodeNil()
 		} else {
 			yyv70.CodecEncodeSelf(e)
 		}
 	}
-	r.EncodeEnd()
+	z.EncSendContainerState(codecSelfer_containerArrayEnd1819)
 }
 
 func (x codecSelfer1819) decNodes(v *Nodes, d *codec1978.Decoder) {
@@ -823,39 +888,44 @@ func (x codecSelfer1819) decNodes(v *Nodes, d *codec1978.Decoder) {
 
 	yyv71 := *v
 	yyh71, yyl71 := z.DecSliceHelperStart()
-
-	var yyrr71, yyrl71 int
-	var yyc71, yyrt71 bool
-	_, _, _ = yyc71, yyrt71, yyrl71
-	yyrr71 = yyl71
-
-	if yyv71 == nil {
-		if yyrl71, yyrt71 = z.DecInferLen(yyl71, z.DecBasicHandle().MaxInitLen, 8); yyrt71 {
-			yyrr71 = yyrl71
-		}
-		yyv71 = make(Nodes, yyrl71)
-		yyc71 = true
-	}
-
+	var yyc71 bool
 	if yyl71 == 0 {
-		if len(yyv71) != 0 {
+		if yyv71 == nil {
+			yyv71 = []*Node{}
+			yyc71 = true
+		} else if len(yyv71) != 0 {
 			yyv71 = yyv71[:0]
 			yyc71 = true
 		}
 	} else if yyl71 > 0 {
-
+		var yyrr71, yyrl71 int
+		var yyrt71 bool
 		if yyl71 > cap(yyv71) {
+
+			yyrg71 := len(yyv71) > 0
+			yyv271 := yyv71
 			yyrl71, yyrt71 = z.DecInferLen(yyl71, z.DecBasicHandle().MaxInitLen, 8)
-			yyv71 = make([]*Node, yyrl71)
+			if yyrt71 {
+				if yyrl71 <= cap(yyv71) {
+					yyv71 = yyv71[:yyrl71]
+				} else {
+					yyv71 = make([]*Node, yyrl71)
+				}
+			} else {
+				yyv71 = make([]*Node, yyrl71)
+			}
 			yyc71 = true
-
 			yyrr71 = len(yyv71)
+			if yyrg71 {
+				copy(yyv71, yyv271)
+			}
 		} else if yyl71 != len(yyv71) {
 			yyv71 = yyv71[:yyl71]
 			yyc71 = true
 		}
 		yyj71 := 0
 		for ; yyj71 < yyrr71; yyj71++ {
+			yyh71.ElemContainerState(yyj71)
 			if r.TryDecodeAsNil() {
 				if yyv71[yyj71] != nil {
 					*yyv71[yyj71] = Node{}
@@ -872,6 +942,7 @@ func (x codecSelfer1819) decNodes(v *Nodes, d *codec1978.Decoder) {
 		if yyrt71 {
 			for ; yyj71 < yyl71; yyj71++ {
 				yyv71 = append(yyv71, nil)
+				yyh71.ElemContainerState(yyj71)
 				if r.TryDecodeAsNil() {
 					if yyv71[yyj71] != nil {
 						*yyv71[yyj71] = Node{}
@@ -888,12 +959,14 @@ func (x codecSelfer1819) decNodes(v *Nodes, d *codec1978.Decoder) {
 		}
 
 	} else {
-		for yyj71 := 0; !r.CheckBreak(); yyj71++ {
+		yyj71 := 0
+		for ; !r.CheckBreak(); yyj71++ {
+
 			if yyj71 >= len(yyv71) {
 				yyv71 = append(yyv71, nil) // var yyz71 *Node
 				yyc71 = true
 			}
-
+			yyh71.ElemContainerState(yyj71)
 			if yyj71 < len(yyv71) {
 				if r.TryDecodeAsNil() {
 					if yyv71[yyj71] != nil {
@@ -912,10 +985,16 @@ func (x codecSelfer1819) decNodes(v *Nodes, d *codec1978.Decoder) {
 			}
 
 		}
-		yyh71.End()
+		if yyj71 < len(yyv71) {
+			yyv71 = yyv71[:yyj71]
+			yyc71 = true
+		} else if yyj71 == 0 && yyv71 == nil {
+			yyv71 = []*Node{}
+			yyc71 = true
+		}
 	}
+	yyh71.End()
 	if yyc71 {
 		*v = yyv71
 	}
-
 }