Merge pull request #3655 from wojtek-t/update_dependency

Update dependency on ugorji/go/codec

Godeps/Godeps.json

@@ -113,7 +113,7 @@
 		},
 		{
 			"ImportPath": "github.com/ugorji/go/codec",
-			"Rev": "5abd4e96a45c386928ed2ca2a7ef63e2533e18ec"
+			"Rev": "45ce7596ace4534e47b69051a92aef7b64ec7b3f"
 		},
 		{
 			"ImportPath": "github.com/xiang90/probing",

Godeps/_workspace/src/github.com/ugorji/go/codec/binc.go

@@ -5,6 +5,7 @@ package codec
 
 import (
 	"math"
+	"reflect"
 	"time"
 )
 
@@ -69,7 +70,15 @@ func (e *bincEncDriver) IsBuiltinType(rt uintptr) bool {
 
 func (e *bincEncDriver) EncodeBuiltin(rt uintptr, v interface{}) {
 	if rt == timeTypId {
-		bs := encodeTime(v.(time.Time))
+		var bs []byte
+		switch x := v.(type) {
+		case time.Time:
+			bs = encodeTime(x)
+		case *time.Time:
+			bs = encodeTime(*x)
+		default:
+			e.e.errorf("binc error encoding builtin: expect time.Time, received %T", v)
+		}
 		e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs)))
 		e.w.writeb(bs)
 	}
@@ -897,5 +906,9 @@ func (h *BincHandle) newDecDriver(d *Decoder) decDriver {
 	return &bincDecDriver{d: d, r: d.r, h: h, br: d.bytes}
 }
 
+func (h *BincHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
+}
+
 var _ decDriver = (*bincDecDriver)(nil)
 var _ encDriver = (*bincEncDriver)(nil)

Godeps/_workspace/src/github.com/ugorji/go/codec/cbor.go

@@ -3,7 +3,10 @@
 
 package codec
 
-import "math"
+import (
+	"math"
+	"reflect"
+)
 
 const (
 	cborMajorUint byte = iota
@@ -158,7 +161,11 @@ func (e *cborEncDriver) EncodeSymbol(v string) {
 }
 
 func (e *cborEncDriver) EncodeStringBytes(c charEncoding, v []byte) {
-	e.encLen(cborBaseBytes, len(v))
+	if c == c_RAW {
+		e.encLen(cborBaseBytes, len(v))
+	} else {
+		e.encLen(cborBaseString, len(v))
+	}
 	e.w.writeb(v)
 }
 
@@ -562,5 +569,9 @@ func (h *CborHandle) newDecDriver(d *Decoder) decDriver {
 	return &cborDecDriver{d: d, r: d.r, h: h, br: d.bytes}
 }
 
+func (h *CborHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{i: ext})
+}
+
 var _ decDriver = (*cborDecDriver)(nil)
 var _ encDriver = (*cborEncDriver)(nil)

Godeps/_workspace/src/github.com/ugorji/go/codec/codec_test.go

@@ -860,6 +860,60 @@ func testCodecRpcOne(t *testing.T, rr Rpc, h Handle, doRequest bool, exitSleepMs
 	return
 }
 
+func doTestMapEncodeForCanonical(t *testing.T, name string, h Handle) {
+	v1 := map[string]interface{}{
+		"a": 1,
+		"b": "hello",
+		"c": map[string]interface{}{
+			"c/a": 1,
+			"c/b": "world",
+			"c/c": []int{1, 2, 3, 4},
+			"c/d": map[string]interface{}{
+				"c/d/a": "fdisajfoidsajfopdjsaopfjdsapofda",
+				"c/d/b": "fdsafjdposakfodpsakfopdsakfpodsakfpodksaopfkdsopafkdopsa",
+				"c/d/c": "poir02  ir30qif4p03qir0pogjfpoaerfgjp ofke[padfk[ewapf kdp[afep[aw",
+				"c/d/d": "fdsopafkd[sa f-32qor-=4qeof -afo-erfo r-eafo 4e-  o r4-qwo ag",
+				"c/d/e": "kfep[a sfkr0[paf[a foe-[wq  ewpfao-q ro3-q ro-4qof4-qor 3-e orfkropzjbvoisdb",
+				"c/d/f": "",
+			},
+			"c/e": map[int]string{
+				1:     "1",
+				22:    "22",
+				333:   "333",
+				4444:  "4444",
+				55555: "55555",
+			},
+			"c/f": map[string]int{
+				"1":     1,
+				"22":    22,
+				"333":   333,
+				"4444":  4444,
+				"55555": 55555,
+			},
+		},
+	}
+	var v2 map[string]interface{}
+	var b1, b2 []byte
+
+	// encode v1 into b1, decode b1 into v2, encode v2 into b2, compare b1 and b2
+
+	bh := h.getBasicHandle()
+	canonical0 := bh.Canonical
+	bh.Canonical = true
+	defer func() { bh.Canonical = canonical0 }()
+
+	e1 := NewEncoderBytes(&b1, h)
+	e1.MustEncode(v1)
+	d1 := NewDecoderBytes(b1, h)
+	d1.MustDecode(&v2)
+	e2 := NewEncoderBytes(&b2, h)
+	e2.MustEncode(v2)
+	if !bytes.Equal(b1, b2) {
+		logT(t, "Unequal bytes: %v VS %v", b1, b2)
+		t.FailNow()
+	}
+}
+
 // Comprehensive testing that generates data encoded from python handle (cbor, msgpack),
 // and validates that our code can read and write it out accordingly.
 // We keep this unexported here, and put actual test in ext_dep_test.go.
@@ -1048,6 +1102,10 @@ func TestCborCodecsEmbeddedPointer(t *testing.T) {
 	testCodecEmbeddedPointer(t, testCborH)
 }
 
+func TestCborMapEncodeForCanonical(t *testing.T) {
+	doTestMapEncodeForCanonical(t, "cbor", testCborH)
+}
+
 func TestJsonCodecsTable(t *testing.T) {
 	testCodecTableOne(t, testJsonH)
 }

Godeps/_workspace/src/github.com/ugorji/go/codec/codecgen/gen.go

@@ -14,6 +14,7 @@ import (
 	"go/build"
 	"go/parser"
 	"go/token"
+	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -23,6 +24,8 @@ import (
 	"time"
 )
 
+const genCodecPkg = "codec1978" // keep this in sync with codec.genCodecPkg
+
 const genFrunMainTmpl = `//+build ignore
 
 package main
@@ -69,7 +72,7 @@ func CodecGenTempWrite{{ .RandString }}() {
 	var t{{ $index }} {{ . }}
 	typs = append(typs, reflect.TypeOf(t{{ $index }}))
 {{ end }}
-	{{ if not .CodecPkgFiles }}{{ .CodecPkgName }}.{{ end }}Gen(&out, "{{ .BuildTag }}", "{{ .PackageName }}", {{ .UseUnsafe }}, typs...)
+	{{ if not .CodecPkgFiles }}{{ .CodecPkgName }}.{{ end }}Gen(&out, "{{ .BuildTag }}", "{{ .PackageName }}", "{{ .RandString }}", {{ .UseUnsafe }}, typs...)
 	bout, err := format.Source(out.Bytes())
 	if err != nil {
 		fout.Write(out.Bytes())
@@ -89,7 +92,7 @@ func CodecGenTempWrite{{ .RandString }}() {
 // Tool then executes: "go run __frun__" which creates fout.
 // fout contains Codec(En|De)codeSelf implementations for every type T.
 //
-func Generate(outfile, buildTag, codecPkgPath string, useUnsafe bool, goRunTag string,
+func Generate(outfile, buildTag, codecPkgPath string, uid int64, useUnsafe bool, goRunTag string,
 	regexName *regexp.Regexp, deleteTempFile bool, infiles ...string) (err error) {
 	// For each file, grab AST, find each type, and write a call to it.
 	if len(infiles) == 0 {
@@ -99,6 +102,13 @@ func Generate(outfile, buildTag, codecPkgPath string, useUnsafe bool, goRunTag s
 		err = errors.New("outfile and codec package path cannot be blank")
 		return
 	}
+	if uid < 0 {
+		uid = -uid
+	}
+	if uid == 0 {
+		rr := rand.New(rand.NewSource(time.Now().UnixNano()))
+		uid = 101 + rr.Int63n(9777)
+	}
 	// We have to parse dir for package, before opening the temp file for writing (else ImportDir fails).
 	// Also, ImportDir(...) must take an absolute path.
 	lastdir := filepath.Dir(outfile)
@@ -123,12 +133,12 @@ func Generate(outfile, buildTag, codecPkgPath string, useUnsafe bool, goRunTag s
 		UseUnsafe       bool
 	}
 	tv := tmplT{
-		CodecPkgName:    "codec1978",
+		CodecPkgName:    genCodecPkg,
 		OutFile:         outfile,
 		CodecImportPath: codecPkgPath,
 		BuildTag:        buildTag,
 		UseUnsafe:       useUnsafe,
-		RandString:      strconv.FormatInt(time.Now().UnixNano(), 10),
+		RandString:      strconv.FormatInt(uid, 10),
 	}
 	tv.ImportPath = pkg.ImportPath
 	if tv.ImportPath == tv.CodecImportPath {
@@ -261,9 +271,9 @@ func main() {
 	rt := flag.String("rt", "", "tags for go run")
 	x := flag.Bool("x", false, "keep temp file")
 	u := flag.Bool("u", false, "Use unsafe, e.g. to avoid unnecessary allocation on []byte->string")
-
+	d := flag.Int64("d", 0, "random identifier for use in generated code")
 	flag.Parse()
-	if err := Generate(*o, *t, *c, *u, *rt,
+	if err := Generate(*o, *t, *c, *d, *u, *rt,
 		regexp.MustCompile(*r), !*x, flag.Args()...); err != nil {
 		fmt.Fprintf(os.Stderr, "codecgen error: %v\n", err)
 		os.Exit(1)

Godeps/_workspace/src/github.com/ugorji/go/codec/encode.go

@@ -6,7 +6,6 @@ package codec
 import (
 	"bytes"
 	"encoding"
-	"errors"
 	"fmt"
 	"io"
 	"reflect"
@@ -63,12 +62,8 @@ type encDriver interface {
 	EncodeRawExt(re *RawExt, e *Encoder)
 	EncodeExt(v interface{}, xtag uint64, ext Ext, e *Encoder)
 	EncodeArrayStart(length int)
-	EncodeArrayEnd()
-	EncodeArrayEntrySeparator()
 	EncodeMapStart(length int)
-	EncodeMapEnd()
-	EncodeMapEntrySeparator()
-	EncodeMapKVSeparator()
+	EncodeEnd()
 	EncodeString(c charEncoding, v string)
 	EncodeSymbol(v string)
 	EncodeStringBytes(c charEncoding, v []byte)
@@ -77,13 +72,13 @@ type encDriver interface {
 	//encStringRunes(c charEncoding, v []rune)
 }
 
+type encDriverAsis interface {
+	EncodeAsis(v []byte)
+}
+
 type encNoSeparator struct{}
 
-func (_ encNoSeparator) EncodeMapEnd()              {}
-func (_ encNoSeparator) EncodeArrayEnd()            {}
-func (_ encNoSeparator) EncodeArrayEntrySeparator() {}
-func (_ encNoSeparator) EncodeMapEntrySeparator()   {}
-func (_ encNoSeparator) EncodeMapKVSeparator()      {}
+func (_ encNoSeparator) EncodeEnd() {}
 
 type encStructFieldBytesV struct {
 	b []byte
@@ -113,8 +108,9 @@ type EncodeOptions struct {
 	// Canonical representation means that encoding a value will always result in the same
 	// sequence of bytes.
 	//
-	// This mostly will apply to maps. In this case, codec will do more work to encode the
-	// map keys out of band, and then sort them, before writing out the map to the stream.
+	// This only affects maps, as the iteration order for maps is random.
+	// In this case, the map keys will first be encoded into []byte, and then sorted,
+	// before writing the sorted keys and the corresponding map values to the stream.
 	Canonical bool
 
 	// AsSymbols defines what should be encoded as symbols.
@@ -249,10 +245,10 @@ func (z *bytesEncWriter) grow(n int) (oldcursor int) {
 	z.c = oldcursor + n
 	if z.c > len(z.b) {
 		if z.c > cap(z.b) {
-			// Tried using appendslice logic: (if cap < 1024, *2, else *1.25).
-			// However, it was too expensive, causing too many iterations of copy.
-			// Using bytes.Buffer model was much better (2*cap + n)
-			bs := make([]byte, 2*cap(z.b)+n)
+			// appendslice logic (if cap < 1024, *2, else *1.25): more expensive. many copy calls.
+			// bytes.Buffer model (2*cap + n): much better
+			// bs := make([]byte, 2*cap(z.b)+n)
+			bs := make([]byte, growCap(cap(z.b), 1, n))
 			copy(bs, z.b[:oldcursor])
 			z.b = bs
 		} else {
@@ -264,7 +260,7 @@ func (z *bytesEncWriter) grow(n int) (oldcursor int) {
 
 // ---------------------------------------------
 
-type encFnInfoX struct {
+type encFnInfo struct {
 	e     *Encoder
 	ti    *typeInfo
 	xfFn  Ext
@@ -272,25 +268,13 @@ type encFnInfoX struct {
 	seq   seqType
 }
 
-type encFnInfo struct {
-	// use encFnInfo as a value receiver.
-	// keep most of it less-used variables accessible via a pointer (*encFnInfoX).
-	// As sweet spot for value-receiver is 3 words, keep everything except
-	// encDriver (which everyone needs) directly accessible.
-	// ensure encFnInfoX is set for everyone who needs it i.e.
-	// rawExt, ext, builtin, (selfer|binary|text)Marshal, kSlice, kStruct, kMap, kInterface, fastpath
-
-	ee encDriver
-	*encFnInfoX
+func (f *encFnInfo) builtin(rv reflect.Value) {
+	f.e.e.EncodeBuiltin(f.ti.rtid, rv.Interface())
 }
 
-func (f encFnInfo) builtin(rv reflect.Value) {
-	f.ee.EncodeBuiltin(f.ti.rtid, rv.Interface())
-}
-
-func (f encFnInfo) rawExt(rv reflect.Value) {
+func (f *encFnInfo) rawExt(rv reflect.Value) {
 	// rev := rv.Interface().(RawExt)
-	// f.ee.EncodeRawExt(&rev, f.e)
+	// f.e.e.EncodeRawExt(&rev, f.e)
 	var re *RawExt
 	if rv.CanAddr() {
 		re = rv.Addr().Interface().(*RawExt)
@@ -298,26 +282,35 @@ func (f encFnInfo) rawExt(rv reflect.Value) {
 		rev := rv.Interface().(RawExt)
 		re = &rev
 	}
-	f.ee.EncodeRawExt(re, f.e)
+	f.e.e.EncodeRawExt(re, f.e)
 }
 
-func (f encFnInfo) ext(rv reflect.Value) {
-	// if this is a struct and it was addressable, then pass the address directly (not the value)
-	if rv.CanAddr() && rv.Kind() == reflect.Struct {
+func (f *encFnInfo) ext(rv reflect.Value) {
+	// if this is a struct|array and it was addressable, then pass the address directly (not the value)
+	if k := rv.Kind(); (k == reflect.Struct || k == reflect.Array) && rv.CanAddr() {
 		rv = rv.Addr()
 	}
-	f.ee.EncodeExt(rv.Interface(), f.xfTag, f.xfFn, f.e)
+	f.e.e.EncodeExt(rv.Interface(), f.xfTag, f.xfFn, f.e)
 }
 
-func (f encFnInfo) getValueForMarshalInterface(rv reflect.Value, indir int8) (v interface{}, proceed bool) {
+func (f *encFnInfo) getValueForMarshalInterface(rv reflect.Value, indir int8) (v interface{}, proceed bool) {
 	if indir == 0 {
 		v = rv.Interface()
 	} else if indir == -1 {
-		v = rv.Addr().Interface()
+		// If a non-pointer was passed to Encode(), then that value is not addressable.
+		// Take addr if addresable, else copy value to an addressable value.
+		if rv.CanAddr() {
+			v = rv.Addr().Interface()
+		} else {
+			rv2 := reflect.New(rv.Type())
+			rv2.Elem().Set(rv)
+			v = rv2.Interface()
+			// fmt.Printf("rv.Type: %v, rv2.Type: %v, v: %v\n", rv.Type(), rv2.Type(), v)
+		}
 	} else {
 		for j := int8(0); j < indir; j++ {
 			if rv.IsNil() {
-				f.ee.EncodeNil()
+				f.e.e.EncodeNil()
 				return
 			}
 			rv = rv.Elem()
@@ -327,74 +320,67 @@ func (f encFnInfo) getValueForMarshalInterface(rv reflect.Value, indir int8) (v
 	return v, true
 }
 
-func (f encFnInfo) selferMarshal(rv reflect.Value) {
+func (f *encFnInfo) selferMarshal(rv reflect.Value) {
 	if v, proceed := f.getValueForMarshalInterface(rv, f.ti.csIndir); proceed {
 		v.(Selfer).CodecEncodeSelf(f.e)
 	}
 }
 
-func (f encFnInfo) binaryMarshal(rv reflect.Value) {
+func (f *encFnInfo) binaryMarshal(rv reflect.Value) {
 	if v, proceed := f.getValueForMarshalInterface(rv, f.ti.bmIndir); proceed {
 		bs, fnerr := v.(encoding.BinaryMarshaler).MarshalBinary()
-		if fnerr != nil {
-			panic(fnerr)
-		}
-		if bs == nil {
-			f.ee.EncodeNil()
-		} else {
-			f.ee.EncodeStringBytes(c_RAW, bs)
-		}
+		f.e.marshal(bs, fnerr, false, c_RAW)
 	}
 }
 
-func (f encFnInfo) textMarshal(rv reflect.Value) {
+func (f *encFnInfo) textMarshal(rv reflect.Value) {
 	if v, proceed := f.getValueForMarshalInterface(rv, f.ti.tmIndir); proceed {
 		// debugf(">>>> encoding.TextMarshaler: %T", rv.Interface())
 		bs, fnerr := v.(encoding.TextMarshaler).MarshalText()
-		if fnerr != nil {
-			panic(fnerr)
-		}
-		if bs == nil {
-			f.ee.EncodeNil()
-		} else {
-			f.ee.EncodeStringBytes(c_UTF8, bs)
-		}
+		f.e.marshal(bs, fnerr, false, c_UTF8)
 	}
 }
 
-func (f encFnInfo) kBool(rv reflect.Value) {
-	f.ee.EncodeBool(rv.Bool())
+func (f *encFnInfo) jsonMarshal(rv reflect.Value) {
+	if v, proceed := f.getValueForMarshalInterface(rv, f.ti.jmIndir); proceed {
+		bs, fnerr := v.(jsonMarshaler).MarshalJSON()
+		f.e.marshal(bs, fnerr, true, c_UTF8)
+	}
+}
+
+func (f *encFnInfo) kBool(rv reflect.Value) {
+	f.e.e.EncodeBool(rv.Bool())
 }
 
-func (f encFnInfo) kString(rv reflect.Value) {
-	f.ee.EncodeString(c_UTF8, rv.String())
+func (f *encFnInfo) kString(rv reflect.Value) {
+	f.e.e.EncodeString(c_UTF8, rv.String())
 }
 
-func (f encFnInfo) kFloat64(rv reflect.Value) {
-	f.ee.EncodeFloat64(rv.Float())
+func (f *encFnInfo) kFloat64(rv reflect.Value) {
+	f.e.e.EncodeFloat64(rv.Float())
 }
 
-func (f encFnInfo) kFloat32(rv reflect.Value) {
-	f.ee.EncodeFloat32(float32(rv.Float()))
+func (f *encFnInfo) kFloat32(rv reflect.Value) {
+	f.e.e.EncodeFloat32(float32(rv.Float()))
 }
 
-func (f encFnInfo) kInt(rv reflect.Value) {
-	f.ee.EncodeInt(rv.Int())
+func (f *encFnInfo) kInt(rv reflect.Value) {
+	f.e.e.EncodeInt(rv.Int())
 }
 
-func (f encFnInfo) kUint(rv reflect.Value) {
-	f.ee.EncodeUint(rv.Uint())
+func (f *encFnInfo) kUint(rv reflect.Value) {
+	f.e.e.EncodeUint(rv.Uint())
 }
 
-func (f encFnInfo) kInvalid(rv reflect.Value) {
-	f.ee.EncodeNil()
+func (f *encFnInfo) kInvalid(rv reflect.Value) {
+	f.e.e.EncodeNil()
 }
 
-func (f encFnInfo) kErr(rv reflect.Value) {
+func (f *encFnInfo) kErr(rv reflect.Value) {
 	f.e.errorf("unsupported kind %s, for %#v", rv.Kind(), rv)
 }
 
-func (f encFnInfo) kSlice(rv reflect.Value) {
+func (f *encFnInfo) kSlice(rv reflect.Value) {
 	ti := f.ti
 	// array may be non-addressable, so we have to manage with care
 	//   (don't call rv.Bytes, rv.Slice, etc).
@@ -402,13 +388,13 @@ func (f encFnInfo) kSlice(rv reflect.Value) {
 	//   Encode(S{}) will bomb on "panic: slice of unaddressable array".
 	if f.seq != seqTypeArray {
 		if rv.IsNil() {
-			f.ee.EncodeNil()
+			f.e.e.EncodeNil()
 			return
 		}
 		// If in this method, then there was no extension function defined.
 		// So it's okay to treat as []byte.
 		if ti.rtid == uint8SliceTypId {
-			f.ee.EncodeStringBytes(c_RAW, rv.Bytes())
+			f.e.e.EncodeStringBytes(c_RAW, rv.Bytes())
 			return
 		}
 	}
@@ -417,9 +403,9 @@ func (f encFnInfo) kSlice(rv reflect.Value) {
 	if rtelem.Kind() == reflect.Uint8 {
 		switch f.seq {
 		case seqTypeArray:
-			// if l == 0 { f.ee.encodeStringBytes(c_RAW, nil) } else
+			// if l == 0 { f.e.e.encodeStringBytes(c_RAW, nil) } else
 			if rv.CanAddr() {
-				f.ee.EncodeStringBytes(c_RAW, rv.Slice(0, l).Bytes())
+				f.e.e.EncodeStringBytes(c_RAW, rv.Slice(0, l).Bytes())
 			} else {
 				var bs []byte
 				if l <= cap(f.e.b) {
@@ -432,10 +418,10 @@ func (f encFnInfo) kSlice(rv reflect.Value) {
 				// for i := 0; i < l; i++ {
 				// 	bs[i] = byte(rv.Index(i).Uint())
 				// }
-				f.ee.EncodeStringBytes(c_RAW, bs)
+				f.e.e.EncodeStringBytes(c_RAW, bs)
 			}
 		case seqTypeSlice:
-			f.ee.EncodeStringBytes(c_RAW, rv.Bytes())
+			f.e.e.EncodeStringBytes(c_RAW, rv.Bytes())
 		case seqTypeChan:
 			bs := f.e.b[:0]
 			// do not use range, so that the number of elements encoded
@@ -447,7 +433,7 @@ func (f encFnInfo) kSlice(rv reflect.Value) {
 			for i := 0; i < l; i++ {
 				bs = append(bs, <-ch)
 			}
-			f.ee.EncodeStringBytes(c_RAW, bs)
+			f.e.e.EncodeStringBytes(c_RAW, bs)
 		}
 		return
 	}
@@ -457,13 +443,12 @@ func (f encFnInfo) kSlice(rv reflect.Value) {
 			f.e.errorf("mapBySlice requires even slice length, but got %v", l)
 			return
 		}
-		f.ee.EncodeMapStart(l / 2)
+		f.e.e.EncodeMapStart(l / 2)
 	} else {
-		f.ee.EncodeArrayStart(l)
+		f.e.e.EncodeArrayStart(l)
 	}
 
 	e := f.e
-	sep := !e.be
 	if l > 0 {
 		for rtelem.Kind() == reflect.Ptr {
 			rtelem = rtelem.Elem()
@@ -471,88 +456,38 @@ func (f encFnInfo) kSlice(rv reflect.Value) {
 		// if kind is reflect.Interface, do not pre-determine the
 		// encoding type, because preEncodeValue may break it down to
 		// a concrete type and kInterface will bomb.
-		var fn encFn
+		var fn *encFn
 		if rtelem.Kind() != reflect.Interface {
 			rtelemid := reflect.ValueOf(rtelem).Pointer()
 			fn = e.getEncFn(rtelemid, rtelem, true, true)
 		}
 		// TODO: Consider perf implication of encoding odd index values as symbols if type is string
-		if sep {
-			for j := 0; j < l; j++ {
-				if j > 0 {
-					if ti.mbs {
-						if j%2 == 0 {
-							f.ee.EncodeMapEntrySeparator()
-						} else {
-							f.ee.EncodeMapKVSeparator()
-						}
-					} else {
-						f.ee.EncodeArrayEntrySeparator()
-					}
-				}
-				if f.seq == seqTypeChan {
-					if rv2, ok2 := rv.Recv(); ok2 {
-						e.encodeValue(rv2, fn)
-					}
-				} else {
-					e.encodeValue(rv.Index(j), fn)
-				}
-			}
-		} else {
-			for j := 0; j < l; j++ {
-				if f.seq == seqTypeChan {
-					if rv2, ok2 := rv.Recv(); ok2 {
-						e.encodeValue(rv2, fn)
-					}
-				} else {
-					e.encodeValue(rv.Index(j), fn)
+		for j := 0; j < l; j++ {
+			if f.seq == seqTypeChan {
+				if rv2, ok2 := rv.Recv(); ok2 {
+					e.encodeValue(rv2, fn)
 				}
+			} else {
+				e.encodeValue(rv.Index(j), fn)
 			}
 		}
-	}
 
-	if sep {
-		if ti.mbs {
-			f.ee.EncodeMapEnd()
-		} else {
-			f.ee.EncodeArrayEnd()
-		}
 	}
+
+	f.e.e.EncodeEnd()
 }
 
-func (f encFnInfo) kStruct(rv reflect.Value) {
+func (f *encFnInfo) kStruct(rv reflect.Value) {
 	fti := f.ti
 	e := f.e
 	tisfi := fti.sfip
 	toMap := !(fti.toArray || e.h.StructToArray)
 	newlen := len(fti.sfi)
+
 	// Use sync.Pool to reduce allocating slices unnecessarily.
 	// The cost of the occasional locking is less than the cost of locking.
+	pool, poolv, fkvs := encStructPoolGet(newlen)
 
-	var fkvs []encStructFieldKV
-	var pool *sync.Pool
-	var poolv interface{}
-	idxpool := newlen / 8
-	if encStructPoolLen != 4 {
-		panic(errors.New("encStructPoolLen must be equal to 4")) // defensive, in case it is changed
-	}
-	if idxpool < encStructPoolLen {
-		pool = &encStructPool[idxpool]
-		poolv = pool.Get()
-		switch vv := poolv.(type) {
-		case *[8]encStructFieldKV:
-			fkvs = vv[:newlen]
-		case *[16]encStructFieldKV:
-			fkvs = vv[:newlen]
-		case *[32]encStructFieldKV:
-			fkvs = vv[:newlen]
-		case *[64]encStructFieldKV:
-			fkvs = vv[:newlen]
-		}
-	}
-	if fkvs == nil {
-		fkvs = make([]encStructFieldKV, newlen)
-	}
 	// if toMap, use the sorted array. If toArray, use unsorted array (to match sequence in struct)
 	if toMap {
 		tisfi = fti.sfi
@@ -587,60 +522,30 @@ func (f encFnInfo) kStruct(rv reflect.Value) {
 	}
 
 	// debugf(">>>> kStruct: newlen: %v", newlen)
-	sep := !e.be
-	ee := f.ee //don't dereference everytime
-	if sep {
-		if toMap {
-			ee.EncodeMapStart(newlen)
-			// asSymbols := e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
-			asSymbols := e.h.AsSymbols == AsSymbolDefault || e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
-			for j := 0; j < newlen; j++ {
-				kv = fkvs[j]
-				if j > 0 {
-					ee.EncodeMapEntrySeparator()
-				}
-				if asSymbols {
-					ee.EncodeSymbol(kv.k)
-				} else {
-					ee.EncodeString(c_UTF8, kv.k)
-				}
-				ee.EncodeMapKVSeparator()
-				e.encodeValue(kv.v, encFn{})
-			}
-			ee.EncodeMapEnd()
-		} else {
-			ee.EncodeArrayStart(newlen)
-			for j := 0; j < newlen; j++ {
-				kv = fkvs[j]
-				if j > 0 {
-					ee.EncodeArrayEntrySeparator()
-				}
-				e.encodeValue(kv.v, encFn{})
+	// sep := !e.be
+	ee := f.e.e //don't dereference everytime
+
+	if toMap {
+		ee.EncodeMapStart(newlen)
+		// asSymbols := e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
+		asSymbols := e.h.AsSymbols == AsSymbolDefault || e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
+		for j := 0; j < newlen; j++ {
+			kv = fkvs[j]
+			if asSymbols {
+				ee.EncodeSymbol(kv.k)
+			} else {
+				ee.EncodeString(c_UTF8, kv.k)
 			}
-			ee.EncodeArrayEnd()
+			e.encodeValue(kv.v, nil)
 		}
 	} else {
-		if toMap {
-			ee.EncodeMapStart(newlen)
-			// asSymbols := e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
-			asSymbols := e.h.AsSymbols == AsSymbolDefault || e.h.AsSymbols&AsSymbolStructFieldNameFlag != 0
-			for j := 0; j < newlen; j++ {
-				kv = fkvs[j]
-				if asSymbols {
-					ee.EncodeSymbol(kv.k)
-				} else {
-					ee.EncodeString(c_UTF8, kv.k)
-				}
-				e.encodeValue(kv.v, encFn{})
-			}
-		} else {
-			ee.EncodeArrayStart(newlen)
-			for j := 0; j < newlen; j++ {
-				kv = fkvs[j]
-				e.encodeValue(kv.v, encFn{})
-			}
+		ee.EncodeArrayStart(newlen)
+		for j := 0; j < newlen; j++ {
+			kv = fkvs[j]
+			e.encodeValue(kv.v, nil)
 		}
 	}
+	ee.EncodeEnd()
 
 	// do not use defer. Instead, use explicit pool return at end of function.
 	// defer has a cost we are trying to avoid.
@@ -650,37 +555,35 @@ func (f encFnInfo) kStruct(rv reflect.Value) {
 	}
 }
 
-// func (f encFnInfo) kPtr(rv reflect.Value) {
+// func (f *encFnInfo) kPtr(rv reflect.Value) {
 // 	debugf(">>>>>>> ??? encode kPtr called - shouldn't get called")
 // 	if rv.IsNil() {
-// 		f.ee.encodeNil()
+// 		f.e.e.encodeNil()
 // 		return
 // 	}
 // 	f.e.encodeValue(rv.Elem())
 // }
 
-func (f encFnInfo) kInterface(rv reflect.Value) {
+func (f *encFnInfo) kInterface(rv reflect.Value) {
 	if rv.IsNil() {
-		f.ee.EncodeNil()
+		f.e.e.EncodeNil()
 		return
 	}
-	f.e.encodeValue(rv.Elem(), encFn{})
+	f.e.encodeValue(rv.Elem(), nil)
 }
 
-func (f encFnInfo) kMap(rv reflect.Value) {
+func (f *encFnInfo) kMap(rv reflect.Value) {
+	ee := f.e.e
 	if rv.IsNil() {
-		f.ee.EncodeNil()
+		ee.EncodeNil()
 		return
 	}
 
 	l := rv.Len()
-	f.ee.EncodeMapStart(l)
+	ee.EncodeMapStart(l)
 	e := f.e
-	sep := !e.be
 	if l == 0 {
-		if sep {
-			f.ee.EncodeMapEnd()
-		}
+		ee.EncodeEnd()
 		return
 	}
 	var asSymbols bool
@@ -691,7 +594,7 @@ func (f encFnInfo) kMap(rv reflect.Value) {
 	// However, if kind is reflect.Interface, do not pre-determine the
 	// encoding type, because preEncodeValue may break it down to
 	// a concrete type and kInterface will bomb.
-	var keyFn, valFn encFn
+	var keyFn, valFn *encFn
 	ti := f.ti
 	rtkey := ti.rt.Key()
 	rtval := ti.rt.Elem()
@@ -718,11 +621,10 @@ func (f encFnInfo) kMap(rv reflect.Value) {
 	}
 	mks := rv.MapKeys()
 	// for j, lmks := 0, len(mks); j < lmks; j++ {
-	ee := f.ee //don't dereference everytime
 	if e.h.Canonical {
 		// first encode each key to a []byte first, then sort them, then record
 		// println(">>>>>>>> CANONICAL <<<<<<<<")
-		var mksv []byte // temporary byte slice for the encoding
+		var mksv []byte = make([]byte, 0, len(mks)*16) // temporary byte slice for the encoding
 		e2 := NewEncoderBytes(&mksv, e.hh)
 		mksbv := make([]encStructFieldBytesV, len(mks))
 		for i, k := range mks {
@@ -730,35 +632,13 @@ func (f encFnInfo) kMap(rv reflect.Value) {
 			e2.MustEncode(k)
 			mksbv[i].v = k
 			mksbv[i].b = mksv[l:]
+			// fmt.Printf(">>>>> %s\n", mksv[l:])
 		}
 		sort.Sort(encStructFieldBytesVslice(mksbv))
 		for j := range mksbv {
-			if j > 0 {
-				ee.EncodeMapEntrySeparator()
-			}
-			e.w.writeb(mksbv[j].b)
-			ee.EncodeMapKVSeparator()
+			e.asis(mksbv[j].b)
 			e.encodeValue(rv.MapIndex(mksbv[j].v), valFn)
 		}
-		ee.EncodeMapEnd()
-	} else if sep {
-		for j := range mks {
-			if j > 0 {
-				ee.EncodeMapEntrySeparator()
-			}
-			if keyTypeIsString {
-				if asSymbols {
-					ee.EncodeSymbol(mks[j].String())
-				} else {
-					ee.EncodeString(c_UTF8, mks[j].String())
-				}
-			} else {
-				e.encodeValue(mks[j], keyFn)
-			}
-			ee.EncodeMapKVSeparator()
-			e.encodeValue(rv.MapIndex(mks[j]), valFn)
-		}
-		ee.EncodeMapEnd()
 	} else {
 		for j := range mks {
 			if keyTypeIsString {
@@ -773,6 +653,7 @@ func (f encFnInfo) kMap(rv reflect.Value) {
 			e.encodeValue(rv.MapIndex(mks[j]), valFn)
 		}
 	}
+	ee.EncodeEnd()
 }
 
 // --------------------------------------------------
@@ -783,12 +664,12 @@ func (f encFnInfo) kMap(rv reflect.Value) {
 // instead of executing the checks every time.
 type encFn struct {
 	i encFnInfo
-	f func(encFnInfo, reflect.Value)
+	f func(*encFnInfo, reflect.Value)
 }
 
 // --------------------------------------------------
 
-type rtidEncFn struct {
+type encRtidFn struct {
 	rtid uintptr
 	fn   encFn
 }
@@ -796,17 +677,21 @@ type rtidEncFn struct {
 // An Encoder writes an object to an output stream in the codec format.
 type Encoder struct {
 	// hopefully, reduce derefencing cost by laying the encWriter inside the Encoder
-	e  encDriver
+	e encDriver
+	// NOTE: Encoder shouldn't call it's write methods,
+	// as the handler MAY need to do some coordination.
 	w  encWriter
-	s  []rtidEncFn
+	s  []encRtidFn
 	be bool // is binary encoding
+	js bool // is json handle
 
 	wi ioEncWriter
 	wb bytesEncWriter
 	h  *BasicHandle
 
+	as encDriverAsis
 	hh Handle
-	f  map[uintptr]encFn
+	f  map[uintptr]*encFn
 	b  [scratchByteArrayLen]byte
 }
 
@@ -826,7 +711,9 @@ func NewEncoder(w io.Writer, h Handle) *Encoder {
 	}
 	e.wi.w = ww
 	e.w = &e.wi
+	_, e.js = h.(*JsonHandle)
 	e.e = h.newEncDriver(e)
+	e.as, _ = e.e.(encDriverAsis)
 	return e
 }
 
@@ -843,7 +730,9 @@ func NewEncoderBytes(out *[]byte, h Handle) *Encoder {
 	}
 	e.wb.b, e.wb.out = in, out
 	e.w = &e.wb
+	_, e.js = h.(*JsonHandle)
 	e.e = h.newEncDriver(e)
+	e.as, _ = e.e.(encDriverAsis)
 	return e
 }
 
@@ -873,8 +762,9 @@ func NewEncoderBytes(out *[]byte, h Handle) *Encoder {
 // The empty values (for omitempty option) are false, 0, any nil pointer
 // or interface value, and any array, slice, map, or string of length zero.
 //
-// Anonymous fields are encoded inline if no struct tag is present.
-// Else they are encoded as regular fields.
+// Anonymous fields are encoded inline except:
+//    - the struct tag specifies a replacement name (first value)
+//    - the field is of an interface type
 //
 // Examples:
 //
@@ -885,6 +775,9 @@ func NewEncoderBytes(out *[]byte, h Handle) *Encoder {
 //          Field2 int      `codec:"myName"`       //Use key "myName" in encode stream
 //          Field3 int32    `codec:",omitempty"`   //use key "Field3". Omit if empty.
 //          Field4 bool     `codec:"f4,omitempty"` //use key "f4". Omit if empty.
+//          io.Reader                              //use key "Reader".
+//          MyStruct        `codec:"my1"           //use key "my1".
+//          MyStruct                               //inline it
 //          ...
 //      }
 //
@@ -894,8 +787,9 @@ func NewEncoderBytes(out *[]byte, h Handle) *Encoder {
 //      }
 //
 // The mode of encoding is based on the type of the value. When a value is seen:
+//   - If a Selfer, call its CodecEncodeSelf method
 //   - If an extension is registered for it, call that extension function
-//   - If it implements BinaryMarshaler, call its MarshalBinary() (data []byte, err error)
+//   - If it implements encoding.(Binary|Text|JSON)Marshaler, call its Marshal(Binary|Text|JSON) method
 //   - Else encode it based on its reflect.Kind
 //
 // Note that struct field names and keys in map[string]XXX will be treated as symbols.
@@ -942,7 +836,7 @@ func (e *Encoder) encode(iv interface{}) {
 		v.CodecEncodeSelf(e)
 
 	case reflect.Value:
-		e.encodeValue(v, encFn{})
+		e.encodeValue(v, nil)
 
 	case string:
 		e.e.EncodeString(c_UTF8, v)
@@ -1010,12 +904,15 @@ func (e *Encoder) encode(iv interface{}) {
 
 	default:
 		// canonical mode is not supported for fastpath of maps (but is fine for slices)
+		const checkCodecSelfer1 = true // in case T is passed, where *T is a Selfer, still checkCodecSelfer
 		if e.h.Canonical {
 			if !fastpathEncodeTypeSwitchSlice(iv, e) {
-				e.encodeI(iv, false, false)
+				e.encodeI(iv, false, checkCodecSelfer1)
+			}
+		} else {
+			if !fastpathEncodeTypeSwitch(iv, e) {
+				e.encodeI(iv, false, checkCodecSelfer1)
 			}
-		} else if !fastpathEncodeTypeSwitch(iv, e) {
-			e.encodeI(iv, false, false)
 		}
 	}
 }
@@ -1025,7 +922,7 @@ func (e *Encoder) encodeI(iv interface{}, checkFastpath, checkCodecSelfer bool)
 		rt := rv.Type()
 		rtid := reflect.ValueOf(rt).Pointer()
 		fn := e.getEncFn(rtid, rt, checkFastpath, checkCodecSelfer)
-		fn.f(fn.i, rv)
+		fn.f(&fn.i, rv)
 	}
 }
 
@@ -1055,27 +952,28 @@ LOOP:
 	return rv, true
 }
 
-func (e *Encoder) encodeValue(rv reflect.Value, fn encFn) {
+func (e *Encoder) encodeValue(rv reflect.Value, fn *encFn) {
 	// if a valid fn is passed, it MUST BE for the dereferenced type of rv
 	if rv, proceed := e.preEncodeValue(rv); proceed {
-		if fn.f == nil {
+		if fn == nil {
 			rt := rv.Type()
 			rtid := reflect.ValueOf(rt).Pointer()
 			fn = e.getEncFn(rtid, rt, true, true)
 		}
-		fn.f(fn.i, rv)
+		fn.f(&fn.i, rv)
 	}
 }
 
-func (e *Encoder) getEncFn(rtid uintptr, rt reflect.Type, checkFastpath, checkCodecSelfer bool) (fn encFn) {
+func (e *Encoder) getEncFn(rtid uintptr, rt reflect.Type, checkFastpath, checkCodecSelfer bool) (fn *encFn) {
 	// rtid := reflect.ValueOf(rt).Pointer()
 	var ok bool
 	if useMapForCodecCache {
 		fn, ok = e.f[rtid]
 	} else {
-		for _, v := range e.s {
+		for i := range e.s {
+			v := &(e.s[i])
 			if v.rtid == rtid {
-				fn, ok = v.fn, true
+				fn, ok = &(v.fn), true
 				break
 			}
 		}
@@ -1083,37 +981,48 @@ func (e *Encoder) getEncFn(rtid uintptr, rt reflect.Type, checkFastpath, checkCo
 	if ok {
 		return
 	}
-	// fi.encFnInfoX = new(encFnInfoX)
+
+	if useMapForCodecCache {
+		if e.f == nil {
+			e.f = make(map[uintptr]*encFn, initCollectionCap)
+		}
+		fn = new(encFn)
+		e.f[rtid] = fn
+	} else {
+		if e.s == nil {
+			e.s = make([]encRtidFn, 0, initCollectionCap)
+		}
+		e.s = append(e.s, encRtidFn{rtid: rtid})
+		fn = &(e.s[len(e.s)-1]).fn
+	}
+
 	ti := getTypeInfo(rtid, rt)
-	var fi encFnInfo
-	fi.ee = e.e
+	fi := &(fn.i)
+	fi.e = e
+	fi.ti = ti
 
 	if checkCodecSelfer && ti.cs {
-		fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-		fn.f = (encFnInfo).selferMarshal
+		fn.f = (*encFnInfo).selferMarshal
 	} else if rtid == rawExtTypId {
-		fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-		fn.f = (encFnInfo).rawExt
+		fn.f = (*encFnInfo).rawExt
 	} else if e.e.IsBuiltinType(rtid) {
-		fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-		fn.f = (encFnInfo).builtin
+		fn.f = (*encFnInfo).builtin
 	} else if xfFn := e.h.getExt(rtid); xfFn != nil {
-		// fi.encFnInfoX = new(encFnInfoX)
-		fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
 		fi.xfTag, fi.xfFn = xfFn.tag, xfFn.ext
-		fn.f = (encFnInfo).ext
+		fn.f = (*encFnInfo).ext
 	} else if supportMarshalInterfaces && e.be && ti.bm {
-		fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-		fn.f = (encFnInfo).binaryMarshal
+		fn.f = (*encFnInfo).binaryMarshal
+	} else if supportMarshalInterfaces && !e.be && e.js && ti.jm {
+		//If JSON, we should check JSONMarshal before textMarshal
+		fn.f = (*encFnInfo).jsonMarshal
 	} else if supportMarshalInterfaces && !e.be && ti.tm {
-		fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-		fn.f = (encFnInfo).textMarshal
+		fn.f = (*encFnInfo).textMarshal
 	} else {
 		rk := rt.Kind()
-		if fastpathEnabled && checkFastpath && (rk == reflect.Map || rk == reflect.Slice) {
+		// if fastpathEnabled && checkFastpath && (rk == reflect.Map || rk == reflect.Slice) {
+		if fastpathEnabled && checkFastpath && (rk == reflect.Slice || (rk == reflect.Map && !e.h.Canonical)) {
 			if rt.PkgPath() == "" {
 				if idx := fastpathAV.index(rtid); idx != -1 {
-					fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
 					fn.f = fastpathAV[idx].encfn
 				}
 			} else {
@@ -1129,8 +1038,7 @@ func (e *Encoder) getEncFn(rtid uintptr, rt reflect.Type, checkFastpath, checkCo
 				if idx := fastpathAV.index(rtuid); idx != -1 {
 					xfnf := fastpathAV[idx].encfn
 					xrt := fastpathAV[idx].rt
-					fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-					fn.f = func(xf encFnInfo, xrv reflect.Value) {
+					fn.f = func(xf *encFnInfo, xrv reflect.Value) {
 						xfnf(xf, xrv.Convert(xrt))
 					}
 				}
@@ -1139,58 +1047,64 @@ func (e *Encoder) getEncFn(rtid uintptr, rt reflect.Type, checkFastpath, checkCo
 		if fn.f == nil {
 			switch rk {
 			case reflect.Bool:
-				fn.f = (encFnInfo).kBool
+				fn.f = (*encFnInfo).kBool
 			case reflect.String:
-				fn.f = (encFnInfo).kString
+				fn.f = (*encFnInfo).kString
 			case reflect.Float64:
-				fn.f = (encFnInfo).kFloat64
+				fn.f = (*encFnInfo).kFloat64
 			case reflect.Float32:
-				fn.f = (encFnInfo).kFloat32
+				fn.f = (*encFnInfo).kFloat32
 			case reflect.Int, reflect.Int8, reflect.Int64, reflect.Int32, reflect.Int16:
-				fn.f = (encFnInfo).kInt
-			case reflect.Uint8, reflect.Uint64, reflect.Uint, reflect.Uint32, reflect.Uint16:
-				fn.f = (encFnInfo).kUint
+				fn.f = (*encFnInfo).kInt
+			case reflect.Uint8, reflect.Uint64, reflect.Uint, reflect.Uint32, reflect.Uint16, reflect.Uintptr:
+				fn.f = (*encFnInfo).kUint
 			case reflect.Invalid:
-				fn.f = (encFnInfo).kInvalid
+				fn.f = (*encFnInfo).kInvalid
 			case reflect.Chan:
-				fi.encFnInfoX = &encFnInfoX{e: e, ti: ti, seq: seqTypeChan}
-				fn.f = (encFnInfo).kSlice
+				fi.seq = seqTypeChan
+				fn.f = (*encFnInfo).kSlice
 			case reflect.Slice:
-				fi.encFnInfoX = &encFnInfoX{e: e, ti: ti, seq: seqTypeSlice}
-				fn.f = (encFnInfo).kSlice
+				fi.seq = seqTypeSlice
+				fn.f = (*encFnInfo).kSlice
 			case reflect.Array:
-				fi.encFnInfoX = &encFnInfoX{e: e, ti: ti, seq: seqTypeArray}
-				fn.f = (encFnInfo).kSlice
+				fi.seq = seqTypeArray
+				fn.f = (*encFnInfo).kSlice
 			case reflect.Struct:
-				fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-				fn.f = (encFnInfo).kStruct
+				fn.f = (*encFnInfo).kStruct
 				// case reflect.Ptr:
-				// 	fn.f = (encFnInfo).kPtr
+				// 	fn.f = (*encFnInfo).kPtr
 			case reflect.Interface:
-				fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-				fn.f = (encFnInfo).kInterface
+				fn.f = (*encFnInfo).kInterface
 			case reflect.Map:
-				fi.encFnInfoX = &encFnInfoX{e: e, ti: ti}
-				fn.f = (encFnInfo).kMap
+				fn.f = (*encFnInfo).kMap
 			default:
-				fn.f = (encFnInfo).kErr
+				fn.f = (*encFnInfo).kErr
 			}
 		}
 	}
-	fn.i = fi
 
-	if useMapForCodecCache {
-		if e.f == nil {
-			e.f = make(map[uintptr]encFn, 32)
-		}
-		e.f[rtid] = fn
+	return
+}
+
+func (e *Encoder) marshal(bs []byte, fnerr error, asis bool, c charEncoding) {
+	if fnerr != nil {
+		panic(fnerr)
+	}
+	if bs == nil {
+		e.e.EncodeNil()
+	} else if asis {
+		e.asis(bs)
 	} else {
-		if e.s == nil {
-			e.s = make([]rtidEncFn, 0, 32)
-		}
-		e.s = append(e.s, rtidEncFn{rtid, fn})
+		e.e.EncodeStringBytes(c, bs)
+	}
+}
+
+func (e *Encoder) asis(v []byte) {
+	if e.as == nil {
+		e.w.writeb(v)
+	} else {
+		e.as.EncodeAsis(v)
 	}
-	return
 }
 
 func (e *Encoder) errorf(format string, params ...interface{}) {
@@ -1205,7 +1119,7 @@ type encStructFieldKV struct {
 	v reflect.Value
 }
 
-const encStructPoolLen = 4
+const encStructPoolLen = 5
 
 // encStructPool is an array of sync.Pool.
 // Each element of the array pools one of encStructPool(8|16|32|64).
@@ -1223,6 +1137,57 @@ func init() {
 	encStructPool[1].New = func() interface{} { return new([16]encStructFieldKV) }
 	encStructPool[2].New = func() interface{} { return new([32]encStructFieldKV) }
 	encStructPool[3].New = func() interface{} { return new([64]encStructFieldKV) }
+	encStructPool[4].New = func() interface{} { return new([128]encStructFieldKV) }
+}
+
+func encStructPoolGet(newlen int) (p *sync.Pool, v interface{}, s []encStructFieldKV) {
+	// if encStructPoolLen != 5 { // constant chec, so removed at build time.
+	// 	panic(errors.New("encStructPoolLen must be equal to 4")) // defensive, in case it is changed
+	// }
+	// idxpool := newlen / 8
+
+	// if pool == nil {
+	// 	fkvs = make([]encStructFieldKV, newlen)
+	// } else {
+	// 	poolv = pool.Get()
+	// 	switch vv := poolv.(type) {
+	// 	case *[8]encStructFieldKV:
+	// 		fkvs = vv[:newlen]
+	// 	case *[16]encStructFieldKV:
+	// 		fkvs = vv[:newlen]
+	// 	case *[32]encStructFieldKV:
+	// 		fkvs = vv[:newlen]
+	// 	case *[64]encStructFieldKV:
+	// 		fkvs = vv[:newlen]
+	// 	case *[128]encStructFieldKV:
+	// 		fkvs = vv[:newlen]
+	// 	}
+	// }
+
+	if newlen <= 8 {
+		p = &encStructPool[0]
+		v = p.Get()
+		s = v.(*[8]encStructFieldKV)[:newlen]
+	} else if newlen <= 16 {
+		p = &encStructPool[1]
+		v = p.Get()
+		s = v.(*[16]encStructFieldKV)[:newlen]
+	} else if newlen <= 32 {
+		p = &encStructPool[2]
+		v = p.Get()
+		s = v.(*[32]encStructFieldKV)[:newlen]
+	} else if newlen <= 64 {
+		p = &encStructPool[3]
+		v = p.Get()
+		s = v.(*[64]encStructFieldKV)[:newlen]
+	} else if newlen <= 128 {
+		p = &encStructPool[4]
+		v = p.Get()
+		s = v.(*[128]encStructFieldKV)[:newlen]
+	} else {
+		s = make([]encStructFieldKV, newlen)
+	}
+	return
 }
 
 // ----------------------------------------

Godeps/_workspace/src/github.com/ugorji/go/codec/fast-path.go.tmpl

@@ -48,8 +48,8 @@ var fastpathTV fastpathT
 type fastpathE struct {
 	rtid uintptr
 	rt reflect.Type 
-	encfn func(encFnInfo, reflect.Value)
-	decfn func(decFnInfo, reflect.Value)
+	encfn func(*encFnInfo, reflect.Value)
+	decfn func(*decFnInfo, reflect.Value)
 }
 
 type fastpathA [{{ .FastpathLen }}]fastpathE
@@ -85,7 +85,7 @@ func init() {
 		return
 	}
 	i := 0
-	fn := func(v interface{}, fe func(encFnInfo, reflect.Value), fd func(decFnInfo, reflect.Value)) (f fastpathE) {
+	fn := func(v interface{}, fe func(*encFnInfo, reflect.Value), fd func(*decFnInfo, reflect.Value)) (f fastpathE) {
 		xrt := reflect.TypeOf(v)
 		xptr := reflect.ValueOf(xrt).Pointer()
 		fastpathAV[i] = fastpathE{xptr, xrt, fe, fd}
@@ -93,11 +93,11 @@ func init() {
 		return
 	}
 	
-	{{range .Values}}{{if not .Primitive}}{{if .Slice }}
-	fn([]{{ .Elem }}(nil), (encFnInfo).{{ .MethodNamePfx "fastpathEnc" false }}R, (decFnInfo).{{ .MethodNamePfx "fastpathDec" false }}R){{end}}{{end}}{{end}}
+	{{range .Values}}{{if not .Primitive}}{{if not .MapKey }}
+	fn([]{{ .Elem }}(nil), (*encFnInfo).{{ .MethodNamePfx "fastpathEnc" false }}R, (*decFnInfo).{{ .MethodNamePfx "fastpathDec" false }}R){{end}}{{end}}{{end}}
 	
-	{{range .Values}}{{if not .Primitive}}{{if not .Slice }}
-	fn(map[{{ .MapKey }}]{{ .Elem }}(nil), (encFnInfo).{{ .MethodNamePfx "fastpathEnc" false }}R, (decFnInfo).{{ .MethodNamePfx "fastpathDec" false }}R){{end}}{{end}}{{end}}
+	{{range .Values}}{{if not .Primitive}}{{if .MapKey }}
+	fn(map[{{ .MapKey }}]{{ .Elem }}(nil), (*encFnInfo).{{ .MethodNamePfx "fastpathEnc" false }}R, (*decFnInfo).{{ .MethodNamePfx "fastpathDec" false }}R){{end}}{{end}}{{end}}
 	
 	sort.Sort(fastpathAslice(fastpathAV[:]))
 }
@@ -107,10 +107,10 @@ func init() {
 // -- -- fast path type switch
 func fastpathEncodeTypeSwitch(iv interface{}, e *Encoder) bool {
 	switch v := iv.(type) {
-{{range .Values}}{{if not .Primitive}}{{if .Slice }}
+{{range .Values}}{{if not .Primitive}}{{if not .MapKey }}
 	case []{{ .Elem }}:{{else}}
 	case map[{{ .MapKey }}]{{ .Elem }}:{{end}}
-		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, fastpathCheckNilTrue, e){{if .Slice }}
+		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, fastpathCheckNilTrue, e){{if not .MapKey }}
 	case *[]{{ .Elem }}:{{else}}
 	case *map[{{ .MapKey }}]{{ .Elem }}:{{end}}
 		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(*v, fastpathCheckNilTrue, e)
@@ -123,7 +123,7 @@ func fastpathEncodeTypeSwitch(iv interface{}, e *Encoder) bool {
 
 func fastpathEncodeTypeSwitchSlice(iv interface{}, e *Encoder) bool {
 	switch v := iv.(type) {
-{{range .Values}}{{if not .Primitive}}{{if .Slice }}
+{{range .Values}}{{if not .Primitive}}{{if not .MapKey }}
 	case []{{ .Elem }}:
 		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, fastpathCheckNilTrue, e)
 	case *[]{{ .Elem }}:
@@ -137,7 +137,7 @@ func fastpathEncodeTypeSwitchSlice(iv interface{}, e *Encoder) bool {
 
 func fastpathEncodeTypeSwitchMap(iv interface{}, e *Encoder) bool {
 	switch v := iv.(type) {
-{{range .Values}}{{if not .Primitive}}{{if not .Slice }}
+{{range .Values}}{{if not .Primitive}}{{if .MapKey }}
 	case map[{{ .MapKey }}]{{ .Elem }}:
 		fastpathTV.{{ .MethodNamePfx "Enc" false }}V(v, fastpathCheckNilTrue, e)
 	case *map[{{ .MapKey }}]{{ .Elem }}:
@@ -150,9 +150,9 @@ func fastpathEncodeTypeSwitchMap(iv interface{}, e *Encoder) bool {
 }
 
 // -- -- fast path functions
-{{range .Values}}{{if not .Primitive}}{{if .Slice }} 
+{{range .Values}}{{if not .Primitive}}{{if not .MapKey }} 
 
-func (f encFnInfo) {{ .MethodNamePfx "fastpathEnc" false }}R(rv reflect.Value) {
+func (f *encFnInfo) {{ .MethodNamePfx "fastpathEnc" false }}R(rv reflect.Value) {
 	fastpathTV.{{ .MethodNamePfx "Enc" false }}V(rv.Interface().([]{{ .Elem }}), fastpathCheckNilFalse, f.e)
 }
 func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v []{{ .Elem }}, checkNil bool, e *Encoder) {
@@ -162,26 +162,17 @@ func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v []{{ .Elem }}, checkNil b
 		return
 	}
 	ee.EncodeArrayStart(len(v))
-	if e.be {
-		for _, v2 := range v {
-			{{ encmd .Elem "v2"}}
-		}
-	} else {
-		for j, v2 := range v {
-			if j > 0 {
-				ee.EncodeArrayEntrySeparator()
-			}
-			{{ encmd .Elem "v2"}}
-		}
-		ee.EncodeArrayEnd()
+	for _, v2 := range v {
+		{{ encmd .Elem "v2"}}
 	}
+	ee.EncodeEnd()
 }
 
 {{end}}{{end}}{{end}}
 
-{{range .Values}}{{if not .Primitive}}{{if not .Slice }}
+{{range .Values}}{{if not .Primitive}}{{if .MapKey }}
 
-func (f encFnInfo) {{ .MethodNamePfx "fastpathEnc" false }}R(rv reflect.Value) {
+func (f *encFnInfo) {{ .MethodNamePfx "fastpathEnc" false }}R(rv reflect.Value) {
 	fastpathTV.{{ .MethodNamePfx "Enc" false }}V(rv.Interface().(map[{{ .MapKey }}]{{ .Elem }}), fastpathCheckNilFalse, f.e)
 }
 func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v map[{{ .MapKey }}]{{ .Elem }}, checkNil bool, e *Encoder) {
@@ -192,32 +183,15 @@ func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v map[{{ .MapKey }}]{{ .Ele
 	}
 	ee.EncodeMapStart(len(v))
 	{{if eq .MapKey "string"}}asSymbols := e.h.AsSymbols&AsSymbolMapStringKeysFlag != 0{{end}}
-	if e.be {
-		for k2, v2 := range v {
-			{{if eq .MapKey "string"}}if asSymbols {
-				ee.EncodeSymbol(k2)
-			} else {
-				ee.EncodeString(c_UTF8, k2)
-			}{{else}}{{ encmd .MapKey "k2"}}{{end}}
-			{{ encmd .Elem "v2"}}
-		}
-	} else {
-		j := 0
-		for k2, v2 := range v {
-			if j > 0 {
-				ee.EncodeMapEntrySeparator()
-			}
-			{{if eq .MapKey "string"}}if asSymbols {
-				ee.EncodeSymbol(k2)
-			} else {
-				ee.EncodeString(c_UTF8, k2)
-			}{{else}}{{ encmd .MapKey "k2"}}{{end}}
-			ee.EncodeMapKVSeparator()
-			{{ encmd .Elem "v2"}}
-			j++
-		}
-		ee.EncodeMapEnd()
+	for k2, v2 := range v {
+		{{if eq .MapKey "string"}}if asSymbols {
+			ee.EncodeSymbol(k2)
+		} else {
+			ee.EncodeString(c_UTF8, k2)
+		}{{else}}{{ encmd .MapKey "k2"}}{{end}}
+		{{ encmd .Elem "v2"}}
 	}
+	ee.EncodeEnd()
 }
 
 {{end}}{{end}}{{end}}
@@ -227,10 +201,10 @@ func (_ fastpathT) {{ .MethodNamePfx "Enc" false }}V(v map[{{ .MapKey }}]{{ .Ele
 // -- -- fast path type switch
 func fastpathDecodeTypeSwitch(iv interface{}, d *Decoder) bool {
 	switch v := iv.(type) {
-{{range .Values}}{{if not .Primitive}}{{if .Slice }}
+{{range .Values}}{{if not .Primitive}}{{if not .MapKey }}
 	case []{{ .Elem }}:{{else}}
 	case map[{{ .MapKey }}]{{ .Elem }}:{{end}}
-		fastpathTV.{{ .MethodNamePfx "Dec" false }}V(v, fastpathCheckNilFalse, false, d){{if .Slice }}
+		fastpathTV.{{ .MethodNamePfx "Dec" false }}V(v, fastpathCheckNilFalse, false, d){{if not .MapKey }}
 	case *[]{{ .Elem }}:{{else}}
 	case *map[{{ .MapKey }}]{{ .Elem }}:{{end}}
 		v2, changed2 := fastpathTV.{{ .MethodNamePfx "Dec" false }}V(*v, fastpathCheckNilFalse, true, d)
@@ -245,16 +219,16 @@ func fastpathDecodeTypeSwitch(iv interface{}, d *Decoder) bool {
 }
 
 // -- -- fast path functions
-{{range .Values}}{{if not .Primitive}}{{if .Slice }}
+{{range .Values}}{{if not .Primitive}}{{if not .MapKey }}
 {{/*
 Slices can change if they 
 - did not come from an array
 - are addressable (from a ptr)
 - are settable (e.g. contained in an interface{})
 */}}
-func (f decFnInfo) {{ .MethodNamePfx "fastpathDec" false }}R(rv reflect.Value) { 
+func (f *decFnInfo) {{ .MethodNamePfx "fastpathDec" false }}R(rv reflect.Value) { 
 	array := f.seq == seqTypeArray
-	if !array && rv.CanAddr() { // CanSet => CanAddr + Exported 
+	if !array && rv.CanAddr() { {{/* // CanSet => CanAddr + Exported */}}
 		vp := rv.Addr().Interface().(*[]{{ .Elem }})
 		v, changed := fastpathTV.{{ .MethodNamePfx "Dec" false }}V(*vp, fastpathCheckNilFalse, !array, f.d)
 		if changed {
@@ -275,7 +249,7 @@ func (f fastpathT) {{ .MethodNamePfx "Dec" false }}X(vp *[]{{ .Elem }}, checkNil
 func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil bool, canChange bool, 
 	d *Decoder) (_ []{{ .Elem }}, changed bool) {
 	dd := d.d
-	// if dd.isContainerType(valueTypeNil) { dd.TryDecodeAsNil()
+	{{/* // if dd.isContainerType(valueTypeNil) { dd.TryDecodeAsNil() */}}
 	if checkNil && dd.TryDecodeAsNil() {
 		if v != nil {
 			changed = true 
@@ -284,47 +258,59 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil b
 	}
 
 	slh, containerLenS := d.decSliceHelperStart()
+    x2read := containerLenS
+    var xtrunc bool 
 	if canChange && v == nil {
-		if containerLenS <= 0 {
-			v = []{{ .Elem }}{}
-		} else {
-			v = make([]{{ .Elem }}, containerLenS, containerLenS)
+		var xlen int 
+		if xlen, xtrunc = decInferLen(containerLenS, d.h.MaxInitLen, {{ .Size }}); xtrunc {
+			x2read = xlen
 		}
+		v = make([]{{ .Elem }}, xlen)
 		changed = true
-	}
+	} 
 	if containerLenS == 0 {
 		if canChange && len(v) != 0 {
 			v = v[:0]
 			changed = true 
 		}{{/*
-		// slh.End() // dd.ReadArrayEnd()
+			// slh.End() // dd.ReadArrayEnd()
 		*/}}
 		return v, changed 
 	}
 	
-	// for j := 0; j < containerLenS; j++ {
+	{{/* // for j := 0; j < containerLenS; j++ { */}}
 	if containerLenS > 0 {
-		decLen := containerLenS
 		if containerLenS > cap(v) {
-			if canChange {
-				s := make([]{{ .Elem }}, containerLenS, containerLenS)
+			if canChange { {{/*
+				// fast-path is for "basic" immutable types, so no need to copy them over
+				// s := make([]{{ .Elem }}, decInferLen(containerLenS, d.h.MaxInitLen))
 				// copy(s, v[:cap(v)])
-				v = s
+				// v = s */}}
+				var xlen int 
+                if xlen, xtrunc = decInferLen(containerLenS, d.h.MaxInitLen, {{ .Size }}); xtrunc {
+					x2read = xlen
+				}
+                v = make([]{{ .Elem }}, xlen)
 				changed = true
 			} else {
 				d.arrayCannotExpand(len(v), containerLenS)
-				decLen = len(v)
+				x2read = len(v)
 			}
 		} else if containerLenS != len(v) {
 			v = v[:containerLenS]
 			changed = true
 		}
-		// all checks done. cannot go past len.
+		{{/* // all checks done. cannot go past len. */}}
 		j := 0
-		for ; j < decLen; j++ { 
+		for ; j < x2read; j++ { 
 			{{ if eq .Elem "interface{}" }}d.decode(&v[j]){{ else }}v[j] = {{ decmd .Elem }}{{ end }}
 		}
-		if !canChange {
+		if xtrunc { {{/* // means canChange=true, changed=true already. */}}
+			for ; j < containerLenS; j++ {
+				v = append(v, {{ zerocmd .Elem }})
+				{{ if eq .Elem "interface{}" }}d.decode(&v[j]){{ else }}v[j] = {{ decmd .Elem }}{{ end }}
+			}
+		} else if !canChange {
 			for ; j < containerLenS; j++ { 
 				d.swallow()
 			}
@@ -340,10 +326,7 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil b
 					d.arrayCannotExpand(len(v), j+1)
 				}
 			} 
-			if j > 0 {
-				slh.Sep(j)
-			}
-			if j < len(v) { // all checks done. cannot go past len.
+			if j < len(v) { {{/* // all checks done. cannot go past len. */}}
 				{{ if eq .Elem "interface{}" }}d.decode(&v[j])
 				{{ else }}v[j] = {{ decmd .Elem }}{{ end }}
 			} else {
@@ -358,13 +341,13 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v []{{ .Elem }}, checkNil b
 {{end}}{{end}}{{end}}
 
 
-{{range .Values}}{{if not .Primitive}}{{if not .Slice }}
+{{range .Values}}{{if not .Primitive}}{{if .MapKey }}
 {{/*
 Maps can change if they are
 - addressable (from a ptr)
 - settable (e.g. contained in an interface{})
 */}}
-func (f decFnInfo) {{ .MethodNamePfx "fastpathDec" false }}R(rv reflect.Value) { 
+func (f *decFnInfo) {{ .MethodNamePfx "fastpathDec" false }}R(rv reflect.Value) { 
 	if rv.CanAddr() {
 		vp := rv.Addr().Interface().(*map[{{ .MapKey }}]{{ .Elem }})
 		v, changed := fastpathTV.{{ .MethodNamePfx "Dec" false }}V(*vp, fastpathCheckNilFalse, true, f.d)
@@ -385,7 +368,7 @@ func (f fastpathT) {{ .MethodNamePfx "Dec" false }}X(vp *map[{{ .MapKey }}]{{ .E
 func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Elem }}, checkNil bool, canChange bool, 
 	d *Decoder) (_ map[{{ .MapKey }}]{{ .Elem }}, changed bool) {
 	dd := d.d
-	// if dd.isContainerType(valueTypeNil) {dd.TryDecodeAsNil()
+	{{/* // if dd.isContainerType(valueTypeNil) {dd.TryDecodeAsNil() */}}
 	if checkNil && dd.TryDecodeAsNil() {
 		if v != nil {
 			changed = true
@@ -395,11 +378,8 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Ele
 
 	containerLen := dd.ReadMapStart()
 	if canChange && v == nil {
-		if containerLen > 0 {
-			v = make(map[{{ .MapKey }}]{{ .Elem }}, containerLen)
-		} else {
-			v = make(map[{{ .MapKey }}]{{ .Elem }}) // supports indefinite-length, etc
-		}
+		xlen, _ := decInferLen(containerLen, d.h.MaxInitLen, {{ .Size }})
+		v = make(map[{{ .MapKey }}]{{ .Elem }}, xlen)
 		changed = true
 	}
 	if containerLen > 0 {
@@ -407,7 +387,7 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Ele
 			{{ if eq .MapKey "interface{}" }}var mk interface{}
 			d.decode(&mk)
 			if bv, bok := mk.([]byte); bok {
-				mk = string(bv) // maps cannot have []byte as key. switch to string.
+				mk = string(bv) {{/* // maps cannot have []byte as key. switch to string. */}}
 			}{{ else }}mk := {{ decmd .MapKey }}{{ end }}
 			mv := v[mk]
 			{{ if eq .Elem "interface{}" }}d.decode(&mv)
@@ -418,15 +398,11 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Ele
 		}
 	} else if containerLen < 0 {
 		for j := 0; !dd.CheckBreak(); j++ {
-			if j > 0 {
-				dd.ReadMapEntrySeparator()
-			}
 			{{ if eq .MapKey "interface{}" }}var mk interface{}
 			d.decode(&mk)
 			if bv, bok := mk.([]byte); bok {
-				mk = string(bv) // maps cannot have []byte as key. switch to string.
+				mk = string(bv) {{/* // maps cannot have []byte as key. switch to string. */}}
 			}{{ else }}mk := {{ decmd .MapKey }}{{ end }}
-			dd.ReadMapKVSeparator()
 			mv := v[mk]
 			{{ if eq .Elem "interface{}" }}d.decode(&mv)
 			{{ else }}mv = {{ decmd .Elem }}{{ end }}
@@ -434,7 +410,7 @@ func (_ fastpathT) {{ .MethodNamePfx "Dec" false }}V(v map[{{ .MapKey }}]{{ .Ele
 				v[mk] = mv
 			}
 		}
-		dd.ReadMapEnd()
+		dd.ReadEnd()
 	}
 	return v, changed
 }

Godeps/_workspace/src/github.com/ugorji/go/codec/gen-dec-array.go.tmpl

@@ -1,15 +1,17 @@
 {{var "v"}} := {{ if not isArray}}*{{ end }}{{ .Varname }}
-{{var "h"}}, {{var "l"}} := z.DecSliceHelperStart()
+{{var "h"}}, {{var "l"}} := z.DecSliceHelperStart() {{/* // helper, containerLenS */}}
 
-var {{var "c"}} bool
-_ = {{var "c"}}
+var {{var "rr"}}, {{var "rl"}} int {{/* // num2read, length of slice/array/chan */}}
+var {{var "c"}}, {{var "rt"}} bool {{/* // changed, truncated */}}
+_, _, _ = {{var "c"}}, {{var "rt"}}, {{var "rl"}}
+{{var "rr"}} = {{var "l"}}
+{{/* rl is NOT used. Only used for getting DecInferLen. len(r) used directly in code */}}
 
 {{ if not isArray }}if {{var "v"}} == nil {
-	if {{var "l"}} <= 0 {
-        {{var "v"}} = make({{ .CTyp }}, 0)
-	} else {
-		{{var "v"}} = make({{ .CTyp }}, {{var "l"}})
+	if {{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }}); {{var "rt"}} {
+		{{var "rr"}} = {{var "rl"}}
 	}
+	{{var "v"}} = make({{ .CTyp }}, {{var "rl"}})
 	{{var "c"}} = true 
 } 
 {{ end }}
@@ -25,31 +27,38 @@ if {{var "l"}} == 0 { {{ if isSlice }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}
 		{{var "v"}} <- {{var "t"}} 
 	{{ else }} 
-	{{var "n"}} := {{var "l"}} 
 	if {{var "l"}} > cap({{var "v"}}) {
 		{{ if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "l"}})
-		{{var "n"}} = len({{var "v"}})
-		{{ else }}{{ if .Immutable }}
+		{{ else }}{{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+		{{ if .Immutable }}
 		{{var "v2"}} := {{var "v"}}
-		{{var "v"}} = make([]{{ .Typ }}, {{var "l"}}, {{var "l"}})
+		{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
 		if len({{var "v"}}) > 0 {
 			copy({{var "v"}}, {{var "v2"}}[:cap({{var "v2"}})])
 		}
-		{{ else }}{{var "v"}} = make([]{{ .Typ }}, {{var "l"}}, {{var "l"}})
+		{{ else }}{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
 		{{ end }}{{var "c"}} = true 
 		{{ end }}
+		{{var "rr"}} = len({{var "v"}})
 	} else if {{var "l"}} != len({{var "v"}}) {
 		{{ if isSlice }}{{var "v"}} = {{var "v"}}[:{{var "l"}}]
 		{{var "c"}} = true {{ end }}
 	}
 	{{var "j"}} := 0
-	for ; {{var "j"}} < {{var "n"}} ; {{var "j"}}++ {
+	for ; {{var "j"}} < {{var "rr"}} ; {{var "j"}}++ {
 		{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
-	} {{ if isArray }}
-	for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
+	}
+	{{ if isArray }}for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
 		z.DecSwallow()
-	}{{ end }}
-	{{ end }}{{/* closing if not chan */}}
+	}
+	{{ else }}if {{var "rt"}} { {{/* means that it is mutable and slice */}}
+		for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
+			{{var "v"}} = append({{var "v"}}, {{ zero}})
+			{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
+		}
+	}
+	{{ end }}
+	{{ end }}{{/* closing 'if not chan' */}}
 } else {
 	for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
 		if {{var "j"}} >= len({{var "v"}}) {
@@ -57,9 +66,6 @@ if {{var "l"}} == 0 { {{ if isSlice }}
 			{{ else if isSlice}}{{var "v"}} = append({{var "v"}}, {{zero}})// var {{var "z"}} {{ .Typ }}
 			{{var "c"}} = true {{ end }}
 		}
-		if {{var "j"}} > 0 {
-			{{var "h"}}.Sep({{var "j"}})
-		}
 		{{ if isChan}}
 		var {{var "t"}} {{ .Typ }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}

Godeps/_workspace/src/github.com/ugorji/go/codec/gen-dec-map.go.tmpl

@@ -1,11 +1,8 @@
 {{var "v"}} := *{{ .Varname }}
 {{var "l"}} := r.ReadMapStart()
 if {{var "v"}} == nil {
-	if {{var "l"}} > 0 {
-		{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}, {{var "l"}})
-	} else {
-		{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}) // supports indefinite-length, etc
-	}
+	{{var "rl"}}, _ := z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+	{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}, {{var "rl"}})
 	*{{ .Varname }} = {{var "v"}}
 }
 if {{var "l"}} > 0  {
@@ -25,9 +22,6 @@ for {{var "j"}} := 0; {{var "j"}} < {{var "l"}}; {{var "j"}}++ {
 }
 } else if {{var "l"}} < 0  {
 for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
-	if {{var "j"}} > 0 {
-		r.ReadMapEntrySeparator()
-	}
 	var {{var "mk"}} {{ .KTyp }} 
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x }}
 {{ if eq .KTyp "interface{}" }}// special case if a byte array.
@@ -35,12 +29,11 @@ for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
 		{{var "mk"}} = string({{var "bv"}})
 	}
 {{ end }}
-	r.ReadMapKVSeparator()
 	{{var "mv"}} := {{var "v"}}[{{var "mk"}}]
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ decLineVar $x }}
 	if {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
-r.ReadMapEnd()
+r.ReadEnd()
 } // else len==0: TODO: Should we clear map entries?

Godeps/_workspace/src/github.com/ugorji/go/codec/gen-helper.generated.go

@@ -10,6 +10,11 @@
 
 package codec
 
+import (
+	"encoding"
+	"reflect"
+)
+
 // This file is used to generate helper code for codecgen.
 // The values here i.e. genHelper(En|De)coder are not to be used directly by
 // library users. They WILL change continously and without notice.
@@ -60,6 +65,56 @@ func (f genHelperEncoder) EncFallback(iv interface{}) {
 	f.e.encodeI(iv, false, false)
 }
 
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncTextMarshal(iv encoding.TextMarshaler) {
+	bs, fnerr := iv.MarshalText()
+	f.e.marshal(bs, fnerr, false, c_UTF8)
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncJSONMarshal(iv jsonMarshaler) {
+	bs, fnerr := iv.MarshalJSON()
+	f.e.marshal(bs, fnerr, true, c_UTF8)
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncBinaryMarshal(iv encoding.BinaryMarshaler) {
+	bs, fnerr := iv.MarshalBinary()
+	f.e.marshal(bs, fnerr, false, c_RAW)
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) TimeRtidIfBinc() uintptr {
+	if _, ok := f.e.hh.(*BincHandle); ok {
+		return timeTypId
+	}
+	return 0
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) IsJSONHandle() bool {
+	return f.e.js
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) HasExtensions() bool {
+	return len(f.e.h.extHandle) != 0
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncExt(v interface{}) (r bool) {
+	rt := reflect.TypeOf(v)
+	if rt.Kind() == reflect.Ptr {
+		rt = rt.Elem()
+	}
+	rtid := reflect.ValueOf(rt).Pointer()
+	if xfFn := f.e.h.getExt(rtid); xfFn != nil {
+		f.e.e.EncodeExt(v, xfFn.tag, xfFn.ext, f.e)
+		return true
+	}
+	return false
+}
+
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) DecBasicHandle() *BasicHandle {
 	return f.d.h
@@ -100,3 +155,66 @@ func (f genHelperDecoder) DecStructFieldNotFound(index int, name string) {
 func (f genHelperDecoder) DecArrayCannotExpand(sliceLen, streamLen int) {
 	f.d.arrayCannotExpand(sliceLen, streamLen)
 }
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecTextUnmarshal(tm encoding.TextUnmarshaler) {
+	fnerr := tm.UnmarshalText(f.d.d.DecodeBytes(f.d.b[:], true, true))
+	if fnerr != nil {
+		panic(fnerr)
+	}
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecJSONUnmarshal(tm jsonUnmarshaler) {
+	// bs := f.dd.DecodeBytes(f.d.b[:], true, true)
+	f.d.r.track()
+	f.d.swallow()
+	bs := f.d.r.stopTrack()
+	// fmt.Printf(">>>>>> CODECGEN JSON: %s\n", bs)
+	fnerr := tm.UnmarshalJSON(bs)
+	if fnerr != nil {
+		panic(fnerr)
+	}
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecBinaryUnmarshal(bm encoding.BinaryUnmarshaler) {
+	fnerr := bm.UnmarshalBinary(f.d.d.DecodeBytes(nil, false, true))
+	if fnerr != nil {
+		panic(fnerr)
+	}
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) TimeRtidIfBinc() uintptr {
+	if _, ok := f.d.hh.(*BincHandle); ok {
+		return timeTypId
+	}
+	return 0
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) IsJSONHandle() bool {
+	return f.d.js
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) HasExtensions() bool {
+	return len(f.d.h.extHandle) != 0
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecExt(v interface{}) (r bool) {
+	rt := reflect.TypeOf(v).Elem()
+	rtid := reflect.ValueOf(rt).Pointer()
+	if xfFn := f.d.h.getExt(rtid); xfFn != nil {
+		f.d.d.DecodeExt(v, xfFn.tag, xfFn.ext)
+		return true
+	}
+	return false
+}
+
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecInferLen(clen, maxlen, unit int) (rvlen int, truncated bool) {
+	return decInferLen(clen, maxlen, unit)
+}

Godeps/_workspace/src/github.com/ugorji/go/codec/gen-helper.go.tmpl

@@ -10,6 +10,11 @@
 
 package codec
 
+import (
+	"encoding"
+	"reflect"
+)
+
 // This file is used to generate helper code for codecgen. 
 // The values here i.e. genHelper(En|De)coder are not to be used directly by 
 // library users. They WILL change continously and without notice.
@@ -48,6 +53,7 @@ type genHelperDecoder struct {
 func (f genHelperEncoder) EncBasicHandle() *BasicHandle {
 	return f.e.h
 }
+
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) EncBinary() bool {
 	return f.e.be // f.e.hh.isBinaryEncoding()
@@ -57,6 +63,49 @@ func (f genHelperEncoder) EncFallback(iv interface{}) {
 	// println(">>>>>>>>> EncFallback")
 	f.e.encodeI(iv, false, false)
 }
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncTextMarshal(iv encoding.TextMarshaler) {
+	bs, fnerr := iv.MarshalText()
+	f.e.marshal(bs, fnerr, false, c_UTF8)
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncJSONMarshal(iv jsonMarshaler) {
+	bs, fnerr := iv.MarshalJSON()
+	f.e.marshal(bs, fnerr, true, c_UTF8)
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncBinaryMarshal(iv encoding.BinaryMarshaler) {
+	bs, fnerr := iv.MarshalBinary()
+	f.e.marshal(bs, fnerr, false, c_RAW)
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) TimeRtidIfBinc() uintptr {
+	if _, ok := f.e.hh.(*BincHandle); ok {
+		return timeTypId 
+	}
+	return 0
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) IsJSONHandle() bool {
+	return f.e.js
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) HasExtensions() bool {
+	return len(f.e.h.extHandle) != 0
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperEncoder) EncExt(v interface{}) (r bool) {
+	rt := reflect.TypeOf(v)
+	if rt.Kind() == reflect.Ptr {
+		rt = rt.Elem()
+	}
+	rtid := reflect.ValueOf(rt).Pointer()
+	if xfFn := f.e.h.getExt(rtid); xfFn != nil {
+		f.e.e.EncodeExt(v, xfFn.tag, xfFn.ext, f.e)
+		return true
+	}
+	return false 
+}
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) DecBasicHandle() *BasicHandle {
@@ -91,7 +140,61 @@ func (f genHelperDecoder) DecStructFieldNotFound(index int, name string) {
 func (f genHelperDecoder) DecArrayCannotExpand(sliceLen, streamLen int) {
 	f.d.arrayCannotExpand(sliceLen, streamLen)
 }
-
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecTextUnmarshal(tm encoding.TextUnmarshaler) {
+	fnerr := tm.UnmarshalText(f.d.d.DecodeBytes(f.d.b[:], true, true))
+	if fnerr != nil {
+		panic(fnerr)
+	}
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecJSONUnmarshal(tm jsonUnmarshaler) {
+	// bs := f.dd.DecodeBytes(f.d.b[:], true, true)
+	f.d.r.track()
+	f.d.swallow()
+	bs := f.d.r.stopTrack()
+	// fmt.Printf(">>>>>> CODECGEN JSON: %s\n", bs)
+	fnerr := tm.UnmarshalJSON(bs)
+	if fnerr != nil {
+		panic(fnerr)
+	}
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecBinaryUnmarshal(bm encoding.BinaryUnmarshaler) {
+	fnerr := bm.UnmarshalBinary(f.d.d.DecodeBytes(nil, false, true))
+	if fnerr != nil {
+		panic(fnerr)
+	}
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) TimeRtidIfBinc() uintptr {
+	if _, ok := f.d.hh.(*BincHandle); ok {
+		return timeTypId 
+	}
+	return 0
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) IsJSONHandle() bool {
+	return f.d.js 
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) HasExtensions() bool {
+	return len(f.d.h.extHandle) != 0
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecExt(v interface{}) (r bool) {
+	rt := reflect.TypeOf(v).Elem()
+	rtid := reflect.ValueOf(rt).Pointer()
+	if xfFn := f.d.h.getExt(rtid); xfFn != nil {
+		f.d.d.DecodeExt(v, xfFn.tag, xfFn.ext)
+		return true
+	}
+	return false 
+}
+// FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
+func (f genHelperDecoder) DecInferLen(clen, maxlen, unit int) (rvlen int, truncated bool) {
+	return decInferLen(clen, maxlen, unit)
+}
 
 {{/*
 

Godeps/_workspace/src/github.com/ugorji/go/codec/gen.generated.go

@@ -9,11 +9,8 @@ const genDecMapTmpl = `
 {{var "v"}} := *{{ .Varname }}
 {{var "l"}} := r.ReadMapStart()
 if {{var "v"}} == nil {
-	if {{var "l"}} > 0 {
-		{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}, {{var "l"}})
-	} else {
-		{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}) // supports indefinite-length, etc
-	}
+	{{var "rl"}}, _ := z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+	{{var "v"}} = make(map[{{ .KTyp }}]{{ .Typ }}, {{var "rl"}})
 	*{{ .Varname }} = {{var "v"}}
 }
 if {{var "l"}} > 0  {
@@ -33,9 +30,6 @@ for {{var "j"}} := 0; {{var "j"}} < {{var "l"}}; {{var "j"}}++ {
 }
 } else if {{var "l"}} < 0  {
 for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
-	if {{var "j"}} > 0 {
-		r.ReadMapEntrySeparator()
-	}
 	var {{var "mk"}} {{ .KTyp }} 
 	{{ $x := printf "%vmk%v" .TempVar .Rand }}{{ decLineVarK $x }}
 {{ if eq .KTyp "interface{}" }}// special case if a byte array.
@@ -43,30 +37,31 @@ for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
 		{{var "mk"}} = string({{var "bv"}})
 	}
 {{ end }}
-	r.ReadMapKVSeparator()
 	{{var "mv"}} := {{var "v"}}[{{var "mk"}}]
 	{{ $x := printf "%vmv%v" .TempVar .Rand }}{{ decLineVar $x }}
 	if {{var "v"}} != nil {
 		{{var "v"}}[{{var "mk"}}] = {{var "mv"}}
 	}
 }
-r.ReadMapEnd()
+r.ReadEnd()
 } // else len==0: TODO: Should we clear map entries?
 `
 
 const genDecListTmpl = `
 {{var "v"}} := {{ if not isArray}}*{{ end }}{{ .Varname }}
-{{var "h"}}, {{var "l"}} := z.DecSliceHelperStart()
+{{var "h"}}, {{var "l"}} := z.DecSliceHelperStart() {{/* // helper, containerLenS */}}
 
-var {{var "c"}} bool
-_ = {{var "c"}}
+var {{var "rr"}}, {{var "rl"}} int {{/* // num2read, length of slice/array/chan */}}
+var {{var "c"}}, {{var "rt"}} bool {{/* // changed, truncated */}}
+_, _, _ = {{var "c"}}, {{var "rt"}}, {{var "rl"}}
+{{var "rr"}} = {{var "l"}}
+{{/* rl is NOT used. Only used for getting DecInferLen. len(r) used directly in code */}}
 
 {{ if not isArray }}if {{var "v"}} == nil {
-	if {{var "l"}} <= 0 {
-        {{var "v"}} = make({{ .CTyp }}, 0)
-	} else {
-		{{var "v"}} = make({{ .CTyp }}, {{var "l"}})
+	if {{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }}); {{var "rt"}} {
+		{{var "rr"}} = {{var "rl"}}
 	}
+	{{var "v"}} = make({{ .CTyp }}, {{var "rl"}})
 	{{var "c"}} = true 
 } 
 {{ end }}
@@ -82,31 +77,38 @@ if {{var "l"}} == 0 { {{ if isSlice }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}
 		{{var "v"}} <- {{var "t"}} 
 	{{ else }} 
-	{{var "n"}} := {{var "l"}} 
 	if {{var "l"}} > cap({{var "v"}}) {
 		{{ if isArray }}z.DecArrayCannotExpand(len({{var "v"}}), {{var "l"}})
-		{{var "n"}} = len({{var "v"}})
-		{{ else }}{{ if .Immutable }}
+		{{ else }}{{var "rl"}}, {{var "rt"}} = z.DecInferLen({{var "l"}}, z.DecBasicHandle().MaxInitLen, {{ .Size }})
+		{{ if .Immutable }}
 		{{var "v2"}} := {{var "v"}}
-		{{var "v"}} = make([]{{ .Typ }}, {{var "l"}}, {{var "l"}})
+		{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
 		if len({{var "v"}}) > 0 {
 			copy({{var "v"}}, {{var "v2"}}[:cap({{var "v2"}})])
 		}
-		{{ else }}{{var "v"}} = make([]{{ .Typ }}, {{var "l"}}, {{var "l"}})
+		{{ else }}{{var "v"}} = make([]{{ .Typ }}, {{var "rl"}})
 		{{ end }}{{var "c"}} = true 
 		{{ end }}
+		{{var "rr"}} = len({{var "v"}})
 	} else if {{var "l"}} != len({{var "v"}}) {
 		{{ if isSlice }}{{var "v"}} = {{var "v"}}[:{{var "l"}}]
 		{{var "c"}} = true {{ end }}
 	}
 	{{var "j"}} := 0
-	for ; {{var "j"}} < {{var "n"}} ; {{var "j"}}++ {
+	for ; {{var "j"}} < {{var "rr"}} ; {{var "j"}}++ {
 		{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
-	} {{ if isArray }}
-	for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
+	}
+	{{ if isArray }}for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
 		z.DecSwallow()
-	}{{ end }}
-	{{ end }}{{/* closing if not chan */}}
+	}
+	{{ else }}if {{var "rt"}} { {{/* means that it is mutable and slice */}}
+		for ; {{var "j"}} < {{var "l"}} ; {{var "j"}}++ {
+			{{var "v"}} = append({{var "v"}}, {{ zero}})
+			{{ $x := printf "%[1]vv%[2]v[%[1]vj%[2]v]" .TempVar .Rand }}{{ decLineVar $x }}
+		}
+	}
+	{{ end }}
+	{{ end }}{{/* closing 'if not chan' */}}
 } else {
 	for {{var "j"}} := 0; !r.CheckBreak(); {{var "j"}}++ {
 		if {{var "j"}} >= len({{var "v"}}) {
@@ -114,9 +116,6 @@ if {{var "l"}} == 0 { {{ if isSlice }}
 			{{ else if isSlice}}{{var "v"}} = append({{var "v"}}, {{zero}})// var {{var "z"}} {{ .Typ }}
 			{{var "c"}} = true {{ end }}
 		}
-		if {{var "j"}} > 0 {
-			{{var "h"}}.Sep({{var "j"}})
-		}
 		{{ if isChan}}
 		var {{var "t"}} {{ .Typ }}
 		{{ $x := printf "%st%s" .TempVar .Rand }}{{ decLineVar $x }}

Godeps/_workspace/src/github.com/ugorji/go/codec/helper.go

@@ -117,6 +117,7 @@ import (
 
 const (
 	scratchByteArrayLen = 32
+	initCollectionCap   = 32 // 32 is defensive. 16 is preferred.
 
 	// Support encoding.(Binary|Text)(Unm|M)arshaler.
 	// This constant flag will enable or disable it.
@@ -147,6 +148,12 @@ const (
 
 	// if derefForIsEmptyValue, deref pointers and interfaces when checking isEmptyValue
 	derefForIsEmptyValue = false
+
+	// if resetSliceElemToZeroValue, then on decoding a slice, reset the element to a zero value first.
+	// Only concern is that, if the slice already contained some garbage, we will decode into that garbage.
+	// The chances of this are slim, so leave this "optimization".
+	// TODO: should this be true, to ensure that we always decode into a "zero" "empty" value?
+	resetSliceElemToZeroValue bool = false
 )
 
 var oneByteArr = [1]byte{0}
@@ -186,6 +193,14 @@ const (
 
 type seqType uint8
 
+// mirror json.Marshaler and json.Unmarshaler here, so we don't import the encoding/json package
+type jsonMarshaler interface {
+	MarshalJSON() ([]byte, error)
+}
+type jsonUnmarshaler interface {
+	UnmarshalJSON([]byte) error
+}
+
 const (
 	_ seqType = iota
 	seqTypeArray
@@ -217,6 +232,9 @@ var (
 	textMarshalerTyp   = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem()
 	textUnmarshalerTyp = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem()
 
+	jsonMarshalerTyp   = reflect.TypeOf((*jsonMarshaler)(nil)).Elem()
+	jsonUnmarshalerTyp = reflect.TypeOf((*jsonUnmarshaler)(nil)).Elem()
+
 	selferTyp = reflect.TypeOf((*Selfer)(nil)).Elem()
 
 	uint8SliceTypId = reflect.ValueOf(uint8SliceTyp).Pointer()
@@ -298,32 +316,41 @@ type RawExt struct {
 	Value interface{}
 }
 
-// Ext handles custom (de)serialization of custom types / extensions.
-type Ext interface {
+// BytesExt handles custom (de)serialization of types to/from []byte.
+// It is used by codecs (e.g. binc, msgpack, simple) which do custom serialization of the types.
+type BytesExt interface {
 	// WriteExt converts a value to a []byte.
-	// It is used by codecs (e.g. binc, msgpack, simple) which do custom serialization of the types.
 	WriteExt(v interface{}) []byte
 
 	// ReadExt updates a value from a []byte.
-	// It is used by codecs (e.g. binc, msgpack, simple) which do custom serialization of the types.
 	ReadExt(dst interface{}, src []byte)
+}
 
+// InterfaceExt handles custom (de)serialization of types to/from another interface{} value.
+// The Encoder or Decoder will then handle the further (de)serialization of that known type.
+//
+// It is used by codecs (e.g. cbor, json) which use the format to do custom serialization of the types.
+type InterfaceExt interface {
 	// ConvertExt converts a value into a simpler interface for easy encoding e.g. convert time.Time to int64.
-	// It is used by codecs (e.g. cbor) which use the format to do custom serialization of the types.
 	ConvertExt(v interface{}) interface{}
 
 	// UpdateExt updates a value from a simpler interface for easy decoding e.g. convert int64 to time.Time.
-	// It is used by codecs (e.g. cbor) which use the format to do custom serialization of the types.
 	UpdateExt(dst interface{}, src interface{})
 }
 
-// bytesExt is a wrapper implementation to support former AddExt exported method.
-type bytesExt struct {
+// Ext handles custom (de)serialization of custom types / extensions.
+type Ext interface {
+	BytesExt
+	InterfaceExt
+}
+
+// addExtWrapper is a wrapper implementation to support former AddExt exported method.
+type addExtWrapper struct {
 	encFn func(reflect.Value) ([]byte, error)
 	decFn func(reflect.Value, []byte) error
 }
 
-func (x bytesExt) WriteExt(v interface{}) []byte {
+func (x addExtWrapper) WriteExt(v interface{}) []byte {
 	// fmt.Printf(">>>>>>>>>> WriteExt: %T, %v\n", v, v)
 	bs, err := x.encFn(reflect.ValueOf(v))
 	if err != nil {
@@ -332,21 +359,57 @@ func (x bytesExt) WriteExt(v interface{}) []byte {
 	return bs
 }
 
-func (x bytesExt) ReadExt(v interface{}, bs []byte) {
+func (x addExtWrapper) ReadExt(v interface{}, bs []byte) {
 	// fmt.Printf(">>>>>>>>>> ReadExt: %T, %v\n", v, v)
 	if err := x.decFn(reflect.ValueOf(v), bs); err != nil {
 		panic(err)
 	}
 }
 
-func (x bytesExt) ConvertExt(v interface{}) interface{} {
+func (x addExtWrapper) ConvertExt(v interface{}) interface{} {
 	return x.WriteExt(v)
 }
 
-func (x bytesExt) UpdateExt(dest interface{}, v interface{}) {
+func (x addExtWrapper) UpdateExt(dest interface{}, v interface{}) {
 	x.ReadExt(dest, v.([]byte))
 }
 
+type setExtWrapper struct {
+	b BytesExt
+	i InterfaceExt
+}
+
+func (x *setExtWrapper) WriteExt(v interface{}) []byte {
+	if x.b == nil {
+		panic("BytesExt.WriteExt is not supported")
+	}
+	return x.b.WriteExt(v)
+}
+
+func (x *setExtWrapper) ReadExt(v interface{}, bs []byte) {
+	if x.b == nil {
+		panic("BytesExt.WriteExt is not supported")
+
+	}
+	x.b.ReadExt(v, bs)
+}
+
+func (x *setExtWrapper) ConvertExt(v interface{}) interface{} {
+	if x.i == nil {
+		panic("InterfaceExt.ConvertExt is not supported")
+
+	}
+	return x.i.ConvertExt(v)
+}
+
+func (x *setExtWrapper) UpdateExt(dest interface{}, v interface{}) {
+	if x.i == nil {
+		panic("InterfaceExxt.UpdateExt is not supported")
+
+	}
+	x.i.UpdateExt(dest, v)
+}
+
 // type errorString string
 // func (x errorString) Error() string { return string(x) }
 
@@ -401,7 +464,7 @@ type extTypeTagFn struct {
 
 type extHandle []*extTypeTagFn
 
-// DEPRECATED: AddExt is deprecated in favor of SetExt. It exists for compatibility only.
+// DEPRECATED: Use SetBytesExt or SetInterfaceExt on the Handle instead.
 //
 // AddExt registes an encode and decode function for a reflect.Type.
 // AddExt internally calls SetExt.
@@ -413,10 +476,10 @@ func (o *extHandle) AddExt(
 	if encfn == nil || decfn == nil {
 		return o.SetExt(rt, uint64(tag), nil)
 	}
-	return o.SetExt(rt, uint64(tag), bytesExt{encfn, decfn})
+	return o.SetExt(rt, uint64(tag), addExtWrapper{encfn, decfn})
 }
 
-// SetExt registers a tag and Ext for a reflect.Type.
+// DEPRECATED: Use SetBytesExt or SetInterfaceExt on the Handle instead.
 //
 // Note that the type must be a named type, and specifically not
 // a pointer or Interface. An error is returned if that is not honored.
@@ -471,6 +534,10 @@ type structFieldInfo struct {
 	toArray   bool // if field is _struct, is the toArray set?
 }
 
+// func (si *structFieldInfo) isZero() bool {
+// 	return si.encName == "" && len(si.is) == 0 && si.i == 0 && !si.omitEmpty && !si.toArray
+// }
+
 // rv returns the field of the struct.
 // If anonymous, it returns an Invalid
 func (si *structFieldInfo) field(v reflect.Value, update bool) (rv2 reflect.Value) {
@@ -516,9 +583,9 @@ func (si *structFieldInfo) setToZeroValue(v reflect.Value) {
 }
 
 func parseStructFieldInfo(fname string, stag string) *structFieldInfo {
-	if fname == "" {
-		panic(noFieldNameToStructFieldInfoErr)
-	}
+	// if fname == "" {
+	// 	panic(noFieldNameToStructFieldInfoErr)
+	// }
 	si := structFieldInfo{
 		encName: fname,
 	}
@@ -589,6 +656,11 @@ type typeInfo struct {
 	tmIndir   int8 // number of indirections to get to textMarshaler type
 	tunmIndir int8 // number of indirections to get to textUnmarshaler type
 
+	jm        bool // base type (T or *T) is a jsonMarshaler
+	junm      bool // base type (T or *T) is a jsonUnmarshaler
+	jmIndir   int8 // number of indirections to get to jsonMarshaler type
+	junmIndir int8 // number of indirections to get to jsonUnmarshaler type
+
 	cs      bool // base type (T or *T) is a Selfer
 	csIndir int8 // number of indirections to get to Selfer type
 
@@ -664,6 +736,12 @@ func getTypeInfo(rtid uintptr, rt reflect.Type) (pti *typeInfo) {
 	if ok, indir = implementsIntf(rt, textUnmarshalerTyp); ok {
 		ti.tunm, ti.tunmIndir = true, indir
 	}
+	if ok, indir = implementsIntf(rt, jsonMarshalerTyp); ok {
+		ti.jm, ti.jmIndir = true, indir
+	}
+	if ok, indir = implementsIntf(rt, jsonUnmarshalerTyp); ok {
+		ti.junm, ti.junmIndir = true, indir
+	}
 	if ok, indir = implementsIntf(rt, selferTyp); ok {
 		ti.cs, ti.csIndir = true, indir
 	}
@@ -723,8 +801,21 @@ func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bo
 		if r1, _ := utf8.DecodeRuneInString(f.Name); r1 == utf8.RuneError || !unicode.IsUpper(r1) {
 			continue
 		}
-		// if anonymous and there is no struct tag and its a struct (or pointer to struct), inline it.
-		if f.Anonymous && stag == "" {
+		var si *structFieldInfo
+		// if anonymous and there is no struct tag (or it's blank)
+		// and its a struct (or pointer to struct), inline it.
+		var doInline bool
+		if f.Anonymous && f.Type.Kind() != reflect.Interface {
+			doInline = stag == ""
+			if !doInline {
+				si = parseStructFieldInfo("", stag)
+				doInline = si.encName == ""
+				// doInline = si.isZero()
+				// fmt.Printf(">>>> doInline for si.isZero: %s: %v\n", f.Name, doInline)
+			}
+		}
+
+		if doInline {
 			ft := f.Type
 			for ft.Kind() == reflect.Ptr {
 				ft = ft.Elem()
@@ -744,7 +835,14 @@ func rgetTypeInfo(rt reflect.Type, indexstack []int, fnameToHastag map[string]bo
 		if _, ok := fnameToHastag[f.Name]; ok {
 			continue
 		}
-		si := parseStructFieldInfo(f.Name, stag)
+		if f.Name == "" {
+			panic(noFieldNameToStructFieldInfoErr)
+		}
+		if si == nil {
+			si = parseStructFieldInfo(f.Name, stag)
+		} else if si.encName == "" {
+			si.encName = f.Name
+		}
 		// si.ikind = int(f.Type.Kind())
 		if len(indexstack) == 0 {
 			si.i = int16(j)
@@ -779,8 +877,9 @@ func panicToErr(err *error) {
 // 	panic(fmt.Errorf("%s: "+format, params2...))
 // }
 
-func isMutableKind(k reflect.Kind) (v bool) {
-	return k == reflect.Int ||
+func isImmutableKind(k reflect.Kind) (v bool) {
+	return false ||
+		k == reflect.Int ||
 		k == reflect.Int8 ||
 		k == reflect.Int16 ||
 		k == reflect.Int32 ||
@@ -790,6 +889,7 @@ func isMutableKind(k reflect.Kind) (v bool) {
 		k == reflect.Uint16 ||
 		k == reflect.Uint32 ||
 		k == reflect.Uint64 ||
+		k == reflect.Uintptr ||
 		k == reflect.Float32 ||
 		k == reflect.Float64 ||
 		k == reflect.Bool ||

Godeps/_workspace/src/github.com/ugorji/go/codec/helper_internal.go

@@ -149,3 +149,94 @@ func halfFloatToFloatBits(yy uint16) (d uint32) {
 	m = m << 13
 	return (s << 31) | (e << 23) | m
 }
+
+// GrowCap will return a new capacity for a slice, given the following:
+//   - oldCap: current capacity
+//   - unit: in-memory size of an element
+//   - num: number of elements to add
+func growCap(oldCap, unit, num int) (newCap int) {
+	// appendslice logic (if cap < 1024, *2, else *1.25):
+	//   leads to many copy calls, especially when copying bytes.
+	//   bytes.Buffer model (2*cap + n): much better for bytes.
+	// smarter way is to take the byte-size of the appended element(type) into account
+
+	// maintain 3 thresholds:
+	// t1: if cap <= t1, newcap = 2x
+	// t2: if cap <= t2, newcap = 1.75x
+	// t3: if cap <= t3, newcap = 1.5x
+	//     else          newcap = 1.25x
+	//
+	// t1, t2, t3 >= 1024 always.
+	// i.e. if unit size >= 16, then always do 2x or 1.25x (ie t1, t2, t3 are all same)
+	//
+	// With this, appending for bytes increase by:
+	//    100% up to 4K
+	//     75% up to 8K
+	//     50% up to 16K
+	//     25% beyond that
+
+	// unit can be 0 e.g. for struct{}{}; handle that appropriately
+	var t1, t2, t3 int // thresholds
+	if unit <= 1 {
+		t1, t2, t3 = 4*1024, 8*1024, 16*1024
+	} else if unit < 16 {
+		t3 = 16 / unit * 1024
+		t1 = t3 * 1 / 4
+		t2 = t3 * 2 / 4
+	} else {
+		t1, t2, t3 = 1024, 1024, 1024
+	}
+
+	var x int // temporary variable
+
+	// x is multiplier here: one of 5, 6, 7 or 8; incr of 25%, 50%, 75% or 100% respectively
+	if oldCap <= t1 { // [0,t1]
+		x = 8
+	} else if oldCap > t3 { // (t3,infinity]
+		x = 5
+	} else if oldCap <= t2 { // (t1,t2]
+		x = 7
+	} else { // (t2,t3]
+		x = 6
+	}
+	newCap = x * oldCap / 4
+
+	if num > 0 {
+		newCap += num
+	}
+
+	// ensure newCap is a multiple of 64 (if it is > 64) or 16.
+	if newCap > 64 {
+		if x = newCap % 64; x != 0 {
+			x = newCap / 64
+			newCap = 64 * (x + 1)
+		}
+	} else {
+		if x = newCap % 16; x != 0 {
+			x = newCap / 16
+			newCap = 16 * (x + 1)
+		}
+	}
+	return
+}
+
+func expandSliceValue(s reflect.Value, num int) reflect.Value {
+	if num <= 0 {
+		return s
+	}
+	l0 := s.Len()
+	l1 := l0 + num // new slice length
+	if l1 < l0 {
+		panic("ExpandSlice: slice overflow")
+	}
+	c0 := s.Cap()
+	if l1 <= c0 {
+		return s.Slice(0, l1)
+	}
+	st := s.Type()
+	c1 := growCap(c0, int(st.Elem().Size()), num)
+	s2 := reflect.MakeSlice(st, l1, c1)
+	// println("expandslicevalue: cap-old: ", c0, ", cap-new: ", c1, ", len-new: ", l1)
+	reflect.Copy(s2, s)
+	return s2
+}

Godeps/_workspace/src/github.com/ugorji/go/codec/helper_unsafe.go

@@ -26,6 +26,9 @@ type unsafeBytes struct {
 // In unsafe mode, it doesn't incur allocation and copying caused by conversion.
 // In regular safe mode, it is an allocation and copy.
 func stringView(v []byte) string {
+	if len(v) == 0 {
+		return ""
+	}
 	x := unsafeString{uintptr(unsafe.Pointer(&v[0])), len(v)}
 	return *(*string)(unsafe.Pointer(&x))
 }
@@ -34,6 +37,9 @@ func stringView(v []byte) string {
 // In unsafe mode, it doesn't incur allocation and copying caused by conversion.
 // In regular safe mode, it is an allocation and copy.
 func bytesView(v string) []byte {
+	if len(v) == 0 {
+		return zeroByteSlice
+	}
 	x := unsafeBytes{uintptr(unsafe.Pointer(&v)), len(v), len(v)}
 	return *(*[]byte)(unsafe.Pointer(&x))
 }

Godeps/_workspace/src/github.com/ugorji/go/codec/json.go

@@ -3,8 +3,9 @@
 
 package codec
 
-// This json support uses base64 encoding for bytes, because you cannot
+// By default, this json support uses base64 encoding for bytes, because you cannot
 // store and read any arbitrary string in json (only unicode).
+// However, the user can configre how to encode/decode bytes.
 //
 // This library specifically supports UTF-8 for encoding and decoding only.
 //
@@ -27,11 +28,18 @@ package codec
 //   - encode does not beautify. There is no whitespace when encoding.
 //   - rpc calls which take single integer arguments or write single numeric arguments will need care.
 
+// Top-level methods of json(End|Dec)Driver (which are implementations of (en|de)cDriver
+// MUST not call one-another.
+// They all must call sep(), and sep() MUST NOT be called more than once for each read.
+// If sep() is called and read is not done, you MUST call retryRead so separator wouldn't be read/written twice.
+
 import (
 	"bytes"
 	"encoding/base64"
 	"fmt"
+	"reflect"
 	"strconv"
+	"sync"
 	"unicode/utf16"
 	"unicode/utf8"
 )
@@ -81,20 +89,122 @@ const (
 	// jsonNumDigitsUint64Largest = 19
 )
 
+// A stack is used to keep track of where we are in the tree.
+// This is necessary, as the Handle must know whether to consume or emit a separator.
+
+type jsonStackElem struct {
+	st byte // top of stack (either '}' or ']' or 0 for map, array or neither).
+	sf bool // NOT first time in that container at top of stack
+	so bool // stack ctr odd
+	sr bool // value has NOT been read, so do not re-send separator
+}
+
+func (x *jsonStackElem) retryRead() {
+	if x != nil && !x.sr {
+		x.sr = true
+	}
+}
+
+func (x *jsonStackElem) sep() (c byte) {
+	// do not use switch, so it's a candidate for inlining.
+	// to inline effectively, this must not be called from within another method.
+	// v := j.st
+	if x == nil || x.st == 0 {
+		return
+	}
+	if x.sr {
+		x.sr = false
+		return
+	}
+	// v == '}' OR ']'
+	if x.st == '}' {
+		// put , or : depending on if even or odd respectively
+		if x.so {
+			c = ':'
+			if !x.sf {
+				x.sf = true
+			}
+		} else if x.sf {
+			c = ','
+		}
+	} else {
+		if x.sf {
+			c = ','
+		} else {
+			x.sf = true
+		}
+	}
+	x.so = !x.so
+	// Note: Anything more, and this function doesn't inline. Keep it tight.
+	// if x.sr {
+	// 	x.sr = false
+	// }
+	return
+}
+
+const jsonStackPoolArrayLen = 32
+
+// pool used to prevent constant allocation of stacks.
+var jsonStackPool = sync.Pool{
+	New: func() interface{} {
+		return new([jsonStackPoolArrayLen]jsonStackElem)
+	},
+}
+
+// jsonStack contains the stack for tracking the state of the container (branch).
+// The same data structure is used during encode and decode, as it is similar functionality.
+type jsonStack struct {
+	s  []jsonStackElem // stack for map or array end tag. map=}, array=]
+	sc *jsonStackElem  // pointer to current (top) element on the stack.
+	sp *[jsonStackPoolArrayLen]jsonStackElem
+}
+
+func (j *jsonStack) start(c byte) {
+	if j.s == nil {
+		// j.s = make([]jsonStackElem, 0, 8)
+		j.sp = jsonStackPool.Get().(*[jsonStackPoolArrayLen]jsonStackElem)
+		j.s = j.sp[:0]
+	}
+	j.s = append(j.s, jsonStackElem{st: c})
+	j.sc = &(j.s[len(j.s)-1])
+}
+
+func (j *jsonStack) end() {
+	l := len(j.s) - 1 // length of new stack after pop'ing
+	if l == 0 {
+		jsonStackPool.Put(j.sp)
+		j.s = nil
+		j.sp = nil
+		j.sc = nil
+	} else {
+		j.s = j.s[:l]
+		j.sc = &(j.s[l-1])
+	}
+	//j.sc = &(j.s[len(j.s)-1])
+}
+
 type jsonEncDriver struct {
 	e  *Encoder
 	w  encWriter
 	h  *JsonHandle
 	b  [64]byte // scratch
 	bs []byte   // scratch
+	se setExtWrapper
+	s  jsonStack
 	noBuiltInTypes
 }
 
 func (e *jsonEncDriver) EncodeNil() {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.w.writeb(jsonLiterals[9:13]) // null
 }
 
 func (e *jsonEncDriver) EncodeBool(b bool) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	if b {
 		e.w.writeb(jsonLiterals[0:4]) // true
 	} else {
@@ -103,78 +213,106 @@ func (e *jsonEncDriver) EncodeBool(b bool) {
 }
 
 func (e *jsonEncDriver) EncodeFloat32(f float32) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.w.writeb(strconv.AppendFloat(e.b[:0], float64(f), 'E', -1, 32))
 }
 
 func (e *jsonEncDriver) EncodeFloat64(f float64) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	// e.w.writestr(strconv.FormatFloat(f, 'E', -1, 64))
 	e.w.writeb(strconv.AppendFloat(e.b[:0], f, 'E', -1, 64))
 }
 
 func (e *jsonEncDriver) EncodeInt(v int64) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.w.writeb(strconv.AppendInt(e.b[:0], v, 10))
 }
 
 func (e *jsonEncDriver) EncodeUint(v uint64) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.w.writeb(strconv.AppendUint(e.b[:0], v, 10))
 }
 
 func (e *jsonEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, en *Encoder) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	if v := ext.ConvertExt(rv); v == nil {
-		e.EncodeNil()
+		e.w.writeb(jsonLiterals[9:13]) // null // e.EncodeNil()
 	} else {
+		e.s.sc.retryRead()
 		en.encode(v)
 	}
 }
 
 func (e *jsonEncDriver) EncodeRawExt(re *RawExt, en *Encoder) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	// only encodes re.Value (never re.Data)
 	if re.Value == nil {
-		e.EncodeNil()
+		e.w.writeb(jsonLiterals[9:13]) // null // e.EncodeNil()
 	} else {
+		e.s.sc.retryRead()
 		en.encode(re.Value)
 	}
 }
 
 func (e *jsonEncDriver) EncodeArrayStart(length int) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
+	e.s.start(']')
 	e.w.writen1('[')
 }
 
-func (e *jsonEncDriver) EncodeArrayEntrySeparator() {
-	e.w.writen1(',')
-}
-
-func (e *jsonEncDriver) EncodeArrayEnd() {
-	e.w.writen1(']')
-}
-
 func (e *jsonEncDriver) EncodeMapStart(length int) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
+	e.s.start('}')
 	e.w.writen1('{')
 }
 
-func (e *jsonEncDriver) EncodeMapEntrySeparator() {
-	e.w.writen1(',')
-}
-
-func (e *jsonEncDriver) EncodeMapKVSeparator() {
-	e.w.writen1(':')
-}
-
-func (e *jsonEncDriver) EncodeMapEnd() {
-	e.w.writen1('}')
+func (e *jsonEncDriver) EncodeEnd() {
+	b := e.s.sc.st
+	e.s.end()
+	e.w.writen1(b)
 }
 
 func (e *jsonEncDriver) EncodeString(c charEncoding, v string) {
 	// e.w.writestr(strconv.Quote(v))
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.quoteStr(v)
 }
 
 func (e *jsonEncDriver) EncodeSymbol(v string) {
 	// e.EncodeString(c_UTF8, v)
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	e.quoteStr(v)
 }
 
 func (e *jsonEncDriver) EncodeStringBytes(c charEncoding, v []byte) {
+	// if encoding raw bytes and RawBytesExt is configured, use it to encode
+	if c == c_RAW && e.se.i != nil {
+		e.EncodeExt(v, 0, &e.se, e.e)
+		return
+	}
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
 	if c == c_RAW {
 		slen := base64.StdEncoding.EncodedLen(len(v))
 		if e.bs == nil {
@@ -195,6 +333,13 @@ func (e *jsonEncDriver) EncodeStringBytes(c charEncoding, v []byte) {
 	}
 }
 
+func (e *jsonEncDriver) EncodeAsis(v []byte) {
+	if c := e.s.sc.sep(); c != 0 {
+		e.w.writen1(c)
+	}
+	e.w.writeb(v)
+}
+
 func (e *jsonEncDriver) quoteStr(s string) {
 	// adapted from std pkg encoding/json
 	const hex = "0123456789abcdef"
@@ -356,9 +501,14 @@ type jsonDecDriver struct {
 	ct   valueType // container type. one of unset, array or map.
 	bstr [8]byte   // scratch used for string \UXXX parsing
 	b    [64]byte  // scratch
+	b2   [64]byte
 
 	wsSkipped bool // whitespace skipped
 
+	se setExtWrapper
+
+	s jsonStack
+
 	n jsonNum
 	noBuiltInTypes
 }
@@ -402,16 +552,27 @@ func (d *jsonDecDriver) readStrIdx(fromIdx, toIdx uint8) {
 }
 
 func (d *jsonDecDriver) TryDecodeAsNil() bool {
-	b := d.skipWhitespace(true)
+	// we mustn't consume the state here, and end up trying to read separator twice.
+	// Instead, we keep track of the state and restore it if we couldn't decode as nil.
+
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
+	b := d.skipWhitespace(false)
 	if b == 'n' {
-		d.readStrIdx(9, 13) // null
+		d.readStrIdx(10, 13) // ull
 		d.ct = valueTypeNil
 		return true
 	}
+	d.r.unreadn1()
+	d.s.sc.retryRead()
 	return false
 }
 
 func (d *jsonDecDriver) DecodeBool() bool {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	b := d.skipWhitespace(false)
 	if b == 'f' {
 		d.readStrIdx(5, 9) // alse
@@ -426,35 +587,35 @@ func (d *jsonDecDriver) DecodeBool() bool {
 }
 
 func (d *jsonDecDriver) ReadMapStart() int {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
+	d.s.start('}')
 	d.expectChar('{')
 	d.ct = valueTypeMap
 	return -1
 }
 
 func (d *jsonDecDriver) ReadArrayStart() int {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
+	d.s.start(']')
 	d.expectChar('[')
 	d.ct = valueTypeArray
 	return -1
 }
-func (d *jsonDecDriver) ReadMapEnd() {
-	d.expectChar('}')
-}
-func (d *jsonDecDriver) ReadArrayEnd() {
-	d.expectChar(']')
-}
-func (d *jsonDecDriver) ReadArrayEntrySeparator() {
-	d.expectChar(',')
-}
-func (d *jsonDecDriver) ReadMapEntrySeparator() {
-	d.expectChar(',')
-}
-func (d *jsonDecDriver) ReadMapKVSeparator() {
-	d.expectChar(':')
+
+func (d *jsonDecDriver) ReadEnd() {
+	b := d.s.sc.st
+	d.s.end()
+	d.expectChar(b)
 }
+
 func (d *jsonDecDriver) expectChar(c uint8) {
 	b := d.skipWhitespace(false)
 	if b != c {
-		d.d.errorf("json: expect char %c but got char %c", c, b)
+		d.d.errorf("json: expect char '%c' but got char '%c'", c, b)
 		return
 	}
 	if jsonTrackSkipWhitespace {
@@ -462,6 +623,17 @@ func (d *jsonDecDriver) expectChar(c uint8) {
 	}
 }
 
+// func (d *jsonDecDriver) maybeChar(c uint8) {
+// 	b := d.skipWhitespace(false)
+// 	if b != c {
+// 		d.r.unreadn1()
+// 		return
+// 	}
+// 	if jsonTrackSkipWhitespace {
+// 		d.wsSkipped = false
+// 	}
+// }
+
 func (d *jsonDecDriver) IsContainerType(vt valueType) bool {
 	// check container type by checking the first char
 	if d.ct == valueTypeUnset {
@@ -635,6 +807,9 @@ LOOP:
 }
 
 func (d *jsonDecDriver) DecodeInt(bitsize uint8) (i int64) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	d.decNum(false)
 	n := &d.n
 	if n.manOverflow {
@@ -667,6 +842,9 @@ func (d *jsonDecDriver) DecodeInt(bitsize uint8) (i int64) {
 }
 
 func (d *jsonDecDriver) DecodeUint(bitsize uint8) (u uint64) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	d.decNum(false)
 	n := &d.n
 	if n.neg {
@@ -698,6 +876,9 @@ func (d *jsonDecDriver) DecodeUint(bitsize uint8) (u uint64) {
 }
 
 func (d *jsonDecDriver) DecodeFloat(chkOverflow32 bool) (f float64) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	d.decNum(true)
 	n := &d.n
 	f = n.floatVal()
@@ -709,6 +890,10 @@ func (d *jsonDecDriver) DecodeFloat(chkOverflow32 bool) (f float64) {
 }
 
 func (d *jsonDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
+	// No need to call sep here, as d.d.decode() handles it
+	// if c := d.s.sc.sep(); c != 0 {
+	// 	d.expectChar(c)
+	// }
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = xtag
@@ -722,14 +907,26 @@ func (d *jsonDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 }
 
 func (d *jsonDecDriver) DecodeBytes(bs []byte, isstring, zerocopy bool) (bsOut []byte) {
-	// zerocopy doesn't matter for json, as the bytes must be parsed.
+	// if decoding into raw bytes, and the RawBytesExt is configured, use it to decode.
+	if !isstring && d.se.i != nil {
+		bsOut = bs
+		d.DecodeExt(&bsOut, 0, &d.se)
+		return
+	}
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	bs0 := d.appendStringAsBytes(d.b[:0])
+	// if isstring, then just return the bytes, even if it is using the scratch buffer.
+	// the bytes will be converted to a string as needed.
 	if isstring {
 		return bs0
 	}
 	slen := base64.StdEncoding.DecodedLen(len(bs0))
-	if cap(bs) >= slen {
+	if slen <= cap(bs) {
 		bsOut = bs[:slen]
+	} else if zerocopy && slen <= cap(d.b2) {
+		bsOut = d.b2[:slen]
 	} else {
 		bsOut = make([]byte, slen)
 	}
@@ -745,6 +942,9 @@ func (d *jsonDecDriver) DecodeBytes(bs []byte, isstring, zerocopy bool) (bsOut [
 }
 
 func (d *jsonDecDriver) DecodeString() (s string) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	return string(d.appendStringAsBytes(d.b[:0]))
 }
 
@@ -816,6 +1016,9 @@ func (d *jsonDecDriver) jsonU4(checkSlashU bool) rune {
 }
 
 func (d *jsonDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurther bool) {
+	if c := d.s.sc.sep(); c != 0 {
+		d.expectChar(c)
+	}
 	n := d.skipWhitespace(true)
 	switch n {
 	case 'n':
@@ -837,7 +1040,7 @@ func (d *jsonDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurthe
 		decodeFurther = true
 	case '"':
 		vt = valueTypeString
-		v = d.DecodeString()
+		v = string(d.appendStringAsBytes(d.b[:0])) // same as d.DecodeString(), but skipping sep() call.
 	default: // number
 		d.decNum(true)
 		n := &d.n
@@ -878,6 +1081,9 @@ func (d *jsonDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurthe
 		}
 		// fmt.Printf("DecodeNaked: Number: %T, %v\n", v, v)
 	}
+	if decodeFurther {
+		d.s.sc.retryRead()
+	}
 	return
 }
 
@@ -887,7 +1093,8 @@ func (d *jsonDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurthe
 //
 // Json is comprehensively supported:
 //    - decodes numbers into interface{} as int, uint or float64
-//    - encodes and decodes []byte using base64 Std Encoding
+//    - configurable way to encode/decode []byte .
+//      by default, encodes and decodes []byte using base64 Std Encoding
 //    - UTF-8 support for encoding and decoding
 //
 // It has better performance than the json library in the standard library,
@@ -901,19 +1108,29 @@ func (d *jsonDecDriver) DecodeNaked() (v interface{}, vt valueType, decodeFurthe
 type JsonHandle struct {
 	BasicHandle
 	textEncodingType
+	// RawBytesExt, if configured, is used to encode and decode raw bytes in a custom way.
+	// If not configured, raw bytes are encoded to/from base64 text.
+	RawBytesExt InterfaceExt
 }
 
 func (h *JsonHandle) newEncDriver(e *Encoder) encDriver {
-	return &jsonEncDriver{e: e, w: e.w, h: h}
+	hd := jsonEncDriver{e: e, w: e.w, h: h}
+	hd.se.i = h.RawBytesExt
+	return &hd
 }
 
 func (h *JsonHandle) newDecDriver(d *Decoder) decDriver {
 	// d := jsonDecDriver{r: r.(*bytesDecReader), h: h}
 	hd := jsonDecDriver{d: d, r: d.r, h: h}
+	hd.se.i = h.RawBytesExt
 	hd.n.bytes = d.b[:]
 	return &hd
 }
 
+func (h *JsonHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{i: ext})
+}
+
 var jsonEncodeTerminate = []byte{' '}
 
 func (h *JsonHandle) rpcEncodeTerminate() []byte {

Godeps/_workspace/src/github.com/ugorji/go/codec/msgpack.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"math"
 	"net/rpc"
+	"reflect"
 )
 
 const (
@@ -536,15 +537,11 @@ func (d *msgpackDecDriver) DecodeBytes(bs []byte, isstring, zerocopy bool) (bsOu
 		d.readNextBd()
 	}
 	var clen int
-	if isstring {
-		clen = d.readContainerLen(msgpackContainerStr)
+	// ignore isstring. Expect that the bytes may be found from msgpackContainerStr or msgpackContainerBin
+	if bd := d.bd; bd == mpBin8 || bd == mpBin16 || bd == mpBin32 {
+		clen = d.readContainerLen(msgpackContainerBin)
 	} else {
-		// bytes can be decoded from msgpackContainerStr or msgpackContainerBin
-		if bd := d.bd; bd == mpBin8 || bd == mpBin16 || bd == mpBin32 {
-			clen = d.readContainerLen(msgpackContainerBin)
-		} else {
-			clen = d.readContainerLen(msgpackContainerStr)
-		}
+		clen = d.readContainerLen(msgpackContainerStr)
 	}
 	// println("DecodeBytes: clen: ", clen)
 	d.bdRead = false
@@ -617,7 +614,7 @@ func (d *msgpackDecDriver) readContainerLen(ct msgpackContainerType) (clen int)
 	} else if (ct.bFixMin & bd) == ct.bFixMin {
 		clen = int(ct.bFixMin ^ bd)
 	} else {
-		d.d.errorf("readContainerLen: %s: hex: %x, dec: %d", msgBadDesc, bd, bd)
+		d.d.errorf("readContainerLen: %s: hex: %x, decimal: %d", msgBadDesc, bd, bd)
 		return
 	}
 	d.bdRead = false
@@ -730,6 +727,10 @@ func (h *MsgpackHandle) newDecDriver(d *Decoder) decDriver {
 	return &msgpackDecDriver{d: d, r: d.r, h: h, br: d.bytes}
 }
 
+func (h *MsgpackHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
+}
+
 //--------------------------------------------------
 
 type msgpackSpecRpcCodec struct {

Godeps/_workspace/src/github.com/ugorji/go/codec/noop.go

@@ -11,6 +11,7 @@ import (
 // NoopHandle returns a no-op handle. It basically does nothing.
 // It is only useful for benchmarking, as it gives an idea of the
 // overhead from the codec framework.
+//
 // LIBRARY USERS: *** DO NOT USE ***
 func NoopHandle(slen int) *noopHandle {
 	h := noopHandle{}
@@ -40,7 +41,8 @@ type noopDrv struct {
 	i    int
 	S    []string
 	B    [][]byte
-	mk   bool      // are we about to read a map key?
+	mks  []bool    // stack. if map (true), else if array (false)
+	mk   bool      // top of stack. what container are we on? map or array?
 	ct   valueType // last request for IsContainerType.
 	cb   bool      // last response for IsContainerType.
 	rand *rand.Rand
@@ -54,21 +56,34 @@ func (h *noopDrv) newDecDriver(_ *Decoder) decDriver { return h }
 
 // --- encDriver
 
-func (h *noopDrv) EncodeBuiltin(rt uintptr, v interface{})    {}
-func (h *noopDrv) EncodeNil()                                 {}
-func (h *noopDrv) EncodeInt(i int64)                          {}
-func (h *noopDrv) EncodeUint(i uint64)                        {}
-func (h *noopDrv) EncodeBool(b bool)                          {}
-func (h *noopDrv) EncodeFloat32(f float32)                    {}
-func (h *noopDrv) EncodeFloat64(f float64)                    {}
-func (h *noopDrv) EncodeRawExt(re *RawExt, e *Encoder)        {}
-func (h *noopDrv) EncodeArrayStart(length int)                {}
-func (h *noopDrv) EncodeArrayEnd()                            {}
-func (h *noopDrv) EncodeArrayEntrySeparator()                 {}
-func (h *noopDrv) EncodeMapStart(length int)                  {}
-func (h *noopDrv) EncodeMapEnd()                              {}
-func (h *noopDrv) EncodeMapEntrySeparator()                   {}
-func (h *noopDrv) EncodeMapKVSeparator()                      {}
+// stack functions (for map and array)
+func (h *noopDrv) start(b bool) {
+	// println("start", len(h.mks)+1)
+	h.mks = append(h.mks, b)
+	h.mk = b
+}
+func (h *noopDrv) end() {
+	// println("end: ", len(h.mks)-1)
+	h.mks = h.mks[:len(h.mks)-1]
+	if len(h.mks) > 0 {
+		h.mk = h.mks[len(h.mks)-1]
+	} else {
+		h.mk = false
+	}
+}
+
+func (h *noopDrv) EncodeBuiltin(rt uintptr, v interface{}) {}
+func (h *noopDrv) EncodeNil()                              {}
+func (h *noopDrv) EncodeInt(i int64)                       {}
+func (h *noopDrv) EncodeUint(i uint64)                     {}
+func (h *noopDrv) EncodeBool(b bool)                       {}
+func (h *noopDrv) EncodeFloat32(f float32)                 {}
+func (h *noopDrv) EncodeFloat64(f float64)                 {}
+func (h *noopDrv) EncodeRawExt(re *RawExt, e *Encoder)     {}
+func (h *noopDrv) EncodeArrayStart(length int)             { h.start(true) }
+func (h *noopDrv) EncodeMapStart(length int)               { h.start(false) }
+func (h *noopDrv) EncodeEnd()                              { h.end() }
+
 func (h *noopDrv) EncodeString(c charEncoding, v string)      {}
 func (h *noopDrv) EncodeSymbol(v string)                      {}
 func (h *noopDrv) EncodeStringBytes(c charEncoding, v []byte) {}
@@ -90,15 +105,11 @@ func (h *noopDrv) DecodeString() (s string)                   { return h.S[h.m(8
 
 func (h *noopDrv) DecodeBytes(bs []byte, isstring, zerocopy bool) []byte { return h.B[h.m(len(h.B))] }
 
-func (h *noopDrv) ReadMapEnd()              { h.mk = false }
-func (h *noopDrv) ReadArrayEnd()            {}
-func (h *noopDrv) ReadArrayEntrySeparator() {}
-func (h *noopDrv) ReadMapEntrySeparator()   { h.mk = true }
-func (h *noopDrv) ReadMapKVSeparator()      { h.mk = false }
+func (h *noopDrv) ReadEnd() { h.end() }
 
 // toggle map/slice
-func (h *noopDrv) ReadMapStart() int   { h.mk = true; return h.m(10) }
-func (h *noopDrv) ReadArrayStart() int { return h.m(10) }
+func (h *noopDrv) ReadMapStart() int   { h.start(true); return h.m(10) }
+func (h *noopDrv) ReadArrayStart() int { h.start(false); return h.m(10) }
 
 func (h *noopDrv) IsContainerType(vt valueType) bool {
 	// return h.m(2) == 0

Godeps/_workspace/src/github.com/ugorji/go/codec/prebuild.sh

@@ -90,8 +90,8 @@ func fastpathEncodeTypeSwitchMap(iv interface{}, e *Encoder) bool { return false
 type fastpathE struct {
 	rtid uintptr
 	rt reflect.Type 
-	encfn func(encFnInfo, reflect.Value)
-	decfn func(decFnInfo, reflect.Value)
+	encfn func(*encFnInfo, reflect.Value)
+	decfn func(*decFnInfo, reflect.Value)
 }
 type fastpathA [0]fastpathE
 func (x fastpathA) index(rtid uintptr) int { return -1 }
@@ -142,9 +142,9 @@ _codegenerators() {
     then
         true && \
             echo "codecgen - !unsafe ... " && \
-            codecgen -rt codecgen -t 'x,codecgen,!unsafe' -o values_codecgen${zsfx} $zfin && \
+            codecgen  -rt codecgen -t 'x,codecgen,!unsafe' -o values_codecgen${zsfx} -d 1978 $zfin && \
             echo "codecgen - unsafe ... " && \
-            codecgen -u -rt codecgen -t 'x,codecgen,unsafe' -o values_codecgen_unsafe${zsfx} $zfin && \
+            codecgen  -u -rt codecgen -t 'x,codecgen,unsafe' -o values_codecgen_unsafe${zsfx} -d 1978 $zfin && \
             echo "msgp ... " && \
             msgp -tests=false -pkg=codec -o=values_msgp${zsfx} -file=$zfin && \
             echo "ffjson ... " && \

Godeps/_workspace/src/github.com/ugorji/go/codec/py_test.go

@@ -6,7 +6,8 @@
 package codec
 
 // These tests are used to verify msgpack and cbor implementations against their python libraries.
-// If you have the library installed, you can enable the tests back by removing the //+build ignore.
+// If you have the library installed, you can enable the tests back by running: go test -tags=x .
+// Look at test.py for how to setup your environment.
 
 import (
 	"testing"

Godeps/_workspace/src/github.com/ugorji/go/codec/simple.go

@@ -3,7 +3,10 @@
 
 package codec
 
-import "math"
+import (
+	"math"
+	"reflect"
+)
 
 const (
 	_               uint8 = iota
@@ -501,5 +504,9 @@ func (h *SimpleHandle) newDecDriver(d *Decoder) decDriver {
 	return &simpleDecDriver{d: d, r: d.r, h: h, br: d.bytes}
 }
 
+func (h *SimpleHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
+	return h.SetExt(rt, tag, &setExtWrapper{b: ext})
+}
+
 var _ decDriver = (*simpleDecDriver)(nil)
 var _ encDriver = (*simpleEncDriver)(nil)

Godeps/_workspace/src/github.com/ugorji/go/codec/test.py

@@ -5,8 +5,9 @@
 # So it can process them (so we don't have to checkin the files).
 
 # Ensure msgpack-python and cbor are installed first, using:
-#   pip install --user msgpack-python
-#   pip install --user cbor
+#   sudo apt-get install python-dev
+#   sudo apt-get install python-pip
+#   pip install --user msgpack-python msgpack-rpc-python cbor
 
 import cbor, msgpack, msgpackrpc, sys, os, threading
 

client/keys.go

@@ -14,7 +14,7 @@
 
 package client
 
-//go:generate codecgen -r "Node|Response|Nodes" -o keys.generated.go keys.go
+//go:generate codecgen -d 1819 -r "Node|Response|Nodes" -o keys.generated.go keys.go
 
 import (
 	"encoding/json"