codec: improve performance 5-20%, and support finalized msgpack spec.

Performance improvement came from using integer keys in maps where reflect.Type
keys were previously used. Gave 5-20% improvement as this was called at every
iteration of recursive function (encode/decode).

Also, support newly finalized msgpack spec, with support for extensions,
Str8, etc. Full binary compatibility flags exist as before (RawToString, WriteExt).

Also minor clean up (comments, docs, etc).

codec/binc.go

@@ -87,7 +87,7 @@ type bincDecDriver struct {
 	vd     byte
 	vs     byte
 	b      [8]byte
-	m      map[uint16]string // symbols (TODO: maybe use uint32 as key, as map optimizes for it)
+	m      map[uint32]string // symbols (use uint32 as key, as map optimizes for it)
 }
 
 func (_ *BincHandle) newEncDriver(w encWriter) encDriver {
@@ -102,9 +102,9 @@ func (_ *BincHandle) writeExt() bool {
 	return true
 }
 
-func (e *bincEncDriver) encodeBuiltinType(rt reflect.Type, rv reflect.Value) bool {
+func (e *bincEncDriver) encodeBuiltinType(rt uintptr, rv reflect.Value) bool {
 	switch rt {
-	case timeTyp:
+	case timeTypId:
 		bs := encodeTime(rv.Interface().(time.Time))
 		e.w.writen1(bincVdTimestamp<<4 | uint8(len(bs)))
 		e.w.writeb(bs)
@@ -259,6 +259,8 @@ func (e *bincEncDriver) encodeSymbol(v string) {
 			e.w.writeUint16(ui)
 		}
 	} else {
+		//e.s++
+		//ui = uint16(e.s)
 		ui = uint16(atomic.AddUint32(&e.s, 1))
 		e.m[v] = ui
 		var lenprec uint8
@@ -353,9 +355,9 @@ func (d *bincDecDriver) currentIsNil() bool {
 	return false
 }
 
-func (d *bincDecDriver) decodeBuiltinType(rt reflect.Type, rv reflect.Value) bool {
+func (d *bincDecDriver) decodeBuiltinType(rt uintptr, rv reflect.Value) bool {
 	switch rt {
-	case timeTyp:
+	case timeTypId:
 		if d.vd != bincVdTimestamp {
 			decErr("Invalid d.vd. Expecting 0x%x. Received: 0x%x", bincVdTimestamp, d.vd)
 		}
@@ -617,16 +619,16 @@ func (d *bincDecDriver) decodeString() (s string) {
 		//extract symbol
 		//if containsStringVal, read it and put in map
 		//else look in map for string value
-		var symbol uint16
+		var symbol uint32
 		vs := d.vs
 		//fmt.Printf(">>>> d.vs: 0b%b, & 0x8: %v, & 0x4: %v\n", d.vs, vs & 0x8, vs & 0x4)
 		if vs&0x8 == 0 {
-			symbol = uint16(d.r.readn1())
+			symbol = uint32(d.r.readn1())
 		} else {
-			symbol = d.r.readUint16()
+			symbol = uint32(d.r.readUint16())
 		}
 		if d.m == nil {
-			d.m = make(map[uint16]string, 16)
+			d.m = make(map[uint32]string, 16)
 		}
 
 		if vs&0x4 == 0 {
@@ -751,15 +753,11 @@ func (d *bincDecDriver) decodeNaked(h decodeHandleI) (rv reflect.Value, ctx deco
 		l := d.decLen()
 		xtag := d.r.readn1()
 		opts := h.(*BincHandle)
-		rt, bfn := opts.getDecodeExtForTag(xtag)
-		if rt == nil {
+		var bfn func(reflect.Value, []byte) error
+		rv, bfn = opts.getDecodeExtForTag(xtag)
+		if bfn == nil {
 			decErr("Unable to find type mapped to extension tag: %v", xtag)
 		}
-		if rt.Kind() == reflect.Ptr {
-			rv = reflect.New(rt.Elem())
-		} else {
-			rv = reflect.New(rt).Elem()
-		}
 		if fnerr := bfn(rv, d.r.readn(l)); fnerr != nil {
 			panic(fnerr)
 		}

codec/decode.go

@@ -37,7 +37,7 @@ type decReader interface {
 type decDriver interface {
 	initReadNext()
 	currentIsNil() bool
-	decodeBuiltinType(rt reflect.Type, rv reflect.Value) bool
+	decodeBuiltinType(rt uintptr, rv reflect.Value) bool
 	//decodeNaked should completely handle extensions, builtins, primitives, etc.
 	//Numbers are decoded as int64, uint64, float64 only (no smaller sized number types).
 	decodeNaked(h decodeHandleI) (rv reflect.Value, ctx decodeNakedContext)
@@ -80,19 +80,20 @@ type decExtTagFn struct {
 }
 
 type decExtTypeTagFn struct {
+	rtid uintptr
 	rt reflect.Type
 	decExtTagFn
 }
 
 type decodeHandleI interface {
-	getDecodeExt(rt reflect.Type) (tag byte, fn func(reflect.Value, []byte) error)
+	getDecodeExt(rt uintptr) (tag byte, fn func(reflect.Value, []byte) error)
 	errorIfNoField() bool
 }
 
 type decHandle struct {
 	// put word-aligned fields first (before bools, etc)
 	exts     []decExtTypeTagFn
-	extFuncs map[reflect.Type]decExtTagFn
+	extFuncs map[uintptr]decExtTagFn
 	// if an extension for byte slice is defined, then always decode Raw as strings
 	rawToStringOverride bool
 }
@@ -116,51 +117,54 @@ func (o *DecodeOptions) errorIfNoField() bool {
 // addDecodeExt registers a function to handle decoding into a given type when an
 // extension type and specific tag byte is detected in the codec stream.
 // To remove an extension, pass fn=nil.
-func (o *decHandle) addDecodeExt(rt reflect.Type, tag byte, fn func(reflect.Value, []byte) error) {
+func (o *decHandle) addDecodeExt(rtid uintptr, rt reflect.Type, tag byte, fn func(reflect.Value, []byte) error) {
 	if o.exts == nil {
 		o.exts = make([]decExtTypeTagFn, 0, 2)
-		o.extFuncs = make(map[reflect.Type]decExtTagFn, 2)
-	}
-	if _, ok := o.extFuncs[rt]; ok {
-		delete(o.extFuncs, rt)
-		if rt == byteSliceTyp {
-			o.rawToStringOverride = false
+		o.extFuncs = make(map[uintptr]decExtTagFn, 2)
+	} else {
+		if _, ok := o.extFuncs[rtid]; ok {
+			delete(o.extFuncs, rtid)
+			for i := 0; i < len(o.exts); i++ {
+				if o.exts[i].rtid == rtid {
+					o.exts = append(o.exts[:i], o.exts[i+1:]...)
+					break
+				}
+			}
+			if rtid == byteSliceTypId {
+				o.rawToStringOverride = false
+			}
 		}
 	}
 	if fn != nil {
-		o.extFuncs[rt] = decExtTagFn{fn, tag}
-		if rt == byteSliceTyp {
+		o.extFuncs[rtid] = decExtTagFn{fn, tag}
+		o.exts = append(o.exts, decExtTypeTagFn{rtid, rt, decExtTagFn{fn, tag}})
+		if rtid == byteSliceTypId {
 			o.rawToStringOverride = true
 		}
 	}
-
-	if leno := len(o.extFuncs); leno > cap(o.exts) {
-		o.exts = make([]decExtTypeTagFn, leno, (leno * 3 / 2))
-	} else {
-		o.exts = o.exts[0:leno]
-	}
-	var i int
-	for k, v := range o.extFuncs {
-		o.exts[i] = decExtTypeTagFn{k, v}
-		i++
-	}
 }
 
-func (o *decHandle) getDecodeExtForTag(tag byte) (rt reflect.Type, fn func(reflect.Value, []byte) error) {
+func (o *decHandle) getDecodeExtForTag(tag byte) (rv reflect.Value, fn func(reflect.Value, []byte) error) {
 	for i, l := 0, len(o.exts); i < l; i++ {
 		if o.exts[i].tag == tag {
-			return o.exts[i].rt, o.exts[i].fn
+			if o.exts[i].rt.Kind() == reflect.Ptr {
+				rv = reflect.New(o.exts[i].rt.Elem())
+			} else {
+				rv = reflect.New(o.exts[i].rt).Elem()
+			}
+			fn = o.exts[i].fn
+			return 
 		}
 	}
 	return
 }
 
-func (o *decHandle) getDecodeExt(rt reflect.Type) (tag byte, fn func(reflect.Value, []byte) error) {
+func (o *decHandle) getDecodeExt(rt uintptr) (tag byte, fn func(reflect.Value, []byte) error) {
 	if l := len(o.exts); l == 0 {
 		return
 	} else if l < mapAccessThreshold {
 		for i := 0; i < l; i++ {
-			if o.exts[i].rt == rt {
+			if o.exts[i].rtid == rt {
 				x := o.exts[i].decExtTagFn
 				return x.tag, x.fn
 			}
@@ -320,6 +324,8 @@ func (d *Decoder) decodeValue(rv reflect.Value) {
 		return
 	}
 
+	rtid := reflect.ValueOf(rt).Pointer()
+	
 	// An extension can be registered for any type, regardless of the Kind
 	// (e.g. type BitSet int64, type MyStruct { / * unexported fields * / }, type X []int, etc.
 	//
@@ -329,10 +335,11 @@ func (d *Decoder) decodeValue(rv reflect.Value) {
 	//
 	// If we are checking for builtin or ext type here, it means we didn't go through decodeNaked,
 	// Because decodeNaked would have handled it. It also means wasNilIntf = false.
-	if dd.decodeBuiltinType(rt, rv) {
+	if dd.decodeBuiltinType(rtid, rv) {
 		return
 	}
-	if bfnTag, bfnFn := d.h.getDecodeExt(rt); bfnFn != nil {
+	
+	if bfnTag, bfnFn := d.h.getDecodeExt(rtid); bfnFn != nil {
 		xbs := dd.decodeExt(bfnTag)
 		if fnerr := bfnFn(rv, xbs); fnerr != nil {
 			panic(fnerr)
@@ -340,6 +347,9 @@ func (d *Decoder) decodeValue(rv reflect.Value) {
 		return
 	}
 
+	// TODO: Decode if type is an encoding.BinaryUnmarshaler: UnmarshalBinary(data []byte) error
+	// There is a cost, as we need to change the rv to an interface{} first.
+	
 	// NOTE: if decoding into a nil interface{}, we return a non-nil
 	// value except even if the container registers a length of 0.
 	//
@@ -396,7 +406,7 @@ func (d *Decoder) decodeValue(rv reflect.Value) {
 			break
 		}
 
-		sfi := getStructFieldInfos(rt)
+		sfi := getStructFieldInfos(rtid, rt)
 		for j := 0; j < containerLen; j++ {
 			// var rvkencname string
 			// ddecode(&rvkencname)

codec/encode.go

@@ -31,7 +31,7 @@ type encWriter interface {
 }
 
 type encDriver interface {
-	encodeBuiltinType(rt reflect.Type, rv reflect.Value) bool
+	encodeBuiltinType(rt uintptr, rv reflect.Value) bool
 	encodeNil()
 	encodeInt(i int64)
 	encodeUint(i uint64)
@@ -50,7 +50,7 @@ type encDriver interface {
 }
 
 type encodeHandleI interface {
-	getEncodeExt(rt reflect.Type) (tag byte, fn func(reflect.Value) ([]byte, error))
+	getEncodeExt(rt uintptr) (tag byte, fn func(reflect.Value) ([]byte, error))
 	writeExt() bool
 }
 
@@ -97,13 +97,13 @@ type encExtTagFn struct {
 }
 
 type encExtTypeTagFn struct {
-	rt reflect.Type
+	rt uintptr
 	encExtTagFn
 }
 
 // EncoderOptions contain options for the encoder, e.g. registered extension functions.
 type encHandle struct {
-	extFuncs map[reflect.Type]encExtTagFn
+	extFuncs map[uintptr]encExtTagFn
 	exts     []encExtTypeTagFn
 }
 
@@ -130,31 +130,28 @@ func (o *simpleIoEncWriterWriter) Write(p []byte) (n int, err error) {
 // addEncodeExt registers a function to handle encoding a given type as an extension
 // with a specific specific tag byte.
 // To remove an extension, pass fn=nil.
-func (o *encHandle) addEncodeExt(rt reflect.Type, tag byte, fn func(reflect.Value) ([]byte, error)) {
+func (o *encHandle) addEncodeExt(rt uintptr, tag byte, fn func(reflect.Value) ([]byte, error)) {
 	if o.exts == nil {
 		o.exts = make([]encExtTypeTagFn, 0, 8)
-		o.extFuncs = make(map[reflect.Type]encExtTagFn, 8)
+		o.extFuncs = make(map[uintptr]encExtTagFn, 8)
+	} else {
+		if _, ok := o.extFuncs[rt]; ok {
+			delete(o.extFuncs, rt)
+			for i := 0; i < len(o.exts); i++ {
+				if o.exts[i].rt == rt {
+					o.exts = append(o.exts[:i], o.exts[i+1:]...)
+					break
+				}
+			}
+		}
 	}
-	delete(o.extFuncs, rt)
-
 	if fn != nil {
 		o.extFuncs[rt] = encExtTagFn{fn, tag}
-	}
-	if leno := len(o.extFuncs); leno > cap(o.exts) {
-		o.exts = make([]encExtTypeTagFn, leno, (leno * 3 / 2))
-	} else {
-		o.exts = o.exts[0:leno]
-	}
-	var i int
-	for k, v := range o.extFuncs {
-		o.exts[i] = encExtTypeTagFn{k, v}
-		i++
+		o.exts = append(o.exts, encExtTypeTagFn{rt, encExtTagFn{fn, tag}})
 	}
 }
 
-func (o *encHandle) getEncodeExt(rt reflect.Type) (tag byte, fn func(reflect.Value) ([]byte, error)) {
-	// For >= 5 elements, map constant cost less than iteration cost.
-	// This is because reflect.Type equality cost is pretty high
+func (o *encHandle) getEncodeExt(rt uintptr) (tag byte, fn func(reflect.Value) ([]byte, error)) {
 	if l := len(o.exts); l == 0 {
 		return
 	} else if l < mapAccessThreshold {
@@ -323,14 +320,16 @@ func (e *Encoder) encode(iv interface{}) {
 
 func (e *Encoder) encodeValue(rv reflect.Value) {
 	rt := rv.Type()
+	rtid := reflect.ValueOf(rt).Pointer()
+	
 	//encode based on type first, since over-rides are based on type.
 	ee := e.e //don't dereference everytime
-	if ee.encodeBuiltinType(rt, rv) {
+	if ee.encodeBuiltinType(rtid, rv) {
 		return
 	}
 
 	//Note: tagFn must handle returning nil if value should be encoded as a nil.
-	if xfTag, xfFn := e.h.getEncodeExt(rt); xfFn != nil {
+	if xfTag, xfFn := e.h.getEncodeExt(rtid); xfFn != nil {
 		bs, fnerr := xfFn(rv)
 		if fnerr != nil {
 			panic(fnerr)
@@ -348,6 +347,9 @@ func (e *Encoder) encodeValue(rv reflect.Value) {
 		return
 	}
 	
+	// TODO: Encode if type is an encoding.BinaryMarshaler: MarshalBinary() (data []byte, err error)
+	// There is a cost, as we need to change the rv to an interface{} first.
+
 	// ensure more common cases appear early in switch.
 	rk := rv.Kind()
 	switch rk {
@@ -400,7 +402,8 @@ func (e *Encoder) encodeValue(rv reflect.Value) {
 			e.encodeValue(rv.MapIndex(mks[j]))
 		}
 	case reflect.Struct:
-		e.encStruct(rt, rv)
+		sis := getStructFieldInfos(rtid, rt)
+		e.encStruct(sis, rv)
 	case reflect.Ptr:
 		if rv.IsNil() {
 			ee.encodeNil()
@@ -425,8 +428,7 @@ func (e *Encoder) encodeValue(rv reflect.Value) {
 	return
 }
 
-func (e *Encoder) encStruct(rt reflect.Type, rv reflect.Value) {
-	sis := getStructFieldInfos(rt)
+func (e *Encoder) encStruct(sis structFieldInfos, rv reflect.Value) {
 	newlen := len(sis)
 	rvals := make([]reflect.Value, newlen)
 	encnames := make([]string, newlen)

codec/helper.go

@@ -18,8 +18,10 @@ import (
 )
 
 const (
-	// For >= 4 elements, map outways cost of linear search (especially for reflect.Type)
-	mapAccessThreshold    = 4
+	// For >= mapAccessThreshold elements, map outways cost of linear search 
+	//   - this was critical for reflect.Type, whose equality cost is pretty high (set to 4)
+	//   - for integers, equality cost is cheap (set to 16, 32 of 64)
+	mapAccessThreshold    = 16 // 4
 	binarySearchThreshold = 16
 	structTagName         = "codec"
 )
@@ -39,7 +41,7 @@ var (
 	bigen               = binary.BigEndian
 	structInfoFieldName = "_struct"
 
-	cachedStructFieldInfos      = make(map[reflect.Type]structFieldInfos, 4)
+	cachedStructFieldInfos      = make(map[uintptr]structFieldInfos, 4)
 	cachedStructFieldInfosMutex sync.RWMutex
 
 	nilIntfSlice     = []interface{}(nil)
@@ -53,6 +55,9 @@ var (
 	ptrTimeTyp       = reflect.TypeOf((*time.Time)(nil))
 	int64SliceTyp    = reflect.TypeOf([]int64(nil))
 
+	timeTypId        = reflect.ValueOf(timeTyp).Pointer()
+	byteSliceTypId   = reflect.ValueOf(byteSliceTyp).Pointer()
+	
 	intBitsize  uint8 = uint8(reflect.TypeOf(int(0)).Bits())
 	uintBitsize uint8 = uint8(reflect.TypeOf(uint(0)).Bits())
 
@@ -71,8 +76,9 @@ func (o *encdecHandle) AddExt(
 	encfn func(reflect.Value) ([]byte, error),
 	decfn func(reflect.Value, []byte) error,
 ) {
-	o.addEncodeExt(rt, tag, encfn)
-	o.addDecodeExt(rt, tag, decfn)
+	rtid := reflect.ValueOf(rt).Pointer()
+	o.addEncodeExt(rtid, tag, encfn)
+	o.addDecodeExt(rtid, rt, tag, decfn)
 }
 
 // Handle is the interface for a specific encoding format.
@@ -142,9 +148,10 @@ func (sis structFieldInfos) indexForEncName(name string) int {
 	return -1
 }
 
-func getStructFieldInfos(rt reflect.Type) (sis structFieldInfos) {
+func getStructFieldInfos(rtid uintptr, rt reflect.Type) (sis structFieldInfos) {
+	var ok bool
 	cachedStructFieldInfosMutex.RLock()
-	sis, ok := cachedStructFieldInfos[rt]
+	sis, ok = cachedStructFieldInfos[rtid]
 	cachedStructFieldInfosMutex.RUnlock()
 	if ok {
 		return
@@ -152,6 +159,9 @@ func getStructFieldInfos(rt reflect.Type) (sis structFieldInfos) {
 
 	cachedStructFieldInfosMutex.Lock()
 	defer cachedStructFieldInfosMutex.Unlock()
+	if sis, ok = cachedStructFieldInfos[rtid]; ok {
+		return
+	}
 
 	var siInfo *structFieldInfo
 	if f, ok := rt.FieldByName(structInfoFieldName); ok {
@@ -167,7 +177,7 @@ func getStructFieldInfos(rt reflect.Type) (sis structFieldInfos) {
 		sis[i] = *sisp[i]
 	}
 	// sis = sisp
-	cachedStructFieldInfos[rt] = sis
+	cachedStructFieldInfos[rtid] = sis
 	return
 }
 

codec/msgpack.go

@@ -19,9 +19,10 @@ const (
 	mpFixMapMax         = 0x8f
 	mpFixArrayMin       = 0x90
 	mpFixArrayMax       = 0x9f
-	mpFixRawMin         = 0xa0
-	mpFixRawMax         = 0xbf
+	mpFixStrMin         = 0xa0
+	mpFixStrMax         = 0xbf
 	mpNil               = 0xc0
+	_                   = 0xc1
 	mpFalse             = 0xc2
 	mpTrue              = 0xc3
 	mpFloat             = 0xca
@@ -34,42 +35,33 @@ const (
 	mpInt16             = 0xd1
 	mpInt32             = 0xd2
 	mpInt64             = 0xd3
-	mpRaw16             = 0xda
-	mpRaw32             = 0xdb
+	
+	// extensions below
+	mpBin8 = 0xc4
+	mpBin16 = 0xc5
+	mpBin32 = 0xc6
+	mpExt8 = 0xc7
+	mpExt16 = 0xc8
+	mpExt32 = 0xc9
+	mpFixExt1 = 0xd4
+	mpFixExt2 = 0xd5
+	mpFixExt4 = 0xd6
+	mpFixExt8 = 0xd7
+	mpFixExt16 = 0xd8
+	
+	mpStr8              = 0xd9 // new
+	mpStr16             = 0xda
+	mpStr32             = 0xdb
+	
 	mpArray16           = 0xdc
 	mpArray32           = 0xdd
+	
 	mpMap16             = 0xde
 	mpMap32             = 0xdf
+	
 	mpNegFixNumMin      = 0xe0
 	mpNegFixNumMax      = 0xff
 
-	// extensions below
-	// mpBin8 = 0xc4
-	// mpBin16 = 0xc5
-	// mpBin32 = 0xc6
-	// mpExt8 = 0xc7
-	// mpExt16 = 0xc8
-	// mpExt32 = 0xc9
-	// mpFixExt1 = 0xd4
-	// mpFixExt2 = 0xd5
-	// mpFixExt4 = 0xd6
-	// mpFixExt8 = 0xd7
-	// mpFixExt16 = 0xd8
-
-	// extensions based off v4: https://gist.github.com/frsyuki/5235364
-	mpXv4Fixext0 = 0xc4
-	mpXv4Fixext1 = 0xc5
-	mpXv4Fixext2 = 0xc6
-	mpXv4Fixext3 = 0xc7
-	mpXv4Fixext4 = 0xc8
-	mpXv4Fixext5 = 0xc9
-
-	mpXv4Ext8m  = 0xd4
-	mpXv4Ext16m = 0xd5
-	mpXv4Ext32m = 0xd6
-	mpXv4Ext8   = 0xd7
-	mpXv4Ext16  = 0xd8
-	mpXv4Ext32  = 0xd9
 )
 
 // MsgpackSpecRpc implements Rpc using the communication protocol defined in
@@ -78,14 +70,16 @@ var MsgpackSpecRpc msgpackSpecRpc
 
 // A MsgpackContainer type specifies the different types of msgpackContainers.
 type msgpackContainerType struct {
-	cutoff     int8
-	b0, b1, b2 byte
+	fixCutoff             int
+	bFixMin, b8, b16, b32 byte
+	hasFixMin, has8, has8Always  bool
 }
 
 var (
-	msgpackContainerRawBytes = msgpackContainerType{32, mpFixRawMin, mpRaw16, mpRaw32}
-	msgpackContainerList     = msgpackContainerType{16, mpFixArrayMin, mpArray16, mpArray32}
-	msgpackContainerMap      = msgpackContainerType{16, mpFixMapMin, mpMap16, mpMap32}
+	msgpackContainerStr  = msgpackContainerType{32, mpFixStrMin, mpStr8, mpStr16, mpStr32, true, true, false}
+	msgpackContainerBin  = msgpackContainerType{32, 0, mpBin8, mpBin16, mpBin32, false, true, true}
+	msgpackContainerList = msgpackContainerType{16, mpFixArrayMin, 0, mpArray16, mpArray32, true, false, false}
+	msgpackContainerMap  = msgpackContainerType{16, mpFixMapMin, 0, mpMap16, mpMap32, true, false, false}
 )
 
 // msgpackSpecRpc is the implementation of Rpc that uses custom communication protocol
@@ -104,11 +98,13 @@ type MsgpackHandle struct {
 	// This setting is used only if an extension func isn't defined for []byte.
 	RawToString bool
 	// WriteExt flag supports encoding configured extensions with extension tags.
+	// It also controls whether other elements of the new spec are encoded (ie Str8).
+	// 
+	// With WriteExt=false, configured extensions are serialized as raw bytes 
+	// and Str8 is not encoded.
 	//
-	// With WriteExt=false, configured extensions are serialized as raw bytes.
-	//
-	// They can still be decoded into a typed object, provided an appropriate one is
-	// provided, but the type cannot be inferred from the stream. If no appropriate
+	// A stream can still be decoded into a typed value, provided an appropriate value
+	// is provided, but the type cannot be inferred from the stream. If no appropriate
 	// type is provided (e.g. decoding into a nil interface{}), you get back
 	// a []byte or string based on the setting of RawToString.
 	WriteExt bool
@@ -119,15 +115,17 @@ type MsgpackHandle struct {
 
 type msgpackEncDriver struct {
 	w encWriter
+	h *MsgpackHandle
 }
 
 type msgpackDecDriver struct {
 	r      decReader
+	h      *MsgpackHandle
 	bd     byte
 	bdRead bool
 }
 
-func (e *msgpackEncDriver) encodeBuiltinType(rt reflect.Type, rv reflect.Value) bool {
+func (e *msgpackEncDriver) encodeBuiltinType(rt uintptr, rv reflect.Value) bool {
 	//no builtin types. All encodings are based on kinds. Types supported as extensions.
 	return false
 }
@@ -195,19 +193,27 @@ func (e *msgpackEncDriver) encodeFloat64(f float64) {
 
 func (e *msgpackEncDriver) encodeExtPreamble(xtag byte, l int) {
 	switch {
-	case l <= 4:
-		e.w.writen2(0xd4|byte(l), xtag)
-	case l <= 8:
-		e.w.writen2(0xc0|byte(l), xtag)
+	case l == 1:
+		e.w.writen2(mpFixExt1, xtag)
+	case l == 2:
+		e.w.writen2(mpFixExt2, xtag)
+	case l == 4:
+		e.w.writen2(mpFixExt4, xtag)
+	case l == 8:
+		e.w.writen2(mpFixExt8, xtag)
+	case l == 16:
+		e.w.writen2(mpFixExt16, xtag)
 	case l < 256:
-		e.w.writen2(mpXv4Fixext5, xtag)
-		e.w.writen1(byte(l))
+		e.w.writen2(mpExt8, byte(l))
+		e.w.writen1(xtag)
 	case l < 65536:
-		e.w.writen2(mpXv4Ext16, xtag)
+		e.w.writen1(mpExt16)
 		e.w.writeUint16(uint16(l))
+		e.w.writen1(xtag)
 	default:
-		e.w.writen2(mpXv4Ext32, xtag)
+		e.w.writen1(mpExt32)
 		e.w.writeUint32(uint32(l))
+		e.w.writen1(xtag)
 	}
 }
 
@@ -220,8 +226,11 @@ func (e *msgpackEncDriver) encodeMapPreamble(length int) {
 }
 
 func (e *msgpackEncDriver) encodeString(c charEncoding, s string) {
-	//ignore charEncoding.
-	e.writeContainerLen(msgpackContainerRawBytes, len(s))
+	if c == c_RAW && e.h.WriteExt {
+		e.writeContainerLen(msgpackContainerBin, len(s))
+	} else {
+		e.writeContainerLen(msgpackContainerStr, len(s))
+	}
 	if len(s) > 0 {
 		e.w.writestr(s)
 	}
@@ -232,8 +241,11 @@ func (e *msgpackEncDriver) encodeSymbol(v string) {
 }
 
 func (e *msgpackEncDriver) encodeStringBytes(c charEncoding, bs []byte) {
-	//ignore charEncoding.
-	e.writeContainerLen(msgpackContainerRawBytes, len(bs))
+	if c == c_RAW && e.h.WriteExt {
+		e.writeContainerLen(msgpackContainerBin, len(bs))
+	} else {
+		e.writeContainerLen(msgpackContainerStr, len(bs))
+	}
 	if len(bs) > 0 {
 		e.w.writeb(bs)
 	}
@@ -241,20 +253,22 @@ func (e *msgpackEncDriver) encodeStringBytes(c charEncoding, bs []byte) {
 
 func (e *msgpackEncDriver) writeContainerLen(ct msgpackContainerType, l int) {
 	switch {
-	case l < int(ct.cutoff):
-		e.w.writen1(ct.b0 | byte(l))
+	case ct.hasFixMin && l < ct.fixCutoff:
+		e.w.writen1(ct.bFixMin | byte(l))
+	case ct.has8 && l < 256 && (ct.has8Always || e.h.WriteExt):
+		e.w.writen2(ct.b8, uint8(l))
 	case l < 65536:
-		e.w.writen1(ct.b1)
+		e.w.writen1(ct.b16)
 		e.w.writeUint16(uint16(l))
 	default:
-		e.w.writen1(ct.b2)
+		e.w.writen1(ct.b32)
 		e.w.writeUint32(uint32(l))
 	}
 }
 
 //---------------------------------------------
 
-func (d *msgpackDecDriver) decodeBuiltinType(rt reflect.Type, rv reflect.Value) bool {
+func (d *msgpackDecDriver) decodeBuiltinType(rt uintptr, rv reflect.Value) bool {
 	return false
 }
 
@@ -309,9 +323,9 @@ func (d *msgpackDecDriver) decodeNaked(h decodeHandleI) (rv reflect.Value, ctx d
 		case bd >= mpNegFixNumMin && bd <= mpNegFixNumMax:
 			// negative fixnum
 			v = int64(int8(bd))
-		case bd == mpRaw16, bd == mpRaw32, bd >= mpFixRawMin && bd <= mpFixRawMax:
+		case bd == mpStr8, bd == mpStr16, bd == mpStr32, bd >= mpFixStrMin && bd <= mpFixStrMax:
 			ctx = dncContainer
-			// v = containerRawBytes
+			// v = containerRaw
 			opts := h.(*MsgpackHandle)
 			if opts.rawToStringOverride || opts.RawToString {
 				var rvm string
@@ -337,20 +351,17 @@ func (d *msgpackDecDriver) decodeNaked(h decodeHandleI) (rv reflect.Value, ctx d
 			} else {
 				rv = reflect.MakeMap(opts.MapType)
 			}
-		case bd >= mpXv4Fixext0 && bd <= mpXv4Fixext5, bd >= mpXv4Ext8m && bd <= mpXv4Ext32:
+		case bd >= mpFixExt1 && bd <= mpFixExt16, bd >= mpExt8 && bd <= mpExt32:
 			//ctx = dncExt
+			clen := d.readExtLen()
 			xtag := d.r.readn1()
 			opts := h.(*MsgpackHandle)
-			rt, bfn := opts.getDecodeExtForTag(xtag)
-			if rt == nil {
+			var bfn func(reflect.Value, []byte) error
+			rv, bfn = opts.getDecodeExtForTag(xtag)
+			if bfn == nil {
 				decErr("Unable to find type mapped to extension tag: %v", xtag)
 			}
-			if rt.Kind() == reflect.Ptr {
-				rv = reflect.New(rt.Elem())
-			} else {
-				rv = reflect.New(rt).Elem()
-			}
-			if fnerr := bfn(rv, d.r.readn(d.readExtLen())); fnerr != nil {
+			if fnerr := bfn(rv, d.r.readn(clen)); fnerr != nil {
 				panic(fnerr)
 			}
 		default:
@@ -499,7 +510,7 @@ func (d *msgpackDecDriver) decodeBool() (b bool) {
 }
 
 func (d *msgpackDecDriver) decodeString() (s string) {
-	clen := d.readContainerLen(msgpackContainerRawBytes)
+	clen := d.readContainerLen(msgpackContainerStr)
 	if clen > 0 {
 		s = string(d.r.readn(clen))
 	}
@@ -509,7 +520,7 @@ func (d *msgpackDecDriver) decodeString() (s string) {
 
 // Callers must check if changed=true (to decide whether to replace the one they have)
 func (d *msgpackDecDriver) decodeBytes(bs []byte) (bsOut []byte, changed bool) {
-	clen := d.readContainerLen(msgpackContainerRawBytes)
+	clen := d.readContainerLen(msgpackContainerBin)
 	// if clen < 0 {
 	// 	changed = true
 	// 	panic("length cannot be zero. this cannot be nil.")
@@ -553,12 +564,14 @@ func (d *msgpackDecDriver) readContainerLen(ct msgpackContainerType) (clen int)
 	switch {
 	case d.bd == mpNil:
 		clen = -1 // to represent nil
-	case d.bd == ct.b1:
+	case d.bd == ct.b8:
+		clen = int(d.r.readn1())
+	case d.bd == ct.b16:
 		clen = int(d.r.readUint16())
-	case d.bd == ct.b2:
+	case d.bd == ct.b32:
 		clen = int(d.r.readUint32())
-	case (ct.b0 & d.bd) == ct.b0:
-		clen = int(ct.b0 ^ d.bd)
+	case (ct.bFixMin & d.bd) == ct.bFixMin:
+		clen = int(ct.bFixMin ^ d.bd)
 	default:
 		decErr("readContainerLen: %s: hex: %x, dec: %d", msgBadDesc, d.bd, d.bd)
 	}
@@ -578,39 +591,42 @@ func (d *msgpackDecDriver) readExtLen() (clen int) {
 	switch d.bd {
 	case mpNil:
 		clen = -1 // to represent nil
-	case mpXv4Fixext5:
+	case mpFixExt1:
+		clen = 1
+	case mpFixExt2:
+		clen = 2
+	case mpFixExt4:
+		clen = 4
+	case mpFixExt8:
+		clen = 8
+	case mpFixExt16:
+		clen = 16
+	case mpExt8:
 		clen = int(d.r.readn1())
-	case mpXv4Ext16:
+	case mpExt16:
 		clen = int(d.r.readUint16())
-	case mpXv4Ext32:
+	case mpExt32:
 		clen = int(d.r.readUint32())
 	default:
-		switch {
-		case d.bd >= mpXv4Fixext0 && d.bd <= mpXv4Fixext4:
-			clen = int(d.bd & 0x0f)
-		case d.bd >= mpXv4Ext8m && d.bd <= mpXv4Ext8:
-			clen = int(d.bd & 0x03)
-		default:
-			decErr("decoding ext bytes: found unexpected byte: %x", d.bd)
-		}
+		decErr("decoding ext bytes: found unexpected byte: %x", d.bd)
 	}
 	return
 }
 
 func (d *msgpackDecDriver) decodeExt(tag byte) (xbs []byte) {
-	// if (d.bd >= mpXv4Fixext0 && d.bd <= mpXv4Fixext5) || 
-	// (d.bd >= mpXv4Ext8m && d.bd <= mpXv4Ext32) {
 	xbd := d.bd
 	switch {
-	case xbd >= mpXv4Fixext0 && xbd <= mpXv4Fixext5, xbd >= mpXv4Ext8m && xbd <= mpXv4Ext32:
+	case xbd == mpBin8, xbd == mpBin16, xbd == mpBin32: 
+		xbs, _ = d.decodeBytes(nil) 
+	case xbd == mpStr8, xbd == mpStr16, xbd == mpStr32, 
+		xbd >= mpFixStrMin && xbd <= mpFixStrMax:
+		xbs = []byte(d.decodeString())
+	default:
+		clen := d.readExtLen()
 		if xtag := d.r.readn1(); xtag != tag {
 			decErr("Wrong extension tag. Got %b. Expecting: %v", xtag, tag)
 		}
-		xbs = d.r.readn(d.readExtLen())
-	case xbd == mpRaw16, xbd == mpRaw32, xbd >= mpFixRawMin && xbd <= mpFixRawMax:
-		xbs, _ = d.decodeBytes(nil)
-	default:
-		decErr("Wrong byte descriptor (Expecting extensions or raw bytes). Got: 0x%x", xbd)
+		xbs = d.r.readn(clen)
 	}
 	d.bdRead = false
 	return
@@ -729,14 +745,24 @@ func (_ *MsgpackHandle) TimeDecodeExt(rv reflect.Value, bs []byte) (err error) {
 	return
 }
 
-func (_ *MsgpackHandle) newEncDriver(w encWriter) encDriver {
-	return &msgpackEncDriver{w: w}
+func (h *MsgpackHandle) newEncDriver(w encWriter) encDriver {
+	return &msgpackEncDriver{w: w, h: h}
 }
 
-func (_ *MsgpackHandle) newDecDriver(r decReader) decDriver {
-	return &msgpackDecDriver{r: r}
+func (h *MsgpackHandle) newDecDriver(r decReader) decDriver {
+	return &msgpackDecDriver{r: r, h: h}
 }
 
-func (o *MsgpackHandle) writeExt() bool {
-	return o.WriteExt
+func (h *MsgpackHandle) writeExt() bool {
+	return h.WriteExt
 }
+
+
+
+
+
+
+
+
+
+