codec: redesign how codecFnInfo is loaded from, and support SelfExt

Having to redesign your values to support extensions is a flawed design.
We have now reverted that, and allow users to just signify within the current
design that they want to support the native encoding/decoding for their extensions.

However, we needed to fix some flaws in the way codecFn was loaded previously.
That (my original) design is flawed and needs to be fixed before we build
anything else atop it.

For one, the fact that we pass checkFastpath and checkCodecSelfer were
code smells, that something wasn't right with the design. Fundamentally,
the fn() should be a static function that always returns the same value
for a given context (since it caches them). I need to just re-architect
it and remove those functions.

Finally, codec.Selfer can be an extension.
Consequently, it cannot delegate to EncodeExtension itself.
Ensure that the check for extension happens before the check whether a Selfer or not.

Summary of changes:

- BasicHandle now keeps 2 rtidFns: one for all, and one for extensions that encode natively
- fn() no longer takes checkFastpath or checkCodecSelfer
- formats all support the sentinel SelfExt
- ext encoding is more efficient as it tries to reuse pools of []byte

Thanks @kostko for his gentle and committed push!

Fixes #274

README.md

@@ -266,11 +266,11 @@ some caveats. See Encode documentation.
 ```go
 const CborStreamBytes byte = 0x5f ...
 const GenVersion = 12
+var SelfExt = &extFailWrapper{}
 var GoRpc goRpc
 var MsgpackSpecRpc msgpackSpecRpc
 func GenHelperDecoder(d *Decoder) (gd genHelperDecoder, dd genHelperDecDriver)
 func GenHelperEncoder(e *Encoder) (ge genHelperEncoder, ee genHelperEncDriver)
-func NewSelfBytesExt(h Handle, bufcap int) *selfBytesExt
 type BasicHandle struct{ ... }
 type BincHandle struct{ ... }
 type BytesExt interface{ ... }
@@ -286,7 +286,6 @@ type Encoder struct{ ... }
 type Ext interface{ ... }
 type Handle interface{ ... }
 type InterfaceExt interface{ ... }
-    var GlobalSelfInterfaceExt InterfaceExt = selfInterfaceExt{}
 type JsonHandle struct{ ... }
 type MapBySlice interface{ ... }
 type MissingFielder interface{ ... }
@@ -296,7 +295,6 @@ type RPCOptions struct{ ... }
 type Raw []byte
 type RawExt struct{ ... }
 type Rpc interface{ ... }
-type SelfExt interface{ ... }
 type Selfer interface{ ... }
 type SimpleHandle struct{ ... }
 type TypeInfos struct{ ... }

codec/bench/bench.sh

@@ -131,10 +131,10 @@ _main() {
         return 1
     fi
     local args=()
-    while getopts "dcsjqp" flag
+    while getopts "dcsjqpg" flag
     do
         case "$flag" in
-            d|c|s|j|q|p) args+=( "$flag" ) ;;
+            d|c|s|j|q|p|g) args+=( "$flag" ) ;;
             *) _usage; return 1 ;;
         esac
     done
@@ -143,6 +143,7 @@ _main() {
     [[ " ${args[*]} " == *"d"* ]] && _go_get "$@"
     [[ " ${args[*]} " == *"c"*  ]] && _gen "$@"
     [[ " ${args[*]} " == *"s"* ]] && _suite "$@" && _suite_gen "$@" 
+    [[ " ${args[*]} " == *"g"* ]] && _suite_gen "$@" 
     [[ " ${args[*]} " == *"j"* ]] && _suite_json "$@"
     [[ " ${args[*]} " == *"q"* ]] && _suite_very_quick_json_trim_output "$@"
     [[ " ${args[*]} " == *"p"* ]] && _suite_very_quick_json_only_profile "$@"

codec/binc.go

@@ -215,17 +215,28 @@ func (e *bincEncDriver) encUint(bd byte, pos bool, v uint64) {
 	}
 }
 
-func (e *bincEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, _ *Encoder) {
-	bs := ext.WriteExt(rv)
+func (e *bincEncDriver) EncodeExt(v interface{}, xtag uint64, ext Ext) {
+	var bs []byte
+	var bufp bytesBufPooler
+	if ext == SelfExt {
+		bs = bufp.get(1024)[:0]
+		e.e.sideEncode(v, &bs)
+		// xdebugf("binc EncodeExt: xbs: len: %d, %v", len(bs), bs)
+	} else {
+		bs = ext.WriteExt(v)
+	}
 	if bs == nil {
 		e.EncodeNil()
 		return
 	}
 	e.encodeExtPreamble(uint8(xtag), len(bs))
 	e.w.writeb(bs)
+	if ext == SelfExt {
+		bufp.end()
+	}
 }
 
-func (e *bincEncDriver) EncodeRawExt(re *RawExt, _ *Encoder) {
+func (e *bincEncDriver) EncodeRawExt(re *RawExt) {
 	e.encodeExtPreamble(uint8(re.Tag), len(re.Data))
 	e.w.writeb(re.Data)
 }
@@ -818,10 +829,13 @@ func (d *bincDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 	}
 	realxtag1, xbs := d.decodeExtV(ext != nil, uint8(xtag))
 	realxtag = uint64(realxtag1)
+	// xdebugf("binc DecodeExt: xbs: len: %d, %v", len(xbs), xbs)
 	if ext == nil {
 		re := rv.(*RawExt)
 		re.Tag = realxtag
 		re.Data = detachZeroCopyBytes(d.br, re.Data, xbs)
+	} else if ext == SelfExt {
+		d.d.sideDecode(rv, xbs)
 	} else {
 		ext.ReadExt(rv, xbs)
 	}
@@ -998,7 +1012,7 @@ func (h *BincHandle) Name() string { return "binc" }
 
 // SetBytesExt sets an extension
 func (h *BincHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
-	return h.SetExt(rt, tag, &bytesExtWrapper{BytesExt: ext})
+	return h.SetExt(rt, tag, makeExt(ext))
 }
 
 func (h *BincHandle) newEncDriver(e *Encoder) encDriver {

codec/cbor.go

@@ -185,23 +185,27 @@ func (e *cborEncDriver) EncodeTime(t time.Time) {
 	}
 }
 
-func (e *cborEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, en *Encoder) {
+func (e *cborEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext) {
+	// xdebugf("cbor EncodeExt: v: %T, %v, ext: %T, %v, ==Self: %v", rv, rv, ext, ext, ext == SelfExt)
 	e.encUint(uint64(xtag), cborBaseTag)
-	if v := ext.ConvertExt(rv); v == nil {
+	if ext == SelfExt {
+		rv2 := baseRV(rv)
+		e.e.encodeValue(rv2, e.h.fnNoExt(rv2.Type()))
+	} else if v := ext.ConvertExt(rv); v == nil {
 		e.EncodeNil()
 	} else {
-		en.encode(v)
+		e.e.encode(v)
 	}
 }
 
-func (e *cborEncDriver) EncodeRawExt(re *RawExt, en *Encoder) {
+func (e *cborEncDriver) EncodeRawExt(re *RawExt) {
 	e.encUint(uint64(re.Tag), cborBaseTag)
 	// only encodes re.Value (never re.Data)
 	// if false && re.Data != nil {
 	// 	en.encode(re.Data)
 	// } else if re.Value != nil {
 	if re.Value != nil {
-		en.encode(re.Value)
+		e.e.encode(re.Value)
 	} else {
 		e.EncodeNil()
 	}
@@ -617,6 +621,7 @@ func (d *cborDecDriver) decodeTime(xtag uint64) (t time.Time) {
 }
 
 func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxtag uint64) {
+	// xdebugf("cbor DecodeExt: v: %T, %v", rv, rv)
 	if !d.bdRead {
 		d.readNextBd()
 	}
@@ -630,13 +635,11 @@ func (d *cborDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 	} else if xtag != realxtag {
 		d.d.errorf("Wrong extension tag. Got %b. Expecting: %v", realxtag, xtag)
 		return
+	} else if ext == SelfExt {
+		rv2 := baseRV(rv)
+		d.d.decodeValue(rv2, d.h.fnNoExt(rv2.Type()))
 	} else {
-		var v interface{}
-		if v2, ok := rv.(SelfExt); ok {
-			v = v2.CodecConvertExt()
-		}
-		d.d.decode(&v)
-		ext.UpdateExt(rv, v)
+		d.d.interfaceExtConvertAndDecode(rv, ext)
 	}
 	d.bdRead = false
 	return
@@ -757,7 +760,7 @@ func (h *CborHandle) Name() string { return "cbor" }
 
 // SetInterfaceExt sets an extension
 func (h *CborHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) {
-	return h.SetExt(rt, tag, &interfaceExtWrapper{InterfaceExt: ext})
+	return h.SetExt(rt, tag, makeExt(ext))
 }
 
 func (h *CborHandle) newEncDriver(e *Encoder) encDriver {

codec/codec_test.go

@@ -425,11 +425,11 @@ func testInit() {
 	// testJsonH.SetInterfaceExt(timeTyp, 1, &testUnixNanoTimeExt{})
 
 	// Add extensions for the testSelfExt
-	chkErr(testSimpleH.SetBytesExt(testSelfExtTyp, 78, NewSelfBytesExt(testSimpleH, 128)))
-	chkErr(testMsgpackH.SetBytesExt(testSelfExtTyp, 78, NewSelfBytesExt(testMsgpackH, 128)))
-	chkErr(testBincH.SetBytesExt(testSelfExtTyp, 78, NewSelfBytesExt(testBincH, 128)))
-	chkErr(testJsonH.SetInterfaceExt(testSelfExtTyp, 78, GlobalSelfInterfaceExt))
-	chkErr(testCborH.SetInterfaceExt(testSelfExtTyp, 78, GlobalSelfInterfaceExt))
+	chkErr(testSimpleH.SetBytesExt(testSelfExtTyp, 78, SelfExt))
+	chkErr(testMsgpackH.SetBytesExt(testSelfExtTyp, 78, SelfExt))
+	chkErr(testBincH.SetBytesExt(testSelfExtTyp, 78, SelfExt))
+	chkErr(testJsonH.SetInterfaceExt(testSelfExtTyp, 78, SelfExt))
+	chkErr(testCborH.SetInterfaceExt(testSelfExtTyp, 78, SelfExt))
 
 	// Now, add extensions for the type wrapInt64 and wrapBytes,
 	// so we can execute the Encode/Decode Ext paths.

codec/decode.go

@@ -1268,7 +1268,7 @@ func (d *Decoder) kInterfaceNaked(f *codecFnInfo) (rvn reflect.Value) {
 				rvn = rvn.Elem()
 			} else {
 				rvn = reflect.New(d.h.MapType).Elem()
-				d.decodeValue(rvn, nil, true)
+				d.decodeValue(rvn, nil)
 			}
 		}
 	case valueTypeArray:
@@ -1288,7 +1288,7 @@ func (d *Decoder) kInterfaceNaked(f *codecFnInfo) (rvn reflect.Value) {
 				rvn = rvn.Elem()
 			} else {
 				rvn = reflect.New(d.h.SliceType).Elem()
-				d.decodeValue(rvn, nil, true)
+				d.decodeValue(rvn, nil)
 			}
 		}
 	case valueTypeExt:
@@ -1375,13 +1375,13 @@ func (d *Decoder) kInterface(f *codecFnInfo, rv reflect.Value) {
 
 	rvn2, canDecode := isDecodeable(rvn)
 	if canDecode {
-		d.decodeValue(rvn2, nil, true)
+		d.decodeValue(rvn2, nil)
 		return
 	}
 
 	rvn2 = reflect.New(rvn.Type()).Elem()
 	rvn2.Set(rvn)
-	d.decodeValue(rvn2, nil, true)
+	d.decodeValue(rvn2, nil)
 	rv.Set(rvn2)
 }
 
@@ -1433,7 +1433,7 @@ func (d *Decoder) kStruct(f *codecFnInfo, rv reflect.Value) {
 				if dd.TryDecodeAsNil() {
 					si.setToZeroValue(rv)
 				} else {
-					d.decodeValue(sfn.field(si), nil, true)
+					d.decodeValue(sfn.field(si), nil)
 				}
 			} else if mf != nil {
 				// store rvkencname in new []byte, as it previously shares Decoder.b, which is used in decode
@@ -1477,7 +1477,7 @@ func (d *Decoder) kStruct(f *codecFnInfo, rv reflect.Value) {
 			if dd.TryDecodeAsNil() {
 				si.setToZeroValue(rv)
 			} else {
-				d.decodeValue(sfn.field(si), nil, true)
+				d.decodeValue(sfn.field(si), nil)
 			}
 		}
 		if (hasLen && containerLen > len(fti.sfiSrc)) || (!hasLen && !checkbreak) {
@@ -1650,9 +1650,9 @@ func (d *Decoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 				rv9 = reflect.New(rtelem0).Elem()
 			}
 			if fn == nil {
-				fn = d.h.fn(rtelem, true, true)
+				fn = d.h.fn(rtelem)
 			}
-			d.decodeValue(rv9, fn, true)
+			d.decodeValue(rv9, fn)
 			rv.Send(rv9)
 		} else {
 			// if indefinite, etc, then expand the slice if necessary
@@ -1695,9 +1695,9 @@ func (d *Decoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 				}
 
 				if fn == nil {
-					fn = d.h.fn(rtelem, true, true)
+					fn = d.h.fn(rtelem)
 				}
-				d.decodeValue(rv9, fn, true)
+				d.decodeValue(rv9, fn)
 			}
 		}
 	}
@@ -1801,9 +1801,9 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 			// NOTE: if doing an insert, you MUST use a real string (not stringview)
 		} else {
 			if keyFn == nil {
-				keyFn = d.h.fn(ktypeLo, true, true)
+				keyFn = d.h.fn(ktypeLo)
 			}
-			d.decodeValue(rvk, keyFn, true)
+			d.decodeValue(rvk, keyFn)
 		}
 		// special case if a byte array.
 		if ktypeIsIntf {
@@ -1869,9 +1869,9 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 			rvk.SetString(d.string(kstrbs))
 		}
 		if valFn == nil {
-			valFn = d.h.fn(vtypeLo, true, true)
+			valFn = d.h.fn(vtypeLo)
 		}
-		d.decodeValue(rvv, valFn, true)
+		d.decodeValue(rvv, valFn)
 		// d.decodeValueFn(rvv, valFn)
 		if mapSet {
 			rv.SetMapIndex(rvk, rvv)
@@ -2684,7 +2684,7 @@ func (d *Decoder) decode(iv interface{}) {
 	// case Selfer:
 	case reflect.Value:
 		v = d.ensureDecodeable(v)
-		d.decodeValue(v, nil, true)
+		d.decodeValue(v, nil)
 
 	case *string:
 		*v = d.d.DecodeString()
@@ -2727,7 +2727,7 @@ func (d *Decoder) decode(iv interface{}) {
 		*v = d.rawBytes()
 
 	case *interface{}:
-		d.decodeValue(reflect.ValueOf(iv).Elem(), nil, true)
+		d.decodeValue(reflect.ValueOf(iv).Elem(), nil)
 		// d.decodeValueNotNil(reflect.ValueOf(iv).Elem())
 
 	default:
@@ -2736,13 +2736,13 @@ func (d *Decoder) decode(iv interface{}) {
 		} else if !fastpathDecodeTypeSwitch(iv, d) {
 			v := reflect.ValueOf(iv)
 			v = d.ensureDecodeable(v)
-			d.decodeValue(v, nil, false)
+			d.decodeValue(v, nil) // TODO: find a way to say: no fast path??? Not necessary???
 			// d.decodeValueFallback(v)
 		}
 	}
 }
 
-func (d *Decoder) decodeValue(rv reflect.Value, fn *codecFn, chkAll bool) {
+func (d *Decoder) decodeValue(rv reflect.Value, fn *codecFn) {
 	// If stream is not containing a nil value, then we can deref to the base
 	// non-pointer value, and decode into that.
 	var rvp reflect.Value
@@ -2762,8 +2762,7 @@ func (d *Decoder) decodeValue(rv reflect.Value, fn *codecFn, chkAll bool) {
 	}
 
 	if fn == nil {
-		// always pass checkCodecSelfer=true, in case T or ****T is passed, where *T is a Selfer
-		fn = d.h.fn(rv.Type(), chkAll, true) // chkAll, chkAll)
+		fn = d.h.fn(rv.Type())
 	}
 	if fn.i.addrD {
 		if rvpValid {
@@ -2960,6 +2959,47 @@ func (d *Decoder) arrayEnd() {
 	d.c = 0
 }
 
+func (d *Decoder) interfaceExtConvertAndDecode(v interface{}, ext Ext) {
+	// var v interface{} = ext.ConvertExt(rv)
+	// d.d.decode(&v)
+	// ext.UpdateExt(rv, v)
+
+	// assume v is a pointer:
+	// - if struct|array, pass as is to ConvertExt
+	// - else make it non-addressable and pass to ConvertExt
+	// - make return value from ConvertExt addressable
+	// - decode into it
+	// - return the interface for passing into UpdateExt.
+	// - interface should be a pointer if struct|array, else a value
+	var s interface{}
+	rv := reflect.ValueOf(v)
+	rv2 := rv.Elem()
+	rvk := rv2.Kind()
+	if rvk == reflect.Struct || rvk == reflect.Array {
+		s = ext.ConvertExt(v)
+	} else {
+		s = ext.ConvertExt(rv2i(rv2))
+	}
+	rv = reflect.ValueOf(s)
+	if !rv.CanAddr() {
+		if rv.Kind() == reflect.Ptr {
+			rv2 = reflect.New(rv.Type().Elem())
+			rv2.Set(rv)
+		} else {
+			rv2 = reflect.New(rv.Type()).Elem()
+			rv2.Set(rv)
+		}
+		rv = rv2
+	}
+	d.decodeValue(rv, nil)
+	ext.UpdateExt(v, rv2i(rv))
+}
+
+func (d *Decoder) sideDecode(v interface{}, bs []byte) {
+	rv := baseRV(v)
+	NewDecoderBytes(bs, d.hh).decodeValue(rv, d.h.fnNoExt(rv.Type()))
+}
+
 // --------------------------------------------------
 
 // decSliceHelper assists when decoding into a slice, from a map or an array in the stream.

codec/encode.go

@@ -47,8 +47,8 @@ type encDriver interface {
 	EncodeFloat32(f float32)
 	EncodeFloat64(f float64)
 	// encodeExtPreamble(xtag byte, length int)
-	EncodeRawExt(re *RawExt, e *Encoder)
-	EncodeExt(v interface{}, xtag uint64, ext Ext, e *Encoder)
+	EncodeRawExt(re *RawExt)
+	EncodeExt(v interface{}, xtag uint64, ext Ext)
 	EncodeStringEnc(c charEncoding, v string) // c cannot be cRAW
 	// EncodeSymbol(v string)
 	EncodeStringBytesRaw(v []byte)
@@ -475,12 +475,12 @@ func (z *bytesEncAppender) reset(in []byte, out *[]byte) {
 // ---------------------------------------------
 
 func (e *Encoder) rawExt(f *codecFnInfo, rv reflect.Value) {
-	e.e.EncodeRawExt(rv2i(rv).(*RawExt), e)
+	e.e.EncodeRawExt(rv2i(rv).(*RawExt))
 }
 
 func (e *Encoder) ext(f *codecFnInfo, rv reflect.Value) {
 	// xdebugf("*Encoder.ext: rv: %v, rv2i: %T, %v", rv, rv2i(rv), rv2i(rv))
-	e.e.EncodeExt(rv2i(rv), f.xfTag, f.xfFn, e)
+	e.e.EncodeExt(rv2i(rv), f.xfTag, f.xfFn)
 }
 
 func (e *Encoder) selferMarshal(f *codecFnInfo, rv reflect.Value) {
@@ -595,7 +595,7 @@ func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 		// encoding type, because preEncodeValue may break it down to
 		// a concrete type and kInterface will bomb.
 		if rtelem.Kind() != reflect.Interface {
-			fn = e.h.fn(rtelem, true, true)
+			fn = e.h.fn(rtelem)
 		}
 		for j := 0; j < l; j++ {
 			if mbs {
@@ -607,7 +607,7 @@ func (e *Encoder) kSlice(f *codecFnInfo, rv reflect.Value) {
 			} else {
 				e.arrayElem()
 			}
-			e.encodeValue(rv.Index(j), fn, true)
+			e.encodeValue(rv.Index(j), fn)
 		}
 	}
 
@@ -692,7 +692,7 @@ func (e *Encoder) kStructNoOmitempty(f *codecFnInfo, rv reflect.Value) {
 		e.arrayStart(len(f.ti.sfiSrc))
 		for _, si := range f.ti.sfiSrc {
 			e.arrayElem()
-			e.encodeValue(sfn.field(si), nil, true)
+			e.encodeValue(sfn.field(si), nil)
 		}
 		e.arrayEnd()
 	} else {
@@ -701,7 +701,7 @@ func (e *Encoder) kStructNoOmitempty(f *codecFnInfo, rv reflect.Value) {
 			e.mapElemKey()
 			e.kStructFieldKey(f.ti.keyType, si.encNameAsciiAlphaNum, si.encName)
 			e.mapElemValue()
-			e.encodeValue(sfn.field(si), nil, true)
+			e.encodeValue(sfn.field(si), nil)
 		}
 		e.mapEnd()
 	}
@@ -785,7 +785,7 @@ func (e *Encoder) kStruct(f *codecFnInfo, rv reflect.Value) {
 			e.mapElemKey()
 			e.kStructFieldKey(f.ti.keyType, kv.v.encNameAsciiAlphaNum, kv.v.encName)
 			e.mapElemValue()
-			e.encodeValue(kv.r, nil, true)
+			e.encodeValue(kv.r, nil)
 		}
 		// now, add the others
 		for k, v := range mf {
@@ -815,7 +815,7 @@ func (e *Encoder) kStruct(f *codecFnInfo, rv reflect.Value) {
 		e.arrayStart(newlen)
 		for j = 0; j < newlen; j++ {
 			e.arrayElem()
-			e.encodeValue(fkvs[j].r, nil, true)
+			e.encodeValue(fkvs[j].r, nil)
 		}
 		e.arrayEnd()
 	}
@@ -853,7 +853,7 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
 		rtval = rtval.Elem()
 	}
 	if rtval.Kind() != reflect.Interface {
-		valFn = e.h.fn(rtval, true, true)
+		valFn = e.h.fn(rtval)
 	}
 	mks := rv.MapKeys()
 
@@ -871,7 +871,7 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
 		}
 		if rtkey.Kind() != reflect.Interface {
 			// rtkeyid = rt2id(rtkey)
-			keyFn = e.h.fn(rtkey, true, true)
+			keyFn = e.h.fn(rtkey)
 		}
 	}
 
@@ -885,10 +885,10 @@ func (e *Encoder) kMap(f *codecFnInfo, rv reflect.Value) {
 				e.e.EncodeStringEnc(cUTF8, mks[j].String())
 			}
 		} else {
-			e.encodeValue(mks[j], keyFn, true)
+			e.encodeValue(mks[j], keyFn)
 		}
 		e.mapElemValue()
-		e.encodeValue(rv.MapIndex(mks[j]), valFn, true)
+		e.encodeValue(rv.MapIndex(mks[j]), valFn)
 	}
 	e.mapEnd()
 }
@@ -910,7 +910,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 			e.mapElemKey()
 			e.e.EncodeBool(mksv[i].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
+			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.String:
 		mksv := make([]stringRv, len(mks))
@@ -928,7 +928,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 				e.e.EncodeStringEnc(cUTF8, mksv[i].v)
 			}
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
+			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint, reflect.Uintptr:
 		mksv := make([]uint64Rv, len(mks))
@@ -942,7 +942,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 			e.mapElemKey()
 			e.e.EncodeUint(mksv[i].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
+			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
 		mksv := make([]int64Rv, len(mks))
@@ -956,7 +956,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 			e.mapElemKey()
 			e.e.EncodeInt(mksv[i].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
+			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Float32:
 		mksv := make([]float64Rv, len(mks))
@@ -970,7 +970,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 			e.mapElemKey()
 			e.e.EncodeFloat32(float32(mksv[i].v))
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
+			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Float64:
 		mksv := make([]float64Rv, len(mks))
@@ -984,7 +984,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 			e.mapElemKey()
 			e.e.EncodeFloat64(mksv[i].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
+			e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 		}
 	case reflect.Struct:
 		if rv.Type() == timeTyp {
@@ -999,7 +999,7 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 				e.mapElemKey()
 				e.e.EncodeTime(mksv[i].v)
 				e.mapElemValue()
-				e.encodeValue(rv.MapIndex(mksv[i].r), valFn, true)
+				e.encodeValue(rv.MapIndex(mksv[i].r), valFn)
 			}
 			break
 		}
@@ -1007,7 +1007,8 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 	default:
 		// out-of-band
 		// first encode each key to a []byte first, then sort them, then record
-		var mksv []byte = make([]byte, 0, len(mks)*16) // temporary byte slice for the encoding
+		var bufp bytesBufPooler
+		var mksv []byte = bufp.get(len(mks) * 16)[:0]
 		e2 := NewEncoderBytes(&mksv, e.hh)
 		mksbv := make([]bytesRv, len(mks))
 		for i, k := range mks {
@@ -1022,8 +1023,9 @@ func (e *Encoder) kMapCanonical(rtkey reflect.Type, rv reflect.Value, mks []refl
 			e.mapElemKey()
 			e.asis(mksbv[j].v)
 			e.mapElemValue()
-			e.encodeValue(rv.MapIndex(mksbv[j].r), valFn, true)
+			e.encodeValue(rv.MapIndex(mksbv[j].r), valFn)
 		}
+		bufp.end()
 	}
 }
 
@@ -1540,7 +1542,7 @@ func (e *Encoder) encode(iv interface{}) {
 	case Raw:
 		e.rawBytes(v)
 	case reflect.Value:
-		e.encodeValue(v, nil, true)
+		e.encodeValue(v, nil)
 
 	case string:
 		if e.h.StringToRaw {
@@ -1629,12 +1631,12 @@ func (e *Encoder) encode(iv interface{}) {
 			v.CodecEncodeSelf(e)
 		} else if !fastpathEncodeTypeSwitch(iv, e) {
 			// checkfastpath=true (not false), as underlying slice/map type may be fast-path
-			e.encodeValue(reflect.ValueOf(iv), nil, true)
+			e.encodeValue(reflect.ValueOf(iv), nil)
 		}
 	}
 }
 
-func (e *Encoder) encodeValue(rv reflect.Value, fn *codecFn, checkFastpath bool) {
+func (e *Encoder) encodeValue(rv reflect.Value, fn *codecFn) {
 	// if a valid fn is passed, it MUST BE for the dereferenced type of rv
 
 	// We considered using a uintptr (a pointer) retrievable via rv.UnsafeAddr.
@@ -1684,8 +1686,7 @@ TOP:
 
 	if fn == nil {
 		rt := rv.Type()
-		// always pass checkCodecSelfer=true, in case T or ****T is passed, where *T is a Selfer
-		fn = e.h.fn(rt, checkFastpath, true)
+		fn = e.h.fn(rt)
 	}
 	if fn.i.addrE {
 		if rvpValid {
@@ -1818,6 +1819,17 @@ func (e *Encoder) arrayEnd() {
 	e.c = containerArrayEnd
 }
 
+// ----------
+
+func (e *Encoder) sideEncode(v interface{}, bs *[]byte) {
+	rv := baseRV(v)
+	e2 := NewEncoderBytes(bs, e.hh)
+	e2.encodeValue(rv, e.h.fnNoExt(rv.Type()))
+	e2.e.atEndOfEncode()
+	e2.w().end()
+	// xdebugf("sideEncode: xbs: len: %d, %v, v: %v", len(*bs), *bs, v)
+}
+
 func encStructFieldKey(encName string, ee encDriver, w *encWriterSwitch,
 	keyType valueType, encNameAsciiAlphaNum bool, js bool) {
 	var m must
@@ -1851,6 +1863,35 @@ func encStructFieldKey(encName string, ee encDriver, w *encWriterSwitch,
 	}
 }
 
+// type encExtPreambler interface {
+// 	encodeExtPreamble(tag uint8, length int)
+// }
+
+// func encBytesExt(rv interface{}, xtag uint64, ext Ext, h Handle, e encDriver) {
+// 	var bs []byte
+// 	var bufp bytesBufPooler
+// 	if ext == SelfExt {
+// 		bs = bufp.get(1024)[:0]
+// 		rv2 := reflect.ValueOf(v)
+// 		NewEncoderBytes(&bs, h).encodeValue(rv2, h.fnNoExt(rv2.Type()))
+// 	} else {
+// 		bs = ext.WriteExt(v)
+// 	}
+// 	if bs == nil {
+// 		e.EncodeNil()
+// 		return
+// 	}
+// 	if e.h.WriteExt {
+// 		e.encodeExtPreamble(uint8(xtag), len(bs))
+// 		e.w.writeb(bs)
+// 	} else {
+// 		e.EncodeStringBytesRaw(bs)
+// 	}
+// 	if ext == SelfExt {
+// 		bufp.end()
+// 	}
+// }
+
 // func encStringAsRawBytesMaybe(ee encDriver, s string, stringToRaw bool) {
 // 	if stringToRaw {
 // 		ee.EncodeStringBytesRaw(bytesView(s))

codec/fast-path.not.go

@@ -35,7 +35,7 @@ type fastpathA [0]fastpathE
 func (x fastpathA) index(rtid uintptr) int { return -1 }
 
 // func (_ fastpathT) DecSliceUint8V(v []uint8, canChange bool, d *Decoder) (_ []uint8, changed bool) {
-// 	fn := d.h.fn(uint8SliceTyp, true, true)
+// 	fn := d.h.fn(uint8SliceTyp)
 // 	d.kSlice(&fn.i, reflect.ValueOf(&v).Elem())
 // 	return v, true
 // }

codec/gen-helper.generated.go

@@ -77,7 +77,7 @@ func (f genHelperEncoder) IsJSONHandle() bool {
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) EncFallback(iv interface{}) {
 	// f.e.encodeI(iv, false, false)
-	f.e.encodeValue(reflect.ValueOf(iv), nil, false)
+	f.e.encodeValue(reflect.ValueOf(iv), nil)
 }
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
@@ -108,12 +108,12 @@ func (f genHelperEncoder) I2Rtid(v interface{}) uintptr {
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) Extension(rtid uintptr) (xfn *extTypeTagFn) {
-	return f.e.h.getExt(rtid)
+	return f.e.h.getExt(rtid, true)
 }
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) EncExtension(v interface{}, xfFn *extTypeTagFn) {
-	f.e.e.EncodeExt(v, xfFn.tag, xfFn.ext, f.e)
+	f.e.e.EncodeExt(v, xfFn.tag, xfFn.ext)
 }
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
@@ -176,7 +176,7 @@ func (f genHelperDecoder) DecFallback(iv interface{}, chkPtr bool) {
 	if chkPtr {
 		rv = f.d.ensureDecodeable(rv)
 	}
-	f.d.decodeValue(rv, nil, false)
+	f.d.decodeValue(rv, nil)
 }
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
@@ -232,7 +232,7 @@ func (f genHelperDecoder) I2Rtid(v interface{}) uintptr {
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) Extension(rtid uintptr) (xfn *extTypeTagFn) {
-	return f.d.h.getExt(rtid)
+	return f.d.h.getExt(rtid, true)
 }
 
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*

codec/gen-helper.go.tmpl

@@ -94,7 +94,7 @@ func (f genHelperEncoder) IsJSONHandle() bool {
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) EncFallback(iv interface{}) {
 	// f.e.encodeI(iv, false, false)
-	f.e.encodeValue(reflect.ValueOf(iv), nil, false)
+	f.e.encodeValue(reflect.ValueOf(iv), nil)
 }
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) EncTextMarshal(iv encoding.TextMarshaler) {
@@ -120,11 +120,11 @@ func (f genHelperEncoder) I2Rtid(v interface{}) uintptr {
 }
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) Extension(rtid uintptr) (xfn *extTypeTagFn) {
-	return f.e.h.getExt(rtid)
+	return f.e.h.getExt(rtid, true)
 }
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) EncExtension(v interface{}, xfFn *extTypeTagFn) {
-	f.e.e.EncodeExt(v, xfFn.tag, xfFn.ext, f.e)
+	f.e.e.EncodeExt(v, xfFn.tag, xfFn.ext)
 }
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperEncoder) WriteStr(s string) {
@@ -174,7 +174,7 @@ func (f genHelperDecoder) DecFallback(iv interface{}, chkPtr bool) {
 	if chkPtr {
 		rv = f.d.ensureDecodeable(rv)
 	}
-	f.d.decodeValue(rv, nil, false)
+	f.d.decodeValue(rv, nil)
 }
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) DecSliceHelperStart() (decSliceHelper, int) {
@@ -221,7 +221,7 @@ func (f genHelperDecoder) I2Rtid(v interface{}) uintptr {
 }
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) Extension(rtid uintptr) (xfn *extTypeTagFn) {
-	return f.d.h.getExt(rtid)
+	return f.d.h.getExt(rtid, true)
 }
 // FOR USE BY CODECGEN ONLY. IT *WILL* CHANGE WITHOUT NOTICE. *DO NOT USE*
 func (f genHelperDecoder) DecExtension(v interface{}, xfFn *extTypeTagFn) {

codec/gen.go

@@ -810,12 +810,14 @@ func (x *genRunner) enc(varname string, t reflect.Type) {
 		return
 	}
 	if t == rawExtTyp {
-		x.linef("%s r.EncodeRawExt(%s, e)", hasIf.c(true), varname)
+		x.linef("%s r.EncodeRawExt(%s)", hasIf.c(true), varname)
 		return
 	}
-	// only check for extensions if the type is named, and has a packagePath.
+	// only check for extensions if extensions are configured,
+	// and the type is named, and has a packagePath,
+	// and this is not the CodecEncodeSelf or CodecDecodeSelf method (i.e. it is not a Selfer)
 	var arrayOrStruct = tk == reflect.Array || tk == reflect.Struct // meaning varname if of type *T
-	if !x.nx && genImportPath(t) != "" && t.Name() != "" {
+	if !x.nx && varname != genTopLevelVarName && genImportPath(t) != "" && t.Name() != "" {
 		yy := fmt.Sprintf("%sxt%s", genTempVarPfx, mi)
 		x.linef("%s %s := z.Extension(z.I2Rtid(%s)); %s != nil { z.EncExtension(%s, %s) ",
 			hasIf.c(false), yy, varname, yy, varname, yy)
@@ -1435,8 +1437,10 @@ func (x *genRunner) dec(varname string, t reflect.Type, isptr bool) {
 		return
 	}
 
-	// only check for extensions if the type is named, and has a packagePath.
-	if !x.nx && genImportPath(t) != "" && t.Name() != "" {
+	// only check for extensions if extensions are configured,
+	// and the type is named, and has a packagePath,
+	// and this is not the CodecEncodeSelf or CodecDecodeSelf method (i.e. it is not a Selfer)
+	if !x.nx && varname != genTopLevelVarName && genImportPath(t) != "" && t.Name() != "" {
 		// first check if extensions are configued, before doing the interface conversion
 		// x.linef("} else if z.HasExtensions() && z.DecExt(%s) {", varname)
 		yy := fmt.Sprintf("%sxt%s", genTempVarPfx, mi)

codec/helper.go

@@ -458,6 +458,15 @@ var immutableKindsSet = [32]bool{
 	// reflect.UnsafePointer
 }
 
+// SelfExt is a sentinel extension signifying that types
+// registered with it SHOULD be encoded and decoded
+// based on the naive mode of the format.
+//
+// This allows users to define a tag for an extension,
+// but signify that the types should be encoded/decoded as the native encoding.
+// This way, users need not also define how to encode or decode the extension.
+var SelfExt = &extFailWrapper{}
+
 // Selfer defines methods by which a value can encode or decode itself.
 //
 // Any type which implements Selfer will be able to encode or decode itself.
@@ -588,8 +597,8 @@ type BasicHandle struct {
 	inited uint32 // holds if inited, and also handle flags (binary encoding, json handler, etc)
 	mu     sync.Mutex
 	// _      uint32 // padding
-	rtidFns atomicRtidFnSlice
-
+	rtidFns      atomicRtidFnSlice
+	rtidFnsNoExt atomicRtidFnSlice
 	// r []uintptr     // rtids mapped to s above
 }
 
@@ -674,16 +683,50 @@ LOOP:
 	return
 }
 
-func (x *BasicHandle) fn(rt reflect.Type, checkFastpath, checkCodecSelfer bool) (fn *codecFn) {
+func (c *BasicHandle) fn(rt reflect.Type) (fn *codecFn) {
+	return c.fnVia(rt, &c.rtidFns, true)
+}
+
+func (c *BasicHandle) fnNoExt(rt reflect.Type) (fn *codecFn) {
+	return c.fnVia(rt, &c.rtidFnsNoExt, false)
+}
+
+func (c *BasicHandle) fnVia(rt reflect.Type, fs *atomicRtidFnSlice, checkExt bool) (fn *codecFn) {
+	// xdebug2f("fnVia: rt: %v, checkExt: %v", rt, checkExt)
 	rtid := rt2id(rt)
-	sp := x.rtidFns.load()
+	sp := fs.load()
 	if sp != nil {
 		if _, fn = findFn(sp, rtid); fn != nil {
 			// xdebugf("<<<< %c: found fn for %v in rtidfns of size: %v", c.n, rt, len(sp))
 			return
 		}
 	}
-	c := x
+	fn = c.fnLoad(rt, rtid, checkExt)
+	c.mu.Lock()
+	var sp2 []codecRtidFn
+	sp = fs.load()
+	if sp == nil {
+		sp2 = []codecRtidFn{{rtid, fn}}
+		fs.store(sp2)
+		// xdebugf(">>>> adding rt: %v to rtidfns of size: %v", rt, len(sp2))
+		// xdebugf(">>>> loading stored rtidfns of size: %v", len(fs.load()))
+	} else {
+		idx, fn2 := findFn(sp, rtid)
+		if fn2 == nil {
+			sp2 = make([]codecRtidFn, len(sp)+1)
+			copy(sp2, sp[:idx])
+			copy(sp2[idx+1:], sp[idx:])
+			sp2[idx] = codecRtidFn{rtid, fn}
+			c.rtidFns.store(sp2)
+			// xdebugf(">>>> adding rt: %v to rtidfns of size: %v", rt, len(sp2))
+
+		}
+	}
+	c.mu.Unlock()
+	return
+}
+
+func (c *BasicHandle) fnLoad(rt reflect.Type, rtid uintptr, checkExt bool) (fn *codecFn) {
 	// xdebugf("#### for %c: load fn for %v in rtidfns of size: %v", c.n, rt, len(sp))
 	fn = new(codecFn)
 	fi := &(fn.i)
@@ -692,13 +735,9 @@ func (x *BasicHandle) fn(rt reflect.Type, checkFastpath, checkCodecSelfer bool)
 
 	rk := reflect.Kind(ti.kind)
 
-	if checkCodecSelfer && (ti.cs || ti.csp) {
-		fn.fe = (*Encoder).selferMarshal
-		fn.fd = (*Decoder).selferUnmarshal
-		fi.addrF = true
-		fi.addrD = ti.csp
-		fi.addrE = ti.csp
-	} else if rtid == timeTypId && !c.TimeNotBuiltin {
+	// anything can be an extension except the built-in ones: time, raw and rawext
+
+	if rtid == timeTypId && !c.TimeNotBuiltin {
 		fn.fe = (*Encoder).kTime
 		fn.fd = (*Decoder).kTime
 	} else if rtid == rawTypId {
@@ -710,7 +749,7 @@ func (x *BasicHandle) fn(rt reflect.Type, checkFastpath, checkCodecSelfer bool)
 		fi.addrF = true
 		fi.addrD = true
 		fi.addrE = true
-	} else if xfFn := c.getExt(rtid); xfFn != nil {
+	} else if xfFn := c.getExt(rtid, checkExt); xfFn != nil {
 		fi.xfTag, fi.xfFn = xfFn.tag, xfFn.ext
 		fn.fe = (*Encoder).ext
 		fn.fd = (*Decoder).ext
@@ -719,6 +758,12 @@ func (x *BasicHandle) fn(rt reflect.Type, checkFastpath, checkCodecSelfer bool)
 		if rk == reflect.Struct || rk == reflect.Array {
 			fi.addrE = true
 		}
+	} else if ti.cs || ti.csp {
+		fn.fe = (*Encoder).selferMarshal
+		fn.fd = (*Decoder).selferUnmarshal
+		fi.addrF = true
+		fi.addrD = ti.csp
+		fi.addrE = ti.csp
 	} else if supportMarshalInterfaces && c.isBe() && (ti.bm || ti.bmp) && (ti.bu || ti.bup) {
 		fn.fe = (*Encoder).binaryMarshal
 		fn.fd = (*Decoder).binaryUnmarshal
@@ -740,7 +785,7 @@ func (x *BasicHandle) fn(rt reflect.Type, checkFastpath, checkCodecSelfer bool)
 		fi.addrD = ti.tup
 		fi.addrE = ti.tmp
 	} else {
-		if fastpathEnabled && checkFastpath && (rk == reflect.Map || rk == reflect.Slice) {
+		if fastpathEnabled && (rk == reflect.Map || rk == reflect.Slice) {
 			if ti.pkgpath == "" { // un-named slice or map
 				if idx := fastpathAV.index(rtid); idx != -1 {
 					fn.fe = fastpathAV[idx].encfn
@@ -841,7 +886,7 @@ func (x *BasicHandle) fn(rt reflect.Type, checkFastpath, checkCodecSelfer bool)
 				fi.addrD = false
 				rt2 := reflect.SliceOf(ti.elem)
 				fn.fd = func(d *Decoder, xf *codecFnInfo, xrv reflect.Value) {
-					d.h.fn(rt2, true, false).fd(d, xf, xrv.Slice(0, xrv.Len()))
+					d.h.fn(rt2).fd(d, xf, xrv.Slice(0, xrv.Len()))
 				}
 				// fn.fd = (*Decoder).kArray
 			case reflect.Struct:
@@ -865,28 +910,6 @@ func (x *BasicHandle) fn(rt reflect.Type, checkFastpath, checkCodecSelfer bool)
 			}
 		}
 	}
-
-	c.mu.Lock()
-	var sp2 []codecRtidFn
-	sp = c.rtidFns.load()
-	if sp == nil {
-		sp2 = []codecRtidFn{{rtid, fn}}
-		c.rtidFns.store(sp2)
-		// xdebugf(">>>> adding rt: %v to rtidfns of size: %v", rt, len(sp2))
-		// xdebugf(">>>> loading stored rtidfns of size: %v", len(c.rtidFns.load()))
-	} else {
-		idx, fn2 := findFn(sp, rtid)
-		if fn2 == nil {
-			sp2 = make([]codecRtidFn, len(sp)+1)
-			copy(sp2, sp[:idx])
-			copy(sp2[idx+1:], sp[idx:])
-			sp2[idx] = codecRtidFn{rtid, fn}
-			c.rtidFns.store(sp2)
-			// xdebugf(">>>> adding rt: %v to rtidfns of size: %v", rt, len(sp2))
-
-		}
-	}
-	c.mu.Unlock()
 	return
 }
 
@@ -1039,6 +1062,11 @@ type interfaceExtWrapper struct {
 	InterfaceExt
 }
 
+type extFailWrapper struct {
+	bytesExtFailer
+	interfaceExtFailer
+}
+
 type binaryEncodingType struct{}
 
 func (binaryEncodingType) isBinary() bool { return true }
@@ -1157,7 +1185,10 @@ func (o *extHandle) SetExt(rt reflect.Type, tag uint64, ext Ext) (err error) {
 	return
 }
 
-func (o extHandle) getExt(rtid uintptr) (v *extTypeTagFn) {
+func (o extHandle) getExt(rtid uintptr, check bool) (v *extTypeTagFn) {
+	if !check {
+		return
+	}
 	for i := range o {
 		v = &o[i]
 		if v.rtid == rtid || v.rtidptr == rtid {
@@ -2095,6 +2126,29 @@ type codecRtidFn struct {
 	fn   *codecFn
 }
 
+func makeExt(ext interface{}) Ext {
+	if ext == nil {
+		return &extFailWrapper{}
+	}
+	switch t := ext.(type) {
+	case nil:
+		return &extFailWrapper{}
+	case Ext:
+		return t
+	case BytesExt:
+		return &bytesExtWrapper{BytesExt: t}
+	case InterfaceExt:
+		return &interfaceExtWrapper{InterfaceExt: t}
+	}
+	return &extFailWrapper{}
+}
+
+func baseRV(v interface{}) (rv reflect.Value) {
+	for rv = reflect.ValueOf(v); rv.Kind() == reflect.Ptr; rv = rv.Elem() {
+	}
+	return
+}
+
 // ----
 
 // these "checkOverflow" functions must be inlinable, and not call anybody.

codec/json.go

@@ -439,20 +439,23 @@ func (e *jsonEncDriver) EncodeTime(t time.Time) {
 	// v, err := t.MarshalJSON(); if err != nil { e.e.error(err) } e.w.writeb(v)
 }
 
-func (e *jsonEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, en *Encoder) {
-	if v := ext.ConvertExt(rv); v == nil {
+func (e *jsonEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext) {
+	if ext == SelfExt {
+		rv2 := baseRV(rv)
+		e.e.encodeValue(rv2, e.h.fnNoExt(rv2.Type()))
+	} else if v := ext.ConvertExt(rv); v == nil {
 		e.EncodeNil()
 	} else {
-		en.encode(v)
+		e.e.encode(v)
 	}
 }
 
-func (e *jsonEncDriver) EncodeRawExt(re *RawExt, en *Encoder) {
+func (e *jsonEncDriver) EncodeRawExt(re *RawExt) {
 	// only encodes re.Value (never re.Data)
 	if re.Value == nil {
 		e.EncodeNil()
 	} else {
-		en.encode(re.Value)
+		e.e.encode(re.Value)
 	}
 }
 
@@ -467,7 +470,7 @@ func (e *jsonEncDriver) EncodeStringBytesRaw(v []byte) {
 		return
 	}
 	if e.se.InterfaceExt != nil {
-		e.EncodeExt(v, 0, &e.se, e.e)
+		e.EncodeExt(v, 0, &e.se)
 		return
 	}
 
@@ -576,7 +579,6 @@ func (e *jsonEncDriver) atEndOfEncode() {
 			e.w.writen1('\n')
 		}
 	}
-
 }
 
 type jsonDecDriver struct {
@@ -944,13 +946,11 @@ func (d *jsonDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realxta
 		re := rv.(*RawExt)
 		re.Tag = xtag
 		d.d.decode(&re.Value)
+	} else if ext == SelfExt {
+		rv2 := baseRV(rv)
+		d.d.decodeValue(rv2, d.h.fnNoExt(rv2.Type()))
 	} else {
-		var v interface{}
-		if v2, ok := rv.(SelfExt); ok {
-			v = v2.CodecConvertExt()
-		}
-		d.d.decode(&v)
-		ext.UpdateExt(rv, v)
+		d.d.interfaceExtConvertAndDecode(rv, ext)
 	}
 	return
 }
@@ -1390,7 +1390,7 @@ func (h *JsonHandle) recreateEncDriver(ed encDriver) (v bool) {
 
 // SetInterfaceExt sets an extension
 func (h *JsonHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) {
-	return h.SetExt(rt, tag, &interfaceExtWrapper{InterfaceExt: ext})
+	return h.SetExt(rt, tag, makeExt(ext))
 }
 
 func (h *JsonHandle) newEncDriver(e *Encoder) (ee encDriver) {

codec/msgpack.go

@@ -318,8 +318,15 @@ func (e *msgpackEncDriver) EncodeTime(t time.Time) {
 	}
 }
 
-func (e *msgpackEncDriver) EncodeExt(v interface{}, xtag uint64, ext Ext, _ *Encoder) {
-	bs := ext.WriteExt(v)
+func (e *msgpackEncDriver) EncodeExt(v interface{}, xtag uint64, ext Ext) {
+	var bs []byte
+	var bufp bytesBufPooler
+	if ext == SelfExt {
+		bs = bufp.get(1024)[:0]
+		e.e.sideEncode(v, &bs)
+	} else {
+		bs = ext.WriteExt(v)
+	}
 	if bs == nil {
 		e.EncodeNil()
 		return
@@ -330,9 +337,12 @@ func (e *msgpackEncDriver) EncodeExt(v interface{}, xtag uint64, ext Ext, _ *Enc
 	} else {
 		e.EncodeStringBytesRaw(bs)
 	}
+	if ext == SelfExt {
+		bufp.end()
+	}
 }
 
-func (e *msgpackEncDriver) EncodeRawExt(re *RawExt, _ *Encoder) {
+func (e *msgpackEncDriver) EncodeRawExt(re *RawExt) {
 	e.encodeExtPreamble(uint8(re.Tag), len(re.Data))
 	e.w.writeb(re.Data)
 }
@@ -904,6 +914,8 @@ func (d *msgpackDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (real
 		re := rv.(*RawExt)
 		re.Tag = realxtag
 		re.Data = detachZeroCopyBytes(d.br, re.Data, xbs)
+	} else if ext == SelfExt {
+		d.d.sideDecode(rv, xbs)
 	} else {
 		ext.ReadExt(rv, xbs)
 	}
@@ -974,7 +986,7 @@ func (h *MsgpackHandle) Name() string { return "msgpack" }
 
 // SetBytesExt sets an extension
 func (h *MsgpackHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
-	return h.SetExt(rt, tag, &bytesExtWrapper{BytesExt: ext})
+	return h.SetExt(rt, tag, makeExt(ext))
 }
 
 func (h *MsgpackHandle) newEncDriver(e *Encoder) encDriver {

codec/selfext.go

@@ -1,134 +0,0 @@
-// Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
-// Use of this source code is governed by a MIT license found in the LICENSE file.
-
-package codec
-
-import "sync"
-
-// This extension expects that types registered with it implement SelfExt interface.
-//
-// This is used by libraries that support BytesExt e.g. cbor, json.
-var GlobalSelfInterfaceExt InterfaceExt = selfInterfaceExt{}
-
-// var selfExtEncPool = sync.Pool{
-// 	New: func() interface{} { return new(Encoder) },
-// }
-// var selfExtDecPool = sync.Pool{
-// 	New: func() interface{} { return new(Decoder) },
-// }
-
-// SelfExt is the interface that users implement so that their types
-// can, with minimal effort, be able to be an extension while allowing the
-// library handling the encoding/decoding needs easily.
-//
-// We now support the ability for an extension to define a tag,
-// but allow itself to be encoded in a default way.
-//
-// Because we use the type to determine how to encode or decode a value,
-// we cannot tell a value to just encode itself, as that will lead to an
-// infinite recursion.
-//
-// Instead, a value can define how it is to be encoded using a delegate value.
-//
-// If your types implement SelfExt, you can use this to define an extension.
-//
-// At encode time, the interface will call CodecConvertExt, and encode that value.
-// At decode time, the library will call CodecConvertExt, decode into that value, and
-// call the primary object's CodecUpdateExt with that value.
-//
-// The easiest way to do this is via struct embedding:
-//
-//    type T struct {
-//        tHelper
-//    }
-//    func (t *T) CodecConvertExt() { return &t.tHelper }
-//    func (t *T) CodecUpdateExt(interface{}) {  } // no-op (as our delegate is interior pointer)
-//    type tHelper struct {
-//        // ... all t fields
-//    }
-//
-// Usage model:
-//
-//    cborHandle.SetInterfaceExt(reflect.TypeOf(T), 122, codec.GlobalSelfInterfaceExt)
-//
-//    msgpackHandle.SetBytesExt(reflect.TypeOf(T), 122, codec.NewSelfBytesExt(msgpackHandle, 1024))
-//
-type SelfExt interface {
-	CodecConvertExt() interface{}
-	CodecUpdateExt(interface{})
-}
-
-type selfBytesExt struct {
-	// For performance and memory utilization, use sync.Pools
-	// They all have to be local to the Ext, as the Ext is bound to a Handle.
-	p sync.Pool // pool of byte buffers
-	e sync.Pool
-	d sync.Pool
-	h Handle
-	// bufcap int     // cap for each byte buffer created
-}
-
-type selfInterfaceExt struct{}
-
-// NewSelfBytesExt will return a BytesExt implementation,
-// that will call an encoder to encode the value to a stream
-// so it can be placed into the encoder stream, and use a decoder
-// to do the same on the other end.
-//
-// Users can specify a buffer size, and we will initialize that
-// buffer for encoding the type. This allows users manage
-// how big the buffer is based on their knowledge of the type being
-// registered.
-//
-// This extension expects that types registered with it implement SelfExt interface.
-//
-// This is used by libraries that support BytesExt e.g. msgpack, binc.
-func NewSelfBytesExt(h Handle, bufcap int) *selfBytesExt {
-	var v = selfBytesExt{h: h}
-	v.p.New = func() interface{} {
-		return make([]byte, 0, bufcap)
-	}
-	v.e.New = func() interface{} { return NewEncoderBytes(nil, v.h) }
-	v.d.New = func() interface{} { return NewDecoderBytes(nil, v.h) }
-	return &v
-}
-
-func (x *selfBytesExt) WriteExt(v interface{}) (s []byte) {
-	ei := x.e.Get()
-	bi := x.p.Get()
-	defer func() {
-		x.e.Put(ei)
-		x.p.Put(bi)
-	}()
-	b := (bi.([]byte))[:0]
-	e := ei.(*Encoder)
-	e.ResetBytes(&b)
-	e.MustEncode(v.(SelfExt).CodecConvertExt())
-	if len(b) > 0 {
-		s = make([]byte, len(b))
-		copy(s, b)
-	}
-	return
-}
-
-func (x *selfBytesExt) ReadExt(dst interface{}, src []byte) {
-	di := x.d.Get()
-	d := di.(*Decoder)
-	defer func() {
-		d.Release()
-		x.d.Put(di)
-	}()
-	d.ResetBytes(src)
-	v := dst.(SelfExt).CodecConvertExt()
-	d.MustDecode(v)
-	dst.(SelfExt).CodecUpdateExt(v)
-	return
-}
-
-func (x selfInterfaceExt) ConvertExt(v interface{}) interface{} {
-	return v.(SelfExt).CodecConvertExt()
-}
-
-func (x selfInterfaceExt) UpdateExt(dst interface{}, src interface{}) {
-	dst.(SelfExt).CodecUpdateExt(src)
-}

codec/simple.go

@@ -127,17 +127,27 @@ func (e *simpleEncDriver) encLen(bd byte, length int) {
 	}
 }
 
-func (e *simpleEncDriver) EncodeExt(rv interface{}, xtag uint64, ext Ext, _ *Encoder) {
-	bs := ext.WriteExt(rv)
+func (e *simpleEncDriver) EncodeExt(v interface{}, xtag uint64, ext Ext) {
+	var bs []byte
+	var bufp bytesBufPooler
+	if ext == SelfExt {
+		bs = bufp.get(1024)[:0]
+		e.e.sideEncode(v, &bs)
+	} else {
+		bs = ext.WriteExt(v)
+	}
 	if bs == nil {
 		e.EncodeNil()
 		return
 	}
 	e.encodeExtPreamble(uint8(xtag), len(bs))
 	e.w.writeb(bs)
+	if ext == SelfExt {
+		bufp.end()
+	}
 }
 
-func (e *simpleEncDriver) EncodeRawExt(re *RawExt, _ *Encoder) {
+func (e *simpleEncDriver) EncodeRawExt(re *RawExt) {
 	e.encodeExtPreamble(uint8(re.Tag), len(re.Data))
 	e.w.writeb(re.Data)
 }
@@ -472,6 +482,8 @@ func (d *simpleDecDriver) DecodeExt(rv interface{}, xtag uint64, ext Ext) (realx
 		re := rv.(*RawExt)
 		re.Tag = realxtag
 		re.Data = detachZeroCopyBytes(d.br, re.Data, xbs)
+	} else if ext == SelfExt {
+		d.d.sideDecode(rv, xbs)
 	} else {
 		ext.ReadExt(rv, xbs)
 	}
@@ -611,7 +623,7 @@ func (h *SimpleHandle) Name() string { return "simple" }
 
 // SetBytesExt sets an extension
 func (h *SimpleHandle) SetBytesExt(rt reflect.Type, tag uint64, ext BytesExt) (err error) {
-	return h.SetExt(rt, tag, &bytesExtWrapper{BytesExt: ext})
+	return h.SetExt(rt, tag, makeExt(ext))
 }
 
 // func (h *SimpleHandle) hasElemSeparators() bool { return true } // as it implements Write(Map|Array)XXX