codec: set up codebase so as to take advantage of mid-stack inlining support

Mid-stack inlining means that we can elide 2 things
- function call to append to a slice, or look at elements of a slice

To support this, we need to remove the interface calls for encWriter and decReader
and just inline the calls with a simple conditional branch.

To do this, each (en|de)cDriver uses a reference to encWriterSwitch or decReaderSwitch.

codec/binc.go

@@ -102,7 +102,7 @@ func bincdesc(vd, vs byte) string {
 type bincEncDriver struct {
 	e *Encoder
 	h *BincHandle
-	w encWriter
+	w *encWriterSwitch
 	m map[string]uint16 // symbols
 	b [16]byte          // scratch, used for encoding numbers - bigendian style
 	s uint16            // symbols sequencer
@@ -110,6 +110,7 @@ type bincEncDriver struct {
 	encDriverTrackContainerWriter
 	noBuiltInTypes
 	// encNoSeparator
+	_ [1]uint64 // padding
 }
 
 func (e *bincEncDriver) EncodeNil() {
@@ -377,7 +378,7 @@ type bincDecDriver struct {
 
 	d      *Decoder
 	h      *BincHandle
-	r      decReader
+	r      *decReaderSwitch
 	br     bool // bytes reader
 	bdRead bool
 	bd     byte
@@ -392,6 +393,7 @@ type bincDecDriver struct {
 	// decNoSeparator
 
 	b [8 * 8]byte // scratch
+	_ [1]uint64   // padding
 }
 
 func (d *bincDecDriver) readNextBd() {

codec/cbor.go

@@ -107,10 +107,10 @@ type cborEncDriver struct {
 	encDriverNoopContainerWriter
 	// encNoSeparator
 	e *Encoder
-	w encWriter
+	w *encWriterSwitch
 	h *CborHandle
 	x [8]byte
-	_ [3]uint64 // padding
+	// _ [3]uint64 // padding
 }
 
 func (e *cborEncDriver) EncodeNil() {
@@ -287,7 +287,7 @@ func (e *cborEncDriver) encStringBytesS(bb byte, v string) {
 type cborDecDriver struct {
 	d *Decoder
 	h *CborHandle
-	r decReader
+	r *decReaderSwitch
 	// b      [scratchByteArrayLen]byte
 	br     bool // bytes reader
 	bdRead bool
@@ -295,7 +295,7 @@ type cborDecDriver struct {
 	noBuiltInTypes
 	// decNoSeparator
 	decDriverNoopContainerReader
-	_ [3]uint64 // padding
+	// _ [3]uint64 // padding
 }
 
 func (d *cborDecDriver) readNextBd() {

codec/decode.go

@@ -1732,7 +1732,9 @@ type rtid2rv struct {
 type decReaderSwitch struct {
 	rb bytesDecReader
 	// ---- cpu cache line boundary?
-	ri       *ioDecReader
+	ri *ioDecReader
+	bi *bufioDecReader
+
 	mtr, str bool // whether maptype or slicetype are known types
 
 	be    bool // is binary encoding
@@ -1740,73 +1742,92 @@ type decReaderSwitch struct {
 	js    bool // is json handle
 	jsms  bool // is json handle, and MapKeyAsString
 	esep  bool // has elem separators
+	bufio bool // is this a bufioDecReader?
 }
 
-// TODO: Uncomment after mid-stack inlining enabled in go 1.11
-//
-// func (z *decReaderSwitch) unreadn1() {
-// 	if z.bytes {
-// 		z.rb.unreadn1()
-// 	} else {
-// 		z.ri.unreadn1()
-// 	}
-// }
-// func (z *decReaderSwitch) readx(n int) []byte {
-// 	if z.bytes {
-// 		return z.rb.readx(n)
-// 	}
-// 	return z.ri.readx(n)
-// }
-// func (z *decReaderSwitch) readb(s []byte) {
-// 	if z.bytes {
-// 		z.rb.readb(s)
-// 	} else {
-// 		z.ri.readb(s)
-// 	}
-// }
-// func (z *decReaderSwitch) readn1() uint8 {
-// 	if z.bytes {
-// 		return z.rb.readn1()
-// 	}
-// 	return z.ri.readn1()
-// }
-// func (z *decReaderSwitch) numread() int {
-// 	if z.bytes {
-// 		return z.rb.numread()
-// 	}
-// 	return z.ri.numread()
-// }
-// func (z *decReaderSwitch) track() {
-// 	if z.bytes {
-// 		z.rb.track()
-// 	} else {
-// 		z.ri.track()
-// 	}
-// }
-// func (z *decReaderSwitch) stopTrack() []byte {
-// 	if z.bytes {
-// 		return z.rb.stopTrack()
-// 	}
-// 	return z.ri.stopTrack()
-// }
-// func (z *decReaderSwitch) skip(accept *bitset256) (token byte) {
-// 	if z.bytes {
-// 		return z.rb.skip(accept)
-// 	}
-// 	return z.ri.skip(accept)
-// }
-// func (z *decReaderSwitch) readTo(in []byte, accept *bitset256) (out []byte) {
-// 	if z.bytes {
-// 		return z.rb.readTo(in, accept)
-// 	}
-// 	return z.ri.readTo(in, accept)
-// }
-// func (z *decReaderSwitch) readUntil(in []byte, stop byte) (out []byte) {
-// 	if z.bytes {
-// 		return z.rb.readUntil(in, stop)
-// 	}
-// 	return z.ri.readUntil(in, stop)
-// }
+func (z *decReaderSwitch) unreadn1() {
+	if z.bytes {
+		z.rb.unreadn1()
+	} else if z.bufio {
+		z.bi.unreadn1()
+	} else {
+		z.ri.unreadn1()
+	}
+}
+func (z *decReaderSwitch) readx(n int) []byte {
+	if z.bytes {
+		return z.rb.readx(n)
+	} else if z.bufio {
+		return z.bi.readx(n)
+	}
+	return z.ri.readx(n)
+}
+func (z *decReaderSwitch) readb(s []byte) {
+	if z.bytes {
+		z.rb.readb(s)
+	} else if z.bufio {
+		z.bi.readb(s)
+	} else {
+		z.ri.readb(s)
+	}
+}
+func (z *decReaderSwitch) readn1() uint8 {
+	if z.bytes {
+		return z.rb.readn1()
+	} else if z.bufio {
+		return z.bi.readn1()
+	}
+	return z.ri.readn1()
+}
+func (z *decReaderSwitch) numread() int {
+	if z.bytes {
+		return z.rb.numread()
+	} else if z.bufio {
+		return z.bi.numread()
+	}
+	return z.ri.numread()
+}
+func (z *decReaderSwitch) track() {
+	if z.bytes {
+		z.rb.track()
+	} else if z.bufio {
+		z.bi.track()
+	} else {
+		z.ri.track()
+	}
+}
+func (z *decReaderSwitch) stopTrack() []byte {
+	if z.bytes {
+		return z.rb.stopTrack()
+	} else if z.bufio {
+		return z.bi.stopTrack()
+	}
+	return z.ri.stopTrack()
+}
+func (z *decReaderSwitch) skip(accept *bitset256) (token byte) {
+	if z.bytes {
+		return z.rb.skip(accept)
+	} else if z.bufio {
+		return z.bi.skip(accept)
+	}
+	return z.ri.skip(accept)
+}
+func (z *decReaderSwitch) readTo(in []byte, accept *bitset256) (out []byte) {
+	if z.bytes {
+		return z.rb.readTo(in, accept)
+	} else if z.bufio {
+		return z.bi.readTo(in, accept)
+	}
+	return z.ri.readTo(in, accept)
+}
+func (z *decReaderSwitch) readUntil(in []byte, stop byte) (out []byte) {
+	if z.bytes {
+		return z.rb.readUntil(in, stop)
+	} else if z.bufio {
+		return z.bi.readUntil(in, stop)
+	}
+	return z.ri.readUntil(in, stop)
+}
 
 // A Decoder reads and decodes an object from an input stream in the codec format.
 type Decoder struct {
@@ -1817,23 +1838,26 @@ type Decoder struct {
 	d decDriver
 	// NOTE: Decoder shouldn't call it's read methods,
 	// as the handler MAY need to do some coordination.
-	r  decReader
-	h  *BasicHandle
-	bi *bufioDecReader
+	r *decReaderSwitch // formerly decReader
+	h *BasicHandle
+	// bi *bufioDecReader
 	// cache the mapTypeId and sliceTypeId for faster comparisons
 	mtid uintptr
 	stid uintptr
 
+	n   *decNaked
+	nsp *sync.Pool
+
 	// ---- cpu cache line boundary?
 	decReaderSwitch
 
 	// ---- cpu cache line boundary?
 	codecFnPooler
 	// cr containerStateRecv
-	n   *decNaked
-	nsp *sync.Pool
 	err error
 
+	_ [1]uint64 // padding
+
 	// ---- cpu cache line boundary?
 	b  [decScratchByteArrayLen]byte // scratch buffer, used by Decoder and xxxEncDrivers
 	is map[string]string            // used for interning strings
@@ -1882,6 +1906,7 @@ func newDecoder(h Handle) *Decoder {
 }
 
 func (d *Decoder) resetCommon() {
+	d.r = &d.decReaderSwitch
 	d.n.reset()
 	d.d.reset()
 	d.err = nil
@@ -1904,14 +1929,18 @@ func (d *Decoder) Reset(r io.Reader) {
 	if r == nil {
 		return
 	}
-	if d.bi == nil {
-		d.bi = new(bufioDecReader)
-	}
 	d.bytes = false
 	if d.h.ReaderBufferSize > 0 {
-		d.bi.buf = make([]byte, 0, d.h.ReaderBufferSize)
+		if d.bi == nil {
+			d.bi = new(bufioDecReader)
+		}
+		if cap(d.bi.buf) < d.h.ReaderBufferSize {
+			d.bi.buf = make([]byte, 0, d.h.ReaderBufferSize)
+		} else {
+			d.bi.buf = d.bi.buf[:0]
+		}
 		d.bi.reset(r)
-		d.r = d.bi
+		d.bufio = true
 	} else {
 		// d.ri.x = &d.b
 		// d.s = d.sa[:0]
@@ -1919,7 +1948,8 @@ func (d *Decoder) Reset(r io.Reader) {
 			d.ri = new(ioDecReader)
 		}
 		d.ri.reset(r)
-		d.r = d.ri
+		// d.r = d.ri
+		d.bufio = false
 	}
 	d.resetCommon()
 }
@@ -1932,7 +1962,7 @@ func (d *Decoder) ResetBytes(in []byte) {
 	}
 	d.bytes = true
 	d.rb.reset(in)
-	d.r = &d.rb
+	// d.r = &d.rb
 	d.resetCommon()
 }
 
@@ -2382,6 +2412,7 @@ func (d *Decoder) wrapErrstr(v interface{}, err *error) {
 	*err = fmt.Errorf("%s decode error [pos %d]: %v", d.hh.Name(), d.r.numread(), v)
 }
 
+// NumBytesRead returns the number of bytes read
 func (d *Decoder) NumBytesRead() int {
 	return d.r.numread()
 }
@@ -2431,7 +2462,7 @@ func (x decSliceHelper) ElemContainerState(index int) {
 	}
 }
 
-func decByteSlice(r decReader, clen, maxInitLen int, bs []byte) (bsOut []byte) {
+func decByteSlice(r *decReaderSwitch, clen, maxInitLen int, bs []byte) (bsOut []byte) {
 	if clen == 0 {
 		return zeroByteSlice
 	}

codec/encode.go

@@ -900,41 +900,50 @@ type encWriterSwitch struct {
 	wi *ioEncWriter
 	// wb bytesEncWriter
 	wb   bytesEncAppender
-	wx   bool // if bytes, wx=true
-	esep bool // whether it has elem separators
-	isas bool // whether e.as != nil
+	wx   bool      // if bytes, wx=true
+	esep bool      // whether it has elem separators
+	isas bool      // whether e.as != nil
+	js   bool      // here, so that no need to piggy back on *codecFner for this
+	be   bool      // here, so that no need to piggy back on *codecFner for this
+	_    [3]byte   // padding
+	_    [2]uint64 // padding
 }
 
-// // TODO: Uncomment after mid-stack inlining enabled in go 1.11
-
-// func (z *encWriterSwitch) writeb(s []byte) {
-// 	if z.wx {
-// 		z.wb.writeb(s)
-// 	} else {
-// 		z.wi.writeb(s)
-// 	}
-// }
-// func (z *encWriterSwitch) writestr(s string) {
-// 	if z.wx {
-// 		z.wb.writestr(s)
-// 	} else {
-// 		z.wi.writestr(s)
-// 	}
-// }
-// func (z *encWriterSwitch) writen1(b1 byte) {
-// 	if z.wx {
-// 		z.wb.writen1(b1)
-// 	} else {
-// 		z.wi.writen1(b1)
-// 	}
-// }
-// func (z *encWriterSwitch) writen2(b1, b2 byte) {
-// 	if z.wx {
-// 		z.wb.writen2(b1, b2)
-// 	} else {
-// 		z.wi.writen2(b1, b2)
-// 	}
-// }
+func (z *encWriterSwitch) writeb(s []byte) {
+	if z.wx {
+		z.wb.writeb(s)
+	} else {
+		z.wi.writeb(s)
+	}
+}
+func (z *encWriterSwitch) writestr(s string) {
+	if z.wx {
+		z.wb.writestr(s)
+	} else {
+		z.wi.writestr(s)
+	}
+}
+func (z *encWriterSwitch) writen1(b1 byte) {
+	if z.wx {
+		z.wb.writen1(b1)
+	} else {
+		z.wi.writen1(b1)
+	}
+}
+func (z *encWriterSwitch) writen2(b1, b2 byte) {
+	if z.wx {
+		z.wb.writen2(b1, b2)
+	} else {
+		z.wi.writen2(b1, b2)
+	}
+}
+func (z *encWriterSwitch) atEndOfEncode() {
+	if z.wx {
+		z.wb.atEndOfEncode()
+	} else {
+		z.wi.atEndOfEncode()
+	}
+}
 
 // An Encoder writes an object to an output stream in the codec format.
 type Encoder struct {
@@ -943,24 +952,20 @@ type Encoder struct {
 	e encDriver
 	// NOTE: Encoder shouldn't call it's write methods,
 	// as the handler MAY need to do some coordination.
-	w encWriter
+	w *encWriterSwitch // formerly encWriter
 
-	h  *BasicHandle
-	bw *bufio.Writer
+	h *BasicHandle
+	// bw *bufio.Writer
 	as encDriverAsis
 
-	// ---- cpu cache line boundary?
+	err error
 
 	// ---- cpu cache line boundary?
 	encWriterSwitch
-	err error
 
 	// ---- cpu cache line boundary?
 	codecFnPooler
 	ci set
-	js bool    // here, so that no need to piggy back on *codecFner for this
-	be bool    // here, so that no need to piggy back on *codecFner for this
-	_  [6]byte // padding
 
 	// ---- writable fields during execution --- *try* to keep in sep cache line
 
@@ -968,6 +973,7 @@ type Encoder struct {
 	// b [scratchByteArrayLen]byte
 	// _ [cacheLineSize - scratchByteArrayLen]byte // padding
 	b [cacheLineSize - 0]byte // used for encoding a chan or (non-addressable) array of bytes
+	_ [1]uint64               // padding
 }
 
 // NewEncoder returns an Encoder for encoding into an io.Writer.
@@ -999,6 +1005,7 @@ func newEncoder(h Handle) *Encoder {
 }
 
 func (e *Encoder) resetCommon() {
+	e.w = &e.encWriterSwitch
 	if e.e == nil || e.hh.recreateEncDriver(e.e) {
 		e.e = e.hh.newEncDriver(e)
 		e.as, e.isas = e.e.(encDriverAsis)
@@ -1025,11 +1032,11 @@ func (e *Encoder) Reset(w io.Writer) {
 	e.wx = false
 	e.wi.w = w
 	if e.h.WriterBufferSize > 0 {
-		e.bw = bufio.NewWriterSize(w, e.h.WriterBufferSize)
-		e.wi.bw = e.bw
-		e.wi.sw = e.bw
-		e.wi.fw = e.bw
-		e.wi.ww = e.bw
+		bw := bufio.NewWriterSize(w, e.h.WriterBufferSize)
+		e.wi.bw = bw
+		e.wi.sw = bw
+		e.wi.fw = bw
+		e.wi.ww = bw
 	} else {
 		if e.wi.bw, ok = w.(io.ByteWriter); !ok {
 			e.wi.bw = e.wi
@@ -1040,7 +1047,7 @@ func (e *Encoder) Reset(w io.Writer) {
 		e.wi.fw, _ = w.(ioFlusher)
 		e.wi.ww = w
 	}
-	e.w = e.wi
+	// e.w = e.wi
 	e.resetCommon()
 }
 
@@ -1058,7 +1065,7 @@ func (e *Encoder) ResetBytes(out *[]byte) {
 	}
 	e.wx = true
 	e.wb.reset(in, out)
-	e.w = &e.wb
+	// e.w = &e.wb
 	e.resetCommon()
 }
 

codec/helper.go

@@ -135,7 +135,7 @@ const (
 	wordSizeBits = 32 << (^uint(0) >> 63) // strconv.IntSize
 	wordSize     = wordSizeBits / 8
 
-	maxLevelsEmbedding = 15 // use this, so structFieldInfo fits into 8 bytes
+	maxLevelsEmbedding = 14 // use this, so structFieldInfo fits into 8 bytes
 )
 
 var (
@@ -628,7 +628,7 @@ func (noElemSeparators) recreateEncDriver(e encDriver) (v bool) { return }
 // Users must already slice the x completely, because we will not reslice.
 type bigenHelper struct {
 	x []byte // must be correctly sliced to appropriate len. slicing is a cost.
-	w encWriter
+	w *encWriterSwitch
 }
 
 func (z bigenHelper) writeUint16(v uint16) {
@@ -817,6 +817,7 @@ type structFieldInfo struct {
 
 	encNameAsciiAlphaNum bool // the encName only contains ascii alphabet and numbers
 	structFieldInfoFlag
+	_ [1]byte // padding
 }
 
 func (si *structFieldInfo) setToZeroValue(v reflect.Value) {

codec/json.go

@@ -115,8 +115,7 @@ func init() {
 // ----------------
 
 type jsonEncDriverTypical struct {
-	w encWriter
-	// w  *encWriterSwitch
+	w  *encWriterSwitch
 	b  *[jsonScratchArrayLen]byte
 	tw bool // term white space
 	c  containerState
@@ -205,7 +204,7 @@ func (e *jsonEncDriverTypical) atEndOfEncode() {
 // ----------------
 
 type jsonEncDriverGeneric struct {
-	w encWriter // encWriter // *encWriterSwitch
+	w *encWriterSwitch
 	b *[jsonScratchArrayLen]byte
 	c containerState
 	// ds string // indent string
@@ -407,12 +406,13 @@ type jsonEncDriver struct {
 	noBuiltInTypes
 	e  *Encoder
 	h  *JsonHandle
-	ew encWriter // encWriter // *encWriterSwitch
+	ew *encWriterSwitch
 	se extWrapper
 	// ---- cpu cache line boundary?
 	bs []byte // scratch
 	// ---- cpu cache line boundary?
 	b [jsonScratchArrayLen]byte // scratch (encode time,
+	_ [2]uint64                 // padding
 }
 
 func (e *jsonEncDriver) EncodeNil() {
@@ -569,7 +569,7 @@ type jsonDecDriver struct {
 	noBuiltInTypes
 	d  *Decoder
 	h  *JsonHandle
-	r  decReader // *decReaderSwitch // decReader
+	r  *decReaderSwitch // decReader
 	se extWrapper
 
 	// ---- writable fields during execution --- *try* to keep in sep cache line
@@ -584,7 +584,7 @@ type jsonDecDriver struct {
 	b  [jsonScratchArrayLen]byte // scratch 1, used for parsing strings or numbers or time.Time
 	b2 [jsonScratchArrayLen]byte // scratch 2, used only for readUntil, decNumBytes
 
-	_ [3]uint64 // padding
+	// _ [3]uint64 // padding
 	// n jsonNum
 }
 

codec/msgpack.go

@@ -199,10 +199,10 @@ type msgpackEncDriver struct {
 	encDriverNoopContainerWriter
 	// encNoSeparator
 	e *Encoder
-	w encWriter
+	w *encWriterSwitch
 	h *MsgpackHandle
 	x [8]byte
-	_ [3]uint64 // padding
+	// _ [3]uint64 // padding
 }
 
 func (e *msgpackEncDriver) EncodeNil() {
@@ -414,7 +414,7 @@ func (e *msgpackEncDriver) writeContainerLen(ct msgpackContainerType, l int) {
 
 type msgpackDecDriver struct {
 	d *Decoder
-	r decReader // *Decoder decReader decReaderT
+	r *decReaderSwitch
 	h *MsgpackHandle
 	// b      [scratchByteArrayLen]byte
 	bd     byte
@@ -424,7 +424,7 @@ type msgpackDecDriver struct {
 	// noStreamingCodec
 	// decNoSeparator
 	decDriverNoopContainerReader
-	_ [3]uint64 // padding
+	// _ [3]uint64 // padding
 }
 
 // Note: This returns either a primitive (int, bool, etc) for non-containers,

codec/simple.go

@@ -36,12 +36,12 @@ type simpleEncDriver struct {
 	// encNoSeparator
 	e *Encoder
 	h *SimpleHandle
-	w encWriter
+	w *encWriterSwitch
 	b [8]byte
 	// c containerState
 	encDriverTrackContainerWriter
 	// encDriverNoopContainerWriter
-	_ [2]uint64 // padding
+	_ [3]uint64 // padding
 }
 
 func (e *simpleEncDriver) EncodeNil() {
@@ -201,7 +201,7 @@ func (e *simpleEncDriver) EncodeTime(t time.Time) {
 type simpleDecDriver struct {
 	d      *Decoder
 	h      *SimpleHandle
-	r      decReader
+	r      *decReaderSwitch
 	bdRead bool
 	bd     byte
 	br     bool // a bytes reader?
@@ -210,7 +210,7 @@ type simpleDecDriver struct {
 	noBuiltInTypes
 	// noStreamingCodec
 	decDriverNoopContainerReader
-	_ [3]uint64 // padding
+	// _ [3]uint64 // padding
 }
 
 func (d *simpleDecDriver) readNextBd() {