codec: add buffered reading/writing support and overall cleanup and fixes

test: add testNumRepeatString flag: default=8, with testlargestrings using value of 32

simplify io.RW handlers

support buffered reading and writing when configured
  - support buffered writing of io.Writer leveraging WriterBufferSize
  - support ReaderBufferSize by implementing a bufioDecReader optimized for codec.
    we couldn't use bufio.Reader because it didn't expose the underlying byte buffer
    which we need for search methods of decReader i.e. skip, readTo and readUntil
  - add tests

test: cleanup and re-organization around build tags
  - changed build tag: codecbench to generated
  - moved generated files to their dedicated files, to only be built as needed
  - separated bench suites into: Codec, X, CodecX and CodecXGen

codec/codec_test.go

@@ -64,12 +64,15 @@ const (
 	testVerifyForPython
 )
 
-const testSkipRPCTests = false
+// const testSkipRPCTests = false
 
 var (
 	testTableNumPrimitives int
 	testTableIdxTime       int
 	testTableNumMaps       int
+
+	// set this when running using bufio, etc
+	testSkipRPCTests = false
 )
 
 var (
@@ -329,7 +332,7 @@ func testVerifyVal(v interface{}, arg testVerifyArg) (v2 interface{}) {
 func testInit() {
 	gob.Register(new(TestStrucFlex))
 	if testInitDebug {
-		ts0 := newTestStrucFlex(2, false, !testSkipIntf, false)
+		ts0 := newTestStrucFlex(2, testNumRepeatString, false, !testSkipIntf, false)
 		logT(nil, "====> depth: %v, ts: %#v\n", 2, ts0)
 	}
 
@@ -473,7 +476,7 @@ ugorji
 	table = append(table, primitives)
 	table = append(table, testMbsT(primitives))
 	table = append(table, maps...)
-	table = append(table, newTestStrucFlex(0, false, !testSkipIntf, false))
+	table = append(table, newTestStrucFlex(0, testNumRepeatString, false, !testSkipIntf, false))
 
 	tableVerify = make([]interface{}, len(table))
 	tableTestNilVerify = make([]interface{}, len(table))
@@ -694,7 +697,7 @@ func testCodecMiscOne(t *testing.T, h Handle) {
 	}
 
 	// func TestMsgpackDecodePtr(t *testing.T) {
-	ts := newTestStrucFlex(testDepth, false, !testSkipIntf, false)
+	ts := newTestStrucFlex(testDepth, testNumRepeatString, false, !testSkipIntf, false)
 	b, err = testMarshalErr(ts, h, t, "pointer-to-struct")
 	if len(b) < 40 {
 		logT(t, "------- Size must be > 40. Size: %d", len(b))
@@ -1455,7 +1458,7 @@ func doTestMsgpackRpcSpecPythonClientToGoSvc(t *testing.T) {
 
 func doTestSwallowAndZero(t *testing.T, h Handle) {
 	testOnce.Do(testInitAll)
-	v1 := newTestStrucFlex(testDepth, false, false, false)
+	v1 := newTestStrucFlex(testDepth, testNumRepeatString, false, false, false)
 	var b1 []byte
 
 	e1 := NewEncoderBytes(&b1, h)
@@ -2110,6 +2113,70 @@ after the new line
 	}
 }
 
+func TestBufioDecReader(t *testing.T) {
+	// try to read 85 bytes in chunks of 7 at a time.
+	var s = strings.Repeat("01234'56789      ", 5)
+	// fmt.Printf("s: %s\n", s)
+	var r = strings.NewReader(s)
+	var br = &bufioDecReader{r: r, buf: make([]byte, 0, 13)}
+	b, err := ioutil.ReadAll(br)
+	if err != nil {
+		panic(err)
+	}
+	var s2 = string(b)
+	// fmt.Printf("s==s2: %v, len(s): %v, len(b): %v, len(s2): %v\n", s == s2, len(s), len(b), len(s2))
+	if s != s2 {
+		logT(t, "not equal: \ns:  %s\ns2: %s", s, s2)
+		failT(t)
+	}
+	// Now, test search functions for skip, readTo and readUntil
+	// readUntil ', readTo ', skip whitespace. 3 times in a loop, each time compare the token and/or outs
+	// readUntil: see: 56789
+	var out []byte
+	var token byte
+	br = &bufioDecReader{r: strings.NewReader(s), buf: make([]byte, 0, 7)}
+	// println()
+	for _, v2 := range [...]string{
+		`01234'`,
+		`56789      01234'`,
+		`56789      01234'`,
+		`56789      01234'`,
+	} {
+		out = br.readUntil(nil, '\'')
+		testDeepEqualErr(string(out), v2, t, "-")
+		// fmt.Printf("readUntil: out: `%s`\n", out)
+	}
+	br = &bufioDecReader{r: strings.NewReader(s), buf: make([]byte, 0, 7)}
+	// println()
+	for range [4]struct{}{} {
+		out = br.readTo(nil, &jsonNumSet)
+		testDeepEqualErr(string(out), `01234`, t, "-")
+		// fmt.Printf("readTo: out: `%s`\n", out)
+		out = br.readUntil(nil, '\'')
+		testDeepEqualErr(string(out), "'", t, "-")
+		// fmt.Printf("readUntil: out: `%s`\n", out)
+		out = br.readTo(nil, &jsonNumSet)
+		testDeepEqualErr(string(out), `56789`, t, "-")
+		// fmt.Printf("readTo: out: `%s`\n", out)
+		out = br.readUntil(nil, '0')
+		testDeepEqualErr(string(out), `      0`, t, "-")
+		// fmt.Printf("readUntil: out: `%s`\n", out)
+		br.UnreadByte()
+	}
+	br = &bufioDecReader{r: strings.NewReader(s), buf: make([]byte, 0, 7)}
+	// println()
+	for range [4]struct{}{} {
+		out = br.readUntil(nil, ' ')
+		testDeepEqualErr(string(out), `01234'56789 `, t, "-")
+		// fmt.Printf("readUntil: out: |%s|\n", out)
+		token = br.skip(&jsonCharWhitespaceSet)
+		testDeepEqualErr(token, byte('0'), t, "-")
+		// fmt.Printf("skip: token: '%c'\n", token)
+		br.UnreadByte()
+	}
+	// println()
+}
+
 // TODO:
 //   Add Tests for:
 //   - decoding empty list/map in stream into a nil slice/map

codec/decode.go

@@ -22,6 +22,10 @@ const (
 var (
 	onlyMapOrArrayCanDecodeIntoStructErr = errors.New("only encoded map or array can be decoded into a struct")
 	cannotDecodeIntoNilErr               = errors.New("cannot decode into nil")
+
+	decUnreadByteNothingToReadErr   = errors.New("cannot unread - nothing has been read")
+	decUnreadByteLastByteNotReadErr = errors.New("cannot unread - last byte has not been read")
+	decUnreadByteUnknownErr         = errors.New("cannot unread - reason unknown")
 )
 
 // decReader abstracts the reading source, allowing implementations that can
@@ -50,10 +54,10 @@ type decReader interface {
 	readUntil(in []byte, stop byte) (out []byte)
 }
 
-type decReaderByteScanner interface {
-	io.Reader
-	io.ByteScanner
-}
+// type decReaderByteScanner interface {
+// 	io.Reader
+// 	io.ByteScanner
+// }
 
 type decDriver interface {
 	// this will check if the next token is a break.
@@ -201,20 +205,366 @@ type DecodeOptions struct {
 	// If true, we will delete the mapping of the key.
 	// Else, just set the mapping to the zero value of the type.
 	DeleteOnNilMapValue bool
+
+	// ReaderBufferSize is the size of the buffer used when reading.
+	//
+	// if > 0, we use a smart buffer internally for performance purposes.
+	ReaderBufferSize int
 }
 
 // ------------------------------------
 
-// ioDecByteScanner implements Read(), ReadByte(...), UnreadByte(...) methods
-// of io.Reader, io.ByteScanner.
-type ioDecByteScanner struct {
-	r  io.Reader
-	l  byte    // last byte
-	ls byte    // last byte status. 0: init-canDoNothing, 1: canRead, 2: canUnread
-	b  [1]byte // tiny buffer for reading single bytes
+type bufioDecReader struct {
+	buf []byte
+	r   io.Reader
+
+	c   int // cursor
+	n   int // num read
+	err error
+
+	trb bool
+	tr  []byte
+
+	b [8]byte
+}
+
+func (z *bufioDecReader) reset(r io.Reader) {
+	z.r, z.c, z.n, z.err, z.trb = r, 0, 0, nil, false
+	if z.tr != nil {
+		z.tr = z.tr[:0]
+	}
+}
+
+func (z *bufioDecReader) Read(p []byte) (n int, err error) {
+	if z.err != nil {
+		return 0, z.err
+	}
+	p0 := p
+	n = copy(p, z.buf[z.c:])
+	z.c += n
+	if z.c == len(z.buf) {
+		z.c = 0
+	}
+	z.n += n
+	if len(p) == n {
+		if z.c == 0 {
+			z.buf = z.buf[:1]
+			z.buf[0] = p[len(p)-1]
+			z.c = 1
+		}
+		if z.trb {
+			z.tr = append(z.tr, p0[:n]...)
+		}
+		return
+	}
+	p = p[n:]
+	var n2 int
+	// if we are here, then z.buf is all read
+	if len(p) > len(z.buf) {
+		n2, err = decReadFull(z.r, p)
+		n += n2
+		z.n += n2
+		z.err = err
+		// don't return EOF if some bytes were read. keep for next time.
+		if n > 0 && err == io.EOF {
+			err = nil
+		}
+		// always keep last byte in z.buf
+		z.buf = z.buf[:1]
+		z.buf[0] = p[len(p)-1]
+		z.c = 1
+		if z.trb {
+			z.tr = append(z.tr, p0[:n]...)
+		}
+		return
+	}
+	// z.c is now 0, and len(p) <= len(z.buf)
+	for len(p) > 0 && z.err == nil {
+		// println("len(p) loop starting ... ")
+		z.c = 0
+		z.buf = z.buf[0:cap(z.buf)]
+		n2, err = z.r.Read(z.buf)
+		if n2 > 0 {
+			if err == io.EOF {
+				err = nil
+			}
+			z.buf = z.buf[:n2]
+			n2 = copy(p, z.buf)
+			z.c = n2
+			n += n2
+			z.n += n2
+			p = p[n2:]
+		}
+		z.err = err
+		// println("... len(p) loop done")
+	}
+	if z.c == 0 {
+		z.buf = z.buf[:1]
+		z.buf[0] = p[len(p)-1]
+		z.c = 1
+	}
+	if z.trb {
+		z.tr = append(z.tr, p0[:n]...)
+	}
+	return
+}
+
+func (z *bufioDecReader) ReadByte() (b byte, err error) {
+	z.b[0] = 0
+	_, err = z.Read(z.b[:1])
+	b = z.b[0]
+	return
+}
+
+func (z *bufioDecReader) UnreadByte() (err error) {
+	if z.err != nil {
+		return z.err
+	}
+	if z.c > 0 {
+		z.c--
+		z.n--
+		if z.trb {
+			z.tr = z.tr[:len(z.tr)-1]
+		}
+		return
+	}
+	return decUnreadByteNothingToReadErr
+}
+
+func (z *bufioDecReader) numread() int {
+	return z.n
+}
+
+func (z *bufioDecReader) readx(n int) (bs []byte) {
+	if n <= 0 || z.err != nil {
+		return
+	}
+	if z.c+n <= len(z.buf) {
+		bs = z.buf[z.c : z.c+n]
+		z.n += n
+		z.c += n
+		if z.trb {
+			z.tr = append(z.tr, bs...)
+		}
+		return
+	}
+	bs = make([]byte, n)
+	_, err := z.Read(bs)
+	if err != nil {
+		panic(err)
+	}
+	return
+}
+
+func (z *bufioDecReader) readb(bs []byte) {
+	_, err := z.Read(bs)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (z *bufioDecReader) readn1eof() (b uint8, eof bool) {
+	b, err := z.ReadByte()
+	if err != nil {
+		if err == io.EOF {
+			eof = true
+		} else {
+			panic(err)
+		}
+	}
+	return
+}
+
+func (z *bufioDecReader) readn1() (b uint8) {
+	b, err := z.ReadByte()
+	if err != nil {
+		panic(err)
+	}
+	return
+}
+
+func (z *bufioDecReader) readn3() (b1, b2, b3 uint8) {
+	z.readb(z.b[:3])
+	return z.b[0], z.b[1], z.b[2]
+}
+
+func (z *bufioDecReader) readn4() (b1, b2, b3, b4 uint8) {
+	z.readb(z.b[:4])
+	return z.b[0], z.b[1], z.b[2], z.b[3]
+}
+
+func (z *bufioDecReader) search(in []byte, accept *bitset256, stop, flag uint8) (token byte, out []byte) {
+	// flag: 1 (skip), 2 (readTo), 4 (readUntil)
+	if flag == 4 {
+		for i := z.c; i < len(z.buf); i++ {
+			if z.buf[i] == stop {
+				token = z.buf[i]
+				z.n = z.n + (i - z.c) - 1
+				i++
+				out = z.buf[z.c:i]
+				if z.trb {
+					z.tr = append(z.tr, z.buf[z.c:i]...)
+				}
+				z.c = i
+				return
+			}
+		}
+	} else {
+		for i := z.c; i < len(z.buf); i++ {
+			if !accept.isset(z.buf[i]) {
+				token = z.buf[i]
+				z.n = z.n + (i - z.c) - 1
+				if flag == 1 {
+					i++
+				} else {
+					out = z.buf[z.c:i]
+				}
+				if z.trb {
+					z.tr = append(z.tr, z.buf[z.c:i]...)
+				}
+				z.c = i
+				return
+			}
+		}
+	}
+	z.n += len(z.buf) - z.c
+	if flag != 1 {
+		out = append(in, z.buf[z.c:]...)
+	}
+	if z.trb {
+		z.tr = append(z.tr, z.buf[z.c:]...)
+	}
+	var n2 int
+	if z.err != nil {
+		return
+	}
+	for {
+		z.c = 0
+		z.buf = z.buf[0:cap(z.buf)]
+		n2, z.err = z.r.Read(z.buf)
+		if n2 > 0 && z.err != nil {
+			z.err = nil
+		}
+		z.buf = z.buf[:n2]
+		if flag == 4 {
+			for i := 0; i < n2; i++ {
+				if z.buf[i] == stop {
+					token = z.buf[i]
+					z.n += i - 1
+					i++
+					out = append(out, z.buf[z.c:i]...)
+					if z.trb {
+						z.tr = append(z.tr, z.buf[z.c:i]...)
+					}
+					z.c = i
+					return
+				}
+			}
+		} else {
+			for i := 0; i < n2; i++ {
+				if !accept.isset(z.buf[i]) {
+					token = z.buf[i]
+					z.n += i - 1
+					if flag == 1 {
+						i++
+					}
+					if flag != 1 {
+						out = append(out, z.buf[z.c:i]...)
+					}
+					if z.trb {
+						z.tr = append(z.tr, z.buf[z.c:i]...)
+					}
+					z.c = i
+					return
+				}
+			}
+		}
+		if flag != 1 {
+			out = append(out, z.buf[:n2]...)
+		}
+		z.n += n2
+		if z.err != nil {
+			return
+		}
+		if z.trb {
+			z.tr = append(z.tr, z.buf[:n2]...)
+		}
+	}
+}
+
+func (z *bufioDecReader) skip(accept *bitset256) (token byte) {
+	token, _ = z.search(nil, accept, 0, 1)
+	return
+}
+
+func (z *bufioDecReader) readTo(in []byte, accept *bitset256) (out []byte) {
+	_, out = z.search(in, accept, 0, 2)
+	return
+}
+
+func (z *bufioDecReader) readUntil(in []byte, stop byte) (out []byte) {
+	_, out = z.search(in, nil, stop, 4)
+	return
+}
+
+func (z *bufioDecReader) unreadn1() {
+	err := z.UnreadByte()
+	if err != nil {
+		panic(err)
+	}
+}
+
+func (z *bufioDecReader) track() {
+	if z.tr != nil {
+		z.tr = z.tr[:0]
+	}
+	z.trb = true
+}
+
+func (z *bufioDecReader) stopTrack() (bs []byte) {
+	z.trb = false
+	return z.tr
+}
+
+// ioDecReader is a decReader that reads off an io.Reader.
+//
+// It also has a fallback implementation of ByteScanner if needed.
+type ioDecReader struct {
+	r io.Reader // the reader passed in
+
+	rr io.Reader
+	br io.ByteScanner
+
+	l   byte    // last byte
+	ls  byte    // last byte status. 0: init-canDoNothing, 1: canRead, 2: canUnread
+	b   [4]byte // tiny buffer for reading single bytes
+	trb bool    // tracking bytes turned on
+
+	// temp byte array re-used internally for efficiency during read.
+	// shares buffer with Decoder, so we keep size of struct within 8 words.
+	x  *[scratchByteArrayLen]byte
+	n  int    // num read
+	tr []byte // tracking bytes read
+}
+
+func (z *ioDecReader) reset(r io.Reader) {
+	z.r = r
+	z.rr = r
+	z.l, z.ls, z.n, z.trb = 0, 0, 0, false
+	if z.tr != nil {
+		z.tr = z.tr[:0]
+	}
+	var ok bool
+	if z.br, ok = r.(io.ByteScanner); !ok {
+		z.br = z
+		z.rr = z
+	}
 }
 
-func (z *ioDecByteScanner) Read(p []byte) (n int, err error) {
+func (z *ioDecReader) Read(p []byte) (n int, err error) {
+	if len(p) == 0 {
+		return
+	}
 	var firstByte bool
 	if z.ls == 1 {
 		z.ls = 2
@@ -240,8 +590,8 @@ func (z *ioDecByteScanner) Read(p []byte) (n int, err error) {
 	return
 }
 
-func (z *ioDecByteScanner) ReadByte() (c byte, err error) {
-	n, err := z.Read(z.b[:])
+func (z *ioDecReader) ReadByte() (c byte, err error) {
+	n, err := z.Read(z.b[:1])
 	if n == 1 {
 		c = z.b[0]
 		if err == io.EOF {
@@ -251,30 +601,20 @@ func (z *ioDecByteScanner) ReadByte() (c byte, err error) {
 	return
 }
 
-func (z *ioDecByteScanner) UnreadByte() (err error) {
-	x := z.ls
-	if x == 0 {
-		err = errors.New("cannot unread - nothing has been read")
-	} else if x == 1 {
-		err = errors.New("cannot unread - last byte has not been read")
-	} else if x == 2 {
+func (z *ioDecReader) UnreadByte() (err error) {
+	switch z.ls {
+	case 2:
 		z.ls = 1
+	case 0:
+		err = decUnreadByteNothingToReadErr
+	case 1:
+		err = decUnreadByteLastByteNotReadErr
+	default:
+		err = decUnreadByteUnknownErr
 	}
 	return
 }
 
-// ioDecReader is a decReader that reads off an io.Reader
-type ioDecReader struct {
-	br decReaderByteScanner
-	// temp byte array re-used internally for efficiency during read.
-	// shares buffer with Decoder, so we keep size of struct within 8 words.
-	x   *[scratchByteArrayLen]byte
-	bs  ioDecByteScanner
-	n   int    // num read
-	tr  []byte // tracking bytes read
-	trb bool
-}
-
 func (z *ioDecReader) numread() int {
 	return z.n
 }
@@ -288,7 +628,7 @@ func (z *ioDecReader) readx(n int) (bs []byte) {
 	} else {
 		bs = make([]byte, n)
 	}
-	if _, err := io.ReadAtLeast(z.br, bs, n); err != nil {
+	if _, err := decReadFull(z.rr, bs); err != nil {
 		panic(err)
 	}
 	z.n += len(bs)
@@ -299,14 +639,13 @@ func (z *ioDecReader) readx(n int) (bs []byte) {
 }
 
 func (z *ioDecReader) readb(bs []byte) {
-	if len(bs) == 0 {
-		return
-	}
-	n, err := io.ReadAtLeast(z.br, bs, len(bs))
-	z.n += n
-	if err != nil {
+	// if len(bs) == 0 {
+	// 	return
+	// }
+	if _, err := decReadFull(z.rr, bs); err != nil {
 		panic(err)
 	}
+	z.n += len(bs)
 	if z.trb {
 		z.tr = append(z.tr, bs...)
 	}
@@ -340,37 +679,13 @@ func (z *ioDecReader) readn1() (b uint8) {
 }
 
 func (z *ioDecReader) readn3() (b1, b2, b3 uint8) {
-	var err error
-	if b1, err = z.br.ReadByte(); err == nil {
-		if b2, err = z.br.ReadByte(); err == nil {
-			if b3, err = z.br.ReadByte(); err == nil {
-				z.n += 3
-				if z.trb {
-					z.tr = append(z.tr, b1, b2, b3)
-				}
-				return
-			}
-		}
-	}
-	panic(err)
+	z.readb(z.b[:3])
+	return z.b[0], z.b[1], z.b[2]
 }
 
 func (z *ioDecReader) readn4() (b1, b2, b3, b4 uint8) {
-	var err error
-	if b1, err = z.br.ReadByte(); err == nil {
-		if b2, err = z.br.ReadByte(); err == nil {
-			if b3, err = z.br.ReadByte(); err == nil {
-				if b4, err = z.br.ReadByte(); err == nil {
-					z.n += 4
-					if z.trb {
-						z.tr = append(z.tr, b1, b2, b3, b4)
-					}
-					return
-				}
-			}
-		}
-	}
-	panic(err)
+	z.readb(z.b[:4])
+	return z.b[0], z.b[1], z.b[2], z.b[3]
 }
 
 func (z *ioDecReader) skip(accept *bitset256) (token byte) {
@@ -549,14 +864,13 @@ func (z *bytesDecReader) skip(accept *bitset256) (token byte) {
 	}
 	blen := len(z.b)
 	for i := z.c; i < blen; i++ {
-		if accept.isset(z.b[i]) {
-			continue
+		if !accept.isset(z.b[i]) {
+			token = z.b[i]
+			i++
+			z.a -= (i - z.c)
+			z.c = i
+			return
 		}
-		token = z.b[i]
-		i++
-		z.a -= (i - z.c)
-		z.c = i
-		return
 	}
 	z.a, z.c = 0, blen
 	return
@@ -1343,7 +1657,6 @@ func (d *Decoder) kMap(f *codecFnInfo, rv reflect.Value) {
 			if valFn == nil {
 				valFn = d.cf.get(vtypeLo, true, true)
 			}
-			// fmt.Printf("decode.kMap: rvv: type: %v (vtype: %v), settable: %v, addressable: %v\n", rvv.Type(), vtype, rvv.CanSet(), rvv.CanAddr())
 			d.decodeValue(rvv, valFn, false, true)
 			// d.decodeValueFn(rvv, valFn)
 		}
@@ -1484,6 +1797,8 @@ type Decoder struct {
 
 	rb bytesDecReader
 	ri ioDecReader
+	bi bufioDecReader
+
 	cr containerStateRecv
 
 	n   *decNaked
@@ -1586,16 +1901,16 @@ func (d *Decoder) resetCommon() {
 }
 
 func (d *Decoder) Reset(r io.Reader) {
-	d.ri.x = &d.b
-	// d.s = d.sa[:0]
-	d.ri.bs.r = nil
-	var ok bool
-	d.ri.br, ok = r.(decReaderByteScanner)
-	if !ok {
-		d.ri.bs.r = r
-		d.ri.br = &d.ri.bs
+	if d.h.ReaderBufferSize > 0 {
+		d.bi.buf = make([]byte, 0, d.h.ReaderBufferSize)
+		d.bi.reset(r)
+		d.r = &d.bi
+	} else {
+		d.ri.x = &d.b
+		// d.s = d.sa[:0]
+		d.ri.reset(r)
+		d.r = &d.ri
 	}
-	d.r = &d.ri
 	d.resetCommon()
 }
 
@@ -2030,11 +2345,12 @@ func (d *Decoder) string(v []byte) (s string) {
 }
 
 // nextValueBytes returns the next value in the stream as a set of bytes.
-func (d *Decoder) nextValueBytes() []byte {
+func (d *Decoder) nextValueBytes() (bs []byte) {
 	d.d.uncacheRead()
 	d.r.track()
 	d.swallow()
-	return d.r.stopTrack()
+	bs = d.r.stopTrack()
+	return
 }
 
 func (d *Decoder) rawBytes() []byte {
@@ -2191,7 +2507,24 @@ func decExpandSliceRV(s reflect.Value, st reflect.Type, stElemSize, num, slen, s
 	scap2 = growCap(scap, stElemSize, num)
 	s2 = reflect.MakeSlice(st, l1, scap2)
 	changed = true
-	// println("expandslicevalue: cap-old: ", c0, ", cap-new: ", c1, ", len-new: ", l1)
 	reflect.Copy(s2, s)
 	return
 }
+
+func decReadFull(r io.Reader, bs []byte) (n int, err error) {
+	var nn int
+	for n < len(bs) && err == nil {
+		nn, err = r.Read(bs[n:])
+		if nn > 0 {
+			if err == io.EOF {
+				// leave EOF for next time
+				err = nil
+			}
+			n += nn
+		}
+	}
+
+	// do not do this - it serves no purpose
+	// if n != len(bs) && err == io.EOF { err = io.ErrUnexpectedEOF }
+	return
+}

codec/encode.go

@@ -4,6 +4,7 @@
 package codec
 
 import (
+	"bufio"
 	"encoding"
 	"fmt"
 	"io"
@@ -80,16 +81,20 @@ type encNoSeparator struct{}
 
 func (_ encNoSeparator) EncodeEnd() {}
 
-type ioEncWriterWriter interface {
-	WriteByte(c byte) error
+type ioEncStringWriter interface {
 	WriteString(s string) (n int, err error)
-	Write(p []byte) (n int, err error)
 }
 
-type ioEncStringWriter interface {
-	WriteString(s string) (n int, err error)
+type ioEncFlusher interface {
+	Flush() error
 }
 
+// type ioEncWriterWriter interface {
+// 	WriteByte(c byte) error
+// 	WriteString(s string) (n int, err error)
+// 	Write(p []byte) (n int, err error)
+// }
+
 type EncodeOptions struct {
 	// Encode a struct as an array, and not as a map
 	StructToArray bool
@@ -146,105 +151,118 @@ type EncodeOptions struct {
 	//   AsSymbolMapStringKeys
 	//   AsSymbolMapStringKeysFlag | AsSymbolStructFieldNameFlag
 	AsSymbols AsSymbolFlag
+
+	// WriterBufferSize is the size of the buffer used when writing.
+	//
+	// if > 0, we use a smart buffer internally for performance purposes.
+	WriterBufferSize int
 }
 
 // ---------------------------------------------
 
-type simpleIoEncWriterWriter struct {
+type simpleIoEncWriter struct {
+	io.Writer
+}
+
+// type bufIoEncWriter struct {
+// 	w   io.Writer
+// 	buf []byte
+// 	err error
+// }
+
+// func (x *bufIoEncWriter) Write(b []byte) (n int, err error) {
+// 	if x.err != nil {
+// 		return 0, x.err
+// 	}
+// 	if cap(x.buf)-len(x.buf) >= len(b) {
+// 		x.buf = append(x.buf, b)
+// 		return len(b), nil
+// 	}
+// 	n, err = x.w.Write(x.buf)
+// 	if err != nil {
+// 		x.err = err
+// 		return 0, x.err
+// 	}
+// 	n, err = x.w.Write(b)
+// 	x.err = err
+// 	return
+// }
+
+// ioEncWriter implements encWriter and can write to an io.Writer implementation
+type ioEncWriter struct {
 	w  io.Writer
+	ww io.Writer
 	bw io.ByteWriter
 	sw ioEncStringWriter
-	bs [1]byte
+	fw ioEncFlusher
+	b  [8]byte
 }
 
-func (o *simpleIoEncWriterWriter) WriteByte(c byte) (err error) {
-	if o.bw != nil {
-		return o.bw.WriteByte(c)
-	}
-	// _, err = o.w.Write([]byte{c})
-	o.bs[0] = c
-	_, err = o.w.Write(o.bs[:])
+func (x *ioEncWriter) WriteByte(b byte) (err error) {
+	// x.bs[0] = b
+	// _, err = x.ww.Write(x.bs[:])
+	var ba = [1]byte{b}
+	_, err = x.w.Write(ba[:])
 	return
 }
 
-func (o *simpleIoEncWriterWriter) WriteString(s string) (n int, err error) {
-	if o.sw != nil {
-		return o.sw.WriteString(s)
-	}
-	// return o.w.Write([]byte(s))
-	return o.w.Write(bytesView(s))
-}
-
-func (o *simpleIoEncWriterWriter) Write(p []byte) (n int, err error) {
-	return o.w.Write(p)
-}
-
-// ----------------------------------------
-
-// ioEncWriter implements encWriter and can write to an io.Writer implementation
-type ioEncWriter struct {
-	w ioEncWriterWriter
-	s simpleIoEncWriterWriter
-	// x [8]byte // temp byte array re-used internally for efficiency
+func (x *ioEncWriter) WriteString(s string) (n int, err error) {
+	return x.w.Write(bytesView(s))
 }
 
 func (z *ioEncWriter) writeb(bs []byte) {
-	if len(bs) == 0 {
-		return
-	}
-	n, err := z.w.Write(bs)
-	if err != nil {
+	// if len(bs) == 0 {
+	// 	return
+	// }
+	if _, err := z.ww.Write(bs); err != nil {
 		panic(err)
 	}
-	if n != len(bs) {
-		panic(fmt.Errorf("incorrect num bytes written. Expecting: %v, Wrote: %v", len(bs), n))
-	}
 }
 
 func (z *ioEncWriter) writestr(s string) {
-	if len(s) == 0 {
-		return
-	}
-	n, err := z.w.WriteString(s)
-	if err != nil {
+	// if len(s) == 0 {
+	// 	return
+	// }
+	if _, err := z.sw.WriteString(s); err != nil {
 		panic(err)
 	}
-	if n != len(s) {
-		panic(fmt.Errorf("incorrect num bytes written. Expecting: %v, Wrote: %v", len(s), n))
-	}
 }
 
 func (z *ioEncWriter) writen1(b byte) {
-	if err := z.w.WriteByte(b); err != nil {
+	if err := z.bw.WriteByte(b); err != nil {
 		panic(err)
 	}
 }
 
 func (z *ioEncWriter) writen2(b1, b2 byte) {
-	for _, b := range [...]byte{b1, b2} {
-		if err := z.w.WriteByte(b); err != nil {
-			panic(err)
+	var err error
+	if err = z.bw.WriteByte(b1); err == nil {
+		if err = z.bw.WriteByte(b2); err == nil {
+			return
 		}
 	}
+	panic(err)
 }
 
 func (z *ioEncWriter) writen4(b1, b2, b3, b4 byte) {
-	for _, b := range [...]byte{b1, b2, b3, b4} {
-		if err := z.w.WriteByte(b); err != nil {
-			panic(err)
-		}
+	z.b[0], z.b[1], z.b[2], z.b[3] = b1, b2, b3, b4
+	if _, err := z.ww.Write(z.b[:4]); err != nil {
+		panic(err)
 	}
 }
 
 func (z *ioEncWriter) writen5(b1, b2, b3, b4, b5 byte) {
-	for _, b := range [...]byte{b1, b2, b3, b4, b5} {
-		if err := z.w.WriteByte(b); err != nil {
-			panic(err)
-		}
+	z.b[0], z.b[1], z.b[2], z.b[3], z.b[4] = b1, b2, b3, b4, b5
+	if _, err := z.ww.Write(z.b[:5]); err != nil {
+		panic(err)
 	}
 }
 
-func (z *ioEncWriter) atEndOfEncode() {}
+func (z *ioEncWriter) atEndOfEncode() {
+	if z.fw != nil {
+		z.fw.Flush()
+	}
+}
 
 // ----------------------------------------
 
@@ -257,25 +275,17 @@ type bytesEncWriter struct {
 }
 
 func (z *bytesEncWriter) writeb(s []byte) {
-	slen := len(s)
-	if slen == 0 {
-		return
-	}
-	oc, a := z.growNoAlloc(slen)
+	oc, a := z.growNoAlloc(len(s))
 	if a {
-		z.growAlloc(slen, oc)
+		z.growAlloc(len(s), oc)
 	}
 	copy(z.b[oc:], s)
 }
 
 func (z *bytesEncWriter) writestr(s string) {
-	slen := len(s)
-	if slen == 0 {
-		return
-	}
-	oc, a := z.growNoAlloc(slen)
+	oc, a := z.growNoAlloc(len(s))
 	if a {
-		z.growAlloc(slen, oc)
+		z.growAlloc(len(s), oc)
 	}
 	copy(z.b[oc:], s)
 }
@@ -1046,6 +1056,7 @@ type Encoder struct {
 
 	wi ioEncWriter
 	wb bytesEncWriter
+	bw bufio.Writer
 
 	cr containerStateRecv
 	as encDriverAsis
@@ -1092,16 +1103,24 @@ func newEncoder(h Handle) *Encoder {
 // This accommodates using the state of the Encoder,
 // where it has "cached" information about sub-engines.
 func (e *Encoder) Reset(w io.Writer) {
-	ww, ok := w.(ioEncWriterWriter)
-	if ok {
-		e.wi.w = ww
+	var ok bool
+	e.wi.w = w
+	if e.h.WriterBufferSize > 0 {
+		bw := bufio.NewWriterSize(w, e.h.WriterBufferSize)
+		e.bw = *bw
+		e.wi.bw = &e.bw
+		e.wi.sw = &e.bw
+		e.wi.fw = &e.bw
+		e.wi.ww = &e.bw
 	} else {
-		sww := &e.wi.s
-		sww.w = w
-		sww.bw, _ = w.(io.ByteWriter)
-		sww.sw, _ = w.(ioEncStringWriter)
-		e.wi.w = sww
-		//ww = bufio.NewWriterSize(w, defEncByteBufSize)
+		if e.wi.bw, ok = w.(io.ByteWriter); !ok {
+			e.wi.bw = &e.wi
+		}
+		if e.wi.sw, ok = w.(ioEncStringWriter); !ok {
+			e.wi.sw = &e.wi
+		}
+		e.wi.fw, _ = w.(ioEncFlusher)
+		e.wi.ww = w
 	}
 	e.w = &e.wi
 	e.e.reset()

codec/json.go

@@ -45,16 +45,16 @@ import (
 var (
 	// jsonLiterals = [...]byte{'t', 'r', 'u', 'e', 'f', 'a', 'l', 's', 'e', 'n', 'u', 'l', 'l'}
 
-	jsonFloat64Pow10 = [...]float64{
-		1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
-		1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
-		1e20, 1e21, 1e22,
-	}
+	// jsonFloat64Pow10 = [...]float64{
+	// 	1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
+	// 	1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
+	// 	1e20, 1e21, 1e22,
+	// }
 
-	jsonUint64Pow10 = [...]uint64{
-		1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
-		1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
-	}
+	// jsonUint64Pow10 = [...]uint64{
+	// 	1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
+	// 	1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
+	// }
 
 	// jsonTabs and jsonSpaces are used as caches for indents
 	jsonTabs, jsonSpaces string

codec/shared_test.go

@@ -157,7 +157,7 @@ func testInitFlags() {
 	flag.BoolVar(&testSkipIntf, "tf", false, "Skip Interfaces")
 	flag.BoolVar(&testUseReset, "tr", false, "Use Reset")
 	flag.IntVar(&testJsonIndent, "td", 0, "Use JSON Indent")
-	flag.IntVar(&testNumRepeatString, "trs", 10, "Create string variables by repeating a string N times")
+	flag.IntVar(&testNumRepeatString, "trs", 8, "Create string variables by repeating a string N times")
 	flag.IntVar(&testMaxInitLen, "tx", 0, "Max Init Len")
 	flag.BoolVar(&testUseMust, "tm", true, "Use Must(En|De)code")
 	flag.BoolVar(&testCheckCircRef, "tl", false, "Use Check Circular Ref")
@@ -194,6 +194,7 @@ func testInitAll() {
 	// only parse it once.
 	if !flag.Parsed() {
 		flag.Parse()
+		testNumRepeatStringMirror = testNumRepeatString
 	}
 	for _, f := range testPreInitFns {
 		f()

codec/values_flex_test.go

@@ -25,7 +25,7 @@ type TestStrucFlex struct {
 	Nteststruc *TestStrucFlex
 }
 
-func newTestStrucFlex(depth int, bench, useInterface, useStringKeyOnly bool) (ts *TestStrucFlex) {
+func newTestStrucFlex(depth, n int, bench, useInterface, useStringKeyOnly bool) (ts *TestStrucFlex) {
 	ts = &TestStrucFlex{
 		Miwu64s: map[int]wrapUint64Slice{
 			5: []wrapUint64{1, 2, 3, 4, 5},
@@ -52,7 +52,7 @@ func newTestStrucFlex(depth int, bench, useInterface, useStringKeyOnly bool) (ts
 			-44: "minus forty four",
 		},
 	}
-	populateTestStrucCommon(&ts.testStrucCommon, bench, useInterface, useStringKeyOnly)
+	populateTestStrucCommon(&ts.testStrucCommon, n, bench, useInterface, useStringKeyOnly)
 	if depth > 0 {
 		depth--
 		if ts.Mtsptr == nil {
@@ -61,7 +61,7 @@ func newTestStrucFlex(depth int, bench, useInterface, useStringKeyOnly bool) (ts
 		if ts.Mts == nil {
 			ts.Mts = make(map[string]TestStrucFlex)
 		}
-		ts.Mtsptr["0"] = newTestStrucFlex(depth, bench, useInterface, useStringKeyOnly)
+		ts.Mtsptr["0"] = newTestStrucFlex(depth, n, bench, useInterface, useStringKeyOnly)
 		ts.Mts["0"] = *(ts.Mtsptr["0"])
 		ts.Its = append(ts.Its, ts.Mtsptr["0"])
 	}

codec/values_test.go

@@ -206,12 +206,14 @@ type Sinterface interface {
 
 var testStrucTime = time.Date(2012, 2, 2, 2, 2, 2, 2000, time.UTC).UTC()
 
-func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useStringKeyOnly bool) {
+var testNumRepeatStringMirror int = 8
+
+func populateTestStrucCommon(ts *testStrucCommon, n int, bench, useInterface, useStringKeyOnly bool) {
 	var i64a, i64b, i64c, i64d int64 = 64, 6464, 646464, 64646464
 
 	var a = AnonInTestStruc{
 		// There's more leeway in altering this.
-		AS:    strRpt("A-String"),
+		AS:    strRpt(n, "A-String"),
 		AI64:  -64646464,
 		AI16:  1616,
 		AUi64: 64646464,
@@ -219,13 +221,13 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 		// single reverse solidus character may be represented in json as "\u005C".
 		// include these in ASslice below.
 		ASslice: []string{
-			strRpt("Aone"),
-			strRpt("Atwo"),
-			strRpt("Athree"),
-			strRpt("Afour.reverse_solidus.\u005c"),
-			strRpt("Afive.Gclef.\U0001d11E\"ugorji\"done.")},
+			strRpt(n, "Aone"),
+			strRpt(n, "Atwo"),
+			strRpt(n, "Athree"),
+			strRpt(n, "Afour.reverse_solidus.\u005c"),
+			strRpt(n, "Afive.Gclef.\U0001d11E\"ugorji\"done.")},
 		AI64slice: []int64{1, -22, 333, -4444, 55555, -666666},
-		AMSU16:    map[string]uint16{strRpt("1"): 1, strRpt("22"): 2, strRpt("333"): 3, strRpt("4444"): 4},
+		AMSU16:    map[string]uint16{strRpt(n, "1"): 1, strRpt(n, "22"): 2, strRpt(n, "333"): 3, strRpt(n, "4444"): 4},
 		AF64slice: []float64{
 			11.11e-11, -11.11e+11,
 			2.222E+12, -2.222E-12,
@@ -241,7 +243,7 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 	}
 
 	*ts = testStrucCommon{
-		S: strRpt(`some really really cool names that are nigerian and american like "ugorji melody nwoke" - get it? `),
+		S: strRpt(n, `some really really cool names that are nigerian and american like "ugorji melody nwoke" - get it? `),
 
 		// set the numbers close to the limits
 		I8:   math.MaxInt8 * 2 / 3,  // 8,
@@ -264,7 +266,7 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 		B:  true,
 		By: 5,
 
-		Sslice:    []string{strRpt("one"), strRpt("two"), strRpt("three")},
+		Sslice:    []string{strRpt(n, "one"), strRpt(n, "two"), strRpt(n, "three")},
 		I64slice:  []int64{1111, 2222, 3333},
 		I16slice:  []int16{44, 55, 66},
 		Ui64slice: []uint64{12121212, 34343434, 56565656},
@@ -273,15 +275,15 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 		Byslice:   []byte{13, 14, 15},
 
 		Msi64: map[string]int64{
-			strRpt("one"):       1,
-			strRpt("two"):       2,
-			strRpt("\"three\""): 3,
+			strRpt(n, "one"):       1,
+			strRpt(n, "two"):       2,
+			strRpt(n, "\"three\""): 3,
 		},
 
 		Ui64array: [4]uint64{4, 16, 64, 256},
 
 		WrapSliceInt64:  []uint64{4, 16, 64, 256},
-		WrapSliceString: []string{strRpt("4"), strRpt("16"), strRpt("64"), strRpt("256")},
+		WrapSliceString: []string{strRpt(n, "4"), strRpt(n, "16"), strRpt(n, "64"), strRpt(n, "256")},
 
 		// DecodeNaked bombs here, because the stringUint64T is decoded as a map,
 		// and a map cannot be the key type of a map.
@@ -293,7 +295,7 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 
 		// make Simplef same as top-level
 		Simplef: testSimpleFields{
-			S: strRpt(`some really really cool names that are nigerian and american like "ugorji melody nwoke" - get it? `),
+			S: strRpt(n, `some really really cool names that are nigerian and american like "ugorji melody nwoke" - get it? `),
 
 			// set the numbers close to the limits
 			I8:   math.MaxInt8 * 2 / 3,  // 8,
@@ -316,7 +318,7 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 			B:  true,
 			By: 5,
 
-			Sslice:    []string{strRpt("one"), strRpt("two"), strRpt("three")},
+			Sslice:    []string{strRpt(n, "one"), strRpt(n, "two"), strRpt(n, "three")},
 			I64slice:  []int64{1111, 2222, 3333},
 			I16slice:  []int16{44, 55, 66},
 			Ui64slice: []uint64{12121212, 34343434, 56565656},
@@ -325,15 +327,15 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 			Byslice:   []byte{13, 14, 15},
 
 			Msi64: map[string]int64{
-				strRpt("one"):       1,
-				strRpt("two"):       2,
-				strRpt("\"three\""): 3,
+				strRpt(n, "one"):       1,
+				strRpt(n, "two"):       2,
+				strRpt(n, "\"three\""): 3,
 			},
 
 			Ui64array: [4]uint64{4, 16, 64, 256},
 
 			WrapSliceInt64:  []uint64{4, 16, 64, 256},
-			WrapSliceString: []string{strRpt("4"), strRpt("16"), strRpt("64"), strRpt("256")},
+			WrapSliceString: []string{strRpt(n, "4"), strRpt(n, "16"), strRpt(n, "64"), strRpt(n, "256")},
 		},
 
 		AnonInTestStruc: a,
@@ -344,10 +346,10 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 
 	if useInterface {
 		ts.AnonInTestStrucIntf = &AnonInTestStrucIntf{
-			Islice: []interface{}{strRpt("true"), true, strRpt("no"), false, uint64(288), float64(0.4)},
+			Islice: []interface{}{strRpt(n, "true"), true, strRpt(n, "no"), false, uint64(288), float64(0.4)},
 			Ms: map[string]interface{}{
-				strRpt("true"):     strRpt("true"),
-				strRpt("int64(9)"): false,
+				strRpt(n, "true"):     strRpt(n, "true"),
+				strRpt(n, "int64(9)"): false,
 			},
 			T: testStrucTime,
 		}
@@ -369,9 +371,9 @@ func populateTestStrucCommon(ts *testStrucCommon, bench, useInterface, useString
 	}
 }
 
-func newTestStruc(depth int, bench, useInterface, useStringKeyOnly bool) (ts *TestStruc) {
+func newTestStruc(depth, n int, bench, useInterface, useStringKeyOnly bool) (ts *TestStruc) {
 	ts = &TestStruc{}
-	populateTestStrucCommon(&ts.testStrucCommon, bench, useInterface, useStringKeyOnly)
+	populateTestStrucCommon(&ts.testStrucCommon, n, bench, useInterface, useStringKeyOnly)
 	if depth > 0 {
 		depth--
 		if ts.Mtsptr == nil {
@@ -380,13 +382,13 @@ func newTestStruc(depth int, bench, useInterface, useStringKeyOnly bool) (ts *Te
 		if ts.Mts == nil {
 			ts.Mts = make(map[string]TestStruc)
 		}
-		ts.Mtsptr[strRpt("0")] = newTestStruc(depth, bench, useInterface, useStringKeyOnly)
-		ts.Mts[strRpt("0")] = *(ts.Mtsptr[strRpt("0")])
-		ts.Its = append(ts.Its, ts.Mtsptr[strRpt("0")])
+		ts.Mtsptr[strRpt(n, "0")] = newTestStruc(depth, n, bench, useInterface, useStringKeyOnly)
+		ts.Mts[strRpt(n, "0")] = *(ts.Mtsptr[strRpt(n, "0")])
+		ts.Its = append(ts.Its, ts.Mtsptr[strRpt(n, "0")])
 	}
 	return
 }
 
-func strRpt(s string) string {
-	return strings.Repeat(s, testNumRepeatString)
+func strRpt(n int, s string) string {
+	return strings.Repeat(s, n)
 }

codec/x_bench_gen_test.go

@@ -0,0 +1,123 @@
+// +build x
+// +build generated
+
+// Copyright (c) 2012-2015 Ugorji Nwoke. All rights reserved.
+// Use of this source code is governed by a MIT license found in the LICENSE file.
+
+package codec
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"testing"
+
+	"github.com/mailru/easyjson"
+	"github.com/pquerna/ffjson/ffjson"
+	"github.com/tinylib/msgp/msgp"
+)
+
+/*
+ To update all these, use:
+ go get -u github.com/tinylib/msgp/msgp github.com/tinylib/msgp \
+           github.com/pquerna/ffjson/ffjson github.com/pquerna/ffjson \
+           github.com/mailru/easyjson/...
+
+ Known Issues with external libraries:
+ - msgp io.R/W support doesn't work. It throws error
+
+*/
+
+func init() {
+	testPreInitFns = append(testPreInitFns, benchXGenPreInit)
+}
+
+func benchXGenPreInit() {
+	benchCheckers = append(benchCheckers,
+		benchChecker{"msgp", fnMsgpEncodeFn, fnMsgpDecodeFn},
+		benchChecker{"easyjson", fnEasyjsonEncodeFn, fnEasyjsonDecodeFn},
+		benchChecker{"ffjson", fnFfjsonEncodeFn, fnFfjsonDecodeFn},
+	)
+}
+
+func fnEasyjsonEncodeFn(ts interface{}, bsIn []byte) ([]byte, error) {
+	if _, ok := ts.(easyjson.Marshaler); !ok {
+		return nil, errors.New("easyjson: input is not a easyjson.Marshaler")
+	}
+	if testUseIoEncDec {
+		buf := new(bytes.Buffer)
+		_, err := easyjson.MarshalToWriter(ts.(easyjson.Marshaler), buf)
+		return buf.Bytes(), err
+	}
+	return easyjson.Marshal(ts.(easyjson.Marshaler))
+	// return ts.(json.Marshaler).MarshalJSON()
+}
+
+func fnEasyjsonDecodeFn(buf []byte, ts interface{}) error {
+	if _, ok := ts.(easyjson.Unmarshaler); !ok {
+		return errors.New("easyjson: input is not a easyjson.Unmarshaler")
+	}
+	if testUseIoEncDec {
+		return easyjson.UnmarshalFromReader(bytes.NewReader(buf), ts.(easyjson.Unmarshaler))
+	}
+	return easyjson.Unmarshal(buf, ts.(easyjson.Unmarshaler))
+	// return ts.(json.Unmarshaler).UnmarshalJSON(buf)
+}
+
+func fnFfjsonEncodeFn(ts interface{}, bsIn []byte) ([]byte, error) {
+	return ffjson.Marshal(ts)
+	// return ts.(json.Marshaler).MarshalJSON()
+}
+
+func fnFfjsonDecodeFn(buf []byte, ts interface{}) error {
+	return ffjson.Unmarshal(buf, ts)
+	// return ts.(json.Unmarshaler).UnmarshalJSON(buf)
+}
+
+func fnMsgpEncodeFn(ts interface{}, bsIn []byte) ([]byte, error) {
+	if _, ok := ts.(msgp.Encodable); !ok {
+		return nil, fmt.Errorf("msgp: input of type %T is not a msgp.Encodable", ts)
+	}
+	if testUseIoEncDec {
+		buf := fnBenchmarkByteBuf(bsIn)
+		err := ts.(msgp.Encodable).EncodeMsg(msgp.NewWriter(buf))
+		return buf.Bytes(), err
+	}
+	return ts.(msgp.Marshaler).MarshalMsg(bsIn[:0]) // msgp appends to slice.
+}
+
+func fnMsgpDecodeFn(buf []byte, ts interface{}) (err error) {
+	if _, ok := ts.(msgp.Decodable); !ok {
+		return fmt.Errorf("msgp: input of type %T is not a msgp.Decodable", ts)
+	}
+	if testUseIoEncDec {
+		err = ts.(msgp.Decodable).DecodeMsg(msgp.NewReader(bytes.NewReader(buf)))
+		return
+	}
+	_, err = ts.(msgp.Unmarshaler).UnmarshalMsg(buf)
+	return
+}
+
+func Benchmark__Msgp_______Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "msgp", benchTs, fnMsgpEncodeFn)
+}
+
+func Benchmark__Msgp_______Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "msgp", benchTs, fnMsgpEncodeFn, fnMsgpDecodeFn, fnBenchNewTs)
+}
+
+func Benchmark__Easyjson___Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "easyjson", benchTs, fnEasyjsonEncodeFn)
+}
+
+func Benchmark__Easyjson___Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "easyjson", benchTs, fnEasyjsonEncodeFn, fnEasyjsonDecodeFn, fnBenchNewTs)
+}
+
+func Benchmark__Ffjson_____Encode(b *testing.B) {
+	fnBenchmarkEncode(b, "ffjson", benchTs, fnFfjsonEncodeFn)
+}
+
+func Benchmark__Ffjson_____Decode(b *testing.B) {
+	fnBenchmarkDecode(b, "ffjson", benchTs, fnFfjsonEncodeFn, fnFfjsonDecodeFn, fnBenchNewTs)
+}

codec/z_all_test.go

@@ -51,7 +51,8 @@ func testSuite(t *testing.T, f func(t *testing.T)) {
 	testMaxInitLen = 0
 	testJsonIndent = 0
 	testUseIoWrapper = false
-	testNumRepeatString = 10
+	testNumRepeatString = 8
+
 	testReinit()
 	t.Run("optionsFalse", f)
 
@@ -87,7 +88,18 @@ func testSuite(t *testing.T, f func(t *testing.T)) {
 	testReinit()
 	t.Run("optionsTrue-deepstruct", f)
 
-	testNumRepeatString = 40
+	// make buffer small enough so that we have to re-fill multiple times.
+	testSkipRPCTests = true
+	testUseIoEncDec = true
+	testDecodeOptions.ReaderBufferSize = 128
+	testEncodeOptions.WriterBufferSize = 128
+	testReinit()
+	t.Run("optionsTrue-bufio", f)
+	testDecodeOptions.ReaderBufferSize = 0
+	testEncodeOptions.WriterBufferSize = 0
+	testSkipRPCTests = false
+
+	testNumRepeatString = 32
 	testReinit()
 	t.Run("optionsTrue-largestrings", f)
 
@@ -163,3 +175,18 @@ func testCodecGroup(t *testing.T) {
 }
 
 func TestCodecSuite(t *testing.T) { testSuite(t, testCodecGroup) }
+
+// func TestCodecSuite(t *testing.T) { testSuite2(t, testCodecGroup2) }
+// func testCodecGroup2(t *testing.T) {
+// 	t.Run("TestJsonCodecsTable", TestJsonCodecsTable)
+// 	t.Run("TestJsonCodecsMisc", TestJsonCodecsMisc)
+// }
+// func testSuite2(t *testing.T, f func(t *testing.T)) {
+// 	testUseIoEncDec = true
+// 	testDecodeOptions = DecodeOptions{}
+// 	testEncodeOptions = EncodeOptions{}
+// 	testDecodeOptions.ReaderBufferSize = 128
+// 	testEncodeOptions.WriterBufferSize = 128
+// 	testReinit()
+// 	t.Run("optionsTrue-bufio", f)
+// }

codec/z_all_x_bench_gen_test.go

@@ -0,0 +1,49 @@
+// +build alltests
+// +build x
+// +build go1.7
+// +build generated
+
+package codec
+
+// see notes in z_all_bench_test.go
+
+import "testing"
+
+func benchmarkCodecXGenGroup(t *testing.B) {
+	logT(nil, "\n-------------------------------\n")
+	t.Run("Benchmark__Msgpack____Encode", Benchmark__Msgpack____Encode)
+	t.Run("Benchmark__Binc_______Encode", Benchmark__Binc_______Encode)
+	t.Run("Benchmark__Simple_____Encode", Benchmark__Simple_____Encode)
+	t.Run("Benchmark__Cbor_______Encode", Benchmark__Cbor_______Encode)
+	t.Run("Benchmark__Json_______Encode", Benchmark__Json_______Encode)
+	t.Run("Benchmark__Std_Json___Encode", Benchmark__Std_Json___Encode)
+	t.Run("Benchmark__Gob________Encode", Benchmark__Gob________Encode)
+	t.Run("Benchmark__JsonIter___Encode", Benchmark__JsonIter___Encode)
+	t.Run("Benchmark__Bson_______Encode", Benchmark__Bson_______Encode)
+	t.Run("Benchmark__VMsgpack___Encode", Benchmark__VMsgpack___Encode)
+	t.Run("Benchmark__Msgp_______Encode", Benchmark__Msgp_______Encode)
+	t.Run("Benchmark__Easyjson___Encode", Benchmark__Easyjson___Encode)
+	t.Run("Benchmark__Ffjson_____Encode", Benchmark__Ffjson_____Encode)
+	t.Run("Benchmark__Gcbor______Encode", Benchmark__Gcbor______Encode)
+	t.Run("Benchmark__Xdr________Encode", Benchmark__Xdr________Encode)
+	t.Run("Benchmark__Sereal_____Encode", Benchmark__Sereal_____Encode)
+
+	t.Run("Benchmark__Msgpack____Decode", Benchmark__Msgpack____Decode)
+	t.Run("Benchmark__Binc_______Decode", Benchmark__Binc_______Decode)
+	t.Run("Benchmark__Simple_____Decode", Benchmark__Simple_____Decode)
+	t.Run("Benchmark__Cbor_______Decode", Benchmark__Cbor_______Decode)
+	t.Run("Benchmark__Json_______Decode", Benchmark__Json_______Decode)
+	t.Run("Benchmark__Std_Json___Decode", Benchmark__Std_Json___Decode)
+	t.Run("Benchmark__Gob________Decode", Benchmark__Gob________Decode)
+	t.Run("Benchmark__JsonIter___Decode", Benchmark__JsonIter___Decode)
+	t.Run("Benchmark__Bson_______Decode", Benchmark__Bson_______Decode)
+	t.Run("Benchmark__VMsgpack___Decode", Benchmark__VMsgpack___Decode)
+	t.Run("Benchmark__Msgp_______Decode", Benchmark__Msgp_______Decode)
+	t.Run("Benchmark__Easyjson___Decode", Benchmark__Easyjson___Decode)
+	t.Run("Benchmark__Ffjson_____Decode", Benchmark__Ffjson_____Decode)
+	t.Run("Benchmark__Gcbor______Decode", Benchmark__Gcbor______Decode)
+	t.Run("Benchmark__Xdr________Decode", Benchmark__Xdr________Decode)
+	t.Run("Benchmark__Sereal_____Decode", Benchmark__Sereal_____Decode)
+}
+
+func BenchmarkCodecXGenSuite(t *testing.B) { benchmarkSuite(t, benchmarkCodecXGenGroup) }