support []byte; marshal without copy

feature_adapter.go

@@ -89,14 +89,12 @@ func UnmarshalAnyFromString(str string) (Any, error) {
 // Marshal returns the JSON encoding of v, adapts to json/encoding Marshal API
 // Refer to https://godoc.org/encoding/json#Marshal for more information
 func Marshal(v interface{}) ([]byte, error) {
-	buf := &bytes.Buffer{}
-	stream := NewStream(buf, 512)
+	stream := NewStream(nil, 256)
 	stream.WriteVal(v)
-	stream.Flush()
 	if stream.Error != nil {
 		return nil, stream.Error
 	}
-	return buf.Bytes(), nil
+	return stream.Buffer(), nil
 }
 
 func MarshalToString(v interface{}) (string, error) {

feature_reflect.go

@@ -159,15 +159,18 @@ func RegisterExtension(extension ExtensionFunc) {
 	extensions = append(extensions, extension)
 }
 
-// CleanDecoders cleans decoders registered
+// CleanDecoders cleans decoders registered or cached
 func CleanDecoders() {
 	typeDecoders = map[string]Decoder{}
 	fieldDecoders = map[string]Decoder{}
+	atomic.StorePointer(&DECODERS, unsafe.Pointer(&map[string]Decoder{}))
 }
 
+// CleanEncoders cleans decoders registered or cached
 func CleanEncoders() {
 	typeEncoders = map[string]Encoder{}
 	fieldEncoders = map[string]Encoder{}
+	atomic.StorePointer(&ENCODERS, unsafe.Pointer(&map[string]Encoder{}))
 }
 
 type optionalDecoder struct {
@@ -339,6 +342,9 @@ func decoderOfType(typ reflect.Type) (Decoder, error) {
 }
 
 func createDecoderOfType(typ reflect.Type) (Decoder, error) {
+	if typ.String() == "[]uint8" {
+		return &base64Codec{}, nil
+	}
 	if typ.AssignableTo(jsonRawMessageType) {
 		return &jsonRawMessageCodec{}, nil
 	}
@@ -426,6 +432,9 @@ func encoderOfType(typ reflect.Type) (Encoder, error) {
 }
 
 func createEncoderOfType(typ reflect.Type) (Encoder, error) {
+	if typ.String() == "[]uint8" {
+		return &base64Codec{}, nil
+	}
 	if typ.AssignableTo(jsonRawMessageType) {
 		return &jsonRawMessageCodec{}, nil
 	}

feature_reflect_native.go

@@ -3,6 +3,7 @@ package jsoniter
 import (
 	"unsafe"
 	"encoding/json"
+	"encoding/base64"
 )
 
 type stringCodec struct {
@@ -379,6 +380,49 @@ func (encoder *jsonRawMessageCodec) isEmpty(ptr unsafe.Pointer) bool {
 	return len(*((*json.RawMessage)(ptr))) == 0
 }
 
+type base64Codec struct {
+}
+
+func (codec *base64Codec) decode(ptr unsafe.Pointer, iter *Iterator) {
+	encoding := base64.StdEncoding
+	src := iter.SkipAndReturnBytes()
+	src = src[1:len(src)-1]
+	decodedLen := encoding.DecodedLen(len(src))
+	dst := make([]byte, decodedLen)
+	_, err := encoding.Decode(dst, src)
+	if err != nil {
+		iter.reportError("decode base64", err.Error())
+	} else {
+		*((*[]byte)(ptr)) = dst
+	}
+}
+
+func (codec *base64Codec) encode(ptr unsafe.Pointer, stream *Stream) {
+	encoding := base64.StdEncoding
+	stream.writeByte('"')
+	src := *((*[]byte)(ptr))
+	toGrow := encoding.EncodedLen(len(src))
+	stream.ensure(toGrow)
+	encoding.Encode(stream.buf[stream.n:], src)
+	stream.n += toGrow
+	stream.writeByte('"')
+}
+
+func (encoder *base64Codec) encodeInterface(val interface{}, stream *Stream) {
+	encoding := base64.StdEncoding
+	stream.writeByte('"')
+	src := val.([]byte)
+	toGrow := encoding.EncodedLen(len(src))
+	stream.ensure(toGrow)
+	encoding.Encode(stream.buf[stream.n:], src)
+	stream.n += toGrow
+	stream.writeByte('"')
+}
+
+func (encoder *base64Codec) isEmpty(ptr unsafe.Pointer) bool {
+	return len(*((*[]byte)(ptr))) == 0
+}
+
 type stringNumberDecoder struct {
 	elemDecoder Decoder
 }
@@ -447,4 +491,4 @@ func (decoder *unmarshalerDecoder) decode(ptr unsafe.Pointer, iter *Iterator) {
 	if err != nil {
 		iter.reportError("unmarshaler", err.Error())
 	}
-}
+}

feature_stream.go

@@ -32,24 +32,32 @@ func (b *Stream) Buffered() int {
 	return b.n
 }
 
+func (b *Stream) Buffer() []byte {
+	return b.buf[:b.n]
+}
+
 // Write writes the contents of p into the buffer.
 // It returns the number of bytes written.
 // If nn < len(p), it also returns an error explaining
 // why the write is short.
 func (b *Stream) Write(p []byte) (nn int, err error) {
 	for len(p) > b.Available() && b.Error == nil {
-		var n int
-		if b.Buffered() == 0 {
-			// Large write, empty buffer.
-			// Write directly from p to avoid copy.
-			n, b.Error = b.out.Write(p)
+		if b.out == nil {
+			b.growAtLeast(len(p))
 		} else {
-			n = copy(b.buf[b.n:], p)
-			b.n += n
-			b.Flush()
+			var n int
+			if b.Buffered() == 0 {
+				// Large write, empty buffer.
+				// Write directly from p to avoid copy.
+				n, b.Error = b.out.Write(p)
+			} else {
+				n = copy(b.buf[b.n:], p)
+				b.n += n
+				b.Flush()
+			}
+			nn += n
+			p = p[n:]
 		}
-		nn += n
-		p = p[n:]
 	}
 	if b.Error != nil {
 		return nn, b.Error
@@ -60,14 +68,13 @@ func (b *Stream) Write(p []byte) (nn int, err error) {
 	return nn, nil
 }
 
-
 // WriteByte writes a single byte.
 func (b *Stream) writeByte(c byte) {
 	if b.Error != nil {
 		return
 	}
-	if b.Available() <= 0 && b.Flush() != nil {
-		return
+	if b.Available() < 1 {
+		b.growAtLeast(1)
 	}
 	b.buf[b.n] = c
 	b.n++
@@ -77,11 +84,11 @@ func (b *Stream) writeTwoBytes(c1 byte, c2 byte) {
 	if b.Error != nil {
 		return
 	}
-	if b.Available() <= 1 && b.Flush() != nil {
-		return
+	if b.Available() < 2 {
+		b.growAtLeast(2)
 	}
 	b.buf[b.n] = c1
-	b.buf[b.n + 1] = c2
+	b.buf[b.n+1] = c2
 	b.n += 2
 }
 
@@ -89,12 +96,12 @@ func (b *Stream) writeThreeBytes(c1 byte, c2 byte, c3 byte) {
 	if b.Error != nil {
 		return
 	}
-	if b.Available() <= 2 && b.Flush() != nil {
-		return
+	if b.Available() < 3 {
+		b.growAtLeast(3)
 	}
 	b.buf[b.n] = c1
-	b.buf[b.n + 1] = c2
-	b.buf[b.n + 2] = c3
+	b.buf[b.n+1] = c2
+	b.buf[b.n+2] = c3
 	b.n += 3
 }
 
@@ -102,13 +109,13 @@ func (b *Stream) writeFourBytes(c1 byte, c2 byte, c3 byte, c4 byte) {
 	if b.Error != nil {
 		return
 	}
-	if b.Available() <= 3 && b.Flush() != nil {
-		return
+	if b.Available() < 4 {
+		b.growAtLeast(4)
 	}
 	b.buf[b.n] = c1
-	b.buf[b.n + 1] = c2
-	b.buf[b.n + 2] = c3
-	b.buf[b.n + 3] = c4
+	b.buf[b.n+1] = c2
+	b.buf[b.n+2] = c3
+	b.buf[b.n+3] = c4
 	b.n += 4
 }
 
@@ -116,19 +123,22 @@ func (b *Stream) writeFiveBytes(c1 byte, c2 byte, c3 byte, c4 byte, c5 byte) {
 	if b.Error != nil {
 		return
 	}
-	if b.Available() <= 3 && b.Flush() != nil {
-		return
+	if b.Available() < 5 {
+		b.growAtLeast(5)
 	}
 	b.buf[b.n] = c1
-	b.buf[b.n + 1] = c2
-	b.buf[b.n + 2] = c3
-	b.buf[b.n + 3] = c4
-	b.buf[b.n + 4] = c5
+	b.buf[b.n+1] = c2
+	b.buf[b.n+2] = c3
+	b.buf[b.n+3] = c4
+	b.buf[b.n+4] = c5
 	b.n += 5
 }
 
 // Flush writes any buffered data to the underlying io.Writer.
 func (b *Stream) Flush() error {
+	if b.out == nil {
+		return nil
+	}
 	if b.Error != nil {
 		return b.Error
 	}
@@ -141,7 +151,7 @@ func (b *Stream) Flush() error {
 	}
 	if err != nil {
 		if n > 0 && n < b.n {
-			copy(b.buf[0:b.n - n], b.buf[n:b.n])
+			copy(b.buf[0:b.n-n], b.buf[n:b.n])
 		}
 		b.n -= n
 		b.Error = err
@@ -151,13 +161,28 @@ func (b *Stream) Flush() error {
 	return nil
 }
 
-func (b *Stream) WriteRaw(s string) {
-	for len(s) > b.Available() && b.Error == nil {
-		n := copy(b.buf[b.n:], s)
-		b.n += n
-		s = s[n:]
-		b.Flush()
+func (b *Stream) ensure(minimal int) {
+	available := b.Available()
+	if available < minimal {
+		if available < 128 {
+			b.Flush()
+		}
+		b.growAtLeast(minimal)
 	}
+}
+
+func (b *Stream) growAtLeast(minimal int) {
+	toGrow := len(b.buf)
+	if toGrow < minimal {
+		toGrow = minimal
+	}
+	newBuf := make([]byte, len(b.buf)+toGrow)
+	copy(newBuf, b.Buffer())
+	b.buf = newBuf
+}
+
+func (b *Stream) WriteRaw(s string) {
+	b.ensure(len(s))
 	if b.Error != nil {
 		return
 	}
@@ -166,18 +191,13 @@ func (b *Stream) WriteRaw(s string) {
 }
 
 func (stream *Stream) WriteString(s string) {
+	stream.ensure(32)
 	valLen := len(s)
 	toWriteLen := valLen
 	bufLengthMinusTwo := len(stream.buf) - 2 // make room for the quotes
-	if stream.n + toWriteLen > bufLengthMinusTwo {
+	if stream.n+toWriteLen > bufLengthMinusTwo {
 		toWriteLen = bufLengthMinusTwo - stream.n
 	}
-	if toWriteLen < 0 {
-		stream.Flush()
-		if stream.n + toWriteLen > bufLengthMinusTwo {
-			toWriteLen = bufLengthMinusTwo - stream.n
-		}
-	}
 	n := stream.n
 	stream.buf[n] = '"'
 	n++
@@ -189,7 +209,7 @@ func (stream *Stream) WriteString(s string) {
 			stream.buf[n] = c
 			n++
 		} else {
-			break;
+			break
 		}
 	}
 	if i == valLen {
@@ -200,14 +220,14 @@ func (stream *Stream) WriteString(s string) {
 	}
 	stream.n = n
 	// for the remaining parts, we process them char by char
-	stream.writeStringSlowPath(s, i, valLen);
+	stream.writeStringSlowPath(s, i, valLen)
 	stream.writeByte('"')
 }
 
 func (stream *Stream) writeStringSlowPath(s string, i int, valLen int) {
 	for ; i < valLen; i++ {
 		c := s[i]
-		switch (c) {
+		switch c {
 		case '"':
 			stream.writeTwoBytes('\\', '"')
 		case '\\':
@@ -223,7 +243,7 @@ func (stream *Stream) writeStringSlowPath(s string, i int, valLen int) {
 		case '\t':
 			stream.writeTwoBytes('\\', 't')
 		default:
-			stream.writeByte(c);
+			stream.writeByte(c)
 		}
 	}
 }
@@ -293,21 +313,14 @@ func (stream *Stream) WriteArrayEnd() {
 }
 
 func (stream *Stream) writeIndention(delta int) {
-	if (stream.indention == 0) {
+	if stream.indention == 0 {
 		return
 	}
 	stream.writeByte('\n')
 	toWrite := stream.indention - delta
-	i := 0
-	for {
-		for ; i < toWrite && stream.n < len(stream.buf); i++ {
-			stream.buf[stream.n] = ' '
-			stream.n ++
-		}
-		if i == toWrite {
-			break;
-		} else {
-			stream.Flush()
-		}
+	stream.ensure(toWrite)
+	for i := 0 ; i < toWrite && stream.n < len(stream.buf); i++ {
+		stream.buf[stream.n] = ' '
+		stream.n ++
 	}
-}
+}

feature_stream_float.go

@@ -33,9 +33,7 @@ func (stream *Stream) WriteFloat32Lossy(val float32) {
 		return
 	}
 	stream.writeByte('.')
-	if stream.Available() < 10 {
-		stream.Flush()
-	}
+	stream.ensure(10)
 	for p := precision - 1; p > 0 && fval < POW10[p]; p-- {
 		stream.writeByte('0')
 	}
@@ -67,9 +65,7 @@ func (stream *Stream) WriteFloat64Lossy(val float64) {
 		return
 	}
 	stream.writeByte('.')
-	if stream.Available() < 10 {
-		stream.Flush()
-	}
+	stream.ensure(10)
 	for p := precision - 1; p > 0 && fval < POW10[p]; p-- {
 		stream.writeByte('0')
 	}

feature_stream_int.go

@@ -1,43 +1,8 @@
 package jsoniter
 
-var digits []uint8
-var digitTens []uint8
-var digitOnes []uint8
 var DIGITS []uint32
 
 func init() {
-	digits = []uint8{
-		'0', '1', '2', '3', '4', '5',
-		'6', '7', '8', '9', 'a', 'b',
-		'c', 'd', 'e', 'f', 'g', 'h',
-		'i', 'j', 'k', 'l', 'm', 'n',
-		'o', 'p', 'q', 'r', 's', 't',
-		'u', 'v', 'w', 'x', 'y', 'z',
-	}
-	digitTens = []uint8{
-		'0', '0', '0', '0', '0', '0', '0', '0', '0', '0',
-		'1', '1', '1', '1', '1', '1', '1', '1', '1', '1',
-		'2', '2', '2', '2', '2', '2', '2', '2', '2', '2',
-		'3', '3', '3', '3', '3', '3', '3', '3', '3', '3',
-		'4', '4', '4', '4', '4', '4', '4', '4', '4', '4',
-		'5', '5', '5', '5', '5', '5', '5', '5', '5', '5',
-		'6', '6', '6', '6', '6', '6', '6', '6', '6', '6',
-		'7', '7', '7', '7', '7', '7', '7', '7', '7', '7',
-		'8', '8', '8', '8', '8', '8', '8', '8', '8', '8',
-		'9', '9', '9', '9', '9', '9', '9', '9', '9', '9',
-	}
-	digitOnes = []uint8{
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-		'0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
-	}
 	DIGITS = make([]uint32, 1000)
 	for i := uint32(0); i < 1000; i++ {
 		DIGITS[i] = (((i / 100) + '0') << 16) + ((((i / 10) % 10) + '0') << 8) + i % 10 + '0';
@@ -72,19 +37,15 @@ func writeBuf(buf []byte, v uint32, n int) {
 }
 
 func (stream *Stream) WriteUint8(val uint8) {
-	if stream.Available() < 3 {
-		stream.Flush()
-	}
+	stream.ensure(3)
 	stream.n = writeFirstBuf(stream.buf, DIGITS[val], stream.n)
 }
 
 func (stream *Stream) WriteInt8(nval int8) {
-	if stream.Available() < 4 {
-		stream.Flush()
-	}
+	stream.ensure(4)
 	n := stream.n
 	var val uint8
-	if (nval < 0) {
+	if nval < 0 {
 		val = uint8(-nval)
 		stream.buf[n] = '-'
 		n++
@@ -95,15 +56,13 @@ func (stream *Stream) WriteInt8(nval int8) {
 }
 
 func (stream *Stream) WriteUint16(val uint16) {
-	if stream.Available() < 5 {
-		stream.Flush()
-	}
+	stream.ensure(5)
 	q1 := val / 1000
 	if q1 == 0 {
 		stream.n = writeFirstBuf(stream.buf, DIGITS[val], stream.n)
 		return
 	}
-	r1 := val - q1 * 1000;
+	r1 := val - q1 * 1000
 	n := writeFirstBuf(stream.buf, DIGITS[q1], stream.n)
 	writeBuf(stream.buf, DIGITS[r1], n)
 	stream.n = n + 3
@@ -111,12 +70,10 @@ func (stream *Stream) WriteUint16(val uint16) {
 }
 
 func (stream *Stream) WriteInt16(nval int16) {
-	if stream.Available() < 6 {
-		stream.Flush()
-	}
+	stream.ensure(6)
 	n := stream.n
 	var val uint16
-	if (nval < 0) {
+	if nval < 0 {
 		val = uint16(-nval)
 		stream.buf[n] = '-'
 		n++
@@ -128,7 +85,7 @@ func (stream *Stream) WriteInt16(nval int16) {
 		stream.n = writeFirstBuf(stream.buf, DIGITS[val], n)
 		return
 	}
-	r1 := val - q1 * 1000;
+	r1 := val - q1 * 1000
 	n = writeFirstBuf(stream.buf, DIGITS[q1], n)
 	writeBuf(stream.buf, DIGITS[r1], n)
 	stream.n = n + 3
@@ -136,16 +93,14 @@ func (stream *Stream) WriteInt16(nval int16) {
 }
 
 func (stream *Stream) WriteUint32(val uint32) {
-	if stream.Available() < 10 {
-		stream.Flush()
-	}
+	stream.ensure(10)
 	n := stream.n
 	q1 := val / 1000
 	if q1 == 0 {
 		stream.n = writeFirstBuf(stream.buf, DIGITS[val], n)
 		return
 	}
-	r1 := val - q1 * 1000;
+	r1 := val - q1 * 1000
 	q2 := q1 / 1000
 	if q2 == 0 {
 		n := writeFirstBuf(stream.buf, DIGITS[q1], n)
@@ -170,9 +125,7 @@ func (stream *Stream) WriteUint32(val uint32) {
 }
 
 func (stream *Stream) WriteInt32(nval int32) {
-	if stream.Available() < 11 {
-		stream.Flush()
-	}
+	stream.ensure(11)
 	n := stream.n
 	var val uint32
 	if (nval < 0) {
@@ -187,7 +140,7 @@ func (stream *Stream) WriteInt32(nval int32) {
 		stream.n = writeFirstBuf(stream.buf, DIGITS[val], n)
 		return
 	}
-	r1 := val - q1 * 1000;
+	r1 := val - q1 * 1000
 	q2 := q1 / 1000
 	if q2 == 0 {
 		n := writeFirstBuf(stream.buf, DIGITS[q1], n)
@@ -212,16 +165,14 @@ func (stream *Stream) WriteInt32(nval int32) {
 }
 
 func (stream *Stream) WriteUint64(val uint64) {
-	if stream.Available() < 20 {
-		stream.Flush()
-	}
+	stream.ensure(20)
 	n := stream.n
 	q1 := val / 1000
 	if q1 == 0 {
 		stream.n = writeFirstBuf(stream.buf, DIGITS[val], n)
 		return
 	}
-	r1 := val - q1 * 1000;
+	r1 := val - q1 * 1000
 	q2 := q1 / 1000
 	if q2 == 0 {
 		n := writeFirstBuf(stream.buf, DIGITS[q1], n)
@@ -278,9 +229,7 @@ func (stream *Stream) WriteUint64(val uint64) {
 }
 
 func (stream *Stream) WriteInt64(nval int64) {
-	if stream.Available() < 20 {
-		stream.Flush()
-	}
+	stream.ensure(20)
 	n := stream.n
 	var val uint64
 	if (nval < 0) {

jsoniter_array_test.go

@@ -264,6 +264,27 @@ func Test_json_RawMessage(t *testing.T) {
 	should.Equal(`[1,2,3]`, str)
 }
 
+func Test_encode_byte_array(t *testing.T) {
+	should := require.New(t)
+	bytes, err := json.Marshal([]byte{1,2,3})
+	should.Nil(err)
+	should.Equal(`"AQID"`, string(bytes))
+	bytes, err = Marshal([]byte{1,2,3})
+	should.Nil(err)
+	should.Equal(`"AQID"`, string(bytes))
+}
+
+func Test_decode_byte_array(t *testing.T) {
+	should := require.New(t)
+	data := []byte{}
+	err := json.Unmarshal([]byte(`"AQID"`), &data)
+	should.Nil(err)
+	should.Equal([]byte{1,2,3}, data)
+	err = Unmarshal([]byte(`"AQID"`), &data)
+	should.Nil(err)
+	should.Equal([]byte{1,2,3}, data)
+}
+
 func Benchmark_jsoniter_array(b *testing.B) {
 	b.ReportAllocs()
 	input := []byte(`[1,2,3,4,5,6,7,8,9]`)

jsoniter_customize_test.go

@@ -45,6 +45,7 @@ func Test_customize_type_encoder(t *testing.T) {
 }
 
 func Test_customize_byte_array_encoder(t *testing.T) {
+	CleanEncoders()
 	should := require.New(t)
 	RegisterTypeEncoder("[]uint8", func(ptr unsafe.Pointer, stream *Stream) {
 		t := *((*[]byte)(ptr))

jsoniter_stream_test.go

@@ -0,0 +1,54 @@
+package jsoniter
+
+import (
+	"testing"
+	"github.com/json-iterator/go/require"
+)
+
+func Test_writeByte_should_grow_buffer(t *testing.T) {
+	should := require.New(t)
+	stream := NewStream(nil, 1)
+	stream.writeByte('1')
+	should.Equal("1", string(stream.Buffer()))
+	should.Equal(1, len(stream.buf))
+	stream.writeByte('2')
+	should.Equal("12", string(stream.Buffer()))
+	should.Equal(2, len(stream.buf))
+	stream.writeThreeBytes('3', '4', '5')
+	should.Equal("12345", string(stream.Buffer()))
+}
+
+func Test_writeBytes_should_grow_buffer(t *testing.T) {
+	should := require.New(t)
+	stream := NewStream(nil, 1)
+	stream.Write([]byte{'1', '2'})
+	should.Equal("12", string(stream.Buffer()))
+	should.Equal(3, len(stream.buf))
+	stream.Write([]byte{'3', '4', '5', '6', '7'})
+	should.Equal("1234567", string(stream.Buffer()))
+	should.Equal(8, len(stream.buf))
+}
+
+func Test_writeIndention_should_grow_buffer(t *testing.T) {
+	should := require.New(t)
+	stream := NewStream(nil, 1)
+	stream.IndentionStep = 2
+	stream.WriteVal([]int{1,2,3})
+	should.Equal("[\n  1,\n  2,\n  3\n]", string(stream.Buffer()))
+}
+
+func Test_writeRaw_should_grow_buffer(t *testing.T) {
+	should := require.New(t)
+	stream := NewStream(nil, 1)
+	stream.WriteRaw("123")
+	should.Nil(stream.Error)
+	should.Equal("123", string(stream.Buffer()))
+}
+
+func Test_writeString_should_grow_buffer(t *testing.T) {
+	should := require.New(t)
+	stream := NewStream(nil, 0)
+	stream.WriteString("123")
+	should.Nil(stream.Error)
+	should.Equal(`"123"`, string(stream.Buffer()))
+}