optimize read int32

feature_iter_float.go

@@ -6,25 +6,25 @@ import (
 	"unsafe"
 )
 
-var zeroToNineDigits []int8
+var floatDigits []int8
 const invalidCharForNumber = int8(-1)
 const endOfNumber = int8(-2)
 const dotInNumber = int8(-3)
-const safeToMultiple10 = uint64(0xffffffffffffffff) / 10 - 10
+const uint64SafeToMultiple10 = uint64(0xffffffffffffffff) / 10 - 10
 
 func init() {
-	zeroToNineDigits = make([]int8, 256)
-	for i := 0; i < len(zeroToNineDigits); i++ {
-		zeroToNineDigits[i] = invalidCharForNumber
+	floatDigits = make([]int8, 256)
+	for i := 0; i < len(floatDigits); i++ {
+		floatDigits[i] = invalidCharForNumber
 	}
-	for i := int8('0'); i < int8('9'); i++ {
-		zeroToNineDigits[i] = i - int8('0')
+	for i := int8('0'); i <= int8('9'); i++ {
+		floatDigits[i] = i - int8('0')
 	}
-	zeroToNineDigits[','] = endOfNumber;
-	zeroToNineDigits[']'] = endOfNumber;
-	zeroToNineDigits['}'] = endOfNumber;
-	zeroToNineDigits[' '] = endOfNumber;
-	zeroToNineDigits['.'] = dotInNumber;
+	floatDigits[','] = endOfNumber;
+	floatDigits[']'] = endOfNumber;
+	floatDigits['}'] = endOfNumber;
+	floatDigits[' '] = endOfNumber;
+	floatDigits['.'] = dotInNumber;
 }
 
 func (iter *Iterator) ReadFloat32() (ret float32) {
@@ -44,7 +44,7 @@ func (iter *Iterator) readPositiveFloat32() (ret float32) {
 	non_decimal_loop:
 	for ; i < iter.tail; i++ {
 		c = iter.buf[i]
-		ind := zeroToNineDigits[c]
+		ind := floatDigits[c]
 		switch ind {
 		case invalidCharForNumber:
 			return iter.readFloat32SlowPath()
@@ -54,7 +54,7 @@ func (iter *Iterator) readPositiveFloat32() (ret float32) {
 		case dotInNumber:
 			break non_decimal_loop
 		}
-		if value > safeToMultiple10 {
+		if value > uint64SafeToMultiple10 {
 			return iter.readFloat32SlowPath()
 		}
 		value = (value << 3) + (value << 1) + uint64(ind); // value = value * 10 + ind;
@@ -64,7 +64,7 @@ func (iter *Iterator) readPositiveFloat32() (ret float32) {
 		decimalPlaces := 0;
 		for ; i < iter.tail; i++ {
 			c = iter.buf[i]
-			ind := zeroToNineDigits[c];
+			ind := floatDigits[c];
 			switch ind {
 			case endOfNumber:
 				if decimalPlaces > 0 && decimalPlaces < len(POW10) {
@@ -79,7 +79,7 @@ func (iter *Iterator) readPositiveFloat32() (ret float32) {
 				return iter.readFloat32SlowPath()
 			}
 			decimalPlaces++
-			if value > safeToMultiple10 {
+			if value > uint64SafeToMultiple10 {
 				return iter.readFloat32SlowPath()
 			}
 			value = (value << 3) + (value << 1) + uint64(ind)
@@ -135,7 +135,7 @@ func (iter *Iterator) readPositiveFloat64() (ret float64) {
 	non_decimal_loop:
 	for ; i < iter.tail; i++ {
 		c = iter.buf[i]
-		ind := zeroToNineDigits[c]
+		ind := floatDigits[c]
 		switch ind {
 		case invalidCharForNumber:
 			return iter.readFloat64SlowPath()
@@ -145,7 +145,7 @@ func (iter *Iterator) readPositiveFloat64() (ret float64) {
 		case dotInNumber:
 			break non_decimal_loop
 		}
-		if value > safeToMultiple10 {
+		if value > uint64SafeToMultiple10 {
 			return iter.readFloat64SlowPath()
 		}
 		value = (value << 3) + (value << 1) + uint64(ind); // value = value * 10 + ind;
@@ -155,7 +155,7 @@ func (iter *Iterator) readPositiveFloat64() (ret float64) {
 		decimalPlaces := 0;
 		for ; i < iter.tail; i++ {
 			c = iter.buf[i]
-			ind := zeroToNineDigits[c];
+			ind := floatDigits[c];
 			switch ind {
 			case endOfNumber:
 				if decimalPlaces > 0 && decimalPlaces < len(POW10) {
@@ -170,7 +170,7 @@ func (iter *Iterator) readPositiveFloat64() (ret float64) {
 				return iter.readFloat64SlowPath()
 			}
 			decimalPlaces++
-			if value > safeToMultiple10 {
+			if value > uint64SafeToMultiple10 {
 				return iter.readFloat64SlowPath()
 			}
 			value = (value << 3) + (value << 1) + uint64(ind)

feature_iter_int.go

@@ -0,0 +1,171 @@
+package jsoniter
+
+
+var intDigits []int8
+const maxUint64 = (1<<64 - 1)
+const cutoffUint64 = maxUint64/10 + 1
+const maxUint32 = (1<<32 - 1)
+const cutoffUint32 = maxUint32/10 + 1
+const int32SafeToMultiply10 = uint32(int32(0x7fffffff)/10 - 10)
+const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 10
+
+func init() {
+	intDigits = make([]int8, 256)
+	for i := 0; i < len(floatDigits); i++ {
+		intDigits[i] = invalidCharForNumber
+	}
+	for i := int8('0'); i <= int8('9'); i++ {
+		intDigits[i] = i - int8('0')
+	}
+}
+
+// ReadUint reads a json object as Uint
+func (iter *Iterator) ReadUint() (ret uint) {
+	val := iter.ReadUint64()
+	converted := uint(val)
+	if uint64(converted) != val {
+		iter.reportError("ReadUint", "int overflow")
+		return
+	}
+	return converted
+}
+
+// ReadUint8 reads a json object as Uint8
+func (iter *Iterator) ReadUint8() (ret uint8) {
+	val := iter.ReadUint64()
+	converted := uint8(val)
+	if uint64(converted) != val {
+		iter.reportError("ReadUint8", "int overflow")
+		return
+	}
+	return converted
+}
+
+// ReadUint16 reads a json object as Uint16
+func (iter *Iterator) ReadUint16() (ret uint16) {
+	val := iter.ReadUint64()
+	converted := uint16(val)
+	if uint64(converted) != val {
+		iter.reportError("ReadUint16", "int overflow")
+		return
+	}
+	return converted
+}
+
+// ReadUint64 reads a json object as Uint64
+func (iter *Iterator) ReadUint64() (ret uint64) {
+	c := iter.nextToken()
+	v := hexDigits[c]
+	if v == 0 {
+		return 0 // single zero
+	}
+	if v == 255 {
+		iter.reportError("ReadUint64", "unexpected character")
+		return
+	}
+	for {
+		if ret >= cutoffUint64 {
+			iter.reportError("ReadUint64", "overflow")
+			return
+		}
+		ret = ret*10 + uint64(v)
+		c = iter.readByte()
+		v = hexDigits[c]
+		if v == 255 {
+			iter.unreadByte()
+			break
+		}
+	}
+	return ret
+}
+
+// ReadInt reads a json object as Int
+func (iter *Iterator) ReadInt() (ret int) {
+	val := iter.ReadInt64()
+	converted := int(val)
+	if int64(converted) != val {
+		iter.reportError("ReadInt", "int overflow")
+		return
+	}
+	return converted
+}
+
+// ReadInt8 reads a json object as Int8
+func (iter *Iterator) ReadInt8() (ret int8) {
+	val := iter.ReadInt64()
+	converted := int8(val)
+	if int64(converted) != val {
+		iter.reportError("ReadInt8", "int overflow")
+		return
+	}
+	return converted
+}
+
+// ReadInt16 reads a json object as Int16
+func (iter *Iterator) ReadInt16() (ret int16) {
+	val := iter.ReadInt64()
+	converted := int16(val)
+	if int64(converted) != val {
+		iter.reportError("ReadInt16", "int overflow")
+		return
+	}
+	return converted
+}
+
+func (iter *Iterator) ReadInt32() int32 {
+	c := iter.nextToken()
+	if c == '-' {
+		return -int32(iter.readUint32(int32SafeToMultiply10, iter.readByte()))
+	} else {
+		return int32(iter.readUint32(int32SafeToMultiply10, c))
+	}
+}
+
+func (iter *Iterator) ReadUint32() uint32 {
+	return iter.readUint32(uint32SafeToMultiply10, iter.nextToken())
+}
+
+func (iter *Iterator) readUint32(safeToMultiply10 uint32, c byte) (ret uint32) {
+	ind := intDigits[c]
+	if ind == 0 {
+		return 0 // single zero
+	}
+	if ind == invalidCharForNumber {
+		iter.reportError("readUint32", "unexpected character: " + string([]byte{byte(ind)}))
+		return
+	}
+	value := uint32(ind)
+	for {
+		for i := iter.head; i < iter.tail; i++ {
+			if value > safeToMultiply10 {
+				iter.reportError("readUint32", "overflow")
+				return
+			}
+			ind = intDigits[iter.buf[i]]
+			if ind == invalidCharForNumber {
+				return value
+			}
+			value =  (value << 3) + (value << 1) + uint32(ind)
+		}
+		if (!iter.loadMore()) {
+			return value
+		}
+	}
+}
+
+// ReadInt64 reads a json object as Int64
+func (iter *Iterator) ReadInt64() (ret int64) {
+	c := iter.nextToken()
+	if iter.Error != nil {
+		return
+	}
+
+	/* optional leading minus */
+	if c == '-' {
+		n := iter.ReadUint64()
+		return -int64(n)
+	}
+	iter.unreadByte()
+	n := iter.ReadUint64()
+	return int64(n)
+}

iterator.go

@@ -19,22 +19,22 @@ const (
 	Object
 )
 
-var atoiDigits []byte
+var hexDigits []byte
 var valueTypes []ValueType
 
 func init() {
-	atoiDigits = make([]byte, 256)
-	for i := 0; i < len(atoiDigits); i++ {
-		atoiDigits[i] = 255
+	hexDigits = make([]byte, 256)
+	for i := 0; i < len(hexDigits); i++ {
+		hexDigits[i] = 255
 	}
 	for i := '0'; i <= '9'; i++ {
-		atoiDigits[i] = byte(i - '0')
+		hexDigits[i] = byte(i - '0')
 	}
 	for i := 'a'; i <= 'f'; i++ {
-		atoiDigits[i] = byte((i - 'a') + 10)
+		hexDigits[i] = byte((i - 'a') + 10)
 	}
 	for i := 'A'; i <= 'F'; i++ {
-		atoiDigits[i] = byte((i - 'A') + 10)
+		hexDigits[i] = byte((i - 'A') + 10)
 	}
 	valueTypes = make([]ValueType, 256)
 	for i := 0; i < len(valueTypes); i++ {
@@ -224,143 +224,6 @@ func (iter *Iterator) unreadByte() {
 	return
 }
 
-const maxUint64 = (1<<64 - 1)
-const cutoffUint64 = maxUint64/10 + 1
-const maxUint32 = (1<<32 - 1)
-const cutoffUint32 = maxUint32/10 + 1
-
-// ReadUint reads a json object as Uint
-func (iter *Iterator) ReadUint() (ret uint) {
-	val := iter.ReadUint64()
-	converted := uint(val)
-	if uint64(converted) != val {
-		iter.reportError("ReadUint", "int overflow")
-		return
-	}
-	return converted
-}
-
-// ReadUint8 reads a json object as Uint8
-func (iter *Iterator) ReadUint8() (ret uint8) {
-	val := iter.ReadUint64()
-	converted := uint8(val)
-	if uint64(converted) != val {
-		iter.reportError("ReadUint8", "int overflow")
-		return
-	}
-	return converted
-}
-
-// ReadUint16 reads a json object as Uint16
-func (iter *Iterator) ReadUint16() (ret uint16) {
-	val := iter.ReadUint64()
-	converted := uint16(val)
-	if uint64(converted) != val {
-		iter.reportError("ReadUint16", "int overflow")
-		return
-	}
-	return converted
-}
-
-// ReadUint32 reads a json object as Uint32
-func (iter *Iterator) ReadUint32() (ret uint32) {
-	val := iter.ReadUint64()
-	converted := uint32(val)
-	if uint64(converted) != val {
-		iter.reportError("ReadUint32", "int overflow")
-		return
-	}
-	return converted
-}
-
-// ReadUint64 reads a json object as Uint64
-func (iter *Iterator) ReadUint64() (ret uint64) {
-	c := iter.nextToken()
-	v := atoiDigits[c]
-	if v == 0 {
-		return 0 // single zero
-	}
-	if v == 255 {
-		iter.reportError("ReadUint64", "unexpected character")
-		return
-	}
-	for {
-		if ret >= cutoffUint64 {
-			iter.reportError("ReadUint64", "overflow")
-			return
-		}
-		ret = ret*10 + uint64(v)
-		c = iter.readByte()
-		v = atoiDigits[c]
-		if v == 255 {
-			iter.unreadByte()
-			break
-		}
-	}
-	return ret
-}
-
-// ReadInt reads a json object as Int
-func (iter *Iterator) ReadInt() (ret int) {
-	val := iter.ReadInt64()
-	converted := int(val)
-	if int64(converted) != val {
-		iter.reportError("ReadInt", "int overflow")
-		return
-	}
-	return converted
-}
-
-// ReadInt8 reads a json object as Int8
-func (iter *Iterator) ReadInt8() (ret int8) {
-	val := iter.ReadInt64()
-	converted := int8(val)
-	if int64(converted) != val {
-		iter.reportError("ReadInt8", "int overflow")
-		return
-	}
-	return converted
-}
-
-// ReadInt16 reads a json object as Int16
-func (iter *Iterator) ReadInt16() (ret int16) {
-	val := iter.ReadInt64()
-	converted := int16(val)
-	if int64(converted) != val {
-		iter.reportError("ReadInt16", "int overflow")
-		return
-	}
-	return converted
-}
-
-// ReadInt32 reads a json object as Int32
-func (iter *Iterator) ReadInt32() (ret int32) {
-	val := iter.ReadInt64()
-	converted := int32(val)
-	if int64(converted) != val {
-		iter.reportError("ReadInt32", "int overflow")
-		return
-	}
-	return converted
-}
-
-// ReadInt64 reads a json object as Int64
-func (iter *Iterator) ReadInt64() (ret int64) {
-	c := iter.nextToken()
-	if iter.Error != nil {
-		return
-	}
-
-	/* optional leading minus */
-	if c == '-' {
-		n := iter.ReadUint64()
-		return -int64(n)
-	}
-	iter.unreadByte()
-	n := iter.ReadUint64()
-	return int64(n)
-}
-
 // ReadString reads a json object as String
 func (iter *Iterator) ReadString() (ret string) {
 	return string(iter.readStringAsBytes())

jsoniter_int_test.go

@@ -72,6 +72,27 @@ func Test_decode_int64_minus_100(t *testing.T) {
 	}
 }
 
+func Test_read_int32(t *testing.T) {
+	inputs := []string{`1`, `12`, `123`}
+	for _, input := range inputs {
+		t.Run(fmt.Sprintf("%v", input), func(t *testing.T) {
+			should := require.New(t)
+			iter := ParseString(input)
+			expected, err := strconv.ParseInt(input, 10, 32)
+			should.Nil(err)
+			should.Equal(int32(expected), iter.ReadInt32())
+		})
+	}
+}
+
+func Test_read_int32_overflow(t *testing.T) {
+	should := require.New(t)
+	input := "123456789123456789"
+	iter := ParseString(input)
+	iter.ReadInt32()
+	should.NotNil(iter.Error)
+}
+
 func Test_write_uint8(t *testing.T) {
 	vals := []uint8{0, 1, 11, 111, 255}
 	for _, val := range vals {
@@ -368,8 +389,10 @@ func Benchmark_itoa(b *testing.B) {
 }
 
 func Benchmark_jsoniter_int(b *testing.B) {
+	iter := NewIterator()
+	input := []byte(`100`)
 	for n := 0; n < b.N; n++ {
-		iter := ParseString(`-100`)
+		iter.ResetBytes(input)
 		iter.ReadInt64()
 	}
 }