codec: json: support encoding large integers as a number or string

Previously, all integers were encoded as json numbers. However, some
large integers (|x| > 2^53) could not be encoded as
json numbers (IEEE double precision) without loss of precision.

To mitigate it, we now support options to define how integers are to be
encoded, either always as json numbers, always as json strings, or
as mostly as json numbers but as json string if |x| > 2^53.

On decode, we now always support decoding a number from a JSON string.

This allows users to ensure that precision is not lost.
It doesn't solve fully the issue where some other libraries some numbers
to be encoded as json strings and others as json numbers. We consciously will
not support it, treating it as an edge case and the library provides the tools
to support the compatibility already (e.g. use a string struct member, pass
the 'A' or 'L' option to IntegerAsString option, etc).

Tests are included.

Updates #128

codec/codec_test.go

@@ -18,8 +18,6 @@ package codec
 //
 // The following manual tests must be done:
 //   - TestCodecUnderlyingType
-//   - Set fastpathEnabled to false and run tests (to ensure that regular reflection works).
-//     We don't want to use a variable there so that code is ellided.
 
 import (
 	"bytes"
@@ -1077,6 +1075,74 @@ func doTestAnonCycle(t *testing.T, name string, h Handle) {
 	logT(t, "pti: %v", pti)
 }
 
+func doTestJsonLargeInteger(t *testing.T, v interface{}, ias uint8) {
+	logT(t, "Running doTestJsonLargeInteger: v: %#v, ias: %c", v, ias)
+	oldIAS := testJsonH.IntegerAsString
+	defer func() { testJsonH.IntegerAsString = oldIAS }()
+	testJsonH.IntegerAsString = ias
+
+	var vu uint
+	var vi int
+	var vb bool
+	var b []byte
+	e := NewEncoderBytes(&b, testJsonH)
+	e.MustEncode(v)
+	e.MustEncode(true)
+	d := NewDecoderBytes(b, testJsonH)
+	// below, we validate that the json string or number was encoded,
+	// then decode, and validate that the correct value was decoded.
+	fnStrChk := func() {
+		// check that output started with ", and ended with "true
+		if !(b[0] == '"' && string(b[len(b)-5:]) == `"true`) {
+			logT(t, "Expecting a JSON string, got: %s", b)
+			failT(t)
+		}
+	}
+
+	switch ias {
+	case 'L':
+		switch v2 := v.(type) {
+		case int:
+			if v2 > 1<<53 || (v2 < 0 && -v2 > 1<<53) {
+				fnStrChk()
+			}
+		case uint:
+			if v2 > 1<<53 {
+				fnStrChk()
+			}
+		}
+	case 'A':
+		fnStrChk()
+	default:
+		// check that output doesn't contain " at all
+		for _, i := range b {
+			if i == '"' {
+				logT(t, "Expecting a JSON Number without quotation: got: %s", b)
+				failT(t)
+			}
+		}
+	}
+	switch v2 := v.(type) {
+	case int:
+		d.MustDecode(&vi)
+		d.MustDecode(&vb)
+		// check that vb = true, and vi == v2
+		if !(vb && vi == v2) {
+			logT(t, "Expecting equal values from %s: got golden: %v, decoded: %v", b, v2, vi)
+			failT(t)
+		}
+	case uint:
+		d.MustDecode(&vu)
+		d.MustDecode(&vb)
+		// check that vb = true, and vi == v2
+		if !(vb && vu == v2) {
+			logT(t, "Expecting equal values from %s: got golden: %v, decoded: %v", b, v2, vu)
+			failT(t)
+		}
+		// fmt.Printf("%v: %s, decode: %d, bool: %v, equal_on_decode: %v\n", v, b, vu, vb, vu == v.(uint))
+	}
+}
+
 // Comprehensive testing that generates data encoded from python handle (cbor, msgpack),
 // and validates that our code can read and write it out accordingly.
 // We keep this unexported here, and put actual test in ext_dep_test.go.
@@ -1356,6 +1422,23 @@ func TestBincUnderlyingType(t *testing.T) {
 	testCodecUnderlyingType(t, testBincH)
 }
 
+func TestJsonLargeInteger(t *testing.T) {
+	for _, i := range []uint8{'L', 'A', 0} {
+		for _, j := range []interface{}{
+			1 << 60,
+			-(1 << 60),
+			0,
+			1 << 20,
+			-(1 << 20),
+			uint(1 << 60),
+			uint(0),
+			uint(1 << 20),
+		} {
+			doTestJsonLargeInteger(t, j, i)
+		}
+	}
+}
+
 // TODO:
 //   Add Tests for:
 //   - decoding empty list/map in stream into a nil slice/map

codec/json.go

@@ -206,10 +206,22 @@ func (e *jsonEncDriver) EncodeFloat64(f float64) {
 }
 
 func (e *jsonEncDriver) EncodeInt(v int64) {
+	if x := e.h.IntegerAsString; x == 'A' || x == 'L' && (v > 1<<53 || v < -(1<<53)) {
+		e.w.writen1('"')
+		e.w.writeb(strconv.AppendInt(e.b[:0], v, 10))
+		e.w.writen1('"')
+		return
+	}
 	e.w.writeb(strconv.AppendInt(e.b[:0], v, 10))
 }
 
 func (e *jsonEncDriver) EncodeUint(v uint64) {
+	if x := e.h.IntegerAsString; x == 'A' || x == 'L' && v > 1<<53 {
+		e.w.writen1('"')
+		e.w.writeb(strconv.AppendUint(e.b[:0], v, 10))
+		e.w.writen1('"')
+		return
+	}
 	e.w.writeb(strconv.AppendUint(e.b[:0], v, 10))
 }
 
@@ -636,6 +648,11 @@ func (d *jsonDecDriver) decNum(storeBytes bool) {
 		d.tok = b
 	}
 	b := d.tok
+	var str bool
+	if b == '"' {
+		str = true
+		b = d.r.readn1()
+	}
 	if !(b == '+' || b == '-' || b == '.' || (b >= '0' && b <= '9')) {
 		d.d.errorf("json: decNum: got first char '%c'", b)
 		return
@@ -650,6 +667,10 @@ func (d *jsonDecDriver) decNum(storeBytes bool) {
 	n.reset()
 	d.bs = d.bs[:0]
 
+	if str && storeBytes {
+		d.bs = append(d.bs, '"')
+	}
+
 	// The format of a number is as below:
 	// parsing:     sign? digit* dot? digit* e?  sign? digit*
 	// states:  0   1*    2      3*   4      5*  6     7
@@ -740,6 +761,14 @@ LOOP:
 			default:
 				break LOOP
 			}
+		case '"':
+			if str {
+				if storeBytes {
+					d.bs = append(d.bs, '"')
+				}
+				b, eof = r.readn1eof()
+			}
+			break LOOP
 		default:
 			break LOOP
 		}
@@ -1110,6 +1139,19 @@ type JsonHandle struct {
 	//   - If positive, indent by that number of spaces.
 	//   - If negative, indent by that number of tabs.
 	Indent int8
+
+	// IntegerAsString controls how integers (signed and unsigned) are encoded.
+	//
+	// Per the JSON Spec, JSON numbers are 64-bit floating point numbers.
+	// Consequently, integers > 2^53 cannot be represented as a JSON number without losing precision.
+	// This can be mitigated by configuring how to encode integers.
+	//
+	// IntegerAsString interpretes the following values:
+	//   - if 'L', then encode integers > 2^53 as a json string.
+	//   - if 'A', then encode all integers as a json string
+	//             containing the exact integer representation as a decimal.
+	//   - else    encode all integers as a json number (default)
+	IntegerAsString uint8
 }
 
 func (h *JsonHandle) SetInterfaceExt(rt reflect.Type, tag uint64, ext InterfaceExt) (err error) {