fix #191 do not always assume the object field is simple string

feature_adapter.go

@@ -110,9 +110,7 @@ type Encoder struct {
 // Encode encode interface{} as JSON to io.Writer
 func (adapter *Encoder) Encode(val interface{}) error {
 	adapter.stream.WriteVal(val)
-	if adapter.stream.cfg.addNewline {
-		adapter.stream.WriteRaw("\n")
-	}
+	adapter.stream.WriteRaw("\n")
 	adapter.stream.Flush()
 	return adapter.stream.Error
 }

feature_config.go

@@ -12,26 +12,26 @@ import (
 // Config customize how the API should behave.
 // The API is created from Config by Froze.
 type Config struct {
-	IndentionStep           int
-	MarshalFloatWith6Digits bool
-	EscapeHTML              bool
-	SortMapKeys             bool
-	UseNumber               bool
-	TagKey                  string
-	ValidateJsonRawMessage  bool
-	AddNewline              bool
+	IndentionStep                 int
+	MarshalFloatWith6Digits       bool
+	EscapeHTML                    bool
+	SortMapKeys                   bool
+	UseNumber                     bool
+	TagKey                        string
+	ValidateJsonRawMessage        bool
+	ObjectFieldMustBeSimpleString bool
 }
 
 type frozenConfig struct {
-	configBeforeFrozen Config
-	sortMapKeys        bool
-	addNewline         bool
-	indentionStep      int
-	decoderCache       unsafe.Pointer
-	encoderCache       unsafe.Pointer
-	extensions         []Extension
-	streamPool         chan *Stream
-	iteratorPool       chan *Iterator
+	configBeforeFrozen            Config
+	sortMapKeys                   bool
+	indentionStep                 int
+	objectFieldMustBeSimpleString bool
+	decoderCache                  unsafe.Pointer
+	encoderCache                  unsafe.Pointer
+	extensions                    []Extension
+	streamPool                    chan *Stream
+	iteratorPool                  chan *Iterator
 }
 
 // API the public interface of this package.
@@ -60,24 +60,24 @@ var ConfigCompatibleWithStandardLibrary = Config{
 	EscapeHTML:             true,
 	SortMapKeys:            true,
 	ValidateJsonRawMessage: true,
-	AddNewline:             true,
 }.Froze()
 
 // ConfigFastest marshals float with only 6 digits precision
 var ConfigFastest = Config{
-	EscapeHTML:              false,
-	MarshalFloatWith6Digits: true,
+	EscapeHTML:                    false,
+	MarshalFloatWith6Digits:       true, // will lose precession
+	ObjectFieldMustBeSimpleString: true, // do not unescape object field
 }.Froze()
 
 // Froze forge API from config
 func (cfg Config) Froze() API {
 	// TODO: cache frozen config
 	frozenConfig := &frozenConfig{
-		sortMapKeys:   cfg.SortMapKeys,
-		indentionStep: cfg.IndentionStep,
-		addNewline:    cfg.AddNewline,
-		streamPool:    make(chan *Stream, 16),
-		iteratorPool:  make(chan *Iterator, 16),
+		sortMapKeys:                   cfg.SortMapKeys,
+		indentionStep:                 cfg.IndentionStep,
+		objectFieldMustBeSimpleString: cfg.ObjectFieldMustBeSimpleString,
+		streamPool:                    make(chan *Stream, 16),
+		iteratorPool:                  make(chan *Iterator, 16),
 	}
 	atomic.StorePointer(&frozenConfig.decoderCache, unsafe.Pointer(&map[string]ValDecoder{}))
 	atomic.StorePointer(&frozenConfig.encoderCache, unsafe.Pointer(&map[string]ValEncoder{}))

feature_iter_object.go

@@ -19,7 +19,16 @@ func (iter *Iterator) ReadObject() (ret string) {
 		c = iter.nextToken()
 		if c == '"' {
 			iter.unreadByte()
-			return string(iter.readObjectFieldAsBytes())
+			if iter.cfg.objectFieldMustBeSimpleString {
+				return string(iter.readObjectFieldAsBytes())
+			} else {
+				field := iter.ReadString()
+				c = iter.nextToken()
+				if c != ':' {
+					iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))
+				}
+				return field
+			}
 		}
 		if c == '}' {
 			return "" // end of object
@@ -27,7 +36,16 @@ func (iter *Iterator) ReadObject() (ret string) {
 		iter.ReportError("ReadObject", `expect " after {, but found `+string([]byte{c}))
 		return
 	case ',':
-		return string(iter.readObjectFieldAsBytes())
+		if iter.cfg.objectFieldMustBeSimpleString {
+			return string(iter.readObjectFieldAsBytes())
+		} else {
+			field := iter.ReadString()
+			c = iter.nextToken()
+			if c != ':' {
+				iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))
+			}
+			return field
+		}
 	case '}':
 		return "" // end of object
 	default:
@@ -44,17 +62,34 @@ func (iter *Iterator) readFieldHash() int32 {
 			for i := iter.head; i < iter.tail; i++ {
 				// require ascii string and no escape
 				b := iter.buf[i]
-				if 'A' <= b && b <= 'Z' {
-					b += 'a' - 'A'
+				if !iter.cfg.objectFieldMustBeSimpleString && b == '\\' {
+					iter.head = i
+					for _, b := range iter.readStringSlowPath() {
+						if 'A' <= b && b <= 'Z' {
+							b += 'a' - 'A'
+						}
+						hash ^= int64(b)
+						hash *= 0x1000193
+					}
+					c = iter.nextToken()
+					if c != ':' {
+						iter.ReportError("readFieldHash", `expect :, but found `+string([]byte{c}))
+						return 0
+					}
+					return int32(hash)
 				}
 				if b == '"' {
 					iter.head = i + 1
 					c = iter.nextToken()
 					if c != ':' {
 						iter.ReportError("readFieldHash", `expect :, but found `+string([]byte{c}))
+						return 0
 					}
 					return int32(hash)
 				}
+				if 'A' <= b && b <= 'Z' {
+					b += 'a' - 'A'
+				}
 				hash ^= int64(b)
 				hash *= 0x1000193
 			}
@@ -80,18 +115,38 @@ func calcHash(str string) int32 {
 // ReadObjectCB read object with callback, the key is ascii only and field name not copied
 func (iter *Iterator) ReadObjectCB(callback func(*Iterator, string) bool) bool {
 	c := iter.nextToken()
+	var fieldBytes []byte
+	var field string
 	if c == '{' {
 		c = iter.nextToken()
 		if c == '"' {
 			iter.unreadByte()
-			field := iter.readObjectFieldAsBytes()
-			if !callback(iter, *(*string)(unsafe.Pointer(&field))) {
+			if iter.cfg.objectFieldMustBeSimpleString {
+				fieldBytes = iter.readObjectFieldAsBytes()
+				field = *(*string)(unsafe.Pointer(&fieldBytes))
+			} else {
+				field = iter.ReadString()
+				c = iter.nextToken()
+				if c != ':' {
+					iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))
+				}
+			}
+			if !callback(iter, field) {
 				return false
 			}
 			c = iter.nextToken()
 			for c == ',' {
-				field = iter.readObjectFieldAsBytes()
-				if !callback(iter, *(*string)(unsafe.Pointer(&field))) {
+				if iter.cfg.objectFieldMustBeSimpleString {
+					fieldBytes = iter.readObjectFieldAsBytes()
+					field = *(*string)(unsafe.Pointer(&fieldBytes))
+				} else {
+					field = iter.ReadString()
+					c = iter.nextToken()
+					if c != ':' {
+						iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))
+					}
+				}
+				if !callback(iter, field) {
 					return false
 				}
 				c = iter.nextToken()

feature_reflect_struct_decoder.go

@@ -427,8 +427,18 @@ func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator)
 	if !iter.readObjectStart() {
 		return
 	}
-	fieldBytes := iter.readObjectFieldAsBytes()
-	field := *(*string)(unsafe.Pointer(&fieldBytes))
+	var fieldBytes []byte
+	var field string
+	if iter.cfg.objectFieldMustBeSimpleString {
+		fieldBytes = iter.readObjectFieldAsBytes()
+		field = *(*string)(unsafe.Pointer(&fieldBytes))
+	} else {
+		field = iter.ReadString()
+		c := iter.nextToken()
+		if c != ':' {
+			iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))
+		}
+	}
 	fieldDecoder := decoder.fields[strings.ToLower(field)]
 	if fieldDecoder == nil {
 		iter.Skip()
@@ -436,8 +446,16 @@ func (decoder *generalStructDecoder) Decode(ptr unsafe.Pointer, iter *Iterator)
 		fieldDecoder.Decode(ptr, iter)
 	}
 	for iter.nextToken() == ',' {
-		fieldBytes = iter.readObjectFieldAsBytes()
-		field = *(*string)(unsafe.Pointer(&fieldBytes))
+		if iter.cfg.objectFieldMustBeSimpleString {
+			fieldBytes := iter.readObjectFieldAsBytes()
+			field = *(*string)(unsafe.Pointer(&fieldBytes))
+		} else {
+			field = iter.ReadString()
+			c := iter.nextToken()
+			if c != ':' {
+				iter.ReportError("ReadObject", "expect : after object field, but found "+string([]byte{c}))
+			}
+		}
 		fieldDecoder = decoder.fields[strings.ToLower(field)]
 		if fieldDecoder == nil {
 			iter.Skip()

jsoniter_1dot8_only_test.go

@@ -22,7 +22,7 @@ func Test_new_encoder(t *testing.T) {
 	encoder2 := NewEncoder(buf2)
 	encoder2.SetEscapeHTML(false)
 	encoder2.Encode([]int{1})
-	should.Equal("[1]", buf2.String())
+	should.Equal("[1]\n", buf2.String())
 }
 
 func Test_string_encode_with_std_without_html_escape(t *testing.T) {

jsoniter_object_test.go

@@ -328,3 +328,15 @@ func Test_decode_nested(t *testing.T) {
 		t.Fatal(slice[2])
 	}
 }
+
+func Test_decode_field_with_escape(t *testing.T) {
+	should := require.New(t)
+	type TestObject struct {
+		Field1 string
+	}
+	var obj TestObject
+	should.Nil(ConfigCompatibleWithStandardLibrary.Unmarshal([]byte(`{"Field\"1":"hello"}`), &obj))
+	should.Equal("", obj.Field1)
+	should.Nil(ConfigCompatibleWithStandardLibrary.Unmarshal([]byte(`{"\u0046ield1":"hello"}`), &obj))
+	should.Equal("hello", obj.Field1)
+}