Lots of enhancements on decoding (setter, tags, errors, etc).

all_test.go

@@ -15,13 +15,7 @@ type S struct{}
 
 var _ = Suite(&S{})
 
-type testItem struct {
-    data string
-    value interface{}
-}
-
-
-var unmarshalTests = []testItem{
+var unmarshalTests = []struct{data string; value interface{}}{
     // It will encode either value as a string if asked for.
     {"hello: world", map[string]string{"hello": "world"}},
     {"hello: true", map[string]string{"hello": "true"}},
@@ -63,9 +57,17 @@ var unmarshalTests = []testItem{
     {"octal: 02472256", map[string]interface{}{"octal": 685230}},
     {"hexa: 0x_0A_74_AE", map[string]interface{}{"hexa": 685230}},
     {"bin: 0b1010_0111_0100_1010_1110", map[string]interface{}{"bin": 685230}},
+    {"bin: -0b101010", map[string]interface{}{"bin": -42}},
     //{"sexa: 190:20:30", map[string]interface{}{"sexa": 0}}, // Unsupported
     {"decimal: +685_230", map[string]int{"decimal": 685230}},
 
+    // Nulls from spec
+    {"empty:", map[string]interface{}{"empty": nil}},
+    {"canonical: ~", map[string]interface{}{"canonical": nil}},
+    {"english: null", map[string]interface{}{"english": nil}},
+    {"~: null key", map[interface{}]string{nil: "null key"}},
+    {"empty:", map[string]*bool{"empty": nil}},
+
     // Sequence
     {"seq: [A,B]", map[string]interface{}{"seq": []interface{}{"A", "B"}}},
     {"seq: [A,B,C]", map[string][]string{"seq": []string{"A", "B", "C"}}},
@@ -86,6 +88,25 @@ var unmarshalTests = []testItem{
     {"a: 1", &struct{B int}{0}},
     {"a: 1", &struct{B int "a"}{1}},
     {"a: y", &struct{A bool}{true}},
+
+    // Some cross type conversions
+    {"v: 42", map[string]uint{"v": 42}},
+    {"v: -42", map[string]uint{}},
+    {"v: 4294967296", map[string]uint64{"v": 4294967296}},
+    {"v: -4294967296", map[string]uint64{}},
+
+    // Overflow cases.
+    {"v: 4294967297", map[string]int32{}},
+    {"v: 128", map[string]int8{}},
+
+    // Quoted values.
+    {"'1': '2'", map[interface{}]interface{}{"1": "2"}},
+
+    // Explicit tags.
+    {"v: !!float '1.1'", map[string]interface{}{"v": 1.1}},
+    {"v: !!null ''", map[string]interface{}{"v": nil}},
+    {"%TAG !y! tag:yaml.org,2002:\n---\nv: !y!int '1'",
+     map[string]interface{}{"v": 1}},
 }
 
 
@@ -106,3 +127,88 @@ func (s *S) TestUnmarshal(c *C) {
         c.Assert(value, Equals, item.value)
     }
 }
+
+var unmarshalErrorTests = []struct{data, error string}{
+    {"v: !!float 'error'", "Can't decode !!str 'error' as a !!float"},
+}
+
+func (s *S) TestUnmarshalErrors(c *C) {
+    for _, item := range unmarshalErrorTests {
+        var value interface{}
+        err := goyaml.Unmarshal([]byte(item.data), &value)
+        c.Assert(err, Matches, item.error)
+    }
+}
+
+var setterTests = []struct{data, tag string; value interface{}}{
+    {"_: {hi: there}", "!!map", map[interface{}]interface{}{"hi": "there"}},
+    {"_: [1,A]", "!!seq", []interface{}{1, "A"}},
+    {"_: 10", "!!int", 10},
+    {"_: null", "!!null", nil},
+    {"_: !!foo 'BAR!'", "!!foo", "BAR!"},
+}
+
+var setterResult = map[int]bool{}
+
+type typeWithSetter struct {
+    tag string
+    value interface{}
+}
+
+func (o *typeWithSetter) SetYAML(tag string, value interface{}) (ok bool) {
+    o.tag = tag
+    o.value = value
+    if i, ok := value.(int); ok {
+        if result, ok := setterResult[i]; ok {
+            return result
+        }
+    }
+    return true
+}
+
+type typeWithSetterField struct {
+    Field *typeWithSetter "_"
+}
+
+func (s *S) TestUnmarshalWithSetter(c *C) {
+    for _, item := range setterTests {
+        obj := &typeWithSetterField{}
+        err := goyaml.Unmarshal([]byte(item.data), obj)
+        c.Assert(err, IsNil)
+        c.Assert(obj.Field, NotNil,
+                 Bug("Pointer not initialized (%#v)", item.value))
+        c.Assert(obj.Field.tag, Equals, item.tag)
+        c.Assert(obj.Field.value, Equals, item.value)
+    }
+}
+
+func (s *S) TestUnmarshalWholeDocumentWithSetter(c *C) {
+    obj := &typeWithSetter{}
+    err := goyaml.Unmarshal([]byte(setterTests[0].data), obj)
+    c.Assert(err, IsNil)
+    c.Assert(obj.tag, Equals, setterTests[0].tag)
+    value, ok := obj.value.(map[interface{}]interface{})
+    c.Assert(ok, Equals, true)
+    c.Assert(value["_"], Equals, setterTests[0].value)
+}
+
+func (s *S) TestUnmarshalWithFalseSetterIgnoresValue(c *C) {
+    setterResult[2] = false
+    setterResult[4] = false
+    defer func() {
+        setterResult[2] = false, false
+        setterResult[4] = false, false
+    }()
+
+    m := map[string]*typeWithSetter{}
+    data := "{abc: 1, def: 2, ghi: 3, jkl: 4}"
+    err := goyaml.Unmarshal([]byte(data), m)
+    c.Assert(err, IsNil)
+    c.Assert(m["abc"], NotNil)
+    c.Assert(m["def"], IsNil)
+    c.Assert(m["ghi"], NotNil)
+    c.Assert(m["jkl"], IsNil)
+
+    c.Assert(m["abc"].value, Equals, 1)
+    c.Assert(m["ghi"].value, Equals, 3)
+}

decode.go

@@ -71,21 +71,65 @@ func (d *decoder) drop() {
     d.unmarshal(blackHole)
 }
 
-func (d *decoder) unmarshal(out reflect.Value) bool {
+// d.setter deals with setters and pointer dereferencing and initialization.
+//
+// It's a slightly convoluted case to handle properly:
+//
+// - Nil pointers should be zeroed out, unless being set to nil
+// - We don't know at this point yet what's the value to SetYAML() with.
+// - We can't separate pointer deref/init and setter checking, because
+//   a setter may be found while going down a pointer chain.
+//
+// Thus, here is how it takes care of it:
+//
+// - out is provided as a pointer, so that it can be replaced.
+// - when looking at a non-setter ptr, *out=ptr.Elem(), unless tag=!!null
+// - when a setter is found, *out=interface{}, and a set() function is
+//   returned to call SetYAML() with the value of *out once it's defined.
+//
+func (d *decoder) setter(tag string, out *reflect.Value, good *bool) (set func()) {
+    again := true
+    for again {
+        again = false
+        setter, _ := (*out).Interface().(Setter)
+        if tag != "!!null" || setter != nil {
+            if pv, ok := (*out).(*reflect.PtrValue); ok {
+                if pv.IsNil() {
+                    *out = reflect.MakeZero(pv.Type().(*reflect.PtrType).Elem())
+                    pv.PointTo(*out)
+                } else {
+                    *out = pv.Elem()
+                }
+                setter, _ = pv.Interface().(Setter)
+                again = true
+            }
+        }
+        if setter != nil {
+            var arg interface{}
+            *out = reflect.NewValue(&arg).(*reflect.PtrValue).Elem()
+            return func() {
+                *good = setter.SetYAML(tag, arg)
+            }
+        }
+    }
+    return nil
+}
+
+func (d *decoder) unmarshal(out reflect.Value) (good bool) {
     switch d.event._type {
     case C.YAML_SCALAR_EVENT:
-        return d.scalar(out)
+        good = d.scalar(out)
     case C.YAML_MAPPING_START_EVENT:
-        return d.mapping(out)
+        good = d.mapping(out)
     case C.YAML_SEQUENCE_START_EVENT:
-        return d.sequence(out)
+        good = d.sequence(out)
     case C.YAML_DOCUMENT_START_EVENT:
-        return d.document(out)
+        good = d.document(out)
     default:
         panic("Attempted to unmarshal unexpected event: " +
               strconv.Itoa(int(d.event._type)))
     }
-    return true
+    return
 }
 
 func (d *decoder) document(out reflect.Value) bool {
@@ -99,46 +143,81 @@ func (d *decoder) document(out reflect.Value) bool {
     return result
 }
 
-func (d *decoder) scalar(out reflect.Value) (ok bool) {
+func (d *decoder) scalar(out reflect.Value) (good bool) {
     scalar := C.event_scalar(d.event)
     str := GoYString(scalar.value)
-    resolved, _ := resolve(str)
+    tag := GoYString(scalar.tag)
+    var resolved interface{}
+    if tag == "" && scalar.plain_implicit == 0 {
+        resolved = str
+    } else {
+        tag, resolved = resolve(tag, str)
+        if set := d.setter(tag, &out, &good); set != nil {
+            defer set()
+        }
+    }
     switch out := out.(type) {
     case *reflect.StringValue:
         out.Set(str)
-        ok = true
+        good = true
     case *reflect.InterfaceValue:
         out.Set(reflect.NewValue(resolved))
-        ok = true
+        good = true
     case *reflect.IntValue:
         switch resolved := resolved.(type) {
         case int:
-            out.Set(int64(resolved))
-            ok = true
+            if !out.Overflow(int64(resolved)) {
+                out.Set(int64(resolved))
+                good = true
+            }
         case int64:
-            out.Set(resolved)
-            ok = true
+            if !out.Overflow(resolved) {
+                out.Set(resolved)
+                good = true
+            }
+        }
+    case *reflect.UintValue:
+        switch resolved := resolved.(type) {
+        case int:
+            if resolved >= 0 {
+                out.Set(uint64(resolved))
+                good = true
+            }
+        case int64:
+            if resolved >= 0 {
+                out.Set(uint64(resolved))
+                good = true
+            }
         }
     case *reflect.BoolValue:
         switch resolved := resolved.(type) {
         case bool:
             out.Set(resolved)
-            ok = true
+            good = true
         }
     case *reflect.FloatValue:
         switch resolved := resolved.(type) {
         case float:
             out.Set(float64(resolved))
-            ok = true
+            good = true
+        }
+    case *reflect.PtrValue:
+        switch resolved := resolved.(type) {
+        case nil:
+            out.PointTo(nil)
+            good = true
         }
     default:
         panic("Can't handle scalar type yet: " + out.Type().String())
     }
     d.next()
-    return ok
+    return good
 }
 
-func (d *decoder) sequence(out reflect.Value) bool {
+func (d *decoder) sequence(out reflect.Value) (good bool) {
+    if set := d.setter("!!seq", &out, &good); set != nil {
+        defer set()
+    }
     if iface, ok := out.(*reflect.InterfaceValue); ok {
         // No type hints. Will have to use a generic sequence.
         out = reflect.NewValue(make([]interface{}, 0))
@@ -164,26 +243,10 @@ func (d *decoder) sequence(out reflect.Value) bool {
     return true
 }
 
-func indirect(out reflect.Value) reflect.Value {
-    for {
-        switch v := out.(type) {
-        case *reflect.PtrValue:
-            if v.IsNil() {
-                out = reflect.MakeZero(v.Type().(*reflect.PtrType).Elem())
-                v.PointTo(out)
-            } else {
-                out = v.Elem()
-            }
-        default:
-            return out
-        }
+func (d *decoder) mapping(out reflect.Value) (good bool) {
+    if set := d.setter("!!map", &out, &good); set != nil {
+        defer set()
     }
-    panic("Unreachable")
-}
-
-func (d *decoder) mapping(out reflect.Value) bool {
-    out = indirect(out)
-
     if s, ok := out.(*reflect.StructValue); ok {
         return d.mappingStruct(s)
     }

goyaml.go

@@ -2,19 +2,43 @@ package goyaml
 
 import (
     "reflect"
+    "runtime"
     "strings"
     "sync"
     "os"
 )
 
+func handleErr(err *os.Error) {
+    if r := recover(); r != nil {
+        if _, ok := r.(runtime.Error); ok {
+            panic(r)
+        } else if s, ok := r.(string); ok {
+            *err = os.ErrorString(s)
+        } else if e, ok := r.(os.Error); ok {
+            *err = e
+        } else {
+            panic(r)
+        }
+    }
+}
+
 
-func Unmarshal(in []byte, out interface{}) os.Error {
+func Unmarshal(in []byte, out interface{}) (err os.Error) {
+    defer handleErr(&err)
     d := newDecoder(in)
     defer d.destroy()
     d.unmarshal(reflect.NewValue(out))
     return nil
 }
 
+type Setter interface {
+    SetYAML(tag string, value interface{}) bool
+}
+
+type Getter interface {
+    GetYAML() (tag string, value interface{})
+}
+
 
 // --------------------------------------------------------------------------
 // Maintain a mapping of keys to structure field indexes

resolve.go

@@ -10,33 +10,9 @@ import (
 // TODO: Support merge, timestamps, and base 60 floats.
 
 
-type stdTag int
-
-var StrTag = stdTag(1)
-var BoolTag = stdTag(2)
-var IntTag = stdTag(3)
-var FloatTag = stdTag(4)
-
-func (t stdTag) String() string {
-    switch t {
-    case StrTag:
-        return "tag:yaml.org,2002:str"
-    case BoolTag:
-        return "tag:yaml.org,2002:bool"
-    case IntTag:
-        return "tag:yaml.org,2002:int"
-    case FloatTag:
-        return "tag:yaml.org,2002:float"
-    default:
-        panic("Internal error: missing tag case")
-    }
-    return ""
-}
-
-
 type resolveMapItem struct {
     value interface{}
-    tag stdTag
+    tag string
 }
 
 var resolveTable = make([]byte, 256)
@@ -50,23 +26,24 @@ func init() {
     for _, c := range "0123456789" {
         t[int(c)] = 'D' // Digit
     }
-    for _, c := range "yYnNtTfFoO" {
+    for _, c := range "yYnNtTfFoO~" {
         t[int(c)] = 'M' // In map
     }
     t[int('.')] = '.' // Float (potentially in map)
     t[int('<')] = '<' // Merge
 
-    var resolveMapList = []struct{v interface{}; tag stdTag; l []string} {
-        {true, BoolTag, []string{"y", "Y", "yes", "Yes", "YES"}},
-        {true, BoolTag, []string{"true", "True", "TRUE"}},
-        {true, BoolTag, []string{"on", "On", "ON"}},
-        {false, BoolTag, []string{"n", "N", "no", "No", "NO"}},
-        {false, BoolTag, []string{"false", "False", "FALSE"}},
-        {false, BoolTag, []string{"off", "Off", "OFF"}},
-        {math.NaN, FloatTag, []string{".nan", ".NaN", ".NAN"}},
-        {math.Inf(+1), FloatTag, []string{".inf", ".Inf", ".INF"}},
-        {math.Inf(+1), FloatTag, []string{"+.inf", "+.Inf", "+.INF"}},
-        {math.Inf(-1), FloatTag, []string{"-.inf", "-.Inf", "-.INF"}},
+    var resolveMapList = []struct{v interface{}; tag string; l []string} {
+        {true, "!!bool", []string{"y", "Y", "yes", "Yes", "YES"}},
+        {true, "!!bool", []string{"true", "True", "TRUE"}},
+        {true, "!!bool", []string{"on", "On", "ON"}},
+        {false, "!!bool", []string{"n", "N", "no", "No", "NO"}},
+        {false, "!!bool", []string{"false", "False", "FALSE"}},
+        {false, "!!bool", []string{"off", "Off", "OFF"}},
+        {nil, "!!null", []string{"~", "null", "Null", "NULL"}},
+        {math.NaN, "!!float", []string{".nan", ".NaN", ".NAN"}},
+        {math.Inf(+1), "!!float", []string{".inf", ".Inf", ".INF"}},
+        {math.Inf(+1), "!!float", []string{"+.inf", "+.Inf", "+.INF"}},
+        {math.Inf(-1), "!!float", []string{"-.inf", "-.Inf", "-.INF"}},
     }
 
     m := resolveMap
@@ -77,19 +54,48 @@ func init() {
     }
 }
 
-func resolve(in string) (out interface{}, tag stdTag) {
+const longTagPrefix = "tag:yaml.org,2002:"
+
+func shortTag(tag string) string {
+    if strings.HasPrefix(tag, longTagPrefix) {
+        return "!!" + tag[len(longTagPrefix):]
+    }
+    return tag
+}
+
+func resolvableTag(tag string) bool {
+    switch tag {
+    case "", "!!str", "!!bool", "!!int", "!!float", "!!null":
+        return true
+    }
+    return false
+}
+
+func resolve(tag string, in string) (rtag string, out interface{}) {
+    tag = shortTag(tag)
+    if !resolvableTag(tag) {
+        return tag, in
+    }
+
+    defer func() {
+        if tag != "" && tag != rtag {
+            panic("Can't decode " + rtag + " '" + in + "' as a " + tag)
+        }
+    }()
+
     if in == "" {
-        return in, tag
+        return "!!null", nil
     }
+
     c := resolveTable[in[0]]
     if c == 0 {
         // It's a string for sure. Nothing to do.
-        return in, StrTag
+        return "!!str", in
     }
 
     // Handle things we can lookup in a map.
     if item, ok := resolveMap[in]; ok {
-        return item.value, item.tag
+        return item.tag, item.value
     }
 
     switch c {
@@ -100,9 +106,9 @@ func resolve(in string) (out interface{}, tag stdTag) {
         // Not in the map, so maybe a normal float.
         floatv, err := strconv.Atof(in)
         if err == nil {
-            return floatv, FloatTag
+            return "!!float", floatv
         }
-        // XXX Handle base 60 floats here.
+        // XXX Handle base 60 floats here (WTF!)
 
     case 'D', 'S':
         // Int, float, or timestamp.
@@ -115,14 +121,25 @@ func resolve(in string) (out interface{}, tag stdTag) {
         intv, err := strconv.Btoi64(in, 0)
         if err == nil {
             if intv == int64(int(intv)) {
-                return int(intv), IntTag
+                return "!!int", int(intv)
             } else {
-                return intv, IntTag
+                return "!!int", intv
             }
         }
         floatv, err := strconv.Atof(in)
         if err == nil {
-            return floatv, FloatTag
+            return "!!float", floatv
+        }
+        if strings.HasPrefix(in, "0b") {
+            intv, err := strconv.Btoi64(in[2:], 2)
+            if err == nil {
+                return "!!int", int(intv)
+            }
+        } else if strings.HasPrefix(in, "-0b") {
+            intv, err := strconv.Btoi64(in[3:], 2)
+            if err == nil {
+                return "!!int", -int(intv)
+            }
         }
         // XXX Handle timestamps here.
 
@@ -133,5 +150,5 @@ func resolve(in string) (out interface{}, tag stdTag) {
         panic("resolveTable item not yet handled: " +
               string([]byte{c}) + " (with " + in +")")
     }
-    return in, StrTag
+    return "!!str", in
 }