yaml.go 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326
  1. // Package yaml provides a wrapper around go-yaml designed to enable a better
  2. // way of handling YAML when marshaling to and from structs.
  3. //
  4. // In short, this package first converts YAML to JSON using go-yaml and then
  5. // uses json.Marshal and json.Unmarshal to convert to or from the struct. This
  6. // means that it effectively reuses the JSON struct tags as well as the custom
  7. // JSON methods MarshalJSON and UnmarshalJSON unlike go-yaml.
  8. //
  9. // See also http://ghodss.com/2014/the-right-way-to-handle-yaml-in-golang
  10. //
  11. package yaml // import "github.com/ghodss/yaml"
  12. import (
  13. "bytes"
  14. "encoding/json"
  15. "fmt"
  16. "io"
  17. "reflect"
  18. "strconv"
  19. "gopkg.in/yaml.v2"
  20. )
  21. // Marshals the object into JSON then converts JSON to YAML and returns the
  22. // YAML.
  23. func Marshal(o interface{}) ([]byte, error) {
  24. j, err := json.Marshal(o)
  25. if err != nil {
  26. return nil, fmt.Errorf("error marshaling into JSON: %v", err)
  27. }
  28. y, err := JSONToYAML(j)
  29. if err != nil {
  30. return nil, fmt.Errorf("error converting JSON to YAML: %v", err)
  31. }
  32. return y, nil
  33. }
  34. // JSONOpt is a decoding option for decoding from JSON format.
  35. type JSONOpt func(*json.Decoder) *json.Decoder
  36. // Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object,
  37. // optionally configuring the behavior of the JSON unmarshal.
  38. func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error {
  39. return unmarshal(yaml.Unmarshal, y, o, opts)
  40. }
  41. // UnmarshalStrict is like Unmarshal except that any mapping keys that are
  42. // duplicates will result in an error.
  43. // To also be strict about unknown fields, add the DisallowUnknownFields option.
  44. func UnmarshalStrict(y []byte, o interface{}, opts ...JSONOpt) error {
  45. return unmarshal(yaml.UnmarshalStrict, y, o, opts)
  46. }
  47. func unmarshal(f func(in []byte, out interface{}) (err error), y []byte, o interface{}, opts []JSONOpt) error {
  48. vo := reflect.ValueOf(o)
  49. j, err := yamlToJSON(y, &vo, f)
  50. if err != nil {
  51. return fmt.Errorf("error converting YAML to JSON: %v", err)
  52. }
  53. err = jsonUnmarshal(bytes.NewReader(j), o, opts...)
  54. if err != nil {
  55. return fmt.Errorf("error unmarshaling JSON: %v", err)
  56. }
  57. return nil
  58. }
  59. // jsonUnmarshal unmarshals the JSON byte stream from the given reader into the
  60. // object, optionally applying decoder options prior to decoding. We are not
  61. // using json.Unmarshal directly as we want the chance to pass in non-default
  62. // options.
  63. func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error {
  64. d := json.NewDecoder(r)
  65. for _, opt := range opts {
  66. d = opt(d)
  67. }
  68. if err := d.Decode(&o); err != nil {
  69. return fmt.Errorf("while decoding JSON: %v", err)
  70. }
  71. return nil
  72. }
  73. // Convert JSON to YAML.
  74. func JSONToYAML(j []byte) ([]byte, error) {
  75. // Convert the JSON to an object.
  76. var jsonObj interface{}
  77. // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the
  78. // Go JSON library doesn't try to pick the right number type (int, float,
  79. // etc.) when unmarshalling to interface{}, it just picks float64
  80. // universally. go-yaml does go through the effort of picking the right
  81. // number type, so we can preserve number type throughout this process.
  82. err := yaml.Unmarshal(j, &jsonObj)
  83. if err != nil {
  84. return nil, err
  85. }
  86. // Marshal this object into YAML.
  87. return yaml.Marshal(jsonObj)
  88. }
  89. // YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML,
  90. // passing JSON through this method should be a no-op.
  91. //
  92. // Things YAML can do that are not supported by JSON:
  93. // * In YAML you can have binary and null keys in your maps. These are invalid
  94. // in JSON. (int and float keys are converted to strings.)
  95. // * Binary data in YAML with the !!binary tag is not supported. If you want to
  96. // use binary data with this library, encode the data as base64 as usual but do
  97. // not use the !!binary tag in your YAML. This will ensure the original base64
  98. // encoded data makes it all the way through to the JSON.
  99. //
  100. // For strict decoding of YAML, use YAMLToJSONStrict.
  101. func YAMLToJSON(y []byte) ([]byte, error) {
  102. return yamlToJSON(y, nil, yaml.Unmarshal)
  103. }
  104. // YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding,
  105. // returning an error on any duplicate field names.
  106. func YAMLToJSONStrict(y []byte) ([]byte, error) {
  107. return yamlToJSON(y, nil, yaml.UnmarshalStrict)
  108. }
  109. func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) {
  110. // Convert the YAML to an object.
  111. var yamlObj interface{}
  112. err := yamlUnmarshal(y, &yamlObj)
  113. if err != nil {
  114. return nil, err
  115. }
  116. // YAML objects are not completely compatible with JSON objects (e.g. you
  117. // can have non-string keys in YAML). So, convert the YAML-compatible object
  118. // to a JSON-compatible object, failing with an error if irrecoverable
  119. // incompatibilities happen along the way.
  120. jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget)
  121. if err != nil {
  122. return nil, err
  123. }
  124. // Convert this object to JSON and return the data.
  125. return json.Marshal(jsonObj)
  126. }
  127. func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) {
  128. var err error
  129. // Resolve jsonTarget to a concrete value (i.e. not a pointer or an
  130. // interface). We pass decodingNull as false because we're not actually
  131. // decoding into the value, we're just checking if the ultimate target is a
  132. // string.
  133. if jsonTarget != nil {
  134. ju, tu, pv := indirect(*jsonTarget, false)
  135. // We have a JSON or Text Umarshaler at this level, so we can't be trying
  136. // to decode into a string.
  137. if ju != nil || tu != nil {
  138. jsonTarget = nil
  139. } else {
  140. jsonTarget = &pv
  141. }
  142. }
  143. // If yamlObj is a number or a boolean, check if jsonTarget is a string -
  144. // if so, coerce. Else return normal.
  145. // If yamlObj is a map or array, find the field that each key is
  146. // unmarshaling to, and when you recurse pass the reflect.Value for that
  147. // field back into this function.
  148. switch typedYAMLObj := yamlObj.(type) {
  149. case map[interface{}]interface{}:
  150. // JSON does not support arbitrary keys in a map, so we must convert
  151. // these keys to strings.
  152. //
  153. // From my reading of go-yaml v2 (specifically the resolve function),
  154. // keys can only have the types string, int, int64, float64, binary
  155. // (unsupported), or null (unsupported).
  156. strMap := make(map[string]interface{})
  157. for k, v := range typedYAMLObj {
  158. // Resolve the key to a string first.
  159. var keyString string
  160. switch typedKey := k.(type) {
  161. case string:
  162. keyString = typedKey
  163. case int:
  164. keyString = strconv.Itoa(typedKey)
  165. case int64:
  166. // go-yaml will only return an int64 as a key if the system
  167. // architecture is 32-bit and the key's value is between 32-bit
  168. // and 64-bit. Otherwise the key type will simply be int.
  169. keyString = strconv.FormatInt(typedKey, 10)
  170. case float64:
  171. // Stolen from go-yaml to use the same conversion to string as
  172. // the go-yaml library uses to convert float to string when
  173. // Marshaling.
  174. s := strconv.FormatFloat(typedKey, 'g', -1, 32)
  175. switch s {
  176. case "+Inf":
  177. s = ".inf"
  178. case "-Inf":
  179. s = "-.inf"
  180. case "NaN":
  181. s = ".nan"
  182. }
  183. keyString = s
  184. case bool:
  185. if typedKey {
  186. keyString = "true"
  187. } else {
  188. keyString = "false"
  189. }
  190. default:
  191. return nil, fmt.Errorf("Unsupported map key of type: %s, key: %+#v, value: %+#v",
  192. reflect.TypeOf(k), k, v)
  193. }
  194. // jsonTarget should be a struct or a map. If it's a struct, find
  195. // the field it's going to map to and pass its reflect.Value. If
  196. // it's a map, find the element type of the map and pass the
  197. // reflect.Value created from that type. If it's neither, just pass
  198. // nil - JSON conversion will error for us if it's a real issue.
  199. if jsonTarget != nil {
  200. t := *jsonTarget
  201. if t.Kind() == reflect.Struct {
  202. keyBytes := []byte(keyString)
  203. // Find the field that the JSON library would use.
  204. var f *field
  205. fields := cachedTypeFields(t.Type())
  206. for i := range fields {
  207. ff := &fields[i]
  208. if bytes.Equal(ff.nameBytes, keyBytes) {
  209. f = ff
  210. break
  211. }
  212. // Do case-insensitive comparison.
  213. if f == nil && ff.equalFold(ff.nameBytes, keyBytes) {
  214. f = ff
  215. }
  216. }
  217. if f != nil {
  218. // Find the reflect.Value of the most preferential
  219. // struct field.
  220. jtf := t.Field(f.index[0])
  221. strMap[keyString], err = convertToJSONableObject(v, &jtf)
  222. if err != nil {
  223. return nil, err
  224. }
  225. continue
  226. }
  227. } else if t.Kind() == reflect.Map {
  228. // Create a zero value of the map's element type to use as
  229. // the JSON target.
  230. jtv := reflect.Zero(t.Type().Elem())
  231. strMap[keyString], err = convertToJSONableObject(v, &jtv)
  232. if err != nil {
  233. return nil, err
  234. }
  235. continue
  236. }
  237. }
  238. strMap[keyString], err = convertToJSONableObject(v, nil)
  239. if err != nil {
  240. return nil, err
  241. }
  242. }
  243. return strMap, nil
  244. case []interface{}:
  245. // We need to recurse into arrays in case there are any
  246. // map[interface{}]interface{}'s inside and to convert any
  247. // numbers to strings.
  248. // If jsonTarget is a slice (which it really should be), find the
  249. // thing it's going to map to. If it's not a slice, just pass nil
  250. // - JSON conversion will error for us if it's a real issue.
  251. var jsonSliceElemValue *reflect.Value
  252. if jsonTarget != nil {
  253. t := *jsonTarget
  254. if t.Kind() == reflect.Slice {
  255. // By default slices point to nil, but we need a reflect.Value
  256. // pointing to a value of the slice type, so we create one here.
  257. ev := reflect.Indirect(reflect.New(t.Type().Elem()))
  258. jsonSliceElemValue = &ev
  259. }
  260. }
  261. // Make and use a new array.
  262. arr := make([]interface{}, len(typedYAMLObj))
  263. for i, v := range typedYAMLObj {
  264. arr[i], err = convertToJSONableObject(v, jsonSliceElemValue)
  265. if err != nil {
  266. return nil, err
  267. }
  268. }
  269. return arr, nil
  270. default:
  271. // If the target type is a string and the YAML type is a number,
  272. // convert the YAML type to a string.
  273. if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String {
  274. // Based on my reading of go-yaml, it may return int, int64,
  275. // float64, or uint64.
  276. var s string
  277. switch typedVal := typedYAMLObj.(type) {
  278. case int:
  279. s = strconv.FormatInt(int64(typedVal), 10)
  280. case int64:
  281. s = strconv.FormatInt(typedVal, 10)
  282. case float64:
  283. s = strconv.FormatFloat(typedVal, 'g', -1, 32)
  284. case uint64:
  285. s = strconv.FormatUint(typedVal, 10)
  286. case bool:
  287. if typedVal {
  288. s = "true"
  289. } else {
  290. s = "false"
  291. }
  292. }
  293. if len(s) > 0 {
  294. yamlObj = interface{}(s)
  295. }
  296. }
  297. return yamlObj, nil
  298. }
  299. return nil, nil
  300. }