xml.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502
  1. // Copyright (c) 2012-2018 Ugorji Nwoke. All rights reserved.
  2. // Use of this source code is governed by a MIT license found in the LICENSE file.
  3. // +build ignore
  4. package codec
  5. /*
  6. A strict Non-validating namespace-aware XML 1.0 parser and (en|de)coder.
  7. We are attempting this due to perceived issues with encoding/xml:
  8. - Complicated. It tried to do too much, and is not as simple to use as json.
  9. - Due to over-engineering, reflection is over-used AND performance suffers:
  10. java is 6X faster:http://fabsk.eu/blog/category/informatique/dev/golang/
  11. even PYTHON performs better: http://outgoing.typepad.com/outgoing/2014/07/exploring-golang.html
  12. codec framework will offer the following benefits
  13. - VASTLY improved performance (when using reflection-mode or codecgen)
  14. - simplicity and consistency: with the rest of the supported formats
  15. - all other benefits of codec framework (streaming, codegeneration, etc)
  16. codec is not a drop-in replacement for encoding/xml.
  17. It is a replacement, based on the simplicity and performance of codec.
  18. Look at it like JAXB for Go.
  19. Challenges:
  20. - Need to output XML preamble, with all namespaces at the right location in the output.
  21. - Each "end" block is dynamic, so we need to maintain a context-aware stack
  22. - How to decide when to use an attribute VS an element
  23. - How to handle chardata, attr, comment EXPLICITLY.
  24. - Should it output fragments?
  25. e.g. encoding a bool should just output true OR false, which is not well-formed XML.
  26. Extend the struct tag. See representative example:
  27. type X struct {
  28. ID uint8 `codec:"http://ugorji.net/x-namespace xid id,omitempty,toarray,attr,cdata"`
  29. // format: [namespace-uri ][namespace-prefix ]local-name, ...
  30. }
  31. Based on this, we encode
  32. - fields as elements, BUT
  33. encode as attributes if struct tag contains ",attr" and is a scalar (bool, number or string)
  34. - text as entity-escaped text, BUT encode as CDATA if struct tag contains ",cdata".
  35. To handle namespaces:
  36. - XMLHandle is denoted as being namespace-aware.
  37. Consequently, we WILL use the ns:name pair to encode and decode if defined, else use the plain name.
  38. - *Encoder and *Decoder know whether the Handle "prefers" namespaces.
  39. - add *Encoder.getEncName(*structFieldInfo).
  40. No one calls *structFieldInfo.indexForEncName directly anymore
  41. - OR better yet: indexForEncName is namespace-aware, and helper.go is all namespace-aware
  42. indexForEncName takes a parameter of the form namespace:local-name OR local-name
  43. - add *Decoder.getStructFieldInfo(encName string) // encName here is either like abc, or h1:nsabc
  44. by being a method on *Decoder, or maybe a method on the Handle itself.
  45. No one accesses .encName anymore
  46. - let encode.go and decode.go use these (for consistency)
  47. - only problem exists for gen.go, where we create a big switch on encName.
  48. Now, we also have to add a switch on strings.endsWith(kName, encNsName)
  49. - gen.go will need to have many more methods, and then double-on the 2 switch loops like:
  50. switch k {
  51. case "abc" : x.abc()
  52. case "def" : x.def()
  53. default {
  54. switch {
  55. case !nsAware: panic(...)
  56. case strings.endsWith(":abc"): x.abc()
  57. case strings.endsWith(":def"): x.def()
  58. default: panic(...)
  59. }
  60. }
  61. }
  62. The structure below accommodates this:
  63. type typeInfo struct {
  64. sfi []*structFieldInfo // sorted by encName
  65. sfins // sorted by namespace
  66. sfia // sorted, to have those with attributes at the top. Needed to write XML appropriately.
  67. sfip // unsorted
  68. }
  69. type structFieldInfo struct {
  70. encName
  71. nsEncName
  72. ns string
  73. attr bool
  74. cdata bool
  75. }
  76. indexForEncName is now an internal helper function that takes a sorted array
  77. (one of ti.sfins or ti.sfi). It is only used by *Encoder.getStructFieldInfo(...)
  78. There will be a separate parser from the builder.
  79. The parser will have a method: next() xmlToken method. It has lookahead support,
  80. so you can pop multiple tokens, make a determination, and push them back in the order popped.
  81. This will be needed to determine whether we are "nakedly" decoding a container or not.
  82. The stack will be implemented using a slice and push/pop happens at the [0] element.
  83. xmlToken has fields:
  84. - type uint8: 0 | ElementStart | ElementEnd | AttrKey | AttrVal | Text
  85. - value string
  86. - ns string
  87. SEE: http://www.xml.com/pub/a/98/10/guide0.html?page=3#ENTDECL
  88. The following are skipped when parsing:
  89. - External Entities (from external file)
  90. - Notation Declaration e.g. <!NOTATION GIF87A SYSTEM "GIF">
  91. - Entity Declarations & References
  92. - XML Declaration (assume UTF-8)
  93. - XML Directive i.e. <! ... >
  94. - Other Declarations: Notation, etc.
  95. - Comment
  96. - Processing Instruction
  97. - schema / DTD for validation:
  98. We are not a VALIDATING parser. Validation is done elsewhere.
  99. However, some parts of the DTD internal subset are used (SEE BELOW).
  100. For Attribute List Declarations e.g.
  101. <!ATTLIST foo:oldjoke name ID #REQUIRED label CDATA #IMPLIED status ( funny | notfunny ) 'funny' >
  102. We considered using the ATTLIST to get "default" value, but not to validate the contents. (VETOED)
  103. The following XML features are supported
  104. - Namespace
  105. - Element
  106. - Attribute
  107. - cdata
  108. - Unicode escape
  109. The following DTD (when as an internal sub-set) features are supported:
  110. - Internal Entities e.g.
  111. <!ELEMENT burns "ugorji is cool" > AND entities for the set: [<>&"']
  112. - Parameter entities e.g.
  113. <!ENTITY % personcontent "ugorji is cool"> <!ELEMENT burns (%personcontent;)*>
  114. At decode time, a structure containing the following is kept
  115. - namespace mapping
  116. - default attribute values
  117. - all internal entities (<>&"' and others written in the document)
  118. When decode starts, it parses XML namespace declarations and creates a map in the
  119. xmlDecDriver. While parsing, that map continuously gets updated.
  120. The only problem happens when a namespace declaration happens on the node that it defines.
  121. e.g. <hn:name xmlns:hn="http://www.ugorji.net" >
  122. To handle this, each Element must be fully parsed at a time,
  123. even if it amounts to multiple tokens which are returned one at a time on request.
  124. xmlns is a special attribute name.
  125. - It is used to define namespaces, including the default
  126. - It is never returned as an AttrKey or AttrVal.
  127. *We may decide later to allow user to use it e.g. you want to parse the xmlns mappings into a field.*
  128. Number, bool, null, mapKey, etc can all be decoded from any xmlToken.
  129. This accommodates map[int]string for example.
  130. It should be possible to create a schema from the types,
  131. or vice versa (generate types from schema with appropriate tags).
  132. This is however out-of-scope from this parsing project.
  133. We should write all namespace information at the first point that it is referenced in the tree,
  134. and use the mapping for all child nodes and attributes. This means that state is maintained
  135. at a point in the tree. This also means that calls to Decode or MustDecode will reset some state.
  136. When decoding, it is important to keep track of entity references and default attribute values.
  137. It seems these can only be stored in the DTD components. We should honor them when decoding.
  138. Configuration for XMLHandle will look like this:
  139. XMLHandle
  140. DefaultNS string
  141. // Encoding:
  142. NS map[string]string // ns URI to key, used for encoding
  143. // Decoding: in case ENTITY declared in external schema or dtd, store info needed here
  144. Entities map[string]string // map of entity rep to character
  145. During encode, if a namespace mapping is not defined for a namespace found on a struct,
  146. then we create a mapping for it using nsN (where N is 1..1000000, and doesn't conflict
  147. with any other namespace mapping).
  148. Note that different fields in a struct can have different namespaces.
  149. However, all fields will default to the namespace on the _struct field (if defined).
  150. An XML document is a name, a map of attributes and a list of children.
  151. Consequently, we cannot "DecodeNaked" into a map[string]interface{} (for example).
  152. We have to "DecodeNaked" into something that resembles XML data.
  153. To support DecodeNaked (decode into nil interface{}), we have to define some "supporting" types:
  154. type Name struct { // Preferred. Less allocations due to conversions.
  155. Local string
  156. Space string
  157. }
  158. type Element struct {
  159. Name Name
  160. Attrs map[Name]string
  161. Children []interface{} // each child is either *Element or string
  162. }
  163. Only two "supporting" types are exposed for XML: Name and Element.
  164. // ------------------
  165. We considered 'type Name string' where Name is like "Space Local" (space-separated).
  166. We decided against it, because each creation of a name would lead to
  167. double allocation (first convert []byte to string, then concatenate them into a string).
  168. The benefit is that it is faster to read Attrs from a map. But given that Element is a value
  169. object, we want to eschew methods and have public exposed variables.
  170. We also considered the following, where xml types were not value objects, and we used
  171. intelligent accessor methods to extract information and for performance.
  172. *** WE DECIDED AGAINST THIS. ***
  173. type Attr struct {
  174. Name Name
  175. Value string
  176. }
  177. // Element is a ValueObject: There are no accessor methods.
  178. // Make element self-contained.
  179. type Element struct {
  180. Name Name
  181. attrsMap map[string]string // where key is "Space Local"
  182. attrs []Attr
  183. childrenT []string
  184. childrenE []Element
  185. childrenI []int // each child is a index into T or E.
  186. }
  187. func (x *Element) child(i) interface{} // returns string or *Element
  188. // ------------------
  189. Per XML spec and our default handling, white space is always treated as
  190. insignificant between elements, except in a text node. The xml:space='preserve'
  191. attribute is ignored.
  192. **Note: there is no xml: namespace. The xml: attributes were defined before namespaces.**
  193. **So treat them as just "directives" that should be interpreted to mean something**.
  194. On encoding, we support indenting aka prettifying markup in the same way we support it for json.
  195. A document or element can only be encoded/decoded from/to a struct. In this mode:
  196. - struct name maps to element name (or tag-info from _struct field)
  197. - fields are mapped to child elements or attributes
  198. A map is either encoded as attributes on current element, or as a set of child elements.
  199. Maps are encoded as attributes iff their keys and values are primitives (number, bool, string).
  200. A list is encoded as a set of child elements.
  201. Primitives (number, bool, string) are encoded as an element, attribute or text
  202. depending on the context.
  203. Extensions must encode themselves as a text string.
  204. Encoding is tough, specifically when encoding mappings, because we need to encode
  205. as either attribute or element. To do this, we need to default to encoding as attributes,
  206. and then let Encoder inform the Handle when to start encoding as nodes.
  207. i.e. Encoder does something like:
  208. h.EncodeMapStart()
  209. h.Encode(), h.Encode(), ...
  210. h.EncodeMapNotAttrSignal() // this is not a bool, because it's a signal
  211. h.Encode(), h.Encode(), ...
  212. h.EncodeEnd()
  213. Only XMLHandle understands this, and will set itself to start encoding as elements.
  214. This support extends to maps. For example, if a struct field is a map, and it has
  215. the struct tag signifying it should be attr, then all its fields are encoded as attributes.
  216. e.g.
  217. type X struct {
  218. M map[string]int `codec:"m,attr"` // encode keys as attributes named
  219. }
  220. Question:
  221. - if encoding a map, what if map keys have spaces in them???
  222. Then they cannot be attributes or child elements. Error.
  223. Options to consider adding later:
  224. - For attribute values, normalize by trimming beginning and ending white space,
  225. and converting every white space sequence to a single space.
  226. - ATTLIST restrictions are enforced.
  227. e.g. default value of xml:space, skipping xml:XYZ style attributes, etc.
  228. - Consider supporting NON-STRICT mode (e.g. to handle HTML parsing).
  229. Some elements e.g. br, hr, etc need not close and should be auto-closed
  230. ... (see http://www.w3.org/TR/html4/loose.dtd)
  231. An expansive set of entities are pre-defined.
  232. - Have easy way to create a HTML parser:
  233. add a HTML() method to XMLHandle, that will set Strict=false, specify AutoClose,
  234. and add HTML Entities to the list.
  235. - Support validating element/attribute XMLName before writing it.
  236. Keep this behind a flag, which is set to false by default (for performance).
  237. type XMLHandle struct {
  238. CheckName bool
  239. }
  240. Misc:
  241. ROADMAP (1 weeks):
  242. - build encoder (1 day)
  243. - build decoder (based off xmlParser) (1 day)
  244. - implement xmlParser (2 days).
  245. Look at encoding/xml for inspiration.
  246. - integrate and TEST (1 days)
  247. - write article and post it (1 day)
  248. // ---------- MORE NOTES FROM 2017-11-30 ------------
  249. when parsing
  250. - parse the attributes first
  251. - then parse the nodes
  252. basically:
  253. - if encoding a field: we use the field name for the wrapper
  254. - if encoding a non-field, then just use the element type name
  255. map[string]string ==> <map><key>abc</key><value>val</value></map>... or
  256. <map key="abc">val</map>... OR
  257. <key1>val1</key1><key2>val2</key2>... <- PREFERED
  258. []string ==> <string>v1</string><string>v2</string>...
  259. string v1 ==> <string>v1</string>
  260. bool true ==> <bool>true</bool>
  261. float 1.0 ==> <float>1.0</float>
  262. ...
  263. F1 map[string]string ==> <F1><key>abc</key><value>val</value></F1>... OR
  264. <F1 key="abc">val</F1>... OR
  265. <F1><abc>val</abc>...</F1> <- PREFERED
  266. F2 []string ==> <F2>v1</F2><F2>v2</F2>...
  267. F3 bool ==> <F3>true</F3>
  268. ...
  269. - a scalar is encoded as:
  270. (value) of type T ==> <T><value/></T>
  271. (value) of field F ==> <F><value/></F>
  272. - A kv-pair is encoded as:
  273. (key,value) ==> <map><key><value/></key></map> OR <map key="value">
  274. (key,value) of field F ==> <F><key><value/></key></F> OR <F key="value">
  275. - A map or struct is just a list of kv-pairs
  276. - A list is encoded as sequences of same node e.g.
  277. <F1 key1="value11">
  278. <F1 key2="value12">
  279. <F2>value21</F2>
  280. <F2>value22</F2>
  281. - we may have to singularize the field name, when entering into xml,
  282. and pluralize them when encoding.
  283. - bi-directional encode->decode->encode is not a MUST.
  284. even encoding/xml cannot decode correctly what was encoded:
  285. see https://play.golang.org/p/224V_nyhMS
  286. func main() {
  287. fmt.Println("Hello, playground")
  288. v := []interface{}{"hello", 1, true, nil, time.Now()}
  289. s, err := xml.Marshal(v)
  290. fmt.Printf("err: %v, \ns: %s\n", err, s)
  291. var v2 []interface{}
  292. err = xml.Unmarshal(s, &v2)
  293. fmt.Printf("err: %v, \nv2: %v\n", err, v2)
  294. type T struct {
  295. V []interface{}
  296. }
  297. v3 := T{V: v}
  298. s, err = xml.Marshal(v3)
  299. fmt.Printf("err: %v, \ns: %s\n", err, s)
  300. var v4 T
  301. err = xml.Unmarshal(s, &v4)
  302. fmt.Printf("err: %v, \nv4: %v\n", err, v4)
  303. }
  304. Output:
  305. err: <nil>,
  306. s: <string>hello</string><int>1</int><bool>true</bool><Time>2009-11-10T23:00:00Z</Time>
  307. err: <nil>,
  308. v2: [<nil>]
  309. err: <nil>,
  310. s: <T><V>hello</V><V>1</V><V>true</V><V>2009-11-10T23:00:00Z</V></T>
  311. err: <nil>,
  312. v4: {[<nil> <nil> <nil> <nil>]}
  313. -
  314. */
  315. // ----------- PARSER -------------------
  316. type xmlTokenType uint8
  317. const (
  318. _ xmlTokenType = iota << 1
  319. xmlTokenElemStart
  320. xmlTokenElemEnd
  321. xmlTokenAttrKey
  322. xmlTokenAttrVal
  323. xmlTokenText
  324. )
  325. type xmlToken struct {
  326. Type xmlTokenType
  327. Value string
  328. Namespace string // blank for AttrVal and Text
  329. }
  330. type xmlParser struct {
  331. r decReader
  332. toks []xmlToken // list of tokens.
  333. ptr int // ptr into the toks slice
  334. done bool // nothing else to parse. r now returns EOF.
  335. }
  336. func (x *xmlParser) next() (t *xmlToken) {
  337. // once x.done, or x.ptr == len(x.toks) == 0, then return nil (to signify finish)
  338. if !x.done && len(x.toks) == 0 {
  339. x.nextTag()
  340. }
  341. // parses one element at a time (into possible many tokens)
  342. if x.ptr < len(x.toks) {
  343. t = &(x.toks[x.ptr])
  344. x.ptr++
  345. if x.ptr == len(x.toks) {
  346. x.ptr = 0
  347. x.toks = x.toks[:0]
  348. }
  349. }
  350. return
  351. }
  352. // nextTag will parses the next element and fill up toks.
  353. // It set done flag if/once EOF is reached.
  354. func (x *xmlParser) nextTag() {
  355. // ...
  356. }
  357. // ----------- ENCODER -------------------
  358. type xmlEncDriver struct {
  359. e *Encoder
  360. w encWriter
  361. h *XMLHandle
  362. b [64]byte // scratch
  363. bs []byte // scratch
  364. // s jsonStack
  365. noBuiltInTypes
  366. }
  367. // ----------- DECODER -------------------
  368. type xmlDecDriver struct {
  369. d *Decoder
  370. h *XMLHandle
  371. r decReader // *bytesDecReader decReader
  372. ct valueType // container type. one of unset, array or map.
  373. bstr [8]byte // scratch used for string \UXXX parsing
  374. b [64]byte // scratch
  375. // wsSkipped bool // whitespace skipped
  376. // s jsonStack
  377. noBuiltInTypes
  378. }
  379. // DecodeNaked will decode into an XMLNode
  380. // XMLName is a value object representing a namespace-aware NAME
  381. type XMLName struct {
  382. Local string
  383. Space string
  384. }
  385. // XMLNode represents a "union" of the different types of XML Nodes.
  386. // Only one of fields (Text or *Element) is set.
  387. type XMLNode struct {
  388. Element *Element
  389. Text string
  390. }
  391. // XMLElement is a value object representing an fully-parsed XML element.
  392. type XMLElement struct {
  393. Name Name
  394. Attrs map[XMLName]string
  395. // Children is a list of child nodes, each being a *XMLElement or string
  396. Children []XMLNode
  397. }
  398. // ----------- HANDLE -------------------
  399. type XMLHandle struct {
  400. BasicHandle
  401. textEncodingType
  402. DefaultNS string
  403. NS map[string]string // ns URI to key, for encoding
  404. Entities map[string]string // entity representation to string, for encoding.
  405. }
  406. func (h *XMLHandle) newEncDriver(e *Encoder) encDriver {
  407. return &xmlEncDriver{e: e, w: e.w, h: h}
  408. }
  409. func (h *XMLHandle) newDecDriver(d *Decoder) decDriver {
  410. // d := xmlDecDriver{r: r.(*bytesDecReader), h: h}
  411. hd := xmlDecDriver{d: d, r: d.r, h: h}
  412. hd.n.bytes = d.b[:]
  413. return &hd
  414. }
  415. var _ decDriver = (*xmlDecDriver)(nil)
  416. var _ encDriver = (*xmlEncDriver)(nil)