go.net/html/charset: new package

Implement retrieving encodings by name, according to the names listed
at http://encoding.spec.whatwg.org/#encodings

This is the first step toward implementing the encoding detection
algorithm.

R=nigeltao
CC=golang-dev
https://golang.org/cl/27110043

html/charset/encoding.go

@@ -0,0 +1,51 @@
+// Package charset provides common text encodings for HTML documents.
+//
+// The mapping from encoding labels to encodings is defined at
+// http://encoding.spec.whatwg.org.
+package charset
+
+import (
+	"errors"
+	"strings"
+
+	"code.google.com/p/go.text/encoding"
+	"code.google.com/p/go.text/transform"
+)
+
+// Encoding returns the encoding with the specified label, or nil if label is
+// not one of the standard encodings for HTML. Matching is case-insensitive
+// and ignores leading and trailing whitespace.
+func Encoding(label string) encoding.Encoding {
+	label = strings.ToLower(strings.Trim(label, "\t\n\r\f "))
+	return encodings[label]
+}
+
+type utf8Encoding struct{}
+
+func (utf8Encoding) NewDecoder() transform.Transformer {
+	return transform.Nop
+}
+
+func (utf8Encoding) NewEncoder() transform.Transformer {
+	return transform.Nop
+}
+
+var ErrReplacementEncoding = errors.New("charset: ISO-2022-CN and ISO-2022-KR are obsolete encodings for HTML")
+
+type errorTransformer struct {
+	err error
+}
+
+func (t errorTransformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	return 0, 0, t.err
+}
+
+type replacementEncoding struct{}
+
+func (replacementEncoding) NewDecoder() transform.Transformer {
+	return errorTransformer{ErrReplacementEncoding}
+}
+
+func (replacementEncoding) NewEncoder() transform.Transformer {
+	return transform.Nop
+}

html/charset/encoding_test.go

@@ -0,0 +1,103 @@
+package charset
+
+import (
+	"io/ioutil"
+	"strings"
+	"testing"
+
+	"code.google.com/p/go.text/transform"
+)
+
+func transformString(t transform.Transformer, s string) (string, error) {
+	r := transform.NewReader(strings.NewReader(s), t)
+	b, err := ioutil.ReadAll(r)
+	return string(b), err
+}
+
+var testCases = []struct {
+	utf8, other, otherEncoding string
+}{
+	{"Résumé", "Résumé", "utf8"},
+	{"Résumé", "R\xe9sum\xe9", "latin1"},
+	{"これは漢字です。", "S0\x8c0o0\"oW[g0Y0\x020", "UTF-16LE"},
+	{"これは漢字です。", "0S0\x8c0oo\"[W0g0Y0\x02", "UTF-16BE"},
+	{"Hello, world", "Hello, world", "ASCII"},
+	{"Gdańsk", "Gda\xf1sk", "ISO-8859-2"},
+	{"Ââ Čč Đđ Ŋŋ Õõ Šš Žž Åå Ää", "\xc2\xe2 \xc8\xe8 \xa9\xb9 \xaf\xbf \xd5\xf5 \xaa\xba \xac\xbc \xc5\xe5 \xc4\xe4", "ISO-8859-10"},
+	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "ISO-8859-11"},
+	{"latviešu", "latvie\xf0u", "ISO-8859-13"},
+	{"Seònaid", "Se\xf2naid", "ISO-8859-14"},
+	{"€1 is cheap", "\xa41 is cheap", "ISO-8859-15"},
+	{"românește", "rom\xe2ne\xbate", "ISO-8859-16"},
+	{"nutraĵo", "nutra\xbco", "ISO-8859-3"},
+	{"Kalâdlit", "Kal\xe2dlit", "ISO-8859-4"},
+	{"русский", "\xe0\xe3\xe1\xe1\xda\xd8\xd9", "ISO-8859-5"},
+	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "ISO-8859-7"},
+	{"Kağan", "Ka\xf0an", "ISO-8859-9"},
+	{"Résumé", "R\x8esum\x8e", "macintosh"},
+	{"Gdańsk", "Gda\xf1sk", "windows-1250"},
+	{"русский", "\xf0\xf3\xf1\xf1\xea\xe8\xe9", "windows-1251"},
+	{"Résumé", "R\xe9sum\xe9", "windows-1252"},
+	{"ελληνικά", "\xe5\xeb\xeb\xe7\xed\xe9\xea\xdc", "windows-1253"},
+	{"Kağan", "Ka\xf0an", "windows-1254"},
+	{"עִבְרִית", "\xf2\xc4\xe1\xc0\xf8\xc4\xe9\xfa", "windows-1255"},
+	{"العربية", "\xc7\xe1\xda\xd1\xc8\xed\xc9", "windows-1256"},
+	{"latviešu", "latvie\xf0u", "windows-1257"},
+	{"Việt", "Vi\xea\xf2t", "windows-1258"},
+	{"สำหรับ", "\xca\xd3\xcb\xc3\u047a", "windows-874"},
+	{"русский", "\xd2\xd5\xd3\xd3\xcb\xc9\xca", "KOI8-R"},
+	{"українська", "\xd5\xcb\xd2\xc1\xa7\xce\xd3\xd8\xcb\xc1", "KOI8-U"},
+	{"Hello 常用國字標準字體表", "Hello \xb1`\xa5\u03b0\xea\xa6r\xbc\u0437\u01e6r\xc5\xe9\xaa\xed", "big5"},
+	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gbk"},
+	{"Hello 常用國字標準字體表", "Hello \xb3\xa3\xd3\xc3\x87\xf8\xd7\xd6\x98\xcb\x9c\xca\xd7\xd6\xf3\x77\xb1\xed", "gb18030"},
+	{"עִבְרִית", "\x81\x30\xfb\x30\x81\x30\xf6\x34\x81\x30\xf9\x33\x81\x30\xf6\x30\x81\x30\xfb\x36\x81\x30\xf6\x34\x81\x30\xfa\x31\x81\x30\xfb\x38", "gb18030"},
+	{"㧯", "\x82\x31\x89\x38", "gb18030"},
+	{"これは漢字です。", "\x82\xb1\x82\xea\x82\xcd\x8a\xbf\x8e\x9a\x82\xc5\x82\xb7\x81B", "SJIS"},
+	{"Hello, 世界!", "Hello, \x90\xa2\x8aE!", "SJIS"},
+	{"ｲｳｴｵｶ", "\xb2\xb3\xb4\xb5\xb6", "SJIS"},
+	{"これは漢字です。", "\xa4\xb3\xa4\xec\xa4\u03f4\xc1\xbb\xfa\xa4\u01e4\xb9\xa1\xa3", "EUC-JP"},
+	{"Hello, 世界!", "Hello, \x1b$B@$3&\x1b(B!", "ISO-2022-JP"},
+	{"네이트 | 즐거움의 시작, 슈파스(Spaβ) NATE", "\xb3\xd7\xc0\xcc\xc6\xae | \xc1\xf1\xb0\xc5\xbf\xf2\xc0\xc7 \xbd\xc3\xc0\xdb, \xbd\xb4\xc6\xc4\xbd\xba(Spa\xa5\xe2) NATE", "EUC-KR"},
+}
+
+func TestDecode(t *testing.T) {
+	for _, tc := range testCases {
+		e := Encoding(tc.otherEncoding)
+		if e == nil {
+			t.Errorf("%s: not found", tc.otherEncoding)
+			continue
+		}
+
+		s, err := transformString(e.NewDecoder(), tc.other)
+
+		if err != nil {
+			t.Errorf("%s: decode %q: %v", tc.otherEncoding, tc.other, err)
+			continue
+		}
+
+		if s != tc.utf8 {
+			t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.utf8)
+		}
+	}
+}
+
+func TestEncode(t *testing.T) {
+	for _, tc := range testCases {
+		e := Encoding(tc.otherEncoding)
+		if e == nil {
+			t.Errorf("%s: not found", tc.otherEncoding)
+			continue
+		}
+
+		s, err := transformString(e.NewEncoder(), tc.utf8)
+
+		if err != nil {
+			t.Errorf("%s: encode %q: %s", tc.otherEncoding, tc.utf8, err)
+			continue
+		}
+
+		if s != tc.other {
+			t.Errorf("%s: got %q, want %q", tc.otherEncoding, s, tc.other)
+		}
+	}
+}

html/charset/gen.go

@@ -0,0 +1,107 @@
+// +build ignore
+
+package main
+
+// Download http://encoding.spec.whatwg.org/encodings.json and use it to
+// generate table.go.
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"strings"
+)
+
+type enc struct {
+	Name   string
+	Labels []string
+}
+
+type group struct {
+	Encodings []enc
+	Heading   string
+}
+
+const specURL = "http://encoding.spec.whatwg.org/encodings.json"
+
+func main() {
+	resp, err := http.Get(specURL)
+	if err != nil {
+		log.Fatalf("error fetching %s: %s", specURL, err)
+	}
+	if resp.StatusCode != 200 {
+		log.Fatalf("error fetching %s: HTTP status %s", specURL, resp.Status)
+	}
+	defer resp.Body.Close()
+
+	var groups []group
+	d := json.NewDecoder(resp.Body)
+	err = d.Decode(&groups)
+	if err != nil {
+		log.Fatalf("error reading encodings.json: %s", err)
+	}
+
+	fmt.Println("// generated by go run gen.go; DO NOT EDIT")
+	fmt.Println()
+	fmt.Println("package charset")
+	fmt.Println()
+
+	fmt.Println("import (")
+	fmt.Println(`"code.google.com/p/go.text/encoding"`)
+	for _, pkg := range []string{"charmap", "japanese", "korean", "simplifiedchinese", "traditionalchinese", "unicode"} {
+		fmt.Printf("\"code.google.com/p/go.text/encoding/%s\"\n", pkg)
+	}
+	fmt.Println(")")
+	fmt.Println()
+
+	fmt.Println("var encodings = map[string]encoding.Encoding{")
+	for _, g := range groups {
+		for _, e := range g.Encodings {
+			goName, ok := miscNames[e.Name]
+			if !ok {
+				for k, v := range prefixes {
+					if strings.HasPrefix(e.Name, k) {
+						goName = v + e.Name[len(k):]
+						break
+					}
+				}
+				if goName == "" {
+					log.Fatalf("unrecognized encoding name: %s", e.Name)
+				}
+			}
+
+			for _, label := range e.Labels {
+				fmt.Printf("%q: %s,\n", label, goName)
+			}
+		}
+	}
+	fmt.Println("}")
+}
+
+var prefixes = map[string]string{
+	"iso-8859-": "charmap.ISO8859_",
+	"windows-":  "charmap.Windows",
+}
+
+var miscNames = map[string]string{
+	"utf-8":          "utf8Encoding{}",
+	"ibm866":         "charmap.CodePage866",
+	"iso-8859-8-i":   "charmap.ISO8859_8",
+	"koi8-r":         "charmap.KOI8R",
+	"koi8-u":         "charmap.KOI8U",
+	"macintosh":      "charmap.Macintosh",
+	"x-mac-cyrillic": "charmap.MacintoshCyrillic",
+	"gbk":            "simplifiedchinese.GBK",
+	"gb18030":        "simplifiedchinese.GB18030",
+	"hz-gb-2312":     "simplifiedchinese.HZGB2312",
+	"big5":           "traditionalchinese.Big5",
+	"euc-jp":         "japanese.EUCJP",
+	"iso-2022-jp":    "japanese.ISO2022JP",
+	"shift_jis":      "japanese.ShiftJIS",
+	"euc-kr":         "korean.EUCKR",
+	"replacement":    "replacementEncoding{}",
+	"utf-16be":       "unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM)",
+	"utf-16le":       "unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM)",
+	"x-user-defined": "charmap.XUserDefined",
+}

html/charset/table.go

@@ -0,0 +1,232 @@
+// generated by go run gen.go; DO NOT EDIT
+
+package charset
+
+import (
+	"code.google.com/p/go.text/encoding"
+	"code.google.com/p/go.text/encoding/charmap"
+	"code.google.com/p/go.text/encoding/japanese"
+	"code.google.com/p/go.text/encoding/korean"
+	"code.google.com/p/go.text/encoding/simplifiedchinese"
+	"code.google.com/p/go.text/encoding/traditionalchinese"
+	"code.google.com/p/go.text/encoding/unicode"
+)
+
+var encodings = map[string]encoding.Encoding{
+	"unicode-1-1-utf-8":   utf8Encoding{},
+	"utf-8":               utf8Encoding{},
+	"utf8":                utf8Encoding{},
+	"866":                 charmap.CodePage866,
+	"cp866":               charmap.CodePage866,
+	"csibm866":            charmap.CodePage866,
+	"ibm866":              charmap.CodePage866,
+	"csisolatin2":         charmap.ISO8859_2,
+	"iso-8859-2":          charmap.ISO8859_2,
+	"iso-ir-101":          charmap.ISO8859_2,
+	"iso8859-2":           charmap.ISO8859_2,
+	"iso88592":            charmap.ISO8859_2,
+	"iso_8859-2":          charmap.ISO8859_2,
+	"iso_8859-2:1987":     charmap.ISO8859_2,
+	"l2":                  charmap.ISO8859_2,
+	"latin2":              charmap.ISO8859_2,
+	"csisolatin3":         charmap.ISO8859_3,
+	"iso-8859-3":          charmap.ISO8859_3,
+	"iso-ir-109":          charmap.ISO8859_3,
+	"iso8859-3":           charmap.ISO8859_3,
+	"iso88593":            charmap.ISO8859_3,
+	"iso_8859-3":          charmap.ISO8859_3,
+	"iso_8859-3:1988":     charmap.ISO8859_3,
+	"l3":                  charmap.ISO8859_3,
+	"latin3":              charmap.ISO8859_3,
+	"csisolatin4":         charmap.ISO8859_4,
+	"iso-8859-4":          charmap.ISO8859_4,
+	"iso-ir-110":          charmap.ISO8859_4,
+	"iso8859-4":           charmap.ISO8859_4,
+	"iso88594":            charmap.ISO8859_4,
+	"iso_8859-4":          charmap.ISO8859_4,
+	"iso_8859-4:1988":     charmap.ISO8859_4,
+	"l4":                  charmap.ISO8859_4,
+	"latin4":              charmap.ISO8859_4,
+	"csisolatincyrillic":  charmap.ISO8859_5,
+	"cyrillic":            charmap.ISO8859_5,
+	"iso-8859-5":          charmap.ISO8859_5,
+	"iso-ir-144":          charmap.ISO8859_5,
+	"iso8859-5":           charmap.ISO8859_5,
+	"iso88595":            charmap.ISO8859_5,
+	"iso_8859-5":          charmap.ISO8859_5,
+	"iso_8859-5:1988":     charmap.ISO8859_5,
+	"arabic":              charmap.ISO8859_6,
+	"asmo-708":            charmap.ISO8859_6,
+	"csiso88596e":         charmap.ISO8859_6,
+	"csiso88596i":         charmap.ISO8859_6,
+	"csisolatinarabic":    charmap.ISO8859_6,
+	"ecma-114":            charmap.ISO8859_6,
+	"iso-8859-6":          charmap.ISO8859_6,
+	"iso-8859-6-e":        charmap.ISO8859_6,
+	"iso-8859-6-i":        charmap.ISO8859_6,
+	"iso-ir-127":          charmap.ISO8859_6,
+	"iso8859-6":           charmap.ISO8859_6,
+	"iso88596":            charmap.ISO8859_6,
+	"iso_8859-6":          charmap.ISO8859_6,
+	"iso_8859-6:1987":     charmap.ISO8859_6,
+	"csisolatingreek":     charmap.ISO8859_7,
+	"ecma-118":            charmap.ISO8859_7,
+	"elot_928":            charmap.ISO8859_7,
+	"greek":               charmap.ISO8859_7,
+	"greek8":              charmap.ISO8859_7,
+	"iso-8859-7":          charmap.ISO8859_7,
+	"iso-ir-126":          charmap.ISO8859_7,
+	"iso8859-7":           charmap.ISO8859_7,
+	"iso88597":            charmap.ISO8859_7,
+	"iso_8859-7":          charmap.ISO8859_7,
+	"iso_8859-7:1987":     charmap.ISO8859_7,
+	"sun_eu_greek":        charmap.ISO8859_7,
+	"csiso88598e":         charmap.ISO8859_8,
+	"csisolatinhebrew":    charmap.ISO8859_8,
+	"hebrew":              charmap.ISO8859_8,
+	"iso-8859-8":          charmap.ISO8859_8,
+	"iso-8859-8-e":        charmap.ISO8859_8,
+	"iso-ir-138":          charmap.ISO8859_8,
+	"iso8859-8":           charmap.ISO8859_8,
+	"iso88598":            charmap.ISO8859_8,
+	"iso_8859-8":          charmap.ISO8859_8,
+	"iso_8859-8:1988":     charmap.ISO8859_8,
+	"visual":              charmap.ISO8859_8,
+	"csiso88598i":         charmap.ISO8859_8,
+	"iso-8859-8-i":        charmap.ISO8859_8,
+	"logical":             charmap.ISO8859_8,
+	"csisolatin6":         charmap.ISO8859_10,
+	"iso-8859-10":         charmap.ISO8859_10,
+	"iso-ir-157":          charmap.ISO8859_10,
+	"iso8859-10":          charmap.ISO8859_10,
+	"iso885910":           charmap.ISO8859_10,
+	"l6":                  charmap.ISO8859_10,
+	"latin6":              charmap.ISO8859_10,
+	"iso-8859-13":         charmap.ISO8859_13,
+	"iso8859-13":          charmap.ISO8859_13,
+	"iso885913":           charmap.ISO8859_13,
+	"iso-8859-14":         charmap.ISO8859_14,
+	"iso8859-14":          charmap.ISO8859_14,
+	"iso885914":           charmap.ISO8859_14,
+	"csisolatin9":         charmap.ISO8859_15,
+	"iso-8859-15":         charmap.ISO8859_15,
+	"iso8859-15":          charmap.ISO8859_15,
+	"iso885915":           charmap.ISO8859_15,
+	"iso_8859-15":         charmap.ISO8859_15,
+	"l9":                  charmap.ISO8859_15,
+	"iso-8859-16":         charmap.ISO8859_16,
+	"cskoi8r":             charmap.KOI8R,
+	"koi":                 charmap.KOI8R,
+	"koi8":                charmap.KOI8R,
+	"koi8-r":              charmap.KOI8R,
+	"koi8_r":              charmap.KOI8R,
+	"koi8-u":              charmap.KOI8U,
+	"csmacintosh":         charmap.Macintosh,
+	"mac":                 charmap.Macintosh,
+	"macintosh":           charmap.Macintosh,
+	"x-mac-roman":         charmap.Macintosh,
+	"dos-874":             charmap.Windows874,
+	"iso-8859-11":         charmap.Windows874,
+	"iso8859-11":          charmap.Windows874,
+	"iso885911":           charmap.Windows874,
+	"tis-620":             charmap.Windows874,
+	"windows-874":         charmap.Windows874,
+	"cp1250":              charmap.Windows1250,
+	"windows-1250":        charmap.Windows1250,
+	"x-cp1250":            charmap.Windows1250,
+	"cp1251":              charmap.Windows1251,
+	"windows-1251":        charmap.Windows1251,
+	"x-cp1251":            charmap.Windows1251,
+	"ansi_x3.4-1968":      charmap.Windows1252,
+	"ascii":               charmap.Windows1252,
+	"cp1252":              charmap.Windows1252,
+	"cp819":               charmap.Windows1252,
+	"csisolatin1":         charmap.Windows1252,
+	"ibm819":              charmap.Windows1252,
+	"iso-8859-1":          charmap.Windows1252,
+	"iso-ir-100":          charmap.Windows1252,
+	"iso8859-1":           charmap.Windows1252,
+	"iso88591":            charmap.Windows1252,
+	"iso_8859-1":          charmap.Windows1252,
+	"iso_8859-1:1987":     charmap.Windows1252,
+	"l1":                  charmap.Windows1252,
+	"latin1":              charmap.Windows1252,
+	"us-ascii":            charmap.Windows1252,
+	"windows-1252":        charmap.Windows1252,
+	"x-cp1252":            charmap.Windows1252,
+	"cp1253":              charmap.Windows1253,
+	"windows-1253":        charmap.Windows1253,
+	"x-cp1253":            charmap.Windows1253,
+	"cp1254":              charmap.Windows1254,
+	"csisolatin5":         charmap.Windows1254,
+	"iso-8859-9":          charmap.Windows1254,
+	"iso-ir-148":          charmap.Windows1254,
+	"iso8859-9":           charmap.Windows1254,
+	"iso88599":            charmap.Windows1254,
+	"iso_8859-9":          charmap.Windows1254,
+	"iso_8859-9:1989":     charmap.Windows1254,
+	"l5":                  charmap.Windows1254,
+	"latin5":              charmap.Windows1254,
+	"windows-1254":        charmap.Windows1254,
+	"x-cp1254":            charmap.Windows1254,
+	"cp1255":              charmap.Windows1255,
+	"windows-1255":        charmap.Windows1255,
+	"x-cp1255":            charmap.Windows1255,
+	"cp1256":              charmap.Windows1256,
+	"windows-1256":        charmap.Windows1256,
+	"x-cp1256":            charmap.Windows1256,
+	"cp1257":              charmap.Windows1257,
+	"windows-1257":        charmap.Windows1257,
+	"x-cp1257":            charmap.Windows1257,
+	"cp1258":              charmap.Windows1258,
+	"windows-1258":        charmap.Windows1258,
+	"x-cp1258":            charmap.Windows1258,
+	"x-mac-cyrillic":      charmap.MacintoshCyrillic,
+	"x-mac-ukrainian":     charmap.MacintoshCyrillic,
+	"chinese":             simplifiedchinese.GBK,
+	"csgb2312":            simplifiedchinese.GBK,
+	"csiso58gb231280":     simplifiedchinese.GBK,
+	"gb2312":              simplifiedchinese.GBK,
+	"gb_2312":             simplifiedchinese.GBK,
+	"gb_2312-80":          simplifiedchinese.GBK,
+	"gbk":                 simplifiedchinese.GBK,
+	"iso-ir-58":           simplifiedchinese.GBK,
+	"x-gbk":               simplifiedchinese.GBK,
+	"gb18030":             simplifiedchinese.GB18030,
+	"hz-gb-2312":          simplifiedchinese.HZGB2312,
+	"big5":                traditionalchinese.Big5,
+	"big5-hkscs":          traditionalchinese.Big5,
+	"cn-big5":             traditionalchinese.Big5,
+	"csbig5":              traditionalchinese.Big5,
+	"x-x-big5":            traditionalchinese.Big5,
+	"cseucpkdfmtjapanese": japanese.EUCJP,
+	"euc-jp":              japanese.EUCJP,
+	"x-euc-jp":            japanese.EUCJP,
+	"csiso2022jp":         japanese.ISO2022JP,
+	"iso-2022-jp":         japanese.ISO2022JP,
+	"csshiftjis":          japanese.ShiftJIS,
+	"ms_kanji":            japanese.ShiftJIS,
+	"shift-jis":           japanese.ShiftJIS,
+	"shift_jis":           japanese.ShiftJIS,
+	"sjis":                japanese.ShiftJIS,
+	"windows-31j":         japanese.ShiftJIS,
+	"x-sjis":              japanese.ShiftJIS,
+	"cseuckr":             korean.EUCKR,
+	"csksc56011987":       korean.EUCKR,
+	"euc-kr":              korean.EUCKR,
+	"iso-ir-149":          korean.EUCKR,
+	"korean":              korean.EUCKR,
+	"ks_c_5601-1987":      korean.EUCKR,
+	"ks_c_5601-1989":      korean.EUCKR,
+	"ksc5601":             korean.EUCKR,
+	"ksc_5601":            korean.EUCKR,
+	"windows-949":         korean.EUCKR,
+	"csiso2022kr":         replacementEncoding{},
+	"iso-2022-kr":         replacementEncoding{},
+	"iso-2022-cn":         replacementEncoding{},
+	"iso-2022-cn-ext":     replacementEncoding{},
+	"utf-16be":            unicode.UTF16(unicode.BigEndian, unicode.IgnoreBOM),
+	"utf-16":              unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM),
+	"utf-16le":            unicode.UTF16(unicode.LittleEndian, unicode.IgnoreBOM),
+	"x-user-defined":      charmap.XUserDefined,
+}