gen.go 3.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115
  1. // Copyright 2015 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build ignore
  5. // This program generates the trie for width operations. The generated table
  6. // includes width category information as well as the normalization mappings.
  7. package main
  8. import (
  9. "bytes"
  10. "fmt"
  11. "io"
  12. "log"
  13. "math"
  14. "unicode/utf8"
  15. "golang.org/x/text/internal/gen"
  16. "golang.org/x/text/internal/triegen"
  17. )
  18. // See gen_common.go for flags.
  19. func main() {
  20. gen.Init()
  21. genTables()
  22. genTests()
  23. gen.Repackage("gen_trieval.go", "trieval.go", "width")
  24. gen.Repackage("gen_common.go", "common_test.go", "width")
  25. }
  26. func genTables() {
  27. t := triegen.NewTrie("width")
  28. // fold and inverse mappings. See mapComment for a description of the format
  29. // of each entry. Add dummy value to make an index of 0 mean no mapping.
  30. inverse := [][4]byte{{}}
  31. mapping := map[[4]byte]int{[4]byte{}: 0}
  32. getWidthData(func(r rune, tag elem, alt rune) {
  33. idx := 0
  34. if alt != 0 {
  35. var buf [4]byte
  36. buf[0] = byte(utf8.EncodeRune(buf[1:], alt))
  37. s := string(r)
  38. buf[buf[0]] ^= s[len(s)-1]
  39. var ok bool
  40. if idx, ok = mapping[buf]; !ok {
  41. idx = len(mapping)
  42. if idx > math.MaxUint8 {
  43. log.Fatalf("Index %d does not fit in a byte.", idx)
  44. }
  45. mapping[buf] = idx
  46. inverse = append(inverse, buf)
  47. }
  48. }
  49. t.Insert(r, uint64(tag|elem(idx)))
  50. })
  51. w := &bytes.Buffer{}
  52. gen.WriteUnicodeVersion(w)
  53. sz, err := t.Gen(w)
  54. if err != nil {
  55. log.Fatal(err)
  56. }
  57. sz += writeMappings(w, inverse)
  58. fmt.Fprintf(w, "// Total table size %d bytes (%dKiB)\n", sz, sz/1024)
  59. gen.WriteVersionedGoFile(*outputFile, "width", w.Bytes())
  60. }
  61. const inverseDataComment = `
  62. // inverseData contains 4-byte entries of the following format:
  63. // <length> <modified UTF-8-encoded rune> <0 padding>
  64. // The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the
  65. // UTF-8 encoding of the original rune. Mappings often have the following
  66. // pattern:
  67. // A -> A (U+FF21 -> U+0041)
  68. // B -> B (U+FF22 -> U+0042)
  69. // ...
  70. // By xor-ing the last byte the same entry can be shared by many mappings. This
  71. // reduces the total number of distinct entries by about two thirds.
  72. // The resulting entry for the aforementioned mappings is
  73. // { 0x01, 0xE0, 0x00, 0x00 }
  74. // Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get
  75. // E0 ^ A1 = 41.
  76. // Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get
  77. // E0 ^ A2 = 42.
  78. // Note that because of the xor-ing, the byte sequence stored in the entry is
  79. // not valid UTF-8.`
  80. func writeMappings(w io.Writer, data [][4]byte) int {
  81. fmt.Fprintln(w, inverseDataComment)
  82. fmt.Fprintf(w, "var inverseData = [%d][4]byte{\n", len(data))
  83. for _, x := range data {
  84. fmt.Fprintf(w, "{ 0x%02x, 0x%02x, 0x%02x, 0x%02x },\n", x[0], x[1], x[2], x[3])
  85. }
  86. fmt.Fprintln(w, "}")
  87. return len(data) * 4
  88. }
  89. func genTests() {
  90. w := &bytes.Buffer{}
  91. fmt.Fprintf(w, "\nvar mapRunes = map[rune]struct{r rune; e elem}{\n")
  92. getWidthData(func(r rune, tag elem, alt rune) {
  93. if alt != 0 {
  94. fmt.Fprintf(w, "\t0x%X: {0x%X, 0x%X},\n", r, alt, tag)
  95. }
  96. })
  97. fmt.Fprintln(w, "}")
  98. gen.WriteGoFile("runes_test.go", "width", w.Bytes())
  99. }