icu_test.go 6.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210
  1. // Copyright 2016 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build icu
  5. package cases
  6. import (
  7. "path"
  8. "strings"
  9. "testing"
  10. "golang.org/x/text/internal/testtext"
  11. "golang.org/x/text/language"
  12. "golang.org/x/text/unicode/norm"
  13. )
  14. func TestICUConformance(t *testing.T) {
  15. // Build test set.
  16. input := []string{
  17. "a.a a_a",
  18. "a\u05d0a",
  19. "\u05d0'a",
  20. "a\u03084a",
  21. "a\u0308a",
  22. "a3\u30a3a",
  23. "a\u303aa",
  24. "a_\u303a_a",
  25. "1_a..a",
  26. "1_a.a",
  27. "a..a.",
  28. "a--a-",
  29. "a-a-",
  30. "a\u200ba",
  31. "a\u200b\u200ba",
  32. "a\u00ad\u00ada", // Format
  33. "a\u00ada",
  34. "a''a", // SingleQuote
  35. "a'a",
  36. "a::a", // MidLetter
  37. "a:a",
  38. "a..a", // MidNumLet
  39. "a.a",
  40. "a;;a", // MidNum
  41. "a;a",
  42. "a__a", // ExtendNumlet
  43. "a_a",
  44. "ΟΣ''a",
  45. }
  46. add := func(x interface{}) {
  47. switch v := x.(type) {
  48. case string:
  49. input = append(input, v)
  50. case []string:
  51. for _, s := range v {
  52. input = append(input, s)
  53. }
  54. }
  55. }
  56. for _, tc := range testCases {
  57. add(tc.src)
  58. add(tc.lower)
  59. add(tc.upper)
  60. add(tc.title)
  61. }
  62. for _, tc := range bufferTests {
  63. add(tc.src)
  64. }
  65. for _, tc := range breakTest {
  66. add(strings.Replace(tc, "|", "", -1))
  67. }
  68. for _, tc := range foldTestCases {
  69. add(tc)
  70. }
  71. // Compare ICU to Go.
  72. for _, c := range []string{"lower", "upper", "title", "fold"} {
  73. for _, tag := range []string{
  74. "und", "af", "az", "el", "lt", "nl", "tr",
  75. } {
  76. for _, s := range input {
  77. if exclude(c, tag, s) {
  78. continue
  79. }
  80. testtext.Run(t, path.Join(c, tag, s), func(t *testing.T) {
  81. want := doICU(tag, c, s)
  82. got := doGo(tag, c, s)
  83. if norm.NFC.String(got) != norm.NFC.String(want) {
  84. t.Errorf("\n in %[3]q (%+[3]q)\n got %[1]q (%+[1]q)\n want %[2]q (%+[2]q)", got, want, s)
  85. }
  86. })
  87. }
  88. }
  89. }
  90. }
  91. // exclude indicates if a string should be excluded from testing.
  92. func exclude(cm, tag, s string) bool {
  93. list := []struct{ cm, tags, pattern string }{
  94. // TODO: Go does not handle certain esoteric breaks correctly. This will be
  95. // fixed once we have a real word break iterator. Alternatively, it
  96. // seems like we're not too far off from making it work, so we could
  97. // fix these last steps. But first verify that using a separate word
  98. // breaker does not hurt performance.
  99. {"title", "af nl", "a''a"},
  100. {"", "", "א'a"},
  101. // All the exclusions below seem to be issues with the ICU
  102. // implementation (at version 57) and thus are not marked as TODO.
  103. // ICU does not handle leading apostrophe for Dutch and
  104. // Afrikaans correctly. See https://unicode.org/cldr/trac/ticket/7078.
  105. {"title", "af nl", "'n"},
  106. {"title", "af nl", "'N"},
  107. // Go terminates the final sigma check after a fixed number of
  108. // ignorables have been found. This ensures that the algorithm can make
  109. // progress in a streaming scenario.
  110. {"lower title", "", "\u039f\u03a3...............................a"},
  111. // This also applies to upper in Greek.
  112. // NOTE: we could fix the following two cases by adding state to elUpper
  113. // and aztrLower. However, considering a modifier to not belong to the
  114. // preceding letter after the maximum modifiers count is reached is
  115. // consistent with the behavior of unicode/norm.
  116. {"upper", "el", "\u03bf" + strings.Repeat("\u0321", 29) + "\u0313"},
  117. {"lower", "az tr lt", "I" + strings.Repeat("\u0321", 30) + "\u0307\u0300"},
  118. {"upper", "lt", "i" + strings.Repeat("\u0321", 30) + "\u0307\u0300"},
  119. {"lower", "lt", "I" + strings.Repeat("\u0321", 30) + "\u0300"},
  120. // ICU title case seems to erroneously removes \u0307 from an upper case
  121. // I unconditionally, instead of only when lowercasing. The ICU
  122. // transform algorithm transforms these cases consistently with our
  123. // implementation.
  124. {"title", "az tr", "\u0307"},
  125. // The spec says to remove \u0307 after Soft-Dotted characters. ICU
  126. // transforms conform but ucasemap_utf8ToUpper does not.
  127. {"upper title", "lt", "i\u0307"},
  128. {"upper title", "lt", "i" + strings.Repeat("\u0321", 29) + "\u0307\u0300"},
  129. // Both Unicode and CLDR prescribe an extra explicit dot above after a
  130. // Soft_Dotted character if there are other modifiers.
  131. // ucasemap_utf8ToUpper does not do this; ICU transforms do.
  132. // The issue with ucasemap_utf8ToUpper seems to be that it does not
  133. // consider the modifiers that are part of composition in the evaluation
  134. // of More_Above. For instance, according to the More_Above rule for lt,
  135. // a dotted capital I (U+0130) becomes i\u0307\u0307 (an small i with
  136. // two additional dots). This seems odd, but is correct. ICU is
  137. // definitely not correct as it produces different results for different
  138. // normal forms. For instance, for an İ:
  139. // \u0130 (NFC) -> i\u0307 (incorrect)
  140. // I\u0307 (NFD) -> i\u0307\u0307 (correct)
  141. // We could argue that we should not add a \u0307 if there already is
  142. // one, but this may be hard to get correct and is not conform the
  143. // standard.
  144. {"lower title", "lt", "\u0130"},
  145. {"lower title", "lt", "\u00cf"},
  146. // We are conform ICU ucasemap_utf8ToUpper if we remove support for
  147. // elUpper. However, this is clearly not conform the spec. Moreover, the
  148. // ICU transforms _do_ implement this transform and produces results
  149. // consistent with our implementation. Note that we still prefer to use
  150. // ucasemap_utf8ToUpper instead of transforms as the latter have
  151. // inconsistencies in the word breaking algorithm.
  152. {"upper", "el", "\u0386"}, // GREEK CAPITAL LETTER ALPHA WITH TONOS
  153. {"upper", "el", "\u0389"}, // GREEK CAPITAL LETTER ETA WITH TONOS
  154. {"upper", "el", "\u038A"}, // GREEK CAPITAL LETTER IOTA WITH TONOS
  155. {"upper", "el", "\u0391"}, // GREEK CAPITAL LETTER ALPHA
  156. {"upper", "el", "\u0397"}, // GREEK CAPITAL LETTER ETA
  157. {"upper", "el", "\u0399"}, // GREEK CAPITAL LETTER IOTA
  158. {"upper", "el", "\u03AC"}, // GREEK SMALL LETTER ALPHA WITH TONOS
  159. {"upper", "el", "\u03AE"}, // GREEK SMALL LETTER ALPHA WITH ETA
  160. {"upper", "el", "\u03AF"}, // GREEK SMALL LETTER ALPHA WITH IOTA
  161. {"upper", "el", "\u03B1"}, // GREEK SMALL LETTER ALPHA
  162. {"upper", "el", "\u03B7"}, // GREEK SMALL LETTER ETA
  163. {"upper", "el", "\u03B9"}, // GREEK SMALL LETTER IOTA
  164. }
  165. for _, x := range list {
  166. if x.cm != "" && strings.Index(x.cm, cm) == -1 {
  167. continue
  168. }
  169. if x.tags != "" && strings.Index(x.tags, tag) == -1 {
  170. continue
  171. }
  172. if strings.Index(s, x.pattern) != -1 {
  173. return true
  174. }
  175. }
  176. return false
  177. }
  178. func doGo(tag, caser, input string) string {
  179. var c Caser
  180. t := language.MustParse(tag)
  181. switch caser {
  182. case "lower":
  183. c = Lower(t)
  184. case "upper":
  185. c = Upper(t)
  186. case "title":
  187. c = Title(t)
  188. case "fold":
  189. c = Fold()
  190. }
  191. return c.String(input)
  192. }