icu.go 4.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209
  1. // Copyright 2012 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. // +build icu
  5. package main
  6. /*
  7. #cgo LDFLAGS: -licui18n -licuuc
  8. #include <stdlib.h>
  9. #include <unicode/ucol.h>
  10. #include <unicode/uiter.h>
  11. #include <unicode/utypes.h>
  12. */
  13. import "C"
  14. import (
  15. "fmt"
  16. "log"
  17. "unicode/utf16"
  18. "unicode/utf8"
  19. "unsafe"
  20. )
  21. func init() {
  22. AddFactory(CollatorFactory{"icu", newUTF16,
  23. "Main ICU collator, using native strings."})
  24. AddFactory(CollatorFactory{"icu8", newUTF8iter,
  25. "ICU collator using ICU iterators to process UTF8."})
  26. AddFactory(CollatorFactory{"icu16", newUTF8conv,
  27. "ICU collation by first converting UTF8 to UTF16."})
  28. }
  29. func icuCharP(s []byte) *C.char {
  30. return (*C.char)(unsafe.Pointer(&s[0]))
  31. }
  32. func icuUInt8P(s []byte) *C.uint8_t {
  33. return (*C.uint8_t)(unsafe.Pointer(&s[0]))
  34. }
  35. func icuUCharP(s []uint16) *C.UChar {
  36. return (*C.UChar)(unsafe.Pointer(&s[0]))
  37. }
  38. func icuULen(s []uint16) C.int32_t {
  39. return C.int32_t(len(s))
  40. }
  41. func icuSLen(s []byte) C.int32_t {
  42. return C.int32_t(len(s))
  43. }
  44. // icuCollator implements a Collator based on ICU.
  45. type icuCollator struct {
  46. loc *C.char
  47. col *C.UCollator
  48. keyBuf []byte
  49. }
  50. const growBufSize = 10 * 1024 * 1024
  51. func (c *icuCollator) init(locale string) error {
  52. err := C.UErrorCode(0)
  53. c.loc = C.CString(locale)
  54. c.col = C.ucol_open(c.loc, &err)
  55. if err > 0 {
  56. return fmt.Errorf("failed opening collator for %q", locale)
  57. } else if err < 0 {
  58. loc := C.ucol_getLocaleByType(c.col, 0, &err)
  59. fmt, ok := map[int]string{
  60. -127: "warning: using default collator: %s",
  61. -128: "warning: using fallback collator: %s",
  62. }[int(err)]
  63. if ok {
  64. log.Printf(fmt, C.GoString(loc))
  65. }
  66. }
  67. c.keyBuf = make([]byte, 0, growBufSize)
  68. return nil
  69. }
  70. func (c *icuCollator) buf() (*C.uint8_t, C.int32_t) {
  71. if len(c.keyBuf) == cap(c.keyBuf) {
  72. c.keyBuf = make([]byte, 0, growBufSize)
  73. }
  74. b := c.keyBuf[len(c.keyBuf):cap(c.keyBuf)]
  75. return icuUInt8P(b), icuSLen(b)
  76. }
  77. func (c *icuCollator) extendBuf(n C.int32_t) []byte {
  78. end := len(c.keyBuf) + int(n)
  79. if end > cap(c.keyBuf) {
  80. if len(c.keyBuf) == 0 {
  81. log.Fatalf("icuCollator: max string size exceeded: %v > %v", n, growBufSize)
  82. }
  83. c.keyBuf = make([]byte, 0, growBufSize)
  84. return nil
  85. }
  86. b := c.keyBuf[len(c.keyBuf):end]
  87. c.keyBuf = c.keyBuf[:end]
  88. return b
  89. }
  90. func (c *icuCollator) Close() error {
  91. C.ucol_close(c.col)
  92. C.free(unsafe.Pointer(c.loc))
  93. return nil
  94. }
  95. // icuUTF16 implements the Collator interface.
  96. type icuUTF16 struct {
  97. icuCollator
  98. }
  99. func newUTF16(locale string) (Collator, error) {
  100. c := &icuUTF16{}
  101. return c, c.init(locale)
  102. }
  103. func (c *icuUTF16) Compare(a, b Input) int {
  104. return int(C.ucol_strcoll(c.col, icuUCharP(a.UTF16), icuULen(a.UTF16), icuUCharP(b.UTF16), icuULen(b.UTF16)))
  105. }
  106. func (c *icuUTF16) Key(s Input) []byte {
  107. bp, bn := c.buf()
  108. n := C.ucol_getSortKey(c.col, icuUCharP(s.UTF16), icuULen(s.UTF16), bp, bn)
  109. if b := c.extendBuf(n); b != nil {
  110. return b
  111. }
  112. return c.Key(s)
  113. }
  114. // icuUTF8iter implements the Collator interface
  115. // This implementation wraps the UTF8 string in an iterator
  116. // which is passed to the collator.
  117. type icuUTF8iter struct {
  118. icuCollator
  119. a, b C.UCharIterator
  120. }
  121. func newUTF8iter(locale string) (Collator, error) {
  122. c := &icuUTF8iter{}
  123. return c, c.init(locale)
  124. }
  125. func (c *icuUTF8iter) Compare(a, b Input) int {
  126. err := C.UErrorCode(0)
  127. C.uiter_setUTF8(&c.a, icuCharP(a.UTF8), icuSLen(a.UTF8))
  128. C.uiter_setUTF8(&c.b, icuCharP(b.UTF8), icuSLen(b.UTF8))
  129. return int(C.ucol_strcollIter(c.col, &c.a, &c.b, &err))
  130. }
  131. func (c *icuUTF8iter) Key(s Input) []byte {
  132. err := C.UErrorCode(0)
  133. state := [2]C.uint32_t{}
  134. C.uiter_setUTF8(&c.a, icuCharP(s.UTF8), icuSLen(s.UTF8))
  135. bp, bn := c.buf()
  136. n := C.ucol_nextSortKeyPart(c.col, &c.a, &(state[0]), bp, bn, &err)
  137. if n >= bn {
  138. // Force failure.
  139. if c.extendBuf(n+1) != nil {
  140. log.Fatal("expected extension to fail")
  141. }
  142. return c.Key(s)
  143. }
  144. return c.extendBuf(n)
  145. }
  146. // icuUTF8conv implements the Collator interface.
  147. // This implementation first converts the give UTF8 string
  148. // to UTF16 and then calls the main ICU collation function.
  149. type icuUTF8conv struct {
  150. icuCollator
  151. }
  152. func newUTF8conv(locale string) (Collator, error) {
  153. c := &icuUTF8conv{}
  154. return c, c.init(locale)
  155. }
  156. func (c *icuUTF8conv) Compare(sa, sb Input) int {
  157. a := encodeUTF16(sa.UTF8)
  158. b := encodeUTF16(sb.UTF8)
  159. return int(C.ucol_strcoll(c.col, icuUCharP(a), icuULen(a), icuUCharP(b), icuULen(b)))
  160. }
  161. func (c *icuUTF8conv) Key(s Input) []byte {
  162. a := encodeUTF16(s.UTF8)
  163. bp, bn := c.buf()
  164. n := C.ucol_getSortKey(c.col, icuUCharP(a), icuULen(a), bp, bn)
  165. if b := c.extendBuf(n); b != nil {
  166. return b
  167. }
  168. return c.Key(s)
  169. }
  170. func encodeUTF16(b []byte) []uint16 {
  171. a := []uint16{}
  172. for len(b) > 0 {
  173. r, sz := utf8.DecodeRune(b)
  174. b = b[sz:]
  175. r1, r2 := utf16.EncodeRune(r)
  176. if r1 != 0xFFFD {
  177. a = append(a, uint16(r1), uint16(r2))
  178. } else {
  179. a = append(a, uint16(r))
  180. }
  181. }
  182. return a
  183. }