snappy_test.go 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725
  1. // Copyright 2011 The Snappy-Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package snappy
  5. import (
  6. "bytes"
  7. "flag"
  8. "fmt"
  9. "io"
  10. "io/ioutil"
  11. "math/rand"
  12. "net/http"
  13. "os"
  14. "path/filepath"
  15. "strings"
  16. "testing"
  17. )
  18. var (
  19. download = flag.Bool("download", false, "If true, download any missing files before running benchmarks")
  20. testdata = flag.String("testdata", "testdata", "Directory containing the test data")
  21. )
  22. func TestMaxEncodedLenOfMaxBlockSize(t *testing.T) {
  23. got := maxEncodedLenOfMaxBlockSize
  24. want := MaxEncodedLen(maxBlockSize)
  25. if got != want {
  26. t.Fatalf("got %d, want %d", got, want)
  27. }
  28. }
  29. func roundtrip(b, ebuf, dbuf []byte) error {
  30. d, err := Decode(dbuf, Encode(ebuf, b))
  31. if err != nil {
  32. return fmt.Errorf("decoding error: %v", err)
  33. }
  34. if !bytes.Equal(b, d) {
  35. return fmt.Errorf("roundtrip mismatch:\n\twant %v\n\tgot %v", b, d)
  36. }
  37. return nil
  38. }
  39. func TestEmpty(t *testing.T) {
  40. if err := roundtrip(nil, nil, nil); err != nil {
  41. t.Fatal(err)
  42. }
  43. }
  44. func TestSmallCopy(t *testing.T) {
  45. for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
  46. for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} {
  47. for i := 0; i < 32; i++ {
  48. s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb"
  49. if err := roundtrip([]byte(s), ebuf, dbuf); err != nil {
  50. t.Errorf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err)
  51. }
  52. }
  53. }
  54. }
  55. }
  56. func TestSmallRand(t *testing.T) {
  57. rng := rand.New(rand.NewSource(1))
  58. for n := 1; n < 20000; n += 23 {
  59. b := make([]byte, n)
  60. for i := range b {
  61. b[i] = uint8(rng.Intn(256))
  62. }
  63. if err := roundtrip(b, nil, nil); err != nil {
  64. t.Fatal(err)
  65. }
  66. }
  67. }
  68. func TestSmallRegular(t *testing.T) {
  69. for n := 1; n < 20000; n += 23 {
  70. b := make([]byte, n)
  71. for i := range b {
  72. b[i] = uint8(i%10 + 'a')
  73. }
  74. if err := roundtrip(b, nil, nil); err != nil {
  75. t.Fatal(err)
  76. }
  77. }
  78. }
  79. func TestInvalidVarint(t *testing.T) {
  80. data := []byte("\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00")
  81. if _, err := DecodedLen(data); err != ErrCorrupt {
  82. t.Errorf("DecodedLen: got %v, want ErrCorrupt", err)
  83. }
  84. if _, err := Decode(nil, data); err != ErrCorrupt {
  85. t.Errorf("Decode: got %v, want ErrCorrupt", err)
  86. }
  87. // The encoded varint overflows 32 bits
  88. data = []byte("\xff\xff\xff\xff\xff\x00")
  89. if _, err := DecodedLen(data); err != ErrCorrupt {
  90. t.Errorf("DecodedLen: got %v, want ErrCorrupt", err)
  91. }
  92. if _, err := Decode(nil, data); err != ErrCorrupt {
  93. t.Errorf("Decode: got %v, want ErrCorrupt", err)
  94. }
  95. }
  96. func TestDecode(t *testing.T) {
  97. testCases := []struct {
  98. desc string
  99. input string
  100. want string
  101. wantErr error
  102. }{{
  103. `decodedLen=0x100000000 is too long`,
  104. "\x80\x80\x80\x80\x10" + "\x00\x41",
  105. "",
  106. ErrCorrupt,
  107. }, {
  108. `decodedLen=3; tagLiteral, 0-byte length; length=3; valid input`,
  109. "\x03" + "\x08\xff\xff\xff",
  110. "\xff\xff\xff",
  111. nil,
  112. }, {
  113. `decodedLen=1; tagLiteral, 1-byte length; not enough length bytes`,
  114. "\x01" + "\xf0",
  115. "",
  116. ErrCorrupt,
  117. }, {
  118. `decodedLen=3; tagLiteral, 1-byte length; length=3; valid input`,
  119. "\x03" + "\xf0\x02\xff\xff\xff",
  120. "\xff\xff\xff",
  121. nil,
  122. }, {
  123. `decodedLen=1; tagLiteral, 2-byte length; not enough length bytes`,
  124. "\x01" + "\xf4\x00",
  125. "",
  126. ErrCorrupt,
  127. }, {
  128. `decodedLen=3; tagLiteral, 2-byte length; length=3; valid input`,
  129. "\x03" + "\xf4\x02\x00\xff\xff\xff",
  130. "\xff\xff\xff",
  131. nil,
  132. }, {
  133. `decodedLen=1; tagLiteral, 3-byte length; not enough length bytes`,
  134. "\x01" + "\xf8\x00\x00",
  135. "",
  136. ErrCorrupt,
  137. }, {
  138. `decodedLen=3; tagLiteral, 3-byte length; length=3; valid input`,
  139. "\x03" + "\xf8\x02\x00\x00\xff\xff\xff",
  140. "\xff\xff\xff",
  141. nil,
  142. }, {
  143. `decodedLen=1; tagLiteral, 4-byte length; not enough length bytes`,
  144. "\x01" + "\xfc\x00\x00\x00",
  145. "",
  146. ErrCorrupt,
  147. }, {
  148. `decodedLen=1; tagLiteral, 4-byte length; length=3; not enough dst bytes`,
  149. "\x01" + "\xfc\x02\x00\x00\x00\xff\xff\xff",
  150. "",
  151. ErrCorrupt,
  152. }, {
  153. `decodedLen=4; tagLiteral, 4-byte length; length=3; not enough src bytes`,
  154. "\x04" + "\xfc\x02\x00\x00\x00\xff",
  155. "",
  156. ErrCorrupt,
  157. }, {
  158. `decodedLen=3; tagLiteral, 4-byte length; length=3; valid input`,
  159. "\x03" + "\xfc\x02\x00\x00\x00\xff\xff\xff",
  160. "\xff\xff\xff",
  161. nil,
  162. }, {
  163. `decodedLen=4; tagCopy1, 1 extra length|offset byte; not enough extra bytes`,
  164. "\x04" + "\x01",
  165. "",
  166. ErrCorrupt,
  167. }, {
  168. `decodedLen=4; tagCopy2, 2 extra length|offset bytes; not enough extra bytes`,
  169. "\x04" + "\x02\x00",
  170. "",
  171. ErrCorrupt,
  172. }, {
  173. `decodedLen=4; tagCopy4; unsupported COPY_4 tag`,
  174. "\x04" + "\x03\x00\x00\x00\x00",
  175. "",
  176. errUnsupportedCopy4Tag,
  177. }, {
  178. `decodedLen=4; tagLiteral (4 bytes "abcd"); valid input`,
  179. "\x04" + "\x0cabcd",
  180. "abcd",
  181. nil,
  182. }, {
  183. `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; valid input`,
  184. "\x08" + "\x0cabcd" + "\x01\x04",
  185. "abcdabcd",
  186. nil,
  187. }, {
  188. `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=2; valid input`,
  189. "\x08" + "\x0cabcd" + "\x01\x02",
  190. "abcdcdcd",
  191. nil,
  192. }, {
  193. `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=1; valid input`,
  194. "\x08" + "\x0cabcd" + "\x01\x01",
  195. "abcddddd",
  196. nil,
  197. }, {
  198. `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=0; zero offset`,
  199. "\x08" + "\x0cabcd" + "\x01\x00",
  200. "",
  201. ErrCorrupt,
  202. }, {
  203. `decodedLen=9; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; inconsistent dLen`,
  204. "\x09" + "\x0cabcd" + "\x01\x04",
  205. "",
  206. ErrCorrupt,
  207. }, {
  208. `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=5; offset too large`,
  209. "\x08" + "\x0cabcd" + "\x01\x05",
  210. "",
  211. ErrCorrupt,
  212. }, {
  213. `decodedLen=7; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; length too large`,
  214. "\x07" + "\x0cabcd" + "\x01\x04",
  215. "",
  216. ErrCorrupt,
  217. }}
  218. for _, tc := range testCases {
  219. g, gotErr := Decode(nil, []byte(tc.input))
  220. if got := string(g); got != tc.want || gotErr != tc.wantErr {
  221. t.Errorf("%s:\ngot %q, %v\nwant %q, %v", tc.desc, got, gotErr, tc.want, tc.wantErr)
  222. }
  223. }
  224. }
  225. // TestEncodeNoiseThenRepeats encodes input for which the first half is very
  226. // incompressible and the second half is very compressible. The encoded form's
  227. // length should be closer to 50% of the original length than 100%.
  228. func TestEncodeNoiseThenRepeats(t *testing.T) {
  229. for _, origLen := range []int{32 * 1024, 256 * 1024, 2048 * 1024} {
  230. src := make([]byte, origLen)
  231. rng := rand.New(rand.NewSource(1))
  232. firstHalf, secondHalf := src[:origLen/2], src[origLen/2:]
  233. for i := range firstHalf {
  234. firstHalf[i] = uint8(rng.Intn(256))
  235. }
  236. for i := range secondHalf {
  237. secondHalf[i] = uint8(i >> 8)
  238. }
  239. dst := Encode(nil, src)
  240. if got, want := len(dst), origLen*3/4; got >= want {
  241. t.Errorf("origLen=%d: got %d encoded bytes, want less than %d", origLen, got, want)
  242. }
  243. }
  244. }
  245. func cmp(a, b []byte) error {
  246. if len(a) != len(b) {
  247. return fmt.Errorf("got %d bytes, want %d", len(a), len(b))
  248. }
  249. for i := range a {
  250. if a[i] != b[i] {
  251. return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i])
  252. }
  253. }
  254. return nil
  255. }
  256. func TestFramingFormat(t *testing.T) {
  257. // src is comprised of alternating 1e5-sized sequences of random
  258. // (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen
  259. // because it is larger than maxBlockSize (64k).
  260. src := make([]byte, 1e6)
  261. rng := rand.New(rand.NewSource(1))
  262. for i := 0; i < 10; i++ {
  263. if i%2 == 0 {
  264. for j := 0; j < 1e5; j++ {
  265. src[1e5*i+j] = uint8(rng.Intn(256))
  266. }
  267. } else {
  268. for j := 0; j < 1e5; j++ {
  269. src[1e5*i+j] = uint8(i)
  270. }
  271. }
  272. }
  273. buf := new(bytes.Buffer)
  274. if _, err := NewWriter(buf).Write(src); err != nil {
  275. t.Fatalf("Write: encoding: %v", err)
  276. }
  277. dst, err := ioutil.ReadAll(NewReader(buf))
  278. if err != nil {
  279. t.Fatalf("ReadAll: decoding: %v", err)
  280. }
  281. if err := cmp(dst, src); err != nil {
  282. t.Fatal(err)
  283. }
  284. }
  285. func TestWriterGoldenOutput(t *testing.T) {
  286. buf := new(bytes.Buffer)
  287. w := NewBufferedWriter(buf)
  288. defer w.Close()
  289. w.Write([]byte("abcd")) // Not compressible.
  290. w.Flush()
  291. w.Write(bytes.Repeat([]byte{'A'}, 100)) // Compressible.
  292. w.Flush()
  293. got := buf.String()
  294. want := strings.Join([]string{
  295. magicChunk,
  296. "\x01\x08\x00\x00", // Uncompressed chunk, 8 bytes long (including 4 byte checksum).
  297. "\x68\x10\xe6\xb6", // Checksum.
  298. "\x61\x62\x63\x64", // Uncompressed payload: "abcd".
  299. "\x00\x0d\x00\x00", // Compressed chunk, 13 bytes long (including 4 byte checksum).
  300. "\x37\xcb\xbc\x9d", // Checksum.
  301. "\x64", // Compressed payload: Uncompressed length (varint encoded): 100.
  302. "\x00\x41", // Compressed payload: tagLiteral, length=1, "A".
  303. "\xfe\x01\x00", // Compressed payload: tagCopy2, length=64, offset=1.
  304. "\x8a\x01\x00", // Compressed payload: tagCopy2, length=35, offset=1.
  305. }, "")
  306. if got != want {
  307. t.Fatalf("\ngot: % x\nwant: % x", got, want)
  308. }
  309. }
  310. func TestNewBufferedWriter(t *testing.T) {
  311. // Test all 32 possible sub-sequences of these 5 input slices.
  312. //
  313. // Their lengths sum to 400,000, which is over 6 times the Writer ibuf
  314. // capacity: 6 * maxBlockSize is 393,216.
  315. inputs := [][]byte{
  316. bytes.Repeat([]byte{'a'}, 40000),
  317. bytes.Repeat([]byte{'b'}, 150000),
  318. bytes.Repeat([]byte{'c'}, 60000),
  319. bytes.Repeat([]byte{'d'}, 120000),
  320. bytes.Repeat([]byte{'e'}, 30000),
  321. }
  322. loop:
  323. for i := 0; i < 1<<uint(len(inputs)); i++ {
  324. var want []byte
  325. buf := new(bytes.Buffer)
  326. w := NewBufferedWriter(buf)
  327. for j, input := range inputs {
  328. if i&(1<<uint(j)) == 0 {
  329. continue
  330. }
  331. if _, err := w.Write(input); err != nil {
  332. t.Errorf("i=%#02x: j=%d: Write: %v", i, j, err)
  333. continue loop
  334. }
  335. want = append(want, input...)
  336. }
  337. if err := w.Close(); err != nil {
  338. t.Errorf("i=%#02x: Close: %v", i, err)
  339. continue
  340. }
  341. got, err := ioutil.ReadAll(NewReader(buf))
  342. if err != nil {
  343. t.Errorf("i=%#02x: ReadAll: %v", i, err)
  344. continue
  345. }
  346. if err := cmp(got, want); err != nil {
  347. t.Errorf("i=%#02x: %v", i, err)
  348. continue
  349. }
  350. }
  351. }
  352. func TestFlush(t *testing.T) {
  353. buf := new(bytes.Buffer)
  354. w := NewBufferedWriter(buf)
  355. defer w.Close()
  356. if _, err := w.Write(bytes.Repeat([]byte{'x'}, 20)); err != nil {
  357. t.Fatalf("Write: %v", err)
  358. }
  359. if n := buf.Len(); n != 0 {
  360. t.Fatalf("before Flush: %d bytes were written to the underlying io.Writer, want 0", n)
  361. }
  362. if err := w.Flush(); err != nil {
  363. t.Fatalf("Flush: %v", err)
  364. }
  365. if n := buf.Len(); n == 0 {
  366. t.Fatalf("after Flush: %d bytes were written to the underlying io.Writer, want non-0", n)
  367. }
  368. }
  369. func TestReaderReset(t *testing.T) {
  370. gold := bytes.Repeat([]byte("All that is gold does not glitter,\n"), 10000)
  371. buf := new(bytes.Buffer)
  372. if _, err := NewWriter(buf).Write(gold); err != nil {
  373. t.Fatalf("Write: %v", err)
  374. }
  375. encoded, invalid, partial := buf.String(), "invalid", "partial"
  376. r := NewReader(nil)
  377. for i, s := range []string{encoded, invalid, partial, encoded, partial, invalid, encoded, encoded} {
  378. if s == partial {
  379. r.Reset(strings.NewReader(encoded))
  380. if _, err := r.Read(make([]byte, 101)); err != nil {
  381. t.Errorf("#%d: %v", i, err)
  382. continue
  383. }
  384. continue
  385. }
  386. r.Reset(strings.NewReader(s))
  387. got, err := ioutil.ReadAll(r)
  388. switch s {
  389. case encoded:
  390. if err != nil {
  391. t.Errorf("#%d: %v", i, err)
  392. continue
  393. }
  394. if err := cmp(got, gold); err != nil {
  395. t.Errorf("#%d: %v", i, err)
  396. continue
  397. }
  398. case invalid:
  399. if err == nil {
  400. t.Errorf("#%d: got nil error, want non-nil", i)
  401. continue
  402. }
  403. }
  404. }
  405. }
  406. func TestWriterReset(t *testing.T) {
  407. gold := bytes.Repeat([]byte("Not all those who wander are lost;\n"), 10000)
  408. const n = 20
  409. for _, buffered := range []bool{false, true} {
  410. var w *Writer
  411. if buffered {
  412. w = NewBufferedWriter(nil)
  413. defer w.Close()
  414. } else {
  415. w = NewWriter(nil)
  416. }
  417. var gots, wants [][]byte
  418. failed := false
  419. for i := 0; i <= n; i++ {
  420. buf := new(bytes.Buffer)
  421. w.Reset(buf)
  422. want := gold[:len(gold)*i/n]
  423. if _, err := w.Write(want); err != nil {
  424. t.Errorf("#%d: Write: %v", i, err)
  425. failed = true
  426. continue
  427. }
  428. if buffered {
  429. if err := w.Flush(); err != nil {
  430. t.Errorf("#%d: Flush: %v", i, err)
  431. failed = true
  432. continue
  433. }
  434. }
  435. got, err := ioutil.ReadAll(NewReader(buf))
  436. if err != nil {
  437. t.Errorf("#%d: ReadAll: %v", i, err)
  438. failed = true
  439. continue
  440. }
  441. gots = append(gots, got)
  442. wants = append(wants, want)
  443. }
  444. if failed {
  445. continue
  446. }
  447. for i := range gots {
  448. if err := cmp(gots[i], wants[i]); err != nil {
  449. t.Errorf("#%d: %v", i, err)
  450. }
  451. }
  452. }
  453. }
  454. func TestWriterResetWithoutFlush(t *testing.T) {
  455. buf0 := new(bytes.Buffer)
  456. buf1 := new(bytes.Buffer)
  457. w := NewBufferedWriter(buf0)
  458. if _, err := w.Write([]byte("xxx")); err != nil {
  459. t.Fatalf("Write #0: %v", err)
  460. }
  461. // Note that we don't Flush the Writer before calling Reset.
  462. w.Reset(buf1)
  463. if _, err := w.Write([]byte("yyy")); err != nil {
  464. t.Fatalf("Write #1: %v", err)
  465. }
  466. if err := w.Flush(); err != nil {
  467. t.Fatalf("Flush: %v", err)
  468. }
  469. got, err := ioutil.ReadAll(NewReader(buf1))
  470. if err != nil {
  471. t.Fatalf("ReadAll: %v", err)
  472. }
  473. if err := cmp(got, []byte("yyy")); err != nil {
  474. t.Fatal(err)
  475. }
  476. }
  477. type writeCounter int
  478. func (c *writeCounter) Write(p []byte) (int, error) {
  479. *c++
  480. return len(p), nil
  481. }
  482. // TestNumUnderlyingWrites tests that each Writer flush only makes one or two
  483. // Write calls on its underlying io.Writer, depending on whether or not the
  484. // flushed buffer was compressible.
  485. func TestNumUnderlyingWrites(t *testing.T) {
  486. testCases := []struct {
  487. input []byte
  488. want int
  489. }{
  490. {bytes.Repeat([]byte{'x'}, 100), 1},
  491. {bytes.Repeat([]byte{'y'}, 100), 1},
  492. {[]byte("ABCDEFGHIJKLMNOPQRST"), 2},
  493. }
  494. var c writeCounter
  495. w := NewBufferedWriter(&c)
  496. defer w.Close()
  497. for i, tc := range testCases {
  498. c = 0
  499. if _, err := w.Write(tc.input); err != nil {
  500. t.Errorf("#%d: Write: %v", i, err)
  501. continue
  502. }
  503. if err := w.Flush(); err != nil {
  504. t.Errorf("#%d: Flush: %v", i, err)
  505. continue
  506. }
  507. if int(c) != tc.want {
  508. t.Errorf("#%d: got %d underlying writes, want %d", i, c, tc.want)
  509. continue
  510. }
  511. }
  512. }
  513. func benchDecode(b *testing.B, src []byte) {
  514. encoded := Encode(nil, src)
  515. // Bandwidth is in amount of uncompressed data.
  516. b.SetBytes(int64(len(src)))
  517. b.ResetTimer()
  518. for i := 0; i < b.N; i++ {
  519. Decode(src, encoded)
  520. }
  521. }
  522. func benchEncode(b *testing.B, src []byte) {
  523. // Bandwidth is in amount of uncompressed data.
  524. b.SetBytes(int64(len(src)))
  525. dst := make([]byte, MaxEncodedLen(len(src)))
  526. b.ResetTimer()
  527. for i := 0; i < b.N; i++ {
  528. Encode(dst, src)
  529. }
  530. }
  531. func readFile(b testing.TB, filename string) []byte {
  532. src, err := ioutil.ReadFile(filename)
  533. if err != nil {
  534. b.Skipf("skipping benchmark: %v", err)
  535. }
  536. if len(src) == 0 {
  537. b.Fatalf("%s has zero length", filename)
  538. }
  539. return src
  540. }
  541. // expand returns a slice of length n containing repeated copies of src.
  542. func expand(src []byte, n int) []byte {
  543. dst := make([]byte, n)
  544. for x := dst; len(x) > 0; {
  545. i := copy(x, src)
  546. x = x[i:]
  547. }
  548. return dst
  549. }
  550. func benchWords(b *testing.B, n int, decode bool) {
  551. // Note: the file is OS-language dependent so the resulting values are not
  552. // directly comparable for non-US-English OS installations.
  553. data := expand(readFile(b, "/usr/share/dict/words"), n)
  554. if decode {
  555. benchDecode(b, data)
  556. } else {
  557. benchEncode(b, data)
  558. }
  559. }
  560. func BenchmarkWordsDecode1e1(b *testing.B) { benchWords(b, 1e1, true) }
  561. func BenchmarkWordsDecode1e2(b *testing.B) { benchWords(b, 1e2, true) }
  562. func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) }
  563. func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) }
  564. func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) }
  565. func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) }
  566. func BenchmarkWordsEncode1e1(b *testing.B) { benchWords(b, 1e1, false) }
  567. func BenchmarkWordsEncode1e2(b *testing.B) { benchWords(b, 1e2, false) }
  568. func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) }
  569. func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) }
  570. func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) }
  571. func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) }
  572. func BenchmarkRandomEncode(b *testing.B) {
  573. rng := rand.New(rand.NewSource(1))
  574. data := make([]byte, 1<<20)
  575. for i := range data {
  576. data[i] = uint8(rng.Intn(256))
  577. }
  578. benchEncode(b, data)
  579. }
  580. // testFiles' values are copied directly from
  581. // https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc
  582. // The label field is unused in snappy-go.
  583. //
  584. // If this list changes (due to the upstream C++ list changing), remember to
  585. // update the .gitignore file in this repository.
  586. var testFiles = []struct {
  587. label string
  588. filename string
  589. sizeLimit int
  590. }{
  591. {"html", "html", 0},
  592. {"urls", "urls.10K", 0},
  593. {"jpg", "fireworks.jpeg", 0},
  594. {"jpg_200", "fireworks.jpeg", 200},
  595. {"pdf", "paper-100k.pdf", 0},
  596. {"html4", "html_x_4", 0},
  597. {"txt1", "alice29.txt", 0},
  598. {"txt2", "asyoulik.txt", 0},
  599. {"txt3", "lcet10.txt", 0},
  600. {"txt4", "plrabn12.txt", 0},
  601. {"pb", "geo.protodata", 0},
  602. {"gaviota", "kppkn.gtb", 0},
  603. }
  604. // The test data files are present at this canonical URL.
  605. const baseURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/"
  606. func downloadTestdata(b *testing.B, basename string) (errRet error) {
  607. filename := filepath.Join(*testdata, basename)
  608. if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 {
  609. return nil
  610. }
  611. if !*download {
  612. b.Skipf("test data not found; skipping benchmark without the -download flag")
  613. }
  614. // Download the official snappy C++ implementation reference test data
  615. // files for benchmarking.
  616. if err := os.Mkdir(*testdata, 0777); err != nil && !os.IsExist(err) {
  617. return fmt.Errorf("failed to create testdata: %s", err)
  618. }
  619. f, err := os.Create(filename)
  620. if err != nil {
  621. return fmt.Errorf("failed to create %s: %s", filename, err)
  622. }
  623. defer f.Close()
  624. defer func() {
  625. if errRet != nil {
  626. os.Remove(filename)
  627. }
  628. }()
  629. url := baseURL + basename
  630. resp, err := http.Get(url)
  631. if err != nil {
  632. return fmt.Errorf("failed to download %s: %s", url, err)
  633. }
  634. defer resp.Body.Close()
  635. if s := resp.StatusCode; s != http.StatusOK {
  636. return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s))
  637. }
  638. _, err = io.Copy(f, resp.Body)
  639. if err != nil {
  640. return fmt.Errorf("failed to download %s to %s: %s", url, filename, err)
  641. }
  642. return nil
  643. }
  644. func benchFile(b *testing.B, n int, decode bool) {
  645. if err := downloadTestdata(b, testFiles[n].filename); err != nil {
  646. b.Fatalf("failed to download testdata: %s", err)
  647. }
  648. data := readFile(b, filepath.Join(*testdata, testFiles[n].filename))
  649. if n := testFiles[n].sizeLimit; 0 < n && n < len(data) {
  650. data = data[:n]
  651. }
  652. if decode {
  653. benchDecode(b, data)
  654. } else {
  655. benchEncode(b, data)
  656. }
  657. }
  658. // Naming convention is kept similar to what snappy's C++ implementation uses.
  659. func Benchmark_UFlat0(b *testing.B) { benchFile(b, 0, true) }
  660. func Benchmark_UFlat1(b *testing.B) { benchFile(b, 1, true) }
  661. func Benchmark_UFlat2(b *testing.B) { benchFile(b, 2, true) }
  662. func Benchmark_UFlat3(b *testing.B) { benchFile(b, 3, true) }
  663. func Benchmark_UFlat4(b *testing.B) { benchFile(b, 4, true) }
  664. func Benchmark_UFlat5(b *testing.B) { benchFile(b, 5, true) }
  665. func Benchmark_UFlat6(b *testing.B) { benchFile(b, 6, true) }
  666. func Benchmark_UFlat7(b *testing.B) { benchFile(b, 7, true) }
  667. func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) }
  668. func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) }
  669. func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) }
  670. func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) }
  671. func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) }
  672. func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) }
  673. func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) }
  674. func Benchmark_ZFlat3(b *testing.B) { benchFile(b, 3, false) }
  675. func Benchmark_ZFlat4(b *testing.B) { benchFile(b, 4, false) }
  676. func Benchmark_ZFlat5(b *testing.B) { benchFile(b, 5, false) }
  677. func Benchmark_ZFlat6(b *testing.B) { benchFile(b, 6, false) }
  678. func Benchmark_ZFlat7(b *testing.B) { benchFile(b, 7, false) }
  679. func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) }
  680. func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) }
  681. func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) }
  682. func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) }