deflate.go 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821
  1. // Copyright 2009 The Go Authors. All rights reserved.
  2. // Copyright (c) 2015 Klaus Post
  3. // Use of this source code is governed by a BSD-style
  4. // license that can be found in the LICENSE file.
  5. package flate
  6. import (
  7. "fmt"
  8. "io"
  9. "math"
  10. )
  11. const (
  12. NoCompression = 0
  13. BestSpeed = 1
  14. BestCompression = 9
  15. DefaultCompression = -1
  16. // HuffmanOnly disables Lempel-Ziv match searching and only performs Huffman
  17. // entropy encoding. This mode is useful in compressing data that has
  18. // already been compressed with an LZ style algorithm (e.g. Snappy or LZ4)
  19. // that lacks an entropy encoder. Compression gains are achieved when
  20. // certain bytes in the input stream occur more frequently than others.
  21. //
  22. // Note that HuffmanOnly produces a compressed output that is
  23. // RFC 1951 compliant. That is, any valid DEFLATE decompressor will
  24. // continue to be able to decompress this output.
  25. HuffmanOnly = -2
  26. ConstantCompression = HuffmanOnly // compatibility alias.
  27. logWindowSize = 15
  28. windowSize = 1 << logWindowSize
  29. windowMask = windowSize - 1
  30. logMaxOffsetSize = 15 // Standard DEFLATE
  31. minMatchLength = 4 // The smallest match that the compressor looks for
  32. maxMatchLength = 258 // The longest match for the compressor
  33. minOffsetSize = 1 // The shortest offset that makes any sense
  34. // The maximum number of tokens we put into a single flat block, just too
  35. // stop things from getting too large.
  36. maxFlateBlockTokens = 1 << 14
  37. maxStoreBlockSize = 65535
  38. hashBits = 17 // After 17 performance degrades
  39. hashSize = 1 << hashBits
  40. hashMask = (1 << hashBits) - 1
  41. hashShift = (hashBits + minMatchLength - 1) / minMatchLength
  42. maxHashOffset = 1 << 24
  43. skipNever = math.MaxInt32
  44. debugDeflate = false
  45. )
  46. type compressionLevel struct {
  47. good, lazy, nice, chain, fastSkipHashing, level int
  48. }
  49. // Compression levels have been rebalanced from zlib deflate defaults
  50. // to give a bigger spread in speed and compression.
  51. // See https://blog.klauspost.com/rebalancing-deflate-compression-levels/
  52. var levels = []compressionLevel{
  53. {}, // 0
  54. // Level 1-6 uses specialized algorithm - values not used
  55. {0, 0, 0, 0, 0, 1},
  56. {0, 0, 0, 0, 0, 2},
  57. {0, 0, 0, 0, 0, 3},
  58. {0, 0, 0, 0, 0, 4},
  59. {0, 0, 0, 0, 0, 5},
  60. {0, 0, 0, 0, 0, 6},
  61. // Levels 7-9 use increasingly more lazy matching
  62. // and increasingly stringent conditions for "good enough".
  63. {8, 8, 24, 16, skipNever, 7},
  64. {10, 16, 24, 64, skipNever, 8},
  65. {32, 258, 258, 4096, skipNever, 9},
  66. }
  67. // advancedState contains state for the advanced levels, with bigger hash tables, etc.
  68. type advancedState struct {
  69. // deflate state
  70. length int
  71. offset int
  72. maxInsertIndex int
  73. // Input hash chains
  74. // hashHead[hashValue] contains the largest inputIndex with the specified hash value
  75. // If hashHead[hashValue] is within the current window, then
  76. // hashPrev[hashHead[hashValue] & windowMask] contains the previous index
  77. // with the same hash value.
  78. chainHead int
  79. hashHead [hashSize]uint32
  80. hashPrev [windowSize]uint32
  81. hashOffset int
  82. // input window: unprocessed data is window[index:windowEnd]
  83. index int
  84. hashMatch [maxMatchLength + minMatchLength]uint32
  85. hash uint32
  86. ii uint16 // position of last match, intended to overflow to reset.
  87. }
  88. type compressor struct {
  89. compressionLevel
  90. w *huffmanBitWriter
  91. // compression algorithm
  92. fill func(*compressor, []byte) int // copy data to window
  93. step func(*compressor) // process window
  94. window []byte
  95. windowEnd int
  96. blockStart int // window index where current tokens start
  97. err error
  98. // queued output tokens
  99. tokens tokens
  100. fast fastEnc
  101. state *advancedState
  102. sync bool // requesting flush
  103. byteAvailable bool // if true, still need to process window[index-1].
  104. }
  105. func (d *compressor) fillDeflate(b []byte) int {
  106. s := d.state
  107. if s.index >= 2*windowSize-(minMatchLength+maxMatchLength) {
  108. // shift the window by windowSize
  109. copy(d.window[:], d.window[windowSize:2*windowSize])
  110. s.index -= windowSize
  111. d.windowEnd -= windowSize
  112. if d.blockStart >= windowSize {
  113. d.blockStart -= windowSize
  114. } else {
  115. d.blockStart = math.MaxInt32
  116. }
  117. s.hashOffset += windowSize
  118. if s.hashOffset > maxHashOffset {
  119. delta := s.hashOffset - 1
  120. s.hashOffset -= delta
  121. s.chainHead -= delta
  122. // Iterate over slices instead of arrays to avoid copying
  123. // the entire table onto the stack (Issue #18625).
  124. for i, v := range s.hashPrev[:] {
  125. if int(v) > delta {
  126. s.hashPrev[i] = uint32(int(v) - delta)
  127. } else {
  128. s.hashPrev[i] = 0
  129. }
  130. }
  131. for i, v := range s.hashHead[:] {
  132. if int(v) > delta {
  133. s.hashHead[i] = uint32(int(v) - delta)
  134. } else {
  135. s.hashHead[i] = 0
  136. }
  137. }
  138. }
  139. }
  140. n := copy(d.window[d.windowEnd:], b)
  141. d.windowEnd += n
  142. return n
  143. }
  144. func (d *compressor) writeBlock(tok *tokens, index int, eof bool) error {
  145. if index > 0 || eof {
  146. var window []byte
  147. if d.blockStart <= index {
  148. window = d.window[d.blockStart:index]
  149. }
  150. d.blockStart = index
  151. d.w.writeBlock(tok, eof, window)
  152. return d.w.err
  153. }
  154. return nil
  155. }
  156. // writeBlockSkip writes the current block and uses the number of tokens
  157. // to determine if the block should be stored on no matches, or
  158. // only huffman encoded.
  159. func (d *compressor) writeBlockSkip(tok *tokens, index int, eof bool) error {
  160. if index > 0 || eof {
  161. if d.blockStart <= index {
  162. window := d.window[d.blockStart:index]
  163. // If we removed less than a 64th of all literals
  164. // we huffman compress the block.
  165. if int(tok.n) > len(window)-int(tok.n>>6) {
  166. d.w.writeBlockHuff(eof, window, d.sync)
  167. } else {
  168. // Write a dynamic huffman block.
  169. d.w.writeBlockDynamic(tok, eof, window, d.sync)
  170. }
  171. } else {
  172. d.w.writeBlock(tok, eof, nil)
  173. }
  174. d.blockStart = index
  175. return d.w.err
  176. }
  177. return nil
  178. }
  179. // fillWindow will fill the current window with the supplied
  180. // dictionary and calculate all hashes.
  181. // This is much faster than doing a full encode.
  182. // Should only be used after a start/reset.
  183. func (d *compressor) fillWindow(b []byte) {
  184. // Do not fill window if we are in store-only or huffman mode.
  185. if d.level <= 0 {
  186. return
  187. }
  188. if d.fast != nil {
  189. // encode the last data, but discard the result
  190. if len(b) > maxMatchOffset {
  191. b = b[len(b)-maxMatchOffset:]
  192. }
  193. d.fast.Encode(&d.tokens, b)
  194. d.tokens.Reset()
  195. return
  196. }
  197. s := d.state
  198. // If we are given too much, cut it.
  199. if len(b) > windowSize {
  200. b = b[len(b)-windowSize:]
  201. }
  202. // Add all to window.
  203. n := copy(d.window[d.windowEnd:], b)
  204. // Calculate 256 hashes at the time (more L1 cache hits)
  205. loops := (n + 256 - minMatchLength) / 256
  206. for j := 0; j < loops; j++ {
  207. startindex := j * 256
  208. end := startindex + 256 + minMatchLength - 1
  209. if end > n {
  210. end = n
  211. }
  212. tocheck := d.window[startindex:end]
  213. dstSize := len(tocheck) - minMatchLength + 1
  214. if dstSize <= 0 {
  215. continue
  216. }
  217. dst := s.hashMatch[:dstSize]
  218. bulkHash4(tocheck, dst)
  219. var newH uint32
  220. for i, val := range dst {
  221. di := i + startindex
  222. newH = val & hashMask
  223. // Get previous value with the same hash.
  224. // Our chain should point to the previous value.
  225. s.hashPrev[di&windowMask] = s.hashHead[newH]
  226. // Set the head of the hash chain to us.
  227. s.hashHead[newH] = uint32(di + s.hashOffset)
  228. }
  229. s.hash = newH
  230. }
  231. // Update window information.
  232. d.windowEnd += n
  233. s.index = n
  234. }
  235. // Try to find a match starting at index whose length is greater than prevSize.
  236. // We only look at chainCount possibilities before giving up.
  237. // pos = s.index, prevHead = s.chainHead-s.hashOffset, prevLength=minMatchLength-1, lookahead
  238. func (d *compressor) findMatch(pos int, prevHead int, prevLength int, lookahead int) (length, offset int, ok bool) {
  239. minMatchLook := maxMatchLength
  240. if lookahead < minMatchLook {
  241. minMatchLook = lookahead
  242. }
  243. win := d.window[0 : pos+minMatchLook]
  244. // We quit when we get a match that's at least nice long
  245. nice := len(win) - pos
  246. if d.nice < nice {
  247. nice = d.nice
  248. }
  249. // If we've got a match that's good enough, only look in 1/4 the chain.
  250. tries := d.chain
  251. length = prevLength
  252. if length >= d.good {
  253. tries >>= 2
  254. }
  255. wEnd := win[pos+length]
  256. wPos := win[pos:]
  257. minIndex := pos - windowSize
  258. for i := prevHead; tries > 0; tries-- {
  259. if wEnd == win[i+length] {
  260. n := matchLen(win[i:i+minMatchLook], wPos)
  261. if n > length && (n > minMatchLength || pos-i <= 4096) {
  262. length = n
  263. offset = pos - i
  264. ok = true
  265. if n >= nice {
  266. // The match is good enough that we don't try to find a better one.
  267. break
  268. }
  269. wEnd = win[pos+n]
  270. }
  271. }
  272. if i == minIndex {
  273. // hashPrev[i & windowMask] has already been overwritten, so stop now.
  274. break
  275. }
  276. i = int(d.state.hashPrev[i&windowMask]) - d.state.hashOffset
  277. if i < minIndex || i < 0 {
  278. break
  279. }
  280. }
  281. return
  282. }
  283. func (d *compressor) writeStoredBlock(buf []byte) error {
  284. if d.w.writeStoredHeader(len(buf), false); d.w.err != nil {
  285. return d.w.err
  286. }
  287. d.w.writeBytes(buf)
  288. return d.w.err
  289. }
  290. // hash4 returns a hash representation of the first 4 bytes
  291. // of the supplied slice.
  292. // The caller must ensure that len(b) >= 4.
  293. func hash4(b []byte) uint32 {
  294. b = b[:4]
  295. return hash4u(uint32(b[3])|uint32(b[2])<<8|uint32(b[1])<<16|uint32(b[0])<<24, hashBits)
  296. }
  297. // bulkHash4 will compute hashes using the same
  298. // algorithm as hash4
  299. func bulkHash4(b []byte, dst []uint32) {
  300. if len(b) < 4 {
  301. return
  302. }
  303. hb := uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24
  304. dst[0] = hash4u(hb, hashBits)
  305. end := len(b) - 4 + 1
  306. for i := 1; i < end; i++ {
  307. hb = (hb << 8) | uint32(b[i+3])
  308. dst[i] = hash4u(hb, hashBits)
  309. }
  310. }
  311. func (d *compressor) initDeflate() {
  312. d.window = make([]byte, 2*windowSize)
  313. d.byteAvailable = false
  314. d.err = nil
  315. if d.state == nil {
  316. return
  317. }
  318. s := d.state
  319. s.index = 0
  320. s.hashOffset = 1
  321. s.length = minMatchLength - 1
  322. s.offset = 0
  323. s.hash = 0
  324. s.chainHead = -1
  325. }
  326. // deflateLazy is the same as deflate, but with d.fastSkipHashing == skipNever,
  327. // meaning it always has lazy matching on.
  328. func (d *compressor) deflateLazy() {
  329. s := d.state
  330. // Sanity enables additional runtime tests.
  331. // It's intended to be used during development
  332. // to supplement the currently ad-hoc unit tests.
  333. const sanity = debugDeflate
  334. if d.windowEnd-s.index < minMatchLength+maxMatchLength && !d.sync {
  335. return
  336. }
  337. s.maxInsertIndex = d.windowEnd - (minMatchLength - 1)
  338. if s.index < s.maxInsertIndex {
  339. s.hash = hash4(d.window[s.index : s.index+minMatchLength])
  340. }
  341. for {
  342. if sanity && s.index > d.windowEnd {
  343. panic("index > windowEnd")
  344. }
  345. lookahead := d.windowEnd - s.index
  346. if lookahead < minMatchLength+maxMatchLength {
  347. if !d.sync {
  348. return
  349. }
  350. if sanity && s.index > d.windowEnd {
  351. panic("index > windowEnd")
  352. }
  353. if lookahead == 0 {
  354. // Flush current output block if any.
  355. if d.byteAvailable {
  356. // There is still one pending token that needs to be flushed
  357. d.tokens.AddLiteral(d.window[s.index-1])
  358. d.byteAvailable = false
  359. }
  360. if d.tokens.n > 0 {
  361. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  362. return
  363. }
  364. d.tokens.Reset()
  365. }
  366. return
  367. }
  368. }
  369. if s.index < s.maxInsertIndex {
  370. // Update the hash
  371. s.hash = hash4(d.window[s.index : s.index+minMatchLength])
  372. ch := s.hashHead[s.hash&hashMask]
  373. s.chainHead = int(ch)
  374. s.hashPrev[s.index&windowMask] = ch
  375. s.hashHead[s.hash&hashMask] = uint32(s.index + s.hashOffset)
  376. }
  377. prevLength := s.length
  378. prevOffset := s.offset
  379. s.length = minMatchLength - 1
  380. s.offset = 0
  381. minIndex := s.index - windowSize
  382. if minIndex < 0 {
  383. minIndex = 0
  384. }
  385. if s.chainHead-s.hashOffset >= minIndex && lookahead > prevLength && prevLength < d.lazy {
  386. if newLength, newOffset, ok := d.findMatch(s.index, s.chainHead-s.hashOffset, minMatchLength-1, lookahead); ok {
  387. s.length = newLength
  388. s.offset = newOffset
  389. }
  390. }
  391. if prevLength >= minMatchLength && s.length <= prevLength {
  392. // There was a match at the previous step, and the current match is
  393. // not better. Output the previous match.
  394. d.tokens.AddMatch(uint32(prevLength-3), uint32(prevOffset-minOffsetSize))
  395. // Insert in the hash table all strings up to the end of the match.
  396. // index and index-1 are already inserted. If there is not enough
  397. // lookahead, the last two strings are not inserted into the hash
  398. // table.
  399. var newIndex int
  400. newIndex = s.index + prevLength - 1
  401. // Calculate missing hashes
  402. end := newIndex
  403. if end > s.maxInsertIndex {
  404. end = s.maxInsertIndex
  405. }
  406. end += minMatchLength - 1
  407. startindex := s.index + 1
  408. if startindex > s.maxInsertIndex {
  409. startindex = s.maxInsertIndex
  410. }
  411. tocheck := d.window[startindex:end]
  412. dstSize := len(tocheck) - minMatchLength + 1
  413. if dstSize > 0 {
  414. dst := s.hashMatch[:dstSize]
  415. bulkHash4(tocheck, dst)
  416. var newH uint32
  417. for i, val := range dst {
  418. di := i + startindex
  419. newH = val & hashMask
  420. // Get previous value with the same hash.
  421. // Our chain should point to the previous value.
  422. s.hashPrev[di&windowMask] = s.hashHead[newH]
  423. // Set the head of the hash chain to us.
  424. s.hashHead[newH] = uint32(di + s.hashOffset)
  425. }
  426. s.hash = newH
  427. }
  428. s.index = newIndex
  429. d.byteAvailable = false
  430. s.length = minMatchLength - 1
  431. if d.tokens.n == maxFlateBlockTokens {
  432. // The block includes the current character
  433. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  434. return
  435. }
  436. d.tokens.Reset()
  437. }
  438. } else {
  439. // Reset, if we got a match this run.
  440. if s.length >= minMatchLength {
  441. s.ii = 0
  442. }
  443. // We have a byte waiting. Emit it.
  444. if d.byteAvailable {
  445. s.ii++
  446. d.tokens.AddLiteral(d.window[s.index-1])
  447. if d.tokens.n == maxFlateBlockTokens {
  448. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  449. return
  450. }
  451. d.tokens.Reset()
  452. }
  453. s.index++
  454. // If we have a long run of no matches, skip additional bytes
  455. // Resets when s.ii overflows after 64KB.
  456. if s.ii > 31 {
  457. n := int(s.ii >> 5)
  458. for j := 0; j < n; j++ {
  459. if s.index >= d.windowEnd-1 {
  460. break
  461. }
  462. d.tokens.AddLiteral(d.window[s.index-1])
  463. if d.tokens.n == maxFlateBlockTokens {
  464. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  465. return
  466. }
  467. d.tokens.Reset()
  468. }
  469. s.index++
  470. }
  471. // Flush last byte
  472. d.tokens.AddLiteral(d.window[s.index-1])
  473. d.byteAvailable = false
  474. // s.length = minMatchLength - 1 // not needed, since s.ii is reset above, so it should never be > minMatchLength
  475. if d.tokens.n == maxFlateBlockTokens {
  476. if d.err = d.writeBlock(&d.tokens, s.index, false); d.err != nil {
  477. return
  478. }
  479. d.tokens.Reset()
  480. }
  481. }
  482. } else {
  483. s.index++
  484. d.byteAvailable = true
  485. }
  486. }
  487. }
  488. }
  489. func (d *compressor) store() {
  490. if d.windowEnd > 0 && (d.windowEnd == maxStoreBlockSize || d.sync) {
  491. d.err = d.writeStoredBlock(d.window[:d.windowEnd])
  492. d.windowEnd = 0
  493. }
  494. }
  495. // fillWindow will fill the buffer with data for huffman-only compression.
  496. // The number of bytes copied is returned.
  497. func (d *compressor) fillBlock(b []byte) int {
  498. n := copy(d.window[d.windowEnd:], b)
  499. d.windowEnd += n
  500. return n
  501. }
  502. // storeHuff will compress and store the currently added data,
  503. // if enough has been accumulated or we at the end of the stream.
  504. // Any error that occurred will be in d.err
  505. func (d *compressor) storeHuff() {
  506. if d.windowEnd < len(d.window) && !d.sync || d.windowEnd == 0 {
  507. return
  508. }
  509. d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync)
  510. d.err = d.w.err
  511. d.windowEnd = 0
  512. }
  513. // storeFast will compress and store the currently added data,
  514. // if enough has been accumulated or we at the end of the stream.
  515. // Any error that occurred will be in d.err
  516. func (d *compressor) storeFast() {
  517. // We only compress if we have maxStoreBlockSize.
  518. if d.windowEnd < len(d.window) {
  519. if !d.sync {
  520. return
  521. }
  522. // Handle extremely small sizes.
  523. if d.windowEnd < 128 {
  524. if d.windowEnd == 0 {
  525. return
  526. }
  527. if d.windowEnd <= 32 {
  528. d.err = d.writeStoredBlock(d.window[:d.windowEnd])
  529. } else {
  530. d.w.writeBlockHuff(false, d.window[:d.windowEnd], true)
  531. d.err = d.w.err
  532. }
  533. d.tokens.Reset()
  534. d.windowEnd = 0
  535. d.fast.Reset()
  536. return
  537. }
  538. }
  539. d.fast.Encode(&d.tokens, d.window[:d.windowEnd])
  540. // If we made zero matches, store the block as is.
  541. if d.tokens.n == 0 {
  542. d.err = d.writeStoredBlock(d.window[:d.windowEnd])
  543. // If we removed less than 1/16th, huffman compress the block.
  544. } else if int(d.tokens.n) > d.windowEnd-(d.windowEnd>>4) {
  545. d.w.writeBlockHuff(false, d.window[:d.windowEnd], d.sync)
  546. d.err = d.w.err
  547. } else {
  548. d.w.writeBlockDynamic(&d.tokens, false, d.window[:d.windowEnd], d.sync)
  549. d.err = d.w.err
  550. }
  551. d.tokens.Reset()
  552. d.windowEnd = 0
  553. }
  554. // write will add input byte to the stream.
  555. // Unless an error occurs all bytes will be consumed.
  556. func (d *compressor) write(b []byte) (n int, err error) {
  557. if d.err != nil {
  558. return 0, d.err
  559. }
  560. n = len(b)
  561. for len(b) > 0 {
  562. d.step(d)
  563. b = b[d.fill(d, b):]
  564. if d.err != nil {
  565. return 0, d.err
  566. }
  567. }
  568. return n, d.err
  569. }
  570. func (d *compressor) syncFlush() error {
  571. d.sync = true
  572. if d.err != nil {
  573. return d.err
  574. }
  575. d.step(d)
  576. if d.err == nil {
  577. d.w.writeStoredHeader(0, false)
  578. d.w.flush()
  579. d.err = d.w.err
  580. }
  581. d.sync = false
  582. return d.err
  583. }
  584. func (d *compressor) init(w io.Writer, level int) (err error) {
  585. d.w = newHuffmanBitWriter(w)
  586. switch {
  587. case level == NoCompression:
  588. d.window = make([]byte, maxStoreBlockSize)
  589. d.fill = (*compressor).fillBlock
  590. d.step = (*compressor).store
  591. case level == ConstantCompression:
  592. d.w.logNewTablePenalty = 4
  593. d.window = make([]byte, maxStoreBlockSize)
  594. d.fill = (*compressor).fillBlock
  595. d.step = (*compressor).storeHuff
  596. case level == DefaultCompression:
  597. level = 5
  598. fallthrough
  599. case level >= 1 && level <= 6:
  600. d.w.logNewTablePenalty = 6
  601. d.fast = newFastEnc(level)
  602. d.window = make([]byte, maxStoreBlockSize)
  603. d.fill = (*compressor).fillBlock
  604. d.step = (*compressor).storeFast
  605. case 7 <= level && level <= 9:
  606. d.w.logNewTablePenalty = 10
  607. d.state = &advancedState{}
  608. d.compressionLevel = levels[level]
  609. d.initDeflate()
  610. d.fill = (*compressor).fillDeflate
  611. d.step = (*compressor).deflateLazy
  612. default:
  613. return fmt.Errorf("flate: invalid compression level %d: want value in range [-2, 9]", level)
  614. }
  615. d.level = level
  616. return nil
  617. }
  618. // reset the state of the compressor.
  619. func (d *compressor) reset(w io.Writer) {
  620. d.w.reset(w)
  621. d.sync = false
  622. d.err = nil
  623. // We only need to reset a few things for Snappy.
  624. if d.fast != nil {
  625. d.fast.Reset()
  626. d.windowEnd = 0
  627. d.tokens.Reset()
  628. return
  629. }
  630. switch d.compressionLevel.chain {
  631. case 0:
  632. // level was NoCompression or ConstantCompresssion.
  633. d.windowEnd = 0
  634. default:
  635. s := d.state
  636. s.chainHead = -1
  637. for i := range s.hashHead {
  638. s.hashHead[i] = 0
  639. }
  640. for i := range s.hashPrev {
  641. s.hashPrev[i] = 0
  642. }
  643. s.hashOffset = 1
  644. s.index, d.windowEnd = 0, 0
  645. d.blockStart, d.byteAvailable = 0, false
  646. d.tokens.Reset()
  647. s.length = minMatchLength - 1
  648. s.offset = 0
  649. s.hash = 0
  650. s.ii = 0
  651. s.maxInsertIndex = 0
  652. }
  653. }
  654. func (d *compressor) close() error {
  655. if d.err != nil {
  656. return d.err
  657. }
  658. d.sync = true
  659. d.step(d)
  660. if d.err != nil {
  661. return d.err
  662. }
  663. if d.w.writeStoredHeader(0, true); d.w.err != nil {
  664. return d.w.err
  665. }
  666. d.w.flush()
  667. d.w.reset(nil)
  668. return d.w.err
  669. }
  670. // NewWriter returns a new Writer compressing data at the given level.
  671. // Following zlib, levels range from 1 (BestSpeed) to 9 (BestCompression);
  672. // higher levels typically run slower but compress more.
  673. // Level 0 (NoCompression) does not attempt any compression; it only adds the
  674. // necessary DEFLATE framing.
  675. // Level -1 (DefaultCompression) uses the default compression level.
  676. // Level -2 (ConstantCompression) will use Huffman compression only, giving
  677. // a very fast compression for all types of input, but sacrificing considerable
  678. // compression efficiency.
  679. //
  680. // If level is in the range [-2, 9] then the error returned will be nil.
  681. // Otherwise the error returned will be non-nil.
  682. func NewWriter(w io.Writer, level int) (*Writer, error) {
  683. var dw Writer
  684. if err := dw.d.init(w, level); err != nil {
  685. return nil, err
  686. }
  687. return &dw, nil
  688. }
  689. // NewWriterDict is like NewWriter but initializes the new
  690. // Writer with a preset dictionary. The returned Writer behaves
  691. // as if the dictionary had been written to it without producing
  692. // any compressed output. The compressed data written to w
  693. // can only be decompressed by a Reader initialized with the
  694. // same dictionary.
  695. func NewWriterDict(w io.Writer, level int, dict []byte) (*Writer, error) {
  696. zw, err := NewWriter(w, level)
  697. if err != nil {
  698. return nil, err
  699. }
  700. zw.d.fillWindow(dict)
  701. zw.dict = append(zw.dict, dict...) // duplicate dictionary for Reset method.
  702. return zw, err
  703. }
  704. // A Writer takes data written to it and writes the compressed
  705. // form of that data to an underlying writer (see NewWriter).
  706. type Writer struct {
  707. d compressor
  708. dict []byte
  709. }
  710. // Write writes data to w, which will eventually write the
  711. // compressed form of data to its underlying writer.
  712. func (w *Writer) Write(data []byte) (n int, err error) {
  713. return w.d.write(data)
  714. }
  715. // Flush flushes any pending data to the underlying writer.
  716. // It is useful mainly in compressed network protocols, to ensure that
  717. // a remote reader has enough data to reconstruct a packet.
  718. // Flush does not return until the data has been written.
  719. // Calling Flush when there is no pending data still causes the Writer
  720. // to emit a sync marker of at least 4 bytes.
  721. // If the underlying writer returns an error, Flush returns that error.
  722. //
  723. // In the terminology of the zlib library, Flush is equivalent to Z_SYNC_FLUSH.
  724. func (w *Writer) Flush() error {
  725. // For more about flushing:
  726. // http://www.bolet.org/~pornin/deflate-flush.html
  727. return w.d.syncFlush()
  728. }
  729. // Close flushes and closes the writer.
  730. func (w *Writer) Close() error {
  731. return w.d.close()
  732. }
  733. // Reset discards the writer's state and makes it equivalent to
  734. // the result of NewWriter or NewWriterDict called with dst
  735. // and w's level and dictionary.
  736. func (w *Writer) Reset(dst io.Writer) {
  737. if len(w.dict) > 0 {
  738. // w was created with NewWriterDict
  739. w.d.reset(dst)
  740. if dst != nil {
  741. w.d.fillWindow(w.dict)
  742. }
  743. } else {
  744. // w was created with NewWriter
  745. w.d.reset(dst)
  746. }
  747. }
  748. // ResetDict discards the writer's state and makes it equivalent to
  749. // the result of NewWriter or NewWriterDict called with dst
  750. // and w's level, but sets a specific dictionary.
  751. func (w *Writer) ResetDict(dst io.Writer, dict []byte) {
  752. w.dict = dict
  753. w.d.reset(dst)
  754. w.d.fillWindow(w.dict)
  755. }