updated command line tool lz4c

fixed Writer.Reset

cmd/lz4c/compress.go

@@ -8,7 +8,7 @@ import (
 	"sync/atomic"
 
 	"code.cloudfoundry.org/bytefmt"
-	"github.com/schollz/progressbar"
+	"github.com/schollz/progressbar/v3"
 
 	"github.com/pierrec/cmdflag"
 	"github.com/pierrec/lz4"
@@ -34,13 +34,16 @@ func Compress(fs *flag.FlagSet) cmdflag.Handler {
 		}
 
 		zw := lz4.NewWriter(nil)
-		zw.Header = lz4.Header{
-			BlockChecksum:    blockChecksum,
-			BlockMaxSize:     int(sz),
-			NoChecksum:       streamChecksum,
-			CompressionLevel: level,
+		options := []lz4.Option{
+			lz4.BlockChecksumOption(blockChecksum),
+			lz4.BlockSizeOption(lz4.BlockSize(sz)),
+			lz4.ChecksumOption(streamChecksum),
+			lz4.CompressionLevelOption(lz4.CompressionLevel(level)),
+			lz4.ConcurrencyOption(concurrency),
+		}
+		if err := zw.Apply(options...); err != nil {
+			return 0, err
 		}
-		zw.WithConcurrency(concurrency)
 
 		// Use stdin/stdout if no file provided.
 		if len(args) == 0 {
@@ -71,7 +74,7 @@ func Compress(fs *flag.FlagSet) cmdflag.Handler {
 			)
 			if size > 0 {
 				// Progress bar setup.
-				numBlocks := int(size) / zw.Header.BlockMaxSize
+				numBlocks := int(size) / int(sz)
 				bar := progressbar.NewOptions(numBlocks,
 					// File transfers are usually slow, make sure we display the bar at 0%.
 					progressbar.OptionSetRenderBlankState(true),
@@ -79,9 +82,14 @@ func Compress(fs *flag.FlagSet) cmdflag.Handler {
 					progressbar.OptionSetDescription(filename),
 					progressbar.OptionClearOnFinish(),
 				)
-				zw.OnBlockDone = func(n int) {
-					_ = bar.Add(1)
-					atomic.AddInt64(&zsize, int64(n))
+				err = zw.Apply(
+					lz4.OnBlockDoneOption(func(n int) {
+						_ = bar.Add(1)
+						atomic.AddInt64(&zsize, int64(n))
+					}),
+				)
+				if err != nil {
+					return 0, err
 				}
 			}
 

cmd/lz4c/uncompress.go

@@ -7,7 +7,7 @@ import (
 	"os"
 	"strings"
 
-	"github.com/schollz/progressbar"
+	"github.com/schollz/progressbar/v3"
 
 	"github.com/pierrec/cmdflag"
 	"github.com/pierrec/lz4"
@@ -64,9 +64,11 @@ func Uncompress(_ *flag.FlagSet) cmdflag.Handler {
 					progressbar.OptionClearOnFinish(),
 				)
 				out = io.MultiWriter(out, bar)
-				zr.OnBlockDone = func(n int) {
-					size += n
-				}
+				_ = zr.Apply(
+					lz4.OnBlockDoneOption(func(n int) {
+						size += n
+					}),
+				)
 			}
 
 			// Uncompress.

go.mod

@@ -4,10 +4,12 @@ go 1.14
 
 require (
 	code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48
-	github.com/frankban/quicktest v1.9.0 // indirect
-	github.com/google/go-cmp v0.4.0
+	github.com/frankban/quicktest v1.9.0
+	github.com/onsi/ginkgo v1.12.0 // indirect
+	github.com/onsi/gomega v1.9.0 // indirect
 	github.com/pierrec/cmdflag v0.0.2
-	github.com/pierrec/packer v0.0.0-20200419211718-decbba9fa6fa // indirect
 	github.com/schollz/progressbar v1.0.0
-	golang.org/x/tools v0.0.0-20200420001825-978e26b7c37c // indirect
+	github.com/schollz/progressbar/v3 v3.1.1
+	golang.org/x/net v0.0.0-20200226121028-0de0cce0169b // indirect
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e // indirect
 )

go.sum

@@ -1,37 +1,63 @@
 code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48 h1:/EMHruHCFXR9xClkGV/t0rmHrdhX4+trQUcBqjwc9xE=
 code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/frankban/quicktest v1.9.0 h1:jfEA+Psfr/pHsRJYPpHiNu7PGJnGctNxvTaM3K1EyXk=
 github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
+github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
 github.com/pierrec/cmdflag v0.0.2 h1:ybjGJnPr/aURn2IKWjO49znx9N0DL6YfGsIxN0PYuVY=
 github.com/pierrec/cmdflag v0.0.2/go.mod h1:a3zKGZ3cdQUfxjd0RGMLZr8xI3nvpJOB+m6o/1X5BmU=
-github.com/pierrec/packer v0.0.0-20200419211718-decbba9fa6fa h1:yVA4oBytvOmjYjiQzeJL83TlMb/g2y7WbnrspRVtPMk=
-github.com/pierrec/packer v0.0.0-20200419211718-decbba9fa6fa/go.mod h1:GKrs5lzNeoNBN0l+jHHePFsqiNgXrZT5vQvGt28rWjI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM=
 github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs=
-github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/schollz/progressbar/v3 v3.1.1 h1:IsKbXcalNftS2i4KUTkf/LTcMIrm1RUoefVaJQaizYw=
+github.com/schollz/progressbar/v3 v3.1.1/go.mod h1:d+PD64vPuv+GL2EhUpvV579FR91WWhRHEnImqGsIBU4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
-golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b h1:0mm1VjtFUOIlE1SbDlwjYaDxZVDP2S5ou6y0gSgXHu8=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e h1:N7DeIrjYszNmSW409R3frPPwglRwMkXSBzwVbkOjLLA=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20200420001825-978e26b7c37c h1:JzwTM5XxGxiCwZEIZQPG46csyhWQxQlu2uSi3bEza34=
-golang.org/x/tools v0.0.0-20200420001825-978e26b7c37c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

internal/lz4block/block_test.go

@@ -103,7 +103,7 @@ func TestCompressUncompressBlock(t *testing.T) {
 			})
 			t.Run(fmt.Sprintf("%s HC", tc.file), func(t *testing.T) {
 				nhc = run(t, tc, func(src, dst []byte) (int, error) {
-					return lz4.CompressBlockHC(src, dst, 10, nil)
+					return lz4.CompressBlockHC(src, dst, 10)
 				})
 			})
 		})

lz4.go

@@ -1,3 +1,13 @@
+// Package lz4 implements reading and writing lz4 compressed data (a frame),
+// as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html.
+//
+// Although the block level compression and decompression functions are exposed and are fully compatible
+// with the lz4 block format definition, they are low level and should not be used directly.
+// For a complete description of an lz4 compressed block, see:
+// http://fastcompression.blogspot.fr/2011/05/lz4-explained.html
+//
+// See https://github.com/lz4/lz4 for the reference C implementation.
+//
 package lz4
 
 import (
@@ -43,7 +53,7 @@ func UncompressBlock(src, dst []byte) (int, error) {
 // the compressed size is 0 and no error, then the data is incompressible.
 //
 // An error is returned if the destination buffer is too small.
-func CompressBlock(src, dst []byte, hashTable []int) (_ int, err error) {
+func CompressBlock(src, dst []byte, hashTable []int) (int, error) {
 	return lz4block.CompressBlock(src, dst, hashTable)
 }
 
@@ -58,7 +68,7 @@ func CompressBlock(src, dst []byte, hashTable []int) (_ int, err error) {
 // the compressed size is 0 and no error, then the data is incompressible.
 //
 // An error is returned if the destination buffer is too small.
-func CompressBlockHC(src, dst []byte, depth CompressionLevel, hashTable []int) (_ int, err error) {
+func CompressBlockHC(src, dst []byte, depth CompressionLevel) (int, error) {
 	return lz4block.CompressBlockHC(src, dst, lz4block.CompressionLevel(depth))
 }
 

options.go

@@ -20,6 +20,7 @@ type (
 	Option func(applier) error
 )
 
+// String returns a string representation of the option with its parameter(s).
 func (o Option) String() string {
 	return o(nil).Error()
 }
@@ -76,7 +77,7 @@ func BlockChecksumOption(flag bool) Option {
 	}
 }
 
-// ChecksumOption enables/disables all blocks checksum (default=true).
+// ChecksumOption enables/disables all blocks or content checksum (default=true).
 func ChecksumOption(flag bool) Option {
 	return func(a applier) error {
 		switch w := a.(type) {
@@ -91,7 +92,8 @@ func ChecksumOption(flag bool) Option {
 	}
 }
 
-// SizeOption sets the size of the original uncompressed data (default=0).
+// SizeOption sets the size of the original uncompressed data (default=0). It is useful to know the size of the
+// whole uncompressed data stream.
 func SizeOption(size uint64) Option {
 	return func(a applier) error {
 		switch w := a.(type) {
@@ -168,7 +170,8 @@ func CompressionLevelOption(level CompressionLevel) Option {
 
 func onBlockDone(int) {}
 
-// OnBlockDoneOption is triggered
+// OnBlockDoneOption is triggered when a block has been processed. For a Writer, it is when is has been compressed,
+// for a Reader, it is when it has been uncompressed.
 func OnBlockDoneOption(handler func(size int)) Option {
 	if handler == nil {
 		handler = onBlockDone

reader.go

@@ -23,6 +23,7 @@ func NewReader(r io.Reader) *Reader {
 	return zr.Reset(r)
 }
 
+// Reader allows reading an LZ4 stream.
 type Reader struct {
 	state   _State
 	src     io.Reader        // source reader

writer.go

@@ -24,6 +24,7 @@ func NewWriter(w io.Writer) *Writer {
 	return zw.Reset(w)
 }
 
+// Writer allows writing an LZ4 stream.
 type Writer struct {
 	state   _State
 	src     io.Writer                 // destination writer
@@ -67,6 +68,13 @@ func (w *Writer) Write(buf []byte) (n int, err error) {
 	case closedState, errorState:
 		return 0, w.state.err
 	case newState:
+		w.frame.InitW(w.src, w.num)
+		size := w.frame.Descriptor.Flags.BlockSizeIndex()
+		w.data = size.Get()
+		w.idx = 0
+		if w.isNotConcurrent() {
+			w.ht = lz4block.HashTablePool.Get().([]int)
+		}
 		if err = w.frame.Descriptor.Write(w.frame, w.src); w.state.next(err) {
 			return
 		}
@@ -111,9 +119,10 @@ func (w *Writer) Write(buf []byte) (n int, err error) {
 
 func (w *Writer) write(data []byte, safe bool) error {
 	if w.isNotConcurrent() {
-		defer w.handler(len(data))
 		block := w.frame.Blocks.Block
-		return block.Compress(w.frame, data, w.ht, w.level).Write(w.frame, w.src)
+		err := block.Compress(w.frame, data, w.ht, w.level).Write(w.frame, w.src)
+		w.handler(len(block.Data))
+		return err
 	}
 	size := w.frame.Descriptor.Flags.BlockSizeIndex()
 	c := make(chan *lz4stream.FrameDataBlock)
@@ -122,7 +131,7 @@ func (w *Writer) write(data []byte, safe bool) error {
 		b := lz4stream.NewFrameDataBlock(size)
 		c <- b.Compress(w.frame, data, nil, w.level)
 		<-c
-		w.handler(len(data))
+		w.handler(len(b.Data))
 		b.CloseW(w.frame)
 		if safe {
 			// safe to put it back as the last usage of it was FrameDataBlock.Write() called before c is closed
@@ -154,6 +163,7 @@ func (w *Writer) Close() (err error) {
 	err = w.frame.CloseW(w.src, w.num)
 	if w.isNotConcurrent() {
 		lz4block.HashTablePool.Put(w.ht)
+		w.ht = nil
 	}
 	// It is now safe to free the buffer.
 	size := w.frame.Descriptor.Flags.BlockSizeIndex()
@@ -177,12 +187,5 @@ func (w *Writer) Reset(writer io.Writer) *Writer {
 	w.state.state = noState
 	w.state.next(nil)
 	w.src = writer
-	w.frame.InitW(w.src, w.num)
-	size := w.frame.Descriptor.Flags.BlockSizeIndex()
-	w.data = size.Get()
-	w.idx = 0
-	if w.isNotConcurrent() {
-		w.ht = lz4block.HashTablePool.Get().([]int)
-	}
 	return w
 }