some cleanups

block.go

@@ -1,6 +1,9 @@
 package lz4
 
-import "errors"
+import (
+	"encoding/binary"
+	"errors"
+)
 
 // block represents a frame data block.
 // Used when compressing or decompressing frame blocks concurrently.
@@ -128,22 +131,32 @@ func CompressBlock(src, dst []byte, soffset int) (int, error) {
 	// Initialise the hash table with the first 64Kb of the input buffer
 	// (used when compressing dependent blocks)
 	for si < soffset {
-		h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+		h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 		si++
 		hashTable[h] = si
 	}
 
 	anchor := si
-	fma := 1<<skipStrength + 3
+	fma := 1 << skipStrength
 	for si < sn-minMatch {
 		// hash the next 4 bytes (sequence)...
-		h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+		h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 		// -1 to separate existing entries from new ones
 		ref := hashTable[h] - 1
 		// ...and store the position of the hash in the hash table (+1 to compensate the -1 upon saving)
 		hashTable[h] = si + 1
+		// no need to check the last 3 bytes in the first literal 4 bytes as
+		// this guarantees that the next match, if any, is compressed with
+		// a lower size, since to have some compression we must have:
+		// ll+ml-overlap > 1 + (ll-15)/255 + (ml-4-15)/255 + 2 (uncompressed size>compressed size)
+		// => ll+ml>3+2*overlap => ll+ml>= 4+2*overlap
+		// and by definition we do have:
+		// ll >= 1, ml >= 4
+		// => ll+ml >= 5
+		// => so overlap must be 0
+
 		// the sequence is new, out of bound (64kb) or not valid: try next sequence
-		if ref < 0 ||
+		if ref < 0 || fma&(1<<skipStrength-1) < 4 ||
 			(si-ref)>>winSizeLog > 0 ||
 			src[ref] != src[si] ||
 			src[ref+1] != src[si+1] ||
@@ -155,7 +168,7 @@ func CompressBlock(src, dst []byte, soffset int) (int, error) {
 			continue
 		}
 		// match found
-		fma = 1<<skipStrength + 3
+		fma = 1 << skipStrength
 		lLen := si - anchor
 		offset := si - ref
 
@@ -284,7 +297,7 @@ func CompressBlockHC(src, dst []byte, soffset int) (int, error) {
 	// Initialise the hash table with the first 64Kb of the input buffer
 	// (used when compressing dependent blocks)
 	for si < soffset {
-		h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+		h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 		chainTable[si&winMask] = hashTable[h]
 		si++
 		hashTable[h] = si
@@ -293,7 +306,7 @@ func CompressBlockHC(src, dst []byte, soffset int) (int, error) {
 	anchor := si
 	for si < sn-minMatch {
 		// hash the next 4 bytes (sequence)...
-		h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+		h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 
 		// follow the chain until out of window and give the longest match
 		mLen := 0
@@ -326,7 +339,7 @@ func CompressBlockHC(src, dst []byte, soffset int) (int, error) {
 		// update hash/chain tables with overlaping bytes:
 		// si already hashed, add everything from si+1 up to the match length
 		for si, ml := si+1, si+mLen; si < ml; {
-			h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+			h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 			chainTable[si&winMask] = hashTable[h]
 			si++
 			hashTable[h] = si

lz4.go

@@ -43,7 +43,8 @@ const (
 	// Its value influences the compression speed and memory usage, the lower the faster,
 	// but at the expense of the compression ratio.
 	// 16 seems to be the best compromise.
-	hashLog = 16
+	hashLog   = 16
+	hashShift = uint((minMatch * 8) - hashLog)
 
 	mfLimit      = 8 + minMatch // The last match cannot start within the last 12 bytes.
 	skipStrength = 6            // variable step for fast scan

lz4_test.go

@@ -225,6 +225,30 @@ func init() {
 	}
 }
 
+func TestBlockIn(t *testing.T) {
+	for _, item := range testDataItems {
+		data := item.data
+		var z []byte
+		z = append(z, data...)
+		if !lz4.CompressInBlock(&z) {
+			// not compressible
+			continue
+		}
+		// fmt.Printf(">> %d z=%v\n", len(z), z)
+		d := make([]byte, len(data))
+		n, err := lz4.UncompressBlock(z, d, 0)
+		if err != nil {
+			t.Errorf("CompressInBlock: UncompressBlock: %s at %d", err, n)
+			t.FailNow()
+		}
+		d = d[:n]
+		if !bytes.Equal(d, data) {
+			t.Errorf("CompressInBlock: invalid decompressed data: %s: %s", item.label, string(d))
+			t.FailNow()
+		}
+	}
+}
+
 // Test low levels core functions:
 // a. compress and compare with supplied data if any
 // b. decompress the previous data and compare it with the original one
@@ -274,31 +298,45 @@ func BenchmarkUncompressBlock(b *testing.B) {
 	}
 }
 
+func BenchmarkCompressInBlock(b *testing.B) {
+	d := append([]byte{}, lorem...)
+	if !lz4.CompressInBlock(&d) {
+		b.Errorf("CompressInBlock: not compressible")
+		b.FailNow()
+	}
+	for i := 0; i < b.N; i++ {
+		d = append([]byte{}, lorem...)
+		lz4.CompressInBlock(&d)
+	}
+}
+
 func BenchmarkCompressBlock(b *testing.B) {
-	d := make([]byte, len(lorem))
+	d := append([]byte{}, lorem...)
 	z := make([]byte, len(lorem))
-	n, err := lz4.CompressBlock(lorem, z, 0)
+	n, err := lz4.CompressBlock(d, z, 0)
 	if err != nil {
 		b.Errorf("CompressBlock: %s", err)
 		b.FailNow()
 	}
 	z = z[:n]
 	for i := 0; i < b.N; i++ {
-		lz4.CompressBlock(z, d, 0)
+		d = append([]byte{}, lorem...)
+		lz4.CompressBlock(d, z, 0)
 	}
 }
 
 func BenchmarkCompressBlockHC(b *testing.B) {
-	d := make([]byte, len(lorem))
+	d := append([]byte{}, lorem...)
 	z := make([]byte, len(lorem))
-	n, err := lz4.CompressBlockHC(lorem, z, 0)
+	n, err := lz4.CompressBlockHC(d, z, 0)
 	if err != nil {
 		b.Errorf("CompressBlock: %s", err)
 		b.FailNow()
 	}
 	z = z[:n]
 	for i := 0; i < b.N; i++ {
-		lz4.CompressBlockHC(z, d, 0)
+		d = append([]byte{}, lorem...)
+		lz4.CompressBlockHC(d, z, 0)
 	}
 }
 
@@ -565,3 +603,21 @@ func TestSkippable(t *testing.T) {
 	}
 
 }
+
+func TestWrittenCountAfterBufferedWrite(t *testing.T) {
+	w := lz4.NewWriter(bytes.NewBuffer(nil))
+	w.Header.BlockDependency = true
+
+	if n, _ := w.Write([]byte{1}); n != 1 {
+		t.Errorf("expected to write 1 byte, wrote %d", n)
+		t.FailNow()
+	}
+
+	forcesWrite := make([]byte, 1<<16)
+
+	if n, _ := w.Write(forcesWrite); n != len(forcesWrite) {
+		t.Errorf("expected to write %d bytes, wrote %d", len(forcesWrite), n)
+		t.FailNow()
+	}
+}
+

reader.go

@@ -12,6 +12,10 @@ import (
 	"sync/atomic"
 )
 
+// ErrInvalid is returned when the data being read is not an LZ4 archive
+// (LZ4 magic number detection failed).
+var ErrInvalid = errors.New("invalid lz4 data")
+
 // errEndOfBlock is returned by readBlock when it has reached the last block of the frame.
 // It is not an error.
 var errEndOfBlock = errors.New("end of block")
@@ -67,7 +71,7 @@ func (z *Reader) readHeader(first bool) error {
 			continue
 		}
 		if magic != frameMagic {
-			return fmt.Errorf("lz4.Read: invalid frame magic number: got %x expected %x", magic, frameMagic)
+			return ErrInvalid
 		}
 		break
 	}