minor cleanups

block.go

@@ -1,6 +1,9 @@
 package lz4
 
-import "errors"
+import (
+	"encoding/binary"
+	"errors"
+)
 
 // block represents a frame data block.
 // Used when compressing or decompressing frame blocks concurrently.
@@ -128,22 +131,32 @@ func CompressBlock(src, dst []byte, soffset int) (int, error) {
 	// Initialise the hash table with the first 64Kb of the input buffer
 	// (used when compressing dependent blocks)
 	for si < soffset {
-		h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+		h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 		si++
 		hashTable[h] = si
 	}
 
 	anchor := si
-	fma := 1<<skipStrength + 3
+	fma := 1 << skipStrength
 	for si < sn-minMatch {
 		// hash the next 4 bytes (sequence)...
-		h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+		h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 		// -1 to separate existing entries from new ones
 		ref := hashTable[h] - 1
 		// ...and store the position of the hash in the hash table (+1 to compensate the -1 upon saving)
 		hashTable[h] = si + 1
+		// no need to check the last 3 bytes in the first literal 4 bytes as
+		// this guarantees that the next match, if any, is compressed with
+		// a lower size, since to have some compression we must have:
+		// ll+ml-overlap > 1 + (ll-15)/255 + (ml-4-15)/255 + 2 (uncompressed size>compressed size)
+		// => ll+ml>3+2*overlap => ll+ml>= 4+2*overlap
+		// and by definition we do have:
+		// ll >= 1, ml >= 4
+		// => ll+ml >= 5
+		// => so overlap must be 0
+
 		// the sequence is new, out of bound (64kb) or not valid: try next sequence
-		if ref < 0 ||
+		if ref < 0 || fma&(1<<skipStrength-1) < 4 ||
 			(si-ref)>>winSizeLog > 0 ||
 			src[ref] != src[si] ||
 			src[ref+1] != src[si+1] ||
@@ -155,7 +168,7 @@ func CompressBlock(src, dst []byte, soffset int) (int, error) {
 			continue
 		}
 		// match found
-		fma = 1<<skipStrength + 3
+		fma = 1 << skipStrength
 		lLen := si - anchor
 		offset := si - ref
 
@@ -284,7 +297,7 @@ func CompressBlockHC(src, dst []byte, soffset int) (int, error) {
 	// Initialise the hash table with the first 64Kb of the input buffer
 	// (used when compressing dependent blocks)
 	for si < soffset {
-		h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+		h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 		chainTable[si&winMask] = hashTable[h]
 		si++
 		hashTable[h] = si
@@ -293,7 +306,7 @@ func CompressBlockHC(src, dst []byte, soffset int) (int, error) {
 	anchor := si
 	for si < sn-minMatch {
 		// hash the next 4 bytes (sequence)...
-		h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+		h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 
 		// follow the chain until out of window and give the longest match
 		mLen := 0
@@ -326,7 +339,7 @@ func CompressBlockHC(src, dst []byte, soffset int) (int, error) {
 		// update hash/chain tables with overlaping bytes:
 		// si already hashed, add everything from si+1 up to the match length
 		for si, ml := si+1, si+mLen; si < ml; {
-			h := (uint32(src[si+3])<<24 | uint32(src[si+2])<<16 | uint32(src[si+1])<<8 | uint32(src[si])) * hasher >> hashShift
+			h := binary.LittleEndian.Uint32(src[si:]) * hasher >> hashShift
 			chainTable[si&winMask] = hashTable[h]
 			si++
 			hashTable[h] = si

lz4.go

@@ -43,7 +43,8 @@ const (
 	// Its value influences the compression speed and memory usage, the lower the faster,
 	// but at the expense of the compression ratio.
 	// 16 seems to be the best compromise.
-	hashLog = 16
+	hashLog   = 16
+	hashShift = uint((minMatch * 8) - hashLog)
 
 	mfLimit      = 8 + minMatch // The last match cannot start within the last 12 bytes.
 	skipStrength = 6            // variable step for fast scan

lz4_test.go

@@ -275,30 +275,32 @@ func BenchmarkUncompressBlock(b *testing.B) {
 }
 
 func BenchmarkCompressBlock(b *testing.B) {
-	d := make([]byte, len(lorem))
+	d := append([]byte{}, lorem...)
 	z := make([]byte, len(lorem))
-	n, err := lz4.CompressBlock(lorem, z, 0)
+	n, err := lz4.CompressBlock(d, z, 0)
 	if err != nil {
 		b.Errorf("CompressBlock: %s", err)
 		b.FailNow()
 	}
 	z = z[:n]
 	for i := 0; i < b.N; i++ {
-		lz4.CompressBlock(z, d, 0)
+		d = append([]byte{}, lorem...)
+		lz4.CompressBlock(d, z, 0)
 	}
 }
 
 func BenchmarkCompressBlockHC(b *testing.B) {
-	d := make([]byte, len(lorem))
+	d := append([]byte{}, lorem...)
 	z := make([]byte, len(lorem))
-	n, err := lz4.CompressBlockHC(lorem, z, 0)
+	n, err := lz4.CompressBlockHC(d, z, 0)
 	if err != nil {
 		b.Errorf("CompressBlock: %s", err)
 		b.FailNow()
 	}
 	z = z[:n]
 	for i := 0; i < b.N; i++ {
-		lz4.CompressBlockHC(z, d, 0)
+		d = append([]byte{}, lorem...)
+		lz4.CompressBlockHC(d, z, 0)
 	}
 }
 
@@ -582,3 +584,4 @@ func TestWrittenCountAfterBufferedWrite(t *testing.T) {
 		t.FailNow()
 	}
 }
+

reader.go

@@ -12,6 +12,10 @@ import (
 	"sync/atomic"
 )
 
+// ErrInvalid is returned when the data being read is not an LZ4 archive
+// (LZ4 magic number detection failed).
+var ErrInvalid = errors.New("invalid lz4 data")
+
 // errEndOfBlock is returned by readBlock when it has reached the last block of the frame.
 // It is not an error.
 var errEndOfBlock = errors.New("end of block")
@@ -67,7 +71,7 @@ func (z *Reader) readHeader(first bool) error {
 			continue
 		}
 		if magic != frameMagic {
-			return fmt.Errorf("lz4.Read: invalid frame magic number: got %x expected %x", magic, frameMagic)
+			return ErrInvalid
 		}
 		break
 	}