Merged pull from arya: Fix compression/decompression bug when BlockDependency=true #6

lz4_test.go

@@ -585,3 +585,52 @@ func TestWrittenCountAfterBufferedWrite(t *testing.T) {
 	}
 }
 
+func TestWrittenBlocksExactlyWindowSize(t *testing.T) {
+	input := make([]byte, 128*1024)
+
+	copy(input[64*1024-1:], []byte{1, 2, 3, 4, 1, 2, 3, 4})
+
+	output := writeReadChunked(t, input, 64*1024)
+
+	if !bytes.Equal(input, output) {
+		t.Errorf("output is not equal to source input")
+		t.FailNow()
+	}
+}
+
+func TestWrittenBlocksLessThanWindowSize(t *testing.T) {
+	input := make([]byte, 80*1024)
+
+	copy(input[64*1024-1:], []byte{1, 2, 3, 4, 1, 2, 3, 4})
+	copy(input[72*1024-1:], []byte{5, 6, 7, 8, 5, 6, 7, 8})
+
+	output := writeReadChunked(t, input, 8*1024)
+	if !bytes.Equal(input, output) {
+		t.Errorf("output is not equal to source input")
+		t.FailNow()
+	}
+}
+
+func writeReadChunked(t *testing.T, in []byte, chunkSize int) []byte {
+	compressed := bytes.NewBuffer(nil)
+	w := lz4.NewWriter(compressed)
+	w.Header.BlockDependency = true
+
+	buf := bytes.NewBuffer(in)
+	for buf.Len() > 0 {
+		_, err := w.Write(buf.Next(chunkSize))
+		if err != nil {
+			t.Errorf("unexpected error: %v", err)
+			t.FailNow()
+		}
+	}
+
+	r := lz4.NewReader(compressed)
+	out := make([]byte, len(in))
+	_, err := io.ReadFull(r, out)
+	if err != nil {
+		t.Errorf("unexpected error: %v", err)
+		t.FailNow()
+	}
+	return out
+}

reader.go

@@ -191,11 +191,14 @@ func (z *Reader) Read(buf []byte) (n int, err error) {
 		// cannot decompress concurrently when dealing with block dependency
 		z.decompressBlock(zb, nil)
 		// the last block may not contain enough data
+		if len(z.window) == 0 {
+			z.window = make([]byte, winSize)
+		}
 		if len(zb.data) >= winSize {
-			if len(z.window) == 0 {
-				z.window = make([]byte, winSize)
-			}
 			copy(z.window, zb.data[len(zb.data)-winSize:])
+		} else {
+			copy(z.window, z.window[len(zb.data):])
+			copy(z.window[len(zb.data)+1:], zb.data)
 		}
 	}
 	z.wg.Wait()

writer.go

@@ -216,8 +216,11 @@ func (z *Writer) Write(buf []byte) (n int, err error) {
 			z.window = make([]byte, winSize)
 		}
 		// last buffer may be shorter than the window
-		if len(buf) > winSize {
+		if len(buf) >= winSize {
 			copy(z.window, buf[len(buf)-winSize:])
+		} else {
+			copy(z.window, z.window[len(buf):])
+			copy(z.window[len(buf)+1:], buf)
 		}
 	}