compress/zstd/encoder_test.go

// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
// Based on work by Yann Collet, released under BSD License.

package zstd

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"runtime"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/klauspost/compress/zip"
	"github.com/klauspost/compress/zstd/internal/xxhash"
)

var testWindowSizes = []int{MinWindowSize, 1 << 16, 1 << 22, 1 << 24}

func TestEncoder_EncodeAllSimple(t *testing.T) {
	in, err := ioutil.ReadFile("testdata/z000028")
	if err != nil {
		t.Fatal(err)
	}
	dec, err := NewReader(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()

	in = append(in, in...)
	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		t.Run(level.String(), func(t *testing.T) {
			e, err := NewWriter(nil, WithEncoderLevel(level), WithEncoderConcurrency(2), WithWindowSize(128<<10), WithZeroFrames(true))
			if err != nil {
				t.Fatal(err)
			}
			defer e.Close()
			start := time.Now()
			dst := e.EncodeAll(in, nil)
			t.Log("Simple Encoder len", len(in), "-> zstd len", len(dst))
			mbpersec := (float64(len(in)) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second)))
			t.Logf("Encoded %d bytes with %.2f MB/s", len(in), mbpersec)

			decoded, err := dec.DecodeAll(dst, nil)
			if err != nil {
				t.Error(err, len(decoded))
			}
			if !bytes.Equal(decoded, in) {
				ioutil.WriteFile("testdata/"+t.Name()+"-z000028.got", decoded, os.ModePerm)
				ioutil.WriteFile("testdata/"+t.Name()+"-z000028.want", in, os.ModePerm)
				t.Fatal("Decoded does not match")
			}
			t.Log("Encoded content matched")
		})
	}
}

func TestEncoder_EncodeAllConcurrent(t *testing.T) {
	in, err := ioutil.ReadFile("testdata/z000028")
	if err != nil {
		t.Fatal(err)
	}
	in = append(in, in...)

	// When running race no more than 8k goroutines allowed.
	n := 4000 / runtime.GOMAXPROCS(0)
	if testing.Short() {
		n = 200 / runtime.GOMAXPROCS(0)
	}
	dec, err := NewReader(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()
	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		t.Run(level.String(), func(t *testing.T) {
			rng := rand.New(rand.NewSource(0x1337))
			e, err := NewWriter(nil, WithEncoderLevel(level), WithZeroFrames(true))
			if err != nil {
				t.Fatal(err)
			}
			defer e.Close()
			var wg sync.WaitGroup
			wg.Add(n)
			for i := 0; i < n; i++ {
				in := in[rng.Int()&1023:]
				in = in[:rng.Intn(len(in))]
				go func() {
					defer wg.Done()
					dst := e.EncodeAll(in, nil)
					//t.Log("Simple Encoder len", len(in), "-> zstd len", len(dst))
					decoded, err := dec.DecodeAll(dst, nil)
					if err != nil {
						t.Error(err, len(decoded))
					}
					if !bytes.Equal(decoded, in) {
						//ioutil.WriteFile("testdata/"+t.Name()+"-z000028.got", decoded, os.ModePerm)
						//ioutil.WriteFile("testdata/"+t.Name()+"-z000028.want", in, os.ModePerm)
						t.Fatal("Decoded does not match")
					}
				}()
			}
			wg.Wait()
			t.Log("Encoded content matched.", n, "goroutines")
		})
	}
}

func TestEncoder_EncodeAllEncodeXML(t *testing.T) {
	f, err := os.Open("testdata/xml.zst")
	if err != nil {
		t.Fatal(err)
	}
	dec, err := NewReader(f)
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()
	in, err := ioutil.ReadAll(dec)
	if err != nil {
		t.Fatal(err)
	}

	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		t.Run(level.String(), func(t *testing.T) {
			e, err := NewWriter(nil, WithEncoderLevel(level))
			if err != nil {
				t.Fatal(err)
			}
			defer e.Close()
			start := time.Now()
			dst := e.EncodeAll(in, nil)
			t.Log("Simple Encoder len", len(in), "-> zstd len", len(dst))
			mbpersec := (float64(len(in)) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second)))
			t.Logf("Encoded %d bytes with %.2f MB/s", len(in), mbpersec)

			decoded, err := dec.DecodeAll(dst, nil)
			if err != nil {
				t.Error(err, len(decoded))
			}
			if !bytes.Equal(decoded, in) {
				ioutil.WriteFile("testdata/"+t.Name()+"-xml.got", decoded, os.ModePerm)
				t.Fatal("Decoded does not match")
			}
			t.Log("Encoded content matched")
		})
	}
}

func TestEncoderRegression(t *testing.T) {
	defer timeout(2 * time.Minute)()
	data, err := ioutil.ReadFile("testdata/comp-crashers.zip")
	if err != nil {
		t.Fatal(err)
	}
	// We can't close the decoder.
	dec, err := NewReader(nil)
	if err != nil {
		t.Error(err)
		return
	}
	defer dec.Close()
	testWindowSizes := testWindowSizes
	if testing.Short() {
		testWindowSizes = []int{1 << 20}
	}
	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		t.Run(level.String(), func(t *testing.T) {
			for _, windowSize := range testWindowSizes {
				t.Run(fmt.Sprintf("window:%d", windowSize), func(t *testing.T) {
					zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
					if err != nil {
						t.Fatal(err)
					}
					enc, err := NewWriter(
						nil,
						WithEncoderCRC(true),
						WithEncoderLevel(level),
						WithWindowSize(windowSize),
					)
					if err != nil {
						t.Fatal(err)
					}
					defer enc.Close()

					for i, tt := range zr.File {
						if !strings.HasSuffix(t.Name(), "") {
							continue
						}
						if testing.Short() && i > 100 {
							break
						}

						t.Run(tt.Name, func(t *testing.T) {
							r, err := tt.Open()
							if err != nil {
								t.Error(err)
								return
							}
							in, err := ioutil.ReadAll(r)
							if err != nil {
								t.Error(err)
							}
							encoded := enc.EncodeAll(in, nil)
							got, err := dec.DecodeAll(encoded, nil)
							if err != nil {
								t.Logf("error: %v\nwant: %v\ngot:  %v", err, len(in), len(got))
								t.Fatal(err)
							}

							// Use the Writer
							var dst bytes.Buffer
							enc.Reset(&dst)
							_, err = enc.Write(in)
							if err != nil {
								t.Error(err)
							}
							err = enc.Close()
							if err != nil {
								t.Error(err)
							}
							encoded = dst.Bytes()
							got, err = dec.DecodeAll(encoded, nil)
							if err != nil {
								t.Logf("error: %v\nwant: %v\ngot:  %v", err, in, got)
								t.Fatal(err)
							}

						})
					}
				})
			}
		})
	}
}

func TestEncoder_EncodeAllTwain(t *testing.T) {
	in, err := ioutil.ReadFile("../testdata/Mark.Twain-Tom.Sawyer.txt")
	if err != nil {
		t.Fatal(err)
	}
	testWindowSizes := testWindowSizes
	if testing.Short() {
		testWindowSizes = []int{1 << 20}
	}

	dec, err := NewReader(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()

	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		t.Run(level.String(), func(t *testing.T) {
			for _, windowSize := range testWindowSizes {
				t.Run(fmt.Sprintf("window:%d", windowSize), func(t *testing.T) {
					e, err := NewWriter(nil, WithEncoderLevel(level), WithWindowSize(windowSize))
					if err != nil {
						t.Fatal(err)
					}
					defer e.Close()
					start := time.Now()
					dst := e.EncodeAll(in, nil)
					t.Log("Simple Encoder len", len(in), "-> zstd len", len(dst))
					mbpersec := (float64(len(in)) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second)))
					t.Logf("Encoded %d bytes with %.2f MB/s", len(in), mbpersec)

					decoded, err := dec.DecodeAll(dst, nil)
					if err != nil {
						t.Error(err, len(decoded))
					}
					if !bytes.Equal(decoded, in) {
						ioutil.WriteFile("testdata/"+t.Name()+"-Mark.Twain-Tom.Sawyer.txt.got", decoded, os.ModePerm)
						t.Fatal("Decoded does not match")
					}
					t.Log("Encoded content matched")
				})
			}
		})
	}
}

func TestEncoder_EncodeAllPi(t *testing.T) {
	in, err := ioutil.ReadFile("../testdata/pi.txt")
	if err != nil {
		t.Fatal(err)
	}
	testWindowSizes := testWindowSizes
	if testing.Short() {
		testWindowSizes = []int{1 << 20}
	}

	dec, err := NewReader(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()

	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		t.Run(level.String(), func(t *testing.T) {
			for _, windowSize := range testWindowSizes {
				t.Run(fmt.Sprintf("window:%d", windowSize), func(t *testing.T) {
					e, err := NewWriter(nil, WithEncoderLevel(level), WithWindowSize(windowSize))
					if err != nil {
						t.Fatal(err)
					}
					defer e.Close()
					start := time.Now()
					dst := e.EncodeAll(in, nil)
					t.Log("Simple Encoder len", len(in), "-> zstd len", len(dst))
					mbpersec := (float64(len(in)) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second)))
					t.Logf("Encoded %d bytes with %.2f MB/s", len(in), mbpersec)

					decoded, err := dec.DecodeAll(dst, nil)
					if err != nil {
						t.Error(err, len(decoded))
					}
					if !bytes.Equal(decoded, in) {
						ioutil.WriteFile("testdata/"+t.Name()+"-pi.txt.got", decoded, os.ModePerm)
						t.Fatal("Decoded does not match")
					}
					t.Log("Encoded content matched")
				})
			}
		})
	}
}

func TestWithEncoderPadding(t *testing.T) {
	n := 100
	if testing.Short() {
		n = 5
	}
	rng := rand.New(rand.NewSource(0x1337))
	d, err := NewReader(nil)
	if err != nil {
		t.Fatal(err)
	}
	defer d.Close()

	for i := 0; i < n; i++ {
		padding := (rng.Int() & 0xfff) + 1
		src := make([]byte, (rng.Int()&0xfffff)+1)
		for i := range src {
			src[i] = uint8(rng.Uint32()) & 7
		}
		e, err := NewWriter(nil, WithEncoderPadding(padding), WithEncoderCRC(rng.Uint32()&1 == 0))
		if err != nil {
			t.Fatal(err)
		}
		// Test the added padding is invisible.
		dst := e.EncodeAll(src, nil)
		if len(dst)%padding != 0 {
			t.Fatalf("wanted size to be mutiple of %d, got size %d with remainder %d", padding, len(dst), len(dst)%padding)
		}
		got, err := d.DecodeAll(dst, nil)
		if err != nil {
			t.Fatal(err)
		}
		if !bytes.Equal(src, got) {
			t.Fatal("output mismatch")
		}
		// Test when we supply data as well.
		dst = e.EncodeAll(src, make([]byte, rng.Int()&255))
		if len(dst)%padding != 0 {
			t.Fatalf("wanted size to be mutiple of %d, got size %d with remainder %d", padding, len(dst), len(dst)%padding)
		}

		// Test using the writer.
		var buf bytes.Buffer
		e.Reset(&buf)
		_, err = io.Copy(e, bytes.NewBuffer(src))
		if err != nil {
			t.Fatal(err)
		}
		err = e.Close()
		if err != nil {
			t.Fatal(err)
		}
		dst = buf.Bytes()
		if len(dst)%padding != 0 {
			t.Fatalf("wanted size to be mutiple of %d, got size %d with remainder %d", padding, len(dst), len(dst)%padding)
		}
		// Test the added padding is invisible.
		got, err = d.DecodeAll(dst, nil)
		if err != nil {
			t.Fatal(err)
		}
		if !bytes.Equal(src, got) {
			t.Fatal("output mismatch")
		}
		// Try after reset
		buf.Reset()
		e.Reset(&buf)
		_, err = io.Copy(e, bytes.NewBuffer(src))
		if err != nil {
			t.Fatal(err)
		}
		err = e.Close()
		if err != nil {
			t.Fatal(err)
		}
		dst = buf.Bytes()
		if len(dst)%padding != 0 {
			t.Fatalf("wanted size to be mutiple of %d, got size %d with remainder %d", padding, len(dst), len(dst)%padding)
		}
		// Test the added padding is invisible.
		got, err = d.DecodeAll(dst, nil)
		if err != nil {
			t.Fatal(err)
		}
		if !bytes.Equal(src, got) {
			t.Fatal("output mismatch")
		}
	}
}
func TestEncoder_EncoderXML(t *testing.T) {
	testEncoderRoundtrip(t, "./testdata/xml.zst", []byte{0x56, 0x54, 0x69, 0x8e, 0x40, 0x50, 0x11, 0xe})
	testEncoderRoundtripWriter(t, "./testdata/xml.zst", []byte{0x56, 0x54, 0x69, 0x8e, 0x40, 0x50, 0x11, 0xe})
}

func TestEncoder_EncoderTwain(t *testing.T) {
	testEncoderRoundtrip(t, "../testdata/Mark.Twain-Tom.Sawyer.txt", []byte{0x12, 0x1f, 0x12, 0x70, 0x79, 0x37, 0x1f, 0xc6})
	testEncoderRoundtripWriter(t, "../testdata/Mark.Twain-Tom.Sawyer.txt", []byte{0x12, 0x1f, 0x12, 0x70, 0x79, 0x37, 0x1f, 0xc6})
}

func TestEncoder_EncoderPi(t *testing.T) {
	testEncoderRoundtrip(t, "../testdata/pi.txt", []byte{0xe7, 0xe5, 0x25, 0x39, 0x92, 0xc7, 0x4a, 0xfb})
	testEncoderRoundtripWriter(t, "../testdata/pi.txt", []byte{0xe7, 0xe5, 0x25, 0x39, 0x92, 0xc7, 0x4a, 0xfb})
}

func TestEncoder_EncoderSilesia(t *testing.T) {
	testEncoderRoundtrip(t, "testdata/silesia.tar", []byte{0xa5, 0x5b, 0x5e, 0xe, 0x5e, 0xea, 0x51, 0x6b})
	testEncoderRoundtripWriter(t, "testdata/silesia.tar", []byte{0xa5, 0x5b, 0x5e, 0xe, 0x5e, 0xea, 0x51, 0x6b})
}

func TestEncoder_EncoderSimple(t *testing.T) {
	testEncoderRoundtrip(t, "testdata/z000028", []byte{0x8b, 0x2, 0x37, 0x70, 0x92, 0xb, 0x98, 0x95})
	testEncoderRoundtripWriter(t, "testdata/z000028", []byte{0x8b, 0x2, 0x37, 0x70, 0x92, 0xb, 0x98, 0x95})
}

func TestEncoder_EncoderHTML(t *testing.T) {
	testEncoderRoundtrip(t, "../testdata/html.txt", []byte{0x35, 0xa9, 0x5c, 0x37, 0x20, 0x9e, 0xc3, 0x37})
	testEncoderRoundtripWriter(t, "../testdata/html.txt", []byte{0x35, 0xa9, 0x5c, 0x37, 0x20, 0x9e, 0xc3, 0x37})
}

func TestEncoder_EncoderEnwik9(t *testing.T) {
	testEncoderRoundtrip(t, "./testdata/enwik9.zst", []byte{0x28, 0xfa, 0xf4, 0x30, 0xca, 0x4b, 0x64, 0x12})
	testEncoderRoundtripWriter(t, "./testdata/enwik9.zst", []byte{0x28, 0xfa, 0xf4, 0x30, 0xca, 0x4b, 0x64, 0x12})
}

// test roundtrip using io.ReaderFrom interface.
func testEncoderRoundtrip(t *testing.T, file string, wantCRC []byte) {
	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		t.Run(level.String(), func(t *testing.T) {
			f, err := os.Open(file)
			if err != nil {
				if os.IsNotExist(err) {
					t.Skip("No input file:", file)
					return
				}
				t.Fatal(err)
			}
			defer f.Close()
			input := io.Reader(f)
			if strings.HasSuffix(file, ".zst") {
				dec, err := NewReader(f)
				if err != nil {
					t.Fatal(err)
				}
				input = dec
				defer dec.Close()
			}

			pr, pw := io.Pipe()
			dec2, err := NewReader(pr)
			if err != nil {
				t.Fatal(err)
			}
			defer dec2.Close()

			enc, err := NewWriter(pw, WithEncoderCRC(true), WithEncoderLevel(level))
			if err != nil {
				t.Fatal(err)
			}
			defer enc.Close()
			var wantSize int64
			start := time.Now()
			go func() {
				n, err := enc.ReadFrom(input)
				if err != nil {
					t.Fatal(err)
				}
				wantSize = n
				err = enc.Close()
				if err != nil {
					t.Fatal(err)
				}
				pw.Close()
			}()
			var gotSize int64

			// Check CRC
			d := xxhash.New()
			if true {
				gotSize, err = io.Copy(d, dec2)
			} else {
				fout, err := os.Create(file + ".got")
				if err != nil {
					t.Fatal(err)
				}
				gotSize, err = io.Copy(io.MultiWriter(fout, d), dec2)
				if err != nil {
					t.Fatal(err)
				}
			}
			if wantSize != gotSize {
				t.Errorf("want size (%d) != got size (%d)", wantSize, gotSize)
			}
			if err != nil {
				t.Fatal(err)
			}
			if gotCRC := d.Sum(nil); len(wantCRC) > 0 && !bytes.Equal(gotCRC, wantCRC) {
				t.Errorf("crc mismatch %#v (want) != %#v (got)", wantCRC, gotCRC)
			} else if len(wantCRC) != 8 {
				t.Logf("Unable to verify CRC: %#v", gotCRC)
			} else {
				t.Logf("CRC Verified: %#v", gotCRC)
			}
			t.Log("Encoder len", wantSize)
			mbpersec := (float64(wantSize) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second)))
			t.Logf("Encoded+Decoded %d bytes with %.2f MB/s", wantSize, mbpersec)
		})
	}
}

type writerWrapper struct {
	w io.Writer
}

func (w writerWrapper) Write(p []byte) (n int, err error) {
	return w.w.Write(p)
}

// test roundtrip using plain io.Writer interface.
func testEncoderRoundtripWriter(t *testing.T, file string, wantCRC []byte) {
	f, err := os.Open(file)
	if err != nil {
		if os.IsNotExist(err) {
			t.Skip("No input file:", file)
			return
		}
		t.Fatal(err)
	}
	defer f.Close()
	input := io.Reader(f)
	if strings.HasSuffix(file, ".zst") {
		dec, err := NewReader(f)
		if err != nil {
			t.Fatal(err)
		}
		input = dec
		defer dec.Close()
	}

	pr, pw := io.Pipe()
	dec2, err := NewReader(pr)
	if err != nil {
		t.Fatal(err)
	}
	defer dec2.Close()

	enc, err := NewWriter(pw, WithEncoderCRC(true))
	if err != nil {
		t.Fatal(err)
	}
	defer enc.Close()
	encW := writerWrapper{w: enc}
	var wantSize int64
	start := time.Now()
	go func() {
		n, err := io.CopyBuffer(encW, input, make([]byte, 1337))
		if err != nil {
			t.Fatal(err)
		}
		wantSize = n
		err = enc.Close()
		if err != nil {
			t.Fatal(err)
		}
		pw.Close()
	}()
	var gotSize int64

	// Check CRC
	d := xxhash.New()
	if true {
		gotSize, err = io.Copy(d, dec2)
	} else {
		fout, err := os.Create(file + ".got")
		if err != nil {
			t.Fatal(err)
		}
		gotSize, err = io.Copy(io.MultiWriter(fout, d), dec2)
		if err != nil {
			t.Fatal(err)
		}
	}
	if wantSize != gotSize {
		t.Errorf("want size (%d) != got size (%d)", wantSize, gotSize)
	}
	if err != nil {
		t.Fatal(err)
	}
	if gotCRC := d.Sum(nil); len(wantCRC) > 0 && !bytes.Equal(gotCRC, wantCRC) {
		t.Errorf("crc mismatch %#v (want) != %#v (got)", wantCRC, gotCRC)
	} else if len(wantCRC) != 8 {
		t.Logf("Unable to verify CRC: %#v", gotCRC)
	} else {
		t.Logf("CRC Verified: %#v", gotCRC)
	}
	t.Log("Fast Encoder len", wantSize)
	mbpersec := (float64(wantSize) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second)))
	t.Logf("Encoded+Decoded %d bytes with %.2f MB/s", wantSize, mbpersec)
}

func TestEncoder_EncodeAllSilesia(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	in, err := ioutil.ReadFile("testdata/silesia.tar")
	if err != nil {
		if os.IsNotExist(err) {
			t.Skip("Missing testdata/silesia.tar")
			return
		}
		t.Fatal(err)
	}

	var e Encoder
	start := time.Now()
	dst := e.EncodeAll(in, nil)
	t.Log("Fast Encoder len", len(in), "-> zstd len", len(dst))
	mbpersec := (float64(len(in)) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second)))
	t.Logf("Encoded %d bytes with %.2f MB/s", len(in), mbpersec)

	dec, err := NewReader(nil, WithDecoderMaxMemory(220<<20))
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()
	decoded, err := dec.DecodeAll(dst, nil)
	if err != nil {
		t.Error(err, len(decoded))
	}
	if !bytes.Equal(decoded, in) {
		ioutil.WriteFile("testdata/"+t.Name()+"-silesia.tar.got", decoded, os.ModePerm)
		t.Fatal("Decoded does not match")
	}
	t.Log("Encoded content matched")
}

func TestEncoder_EncodeAllEmpty(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	var in []byte

	e, err := NewWriter(nil, WithZeroFrames(true))
	if err != nil {
		t.Fatal(err)
	}
	defer e.Close()
	dst := e.EncodeAll(in, nil)
	if len(dst) == 0 {
		t.Fatal("Requested zero frame, but got nothing.")
	}
	t.Log("Block Encoder len", len(in), "-> zstd len", len(dst), dst)

	dec, err := NewReader(nil, WithDecoderMaxMemory(220<<20))
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()
	decoded, err := dec.DecodeAll(dst, nil)
	if err != nil {
		t.Error(err, len(decoded))
	}
	if !bytes.Equal(decoded, in) {
		t.Fatal("Decoded does not match")
	}

	// Test buffer writer.
	var buf bytes.Buffer
	e.Reset(&buf)
	err = e.Close()
	if err != nil {
		t.Fatal(err)
	}
	dst = buf.Bytes()
	if len(dst) == 0 {
		t.Fatal("Requested zero frame, but got nothing.")
	}
	t.Log("Buffer Encoder len", len(in), "-> zstd len", len(dst))

	decoded, err = dec.DecodeAll(dst, nil)
	if err != nil {
		t.Error(err, len(decoded))
	}
	if !bytes.Equal(decoded, in) {
		t.Fatal("Decoded does not match")
	}

	t.Log("Encoded content matched")
}

func TestEncoder_EncodeAllEnwik9(t *testing.T) {
	if false || testing.Short() {
		t.SkipNow()
	}
	file := "testdata/enwik9.zst"
	f, err := os.Open(file)
	if err != nil {
		if os.IsNotExist(err) {
			t.Skip("To run extended tests, download http://mattmahoney.net/dc/enwik9.zip unzip it \n" +
				"compress it with 'zstd -15 -T0 enwik9' and place it in " + file)
		}
	}
	dec, err := NewReader(f)
	if err != nil {
		t.Fatal(err)
	}
	defer dec.Close()
	in, err := ioutil.ReadAll(dec)
	if err != nil {
		t.Fatal(err)
	}

	start := time.Now()
	var e Encoder
	dst := e.EncodeAll(in, nil)
	t.Log("Simple Encoder len", len(in), "-> zstd len", len(dst))
	mbpersec := (float64(len(in)) / (1024 * 1024)) / (float64(time.Since(start)) / (float64(time.Second)))
	t.Logf("Encoded %d bytes with %.2f MB/s", len(in), mbpersec)
	decoded, err := dec.DecodeAll(dst, nil)
	if err != nil {
		t.Error(err, len(decoded))
	}
	if !bytes.Equal(decoded, in) {
		ioutil.WriteFile("testdata/"+t.Name()+"-enwik9.got", decoded, os.ModePerm)
		t.Fatal("Decoded does not match")
	}
	t.Log("Encoded content matched")
}

func BenchmarkEncoder_EncodeAllXML(b *testing.B) {
	f, err := os.Open("testdata/xml.zst")
	if err != nil {
		b.Fatal(err)
	}
	dec, err := NewReader(f)
	if err != nil {
		b.Fatal(err)
	}
	in, err := ioutil.ReadAll(dec)
	if err != nil {
		b.Fatal(err)
	}
	dec.Close()

	enc := Encoder{}
	dst := enc.EncodeAll(in, nil)
	wantSize := len(dst)
	b.Log("Output size:", len(dst))
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(int64(len(in)))
	for i := 0; i < b.N; i++ {
		dst := enc.EncodeAll(in, dst[:0])
		if len(dst) != wantSize {
			b.Fatal(len(dst), "!=", wantSize)
		}
	}
}

func BenchmarkEncoder_EncodeAllSimple(b *testing.B) {
	f, err := os.Open("testdata/z000028")
	if err != nil {
		b.Fatal(err)
	}
	in, err := ioutil.ReadAll(f)
	if err != nil {
		b.Fatal(err)
	}

	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		b.Run(level.String(), func(b *testing.B) {
			enc, err := NewWriter(nil, WithEncoderConcurrency(1), WithEncoderLevel(level))
			if err != nil {
				b.Fatal(err)
			}
			defer enc.Close()
			dst := enc.EncodeAll(in, nil)
			wantSize := len(dst)
			b.ResetTimer()
			b.ReportAllocs()
			b.SetBytes(int64(len(in)))
			for i := 0; i < b.N; i++ {
				dst := enc.EncodeAll(in, dst[:0])
				if len(dst) != wantSize {
					b.Fatal(len(dst), "!=", wantSize)
				}
			}
		})
	}
}

func BenchmarkEncoder_EncodeAllSimple4K(b *testing.B) {
	f, err := os.Open("testdata/z000028")
	if err != nil {
		b.Fatal(err)
	}
	in, err := ioutil.ReadAll(f)
	if err != nil {
		b.Fatal(err)
	}
	in = in[:4096]

	for level := EncoderLevel(speedNotSet + 1); level < speedLast; level++ {
		b.Run(level.String(), func(b *testing.B) {
			enc, err := NewWriter(nil, WithEncoderConcurrency(1), WithEncoderLevel(level))
			if err != nil {
				b.Fatal(err)
			}
			defer enc.Close()
			dst := enc.EncodeAll(in, nil)
			wantSize := len(dst)
			b.ResetTimer()
			b.ReportAllocs()
			b.SetBytes(int64(len(in)))
			for i := 0; i < b.N; i++ {
				dst := enc.EncodeAll(in, dst[:0])
				if len(dst) != wantSize {
					b.Fatal(len(dst), "!=", wantSize)
				}
			}
		})
	}
}

func BenchmarkEncoder_EncodeAllHTML(b *testing.B) {
	f, err := os.Open("../testdata/html.txt")
	if err != nil {
		b.Fatal(err)
	}
	in, err := ioutil.ReadAll(f)
	if err != nil {
		b.Fatal(err)
	}

	enc := Encoder{}
	dst := enc.EncodeAll(in, nil)
	wantSize := len(dst)
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(int64(len(in)))
	for i := 0; i < b.N; i++ {
		dst := enc.EncodeAll(in, dst[:0])
		if len(dst) != wantSize {
			b.Fatal(len(dst), "!=", wantSize)
		}
	}
}

func BenchmarkEncoder_EncodeAllTwain(b *testing.B) {
	f, err := os.Open("../testdata/Mark.Twain-Tom.Sawyer.txt")
	if err != nil {
		b.Fatal(err)
	}
	in, err := ioutil.ReadAll(f)
	if err != nil {
		b.Fatal(err)
	}

	enc := Encoder{}
	dst := enc.EncodeAll(in, nil)
	wantSize := len(dst)
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(int64(len(in)))
	for i := 0; i < b.N; i++ {
		dst := enc.EncodeAll(in, dst[:0])
		if len(dst) != wantSize {
			b.Fatal(len(dst), "!=", wantSize)
		}
	}
}

func BenchmarkEncoder_EncodeAllPi(b *testing.B) {
	f, err := os.Open("../testdata/pi.txt")
	if err != nil {
		b.Fatal(err)
	}
	in, err := ioutil.ReadAll(f)
	if err != nil {
		b.Fatal(err)
	}

	enc := Encoder{}
	dst := enc.EncodeAll(in, nil)
	wantSize := len(dst)
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(int64(len(in)))
	for i := 0; i < b.N; i++ {
		dst := enc.EncodeAll(in, dst[:0])
		if len(dst) != wantSize {
			b.Fatal(len(dst), "!=", wantSize)
		}
	}
}

func BenchmarkRandomEncodeAllFastest(b *testing.B) {
	rng := rand.New(rand.NewSource(1))
	data := make([]byte, 10<<20)
	for i := range data {
		data[i] = uint8(rng.Intn(256))
	}
	enc, _ := NewWriter(nil, WithEncoderLevel(SpeedFastest), WithEncoderConcurrency(1))
	defer enc.Close()
	dst := enc.EncodeAll(data, nil)
	wantSize := len(dst)
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(int64(len(data)))
	for i := 0; i < b.N; i++ {
		dst := enc.EncodeAll(data, dst[:0])
		if len(dst) != wantSize {
			b.Fatal(len(dst), "!=", wantSize)
		}
	}
}

func BenchmarkRandomEncodeAllDefault(b *testing.B) {
	rng := rand.New(rand.NewSource(1))
	data := make([]byte, 10<<20)
	for i := range data {
		data[i] = uint8(rng.Intn(256))
	}
	enc, _ := NewWriter(nil, WithEncoderLevel(SpeedDefault), WithEncoderConcurrency(1))
	defer enc.Close()
	dst := enc.EncodeAll(data, nil)
	wantSize := len(dst)
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(int64(len(data)))
	for i := 0; i < b.N; i++ {
		dst := enc.EncodeAll(data, dst[:0])
		if len(dst) != wantSize {
			b.Fatal(len(dst), "!=", wantSize)
		}
	}
}

func BenchmarkRandomEncoderFastest(b *testing.B) {
	rng := rand.New(rand.NewSource(1))
	data := make([]byte, 10<<20)
	for i := range data {
		data[i] = uint8(rng.Intn(256))
	}
	wantSize := int64(len(data))
	enc, _ := NewWriter(ioutil.Discard, WithEncoderLevel(SpeedFastest))
	defer enc.Close()
	n, err := io.Copy(enc, bytes.NewBuffer(data))
	if err != nil {
		b.Fatal(err)
	}
	if n != wantSize {
		b.Fatal(n, "!=", wantSize)
	}
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(wantSize)
	for i := 0; i < b.N; i++ {
		enc.Reset(ioutil.Discard)
		n, err := io.Copy(enc, bytes.NewBuffer(data))
		if err != nil {
			b.Fatal(err)
		}
		if n != wantSize {
			b.Fatal(n, "!=", wantSize)
		}
	}
}

func BenchmarkRandomEncoderDefault(b *testing.B) {
	rng := rand.New(rand.NewSource(1))
	data := make([]byte, 10<<20)
	for i := range data {
		data[i] = uint8(rng.Intn(256))
	}
	wantSize := int64(len(data))
	enc, _ := NewWriter(ioutil.Discard, WithEncoderLevel(SpeedDefault))
	defer enc.Close()
	n, err := io.Copy(enc, bytes.NewBuffer(data))
	if err != nil {
		b.Fatal(err)
	}
	if n != wantSize {
		b.Fatal(n, "!=", wantSize)
	}
	b.ResetTimer()
	b.ReportAllocs()
	b.SetBytes(wantSize)
	for i := 0; i < b.N; i++ {
		enc.Reset(ioutil.Discard)
		n, err := io.Copy(enc, bytes.NewBuffer(data))
		if err != nil {
			b.Fatal(err)
		}
		if n != wantSize {
			b.Fatal(n, "!=", wantSize)
		}
	}
}