aliyun-oss-go-sdk/oss/crc_test.go

package oss

import (
	"crypto/md5"
	"encoding/base64"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"strings"

	. "gopkg.in/check.v1"
)

type OssCrcSuite struct {
	client *Client
	bucket *Bucket
}

var _ = Suite(&OssCrcSuite{})

// SetUpSuite runs once when the suite starts running
func (s *OssCrcSuite) SetUpSuite(c *C) {
	client, err := New(endpoint, accessID, accessKey)
	c.Assert(err, IsNil)
	s.client = client

	s.client.CreateBucket(bucketName)

	bucket, err := s.client.Bucket(bucketName)
	c.Assert(err, IsNil)
	s.bucket = bucket

	testLogger.Println("test crc started")
}

// TearDownSuite runs before each test or benchmark starts running
func (s *OssCrcSuite) TearDownSuite(c *C) {
	// Delete part
	keyMarker := KeyMarker("")
	uploadIDMarker := UploadIDMarker("")
	for {
		lmur, err := s.bucket.ListMultipartUploads(keyMarker, uploadIDMarker)
		c.Assert(err, IsNil)
		for _, upload := range lmur.Uploads {
			var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName,
				Key: upload.Key, UploadID: upload.UploadID}
			err = s.bucket.AbortMultipartUpload(imur)
			c.Assert(err, IsNil)
		}
		keyMarker = KeyMarker(lmur.NextKeyMarker)
		uploadIDMarker = UploadIDMarker(lmur.NextUploadIDMarker)
		if !lmur.IsTruncated {
			break
		}
	}

	// Delete objects
	marker := Marker("")
	for {
		lor, err := s.bucket.ListObjects(marker)
		c.Assert(err, IsNil)
		for _, object := range lor.Objects {
			err = s.bucket.DeleteObject(object.Key)
			c.Assert(err, IsNil)
		}
		marker = Marker(lor.NextMarker)
		if !lor.IsTruncated {
			break
		}
	}

	// Delete bucket
	err := s.client.DeleteBucket(s.bucket.BucketName)
	c.Assert(err, IsNil)

	testLogger.Println("test crc completed")
}

// SetUpTest runs after each test or benchmark runs
func (s *OssCrcSuite) SetUpTest(c *C) {
	err := removeTempFiles("../oss", ".jpg")
	c.Assert(err, IsNil)
}

// TearDownTest runs once after all tests or benchmarks have finished running
func (s *OssCrcSuite) TearDownTest(c *C) {
	err := removeTempFiles("../oss", ".jpg")
	c.Assert(err, IsNil)
}

// TestCRCGolden tests OSS's CRC64
func (s *OssCrcSuite) TestCRCGolden(c *C) {
	type crcTest struct {
		out uint64
		in  string
	}

	var crcGolden = []crcTest{
		{0x0, ""},
		{0x3420000000000000, "a"},
		{0x36c4200000000000, "ab"},
		{0x3776c42000000000, "abc"},
		{0x336776c420000000, "abcd"},
		{0x32d36776c4200000, "abcde"},
		{0x3002d36776c42000, "abcdef"},
		{0x31b002d36776c420, "abcdefg"},
		{0xe21b002d36776c4, "abcdefgh"},
		{0x8b6e21b002d36776, "abcdefghi"},
		{0x7f5b6e21b002d367, "abcdefghij"},
		{0x8ec0e7c835bf9cdf, "Discard medicine more than two years old."},
		{0xc7db1759e2be5ab4, "He who has a shady past knows that nice guys finish last."},
		{0xfbf9d9603a6fa020, "I wouldn't marry him with a ten foot pole."},
		{0xeafc4211a6daa0ef, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
		{0x3e05b21c7a4dc4da, "The days of the digital watch are numbered.  -Tom Stoppard"},
		{0x5255866ad6ef28a6, "Nepal premier won't resign."},
		{0x8a79895be1e9c361, "For every action there is an equal and opposite government program."},
		{0x8878963a649d4916, "His money is twice tainted: 'taint yours and 'taint mine."},
		{0xa7b9d53ea87eb82f, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
		{0xdb6805c0966a2f9c, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
		{0xf3553c65dacdadd2, "size:  a.out:  bad magic"},
		{0x9d5e034087a676b9, "The major problem is with sendmail.  -Mark Horton"},
		{0xa6db2d7f8da96417, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
		{0x325e00cd2fe819f9, "If the enemy is within range, then so are you."},
		{0x88c6600ce58ae4c6, "It's well we cannot hear the screams/That we create in others' dreams."},
		{0x28c4a3f3b769e078, "You remind me of a TV show, but that's all right: I watch it anyway."},
		{0xa698a34c9d9f1dca, "C is as portable as Stonehedge!!"},
		{0xf6c1e2a8c26c5cfc, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
		{0xd402559dfe9b70c, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
		{0xdb6efff26aa94946, "How can you write a big system without C++?  -Paul Glick"},
	}

	var tab = crc64.MakeTable(crc64.ISO)

	for i := 0; i < len(crcGolden); i++ {
		golden := crcGolden[i]
		crc := NewCRC(tab, 0)
		io.WriteString(crc, golden.in)
		sum := crc.Sum64()

		c.Assert(sum, Equals, golden.out)
	}
}

// testCRC64Combine tests CRC64 on vector[0..pos] which should have CRC64 crc.
// Also test CRC64Combine on vector[] split in two.
func testCRC64Combine(c *C, str string, pos int, crc uint64) {
	tabECMA := crc64.MakeTable(crc64.ECMA)

	// Test CRC64
	hash := crc64.New(tabECMA)
	io.WriteString(hash, str)
	crc1 := hash.Sum64()
	c.Assert(crc1, Equals, crc)

	// Test CRC64 combine
	hash = crc64.New(tabECMA)
	io.WriteString(hash, str[0:pos])
	crc1 = hash.Sum64()

	hash = crc64.New(tabECMA)
	io.WriteString(hash, str[pos:len(str)])
	crc2 := hash.Sum64()

	crc1 = CRC64Combine(crc1, crc2, uint64(len(str)-pos))
	c.Assert(crc1, Equals, crc)
}

// TestCRCCombine tests CRC64Combine
func (s *OssCrcSuite) TestCRCCombine(c *C) {
	str := "123456789"
	testCRC64Combine(c, str, (len(str)+1)>>1, 0x995DC9BBDF1939FA)

	str = "This is a test of the emergency broadcast system."
	testCRC64Combine(c, str, (len(str)+1)>>1, 0x27DB187FC15BBC72)
}

// TestCRCRepeatedCombine tests CRC64Combine
func (s *OssCrcSuite) TestCRCRepeatedCombine(c *C) {
	tab := crc64.MakeTable(crc64.ECMA)
	str := "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"

	for i := 0; i <= len(str); i++ {
		hash := crc64.New(tab)
		io.WriteString(hash, string(str[0:i]))
		prev := hash.Sum64()

		hash = crc64.New(tab)
		io.WriteString(hash, string(str[i:len(str)]))
		post := hash.Sum64()

		crc := CRC64Combine(prev, post, uint64(len(str)-i))
		testLogger.Println("TestCRCRepeatedCombine:", prev, post, crc, i, len(str))
		c.Assert(crc == 0x7AD25FAFA1710407, Equals, true)
	}
}

// TestCRCRandomCombine tests CRC64Combine
func (s *OssCrcSuite) TestCRCRandomCombine(c *C) {
	tab := crc64.MakeTable(crc64.ECMA)
	fileName := "../sample/BingWallpaper-2015-11-07.jpg"

	body, err := ioutil.ReadFile(fileName)
	c.Assert(err, IsNil)

	for i := 0; i < 10; i++ {
		fileParts, err := SplitFileByPartNum(fileName, 1+rand.Intn(9999))
		c.Assert(err, IsNil)

		var crc uint64
		for _, part := range fileParts {
			calc := NewCRC(tab, 0)
			calc.Write(body[part.Offset : part.Offset+part.Size])
			crc = CRC64Combine(crc, calc.Sum64(), (uint64)(part.Size))
		}

		testLogger.Println("TestCRCRandomCombine:", crc, i, fileParts)
		c.Assert(crc == 0x2B612D24FFF64222, Equals, true)
	}
}

// TestEnableCRCAndMD5 tests MD5 and CRC check
func (s *OssCrcSuite) TestEnableCRCAndMD5(c *C) {
	objectName := objectNamePrefix + randStr(8)
	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
	newFileName := "BingWallpaper-2015-11-07-2.jpg"
	objectValue := "空山新雨后，天气晚来秋。明月松间照，清泉石上流。竹喧归浣女，莲动下渔舟。随意春芳歇，王孙自可留。"

	client, err := New(endpoint, accessID, accessKey, EnableCRC(true), EnableMD5(true), MD5ThresholdCalcInMemory(200*1024))
	c.Assert(err, IsNil)
	bucket, err := client.Bucket(bucketName)
	c.Assert(err, IsNil)

	// PutObject
	err = bucket.PutObject(objectName, strings.NewReader(objectValue))
	c.Assert(err, IsNil)

	// GetObject
	body, err := bucket.GetObject(objectName)
	c.Assert(err, IsNil)
	_, err = ioutil.ReadAll(body)
	c.Assert(err, IsNil)
	body.Close()

	// GetObjectWithCRC
	getResult, err := bucket.DoGetObject(&GetObjectRequest{objectName}, nil)
	c.Assert(err, IsNil)
	str, err := readBody(getResult.Response.Body)
	c.Assert(err, IsNil)
	c.Assert(str, Equals, objectValue)
	c.Assert(getResult.ClientCRC.Sum64(), Equals, getResult.ServerCRC)

	// PutObjectFromFile
	err = bucket.PutObjectFromFile(objectName, fileName)
	c.Assert(err, IsNil)

	// GetObjectToFile
	err = bucket.GetObjectToFile(objectName, newFileName)
	c.Assert(err, IsNil)
	eq, err := compareFiles(fileName, newFileName)
	c.Assert(err, IsNil)
	c.Assert(eq, Equals, true)

	// DeleteObject
	err = bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	// AppendObject
	var nextPos int64
	nextPos, err = bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
	c.Assert(err, IsNil)
	nextPos, err = bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
	c.Assert(err, IsNil)

	err = bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	request := &AppendObjectRequest{
		ObjectKey: objectName,
		Reader:    strings.NewReader(objectValue),
		Position:  0,
	}
	appendResult, err := bucket.DoAppendObject(request, []Option{InitCRC(0)})
	c.Assert(err, IsNil)
	request.Position = appendResult.NextPosition
	appendResult, err = bucket.DoAppendObject(request, []Option{InitCRC(appendResult.CRC)})
	c.Assert(err, IsNil)

	err = s.bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	// MultipartUpload
	chunks, err := SplitFileByPartSize(fileName, 100*1024)
	imurUpload, err := bucket.InitiateMultipartUpload(objectName)
	c.Assert(err, IsNil)
	var partsUpload []UploadPart

	for _, chunk := range chunks {
		part, err := bucket.UploadPartFromFile(imurUpload, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number))
		c.Assert(err, IsNil)
		partsUpload = append(partsUpload, part)
	}

	_, err = bucket.CompleteMultipartUpload(imurUpload, partsUpload)
	c.Assert(err, IsNil)

	// Check MultipartUpload
	err = bucket.GetObjectToFile(objectName, newFileName)
	c.Assert(err, IsNil)
	eq, err = compareFiles(fileName, newFileName)
	c.Assert(err, IsNil)
	c.Assert(eq, Equals, true)

	// DeleteObjects
	_, err = bucket.DeleteObjects([]string{objectName})
	c.Assert(err, IsNil)
}

// TestDisableCRCAndMD5 disables MD5 and CRC
func (s *OssCrcSuite) TestDisableCRCAndMD5(c *C) {
	objectName := objectNamePrefix + randStr(8)
	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
	newFileName := "BingWallpaper-2015-11-07-3.jpg"
	objectValue := "中岁颇好道，晚家南山陲。兴来每独往，胜事空自知。行到水穷处，坐看云起时。偶然值林叟，谈笑无还期。"

	client, err := New(endpoint, accessID, accessKey, EnableCRC(false), EnableMD5(false))
	c.Assert(err, IsNil)
	bucket, err := client.Bucket(bucketName)
	c.Assert(err, IsNil)

	// PutObject
	err = bucket.PutObject(objectName, strings.NewReader(objectValue))
	c.Assert(err, IsNil)

	// GetObject
	body, err := bucket.GetObject(objectName)
	c.Assert(err, IsNil)
	_, err = ioutil.ReadAll(body)
	c.Assert(err, IsNil)
	body.Close()

	// GetObjectWithCRC
	getResult, err := bucket.DoGetObject(&GetObjectRequest{objectName}, nil)
	c.Assert(err, IsNil)
	str, err := readBody(getResult.Response.Body)
	c.Assert(err, IsNil)
	c.Assert(str, Equals, objectValue)

	// PutObjectFromFile
	err = bucket.PutObjectFromFile(objectName, fileName)
	c.Assert(err, IsNil)

	// GetObjectToFile
	err = bucket.GetObjectToFile(objectName, newFileName)
	c.Assert(err, IsNil)
	eq, err := compareFiles(fileName, newFileName)
	c.Assert(err, IsNil)
	c.Assert(eq, Equals, true)

	// DeleteObject
	err = bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	// AppendObject
	var nextPos int64
	nextPos, err = bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
	c.Assert(err, IsNil)
	nextPos, err = bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
	c.Assert(err, IsNil)

	err = bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	request := &AppendObjectRequest{
		ObjectKey: objectName,
		Reader:    strings.NewReader(objectValue),
		Position:  0,
	}
	appendResult, err := bucket.DoAppendObject(request, []Option{InitCRC(0)})
	c.Assert(err, IsNil)
	request.Position = appendResult.NextPosition
	appendResult, err = bucket.DoAppendObject(request, []Option{InitCRC(appendResult.CRC)})
	c.Assert(err, IsNil)

	err = s.bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	// MultipartUpload
	chunks, err := SplitFileByPartSize(fileName, 100*1024)
	imurUpload, err := bucket.InitiateMultipartUpload(objectName)
	c.Assert(err, IsNil)
	var partsUpload []UploadPart

	for _, chunk := range chunks {
		part, err := bucket.UploadPartFromFile(imurUpload, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number))
		c.Assert(err, IsNil)
		partsUpload = append(partsUpload, part)
	}

	_, err = bucket.CompleteMultipartUpload(imurUpload, partsUpload)
	c.Assert(err, IsNil)

	// Check MultipartUpload
	err = bucket.GetObjectToFile(objectName, newFileName)
	c.Assert(err, IsNil)
	eq, err = compareFiles(fileName, newFileName)
	c.Assert(err, IsNil)
	c.Assert(eq, Equals, true)

	// DeleteObjects
	_, err = bucket.DeleteObjects([]string{objectName})
	c.Assert(err, IsNil)
}

// TestSpecifyContentMD5 specifies MD5
func (s *OssCrcSuite) TestSpecifyContentMD5(c *C) {
	objectName := objectNamePrefix + randStr(8)
	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
	objectValue := "积雨空林烟火迟，蒸藜炊黍饷东菑。漠漠水田飞白鹭，阴阴夏木啭黄鹂。山中习静观朝槿，松下清斋折露葵。野老与人争席罢，海鸥何事更相疑。"

	mh := md5.Sum([]byte(objectValue))
	md5B64 := base64.StdEncoding.EncodeToString(mh[:])

	// PutObject
	err := s.bucket.PutObject(objectName, strings.NewReader(objectValue), ContentMD5(md5B64))
	c.Assert(err, IsNil)

	// PutObjectFromFile
	file, err := os.Open(fileName)
	md5 := md5.New()
	io.Copy(md5, file)
	mdHex := base64.StdEncoding.EncodeToString(md5.Sum(nil)[:])
	err = s.bucket.PutObjectFromFile(objectName, fileName, ContentMD5(mdHex))
	c.Assert(err, IsNil)

	err = s.bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	// AppendObject
	var nextPos int64
	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
	c.Assert(err, IsNil)
	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
	c.Assert(err, IsNil)

	err = s.bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	request := &AppendObjectRequest{
		ObjectKey: objectName,
		Reader:    strings.NewReader(objectValue),
		Position:  0,
	}
	appendResult, err := s.bucket.DoAppendObject(request, []Option{InitCRC(0)})
	c.Assert(err, IsNil)
	request.Position = appendResult.NextPosition
	appendResult, err = s.bucket.DoAppendObject(request, []Option{InitCRC(appendResult.CRC)})
	c.Assert(err, IsNil)

	err = s.bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)

	// MultipartUpload
	imurUpload, err := s.bucket.InitiateMultipartUpload(objectName)
	c.Assert(err, IsNil)

	var partsUpload []UploadPart
	part, err := s.bucket.UploadPart(imurUpload, strings.NewReader(objectValue), (int64)(len([]byte(objectValue))), 1)
	c.Assert(err, IsNil)
	partsUpload = append(partsUpload, part)

	_, err = s.bucket.CompleteMultipartUpload(imurUpload, partsUpload)
	c.Assert(err, IsNil)

	// DeleteObject
	err = s.bucket.DeleteObject(objectName)
	c.Assert(err, IsNil)
}

// TestAppendObjectNegative
func (s *OssCrcSuite) TestAppendObjectNegative(c *C) {
	objectName := objectNamePrefix + randStr(8)
	objectValue := "空山不见人，但闻人语响。返影入深林，复照青苔上。"

	nextPos, err := s.bucket.AppendObject(objectName, strings.NewReader(objectValue), 0, InitCRC(0))
	c.Assert(err, IsNil)

	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos, InitCRC(0))
	c.Assert(err, NotNil)
	c.Assert(strings.HasPrefix(err.Error(), "oss: the crc"), Equals, true)
}