crc

oss/bucket.go

@@ -250,7 +250,7 @@ func (bucket Bucket) copy(srcObjectKey, destBucketName, destObjectKey string, op
 	if err != nil {
 		return out, err
 	}
-	resp, err := bucket.Client.Conn.Do("PUT", destBucketName, destObjectKey, "", "", headers, nil)
+	resp, err := bucket.Client.Conn.Do("PUT", destBucketName, destObjectKey, "", "", headers, nil, 0)
 	if err != nil {
 		return out, err
 	}
@@ -273,8 +273,8 @@ func (bucket Bucket) copy(srcObjectKey, destBucketName, destObjectKey string, op
 // appendPosition  object追加的起始位置。
 // destObjectProperties  第一次追加时指定新对象的属性，如CacheControl、ContentDisposition、ContentEncoding、
 // Expires、ServerSideEncryption、ObjectACL。
-// int64 下次追加的开始位置，error为nil空时有效。
 //
+// int64 下次追加的开始位置，error为nil空时有效。
 // error 操作无错误为nil，非nil为错误信息。
 //
 func (bucket Bucket) AppendObject(objectKey string, reader io.Reader, appendPosition int64, options ...Option) (int64, error) {
@@ -283,18 +283,65 @@ func (bucket Bucket) AppendObject(objectKey string, reader io.Reader, appendPosi
 	opts := addContentType(options, objectKey)
 	resp, err := bucket.do("POST", objectKey, params, params, opts, reader)
 	if err != nil {
-		return nextAppendPosition, err
+		return appendPosition, err
 	}
 	defer resp.body.Close()
 
-	napint, err := strconv.Atoi(resp.headers.Get(HTTPHeaderOssNextAppendPosition))
+	nextAppendPosition, err = strconv.ParseInt(resp.headers.Get(HTTPHeaderOssNextAppendPosition), 10, 64)
 	if err != nil {
 		return nextAppendPosition, err
 	}
-	nextAppendPosition = int64(napint)
+
 	return nextAppendPosition, nil
 }
 
+//
+// AppendObjectWithCRC 追加方式上传。
+//
+// AppendObject参数必须包含position，其值指定从何处进行追加。首次追加操作的position必须为0，
+// 后续追加操作的position是Object的当前长度。例如，第一次Append Object请求指定position值为0，
+// content-length是65536；那么，第二次Append Object需要指定position为65536。
+// 每次操作成功后，响应头部x-oss-next-append-position也会标明下一次追加的position。
+//
+// objectKey  需要追加的Object。
+// reader     io.Reader，读取追的内容。
+// appendPosition  object追加的起始位置。
+// initCRC  上次追加的返回的CRC校验值，第一次填0。
+// destObjectProperties  第一次追加时指定新对象的属性，如CacheControl、ContentDisposition、ContentEncoding、
+// Expires、ServerSideEncryption、ObjectACL。
+//
+// int64 下次追加的开始位置，error为nil空时有效。
+// int64 本次追加后文件的CRC值，作为下次的初始化值。
+// error 操作无错误为nil，非nil为错误信息。
+//
+func (bucket Bucket) AppendObjectWithCRC(objectKey string, reader io.Reader, appendPosition int64, initCRC uint64, options ...Option) (int64, uint64, error) {
+	var nextAppendPosition int64
+	params := "append&position=" + strconv.Itoa(int(appendPosition))
+	opts := addContentType(options, objectKey)
+	headers := make(map[string]string)
+
+	handleOptions(headers, opts)
+	resp, err := bucket.Client.Conn.Do("POST", bucket.BucketName, objectKey, params, params, headers, reader, initCRC)
+	if err != nil {
+		return nextAppendPosition, initCRC, err
+	}
+	defer resp.body.Close()
+
+	nextAppendPosition, err = strconv.ParseInt(resp.headers.Get(HTTPHeaderOssNextAppendPosition), 10, 64)
+	if err != nil {
+		return nextAppendPosition, initCRC, err
+	}
+
+	if bucket.getConfig().IsEnableCRC {
+		err = checkCRC(resp, "AppendObject")
+		if err != nil {
+			return nextAppendPosition, resp.serverCRC, err
+		}
+	}
+
+	return nextAppendPosition, resp.serverCRC, nil
+}
+
 //
 // DeleteObject 删除Object。
 //
@@ -531,14 +578,13 @@ func (bucket Bucket) do(method, objectName, urlParams, subResource string,
 		return nil, err
 	}
 	return bucket.Client.Conn.Do(method, bucket.BucketName, objectName,
-		urlParams, subResource, headers, data)
+		urlParams, subResource, headers, data, 0)
 }
 
 func (bucket Bucket) getConfig() *Config {
 	return bucket.Client.Config
 }
 
-// Private
 func addContentType(options []Option, keys ...string) []Option {
 	typ := TypeByExtension("")
 	for _, key := range keys {

oss/client.go

@@ -689,6 +689,17 @@ func EnableCRC(isEnableCRC bool) ClientOption {
 	}
 }
 
+//
+// UserAgent 指定UserAgent，默认如下aliyun-sdk-go/1.2.0 (windows/-/amd64;go1.5.2)。
+//
+// userAgent user agent字符串。
+//
+func UserAgent(userAgent string) ClientOption {
+	return func(client *Client) {
+		client.Config.UserAgent = userAgent
+	}
+}
+
 //
 // Proxy 设置代理服务器，默认不使用代理。
 //
@@ -724,5 +735,5 @@ func AuthProxy(proxyHost, proxyUser, proxyPassword string) ClientOption {
 func (client Client) do(method, bucketName, urlParams, subResource string,
 	headers map[string]string, data io.Reader) (*Response, error) {
 	return client.Conn.Do(method, bucketName, "", urlParams,
-		subResource, headers, data)
+		subResource, headers, data, 0)
 }

oss/client_test.go

@@ -1319,6 +1319,10 @@ func (s *OssClientSuite) TestClientOption(c *C) {
 	c.Assert(client.Conn.config.IsAuthProxy, Equals, true)
 	c.Assert(client.Conn.config.ProxyUser, Equals, proxyUser)
 	c.Assert(client.Conn.config.ProxyPassword, Equals, proxyPasswd)
+
+	client, err = New(endpoint, accessID, accessKey, UserAgent("go sdk user agent"))
+
+	c.Assert(client.Conn.config.UserAgent, Equals, "go sdk user agent")
 }
 
 // _TestProxy

oss/conn.go

@@ -7,7 +7,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"hash"
-	"hash/crc64"
 	"io"
 	"io/ioutil"
 	"net"
@@ -36,14 +35,14 @@ type Response struct {
 
 // Do 处理请求，返回响应结果。
 func (conn Conn) Do(method, bucketName, objectName, urlParams, subResource string,
-	headers map[string]string, data io.Reader) (*Response, error) {
+	headers map[string]string, data io.Reader, initCRC uint64) (*Response, error) {
 	uri := conn.url.getURL(bucketName, objectName, urlParams)
 	resource := conn.url.getResource(bucketName, objectName, subResource)
-	return conn.doRequest(method, uri, resource, headers, data)
+	return conn.doRequest(method, uri, resource, headers, data, initCRC)
 }
 
 func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource string,
-	headers map[string]string, data io.Reader) (*Response, error) {
+	headers map[string]string, data io.Reader, initCRC uint64) (*Response, error) {
 	httpTimeOut := conn.config.HTTPTimeout
 	method = strings.ToUpper(method)
 	if !conn.config.IsUseProxy {
@@ -59,7 +58,7 @@ func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource st
 		Host:       uri.Host,
 	}
 
-	fd, crc := conn.handleBody(req, data)
+	fd, crc := conn.handleBody(req, data, initCRC)
 	if fd != nil {
 		defer func() {
 			fd.Close()
@@ -135,7 +134,7 @@ func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource st
 }
 
 // handle request body
-func (conn Conn) handleBody(req *http.Request, body io.Reader) (*os.File, hash.Hash64) {
+func (conn Conn) handleBody(req *http.Request, body io.Reader, initCRC uint64) (*os.File, hash.Hash64) {
 	var file *os.File
 	var crc hash.Hash64
 	reader := body
@@ -180,7 +179,7 @@ func (conn Conn) handleBody(req *http.Request, body io.Reader) (*os.File, hash.H
 	}
 
 	if reader != nil && conn.config.IsEnableCRC {
-		crc = crc64.New(crcTable())
+		crc = NewCRC(crcTable(), initCRC)
 		reader = io.TeeReader(reader, crc)
 	}
 

oss/crc.go

@@ -0,0 +1,44 @@
+package oss
+
+import (
+	"hash"
+	"hash/crc64"
+)
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	crc uint64
+	tab *crc64.Table
+}
+
+// NewCRC creates a new hash.Hash64 computing the CRC-64 checksum
+// using the polynomial represented by the Table.
+func NewCRC(tab *crc64.Table, init uint64) hash.Hash64 { return &digest{init, tab} }
+
+// Size returns the number of bytes Sum will return.
+func (d *digest) Size() int { return crc64.Size }
+
+// BlockSize returns the hash's underlying block size.
+// The Write method must be able to accept any amount
+// of data, but it may operate more efficiently if all writes
+// are a multiple of the block size.
+func (d *digest) BlockSize() int { return 1 }
+
+// Reset resets the Hash to its initial state.
+func (d *digest) Reset() { d.crc = 0 }
+
+// Write (via the embedded io.Writer interface) adds more data to the running hash.
+// It never returns an error.
+func (d *digest) Write(p []byte) (n int, err error) {
+	d.crc = crc64.Update(d.crc, d.tab, p)
+	return len(p), nil
+}
+
+// Sum64 returns crc64 value.
+func (d *digest) Sum64() uint64 { return d.crc }
+
+// Sum returns hash value.
+func (d *digest) Sum(in []byte) []byte {
+	s := d.Sum64()
+	return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s))
+}

oss/crc_test.go

@@ -3,6 +3,7 @@ package oss
 import (
 	"crypto/md5"
 	"encoding/base64"
+	"hash/crc64"
 	"io"
 	"io/ioutil"
 	"os"
@@ -72,13 +73,65 @@ func (s *OssCrcSuite) TearDownTest(c *C) {
 	c.Assert(err, IsNil)
 }
 
+// TestCRCGolden 测试OSS实现的CRC64
+func (s *OssCrcSuite) TestCRCGolden(c *C) {
+	type crcTest struct {
+		out uint64
+		in  string
+	}
+
+	var crcGolden = []crcTest{
+		{0x0, ""},
+		{0x3420000000000000, "a"},
+		{0x36c4200000000000, "ab"},
+		{0x3776c42000000000, "abc"},
+		{0x336776c420000000, "abcd"},
+		{0x32d36776c4200000, "abcde"},
+		{0x3002d36776c42000, "abcdef"},
+		{0x31b002d36776c420, "abcdefg"},
+		{0xe21b002d36776c4, "abcdefgh"},
+		{0x8b6e21b002d36776, "abcdefghi"},
+		{0x7f5b6e21b002d367, "abcdefghij"},
+		{0x8ec0e7c835bf9cdf, "Discard medicine more than two years old."},
+		{0xc7db1759e2be5ab4, "He who has a shady past knows that nice guys finish last."},
+		{0xfbf9d9603a6fa020, "I wouldn't marry him with a ten foot pole."},
+		{0xeafc4211a6daa0ef, "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+		{0x3e05b21c7a4dc4da, "The days of the digital watch are numbered.  -Tom Stoppard"},
+		{0x5255866ad6ef28a6, "Nepal premier won't resign."},
+		{0x8a79895be1e9c361, "For every action there is an equal and opposite government program."},
+		{0x8878963a649d4916, "His money is twice tainted: 'taint yours and 'taint mine."},
+		{0xa7b9d53ea87eb82f, "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+		{0xdb6805c0966a2f9c, "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+		{0xf3553c65dacdadd2, "size:  a.out:  bad magic"},
+		{0x9d5e034087a676b9, "The major problem is with sendmail.  -Mark Horton"},
+		{0xa6db2d7f8da96417, "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+		{0x325e00cd2fe819f9, "If the enemy is within range, then so are you."},
+		{0x88c6600ce58ae4c6, "It's well we cannot hear the screams/That we create in others' dreams."},
+		{0x28c4a3f3b769e078, "You remind me of a TV show, but that's all right: I watch it anyway."},
+		{0xa698a34c9d9f1dca, "C is as portable as Stonehedge!!"},
+		{0xf6c1e2a8c26c5cfc, "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+		{0xd402559dfe9b70c, "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+		{0xdb6efff26aa94946, "How can you write a big system without C++?  -Paul Glick"},
+	}
+
+	var tab = crc64.MakeTable(crc64.ISO)
+
+	for i := 0; i < len(crcGolden); i++ {
+		golden := crcGolden[i]
+		crc := NewCRC(tab, 0)
+		io.WriteString(crc, golden.in)
+		sum := crc.Sum64()
+
+		c.Assert(sum, Equals, golden.out)
+	}
+}
+
 // TestEnableCRCAndMD5 开启MD5和CRC校验
 func (s *OssCrcSuite) TestEnableCRCAndMD5(c *C) {
 	objectName := objectNamePrefix + "tecam"
 	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
 	newFileName := "BingWallpaper-2015-11-07-2.jpg"
-	objectValue := "大江东去，浪淘尽，千古风流人物。 故垒西边，人道是、三国周郎赤壁。 乱石穿空，惊涛拍岸，卷起千堆雪。 江山如画，一时多少豪杰。" +
-		"遥想公谨当年，小乔初嫁了，雄姿英发。 羽扇纶巾，谈笑间、樯橹灰飞烟灭。故国神游，多情应笑我，早生华发，人生如梦，一尊还酹江月。"
+	objectValue := "空山新雨后，天气晚来秋。明月松间照，清泉石上流。竹喧归浣女，莲动下渔舟。随意春芳歇，王孙自可留。"
 
 	client, err := New(endpoint, accessID, accessKey, EnableCRC(true), EnableMD5(true), MD5ThresholdCalcInMemory(200*1024))
 	c.Assert(err, IsNil)
@@ -123,6 +176,18 @@ func (s *OssCrcSuite) TestEnableCRCAndMD5(c *C) {
 	var nextPos int64
 	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
 	c.Assert(err, IsNil)
+	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
+	c.Assert(err, IsNil)
+
+	err = s.bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	nextPos = 0
+	var initCRC uint64
+	nextPos, initCRC, err = s.bucket.AppendObjectWithCRC(objectName, strings.NewReader(objectValue), nextPos, initCRC)
+	c.Assert(err, IsNil)
+	nextPos, initCRC, err = s.bucket.AppendObjectWithCRC(objectName, strings.NewReader(objectValue), nextPos, initCRC)
+	c.Assert(err, IsNil)
 
 	err = s.bucket.DeleteObject(objectName)
 	c.Assert(err, IsNil)
@@ -159,8 +224,7 @@ func (s *OssCrcSuite) TestDisableCRCAndMD5(c *C) {
 	objectName := objectNamePrefix + "tdcam"
 	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
 	newFileName := "BingWallpaper-2015-11-07-3.jpg"
-	objectValue := "大江东去，浪淘尽，千古风流人物。 故垒西边，人道是、三国周郎赤壁。 乱石穿空，惊涛拍岸，卷起千堆雪。 江山如画，一时多少豪杰。" +
-		"遥想公谨当年，小乔初嫁了，雄姿英发。 羽扇纶巾，谈笑间、樯橹灰飞烟灭。故国神游，多情应笑我，早生华发，人生如梦，一尊还酹江月。"
+	objectValue := "中岁颇好道，晚家南山陲。兴来每独往，胜事空自知。行到水穷处，坐看云起时。偶然值林叟，谈笑无还期。"
 
 	client, err := New(endpoint, accessID, accessKey, EnableCRC(false), EnableMD5(false))
 	c.Assert(err, IsNil)
@@ -204,6 +268,18 @@ func (s *OssCrcSuite) TestDisableCRCAndMD5(c *C) {
 	var nextPos int64
 	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
 	c.Assert(err, IsNil)
+	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
+	c.Assert(err, IsNil)
+
+	err = s.bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	nextPos = 0
+	var initCRC uint64
+	nextPos, initCRC, err = s.bucket.AppendObjectWithCRC(objectName, strings.NewReader(objectValue), nextPos, initCRC)
+	c.Assert(err, IsNil)
+	nextPos, initCRC, err = s.bucket.AppendObjectWithCRC(objectName, strings.NewReader(objectValue), nextPos, initCRC)
+	c.Assert(err, IsNil)
 
 	err = s.bucket.DeleteObject(objectName)
 	c.Assert(err, IsNil)
@@ -239,8 +315,7 @@ func (s *OssCrcSuite) TestDisableCRCAndMD5(c *C) {
 func (s *OssCrcSuite) TestSpecifyContentMD5(c *C) {
 	objectName := objectNamePrefix + "tdcam"
 	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
-	objectValue := "大江东去，浪淘尽，千古风流人物。 故垒西边，人道是、三国周郎赤壁。 乱石穿空，惊涛拍岸，卷起千堆雪。 江山如画，一时多少豪杰。" +
-		"遥想公谨当年，小乔初嫁了，雄姿英发。 羽扇纶巾，谈笑间、樯橹灰飞烟灭。故国神游，多情应笑我，早生华发，人生如梦，一尊还酹江月。"
+	objectValue := "积雨空林烟火迟，蒸藜炊黍饷东菑。漠漠水田飞白鹭，阴阴夏木啭黄鹂。山中习静观朝槿，松下清斋折露葵。野老与人争席罢，海鸥何事更相疑。"
 
 	mh := md5.Sum([]byte(objectValue))
 	md5B64 := base64.StdEncoding.EncodeToString(mh[:])
@@ -262,7 +337,19 @@ func (s *OssCrcSuite) TestSpecifyContentMD5(c *C) {
 
 	// AppendObject
 	var nextPos int64
-	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos, ContentMD5(md5B64))
+	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
+	c.Assert(err, IsNil)
+	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
+	c.Assert(err, IsNil)
+
+	err = s.bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	nextPos = 0
+	var initCRC uint64
+	nextPos, initCRC, err = s.bucket.AppendObjectWithCRC(objectName, strings.NewReader(objectValue), nextPos, initCRC)
+	c.Assert(err, IsNil)
+	nextPos, initCRC, err = s.bucket.AppendObjectWithCRC(objectName, strings.NewReader(objectValue), nextPos, initCRC)
 	c.Assert(err, IsNil)
 
 	err = s.bucket.DeleteObject(objectName)
@@ -284,3 +371,17 @@ func (s *OssCrcSuite) TestSpecifyContentMD5(c *C) {
 	err = s.bucket.DeleteObject(objectName)
 	c.Assert(err, IsNil)
 }
+
+// TestCopyObjectToOrFromNegative
+func (s *OssCrcSuite) TestAppendObjectNegative(c *C) {
+	objectName := objectNamePrefix + "taoncrc"
+	objectValue := "空山不见人，但闻人语响。返影入深林，复照青苔上。"
+	var nextPos int64
+	var initCRC uint64
+
+	nextPos, initCRC, err := s.bucket.AppendObjectWithCRC(objectName, strings.NewReader(objectValue), nextPos, initCRC)
+	c.Assert(err, IsNil)
+
+	nextPos, initCRC, err = s.bucket.AppendObjectWithCRC(objectName, strings.NewReader(objectValue), nextPos, initCRC-1)
+	c.Assert(err, NotNil)
+}