crc

oss/bucket.go

@@ -2,8 +2,13 @@ package oss
 
 import (
 	"bytes"
+	"crypto/md5"
+	"encoding/base64"
 	"encoding/xml"
+	"hash"
+	"hash/crc64"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"strconv"
@@ -32,8 +37,15 @@ func (bucket Bucket) PutObject(objectKey string, reader io.Reader, options ...Op
 	if err != nil {
 		return err
 	}
-
 	defer resp.body.Close()
+
+	if bucket.getConfig().IsEnableCRC {
+		err = checkCRC(resp, "PutObject")
+		if err != nil {
+			return err
+		}
+	}
+
 	return checkRespCode(resp.statusCode, []int{http.StatusOK})
 }
 
@@ -60,6 +72,14 @@ func (bucket Bucket) PutObjectFromFile(objectKey, filePath string, options ...Op
 		return err
 	}
 	defer resp.body.Close()
+
+	if bucket.getConfig().IsEnableCRC {
+		err = checkCRC(resp, "PutObjectFromFile")
+		if err != nil {
+			return err
+		}
+	}
+
 	return checkRespCode(resp.statusCode, []int{http.StatusOK})
 }
 
@@ -82,6 +102,35 @@ func (bucket Bucket) GetObject(objectKey string, options ...Option) (io.ReadClos
 	return resp.body, nil
 }
 
+//
+// GetObjectWithCRC 下载文件同时计算CRC。
+//
+// objectKey 下载的文件名称。
+// options   对象的属性限制项，可选值有Range、IfModifiedSince、IfUnmodifiedSince、IfMatch、
+// IfNoneMatch、AcceptEncoding，详细请参考
+// https://help.aliyun.com/document_detail/oss/api-reference/object/GetObject.html
+//
+// io.ReadCloser  body，文件内容，读取数据后需要close。error为nil时有效。
+// hash.Hash64 clientCRCCalculator，读取数据的CRC计算器。数据读取介绍通过clientCRCCalculator.Sum64()获取。
+// int64 serverCRC 服务器端存储数据的CRC值。
+// error  操作无错误为nil，非nil为错误信息。
+//
+func (bucket Bucket) GetObjectWithCRC(objectKey string, options ...Option) (body io.ReadCloser, clientCRCCalculator hash.Hash64, serverCRC uint64, err error) {
+	resp, err := bucket.do("GET", objectKey, "", "", options, nil)
+	if err != nil {
+		return
+	}
+
+	body = resp.body
+	hasRange, _, _ := isOptionSet(options, HTTPHeaderRange)
+	if bucket.getConfig().IsEnableCRC && !hasRange {
+		serverCRC = resp.serverCRC
+		clientCRCCalculator = crc64.New(crcTable())
+		body = ioutil.NopCloser(io.TeeReader(body, clientCRCCalculator))
+	}
+	return
+}
+
 //
 // GetObjectToFile 下载文件。
 //
@@ -97,7 +146,7 @@ func (bucket Bucket) GetObjectToFile(objectKey, filePath string, options ...Opti
 		return err
 	}
 	defer resp.body.Close()
-	
+
 	// 如果文件不存在则创建，存在则清空
 	fd, err := os.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0660)
 	if err != nil {
@@ -105,11 +154,29 @@ func (bucket Bucket) GetObjectToFile(objectKey, filePath string, options ...Opti
 	}
 	defer fd.Close()
 
-	_, err = io.Copy(fd, resp.body)
+	// 读取数据同时计算CRC
+	body := resp.body
+	var crc hash.Hash64
+	hasRange, _, _ := isOptionSet(options, HTTPHeaderRange)
+	if bucket.getConfig().IsEnableCRC && !hasRange {
+		crc = crc64.New(crcTable())
+		body = ioutil.NopCloser(io.TeeReader(body, crc))
+	}
+
+	_, err = io.Copy(fd, body)
 	if err != nil {
 		return err
 	}
 
+	// 比较CRC值
+	if bucket.getConfig().IsEnableCRC && !hasRange {
+		resp.clientCRC = crc.Sum64()
+		err = checkCRC(resp, "GetObjectToFile")
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -171,7 +238,7 @@ func (bucket Bucket) CopyObjectFrom(srcBucketName, srcObjectKey, destObjectKey s
 		return out, err
 	}
 
-	return srcBucket.copy(srcObjectKey, destBucketName, destObjectKey, options...);
+	return srcBucket.copy(srcObjectKey, destBucketName, destObjectKey, options...)
 }
 
 func (bucket Bucket) copy(srcObjectKey, destBucketName, destObjectKey string, options ...Option) (CopyObjectResult, error) {
@@ -272,6 +339,9 @@ func (bucket Bucket) DeleteObjects(objectKeys []string, options ...Option) (Dele
 
 	contentType := http.DetectContentType(buffer.Bytes())
 	options = append(options, ContentType(contentType))
+	sum := md5.Sum(bs)
+	b64 := base64.StdEncoding.EncodeToString(sum[:])
+	options = append(options, ContentMD5(b64))
 	resp, err := bucket.do("POST", "", "delete"+encode, "delete", options, buffer)
 	if err != nil {
 		return out, err

oss/client.go

@@ -5,7 +5,6 @@ package oss
 import (
 	"bytes"
 	"encoding/xml"
-	"fmt"
 	"io"
 	"net/http"
 	"strings"
@@ -248,8 +247,6 @@ func (client Client) SetBucketLifecycle(bucketName string, rules []LifecycleRule
 	buffer := new(bytes.Buffer)
 	buffer.Write(bs)
 
-	fmt.Println("xml:", string(bs))
-
 	contentType := http.DetectContentType(buffer.Bytes())
 	headers := map[string]string{}
 	headers[HTTPHeaderContentType] = contentType
@@ -398,7 +395,6 @@ func (client Client) SetBucketLogging(bucketName, targetBucket, targetPrefix str
 
 	buffer := new(bytes.Buffer)
 	buffer.Write(bs)
-	fmt.Println(isEnable, "; xml: ", string(bs))
 
 	contentType := http.DetectContentType(buffer.Bytes())
 	headers := map[string]string{}
@@ -650,7 +646,7 @@ func Timeout(connectTimeoutSec, readWriteTimeout int64) ClientOption {
 }
 
 //
-// SecurityToken 临时用户设置SecurityToken
+// SecurityToken 临时用户设置SecurityToken。
 //
 // token STS token
 //
@@ -663,7 +659,7 @@ func SecurityToken(token string) ClientOption {
 //
 // EnableMD5 是否启用MD5校验，默认启用。
 //
-// isEnableMD5 true启用MD5校验，false不启用MD5校验，默认true
+// isEnableMD5 true启用MD5校验，false不启用MD5校验
 //
 func EnableMD5(isEnableMD5 bool) ClientOption {
 	return func(client *Client) {
@@ -671,6 +667,28 @@ func EnableMD5(isEnableMD5 bool) ClientOption {
 	}
 }
 
+//
+// MD5ThresholdCalcInMemory 使用内存计算MD5值的上限，默认16MB。
+//
+// threshold 单位Byte。上传内容小于threshold在MD5在内存中计算，大于使用临时文件计算MD5
+//
+func MD5ThresholdCalcInMemory(threshold int64) ClientOption {
+	return func(client *Client) {
+		client.Config.MD5Threshold = threshold
+	}
+}
+
+//
+// EnableCRC 上传是否启用CRC校验，默认启用。
+//
+// isEnableCRC true启用CRC校验，false不启用CRC校验
+//
+func EnableCRC(isEnableCRC bool) ClientOption {
+	return func(client *Client) {
+		client.Config.IsEnableCRC = isEnableCRC
+	}
+}
+
 //
 // Proxy 设置代理服务器，默认不使用代理。
 //

oss/conf.go

@@ -24,12 +24,14 @@ type Config struct {
 	SecurityToken   string      // STS Token
 	IsCname         bool        // Endpoint是否是CNAME
 	HTTPTimeout     HTTPTimeout // HTTP的超时时间设置
-	IsEnableMD5     bool        // 上传数据时是否启用MD5校验
 	IsUseProxy      bool        // 是否使用代理
 	ProxyHost       string      // 代理服务器地址
 	IsAuthProxy     bool        // 代理服务器是否使用用户认证
 	ProxyUser       string      // 代理服务器认证用户名
 	ProxyPassword   string      // 代理服务器认证密码
+	IsEnableMD5     bool        // 上传数据时是否启用MD5校验
+	MD5Threshold    int64       // 内存中计算MD5的上线大小，大于该值启用临时文件，单位Byte
+	IsEnableCRC     bool        // 上传数据时是否启用CRC64校验
 }
 
 // 获取默认配置
@@ -45,7 +47,6 @@ func getDefaultOssConfig() *Config {
 	config.Timeout = 60 // seconds
 	config.SecurityToken = ""
 	config.IsCname = false
-	config.IsEnableMD5 = true
 
 	config.HTTPTimeout.ConnectTimeout = time.Second * 30   // 30s
 	config.HTTPTimeout.ReadWriteTimeout = time.Second * 60 // 60s
@@ -58,5 +59,9 @@ func getDefaultOssConfig() *Config {
 	config.ProxyUser = ""
 	config.ProxyPassword = ""
 
+	config.MD5Threshold = 16 * 1024 * 1024 // 16MB
+	config.IsEnableMD5 = false
+	config.IsEnableCRC = true
+
 	return &config
 }

oss/conn.go

@@ -6,6 +6,8 @@ import (
 	"encoding/base64"
 	"encoding/xml"
 	"fmt"
+	"hash"
+	"hash/crc64"
 	"io"
 	"io/ioutil"
 	"net"
@@ -28,6 +30,8 @@ type Response struct {
 	statusCode int
 	headers    http.Header
 	body       io.ReadCloser
+	clientCRC  uint64
+	serverCRC  uint64
 }
 
 // Do 处理请求，返回响应结果。
@@ -54,7 +58,14 @@ func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource st
 		Header:     make(http.Header),
 		Host:       uri.Host,
 	}
-	conn.handleBody(req, data)
+
+	fd, crc := conn.handleBody(req, data)
+	if fd != nil {
+		defer func() {
+			fd.Close()
+			os.Remove(fd.Name())
+		}()
+	}
 
 	date := time.Now().UTC().Format(http.TimeFormat)
 	req.Header.Set(HTTPHeaderDate, date)
@@ -120,16 +131,16 @@ func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource st
 		return nil, err
 	}
 
-	return conn.handleResponse(resp)
+	return conn.handleResponse(resp, crc)
 }
 
 // handle request body
-func (conn Conn) handleBody(req *http.Request, body io.Reader) {
-	rc, ok := body.(io.ReadCloser)
-	if !ok && body != nil {
-		rc = ioutil.NopCloser(body)
-	}
-	req.Body = rc
+func (conn Conn) handleBody(req *http.Request, body io.Reader) (*os.File, hash.Hash64) {
+	var file *os.File
+	var crc hash.Hash64
+	reader := body
+
+	// length
 	switch v := body.(type) {
 	case *bytes.Buffer:
 		req.ContentLength = int64(v.Len())
@@ -143,13 +154,43 @@ func (conn Conn) handleBody(req *http.Request, body io.Reader) {
 	req.Header.Set(HTTPHeaderContentLength, strconv.FormatInt(req.ContentLength, 10))
 
 	// md5
-	if req.Body != nil && conn.config.IsEnableMD5 {
-		buf, _ := ioutil.ReadAll(req.Body)
-		req.Body = ioutil.NopCloser(bytes.NewReader(buf))
-		sum := md5.Sum(buf)
-		b64 := base64.StdEncoding.EncodeToString(sum[:])
-		req.Header.Set(HTTPHeaderContentMD5, b64)
+	if body != nil && conn.config.IsEnableMD5 && req.Header.Get(HTTPHeaderContentMD5) == "" {
+		if req.ContentLength == 0 || req.ContentLength > conn.config.MD5Threshold {
+			// huge body, use temporary file
+			file, _ = ioutil.TempFile(os.TempDir(), TempFilePrefix)
+			if file != nil {
+				io.Copy(file, body)
+				file.Seek(0, os.SEEK_SET)
+				md5 := md5.New()
+				io.Copy(md5, file)
+				sum := md5.Sum(nil)
+				b64 := base64.StdEncoding.EncodeToString(sum[:])
+				req.Header.Set(HTTPHeaderContentMD5, b64)
+				file.Seek(0, os.SEEK_SET)
+				reader = file
+			}
+		} else {
+			// small body, use memory
+			buf, _ := ioutil.ReadAll(body)
+			sum := md5.Sum(buf)
+			b64 := base64.StdEncoding.EncodeToString(sum[:])
+			req.Header.Set(HTTPHeaderContentMD5, b64)
+			reader = bytes.NewReader(buf)
+		}
 	}
+
+	if reader != nil && conn.config.IsEnableCRC {
+		crc = crc64.New(crcTable())
+		reader = io.TeeReader(reader, crc)
+	}
+
+	rc, ok := reader.(io.ReadCloser)
+	if !ok && reader != nil {
+		rc = ioutil.NopCloser(reader)
+	}
+	req.Body = rc
+
+	return file, crc
 }
 
 func tryGetFileSize(f *os.File) int64 {
@@ -158,7 +199,10 @@ func tryGetFileSize(f *os.File) int64 {
 }
 
 // handle response
-func (conn Conn) handleResponse(resp *http.Response) (*Response, error) {
+func (conn Conn) handleResponse(resp *http.Response, crc hash.Hash64) (*Response, error) {
+	var cliCRC uint64
+	var srvCRC uint64
+
 	statusCode := resp.StatusCode
 	if statusCode >= 400 && statusCode <= 505 {
 		// 4xx and 5xx indicate that the operation has error occurred
@@ -183,7 +227,7 @@ func (conn Conn) handleResponse(resp *http.Response) (*Response, error) {
 		return &Response{
 			statusCode: resp.StatusCode,
 			headers:    resp.Header,
-			body:       ioutil.NopCloser(bytes.NewReader(respBody)), // restore the body//
+			body:       ioutil.NopCloser(bytes.NewReader(respBody)), // restore the body
 		}, err
 	} else if statusCode >= 300 && statusCode <= 307 {
 		// oss use 3xx, but response has no body
@@ -195,11 +239,18 @@ func (conn Conn) handleResponse(resp *http.Response) (*Response, error) {
 		}, err
 	}
 
+	if conn.config.IsEnableCRC && crc != nil {
+		cliCRC = crc.Sum64()
+	}
+	srvCRC, _ = strconv.ParseUint(resp.Header.Get(HTTPHeaderOssCRC64), 10, 64)
+
 	// 2xx, successful
 	return &Response{
 		statusCode: resp.StatusCode,
 		headers:    resp.Header,
 		body:       resp.Body,
+		clientCRC:  cliCRC,
+		serverCRC:  srvCRC,
 	}, nil
 }
 

oss/const.go

@@ -68,6 +68,7 @@ const (
 	HTTPHeaderOssMetadataDirective           = "X-Oss-Metadata-Directive"
 	HTTPHeaderOssNextAppendPosition          = "X-Oss-Next-Append-Position"
 	HTTPHeaderOssRequestID                   = "X-Oss-Request-Id"
+	HTTPHeaderOssCRC64                       = "X-Oss-Hash-Crc64ecma"
 )
 
 // 其它常量
@@ -75,5 +76,7 @@ const (
 	MaxPartSize = 5 * 1024 * 1024 * 1024 // 文件片最大值，5GB
 	MinPartSize = 100 * 1024             // 文件片最小值，100KB
 
-	Version = "1.1.0" // Go sdk版本
+	TempFilePrefix = "oss-go-temp-" // 临时文件前缀
+
+	Version = "1.2.0" // Go sdk版本
 )

oss/crc_test.go

@@ -0,0 +1,286 @@
+package oss
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	. "gopkg.in/check.v1"
+)
+
+type OssCrcSuite struct {
+	client *Client
+	bucket *Bucket
+}
+
+var _ = Suite(&OssCrcSuite{})
+
+// Run once when the suite starts running
+func (s *OssCrcSuite) SetUpSuite(c *C) {
+	client, err := New(endpoint, accessID, accessKey)
+	c.Assert(err, IsNil)
+	s.client = client
+
+	s.client.CreateBucket(bucketName)
+	time.Sleep(5 * time.Second)
+
+	bucket, err := s.client.Bucket(bucketName)
+	c.Assert(err, IsNil)
+	s.bucket = bucket
+
+	testLogger.Println("test crc started")
+}
+
+// Run before each test or benchmark starts running
+func (s *OssCrcSuite) TearDownSuite(c *C) {
+	// Delete Part
+	lmur, err := s.bucket.ListMultipartUploads()
+	c.Assert(err, IsNil)
+
+	for _, upload := range lmur.Uploads {
+		var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName,
+			Key: upload.Key, UploadID: upload.UploadID}
+		err = s.bucket.AbortMultipartUpload(imur)
+		c.Assert(err, IsNil)
+	}
+
+	// Delete Objects
+	lor, err := s.bucket.ListObjects()
+	c.Assert(err, IsNil)
+
+	for _, object := range lor.Objects {
+		err = s.bucket.DeleteObject(object.Key)
+		c.Assert(err, IsNil)
+	}
+
+	testLogger.Println("test crc completed")
+}
+
+// Run after each test or benchmark runs
+func (s *OssCrcSuite) SetUpTest(c *C) {
+	err := removeTempFiles("../oss", ".jpg")
+	c.Assert(err, IsNil)
+}
+
+// Run once after all tests or benchmarks have finished running
+func (s *OssCrcSuite) TearDownTest(c *C) {
+	err := removeTempFiles("../oss", ".jpg")
+	c.Assert(err, IsNil)
+}
+
+// TestEnableCRCAndMD5 开启MD5和CRC校验
+func (s *OssCrcSuite) TestEnableCRCAndMD5(c *C) {
+	objectName := objectNamePrefix + "tecam"
+	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
+	newFileName := "BingWallpaper-2015-11-07-2.jpg"
+	objectValue := "大江东去，浪淘尽，千古风流人物。 故垒西边，人道是、三国周郎赤壁。 乱石穿空，惊涛拍岸，卷起千堆雪。 江山如画，一时多少豪杰。" +
+		"遥想公谨当年，小乔初嫁了，雄姿英发。 羽扇纶巾，谈笑间、樯橹灰飞烟灭。故国神游，多情应笑我，早生华发，人生如梦，一尊还酹江月。"
+
+	client, err := New(endpoint, accessID, accessKey, EnableCRC(true), EnableMD5(true), MD5ThresholdCalcInMemory(200*1024))
+	c.Assert(err, IsNil)
+	bucket, err := client.Bucket(bucketName)
+	c.Assert(err, IsNil)
+
+	// PutObject
+	err = bucket.PutObject(objectName, strings.NewReader(objectValue))
+	c.Assert(err, IsNil)
+
+	// GetObject
+	body, err := bucket.GetObject(objectName)
+	c.Assert(err, IsNil)
+	_, err = ioutil.ReadAll(body)
+	c.Assert(err, IsNil)
+	body.Close()
+
+	// GetObjectWithCRC
+	body, calcCRC, srvCRC, err := bucket.GetObjectWithCRC(objectName)
+	c.Assert(err, IsNil)
+	str, err := readBody(body)
+	c.Assert(err, IsNil)
+	c.Assert(str, Equals, objectValue)
+	c.Assert(calcCRC.Sum64(), Equals, srvCRC)
+
+	// PutObjectFromFile
+	err = bucket.PutObjectFromFile(objectName, fileName)
+	c.Assert(err, IsNil)
+
+	// GetObjectToFile
+	err = bucket.GetObjectToFile(objectName, newFileName)
+	c.Assert(err, IsNil)
+	eq, err := compareFiles(fileName, newFileName)
+	c.Assert(err, IsNil)
+	c.Assert(eq, Equals, true)
+
+	// DeleteObject
+	err = bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	// AppendObject
+	var nextPos int64
+	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
+	c.Assert(err, IsNil)
+
+	err = s.bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	//	MultipartUpload
+	chunks, err := SplitFileByPartSize(fileName, 100*1024)
+	imurUpload, err := bucket.InitiateMultipartUpload(objectName)
+	c.Assert(err, IsNil)
+	var partsUpload []UploadPart
+
+	for _, chunk := range chunks {
+		part, err := bucket.UploadPartFromFile(imurUpload, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number))
+		c.Assert(err, IsNil)
+		partsUpload = append(partsUpload, part)
+	}
+
+	_, err = bucket.CompleteMultipartUpload(imurUpload, partsUpload)
+	c.Assert(err, IsNil)
+
+	// Check MultipartUpload
+	err = bucket.GetObjectToFile(objectName, newFileName)
+	c.Assert(err, IsNil)
+	eq, err = compareFiles(fileName, newFileName)
+	c.Assert(err, IsNil)
+	c.Assert(eq, Equals, true)
+
+	// DeleteObjects
+	_, err = bucket.DeleteObjects([]string{objectName})
+	c.Assert(err, IsNil)
+}
+
+// TestDisableCRCAndMD5 关闭MD5和CRC校验
+func (s *OssCrcSuite) TestDisableCRCAndMD5(c *C) {
+	objectName := objectNamePrefix + "tdcam"
+	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
+	newFileName := "BingWallpaper-2015-11-07-3.jpg"
+	objectValue := "大江东去，浪淘尽，千古风流人物。 故垒西边，人道是、三国周郎赤壁。 乱石穿空，惊涛拍岸，卷起千堆雪。 江山如画，一时多少豪杰。" +
+		"遥想公谨当年，小乔初嫁了，雄姿英发。 羽扇纶巾，谈笑间、樯橹灰飞烟灭。故国神游，多情应笑我，早生华发，人生如梦，一尊还酹江月。"
+
+	client, err := New(endpoint, accessID, accessKey, EnableCRC(false), EnableMD5(false))
+	c.Assert(err, IsNil)
+	bucket, err := client.Bucket(bucketName)
+	c.Assert(err, IsNil)
+
+	// PutObject
+	err = bucket.PutObject(objectName, strings.NewReader(objectValue))
+	c.Assert(err, IsNil)
+
+	// GetObject
+	body, err := bucket.GetObject(objectName)
+	c.Assert(err, IsNil)
+	_, err = ioutil.ReadAll(body)
+	c.Assert(err, IsNil)
+	body.Close()
+
+	// GetObjectWithCRC
+	body, _, _, err = bucket.GetObjectWithCRC(objectName)
+	c.Assert(err, IsNil)
+	str, err := readBody(body)
+	c.Assert(err, IsNil)
+	c.Assert(str, Equals, objectValue)
+
+	// PutObjectFromFile
+	err = bucket.PutObjectFromFile(objectName, fileName)
+	c.Assert(err, IsNil)
+
+	// GetObjectToFile
+	err = bucket.GetObjectToFile(objectName, newFileName)
+	c.Assert(err, IsNil)
+	eq, err := compareFiles(fileName, newFileName)
+	c.Assert(err, IsNil)
+	c.Assert(eq, Equals, true)
+
+	// DeleteObject
+	err = bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	// AppendObject
+	var nextPos int64
+	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos)
+	c.Assert(err, IsNil)
+
+	err = s.bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	//	MultipartUpload
+	chunks, err := SplitFileByPartSize(fileName, 100*1024)
+	imurUpload, err := bucket.InitiateMultipartUpload(objectName)
+	c.Assert(err, IsNil)
+	var partsUpload []UploadPart
+
+	for _, chunk := range chunks {
+		part, err := bucket.UploadPartFromFile(imurUpload, fileName, chunk.Offset, chunk.Size, (int)(chunk.Number))
+		c.Assert(err, IsNil)
+		partsUpload = append(partsUpload, part)
+	}
+
+	_, err = bucket.CompleteMultipartUpload(imurUpload, partsUpload)
+	c.Assert(err, IsNil)
+
+	// Check MultipartUpload
+	err = bucket.GetObjectToFile(objectName, newFileName)
+	c.Assert(err, IsNil)
+	eq, err = compareFiles(fileName, newFileName)
+	c.Assert(err, IsNil)
+	c.Assert(eq, Equals, true)
+
+	// DeleteObjects
+	_, err = bucket.DeleteObjects([]string{objectName})
+	c.Assert(err, IsNil)
+}
+
+// TestDisableCRCAndMD5 关闭MD5和CRC校验
+func (s *OssCrcSuite) TestSpecifyContentMD5(c *C) {
+	objectName := objectNamePrefix + "tdcam"
+	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
+	objectValue := "大江东去，浪淘尽，千古风流人物。 故垒西边，人道是、三国周郎赤壁。 乱石穿空，惊涛拍岸，卷起千堆雪。 江山如画，一时多少豪杰。" +
+		"遥想公谨当年，小乔初嫁了，雄姿英发。 羽扇纶巾，谈笑间、樯橹灰飞烟灭。故国神游，多情应笑我，早生华发，人生如梦，一尊还酹江月。"
+
+	mh := md5.Sum([]byte(objectValue))
+	md5B64 := base64.StdEncoding.EncodeToString(mh[:])
+
+	// PutObject
+	err := s.bucket.PutObject(objectName, strings.NewReader(objectValue), ContentMD5(md5B64))
+	c.Assert(err, IsNil)
+
+	// PutObjectFromFile
+	file, err := os.Open(fileName)
+	md5 := md5.New()
+	io.Copy(md5, file)
+	mdHex := base64.StdEncoding.EncodeToString(md5.Sum(nil)[:])
+	err = s.bucket.PutObjectFromFile(objectName, fileName, ContentMD5(mdHex))
+	c.Assert(err, IsNil)
+
+	err = s.bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	// AppendObject
+	var nextPos int64
+	nextPos, err = s.bucket.AppendObject(objectName, strings.NewReader(objectValue), nextPos, ContentMD5(md5B64))
+	c.Assert(err, IsNil)
+
+	err = s.bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+
+	//	MultipartUpload
+	imurUpload, err := s.bucket.InitiateMultipartUpload(objectName)
+	c.Assert(err, IsNil)
+
+	var partsUpload []UploadPart
+	part, err := s.bucket.UploadPart(imurUpload, strings.NewReader(objectValue), (int64)(len([]byte(objectValue))), 1)
+	c.Assert(err, IsNil)
+	partsUpload = append(partsUpload, part)
+
+	_, err = s.bucket.CompleteMultipartUpload(imurUpload, partsUpload)
+	c.Assert(err, IsNil)
+
+	// DeleteObject
+	err = s.bucket.DeleteObject(objectName)
+	c.Assert(err, IsNil)
+}

oss/error.go

@@ -20,7 +20,7 @@ type ServiceError struct {
 
 // Implement interface error
 func (e ServiceError) Error() string {
-	return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s", 
+	return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s",
 		e.StatusCode, e.Code, e.Message, e.RequestID)
 }
 
@@ -59,3 +59,24 @@ func checkRespCode(respCode int, allowed []int) error {
 	}
 	return UnexpectedStatusCodeError{allowed, respCode}
 }
+
+// CRCCheckError is returned when crc check is inconsistent between client and server
+type CRCCheckError struct {
+	clientCRC uint64 // 客户端计算的CRC64值
+	serverCRC uint64 // 服务端计算的CRC64值
+	operation string // 上传操作，如PutObject/AppendObject/UploadPart等
+	requestID string // 本次操作的RequestID
+}
+
+// Implement interface error
+func (e CRCCheckError) Error() string {
+	return fmt.Sprintf("oss: the crc of %s is inconsistent, client %d but server %d; request id is %s",
+		e.operation, e.clientCRC, e.serverCRC, e.requestID)
+}
+
+func checkCRC(resp *Response, operation string) error {
+	if resp.clientCRC == resp.serverCRC {
+		return nil
+	}
+	return CRCCheckError{resp.clientCRC, resp.serverCRC, operation, resp.headers.Get(HTTPHeaderOssRequestID)}
+}

oss/multipart.go

@@ -63,6 +63,13 @@ func (bucket Bucket) UploadPart(imur InitiateMultipartUploadResult, reader io.Re
 	}
 	defer resp.body.Close()
 
+	if bucket.getConfig().IsEnableCRC {
+		err = checkCRC(resp, "UploadPart")
+		if err != nil {
+			return part, err
+		}
+	}
+
 	part.ETag = resp.headers.Get(HTTPHeaderEtag)
 	part.PartNumber = partNumber
 	return part, nil
@@ -98,6 +105,13 @@ func (bucket Bucket) UploadPartFromFile(imur InitiateMultipartUploadResult, file
 	}
 	defer resp.body.Close()
 
+	if bucket.getConfig().IsEnableCRC {
+		err = checkCRC(resp, "UploadPartFromFile")
+		if err != nil {
+			return part, err
+		}
+	}
+
 	part.ETag = resp.headers.Get(HTTPHeaderEtag)
 	part.PartNumber = partNumber
 	return part, nil

oss/option.go

@@ -65,6 +65,11 @@ func ContentEncoding(value string) Option {
 	return setHeader(HTTPHeaderContentEncoding, value)
 }
 
+// ContentMD5 is an option to set Content-MD5 header
+func ContentMD5(value string) Option {
+	return setHeader(HTTPHeaderContentMD5, value)
+}
+
 // Expires is an option to set Expires header
 func Expires(t time.Time) Option {
 	return setHeader(HTTPHeaderExpires, t.Format(http.TimeFormat))
@@ -317,3 +322,19 @@ func findOption(options []Option, param, defaultVal string) (string, error) {
 	}
 	return defaultVal, nil
 }
+
+func isOptionSet(options []Option, option string) (bool, string, error) {
+	params := map[string]optionValue{}
+	for _, option := range options {
+		if option != nil {
+			if err := option(params); err != nil {
+				return false, "", err
+			}
+		}
+	}
+
+	if val, ok := params[option]; ok {
+		return true, val.Value, nil
+	}
+	return false, "", nil
+}

oss/upload_test.go

@@ -395,7 +395,7 @@ func (s *OssUploadSuite) TestUploadRoutineWithRecoveryNegative(c *C) {
 func (s *OssUploadSuite) TestUploadLocalFileChange(c *C) {
 	objectName := objectNamePrefix + "tulfc"
 	fileName := "../sample/BingWallpaper-2015-11-07.jpg"
-	localFile := "../sample/BingWallpaper-2015-11-07-2.jpg"
+	localFile := "BingWallpaper-2015-11-07.jpg"
 	newFile := "upload-new-file-3.jpg"
 
 	os.Remove(localFile)

oss/utils.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"hash/crc64"
 	"net/http"
 	"os"
 	"os/exec"
@@ -157,3 +158,8 @@ func GetPartEnd(begin int64, total int64, per int64) int64 {
 	}
 	return begin + per - 1
 }
+
+// crcTable returns the Table constructed from the specified polynomial
+var crcTable = func() *crc64.Table {
+	return crc64.MakeTable(crc64.ECMA)
+}