Merge pull request #139 from aliyun/preview_1.9.3

Preview 1.9.3

oss/bucket.go

@@ -457,7 +457,7 @@ func (bucket Bucket) IsObjectExist(objectKey string) (bool, error) {
 
 	switch err.(type) {
 	case ServiceError:
-		if err.(ServiceError).StatusCode == 404 && err.(ServiceError).Code == "NoSuchKey" {
+		if err.(ServiceError).StatusCode == 404 {
 			return false, nil
 		}
 	}
@@ -557,7 +557,7 @@ func (bucket Bucket) GetObjectMeta(objectKey string, options ...Option) (http.He
 	params := map[string]interface{}{}
 	params["objectMeta"] = nil
 	//resp, err := bucket.do("GET", objectKey, "?objectMeta", "", nil, nil, nil)
-	resp, err := bucket.do("GET", objectKey, params, options, nil, nil)
+	resp, err := bucket.do("HEAD", objectKey, params, options, nil, nil)
 	if err != nil {
 		return nil, err
 	}

oss/client.go

@@ -18,8 +18,9 @@ import (
 type (
 	// Client OSS client
 	Client struct {
-		Config *Config // OSS client configuration
-		Conn   *Conn   // Send HTTP request
+		Config     *Config      // OSS client configuration
+		Conn       *Conn        // Send HTTP request
+		HTTPClient *http.Client //http.Client to use - if nil will make its own
 	}
 
 	// ClientOption client option such as UseCname, Timeout, SecurityToken.
@@ -51,8 +52,8 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption)
 
 	// OSS client
 	client := &Client{
-		config,
-		conn,
+		Config: config,
+		Conn:   conn,
 	}
 
 	// Client options parse
@@ -61,7 +62,7 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption)
 	}
 
 	// Create HTTP connection
-	err := conn.init(config, url)
+	err := conn.init(config, url, client.HTTPClient)
 
 	return client, err
 }
@@ -149,7 +150,7 @@ func (client Client) ListBuckets(options ...Option) (ListBucketsResult, error) {
 // IsBucketExist checks if the bucket exists
 //
 // bucketName    the bucket name.
-// 
+//
 // bool    true if it exists, and it's only valid when error is nil.
 // error    it's nil if no error, otherwise it's an error object.
 //
@@ -184,7 +185,7 @@ func (client Client) DeleteBucket(bucketName string) error {
 
 // GetBucketLocation gets the bucket location.
 //
-// Checks out the following link for more information : 
+// Checks out the following link for more information :
 // https://help.aliyun.com/document_detail/oss/user_guide/oss_concept/endpoint.html
 //
 // bucketName    the bucket name
@@ -253,7 +254,7 @@ func (client Client) GetBucketACL(bucketName string) (GetBucketACLResult, error)
 // bucketName    the bucket name.
 // rules    the lifecycle rules. There're two kind of rules: absolute time expiration and relative time expiration in days and day/month/year respectively.
 //          Check out sample/bucket_lifecycle.go for more details.
-// 
+//
 // error    it's nil if no error, otherwise it's an error object.
 //
 func (client Client) SetBucketLifecycle(bucketName string, rules []LifecycleRule) error {
@@ -300,7 +301,7 @@ func (client Client) DeleteBucketLifecycle(bucketName string) error {
 // GetBucketLifecycle gets the bucket's lifecycle settings.
 //
 // bucketName    the bucket name.
-// 
+//
 // GetBucketLifecycleResponse    the result object upon successful request. It's only valid when error is nil.
 // error    it's nil if no error, otherwise it's an error object.
 //
@@ -757,6 +758,15 @@ func AuthProxy(proxyHost, proxyUser, proxyPassword string) ClientOption {
 	}
 }
 
+//
+// HTTPClient sets the http.Client in use to the one passed in
+//
+func HTTPClient(HTTPClient *http.Client) ClientOption {
+	return func(client *Client) {
+		client.HTTPClient = HTTPClient
+	}
+}
+
 // Private
 func (client Client) do(method, bucketName string, params map[string]interface{},
 	headers map[string]string, data io.Reader) (*Response, error) {

oss/client_test.go

@@ -7,6 +7,7 @@ package oss
 import (
 	"log"
 	"math/rand"
+	"net/http"
 	"os"
 	"strings"
 	"testing"
@@ -1411,8 +1412,15 @@ func (s *OssClientSuite) TestClientOption(c *C) {
 	c.Assert(client.Conn.config.ProxyPassword, Equals, proxyPasswd)
 
 	client, err = New(endpoint, accessID, accessKey, UserAgent("go sdk user agent"))
-
 	c.Assert(client.Conn.config.UserAgent, Equals, "go sdk user agent")
+
+	// Check we can overide the http.Client
+	httpClient := new(http.Client)
+	client, err = New(endpoint, accessID, accessKey, HTTPClient(httpClient))
+	c.Assert(client.HTTPClient, Equals, httpClient)
+	c.Assert(client.Conn.client, Equals, httpClient)
+	client, err = New(endpoint, accessID, accessKey)
+	c.Assert(client.HTTPClient, IsNil)
 }
 
 // TestProxy

oss/conf.go

@@ -13,26 +13,32 @@ type HTTPTimeout struct {
 	IdleConnTimeout  time.Duration
 }
 
+type HTTPMaxConns struct {
+	MaxIdleConns        int
+	MaxIdleConnsPerHost int
+}
+
 // Config defines oss configuration
 type Config struct {
-	Endpoint        string      // OSS endpoint
-	AccessKeyID     string      // AccessId
-	AccessKeySecret string      // AccessKey
-	RetryTimes      uint        // Retry count by default it's 5.
-	UserAgent       string      // SDK name/version/system information
-	IsDebug         bool        // Enable debug mode. Default is false.
-	Timeout         uint        // Timeout in seconds. By default it's 60.
-	SecurityToken   string      // STS Token
-	IsCname         bool        // If cname is in the endpoint.
-	HTTPTimeout     HTTPTimeout // HTTP timeout
-	IsUseProxy      bool        // Flag of using proxy.
-	ProxyHost       string      // Flag of using proxy host.
-	IsAuthProxy     bool        // Flag of needing authentication.
-	ProxyUser       string      // Proxy user
-	ProxyPassword   string      // Proxy password
-	IsEnableMD5     bool        // Flag of enabling MD5 for upload.
-	MD5Threshold    int64       // Memory footprint threshold for each MD5 computation (16MB is the default), in byte. When the data is more than that, temp file is used.
-	IsEnableCRC     bool        // Flag of enabling CRC for upload.
+	Endpoint        string       // OSS endpoint
+	AccessKeyID     string       // AccessId
+	AccessKeySecret string       // AccessKey
+	RetryTimes      uint         // Retry count by default it's 5.
+	UserAgent       string       // SDK name/version/system information
+	IsDebug         bool         // Enable debug mode. Default is false.
+	Timeout         uint         // Timeout in seconds. By default it's 60.
+	SecurityToken   string       // STS Token
+	IsCname         bool         // If cname is in the endpoint.
+	HTTPTimeout     HTTPTimeout  // HTTP timeout
+	HTTPMaxConns    HTTPMaxConns // Http max connections
+	IsUseProxy      bool         // Flag of using proxy.
+	ProxyHost       string       // Flag of using proxy host.
+	IsAuthProxy     bool         // Flag of needing authentication.
+	ProxyUser       string       // Proxy user
+	ProxyPassword   string       // Proxy password
+	IsEnableMD5     bool         // Flag of enabling MD5 for upload.
+	MD5Threshold    int64        // Memory footprint threshold for each MD5 computation (16MB is the default), in byte. When the data is more than that, temp file is used.
+	IsEnableCRC     bool         // Flag of enabling CRC for upload.
 }
 
 // getDefaultOssConfig gets the default configuration.
@@ -44,8 +50,8 @@ func getDefaultOssConfig() *Config {
 	config.AccessKeySecret = ""
 	config.RetryTimes = 5
 	config.IsDebug = false
-	config.UserAgent = userAgent
-	config.Timeout = 60  // Seconds
+	config.UserAgent = userAgent()
+	config.Timeout = 60 // Seconds
 	config.SecurityToken = ""
 	config.IsCname = false
 
@@ -54,6 +60,8 @@ func getDefaultOssConfig() *Config {
 	config.HTTPTimeout.HeaderTimeout = time.Second * 60    // 60s
 	config.HTTPTimeout.LongTimeout = time.Second * 300     // 300s
 	config.HTTPTimeout.IdleConnTimeout = time.Second * 50  // 50s
+	config.HTTPMaxConns.MaxIdleConns = 100
+	config.HTTPMaxConns.MaxIdleConnsPerHost = 100
 
 	config.IsUseProxy = false
 	config.ProxyHost = ""

oss/conn.go

@@ -30,22 +30,25 @@ type Conn struct {
 var signKeyList = []string{"acl", "uploads", "location", "cors", "logging", "website", "referer", "lifecycle", "delete", "append", "tagging", "objectMeta", "uploadId", "partNumber", "security-token", "position", "img", "style", "styleName", "replication", "replicationProgress", "replicationLocation", "cname", "bucketInfo", "comp", "qos", "live", "status", "vod", "startTime", "endTime", "symlink", "x-oss-process", "response-content-type", "response-content-language", "response-expires", "response-cache-control", "response-content-disposition", "response-content-encoding", "udf", "udfName", "udfImage", "udfId", "udfImageDesc", "udfApplication", "comp", "udfApplicationLog", "restore", "callback", "callback-var"}
 
 // init initializes Conn
-func (conn *Conn) init(config *Config, urlMaker *urlMaker) error {
-	// New transport
-	transport := newTransport(conn, config)
-
-	// Proxy
-	if conn.config.IsUseProxy {
-		proxyURL, err := url.Parse(config.ProxyHost)
-		if err != nil {
-			return err
+func (conn *Conn) init(config *Config, urlMaker *urlMaker, client *http.Client) error {
+	if client == nil {
+		// New transport
+		transport := newTransport(conn, config)
+
+		// Proxy
+		if conn.config.IsUseProxy {
+			proxyURL, err := url.Parse(config.ProxyHost)
+			if err != nil {
+				return err
+			}
+			transport.Proxy = http.ProxyURL(proxyURL)
 		}
-		transport.Proxy = http.ProxyURL(proxyURL)
+		client = &http.Client{Transport: transport}
 	}
 
 	conn.config = config
 	conn.url = urlMaker
-	conn.client = &http.Client{Transport: transport}
+	conn.client = client
 
 	return nil
 }
@@ -347,8 +350,10 @@ func (conn Conn) handleResponse(resp *http.Response, crc hash.Hash64) (*Response
 		}
 
 		if len(respBody) == 0 {
-			// No error in response body
-			err = fmt.Errorf("oss: service returned empty response body, status = %s, RequestId = %s", resp.Status, resp.Header.Get(HTTPHeaderOssRequestID))
+			err = ServiceError{
+				StatusCode: statusCode,
+				RequestID:  resp.Header.Get(HTTPHeaderOssRequestID),
+			}
 		} else {
 			// Response contains storage service error object, unmarshal
 			srvErr, errIn := serviceErrFromXML(respBody, resp.StatusCode,

oss/const.go

@@ -103,6 +103,7 @@ const (
 	HTTPHeaderOssObjectACL                   = "X-Oss-Object-Acl"
 	HTTPHeaderOssSecurityToken               = "X-Oss-Security-Token"
 	HTTPHeaderOssServerSideEncryption        = "X-Oss-Server-Side-Encryption"
+	HTTPHeaderOssServerSideEncryptionKeyID   = "X-Oss-Server-Side-Encryption-Key-Id"
 	HTTPHeaderOssCopySource                  = "X-Oss-Copy-Source"
 	HTTPHeaderOssCopySourceRange             = "X-Oss-Copy-Source-Range"
 	HTTPHeaderOssCopySourceIfMatch           = "X-Oss-Copy-Source-If-Match"

oss/error.go

@@ -14,14 +14,19 @@ type ServiceError struct {
 	Message    string   `xml:"Message"`   // The detail error message from OSS
 	RequestID  string   `xml:"RequestId"` // The UUID used to uniquely identify the request
 	HostID     string   `xml:"HostId"`    // The OSS server cluster's Id
+	Endpoint   string   `xml:"Endpoint"`
 	RawMessage string   // The raw messages from OSS
 	StatusCode int      // HTTP status code
 }
 
 // Error implements interface error
 func (e ServiceError) Error() string {
-	return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=%s, RequestId=%s",
-		e.StatusCode, e.Code, e.Message, e.RequestID)
+	if e.Endpoint == "" {
+		return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=\"%s\", RequestId=%s",
+			e.StatusCode, e.Code, e.Message, e.RequestID)
+	}
+	return fmt.Sprintf("oss: service returned error: StatusCode=%d, ErrorCode=%s, ErrorMessage=\"%s\", RequestId=%s, Endpoint=%s",
+		e.StatusCode, e.Code, e.Message, e.RequestID, e.Endpoint)
 }
 
 // UnexpectedStatusCodeError is returned when a storage service responds with neither an error

oss/error_test.go

@@ -3,6 +3,7 @@ package oss
 import (
 	"math"
 	"net/http"
+	"strings"
 
 	. "gopkg.in/check.v1"
 )
@@ -82,3 +83,29 @@ func (s *OssErrorSuite) TestCheckDownloadCRC(c *C) {
 	c.Assert(err, NotNil)
 	testLogger.Println("error:", err)
 }
+
+func (s *OssErrorSuite) TestServiceErrorEndPoint(c *C) {
+	xmlBody := `<?xml version="1.0" encoding="UTF-8"?>
+	<Error>
+	  <Code>AccessDenied</Code>
+	  <Message>The bucket you visit is not belong to you.</Message>
+	  <RequestId>5C1B5E9BD79A6B9B6466166E</RequestId>
+	  <HostId>oss-c-sdk-test-verify-b.oss-cn-shenzhen.aliyuncs.com</HostId>
+	</Error>`
+	serverError, _ := serviceErrFromXML([]byte(xmlBody), 403, "5C1B5E9BD79A6B9B6466166E")
+	errMsg := serverError.Error()
+	c.Assert(strings.Contains(errMsg, "Endpoint="), Equals, false)
+
+	xmlBodyWithEndPoint := `<?xml version="1.0" encoding="UTF-8"?>
+	<Error>
+      <Code>AccessDenied</Code>
+	  <Message>The bucket you are attempting to access must be addressed using the specified endpoint. Please send all future requests to this endpoint.</Message>
+	  <RequestId>5C1B595ED51820B569C6A12F</RequestId>
+	  <HostId>hello-hangzws.oss-cn-qingdao.aliyuncs.com</HostId>
+	  <Bucket>hello-hangzws</Bucket>
+	  <Endpoint>oss-cn-shenzhen.aliyuncs.com</Endpoint>
+	</Error>`
+	serverError, _ = serviceErrFromXML([]byte(xmlBodyWithEndPoint), 406, "5C1B595ED51820B569C6A12F")
+	errMsg = serverError.Error()
+	c.Assert(strings.Contains(errMsg, "Endpoint=oss-cn-shenzhen.aliyuncs.com"), Equals, true)
+}

oss/option.go

@@ -65,6 +65,11 @@ func ContentEncoding(value string) Option {
 	return setHeader(HTTPHeaderContentEncoding, value)
 }
 
+// ContentLanguage is an option to set Content-Language header
+func ContentLanguage(value string) Option {
+	return setHeader(HTTPHeaderContentLanguage, value)
+}
+
 // ContentMD5 is an option to set Content-MD5 header
 func ContentMD5(value string) Option {
 	return setHeader(HTTPHeaderContentMD5, value)
@@ -157,6 +162,11 @@ func ServerSideEncryption(value string) Option {
 	return setHeader(HTTPHeaderOssServerSideEncryption, value)
 }
 
+// ServerSideEncryptionKeyID is an option to set X-Oss-Server-Side-Encryption-Key-Id header
+func ServerSideEncryptionKeyID(value string) Option {
+	return setHeader(HTTPHeaderOssServerSideEncryptionKeyID, value)
+}
+
 // ObjectACL is an option to set X-Oss-Object-Acl header
 func ObjectACL(acl ACLType) Option {
 	return setHeader(HTTPHeaderOssObjectACL, string(acl))

oss/option_test.go

@@ -142,6 +142,16 @@ var headerTestcases = []optionTestCase{
 		key:    "X-Oss-Callback-Var",
 		value:  "JTdCJTIyeCUzQXZhcjElMjIlM0ElMjJ2YWx1ZTElMjIlMkMlMjJ4JTNBdmFyMiUyMiUzQSUyMnZhbHVlMiUyMiU3RA==",
 	},
+	{
+		option: ContentLanguage("zh-CN"),
+		key:    "Content-Language",
+		value:  "zh-CN",
+	},
+	{
+		option: ServerSideEncryptionKeyID("xossekid"),
+		key:    "X-Oss-Server-Side-Encryption-Key-Id",
+		value:  "xossekid",
+	},
 }
 
 func (s *OssOptionSuite) TestHeaderOptions(c *C) {

oss/transport_1_6.go

@@ -9,6 +9,7 @@ import (
 
 func newTransport(conn *Conn, config *Config) *http.Transport {
 	httpTimeOut := conn.config.HTTPTimeout
+	httpMaxConns := conn.config.HTTPMaxConns
 	// New Transport
 	transport := &http.Transport{
 		Dial: func(netw, addr string) (net.Conn, error) {
@@ -18,6 +19,7 @@ func newTransport(conn *Conn, config *Config) *http.Transport {
 			}
 			return newTimeoutConn(conn, httpTimeOut.ReadWriteTimeout, httpTimeOut.LongTimeout), nil
 		},
+		MaxIdleConnsPerHost:   httpMaxConns.MaxIdleConnsPerHost,
 		ResponseHeaderTimeout: httpTimeOut.HeaderTimeout,
 	}
 	return transport

oss/transport_1_7.go

@@ -9,6 +9,7 @@ import (
 
 func newTransport(conn *Conn, config *Config) *http.Transport {
 	httpTimeOut := conn.config.HTTPTimeout
+	httpMaxConns := conn.config.HTTPMaxConns
 	// New Transport
 	transport := &http.Transport{
 		Dial: func(netw, addr string) (net.Conn, error) {
@@ -18,6 +19,8 @@ func newTransport(conn *Conn, config *Config) *http.Transport {
 			}
 			return newTimeoutConn(conn, httpTimeOut.ReadWriteTimeout, httpTimeOut.LongTimeout), nil
 		},
+		MaxIdleConns:          httpMaxConns.MaxIdleConns,
+		MaxIdleConnsPerHost:   httpMaxConns.MaxIdleConnsPerHost,
 		IdleConnTimeout:       httpTimeOut.IdleConnTimeout,
 		ResponseHeaderTimeout: httpTimeOut.HeaderTimeout,
 	}

oss/upload.go

@@ -24,7 +24,7 @@ import (
 //
 func (bucket Bucket) UploadFile(objectKey, filePath string, partSize int64, options ...Option) error {
 	if partSize < MinPartSize || partSize > MaxPartSize {
-		return errors.New("oss: part size invalid range (1024KB, 5GB]")
+		return errors.New("oss: part size invalid range (100KB, 5GB]")
 	}
 
 	cpConf := getCpConfig(options)

oss/utils.go

@@ -16,11 +16,11 @@ import (
 
 // userAgent gets user agent
 // It has the SDK version information, OS information and GO version
-var userAgent = func() string {
+func userAgent() string {
 	sys := getSysInfo()
 	return fmt.Sprintf("aliyun-sdk-go/%s (%s/%s/%s;%s)", Version, sys.name,
 		sys.release, sys.machine, runtime.Version())
-}()
+}
 
 type sysInfo struct {
 	name    string // OS name such as windows/Linux

sample/config.go

@@ -6,6 +6,7 @@ const (
 	accessID   string = "<AccessKeyId>"
 	accessKey  string = "<AccessKeySecret>"
 	bucketName string = "<my-bucket>"
+	kmsID      string = "<KmsID>"
 
 	// The cname endpoint
 	// These information are required to run sample/cname_sample

sample/put_object.go

@@ -2,6 +2,8 @@ package sample
 
 import (
 	"bytes"
+	"encoding/base64"
+	"encoding/json"
 	"fmt"
 	"os"
 	"strings"
@@ -66,7 +68,35 @@ func PutObjectSample() {
 	}
 	fmt.Println("Object Meta:", props)
 
-	// Case 6: Big file's multipart upload. It supports concurrent upload with resumable upload.
+	// Case 6: Upload an object with sever side encrpytion kms and kms id specified
+	err = bucket.PutObject(objectKey, strings.NewReader(val), oss.ServerSideEncryption("KMS"), oss.ServerSideEncryptionKeyID(kmsID))
+	if err != nil {
+		HandleError(err)
+	}
+
+	// Case 7: Upload an object with callback
+	callbackMap := map[string]string{}
+	callbackMap["callbackUrl"] = "http://oss-demo.aliyuncs.com:23450"
+	callbackMap["callbackHost"] = "oss-cn-hangzhou.aliyuncs.com"
+	callbackMap["callbackBody"] = "filename=${object}&size=${size}&mimeType=${mimeType}"
+	callbackMap["callbackBodyType"] = "application/x-www-form-urlencoded"
+
+	callbackBuffer := bytes.NewBuffer([]byte{})
+	callbackEncoder := json.NewEncoder(callbackBuffer)
+	//do not encode '&' to "\u0026"
+	callbackEncoder.SetEscapeHTML(false)
+	err = callbackEncoder.Encode(callbackMap)
+	if err != nil {
+		HandleError(err)
+	}
+
+	callbackVal := base64.StdEncoding.EncodeToString(callbackBuffer.Bytes())
+	err = bucket.PutObject(objectKey, strings.NewReader(val), oss.Callback(callbackVal))
+	if err != nil {
+		HandleError(err)
+	}
+
+	// Case 8: Big file's multipart upload. It supports concurrent upload with resumable upload.
 	// multipart upload with 100K as part size. By default 1 coroutine is used and no checkpoint is used.
 	err = bucket.UploadFile(objectKey, localFile, 100*1024)
 	if err != nil {
@@ -85,7 +115,7 @@ func PutObjectSample() {
 		HandleError(err)
 	}
 
-	// Specify the local file path for checkpoint files. 
+	// Specify the local file path for checkpoint files.
 	// the 2nd parameter of Checkpoint can specify the file path, when the file path is empty, it will upload the directory.
 	err = bucket.UploadFile(objectKey, localFile, 100*1024, oss.Checkpoint(true, localFile+".cp"))
 	if err != nil {