Support for setting the maximum number of idle connections

oss/client.go

@@ -18,8 +18,9 @@ import (
 type (
 	// Client OSS client
 	Client struct {
-		Config *Config // OSS client configuration
-		Conn   *Conn   // Send HTTP request
+		Config     *Config      // OSS client configuration
+		Conn       *Conn        // Send HTTP request
+		HTTPClient *http.Client //http.Client to use - if nil will make its own
 	}
 
 	// ClientOption client option such as UseCname, Timeout, SecurityToken.
@@ -51,8 +52,8 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption)
 
 	// OSS client
 	client := &Client{
-		config,
-		conn,
+		Config: config,
+		Conn:   conn,
 	}
 
 	// Client options parse
@@ -61,7 +62,7 @@ func New(endpoint, accessKeyID, accessKeySecret string, options ...ClientOption)
 	}
 
 	// Create HTTP connection
-	err := conn.init(config, url)
+	err := conn.init(config, url, client.HTTPClient)
 
 	return client, err
 }
@@ -149,7 +150,7 @@ func (client Client) ListBuckets(options ...Option) (ListBucketsResult, error) {
 // IsBucketExist checks if the bucket exists
 //
 // bucketName    the bucket name.
-// 
+//
 // bool    true if it exists, and it's only valid when error is nil.
 // error    it's nil if no error, otherwise it's an error object.
 //
@@ -184,7 +185,7 @@ func (client Client) DeleteBucket(bucketName string) error {
 
 // GetBucketLocation gets the bucket location.
 //
-// Checks out the following link for more information : 
+// Checks out the following link for more information :
 // https://help.aliyun.com/document_detail/oss/user_guide/oss_concept/endpoint.html
 //
 // bucketName    the bucket name
@@ -253,7 +254,7 @@ func (client Client) GetBucketACL(bucketName string) (GetBucketACLResult, error)
 // bucketName    the bucket name.
 // rules    the lifecycle rules. There're two kind of rules: absolute time expiration and relative time expiration in days and day/month/year respectively.
 //          Check out sample/bucket_lifecycle.go for more details.
-// 
+//
 // error    it's nil if no error, otherwise it's an error object.
 //
 func (client Client) SetBucketLifecycle(bucketName string, rules []LifecycleRule) error {
@@ -300,7 +301,7 @@ func (client Client) DeleteBucketLifecycle(bucketName string) error {
 // GetBucketLifecycle gets the bucket's lifecycle settings.
 //
 // bucketName    the bucket name.
-// 
+//
 // GetBucketLifecycleResponse    the result object upon successful request. It's only valid when error is nil.
 // error    it's nil if no error, otherwise it's an error object.
 //
@@ -757,6 +758,15 @@ func AuthProxy(proxyHost, proxyUser, proxyPassword string) ClientOption {
 	}
 }
 
+//
+// HTTPClient sets the http.Client in use to the one passed in
+//
+func HTTPClient(HTTPClient *http.Client) ClientOption {
+	return func(client *Client) {
+		client.HTTPClient = HTTPClient
+	}
+}
+
 // Private
 func (client Client) do(method, bucketName string, params map[string]interface{},
 	headers map[string]string, data io.Reader) (*Response, error) {

oss/client_test.go

@@ -7,6 +7,7 @@ package oss
 import (
 	"log"
 	"math/rand"
+	"net/http"
 	"os"
 	"strings"
 	"testing"
@@ -1411,8 +1412,15 @@ func (s *OssClientSuite) TestClientOption(c *C) {
 	c.Assert(client.Conn.config.ProxyPassword, Equals, proxyPasswd)
 
 	client, err = New(endpoint, accessID, accessKey, UserAgent("go sdk user agent"))
-
 	c.Assert(client.Conn.config.UserAgent, Equals, "go sdk user agent")
+
+	// Check we can overide the http.Client
+	httpClient := new(http.Client)
+	client, err = New(endpoint, accessID, accessKey, HTTPClient(httpClient))
+	c.Assert(client.HTTPClient, Equals, httpClient)
+	c.Assert(client.Conn.client, Equals, httpClient)
+	client, err = New(endpoint, accessID, accessKey)
+	c.Assert(client.HTTPClient, IsNil)
 }
 
 // TestProxy

oss/conf.go

@@ -13,26 +13,32 @@ type HTTPTimeout struct {
 	IdleConnTimeout  time.Duration
 }
 
+type HTTPMaxConns struct {
+	MaxIdleConns        int
+	MaxIdleConnsPerHost int
+}
+
 // Config defines oss configuration
 type Config struct {
-	Endpoint        string      // OSS endpoint
-	AccessKeyID     string      // AccessId
-	AccessKeySecret string      // AccessKey
-	RetryTimes      uint        // Retry count by default it's 5.
-	UserAgent       string      // SDK name/version/system information
-	IsDebug         bool        // Enable debug mode. Default is false.
-	Timeout         uint        // Timeout in seconds. By default it's 60.
-	SecurityToken   string      // STS Token
-	IsCname         bool        // If cname is in the endpoint.
-	HTTPTimeout     HTTPTimeout // HTTP timeout
-	IsUseProxy      bool        // Flag of using proxy.
-	ProxyHost       string      // Flag of using proxy host.
-	IsAuthProxy     bool        // Flag of needing authentication.
-	ProxyUser       string      // Proxy user
-	ProxyPassword   string      // Proxy password
-	IsEnableMD5     bool        // Flag of enabling MD5 for upload.
-	MD5Threshold    int64       // Memory footprint threshold for each MD5 computation (16MB is the default), in byte. When the data is more than that, temp file is used.
-	IsEnableCRC     bool        // Flag of enabling CRC for upload.
+	Endpoint        string       // OSS endpoint
+	AccessKeyID     string       // AccessId
+	AccessKeySecret string       // AccessKey
+	RetryTimes      uint         // Retry count by default it's 5.
+	UserAgent       string       // SDK name/version/system information
+	IsDebug         bool         // Enable debug mode. Default is false.
+	Timeout         uint         // Timeout in seconds. By default it's 60.
+	SecurityToken   string       // STS Token
+	IsCname         bool         // If cname is in the endpoint.
+	HTTPTimeout     HTTPTimeout  // HTTP timeout
+	HTTPMaxConns    HTTPMaxConns // Http max connections
+	IsUseProxy      bool         // Flag of using proxy.
+	ProxyHost       string       // Flag of using proxy host.
+	IsAuthProxy     bool         // Flag of needing authentication.
+	ProxyUser       string       // Proxy user
+	ProxyPassword   string       // Proxy password
+	IsEnableMD5     bool         // Flag of enabling MD5 for upload.
+	MD5Threshold    int64        // Memory footprint threshold for each MD5 computation (16MB is the default), in byte. When the data is more than that, temp file is used.
+	IsEnableCRC     bool         // Flag of enabling CRC for upload.
 }
 
 // getDefaultOssConfig gets the default configuration.
@@ -54,6 +60,8 @@ func getDefaultOssConfig() *Config {
 	config.HTTPTimeout.HeaderTimeout = time.Second * 60    // 60s
 	config.HTTPTimeout.LongTimeout = time.Second * 300     // 300s
 	config.HTTPTimeout.IdleConnTimeout = time.Second * 50  // 50s
+	config.HTTPMaxConns.MaxIdleConns = 100
+	config.HTTPMaxConns.MaxIdleConnsPerHost = 100
 
 	config.IsUseProxy = false
 	config.ProxyHost = ""

oss/conn.go

@@ -30,22 +30,25 @@ type Conn struct {
 var signKeyList = []string{"acl", "uploads", "location", "cors", "logging", "website", "referer", "lifecycle", "delete", "append", "tagging", "objectMeta", "uploadId", "partNumber", "security-token", "position", "img", "style", "styleName", "replication", "replicationProgress", "replicationLocation", "cname", "bucketInfo", "comp", "qos", "live", "status", "vod", "startTime", "endTime", "symlink", "x-oss-process", "response-content-type", "response-content-language", "response-expires", "response-cache-control", "response-content-disposition", "response-content-encoding", "udf", "udfName", "udfImage", "udfId", "udfImageDesc", "udfApplication", "comp", "udfApplicationLog", "restore", "callback", "callback-var"}
 
 // init initializes Conn
-func (conn *Conn) init(config *Config, urlMaker *urlMaker) error {
-	// New transport
-	transport := newTransport(conn, config)
-
-	// Proxy
-	if conn.config.IsUseProxy {
-		proxyURL, err := url.Parse(config.ProxyHost)
-		if err != nil {
-			return err
+func (conn *Conn) init(config *Config, urlMaker *urlMaker, client *http.Client) error {
+	if client == nil {
+		// New transport
+		transport := newTransport(conn, config)
+
+		// Proxy
+		if conn.config.IsUseProxy {
+			proxyURL, err := url.Parse(config.ProxyHost)
+			if err != nil {
+				return err
+			}
+			transport.Proxy = http.ProxyURL(proxyURL)
 		}
-		transport.Proxy = http.ProxyURL(proxyURL)
+		client = &http.Client{Transport: transport}
 	}
 
 	conn.config = config
 	conn.url = urlMaker
-	conn.client = &http.Client{Transport: transport}
+	conn.client = client
 
 	return nil
 }

oss/transport_1_6.go

@@ -9,6 +9,7 @@ import (
 
 func newTransport(conn *Conn, config *Config) *http.Transport {
 	httpTimeOut := conn.config.HTTPTimeout
+	httpMaxConns := conn.config.HTTPMaxConns
 	// New Transport
 	transport := &http.Transport{
 		Dial: func(netw, addr string) (net.Conn, error) {
@@ -18,6 +19,8 @@ func newTransport(conn *Conn, config *Config) *http.Transport {
 			}
 			return newTimeoutConn(conn, httpTimeOut.ReadWriteTimeout, httpTimeOut.LongTimeout), nil
 		},
+		MaxIdleConns:          httpMaxConns.MaxIdleConns,
+		MaxIdleConnsPerHost:   httpMaxConns.MaxIdleConnsPerHost,
 		ResponseHeaderTimeout: httpTimeOut.HeaderTimeout,
 	}
 	return transport

oss/transport_1_7.go

@@ -9,6 +9,7 @@ import (
 
 func newTransport(conn *Conn, config *Config) *http.Transport {
 	httpTimeOut := conn.config.HTTPTimeout
+	httpMaxConns := conn.config.HTTPMaxConns
 	// New Transport
 	transport := &http.Transport{
 		Dial: func(netw, addr string) (net.Conn, error) {
@@ -18,6 +19,8 @@ func newTransport(conn *Conn, config *Config) *http.Transport {
 			}
 			return newTimeoutConn(conn, httpTimeOut.ReadWriteTimeout, httpTimeOut.LongTimeout), nil
 		},
+		MaxIdleConns:          httpMaxConns.MaxIdleConns,
+		MaxIdleConnsPerHost:   httpMaxConns.MaxIdleConnsPerHost,
 		IdleConnTimeout:       httpTimeOut.IdleConnTimeout,
 		ResponseHeaderTimeout: httpTimeOut.HeaderTimeout,
 	}