conn.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798
  1. package oss
  2. import (
  3. "bytes"
  4. "crypto/md5"
  5. "encoding/base64"
  6. "encoding/json"
  7. "encoding/xml"
  8. "fmt"
  9. "hash"
  10. "io"
  11. "io/ioutil"
  12. "net"
  13. "net/http"
  14. "net/url"
  15. "os"
  16. "sort"
  17. "strconv"
  18. "strings"
  19. "time"
  20. )
  21. // Conn defines OSS Conn
  22. type Conn struct {
  23. config *Config
  24. url *urlMaker
  25. client *http.Client
  26. }
  27. var signKeyList = []string{"acl", "uploads", "location", "cors",
  28. "logging", "website", "referer", "lifecycle",
  29. "delete", "append", "tagging", "objectMeta",
  30. "uploadId", "partNumber", "security-token",
  31. "position", "img", "style", "styleName",
  32. "replication", "replicationProgress",
  33. "replicationLocation", "cname", "bucketInfo",
  34. "comp", "qos", "live", "status", "vod",
  35. "startTime", "endTime", "symlink",
  36. "x-oss-process", "response-content-type", "x-oss-traffic-limit",
  37. "response-content-language", "response-expires",
  38. "response-cache-control", "response-content-disposition",
  39. "response-content-encoding", "udf", "udfName", "udfImage",
  40. "udfId", "udfImageDesc", "udfApplication", "comp",
  41. "udfApplicationLog", "restore", "callback", "callback-var", "qosInfo",
  42. "policy", "stat", "encryption", "versions", "versioning", "versionId", "requestPayment",
  43. "x-oss-request-payer", "sequential"}
  44. // init initializes Conn
  45. func (conn *Conn) init(config *Config, urlMaker *urlMaker, client *http.Client) error {
  46. if client == nil {
  47. // New transport
  48. transport := newTransport(conn, config)
  49. // Proxy
  50. if conn.config.IsUseProxy {
  51. proxyURL, err := url.Parse(config.ProxyHost)
  52. if err != nil {
  53. return err
  54. }
  55. if config.IsAuthProxy {
  56. if config.ProxyPassword != "" {
  57. proxyURL.User = url.UserPassword(config.ProxyUser, config.ProxyPassword)
  58. } else {
  59. proxyURL.User = url.User(config.ProxyUser)
  60. }
  61. }
  62. transport.Proxy = http.ProxyURL(proxyURL)
  63. }
  64. client = &http.Client{Transport: transport}
  65. }
  66. conn.config = config
  67. conn.url = urlMaker
  68. conn.client = client
  69. return nil
  70. }
  71. // Do sends request and returns the response
  72. func (conn Conn) Do(method, bucketName, objectName string, params map[string]interface{}, headers map[string]string,
  73. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  74. urlParams := conn.getURLParams(params)
  75. subResource := conn.getSubResource(params)
  76. uri := conn.url.getURL(bucketName, objectName, urlParams)
  77. resource := conn.getResource(bucketName, objectName, subResource)
  78. return conn.doRequest(method, uri, resource, headers, data, initCRC, listener)
  79. }
  80. // DoURL sends the request with signed URL and returns the response result.
  81. func (conn Conn) DoURL(method HTTPMethod, signedURL string, headers map[string]string,
  82. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  83. // Get URI from signedURL
  84. uri, err := url.ParseRequestURI(signedURL)
  85. if err != nil {
  86. return nil, err
  87. }
  88. m := strings.ToUpper(string(method))
  89. req := &http.Request{
  90. Method: m,
  91. URL: uri,
  92. Proto: "HTTP/1.1",
  93. ProtoMajor: 1,
  94. ProtoMinor: 1,
  95. Header: make(http.Header),
  96. Host: uri.Host,
  97. }
  98. tracker := &readerTracker{completedBytes: 0}
  99. fd, crc := conn.handleBody(req, data, initCRC, listener, tracker)
  100. if fd != nil {
  101. defer func() {
  102. fd.Close()
  103. os.Remove(fd.Name())
  104. }()
  105. }
  106. if conn.config.IsAuthProxy {
  107. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  108. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  109. req.Header.Set("Proxy-Authorization", basic)
  110. }
  111. req.Header.Set(HTTPHeaderHost, req.Host)
  112. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  113. if headers != nil {
  114. for k, v := range headers {
  115. req.Header.Set(k, v)
  116. }
  117. }
  118. // Transfer started
  119. event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength, 0)
  120. publishProgress(listener, event)
  121. if conn.config.LogLevel >= Debug {
  122. conn.LoggerHTTPReq(req)
  123. }
  124. resp, err := conn.client.Do(req)
  125. if err != nil {
  126. // Transfer failed
  127. event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0)
  128. publishProgress(listener, event)
  129. conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error())
  130. return nil, err
  131. }
  132. if conn.config.LogLevel >= Debug {
  133. //print out http resp
  134. conn.LoggerHTTPResp(req, resp)
  135. }
  136. // Transfer completed
  137. event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength, 0)
  138. publishProgress(listener, event)
  139. return conn.handleResponse(resp, crc)
  140. }
  141. func (conn Conn) getURLParams(params map[string]interface{}) string {
  142. // Sort
  143. keys := make([]string, 0, len(params))
  144. for k := range params {
  145. keys = append(keys, k)
  146. }
  147. sort.Strings(keys)
  148. // Serialize
  149. var buf bytes.Buffer
  150. for _, k := range keys {
  151. if buf.Len() > 0 {
  152. buf.WriteByte('&')
  153. }
  154. buf.WriteString(url.QueryEscape(k))
  155. if params[k] != nil {
  156. buf.WriteString("=" + strings.Replace(url.QueryEscape(params[k].(string)), "+", "%20", -1))
  157. }
  158. }
  159. return buf.String()
  160. }
  161. func (conn Conn) getSubResource(params map[string]interface{}) string {
  162. // Sort
  163. keys := make([]string, 0, len(params))
  164. signParams := make(map[string]string)
  165. for k := range params {
  166. if conn.config.AuthVersion == AuthV2 {
  167. encodedKey := url.QueryEscape(k)
  168. keys = append(keys, encodedKey)
  169. if params[k] != nil && params[k] != "" {
  170. signParams[encodedKey] = strings.Replace(url.QueryEscape(params[k].(string)), "+", "%20", -1)
  171. }
  172. } else if conn.isParamSign(k) {
  173. keys = append(keys, k)
  174. if params[k] != nil {
  175. signParams[k] = params[k].(string)
  176. }
  177. }
  178. }
  179. sort.Strings(keys)
  180. // Serialize
  181. var buf bytes.Buffer
  182. for _, k := range keys {
  183. if buf.Len() > 0 {
  184. buf.WriteByte('&')
  185. }
  186. buf.WriteString(k)
  187. if _, ok := signParams[k]; ok {
  188. buf.WriteString("=" + signParams[k])
  189. }
  190. }
  191. return buf.String()
  192. }
  193. func (conn Conn) isParamSign(paramKey string) bool {
  194. for _, k := range signKeyList {
  195. if paramKey == k {
  196. return true
  197. }
  198. }
  199. return false
  200. }
  201. // getResource gets canonicalized resource
  202. func (conn Conn) getResource(bucketName, objectName, subResource string) string {
  203. if subResource != "" {
  204. subResource = "?" + subResource
  205. }
  206. if bucketName == "" {
  207. if conn.config.AuthVersion == AuthV2 {
  208. return url.QueryEscape("/") + subResource
  209. }
  210. return fmt.Sprintf("/%s%s", bucketName, subResource)
  211. }
  212. if conn.config.AuthVersion == AuthV2 {
  213. return url.QueryEscape("/"+bucketName+"/") + strings.Replace(url.QueryEscape(objectName), "+", "%20", -1) + subResource
  214. }
  215. return fmt.Sprintf("/%s/%s%s", bucketName, objectName, subResource)
  216. }
  217. func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource string, headers map[string]string,
  218. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  219. method = strings.ToUpper(method)
  220. req := &http.Request{
  221. Method: method,
  222. URL: uri,
  223. Proto: "HTTP/1.1",
  224. ProtoMajor: 1,
  225. ProtoMinor: 1,
  226. Header: make(http.Header),
  227. Host: uri.Host,
  228. }
  229. tracker := &readerTracker{completedBytes: 0}
  230. fd, crc := conn.handleBody(req, data, initCRC, listener, tracker)
  231. if fd != nil {
  232. defer func() {
  233. fd.Close()
  234. os.Remove(fd.Name())
  235. }()
  236. }
  237. if conn.config.IsAuthProxy {
  238. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  239. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  240. req.Header.Set("Proxy-Authorization", basic)
  241. }
  242. date := time.Now().UTC().Format(http.TimeFormat)
  243. req.Header.Set(HTTPHeaderDate, date)
  244. req.Header.Set(HTTPHeaderHost, req.Host)
  245. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  246. akIf := conn.config.GetCredentials()
  247. if akIf.GetSecurityToken() != "" {
  248. req.Header.Set(HTTPHeaderOssSecurityToken, akIf.GetSecurityToken())
  249. }
  250. if headers != nil {
  251. for k, v := range headers {
  252. req.Header.Set(k, v)
  253. }
  254. }
  255. conn.signHeader(req, canonicalizedResource)
  256. // Transfer started
  257. event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength, 0)
  258. publishProgress(listener, event)
  259. if conn.config.LogLevel >= Debug {
  260. conn.LoggerHTTPReq(req)
  261. }
  262. resp, err := conn.client.Do(req)
  263. if err != nil {
  264. // Transfer failed
  265. event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0)
  266. publishProgress(listener, event)
  267. conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error())
  268. return nil, err
  269. }
  270. if conn.config.LogLevel >= Debug {
  271. //print out http resp
  272. conn.LoggerHTTPResp(req, resp)
  273. }
  274. // Transfer completed
  275. event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength, 0)
  276. publishProgress(listener, event)
  277. return conn.handleResponse(resp, crc)
  278. }
  279. func (conn Conn) signURL(method HTTPMethod, bucketName, objectName string, expiration int64, params map[string]interface{}, headers map[string]string) string {
  280. akIf := conn.config.GetCredentials()
  281. if akIf.GetSecurityToken() != "" {
  282. params[HTTPParamSecurityToken] = akIf.GetSecurityToken()
  283. }
  284. m := strings.ToUpper(string(method))
  285. req := &http.Request{
  286. Method: m,
  287. Header: make(http.Header),
  288. }
  289. if conn.config.IsAuthProxy {
  290. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  291. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  292. req.Header.Set("Proxy-Authorization", basic)
  293. }
  294. req.Header.Set(HTTPHeaderDate, strconv.FormatInt(expiration, 10))
  295. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  296. if headers != nil {
  297. for k, v := range headers {
  298. req.Header.Set(k, v)
  299. }
  300. }
  301. if conn.config.AuthVersion == AuthV2 {
  302. params[HTTPParamSignatureVersion] = "OSS2"
  303. params[HTTPParamExpiresV2] = strconv.FormatInt(expiration, 10)
  304. params[HTTPParamAccessKeyIDV2] = conn.config.AccessKeyID
  305. additionalList, _ := conn.getAdditionalHeaderKeys(req)
  306. if len(additionalList) > 0 {
  307. params[HTTPParamAdditionalHeadersV2] = strings.Join(additionalList, ";")
  308. }
  309. }
  310. subResource := conn.getSubResource(params)
  311. canonicalizedResource := conn.getResource(bucketName, objectName, subResource)
  312. signedStr := conn.getSignedStr(req, canonicalizedResource, akIf.GetAccessKeySecret())
  313. if conn.config.AuthVersion == AuthV1 {
  314. params[HTTPParamExpires] = strconv.FormatInt(expiration, 10)
  315. params[HTTPParamAccessKeyID] = akIf.GetAccessKeyID()
  316. params[HTTPParamSignature] = signedStr
  317. } else if conn.config.AuthVersion == AuthV2 {
  318. params[HTTPParamSignatureV2] = signedStr
  319. }
  320. urlParams := conn.getURLParams(params)
  321. return conn.url.getSignURL(bucketName, objectName, urlParams)
  322. }
  323. func (conn Conn) signRtmpURL(bucketName, channelName, playlistName string, expiration int64) string {
  324. params := map[string]interface{}{}
  325. if playlistName != "" {
  326. params[HTTPParamPlaylistName] = playlistName
  327. }
  328. expireStr := strconv.FormatInt(expiration, 10)
  329. params[HTTPParamExpires] = expireStr
  330. akIf := conn.config.GetCredentials()
  331. if akIf.GetAccessKeyID() != "" {
  332. params[HTTPParamAccessKeyID] = akIf.GetAccessKeyID()
  333. if akIf.GetSecurityToken() != "" {
  334. params[HTTPParamSecurityToken] = akIf.GetSecurityToken()
  335. }
  336. signedStr := conn.getRtmpSignedStr(bucketName, channelName, playlistName, expiration, akIf.GetAccessKeySecret(), params)
  337. params[HTTPParamSignature] = signedStr
  338. }
  339. urlParams := conn.getURLParams(params)
  340. return conn.url.getSignRtmpURL(bucketName, channelName, urlParams)
  341. }
  342. // handleBody handles request body
  343. func (conn Conn) handleBody(req *http.Request, body io.Reader, initCRC uint64,
  344. listener ProgressListener, tracker *readerTracker) (*os.File, hash.Hash64) {
  345. var file *os.File
  346. var crc hash.Hash64
  347. reader := body
  348. // Length
  349. switch v := body.(type) {
  350. case *bytes.Buffer:
  351. req.ContentLength = int64(v.Len())
  352. case *bytes.Reader:
  353. req.ContentLength = int64(v.Len())
  354. case *strings.Reader:
  355. req.ContentLength = int64(v.Len())
  356. case *os.File:
  357. req.ContentLength = tryGetFileSize(v)
  358. case *io.LimitedReader:
  359. req.ContentLength = int64(v.N)
  360. }
  361. req.Header.Set(HTTPHeaderContentLength, strconv.FormatInt(req.ContentLength, 10))
  362. // MD5
  363. if body != nil && conn.config.IsEnableMD5 && req.Header.Get(HTTPHeaderContentMD5) == "" {
  364. md5 := ""
  365. reader, md5, file, _ = calcMD5(body, req.ContentLength, conn.config.MD5Threshold)
  366. req.Header.Set(HTTPHeaderContentMD5, md5)
  367. }
  368. // CRC
  369. if reader != nil && conn.config.IsEnableCRC {
  370. crc = NewCRC(crcTable(), initCRC)
  371. reader = TeeReader(reader, crc, req.ContentLength, listener, tracker)
  372. }
  373. // HTTP body
  374. rc, ok := reader.(io.ReadCloser)
  375. if !ok && reader != nil {
  376. rc = ioutil.NopCloser(reader)
  377. }
  378. if conn.isUploadLimitReq(req) {
  379. limitReader := &LimitSpeedReader{
  380. reader: rc,
  381. ossLimiter: conn.config.UploadLimiter,
  382. }
  383. req.Body = limitReader
  384. } else {
  385. req.Body = rc
  386. }
  387. return file, crc
  388. }
  389. // isUploadLimitReq: judge limit upload speed or not
  390. func (conn Conn) isUploadLimitReq(req *http.Request) bool {
  391. if conn.config.UploadLimitSpeed == 0 || conn.config.UploadLimiter == nil {
  392. return false
  393. }
  394. if req.Method != "GET" && req.Method != "DELETE" && req.Method != "HEAD" {
  395. if req.ContentLength > 0 {
  396. return true
  397. }
  398. }
  399. return false
  400. }
  401. func tryGetFileSize(f *os.File) int64 {
  402. fInfo, _ := f.Stat()
  403. return fInfo.Size()
  404. }
  405. // handleResponse handles response
  406. func (conn Conn) handleResponse(resp *http.Response, crc hash.Hash64) (*Response, error) {
  407. var cliCRC uint64
  408. var srvCRC uint64
  409. statusCode := resp.StatusCode
  410. if statusCode >= 400 && statusCode <= 505 {
  411. // 4xx and 5xx indicate that the operation has error occurred
  412. var respBody []byte
  413. respBody, err := readResponseBody(resp)
  414. if err != nil {
  415. return nil, err
  416. }
  417. if len(respBody) == 0 {
  418. err = ServiceError{
  419. StatusCode: statusCode,
  420. RequestID: resp.Header.Get(HTTPHeaderOssRequestID),
  421. }
  422. } else {
  423. // Response contains storage service error object, unmarshal
  424. srvErr, errIn := serviceErrFromXML(respBody, resp.StatusCode,
  425. resp.Header.Get(HTTPHeaderOssRequestID))
  426. if errIn != nil { // error unmarshaling the error response
  427. err = fmt.Errorf("oss: service returned invalid response body, status = %s, RequestId = %s", resp.Status, resp.Header.Get(HTTPHeaderOssRequestID))
  428. } else {
  429. err = srvErr
  430. }
  431. }
  432. return &Response{
  433. StatusCode: resp.StatusCode,
  434. Headers: resp.Header,
  435. Body: ioutil.NopCloser(bytes.NewReader(respBody)), // restore the body
  436. }, err
  437. } else if statusCode >= 300 && statusCode <= 307 {
  438. // OSS use 3xx, but response has no body
  439. err := fmt.Errorf("oss: service returned %d,%s", resp.StatusCode, resp.Status)
  440. return &Response{
  441. StatusCode: resp.StatusCode,
  442. Headers: resp.Header,
  443. Body: resp.Body,
  444. }, err
  445. }
  446. if conn.config.IsEnableCRC && crc != nil {
  447. cliCRC = crc.Sum64()
  448. }
  449. srvCRC, _ = strconv.ParseUint(resp.Header.Get(HTTPHeaderOssCRC64), 10, 64)
  450. // 2xx, successful
  451. return &Response{
  452. StatusCode: resp.StatusCode,
  453. Headers: resp.Header,
  454. Body: resp.Body,
  455. ClientCRC: cliCRC,
  456. ServerCRC: srvCRC,
  457. }, nil
  458. }
  459. // LoggerHTTPReq Print the header information of the http request
  460. func (conn Conn) LoggerHTTPReq(req *http.Request) {
  461. var logBuffer bytes.Buffer
  462. logBuffer.WriteString(fmt.Sprintf("[Req:%p]Method:%s\t", req, req.Method))
  463. logBuffer.WriteString(fmt.Sprintf("Host:%s\t", req.URL.Host))
  464. logBuffer.WriteString(fmt.Sprintf("Path:%s\t", req.URL.Path))
  465. logBuffer.WriteString(fmt.Sprintf("Query:%s\t", req.URL.RawQuery))
  466. logBuffer.WriteString(fmt.Sprintf("Header info:"))
  467. for k, v := range req.Header {
  468. var valueBuffer bytes.Buffer
  469. for j := 0; j < len(v); j++ {
  470. if j > 0 {
  471. valueBuffer.WriteString(" ")
  472. }
  473. valueBuffer.WriteString(v[j])
  474. }
  475. logBuffer.WriteString(fmt.Sprintf("\t%s:%s", k, valueBuffer.String()))
  476. }
  477. conn.config.WriteLog(Debug, "%s\n", logBuffer.String())
  478. }
  479. // LoggerHTTPResp Print Response to http request
  480. func (conn Conn) LoggerHTTPResp(req *http.Request, resp *http.Response) {
  481. var logBuffer bytes.Buffer
  482. logBuffer.WriteString(fmt.Sprintf("[Resp:%p]StatusCode:%d\t", req, resp.StatusCode))
  483. logBuffer.WriteString(fmt.Sprintf("Header info:"))
  484. for k, v := range resp.Header {
  485. var valueBuffer bytes.Buffer
  486. for j := 0; j < len(v); j++ {
  487. if j > 0 {
  488. valueBuffer.WriteString(" ")
  489. }
  490. valueBuffer.WriteString(v[j])
  491. }
  492. logBuffer.WriteString(fmt.Sprintf("\t%s:%s", k, valueBuffer.String()))
  493. }
  494. conn.config.WriteLog(Debug, "%s\n", logBuffer.String())
  495. }
  496. func calcMD5(body io.Reader, contentLen, md5Threshold int64) (reader io.Reader, b64 string, tempFile *os.File, err error) {
  497. if contentLen == 0 || contentLen > md5Threshold {
  498. // Huge body, use temporary file
  499. tempFile, err = ioutil.TempFile(os.TempDir(), TempFilePrefix)
  500. if tempFile != nil {
  501. io.Copy(tempFile, body)
  502. tempFile.Seek(0, os.SEEK_SET)
  503. md5 := md5.New()
  504. io.Copy(md5, tempFile)
  505. sum := md5.Sum(nil)
  506. b64 = base64.StdEncoding.EncodeToString(sum[:])
  507. tempFile.Seek(0, os.SEEK_SET)
  508. reader = tempFile
  509. }
  510. } else {
  511. // Small body, use memory
  512. buf, _ := ioutil.ReadAll(body)
  513. sum := md5.Sum(buf)
  514. b64 = base64.StdEncoding.EncodeToString(sum[:])
  515. reader = bytes.NewReader(buf)
  516. }
  517. return
  518. }
  519. func readResponseBody(resp *http.Response) ([]byte, error) {
  520. defer resp.Body.Close()
  521. out, err := ioutil.ReadAll(resp.Body)
  522. if err == io.EOF {
  523. err = nil
  524. }
  525. return out, err
  526. }
  527. func serviceErrFromXML(body []byte, statusCode int, requestID string) (ServiceError, error) {
  528. var storageErr ServiceError
  529. if err := xml.Unmarshal(body, &storageErr); err != nil {
  530. return storageErr, err
  531. }
  532. storageErr.StatusCode = statusCode
  533. storageErr.RequestID = requestID
  534. storageErr.RawMessage = string(body)
  535. return storageErr, nil
  536. }
  537. func xmlUnmarshal(body io.Reader, v interface{}) error {
  538. data, err := ioutil.ReadAll(body)
  539. if err != nil {
  540. return err
  541. }
  542. return xml.Unmarshal(data, v)
  543. }
  544. func jsonUnmarshal(body io.Reader, v interface{}) error {
  545. data, err := ioutil.ReadAll(body)
  546. if err != nil {
  547. return err
  548. }
  549. return json.Unmarshal(data, v)
  550. }
  551. // timeoutConn handles HTTP timeout
  552. type timeoutConn struct {
  553. conn net.Conn
  554. timeout time.Duration
  555. longTimeout time.Duration
  556. }
  557. func newTimeoutConn(conn net.Conn, timeout time.Duration, longTimeout time.Duration) *timeoutConn {
  558. conn.SetReadDeadline(time.Now().Add(longTimeout))
  559. return &timeoutConn{
  560. conn: conn,
  561. timeout: timeout,
  562. longTimeout: longTimeout,
  563. }
  564. }
  565. func (c *timeoutConn) Read(b []byte) (n int, err error) {
  566. c.SetReadDeadline(time.Now().Add(c.timeout))
  567. n, err = c.conn.Read(b)
  568. c.SetReadDeadline(time.Now().Add(c.longTimeout))
  569. return n, err
  570. }
  571. func (c *timeoutConn) Write(b []byte) (n int, err error) {
  572. c.SetWriteDeadline(time.Now().Add(c.timeout))
  573. n, err = c.conn.Write(b)
  574. c.SetReadDeadline(time.Now().Add(c.longTimeout))
  575. return n, err
  576. }
  577. func (c *timeoutConn) Close() error {
  578. return c.conn.Close()
  579. }
  580. func (c *timeoutConn) LocalAddr() net.Addr {
  581. return c.conn.LocalAddr()
  582. }
  583. func (c *timeoutConn) RemoteAddr() net.Addr {
  584. return c.conn.RemoteAddr()
  585. }
  586. func (c *timeoutConn) SetDeadline(t time.Time) error {
  587. return c.conn.SetDeadline(t)
  588. }
  589. func (c *timeoutConn) SetReadDeadline(t time.Time) error {
  590. return c.conn.SetReadDeadline(t)
  591. }
  592. func (c *timeoutConn) SetWriteDeadline(t time.Time) error {
  593. return c.conn.SetWriteDeadline(t)
  594. }
  595. // UrlMaker builds URL and resource
  596. const (
  597. urlTypeCname = 1
  598. urlTypeIP = 2
  599. urlTypeAliyun = 3
  600. )
  601. type urlMaker struct {
  602. Scheme string // HTTP or HTTPS
  603. NetLoc string // Host or IP
  604. Type int // 1 CNAME, 2 IP, 3 ALIYUN
  605. IsProxy bool // Proxy
  606. }
  607. // Init parses endpoint
  608. func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) error {
  609. if strings.HasPrefix(endpoint, "http://") {
  610. um.Scheme = "http"
  611. um.NetLoc = endpoint[len("http://"):]
  612. } else if strings.HasPrefix(endpoint, "https://") {
  613. um.Scheme = "https"
  614. um.NetLoc = endpoint[len("https://"):]
  615. } else {
  616. um.Scheme = "http"
  617. um.NetLoc = endpoint
  618. }
  619. //use url.Parse() to get real host
  620. strUrl := um.Scheme + "://" + um.NetLoc
  621. url, err := url.Parse(strUrl)
  622. if err != nil {
  623. return err
  624. }
  625. um.NetLoc = url.Host
  626. host, _, err := net.SplitHostPort(um.NetLoc)
  627. if err != nil {
  628. host = um.NetLoc
  629. if host[0] == '[' && host[len(host)-1] == ']' {
  630. host = host[1 : len(host)-1]
  631. }
  632. }
  633. ip := net.ParseIP(host)
  634. if ip != nil {
  635. um.Type = urlTypeIP
  636. } else if isCname {
  637. um.Type = urlTypeCname
  638. } else {
  639. um.Type = urlTypeAliyun
  640. }
  641. um.IsProxy = isProxy
  642. return nil
  643. }
  644. // getURL gets URL
  645. func (um urlMaker) getURL(bucket, object, params string) *url.URL {
  646. host, path := um.buildURL(bucket, object)
  647. addr := ""
  648. if params == "" {
  649. addr = fmt.Sprintf("%s://%s%s", um.Scheme, host, path)
  650. } else {
  651. addr = fmt.Sprintf("%s://%s%s?%s", um.Scheme, host, path, params)
  652. }
  653. uri, _ := url.ParseRequestURI(addr)
  654. return uri
  655. }
  656. // getSignURL gets sign URL
  657. func (um urlMaker) getSignURL(bucket, object, params string) string {
  658. host, path := um.buildURL(bucket, object)
  659. return fmt.Sprintf("%s://%s%s?%s", um.Scheme, host, path, params)
  660. }
  661. // getSignRtmpURL Build Sign Rtmp URL
  662. func (um urlMaker) getSignRtmpURL(bucket, channelName, params string) string {
  663. host, path := um.buildURL(bucket, "live")
  664. channelName = url.QueryEscape(channelName)
  665. channelName = strings.Replace(channelName, "+", "%20", -1)
  666. return fmt.Sprintf("rtmp://%s%s/%s?%s", host, path, channelName, params)
  667. }
  668. // buildURL builds URL
  669. func (um urlMaker) buildURL(bucket, object string) (string, string) {
  670. var host = ""
  671. var path = ""
  672. object = url.QueryEscape(object)
  673. object = strings.Replace(object, "+", "%20", -1)
  674. if um.Type == urlTypeCname {
  675. host = um.NetLoc
  676. path = "/" + object
  677. } else if um.Type == urlTypeIP {
  678. if bucket == "" {
  679. host = um.NetLoc
  680. path = "/"
  681. } else {
  682. host = um.NetLoc
  683. path = fmt.Sprintf("/%s/%s", bucket, object)
  684. }
  685. } else {
  686. if bucket == "" {
  687. host = um.NetLoc
  688. path = "/"
  689. } else {
  690. host = bucket + "." + um.NetLoc
  691. path = "/" + object
  692. }
  693. }
  694. return host, path
  695. }