conn.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792
  1. package oss
  2. import (
  3. "bytes"
  4. "crypto/md5"
  5. "encoding/base64"
  6. "encoding/json"
  7. "encoding/xml"
  8. "fmt"
  9. "hash"
  10. "io"
  11. "io/ioutil"
  12. "net"
  13. "net/http"
  14. "net/url"
  15. "os"
  16. "sort"
  17. "strconv"
  18. "strings"
  19. "time"
  20. )
  21. // Conn defines OSS Conn
  22. type Conn struct {
  23. config *Config
  24. url *urlMaker
  25. client *http.Client
  26. }
  27. var signKeyList = []string{"acl", "uploads", "location", "cors",
  28. "logging", "website", "referer", "lifecycle",
  29. "delete", "append", "tagging", "objectMeta",
  30. "uploadId", "partNumber", "security-token",
  31. "position", "img", "style", "styleName",
  32. "replication", "replicationProgress",
  33. "replicationLocation", "cname", "bucketInfo",
  34. "comp", "qos", "live", "status", "vod",
  35. "startTime", "endTime", "symlink",
  36. "x-oss-process", "response-content-type", "x-oss-traffic-limit",
  37. "response-content-language", "response-expires",
  38. "response-cache-control", "response-content-disposition",
  39. "response-content-encoding", "udf", "udfName", "udfImage",
  40. "udfId", "udfImageDesc", "udfApplication", "comp",
  41. "udfApplicationLog", "restore", "callback", "callback-var", "qosInfo",
  42. "policy", "stat", "encryption", "versions", "versioning", "versionId", "requestPayment",
  43. "x-oss-request-payer", "sequential",
  44. "inventory", "inventoryId", "continuation-token", "asyncFetch"}
  45. // init initializes Conn
  46. func (conn *Conn) init(config *Config, urlMaker *urlMaker, client *http.Client) error {
  47. if client == nil {
  48. // New transport
  49. transport := newTransport(conn, config)
  50. // Proxy
  51. if conn.config.IsUseProxy {
  52. proxyURL, err := url.Parse(config.ProxyHost)
  53. if err != nil {
  54. return err
  55. }
  56. if config.IsAuthProxy {
  57. if config.ProxyPassword != "" {
  58. proxyURL.User = url.UserPassword(config.ProxyUser, config.ProxyPassword)
  59. } else {
  60. proxyURL.User = url.User(config.ProxyUser)
  61. }
  62. }
  63. transport.Proxy = http.ProxyURL(proxyURL)
  64. }
  65. client = &http.Client{Transport: transport}
  66. if !config.RedirectEnabled {
  67. disableHTTPRedirect(client)
  68. }
  69. }
  70. conn.config = config
  71. conn.url = urlMaker
  72. conn.client = client
  73. return nil
  74. }
  75. // Do sends request and returns the response
  76. func (conn Conn) Do(method, bucketName, objectName string, params map[string]interface{}, headers map[string]string,
  77. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  78. urlParams := conn.getURLParams(params)
  79. subResource := conn.getSubResource(params)
  80. uri := conn.url.getURL(bucketName, objectName, urlParams)
  81. resource := conn.getResource(bucketName, objectName, subResource)
  82. return conn.doRequest(method, uri, resource, headers, data, initCRC, listener)
  83. }
  84. // DoURL sends the request with signed URL and returns the response result.
  85. func (conn Conn) DoURL(method HTTPMethod, signedURL string, headers map[string]string,
  86. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  87. // Get URI from signedURL
  88. uri, err := url.ParseRequestURI(signedURL)
  89. if err != nil {
  90. return nil, err
  91. }
  92. m := strings.ToUpper(string(method))
  93. req := &http.Request{
  94. Method: m,
  95. URL: uri,
  96. Proto: "HTTP/1.1",
  97. ProtoMajor: 1,
  98. ProtoMinor: 1,
  99. Header: make(http.Header),
  100. Host: uri.Host,
  101. }
  102. tracker := &readerTracker{completedBytes: 0}
  103. fd, crc := conn.handleBody(req, data, initCRC, listener, tracker)
  104. if fd != nil {
  105. defer func() {
  106. fd.Close()
  107. os.Remove(fd.Name())
  108. }()
  109. }
  110. if conn.config.IsAuthProxy {
  111. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  112. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  113. req.Header.Set("Proxy-Authorization", basic)
  114. }
  115. req.Header.Set(HTTPHeaderHost, req.Host)
  116. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  117. if headers != nil {
  118. for k, v := range headers {
  119. req.Header.Set(k, v)
  120. }
  121. }
  122. // Transfer started
  123. event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength, 0)
  124. publishProgress(listener, event)
  125. if conn.config.LogLevel >= Debug {
  126. conn.LoggerHTTPReq(req)
  127. }
  128. resp, err := conn.client.Do(req)
  129. if err != nil {
  130. // Transfer failed
  131. event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0)
  132. publishProgress(listener, event)
  133. conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error())
  134. return nil, err
  135. }
  136. if conn.config.LogLevel >= Debug {
  137. //print out http resp
  138. conn.LoggerHTTPResp(req, resp)
  139. }
  140. // Transfer completed
  141. event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength, 0)
  142. publishProgress(listener, event)
  143. return conn.handleResponse(resp, crc)
  144. }
  145. func (conn Conn) getURLParams(params map[string]interface{}) string {
  146. // Sort
  147. keys := make([]string, 0, len(params))
  148. for k := range params {
  149. keys = append(keys, k)
  150. }
  151. sort.Strings(keys)
  152. // Serialize
  153. var buf bytes.Buffer
  154. for _, k := range keys {
  155. if buf.Len() > 0 {
  156. buf.WriteByte('&')
  157. }
  158. buf.WriteString(url.QueryEscape(k))
  159. if params[k] != nil {
  160. buf.WriteString("=" + strings.Replace(url.QueryEscape(params[k].(string)), "+", "%20", -1))
  161. }
  162. }
  163. return buf.String()
  164. }
  165. func (conn Conn) getSubResource(params map[string]interface{}) string {
  166. // Sort
  167. keys := make([]string, 0, len(params))
  168. signParams := make(map[string]string)
  169. for k := range params {
  170. if conn.config.AuthVersion == AuthV2 {
  171. encodedKey := url.QueryEscape(k)
  172. keys = append(keys, encodedKey)
  173. if params[k] != nil && params[k] != "" {
  174. signParams[encodedKey] = strings.Replace(url.QueryEscape(params[k].(string)), "+", "%20", -1)
  175. }
  176. } else if conn.isParamSign(k) {
  177. keys = append(keys, k)
  178. if params[k] != nil {
  179. signParams[k] = params[k].(string)
  180. }
  181. }
  182. }
  183. sort.Strings(keys)
  184. // Serialize
  185. var buf bytes.Buffer
  186. for _, k := range keys {
  187. if buf.Len() > 0 {
  188. buf.WriteByte('&')
  189. }
  190. buf.WriteString(k)
  191. if _, ok := signParams[k]; ok {
  192. buf.WriteString("=" + signParams[k])
  193. }
  194. }
  195. return buf.String()
  196. }
  197. func (conn Conn) isParamSign(paramKey string) bool {
  198. for _, k := range signKeyList {
  199. if paramKey == k {
  200. return true
  201. }
  202. }
  203. return false
  204. }
  205. // getResource gets canonicalized resource
  206. func (conn Conn) getResource(bucketName, objectName, subResource string) string {
  207. if subResource != "" {
  208. subResource = "?" + subResource
  209. }
  210. if bucketName == "" {
  211. if conn.config.AuthVersion == AuthV2 {
  212. return url.QueryEscape("/") + subResource
  213. }
  214. return fmt.Sprintf("/%s%s", bucketName, subResource)
  215. }
  216. if conn.config.AuthVersion == AuthV2 {
  217. return url.QueryEscape("/"+bucketName+"/") + strings.Replace(url.QueryEscape(objectName), "+", "%20", -1) + subResource
  218. }
  219. return fmt.Sprintf("/%s/%s%s", bucketName, objectName, subResource)
  220. }
  221. func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource string, headers map[string]string,
  222. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  223. method = strings.ToUpper(method)
  224. req := &http.Request{
  225. Method: method,
  226. URL: uri,
  227. Proto: "HTTP/1.1",
  228. ProtoMajor: 1,
  229. ProtoMinor: 1,
  230. Header: make(http.Header),
  231. Host: uri.Host,
  232. }
  233. tracker := &readerTracker{completedBytes: 0}
  234. fd, crc := conn.handleBody(req, data, initCRC, listener, tracker)
  235. if fd != nil {
  236. defer func() {
  237. fd.Close()
  238. os.Remove(fd.Name())
  239. }()
  240. }
  241. if conn.config.IsAuthProxy {
  242. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  243. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  244. req.Header.Set("Proxy-Authorization", basic)
  245. }
  246. date := time.Now().UTC().Format(http.TimeFormat)
  247. req.Header.Set(HTTPHeaderDate, date)
  248. req.Header.Set(HTTPHeaderHost, req.Host)
  249. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  250. akIf := conn.config.GetCredentials()
  251. if akIf.GetSecurityToken() != "" {
  252. req.Header.Set(HTTPHeaderOssSecurityToken, akIf.GetSecurityToken())
  253. }
  254. if headers != nil {
  255. for k, v := range headers {
  256. req.Header.Set(k, v)
  257. }
  258. }
  259. conn.signHeader(req, canonicalizedResource)
  260. // Transfer started
  261. event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength, 0)
  262. publishProgress(listener, event)
  263. if conn.config.LogLevel >= Debug {
  264. conn.LoggerHTTPReq(req)
  265. }
  266. resp, err := conn.client.Do(req)
  267. if err != nil {
  268. // Transfer failed
  269. event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0)
  270. publishProgress(listener, event)
  271. conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error())
  272. return nil, err
  273. }
  274. if conn.config.LogLevel >= Debug {
  275. //print out http resp
  276. conn.LoggerHTTPResp(req, resp)
  277. }
  278. // Transfer completed
  279. event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength, 0)
  280. publishProgress(listener, event)
  281. return conn.handleResponse(resp, crc)
  282. }
  283. func (conn Conn) signURL(method HTTPMethod, bucketName, objectName string, expiration int64, params map[string]interface{}, headers map[string]string) string {
  284. akIf := conn.config.GetCredentials()
  285. if akIf.GetSecurityToken() != "" {
  286. params[HTTPParamSecurityToken] = akIf.GetSecurityToken()
  287. }
  288. m := strings.ToUpper(string(method))
  289. req := &http.Request{
  290. Method: m,
  291. Header: make(http.Header),
  292. }
  293. if conn.config.IsAuthProxy {
  294. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  295. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  296. req.Header.Set("Proxy-Authorization", basic)
  297. }
  298. req.Header.Set(HTTPHeaderDate, strconv.FormatInt(expiration, 10))
  299. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  300. if headers != nil {
  301. for k, v := range headers {
  302. req.Header.Set(k, v)
  303. }
  304. }
  305. if conn.config.AuthVersion == AuthV2 {
  306. params[HTTPParamSignatureVersion] = "OSS2"
  307. params[HTTPParamExpiresV2] = strconv.FormatInt(expiration, 10)
  308. params[HTTPParamAccessKeyIDV2] = conn.config.AccessKeyID
  309. additionalList, _ := conn.getAdditionalHeaderKeys(req)
  310. if len(additionalList) > 0 {
  311. params[HTTPParamAdditionalHeadersV2] = strings.Join(additionalList, ";")
  312. }
  313. }
  314. subResource := conn.getSubResource(params)
  315. canonicalizedResource := conn.getResource(bucketName, objectName, subResource)
  316. signedStr := conn.getSignedStr(req, canonicalizedResource, akIf.GetAccessKeySecret())
  317. if conn.config.AuthVersion == AuthV1 {
  318. params[HTTPParamExpires] = strconv.FormatInt(expiration, 10)
  319. params[HTTPParamAccessKeyID] = akIf.GetAccessKeyID()
  320. params[HTTPParamSignature] = signedStr
  321. } else if conn.config.AuthVersion == AuthV2 {
  322. params[HTTPParamSignatureV2] = signedStr
  323. }
  324. urlParams := conn.getURLParams(params)
  325. return conn.url.getSignURL(bucketName, objectName, urlParams)
  326. }
  327. func (conn Conn) signRtmpURL(bucketName, channelName, playlistName string, expiration int64) string {
  328. params := map[string]interface{}{}
  329. if playlistName != "" {
  330. params[HTTPParamPlaylistName] = playlistName
  331. }
  332. expireStr := strconv.FormatInt(expiration, 10)
  333. params[HTTPParamExpires] = expireStr
  334. akIf := conn.config.GetCredentials()
  335. if akIf.GetAccessKeyID() != "" {
  336. params[HTTPParamAccessKeyID] = akIf.GetAccessKeyID()
  337. if akIf.GetSecurityToken() != "" {
  338. params[HTTPParamSecurityToken] = akIf.GetSecurityToken()
  339. }
  340. signedStr := conn.getRtmpSignedStr(bucketName, channelName, playlistName, expiration, akIf.GetAccessKeySecret(), params)
  341. params[HTTPParamSignature] = signedStr
  342. }
  343. urlParams := conn.getURLParams(params)
  344. return conn.url.getSignRtmpURL(bucketName, channelName, urlParams)
  345. }
  346. // handleBody handles request body
  347. func (conn Conn) handleBody(req *http.Request, body io.Reader, initCRC uint64,
  348. listener ProgressListener, tracker *readerTracker) (*os.File, hash.Hash64) {
  349. var file *os.File
  350. var crc hash.Hash64
  351. reader := body
  352. readerLen, err := GetReaderLen(reader)
  353. if err == nil {
  354. req.ContentLength = readerLen
  355. }
  356. req.Header.Set(HTTPHeaderContentLength, strconv.FormatInt(req.ContentLength, 10))
  357. // MD5
  358. if body != nil && conn.config.IsEnableMD5 && req.Header.Get(HTTPHeaderContentMD5) == "" {
  359. md5 := ""
  360. reader, md5, file, _ = calcMD5(body, req.ContentLength, conn.config.MD5Threshold)
  361. req.Header.Set(HTTPHeaderContentMD5, md5)
  362. }
  363. // CRC
  364. if reader != nil && conn.config.IsEnableCRC {
  365. crc = NewCRC(CrcTable(), initCRC)
  366. reader = TeeReader(reader, crc, req.ContentLength, listener, tracker)
  367. }
  368. // HTTP body
  369. rc, ok := reader.(io.ReadCloser)
  370. if !ok && reader != nil {
  371. rc = ioutil.NopCloser(reader)
  372. }
  373. if conn.isUploadLimitReq(req) {
  374. limitReader := &LimitSpeedReader{
  375. reader: rc,
  376. ossLimiter: conn.config.UploadLimiter,
  377. }
  378. req.Body = limitReader
  379. } else {
  380. req.Body = rc
  381. }
  382. return file, crc
  383. }
  384. // isUploadLimitReq: judge limit upload speed or not
  385. func (conn Conn) isUploadLimitReq(req *http.Request) bool {
  386. if conn.config.UploadLimitSpeed == 0 || conn.config.UploadLimiter == nil {
  387. return false
  388. }
  389. if req.Method != "GET" && req.Method != "DELETE" && req.Method != "HEAD" {
  390. if req.ContentLength > 0 {
  391. return true
  392. }
  393. }
  394. return false
  395. }
  396. func tryGetFileSize(f *os.File) int64 {
  397. fInfo, _ := f.Stat()
  398. return fInfo.Size()
  399. }
  400. // handleResponse handles response
  401. func (conn Conn) handleResponse(resp *http.Response, crc hash.Hash64) (*Response, error) {
  402. var cliCRC uint64
  403. var srvCRC uint64
  404. statusCode := resp.StatusCode
  405. if statusCode >= 400 && statusCode <= 505 {
  406. // 4xx and 5xx indicate that the operation has error occurred
  407. var respBody []byte
  408. respBody, err := readResponseBody(resp)
  409. if err != nil {
  410. return nil, err
  411. }
  412. if len(respBody) == 0 {
  413. err = ServiceError{
  414. StatusCode: statusCode,
  415. RequestID: resp.Header.Get(HTTPHeaderOssRequestID),
  416. }
  417. } else {
  418. // Response contains storage service error object, unmarshal
  419. srvErr, errIn := serviceErrFromXML(respBody, resp.StatusCode,
  420. resp.Header.Get(HTTPHeaderOssRequestID))
  421. if errIn != nil { // error unmarshaling the error response
  422. err = fmt.Errorf("oss: service returned invalid response body, status = %s, RequestId = %s", resp.Status, resp.Header.Get(HTTPHeaderOssRequestID))
  423. } else {
  424. err = srvErr
  425. }
  426. }
  427. return &Response{
  428. StatusCode: resp.StatusCode,
  429. Headers: resp.Header,
  430. Body: ioutil.NopCloser(bytes.NewReader(respBody)), // restore the body
  431. }, err
  432. } else if statusCode >= 300 && statusCode <= 307 {
  433. // OSS use 3xx, but response has no body
  434. err := fmt.Errorf("oss: service returned %d,%s", resp.StatusCode, resp.Status)
  435. return &Response{
  436. StatusCode: resp.StatusCode,
  437. Headers: resp.Header,
  438. Body: resp.Body,
  439. }, err
  440. }
  441. if conn.config.IsEnableCRC && crc != nil {
  442. cliCRC = crc.Sum64()
  443. }
  444. srvCRC, _ = strconv.ParseUint(resp.Header.Get(HTTPHeaderOssCRC64), 10, 64)
  445. // 2xx, successful
  446. return &Response{
  447. StatusCode: resp.StatusCode,
  448. Headers: resp.Header,
  449. Body: resp.Body,
  450. ClientCRC: cliCRC,
  451. ServerCRC: srvCRC,
  452. }, nil
  453. }
  454. // LoggerHTTPReq Print the header information of the http request
  455. func (conn Conn) LoggerHTTPReq(req *http.Request) {
  456. var logBuffer bytes.Buffer
  457. logBuffer.WriteString(fmt.Sprintf("[Req:%p]Method:%s\t", req, req.Method))
  458. logBuffer.WriteString(fmt.Sprintf("Host:%s\t", req.URL.Host))
  459. logBuffer.WriteString(fmt.Sprintf("Path:%s\t", req.URL.Path))
  460. logBuffer.WriteString(fmt.Sprintf("Query:%s\t", req.URL.RawQuery))
  461. logBuffer.WriteString(fmt.Sprintf("Header info:"))
  462. for k, v := range req.Header {
  463. var valueBuffer bytes.Buffer
  464. for j := 0; j < len(v); j++ {
  465. if j > 0 {
  466. valueBuffer.WriteString(" ")
  467. }
  468. valueBuffer.WriteString(v[j])
  469. }
  470. logBuffer.WriteString(fmt.Sprintf("\t%s:%s", k, valueBuffer.String()))
  471. }
  472. conn.config.WriteLog(Debug, "%s\n", logBuffer.String())
  473. }
  474. // LoggerHTTPResp Print Response to http request
  475. func (conn Conn) LoggerHTTPResp(req *http.Request, resp *http.Response) {
  476. var logBuffer bytes.Buffer
  477. logBuffer.WriteString(fmt.Sprintf("[Resp:%p]StatusCode:%d\t", req, resp.StatusCode))
  478. logBuffer.WriteString(fmt.Sprintf("Header info:"))
  479. for k, v := range resp.Header {
  480. var valueBuffer bytes.Buffer
  481. for j := 0; j < len(v); j++ {
  482. if j > 0 {
  483. valueBuffer.WriteString(" ")
  484. }
  485. valueBuffer.WriteString(v[j])
  486. }
  487. logBuffer.WriteString(fmt.Sprintf("\t%s:%s", k, valueBuffer.String()))
  488. }
  489. conn.config.WriteLog(Debug, "%s\n", logBuffer.String())
  490. }
  491. func calcMD5(body io.Reader, contentLen, md5Threshold int64) (reader io.Reader, b64 string, tempFile *os.File, err error) {
  492. if contentLen == 0 || contentLen > md5Threshold {
  493. // Huge body, use temporary file
  494. tempFile, err = ioutil.TempFile(os.TempDir(), TempFilePrefix)
  495. if tempFile != nil {
  496. io.Copy(tempFile, body)
  497. tempFile.Seek(0, os.SEEK_SET)
  498. md5 := md5.New()
  499. io.Copy(md5, tempFile)
  500. sum := md5.Sum(nil)
  501. b64 = base64.StdEncoding.EncodeToString(sum[:])
  502. tempFile.Seek(0, os.SEEK_SET)
  503. reader = tempFile
  504. }
  505. } else {
  506. // Small body, use memory
  507. buf, _ := ioutil.ReadAll(body)
  508. sum := md5.Sum(buf)
  509. b64 = base64.StdEncoding.EncodeToString(sum[:])
  510. reader = bytes.NewReader(buf)
  511. }
  512. return
  513. }
  514. func readResponseBody(resp *http.Response) ([]byte, error) {
  515. defer resp.Body.Close()
  516. out, err := ioutil.ReadAll(resp.Body)
  517. if err == io.EOF {
  518. err = nil
  519. }
  520. return out, err
  521. }
  522. func serviceErrFromXML(body []byte, statusCode int, requestID string) (ServiceError, error) {
  523. var storageErr ServiceError
  524. if err := xml.Unmarshal(body, &storageErr); err != nil {
  525. return storageErr, err
  526. }
  527. storageErr.StatusCode = statusCode
  528. storageErr.RequestID = requestID
  529. storageErr.RawMessage = string(body)
  530. return storageErr, nil
  531. }
  532. func xmlUnmarshal(body io.Reader, v interface{}) error {
  533. data, err := ioutil.ReadAll(body)
  534. if err != nil {
  535. return err
  536. }
  537. return xml.Unmarshal(data, v)
  538. }
  539. func jsonUnmarshal(body io.Reader, v interface{}) error {
  540. data, err := ioutil.ReadAll(body)
  541. if err != nil {
  542. return err
  543. }
  544. return json.Unmarshal(data, v)
  545. }
  546. // timeoutConn handles HTTP timeout
  547. type timeoutConn struct {
  548. conn net.Conn
  549. timeout time.Duration
  550. longTimeout time.Duration
  551. }
  552. func newTimeoutConn(conn net.Conn, timeout time.Duration, longTimeout time.Duration) *timeoutConn {
  553. conn.SetReadDeadline(time.Now().Add(longTimeout))
  554. return &timeoutConn{
  555. conn: conn,
  556. timeout: timeout,
  557. longTimeout: longTimeout,
  558. }
  559. }
  560. func (c *timeoutConn) Read(b []byte) (n int, err error) {
  561. c.SetReadDeadline(time.Now().Add(c.timeout))
  562. n, err = c.conn.Read(b)
  563. c.SetReadDeadline(time.Now().Add(c.longTimeout))
  564. return n, err
  565. }
  566. func (c *timeoutConn) Write(b []byte) (n int, err error) {
  567. c.SetWriteDeadline(time.Now().Add(c.timeout))
  568. n, err = c.conn.Write(b)
  569. c.SetReadDeadline(time.Now().Add(c.longTimeout))
  570. return n, err
  571. }
  572. func (c *timeoutConn) Close() error {
  573. return c.conn.Close()
  574. }
  575. func (c *timeoutConn) LocalAddr() net.Addr {
  576. return c.conn.LocalAddr()
  577. }
  578. func (c *timeoutConn) RemoteAddr() net.Addr {
  579. return c.conn.RemoteAddr()
  580. }
  581. func (c *timeoutConn) SetDeadline(t time.Time) error {
  582. return c.conn.SetDeadline(t)
  583. }
  584. func (c *timeoutConn) SetReadDeadline(t time.Time) error {
  585. return c.conn.SetReadDeadline(t)
  586. }
  587. func (c *timeoutConn) SetWriteDeadline(t time.Time) error {
  588. return c.conn.SetWriteDeadline(t)
  589. }
  590. // UrlMaker builds URL and resource
  591. const (
  592. urlTypeCname = 1
  593. urlTypeIP = 2
  594. urlTypeAliyun = 3
  595. )
  596. type urlMaker struct {
  597. Scheme string // HTTP or HTTPS
  598. NetLoc string // Host or IP
  599. Type int // 1 CNAME, 2 IP, 3 ALIYUN
  600. IsProxy bool // Proxy
  601. }
  602. // Init parses endpoint
  603. func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) error {
  604. if strings.HasPrefix(endpoint, "http://") {
  605. um.Scheme = "http"
  606. um.NetLoc = endpoint[len("http://"):]
  607. } else if strings.HasPrefix(endpoint, "https://") {
  608. um.Scheme = "https"
  609. um.NetLoc = endpoint[len("https://"):]
  610. } else {
  611. um.Scheme = "http"
  612. um.NetLoc = endpoint
  613. }
  614. //use url.Parse() to get real host
  615. strUrl := um.Scheme + "://" + um.NetLoc
  616. url, err := url.Parse(strUrl)
  617. if err != nil {
  618. return err
  619. }
  620. um.NetLoc = url.Host
  621. host, _, err := net.SplitHostPort(um.NetLoc)
  622. if err != nil {
  623. host = um.NetLoc
  624. if host[0] == '[' && host[len(host)-1] == ']' {
  625. host = host[1 : len(host)-1]
  626. }
  627. }
  628. ip := net.ParseIP(host)
  629. if ip != nil {
  630. um.Type = urlTypeIP
  631. } else if isCname {
  632. um.Type = urlTypeCname
  633. } else {
  634. um.Type = urlTypeAliyun
  635. }
  636. um.IsProxy = isProxy
  637. return nil
  638. }
  639. // getURL gets URL
  640. func (um urlMaker) getURL(bucket, object, params string) *url.URL {
  641. host, path := um.buildURL(bucket, object)
  642. addr := ""
  643. if params == "" {
  644. addr = fmt.Sprintf("%s://%s%s", um.Scheme, host, path)
  645. } else {
  646. addr = fmt.Sprintf("%s://%s%s?%s", um.Scheme, host, path, params)
  647. }
  648. uri, _ := url.ParseRequestURI(addr)
  649. return uri
  650. }
  651. // getSignURL gets sign URL
  652. func (um urlMaker) getSignURL(bucket, object, params string) string {
  653. host, path := um.buildURL(bucket, object)
  654. return fmt.Sprintf("%s://%s%s?%s", um.Scheme, host, path, params)
  655. }
  656. // getSignRtmpURL Build Sign Rtmp URL
  657. func (um urlMaker) getSignRtmpURL(bucket, channelName, params string) string {
  658. host, path := um.buildURL(bucket, "live")
  659. channelName = url.QueryEscape(channelName)
  660. channelName = strings.Replace(channelName, "+", "%20", -1)
  661. return fmt.Sprintf("rtmp://%s%s/%s?%s", host, path, channelName, params)
  662. }
  663. // buildURL builds URL
  664. func (um urlMaker) buildURL(bucket, object string) (string, string) {
  665. var host = ""
  666. var path = ""
  667. object = url.QueryEscape(object)
  668. object = strings.Replace(object, "+", "%20", -1)
  669. if um.Type == urlTypeCname {
  670. host = um.NetLoc
  671. path = "/" + object
  672. } else if um.Type == urlTypeIP {
  673. if bucket == "" {
  674. host = um.NetLoc
  675. path = "/"
  676. } else {
  677. host = um.NetLoc
  678. path = fmt.Sprintf("/%s/%s", bucket, object)
  679. }
  680. } else {
  681. if bucket == "" {
  682. host = um.NetLoc
  683. path = "/"
  684. } else {
  685. host = bucket + "." + um.NetLoc
  686. path = "/" + object
  687. }
  688. }
  689. return host, path
  690. }