conn.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794
  1. package oss
  2. import (
  3. "bytes"
  4. "crypto/md5"
  5. "encoding/base64"
  6. "encoding/json"
  7. "encoding/xml"
  8. "fmt"
  9. "hash"
  10. "io"
  11. "io/ioutil"
  12. "net"
  13. "net/http"
  14. "net/url"
  15. "os"
  16. "sort"
  17. "strconv"
  18. "strings"
  19. "time"
  20. )
  21. // Conn defines OSS Conn
  22. type Conn struct {
  23. config *Config
  24. url *urlMaker
  25. client *http.Client
  26. }
  27. var signKeyList = []string{"acl", "uploads", "location", "cors",
  28. "logging", "website", "referer", "lifecycle",
  29. "delete", "append", "tagging", "objectMeta",
  30. "uploadId", "partNumber", "security-token",
  31. "position", "img", "style", "styleName",
  32. "replication", "replicationProgress",
  33. "replicationLocation", "cname", "bucketInfo",
  34. "comp", "qos", "live", "status", "vod",
  35. "startTime", "endTime", "symlink",
  36. "x-oss-process", "response-content-type", "x-oss-traffic-limit",
  37. "response-content-language", "response-expires",
  38. "response-cache-control", "response-content-disposition",
  39. "response-content-encoding", "udf", "udfName", "udfImage",
  40. "udfId", "udfImageDesc", "udfApplication", "comp",
  41. "udfApplicationLog", "restore", "callback", "callback-var", "qosInfo",
  42. "policy", "stat", "encryption", "versions", "versioning", "versionId", "requestPayment",
  43. "x-oss-request-payer", "sequential",
  44. "inventory", "inventoryId", "continuation-token", "asyncFetch"}
  45. // init initializes Conn
  46. func (conn *Conn) init(config *Config, urlMaker *urlMaker, client *http.Client) error {
  47. if client == nil {
  48. // New transport
  49. transport := newTransport(conn, config)
  50. // Proxy
  51. if conn.config.IsUseProxy {
  52. proxyURL, err := url.Parse(config.ProxyHost)
  53. if err != nil {
  54. return err
  55. }
  56. if config.IsAuthProxy {
  57. if config.ProxyPassword != "" {
  58. proxyURL.User = url.UserPassword(config.ProxyUser, config.ProxyPassword)
  59. } else {
  60. proxyURL.User = url.User(config.ProxyUser)
  61. }
  62. }
  63. transport.Proxy = http.ProxyURL(proxyURL)
  64. }
  65. client = &http.Client{Transport: transport}
  66. if !config.RedirectEnabled {
  67. client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
  68. return http.ErrUseLastResponse
  69. }
  70. }
  71. }
  72. conn.config = config
  73. conn.url = urlMaker
  74. conn.client = client
  75. return nil
  76. }
  77. // Do sends request and returns the response
  78. func (conn Conn) Do(method, bucketName, objectName string, params map[string]interface{}, headers map[string]string,
  79. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  80. urlParams := conn.getURLParams(params)
  81. subResource := conn.getSubResource(params)
  82. uri := conn.url.getURL(bucketName, objectName, urlParams)
  83. resource := conn.getResource(bucketName, objectName, subResource)
  84. return conn.doRequest(method, uri, resource, headers, data, initCRC, listener)
  85. }
  86. // DoURL sends the request with signed URL and returns the response result.
  87. func (conn Conn) DoURL(method HTTPMethod, signedURL string, headers map[string]string,
  88. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  89. // Get URI from signedURL
  90. uri, err := url.ParseRequestURI(signedURL)
  91. if err != nil {
  92. return nil, err
  93. }
  94. m := strings.ToUpper(string(method))
  95. req := &http.Request{
  96. Method: m,
  97. URL: uri,
  98. Proto: "HTTP/1.1",
  99. ProtoMajor: 1,
  100. ProtoMinor: 1,
  101. Header: make(http.Header),
  102. Host: uri.Host,
  103. }
  104. tracker := &readerTracker{completedBytes: 0}
  105. fd, crc := conn.handleBody(req, data, initCRC, listener, tracker)
  106. if fd != nil {
  107. defer func() {
  108. fd.Close()
  109. os.Remove(fd.Name())
  110. }()
  111. }
  112. if conn.config.IsAuthProxy {
  113. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  114. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  115. req.Header.Set("Proxy-Authorization", basic)
  116. }
  117. req.Header.Set(HTTPHeaderHost, req.Host)
  118. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  119. if headers != nil {
  120. for k, v := range headers {
  121. req.Header.Set(k, v)
  122. }
  123. }
  124. // Transfer started
  125. event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength, 0)
  126. publishProgress(listener, event)
  127. if conn.config.LogLevel >= Debug {
  128. conn.LoggerHTTPReq(req)
  129. }
  130. resp, err := conn.client.Do(req)
  131. if err != nil {
  132. // Transfer failed
  133. event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0)
  134. publishProgress(listener, event)
  135. conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error())
  136. return nil, err
  137. }
  138. if conn.config.LogLevel >= Debug {
  139. //print out http resp
  140. conn.LoggerHTTPResp(req, resp)
  141. }
  142. // Transfer completed
  143. event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength, 0)
  144. publishProgress(listener, event)
  145. return conn.handleResponse(resp, crc)
  146. }
  147. func (conn Conn) getURLParams(params map[string]interface{}) string {
  148. // Sort
  149. keys := make([]string, 0, len(params))
  150. for k := range params {
  151. keys = append(keys, k)
  152. }
  153. sort.Strings(keys)
  154. // Serialize
  155. var buf bytes.Buffer
  156. for _, k := range keys {
  157. if buf.Len() > 0 {
  158. buf.WriteByte('&')
  159. }
  160. buf.WriteString(url.QueryEscape(k))
  161. if params[k] != nil {
  162. buf.WriteString("=" + strings.Replace(url.QueryEscape(params[k].(string)), "+", "%20", -1))
  163. }
  164. }
  165. return buf.String()
  166. }
  167. func (conn Conn) getSubResource(params map[string]interface{}) string {
  168. // Sort
  169. keys := make([]string, 0, len(params))
  170. signParams := make(map[string]string)
  171. for k := range params {
  172. if conn.config.AuthVersion == AuthV2 {
  173. encodedKey := url.QueryEscape(k)
  174. keys = append(keys, encodedKey)
  175. if params[k] != nil && params[k] != "" {
  176. signParams[encodedKey] = strings.Replace(url.QueryEscape(params[k].(string)), "+", "%20", -1)
  177. }
  178. } else if conn.isParamSign(k) {
  179. keys = append(keys, k)
  180. if params[k] != nil {
  181. signParams[k] = params[k].(string)
  182. }
  183. }
  184. }
  185. sort.Strings(keys)
  186. // Serialize
  187. var buf bytes.Buffer
  188. for _, k := range keys {
  189. if buf.Len() > 0 {
  190. buf.WriteByte('&')
  191. }
  192. buf.WriteString(k)
  193. if _, ok := signParams[k]; ok {
  194. buf.WriteString("=" + signParams[k])
  195. }
  196. }
  197. return buf.String()
  198. }
  199. func (conn Conn) isParamSign(paramKey string) bool {
  200. for _, k := range signKeyList {
  201. if paramKey == k {
  202. return true
  203. }
  204. }
  205. return false
  206. }
  207. // getResource gets canonicalized resource
  208. func (conn Conn) getResource(bucketName, objectName, subResource string) string {
  209. if subResource != "" {
  210. subResource = "?" + subResource
  211. }
  212. if bucketName == "" {
  213. if conn.config.AuthVersion == AuthV2 {
  214. return url.QueryEscape("/") + subResource
  215. }
  216. return fmt.Sprintf("/%s%s", bucketName, subResource)
  217. }
  218. if conn.config.AuthVersion == AuthV2 {
  219. return url.QueryEscape("/"+bucketName+"/") + strings.Replace(url.QueryEscape(objectName), "+", "%20", -1) + subResource
  220. }
  221. return fmt.Sprintf("/%s/%s%s", bucketName, objectName, subResource)
  222. }
  223. func (conn Conn) doRequest(method string, uri *url.URL, canonicalizedResource string, headers map[string]string,
  224. data io.Reader, initCRC uint64, listener ProgressListener) (*Response, error) {
  225. method = strings.ToUpper(method)
  226. req := &http.Request{
  227. Method: method,
  228. URL: uri,
  229. Proto: "HTTP/1.1",
  230. ProtoMajor: 1,
  231. ProtoMinor: 1,
  232. Header: make(http.Header),
  233. Host: uri.Host,
  234. }
  235. tracker := &readerTracker{completedBytes: 0}
  236. fd, crc := conn.handleBody(req, data, initCRC, listener, tracker)
  237. if fd != nil {
  238. defer func() {
  239. fd.Close()
  240. os.Remove(fd.Name())
  241. }()
  242. }
  243. if conn.config.IsAuthProxy {
  244. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  245. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  246. req.Header.Set("Proxy-Authorization", basic)
  247. }
  248. date := time.Now().UTC().Format(http.TimeFormat)
  249. req.Header.Set(HTTPHeaderDate, date)
  250. req.Header.Set(HTTPHeaderHost, req.Host)
  251. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  252. akIf := conn.config.GetCredentials()
  253. if akIf.GetSecurityToken() != "" {
  254. req.Header.Set(HTTPHeaderOssSecurityToken, akIf.GetSecurityToken())
  255. }
  256. if headers != nil {
  257. for k, v := range headers {
  258. req.Header.Set(k, v)
  259. }
  260. }
  261. conn.signHeader(req, canonicalizedResource)
  262. // Transfer started
  263. event := newProgressEvent(TransferStartedEvent, 0, req.ContentLength, 0)
  264. publishProgress(listener, event)
  265. if conn.config.LogLevel >= Debug {
  266. conn.LoggerHTTPReq(req)
  267. }
  268. resp, err := conn.client.Do(req)
  269. if err != nil {
  270. // Transfer failed
  271. event = newProgressEvent(TransferFailedEvent, tracker.completedBytes, req.ContentLength, 0)
  272. publishProgress(listener, event)
  273. conn.config.WriteLog(Debug, "[Resp:%p]http error:%s\n", req, err.Error())
  274. return nil, err
  275. }
  276. if conn.config.LogLevel >= Debug {
  277. //print out http resp
  278. conn.LoggerHTTPResp(req, resp)
  279. }
  280. // Transfer completed
  281. event = newProgressEvent(TransferCompletedEvent, tracker.completedBytes, req.ContentLength, 0)
  282. publishProgress(listener, event)
  283. return conn.handleResponse(resp, crc)
  284. }
  285. func (conn Conn) signURL(method HTTPMethod, bucketName, objectName string, expiration int64, params map[string]interface{}, headers map[string]string) string {
  286. akIf := conn.config.GetCredentials()
  287. if akIf.GetSecurityToken() != "" {
  288. params[HTTPParamSecurityToken] = akIf.GetSecurityToken()
  289. }
  290. m := strings.ToUpper(string(method))
  291. req := &http.Request{
  292. Method: m,
  293. Header: make(http.Header),
  294. }
  295. if conn.config.IsAuthProxy {
  296. auth := conn.config.ProxyUser + ":" + conn.config.ProxyPassword
  297. basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(auth))
  298. req.Header.Set("Proxy-Authorization", basic)
  299. }
  300. req.Header.Set(HTTPHeaderDate, strconv.FormatInt(expiration, 10))
  301. req.Header.Set(HTTPHeaderUserAgent, conn.config.UserAgent)
  302. if headers != nil {
  303. for k, v := range headers {
  304. req.Header.Set(k, v)
  305. }
  306. }
  307. if conn.config.AuthVersion == AuthV2 {
  308. params[HTTPParamSignatureVersion] = "OSS2"
  309. params[HTTPParamExpiresV2] = strconv.FormatInt(expiration, 10)
  310. params[HTTPParamAccessKeyIDV2] = conn.config.AccessKeyID
  311. additionalList, _ := conn.getAdditionalHeaderKeys(req)
  312. if len(additionalList) > 0 {
  313. params[HTTPParamAdditionalHeadersV2] = strings.Join(additionalList, ";")
  314. }
  315. }
  316. subResource := conn.getSubResource(params)
  317. canonicalizedResource := conn.getResource(bucketName, objectName, subResource)
  318. signedStr := conn.getSignedStr(req, canonicalizedResource, akIf.GetAccessKeySecret())
  319. if conn.config.AuthVersion == AuthV1 {
  320. params[HTTPParamExpires] = strconv.FormatInt(expiration, 10)
  321. params[HTTPParamAccessKeyID] = akIf.GetAccessKeyID()
  322. params[HTTPParamSignature] = signedStr
  323. } else if conn.config.AuthVersion == AuthV2 {
  324. params[HTTPParamSignatureV2] = signedStr
  325. }
  326. urlParams := conn.getURLParams(params)
  327. return conn.url.getSignURL(bucketName, objectName, urlParams)
  328. }
  329. func (conn Conn) signRtmpURL(bucketName, channelName, playlistName string, expiration int64) string {
  330. params := map[string]interface{}{}
  331. if playlistName != "" {
  332. params[HTTPParamPlaylistName] = playlistName
  333. }
  334. expireStr := strconv.FormatInt(expiration, 10)
  335. params[HTTPParamExpires] = expireStr
  336. akIf := conn.config.GetCredentials()
  337. if akIf.GetAccessKeyID() != "" {
  338. params[HTTPParamAccessKeyID] = akIf.GetAccessKeyID()
  339. if akIf.GetSecurityToken() != "" {
  340. params[HTTPParamSecurityToken] = akIf.GetSecurityToken()
  341. }
  342. signedStr := conn.getRtmpSignedStr(bucketName, channelName, playlistName, expiration, akIf.GetAccessKeySecret(), params)
  343. params[HTTPParamSignature] = signedStr
  344. }
  345. urlParams := conn.getURLParams(params)
  346. return conn.url.getSignRtmpURL(bucketName, channelName, urlParams)
  347. }
  348. // handleBody handles request body
  349. func (conn Conn) handleBody(req *http.Request, body io.Reader, initCRC uint64,
  350. listener ProgressListener, tracker *readerTracker) (*os.File, hash.Hash64) {
  351. var file *os.File
  352. var crc hash.Hash64
  353. reader := body
  354. readerLen, err := GetReaderLen(reader)
  355. if err == nil {
  356. req.ContentLength = readerLen
  357. }
  358. req.Header.Set(HTTPHeaderContentLength, strconv.FormatInt(req.ContentLength, 10))
  359. // MD5
  360. if body != nil && conn.config.IsEnableMD5 && req.Header.Get(HTTPHeaderContentMD5) == "" {
  361. md5 := ""
  362. reader, md5, file, _ = calcMD5(body, req.ContentLength, conn.config.MD5Threshold)
  363. req.Header.Set(HTTPHeaderContentMD5, md5)
  364. }
  365. // CRC
  366. if reader != nil && conn.config.IsEnableCRC {
  367. crc = NewCRC(CrcTable(), initCRC)
  368. reader = TeeReader(reader, crc, req.ContentLength, listener, tracker)
  369. }
  370. // HTTP body
  371. rc, ok := reader.(io.ReadCloser)
  372. if !ok && reader != nil {
  373. rc = ioutil.NopCloser(reader)
  374. }
  375. if conn.isUploadLimitReq(req) {
  376. limitReader := &LimitSpeedReader{
  377. reader: rc,
  378. ossLimiter: conn.config.UploadLimiter,
  379. }
  380. req.Body = limitReader
  381. } else {
  382. req.Body = rc
  383. }
  384. return file, crc
  385. }
  386. // isUploadLimitReq: judge limit upload speed or not
  387. func (conn Conn) isUploadLimitReq(req *http.Request) bool {
  388. if conn.config.UploadLimitSpeed == 0 || conn.config.UploadLimiter == nil {
  389. return false
  390. }
  391. if req.Method != "GET" && req.Method != "DELETE" && req.Method != "HEAD" {
  392. if req.ContentLength > 0 {
  393. return true
  394. }
  395. }
  396. return false
  397. }
  398. func tryGetFileSize(f *os.File) int64 {
  399. fInfo, _ := f.Stat()
  400. return fInfo.Size()
  401. }
  402. // handleResponse handles response
  403. func (conn Conn) handleResponse(resp *http.Response, crc hash.Hash64) (*Response, error) {
  404. var cliCRC uint64
  405. var srvCRC uint64
  406. statusCode := resp.StatusCode
  407. if statusCode >= 400 && statusCode <= 505 {
  408. // 4xx and 5xx indicate that the operation has error occurred
  409. var respBody []byte
  410. respBody, err := readResponseBody(resp)
  411. if err != nil {
  412. return nil, err
  413. }
  414. if len(respBody) == 0 {
  415. err = ServiceError{
  416. StatusCode: statusCode,
  417. RequestID: resp.Header.Get(HTTPHeaderOssRequestID),
  418. }
  419. } else {
  420. // Response contains storage service error object, unmarshal
  421. srvErr, errIn := serviceErrFromXML(respBody, resp.StatusCode,
  422. resp.Header.Get(HTTPHeaderOssRequestID))
  423. if errIn != nil { // error unmarshaling the error response
  424. err = fmt.Errorf("oss: service returned invalid response body, status = %s, RequestId = %s", resp.Status, resp.Header.Get(HTTPHeaderOssRequestID))
  425. } else {
  426. err = srvErr
  427. }
  428. }
  429. return &Response{
  430. StatusCode: resp.StatusCode,
  431. Headers: resp.Header,
  432. Body: ioutil.NopCloser(bytes.NewReader(respBody)), // restore the body
  433. }, err
  434. } else if statusCode >= 300 && statusCode <= 307 {
  435. // OSS use 3xx, but response has no body
  436. err := fmt.Errorf("oss: service returned %d,%s", resp.StatusCode, resp.Status)
  437. return &Response{
  438. StatusCode: resp.StatusCode,
  439. Headers: resp.Header,
  440. Body: resp.Body,
  441. }, err
  442. }
  443. if conn.config.IsEnableCRC && crc != nil {
  444. cliCRC = crc.Sum64()
  445. }
  446. srvCRC, _ = strconv.ParseUint(resp.Header.Get(HTTPHeaderOssCRC64), 10, 64)
  447. // 2xx, successful
  448. return &Response{
  449. StatusCode: resp.StatusCode,
  450. Headers: resp.Header,
  451. Body: resp.Body,
  452. ClientCRC: cliCRC,
  453. ServerCRC: srvCRC,
  454. }, nil
  455. }
  456. // LoggerHTTPReq Print the header information of the http request
  457. func (conn Conn) LoggerHTTPReq(req *http.Request) {
  458. var logBuffer bytes.Buffer
  459. logBuffer.WriteString(fmt.Sprintf("[Req:%p]Method:%s\t", req, req.Method))
  460. logBuffer.WriteString(fmt.Sprintf("Host:%s\t", req.URL.Host))
  461. logBuffer.WriteString(fmt.Sprintf("Path:%s\t", req.URL.Path))
  462. logBuffer.WriteString(fmt.Sprintf("Query:%s\t", req.URL.RawQuery))
  463. logBuffer.WriteString(fmt.Sprintf("Header info:"))
  464. for k, v := range req.Header {
  465. var valueBuffer bytes.Buffer
  466. for j := 0; j < len(v); j++ {
  467. if j > 0 {
  468. valueBuffer.WriteString(" ")
  469. }
  470. valueBuffer.WriteString(v[j])
  471. }
  472. logBuffer.WriteString(fmt.Sprintf("\t%s:%s", k, valueBuffer.String()))
  473. }
  474. conn.config.WriteLog(Debug, "%s\n", logBuffer.String())
  475. }
  476. // LoggerHTTPResp Print Response to http request
  477. func (conn Conn) LoggerHTTPResp(req *http.Request, resp *http.Response) {
  478. var logBuffer bytes.Buffer
  479. logBuffer.WriteString(fmt.Sprintf("[Resp:%p]StatusCode:%d\t", req, resp.StatusCode))
  480. logBuffer.WriteString(fmt.Sprintf("Header info:"))
  481. for k, v := range resp.Header {
  482. var valueBuffer bytes.Buffer
  483. for j := 0; j < len(v); j++ {
  484. if j > 0 {
  485. valueBuffer.WriteString(" ")
  486. }
  487. valueBuffer.WriteString(v[j])
  488. }
  489. logBuffer.WriteString(fmt.Sprintf("\t%s:%s", k, valueBuffer.String()))
  490. }
  491. conn.config.WriteLog(Debug, "%s\n", logBuffer.String())
  492. }
  493. func calcMD5(body io.Reader, contentLen, md5Threshold int64) (reader io.Reader, b64 string, tempFile *os.File, err error) {
  494. if contentLen == 0 || contentLen > md5Threshold {
  495. // Huge body, use temporary file
  496. tempFile, err = ioutil.TempFile(os.TempDir(), TempFilePrefix)
  497. if tempFile != nil {
  498. io.Copy(tempFile, body)
  499. tempFile.Seek(0, os.SEEK_SET)
  500. md5 := md5.New()
  501. io.Copy(md5, tempFile)
  502. sum := md5.Sum(nil)
  503. b64 = base64.StdEncoding.EncodeToString(sum[:])
  504. tempFile.Seek(0, os.SEEK_SET)
  505. reader = tempFile
  506. }
  507. } else {
  508. // Small body, use memory
  509. buf, _ := ioutil.ReadAll(body)
  510. sum := md5.Sum(buf)
  511. b64 = base64.StdEncoding.EncodeToString(sum[:])
  512. reader = bytes.NewReader(buf)
  513. }
  514. return
  515. }
  516. func readResponseBody(resp *http.Response) ([]byte, error) {
  517. defer resp.Body.Close()
  518. out, err := ioutil.ReadAll(resp.Body)
  519. if err == io.EOF {
  520. err = nil
  521. }
  522. return out, err
  523. }
  524. func serviceErrFromXML(body []byte, statusCode int, requestID string) (ServiceError, error) {
  525. var storageErr ServiceError
  526. if err := xml.Unmarshal(body, &storageErr); err != nil {
  527. return storageErr, err
  528. }
  529. storageErr.StatusCode = statusCode
  530. storageErr.RequestID = requestID
  531. storageErr.RawMessage = string(body)
  532. return storageErr, nil
  533. }
  534. func xmlUnmarshal(body io.Reader, v interface{}) error {
  535. data, err := ioutil.ReadAll(body)
  536. if err != nil {
  537. return err
  538. }
  539. return xml.Unmarshal(data, v)
  540. }
  541. func jsonUnmarshal(body io.Reader, v interface{}) error {
  542. data, err := ioutil.ReadAll(body)
  543. if err != nil {
  544. return err
  545. }
  546. return json.Unmarshal(data, v)
  547. }
  548. // timeoutConn handles HTTP timeout
  549. type timeoutConn struct {
  550. conn net.Conn
  551. timeout time.Duration
  552. longTimeout time.Duration
  553. }
  554. func newTimeoutConn(conn net.Conn, timeout time.Duration, longTimeout time.Duration) *timeoutConn {
  555. conn.SetReadDeadline(time.Now().Add(longTimeout))
  556. return &timeoutConn{
  557. conn: conn,
  558. timeout: timeout,
  559. longTimeout: longTimeout,
  560. }
  561. }
  562. func (c *timeoutConn) Read(b []byte) (n int, err error) {
  563. c.SetReadDeadline(time.Now().Add(c.timeout))
  564. n, err = c.conn.Read(b)
  565. c.SetReadDeadline(time.Now().Add(c.longTimeout))
  566. return n, err
  567. }
  568. func (c *timeoutConn) Write(b []byte) (n int, err error) {
  569. c.SetWriteDeadline(time.Now().Add(c.timeout))
  570. n, err = c.conn.Write(b)
  571. c.SetReadDeadline(time.Now().Add(c.longTimeout))
  572. return n, err
  573. }
  574. func (c *timeoutConn) Close() error {
  575. return c.conn.Close()
  576. }
  577. func (c *timeoutConn) LocalAddr() net.Addr {
  578. return c.conn.LocalAddr()
  579. }
  580. func (c *timeoutConn) RemoteAddr() net.Addr {
  581. return c.conn.RemoteAddr()
  582. }
  583. func (c *timeoutConn) SetDeadline(t time.Time) error {
  584. return c.conn.SetDeadline(t)
  585. }
  586. func (c *timeoutConn) SetReadDeadline(t time.Time) error {
  587. return c.conn.SetReadDeadline(t)
  588. }
  589. func (c *timeoutConn) SetWriteDeadline(t time.Time) error {
  590. return c.conn.SetWriteDeadline(t)
  591. }
  592. // UrlMaker builds URL and resource
  593. const (
  594. urlTypeCname = 1
  595. urlTypeIP = 2
  596. urlTypeAliyun = 3
  597. )
  598. type urlMaker struct {
  599. Scheme string // HTTP or HTTPS
  600. NetLoc string // Host or IP
  601. Type int // 1 CNAME, 2 IP, 3 ALIYUN
  602. IsProxy bool // Proxy
  603. }
  604. // Init parses endpoint
  605. func (um *urlMaker) Init(endpoint string, isCname bool, isProxy bool) error {
  606. if strings.HasPrefix(endpoint, "http://") {
  607. um.Scheme = "http"
  608. um.NetLoc = endpoint[len("http://"):]
  609. } else if strings.HasPrefix(endpoint, "https://") {
  610. um.Scheme = "https"
  611. um.NetLoc = endpoint[len("https://"):]
  612. } else {
  613. um.Scheme = "http"
  614. um.NetLoc = endpoint
  615. }
  616. //use url.Parse() to get real host
  617. strUrl := um.Scheme + "://" + um.NetLoc
  618. url, err := url.Parse(strUrl)
  619. if err != nil {
  620. return err
  621. }
  622. um.NetLoc = url.Host
  623. host, _, err := net.SplitHostPort(um.NetLoc)
  624. if err != nil {
  625. host = um.NetLoc
  626. if host[0] == '[' && host[len(host)-1] == ']' {
  627. host = host[1 : len(host)-1]
  628. }
  629. }
  630. ip := net.ParseIP(host)
  631. if ip != nil {
  632. um.Type = urlTypeIP
  633. } else if isCname {
  634. um.Type = urlTypeCname
  635. } else {
  636. um.Type = urlTypeAliyun
  637. }
  638. um.IsProxy = isProxy
  639. return nil
  640. }
  641. // getURL gets URL
  642. func (um urlMaker) getURL(bucket, object, params string) *url.URL {
  643. host, path := um.buildURL(bucket, object)
  644. addr := ""
  645. if params == "" {
  646. addr = fmt.Sprintf("%s://%s%s", um.Scheme, host, path)
  647. } else {
  648. addr = fmt.Sprintf("%s://%s%s?%s", um.Scheme, host, path, params)
  649. }
  650. uri, _ := url.ParseRequestURI(addr)
  651. return uri
  652. }
  653. // getSignURL gets sign URL
  654. func (um urlMaker) getSignURL(bucket, object, params string) string {
  655. host, path := um.buildURL(bucket, object)
  656. return fmt.Sprintf("%s://%s%s?%s", um.Scheme, host, path, params)
  657. }
  658. // getSignRtmpURL Build Sign Rtmp URL
  659. func (um urlMaker) getSignRtmpURL(bucket, channelName, params string) string {
  660. host, path := um.buildURL(bucket, "live")
  661. channelName = url.QueryEscape(channelName)
  662. channelName = strings.Replace(channelName, "+", "%20", -1)
  663. return fmt.Sprintf("rtmp://%s%s/%s?%s", host, path, channelName, params)
  664. }
  665. // buildURL builds URL
  666. func (um urlMaker) buildURL(bucket, object string) (string, string) {
  667. var host = ""
  668. var path = ""
  669. object = url.QueryEscape(object)
  670. object = strings.Replace(object, "+", "%20", -1)
  671. if um.Type == urlTypeCname {
  672. host = um.NetLoc
  673. path = "/" + object
  674. } else if um.Type == urlTypeIP {
  675. if bucket == "" {
  676. host = um.NetLoc
  677. path = "/"
  678. } else {
  679. host = um.NetLoc
  680. path = fmt.Sprintf("/%s/%s", bucket, object)
  681. }
  682. } else {
  683. if bucket == "" {
  684. host = um.NetLoc
  685. path = "/"
  686. } else {
  687. host = bucket + "." + um.NetLoc
  688. path = "/" + object
  689. }
  690. }
  691. return host, path
  692. }