progress_test.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
  1. // bucket test
  2. package oss
  3. import (
  4. "bytes"
  5. "io/ioutil"
  6. "math/rand"
  7. "os"
  8. "strings"
  9. . "gopkg.in/check.v1"
  10. )
  11. type OssProgressSuite struct {
  12. client *Client
  13. bucket *Bucket
  14. }
  15. var _ = Suite(&OssProgressSuite{})
  16. // SetUpSuite runs once when the suite starts running
  17. func (s *OssProgressSuite) SetUpSuite(c *C) {
  18. client, err := New(endpoint, accessID, accessKey)
  19. c.Assert(err, IsNil)
  20. s.client = client
  21. s.client.CreateBucket(bucketName)
  22. bucket, err := s.client.Bucket(bucketName)
  23. c.Assert(err, IsNil)
  24. s.bucket = bucket
  25. testLogger.Println("test progress started")
  26. }
  27. // TearDownSuite runs before each test or benchmark starts running
  28. func (s *OssProgressSuite) TearDownSuite(c *C) {
  29. // Abort multipart uploads
  30. keyMarker := KeyMarker("")
  31. uploadIDMarker := UploadIDMarker("")
  32. for {
  33. lmu, err := s.bucket.ListMultipartUploads(keyMarker, uploadIDMarker)
  34. c.Assert(err, IsNil)
  35. for _, upload := range lmu.Uploads {
  36. imur := InitiateMultipartUploadResult{Bucket: bucketName, Key: upload.Key, UploadID: upload.UploadID}
  37. err = s.bucket.AbortMultipartUpload(imur)
  38. c.Assert(err, IsNil)
  39. }
  40. keyMarker = KeyMarker(lmu.NextKeyMarker)
  41. uploadIDMarker = UploadIDMarker(lmu.NextUploadIDMarker)
  42. if !lmu.IsTruncated {
  43. break
  44. }
  45. }
  46. // Delete objects
  47. marker := Marker("")
  48. for {
  49. lor, err := s.bucket.ListObjects(marker)
  50. c.Assert(err, IsNil)
  51. for _, object := range lor.Objects {
  52. err = s.bucket.DeleteObject(object.Key)
  53. c.Assert(err, IsNil)
  54. }
  55. marker = Marker(lor.NextMarker)
  56. if !lor.IsTruncated {
  57. break
  58. }
  59. }
  60. // Delete bucket
  61. err := s.client.DeleteBucket(s.bucket.BucketName)
  62. c.Assert(err, IsNil)
  63. testLogger.Println("test progress completed")
  64. }
  65. // SetUpTest runs after each test or benchmark runs
  66. func (s *OssProgressSuite) SetUpTest(c *C) {
  67. err := removeTempFiles("../oss", ".jpg")
  68. c.Assert(err, IsNil)
  69. err = removeTempFiles("../oss", ".txt")
  70. c.Assert(err, IsNil)
  71. err = removeTempFiles("../oss", ".html")
  72. c.Assert(err, IsNil)
  73. }
  74. // TearDownTest runs once after all tests or benchmarks have finished running
  75. func (s *OssProgressSuite) TearDownTest(c *C) {
  76. err := removeTempFiles("../oss", ".jpg")
  77. c.Assert(err, IsNil)
  78. err = removeTempFiles("../oss", ".txt")
  79. c.Assert(err, IsNil)
  80. err = removeTempFiles("../oss", ".html")
  81. c.Assert(err, IsNil)
  82. }
  83. // OssProgressListener is the progress listener
  84. type OssProgressListener struct {
  85. }
  86. // ProgressChanged handles progress event
  87. func (listener *OssProgressListener) ProgressChanged(event *ProgressEvent) {
  88. switch event.EventType {
  89. case TransferStartedEvent:
  90. testLogger.Printf("Transfer Started, ConsumedBytes: %d, TotalBytes %d.\n",
  91. event.ConsumedBytes, event.TotalBytes)
  92. case TransferDataEvent:
  93. testLogger.Printf("Transfer Data, ConsumedBytes: %d, TotalBytes %d, %d%%.\n",
  94. event.ConsumedBytes, event.TotalBytes, event.ConsumedBytes*100/event.TotalBytes)
  95. case TransferCompletedEvent:
  96. testLogger.Printf("Transfer Completed, ConsumedBytes: %d, TotalBytes %d.\n",
  97. event.ConsumedBytes, event.TotalBytes)
  98. case TransferFailedEvent:
  99. testLogger.Printf("Transfer Failed, ConsumedBytes: %d, TotalBytes %d.\n",
  100. event.ConsumedBytes, event.TotalBytes)
  101. default:
  102. }
  103. }
  104. // TestPutObject
  105. func (s *OssProgressSuite) TestPutObject(c *C) {
  106. objectName := randStr(8) + ".jpg"
  107. localFile := "../sample/The Go Programming Language.html"
  108. // PutObject
  109. fd, err := os.Open(localFile)
  110. c.Assert(err, IsNil)
  111. defer fd.Close()
  112. err = s.bucket.PutObject(objectName, fd, Progress(&OssProgressListener{}))
  113. c.Assert(err, IsNil)
  114. // PutObjectFromFile
  115. err = s.bucket.PutObjectFromFile(objectName, localFile, Progress(&OssProgressListener{}))
  116. c.Assert(err, IsNil)
  117. // DoPutObject
  118. fd, err = os.Open(localFile)
  119. c.Assert(err, IsNil)
  120. defer fd.Close()
  121. request := &PutObjectRequest{
  122. ObjectKey: objectName,
  123. Reader: fd,
  124. }
  125. options := []Option{Progress(&OssProgressListener{})}
  126. _, err = s.bucket.DoPutObject(request, options)
  127. c.Assert(err, IsNil)
  128. // PutObject size is 0
  129. err = s.bucket.PutObject(objectName, strings.NewReader(""), Progress(&OssProgressListener{}))
  130. c.Assert(err, IsNil)
  131. testLogger.Println("OssProgressSuite.TestPutObject")
  132. }
  133. // TestSignURL
  134. func (s *OssProgressSuite) TestSignURL(c *C) {
  135. objectName := objectNamePrefix + randStr(8)
  136. filePath := randLowStr(10)
  137. content := randStr(20)
  138. createFile(filePath, content, c)
  139. // Sign URL for put
  140. str, err := s.bucket.SignURL(objectName, HTTPPut, 60, Progress(&OssProgressListener{}))
  141. c.Assert(err, IsNil)
  142. c.Assert(strings.Contains(str, HTTPParamExpires+"="), Equals, true)
  143. c.Assert(strings.Contains(str, HTTPParamAccessKeyID+"="), Equals, true)
  144. c.Assert(strings.Contains(str, HTTPParamSignature+"="), Equals, true)
  145. // Put object with URL
  146. fd, err := os.Open(filePath)
  147. c.Assert(err, IsNil)
  148. defer fd.Close()
  149. err = s.bucket.PutObjectWithURL(str, fd, Progress(&OssProgressListener{}))
  150. c.Assert(err, IsNil)
  151. // Put object from file with URL
  152. err = s.bucket.PutObjectFromFileWithURL(str, filePath, Progress(&OssProgressListener{}))
  153. c.Assert(err, IsNil)
  154. // DoPutObject
  155. fd, err = os.Open(filePath)
  156. c.Assert(err, IsNil)
  157. defer fd.Close()
  158. options := []Option{Progress(&OssProgressListener{})}
  159. _, err = s.bucket.DoPutObjectWithURL(str, fd, options)
  160. c.Assert(err, IsNil)
  161. // Sign URL for get
  162. str, err = s.bucket.SignURL(objectName, HTTPGet, 60, Progress(&OssProgressListener{}))
  163. c.Assert(err, IsNil)
  164. c.Assert(strings.Contains(str, HTTPParamExpires+"="), Equals, true)
  165. c.Assert(strings.Contains(str, HTTPParamAccessKeyID+"="), Equals, true)
  166. c.Assert(strings.Contains(str, HTTPParamSignature+"="), Equals, true)
  167. // Get object with URL
  168. body, err := s.bucket.GetObjectWithURL(str, Progress(&OssProgressListener{}))
  169. c.Assert(err, IsNil)
  170. str, err = readBody(body)
  171. c.Assert(err, IsNil)
  172. c.Assert(str, Equals, content)
  173. // Get object to file with URL
  174. str, err = s.bucket.SignURL(objectName, HTTPGet, 10, Progress(&OssProgressListener{}))
  175. c.Assert(err, IsNil)
  176. newFile := randStr(10)
  177. err = s.bucket.GetObjectToFileWithURL(str, newFile, Progress(&OssProgressListener{}))
  178. c.Assert(err, IsNil)
  179. eq, err := compareFiles(filePath, newFile)
  180. c.Assert(err, IsNil)
  181. c.Assert(eq, Equals, true)
  182. os.Remove(filePath)
  183. os.Remove(newFile)
  184. err = s.bucket.DeleteObject(objectName)
  185. c.Assert(err, IsNil)
  186. testLogger.Println("OssProgressSuite.TestSignURL")
  187. }
  188. func (s *OssProgressSuite) TestPutObjectNegative(c *C) {
  189. objectName := objectNamePrefix + randStr(8)
  190. localFile := "../sample/The Go Programming Language.html"
  191. // Invalid endpoint
  192. client, err := New("http://oss-cn-taikang.aliyuncs.com", accessID, accessKey)
  193. c.Assert(err, IsNil)
  194. bucket, err := client.Bucket(bucketName)
  195. c.Assert(err, IsNil)
  196. err = bucket.PutObjectFromFile(objectName, localFile, Progress(&OssProgressListener{}))
  197. testLogger.Println(err)
  198. c.Assert(err, NotNil)
  199. testLogger.Println("OssProgressSuite.TestPutObjectNegative")
  200. }
  201. // TestAppendObject
  202. func (s *OssProgressSuite) TestAppendObject(c *C) {
  203. objectName := objectNamePrefix + randStr(8)
  204. objectValue := "昨夜雨疏风骤,浓睡不消残酒。试问卷帘人,却道海棠依旧。知否?知否?应是绿肥红瘦。"
  205. var val = []byte(objectValue)
  206. var nextPos int64
  207. var midPos = 1 + rand.Intn(len(val)-1)
  208. // AppendObject
  209. nextPos, err := s.bucket.AppendObject(objectName, bytes.NewReader(val[0:midPos]), nextPos, Progress(&OssProgressListener{}))
  210. c.Assert(err, IsNil)
  211. // DoAppendObject
  212. request := &AppendObjectRequest{
  213. ObjectKey: objectName,
  214. Reader: bytes.NewReader(val[midPos:]),
  215. Position: nextPos,
  216. }
  217. options := []Option{Progress(&OssProgressListener{})}
  218. _, err = s.bucket.DoAppendObject(request, options)
  219. c.Assert(err, IsNil)
  220. testLogger.Println("OssProgressSuite.TestAppendObject")
  221. }
  222. // TestMultipartUpload
  223. func (s *OssProgressSuite) TestMultipartUpload(c *C) {
  224. objectName := objectNamePrefix + randStr(8)
  225. var fileName = "../sample/BingWallpaper-2015-11-07.jpg"
  226. chunks, err := SplitFileByPartNum(fileName, 3)
  227. c.Assert(err, IsNil)
  228. testLogger.Println("chunks:", chunks)
  229. fd, err := os.Open(fileName)
  230. c.Assert(err, IsNil)
  231. defer fd.Close()
  232. // Initiate
  233. imur, err := s.bucket.InitiateMultipartUpload(objectName)
  234. c.Assert(err, IsNil)
  235. // UploadPart
  236. var parts []UploadPart
  237. for _, chunk := range chunks {
  238. fd.Seek(chunk.Offset, os.SEEK_SET)
  239. part, err := s.bucket.UploadPart(imur, fd, chunk.Size, chunk.Number, Progress(&OssProgressListener{}))
  240. c.Assert(err, IsNil)
  241. parts = append(parts, part)
  242. }
  243. // Complete
  244. _, err = s.bucket.CompleteMultipartUpload(imur, parts)
  245. c.Assert(err, IsNil)
  246. err = s.bucket.DeleteObject(objectName)
  247. c.Assert(err, IsNil)
  248. testLogger.Println("OssProgressSuite.TestMultipartUpload")
  249. }
  250. // TestMultipartUploadFromFile
  251. func (s *OssProgressSuite) TestMultipartUploadFromFile(c *C) {
  252. objectName := objectNamePrefix + randStr(8)
  253. var fileName = "../sample/BingWallpaper-2015-11-07.jpg"
  254. chunks, err := SplitFileByPartNum(fileName, 3)
  255. c.Assert(err, IsNil)
  256. // Initiate
  257. imur, err := s.bucket.InitiateMultipartUpload(objectName)
  258. c.Assert(err, IsNil)
  259. // UploadPart
  260. var parts []UploadPart
  261. for _, chunk := range chunks {
  262. part, err := s.bucket.UploadPartFromFile(imur, fileName, chunk.Offset, chunk.Size, chunk.Number, Progress(&OssProgressListener{}))
  263. c.Assert(err, IsNil)
  264. parts = append(parts, part)
  265. }
  266. // Complete
  267. _, err = s.bucket.CompleteMultipartUpload(imur, parts)
  268. c.Assert(err, IsNil)
  269. err = s.bucket.DeleteObject(objectName)
  270. c.Assert(err, IsNil)
  271. testLogger.Println("OssProgressSuite.TestMultipartUploadFromFile")
  272. }
  273. // TestGetObject
  274. func (s *OssProgressSuite) TestGetObject(c *C) {
  275. objectName := objectNamePrefix + randStr(8)
  276. localFile := "../sample/BingWallpaper-2015-11-07.jpg"
  277. newFile := "newpic-progress-1.jpg"
  278. // PutObject
  279. err := s.bucket.PutObjectFromFile(objectName, localFile, Progress(&OssProgressListener{}))
  280. c.Assert(err, IsNil)
  281. // GetObject
  282. body, err := s.bucket.GetObject(objectName, Progress(&OssProgressListener{}))
  283. c.Assert(err, IsNil)
  284. _, err = ioutil.ReadAll(body)
  285. c.Assert(err, IsNil)
  286. body.Close()
  287. // GetObjectToFile
  288. err = s.bucket.GetObjectToFile(objectName, newFile, Progress(&OssProgressListener{}))
  289. c.Assert(err, IsNil)
  290. // DoGetObject
  291. request := &GetObjectRequest{objectName}
  292. options := []Option{Progress(&OssProgressListener{})}
  293. result, err := s.bucket.DoGetObject(request, options)
  294. c.Assert(err, IsNil)
  295. _, err = ioutil.ReadAll(result.Response.Body)
  296. c.Assert(err, IsNil)
  297. result.Response.Body.Close()
  298. // GetObject with range
  299. body, err = s.bucket.GetObject(objectName, Range(1024, 4*1024), Progress(&OssProgressListener{}))
  300. c.Assert(err, IsNil)
  301. _, err = ioutil.ReadAll(body)
  302. c.Assert(err, IsNil)
  303. body.Close()
  304. // PutObject size is 0
  305. err = s.bucket.PutObject(objectName, strings.NewReader(""), Progress(&OssProgressListener{}))
  306. c.Assert(err, IsNil)
  307. // GetObject size is 0
  308. body, err = s.bucket.GetObject(objectName, Progress(&OssProgressListener{}))
  309. c.Assert(err, IsNil)
  310. _, err = ioutil.ReadAll(body)
  311. c.Assert(err, IsNil)
  312. body.Close()
  313. testLogger.Println("OssProgressSuite.TestGetObject")
  314. }
  315. // TestGetObjectNegative
  316. func (s *OssProgressSuite) TestGetObjectNegative(c *C) {
  317. objectName := objectNamePrefix + randStr(8)
  318. localFile := "../sample/BingWallpaper-2015-11-07.jpg"
  319. // PutObject
  320. err := s.bucket.PutObjectFromFile(objectName, localFile)
  321. c.Assert(err, IsNil)
  322. // GetObject
  323. body, err := s.bucket.GetObject(objectName, Progress(&OssProgressListener{}))
  324. c.Assert(err, IsNil)
  325. buf := make([]byte, 4*1024)
  326. n, err := body.Read(buf)
  327. c.Assert(err, IsNil)
  328. //time.Sleep(70 * time.Second) TODO
  329. // Read should fail
  330. for err == nil {
  331. n, err = body.Read(buf)
  332. n += n
  333. }
  334. c.Assert(err, NotNil)
  335. body.Close()
  336. testLogger.Println("OssProgressSuite.TestGetObjectNegative")
  337. }
  338. // TestUploadFile
  339. func (s *OssProgressSuite) TestUploadFile(c *C) {
  340. objectName := objectNamePrefix + randStr(8)
  341. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  342. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(5), Progress(&OssProgressListener{}))
  343. c.Assert(err, IsNil)
  344. err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, objectName+".cp"), Progress(&OssProgressListener{}))
  345. c.Assert(err, IsNil)
  346. testLogger.Println("OssProgressSuite.TestUploadFile")
  347. }
  348. // TestDownloadFile
  349. func (s *OssProgressSuite) TestDownloadFile(c *C) {
  350. objectName := objectNamePrefix + randStr(8)
  351. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  352. newFile := "down-new-file-progress-2.jpg"
  353. // Upload
  354. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  355. c.Assert(err, IsNil)
  356. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(5), Progress(&OssProgressListener{}))
  357. c.Assert(err, IsNil)
  358. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), Progress(&OssProgressListener{}))
  359. c.Assert(err, IsNil)
  360. err = s.bucket.DownloadFile(objectName, newFile, 50*1024, Routines(3), Checkpoint(true, ""), Progress(&OssProgressListener{}))
  361. c.Assert(err, IsNil)
  362. testLogger.Println("OssProgressSuite.TestDownloadFile")
  363. }
  364. // TestCopyFile
  365. func (s *OssProgressSuite) TestCopyFile(c *C) {
  366. srcObjectName := objectNamePrefix + randStr(8)
  367. destObjectName := srcObjectName + "-copy"
  368. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  369. // Upload
  370. err := s.bucket.UploadFile(srcObjectName, fileName, 100*1024, Routines(3))
  371. c.Assert(err, IsNil)
  372. err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 100*1024, Routines(5), Progress(&OssProgressListener{}))
  373. c.Assert(err, IsNil)
  374. err = s.bucket.CopyFile(bucketName, srcObjectName, destObjectName, 1024*100, Routines(3), Checkpoint(true, ""), Progress(&OssProgressListener{}))
  375. c.Assert(err, IsNil)
  376. testLogger.Println("OssProgressSuite.TestCopyFile")
  377. }