download_test.go 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676
  1. package oss
  2. import (
  3. "bytes"
  4. "fmt"
  5. "os"
  6. "time"
  7. . "gopkg.in/check.v1"
  8. )
  9. type OssDownloadSuite struct {
  10. client *Client
  11. bucket *Bucket
  12. }
  13. var _ = Suite(&OssDownloadSuite{})
  14. // SetUpSuite runs once when the suite starts running
  15. func (s *OssDownloadSuite) SetUpSuite(c *C) {
  16. client, err := New(endpoint, accessID, accessKey)
  17. c.Assert(err, IsNil)
  18. s.client = client
  19. s.client.CreateBucket(bucketName)
  20. // time.Sleep(timeoutInOperation)
  21. bucket, err := s.client.Bucket(bucketName)
  22. c.Assert(err, IsNil)
  23. s.bucket = bucket
  24. testLogger.Println("test download started")
  25. }
  26. // TearDownSuite runs before each test or benchmark starts running
  27. func (s *OssDownloadSuite) TearDownSuite(c *C) {
  28. // Delete part
  29. keyMarker := KeyMarker("")
  30. uploadIDMarker := UploadIDMarker("")
  31. for {
  32. lmur, err := s.bucket.ListMultipartUploads(keyMarker, uploadIDMarker)
  33. c.Assert(err, IsNil)
  34. for _, upload := range lmur.Uploads {
  35. var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName,
  36. Key: upload.Key, UploadID: upload.UploadID}
  37. err = s.bucket.AbortMultipartUpload(imur)
  38. c.Assert(err, IsNil)
  39. }
  40. keyMarker = KeyMarker(lmur.NextKeyMarker)
  41. uploadIDMarker = UploadIDMarker(lmur.NextUploadIDMarker)
  42. if !lmur.IsTruncated {
  43. break
  44. }
  45. }
  46. // Delete objects
  47. marker := Marker("")
  48. for {
  49. lor, err := s.bucket.ListObjects(marker)
  50. c.Assert(err, IsNil)
  51. for _, object := range lor.Objects {
  52. err = s.bucket.DeleteObject(object.Key)
  53. c.Assert(err, IsNil)
  54. }
  55. marker = Marker(lor.NextMarker)
  56. if !lor.IsTruncated {
  57. break
  58. }
  59. }
  60. // Delete bucket
  61. err := s.client.DeleteBucket(s.bucket.BucketName)
  62. c.Assert(err, IsNil)
  63. testLogger.Println("test download completed")
  64. }
  65. // SetUpTest runs after each test or benchmark runs
  66. func (s *OssDownloadSuite) SetUpTest(c *C) {
  67. err := removeTempFiles("../oss", ".jpg")
  68. c.Assert(err, IsNil)
  69. }
  70. // TearDownTest runs once after all tests or benchmarks have finished running
  71. func (s *OssDownloadSuite) TearDownTest(c *C) {
  72. err := removeTempFiles("../oss", ".jpg")
  73. c.Assert(err, IsNil)
  74. err = removeTempFiles("../oss", ".temp")
  75. c.Assert(err, IsNil)
  76. }
  77. // TestDownloadRoutineWithoutRecovery multipart downloads without checkpoint
  78. func (s *OssDownloadSuite) TestDownloadRoutineWithoutRecovery(c *C) {
  79. objectName := objectNamePrefix + "tdrwr"
  80. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  81. newFile := "down-new-file.jpg"
  82. // Upload a file
  83. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  84. c.Assert(err, IsNil)
  85. // Download the file by default
  86. err = s.bucket.DownloadFile(objectName, newFile, 100*1024)
  87. c.Assert(err, IsNil)
  88. // Check
  89. eq, err := compareFiles(fileName, newFile)
  90. c.Assert(err, IsNil)
  91. c.Assert(eq, Equals, true)
  92. // Use 2 coroutines to download the file and total parts count is 5
  93. os.Remove(newFile)
  94. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2))
  95. c.Assert(err, IsNil)
  96. // Check
  97. eq, err = compareFiles(fileName, newFile)
  98. c.Assert(err, IsNil)
  99. c.Assert(eq, Equals, true)
  100. // Use 5 coroutines to download the file and the total parts count is 5.
  101. os.Remove(newFile)
  102. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(5))
  103. c.Assert(err, IsNil)
  104. // Check
  105. eq, err = compareFiles(fileName, newFile)
  106. c.Assert(err, IsNil)
  107. c.Assert(eq, Equals, true)
  108. // Use 10 coroutines to download the file and the total parts count is 5.
  109. os.Remove(newFile)
  110. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10))
  111. c.Assert(err, IsNil)
  112. // Check
  113. eq, err = compareFiles(fileName, newFile)
  114. c.Assert(err, IsNil)
  115. c.Assert(eq, Equals, true)
  116. err = s.bucket.DeleteObject(objectName)
  117. c.Assert(err, IsNil)
  118. }
  119. // DownErrorHooker requests hook by downloadPart
  120. func DownErrorHooker(part downloadPart) error {
  121. if part.Index == 4 {
  122. time.Sleep(time.Second)
  123. return fmt.Errorf("ErrorHooker")
  124. }
  125. return nil
  126. }
  127. // TestDownloadRoutineWithRecovery multi-routine resumable download
  128. func (s *OssDownloadSuite) TestDownloadRoutineWithRecovery(c *C) {
  129. objectName := objectNamePrefix + "tdrtr"
  130. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  131. newFile := "down-new-file-2.jpg"
  132. // Upload a file
  133. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  134. c.Assert(err, IsNil)
  135. // Download a file with default checkpoint
  136. downloadPartHooker = DownErrorHooker
  137. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  138. c.Assert(err, NotNil)
  139. c.Assert(err.Error(), Equals, "ErrorHooker")
  140. downloadPartHooker = defaultDownloadPartHook
  141. // Check
  142. dcp := downloadCheckpoint{}
  143. err = dcp.load(newFile + ".cp")
  144. c.Assert(err, IsNil)
  145. c.Assert(dcp.Magic, Equals, downloadCpMagic)
  146. c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ=="))
  147. c.Assert(dcp.FilePath, Equals, newFile)
  148. c.Assert(dcp.ObjStat.Size, Equals, int64(482048))
  149. c.Assert(len(dcp.ObjStat.LastModified) > 0, Equals, true)
  150. c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"")
  151. c.Assert(dcp.Object, Equals, objectName)
  152. c.Assert(len(dcp.Parts), Equals, 5)
  153. c.Assert(len(dcp.todoParts()), Equals, 1)
  154. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  155. c.Assert(err, IsNil)
  156. //download success, checkpoint file has been deleted
  157. err = dcp.load(newFile + ".cp")
  158. c.Assert(err, NotNil)
  159. eq, err := compareFiles(fileName, newFile)
  160. c.Assert(err, IsNil)
  161. c.Assert(eq, Equals, true)
  162. // Resumable download with empty checkpoint file path
  163. downloadPartHooker = DownErrorHooker
  164. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, ""))
  165. c.Assert(err, NotNil)
  166. c.Assert(err.Error(), Equals, "ErrorHooker")
  167. downloadPartHooker = defaultDownloadPartHook
  168. dcp = downloadCheckpoint{}
  169. err = dcp.load(newFile + ".cp")
  170. c.Assert(err, NotNil)
  171. // Resumable download with checkpoint dir
  172. os.Remove(newFile)
  173. downloadPartHooker = DownErrorHooker
  174. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, CheckpointDir(true, "./"))
  175. c.Assert(err, NotNil)
  176. c.Assert(err.Error(), Equals, "ErrorHooker")
  177. downloadPartHooker = defaultDownloadPartHook
  178. // Check
  179. dcp = downloadCheckpoint{}
  180. cpConf := cpConfig{IsEnable: true, DirPath: "./"}
  181. cpFilePath := getDownloadCpFilePath(&cpConf, s.bucket.BucketName, objectName, newFile)
  182. err = dcp.load(cpFilePath)
  183. c.Assert(err, IsNil)
  184. c.Assert(dcp.Magic, Equals, downloadCpMagic)
  185. c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ=="))
  186. c.Assert(dcp.FilePath, Equals, newFile)
  187. c.Assert(dcp.ObjStat.Size, Equals, int64(482048))
  188. c.Assert(len(dcp.ObjStat.LastModified) > 0, Equals, true)
  189. c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"")
  190. c.Assert(dcp.Object, Equals, objectName)
  191. c.Assert(len(dcp.Parts), Equals, 5)
  192. c.Assert(len(dcp.todoParts()), Equals, 1)
  193. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, CheckpointDir(true, "./"))
  194. c.Assert(err, IsNil)
  195. //download success, checkpoint file has been deleted
  196. err = dcp.load(cpFilePath)
  197. c.Assert(err, NotNil)
  198. eq, err = compareFiles(fileName, newFile)
  199. c.Assert(err, IsNil)
  200. c.Assert(eq, Equals, true)
  201. // Resumable download with checkpoint at a time. No error is expected in the download procedure.
  202. os.Remove(newFile)
  203. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  204. c.Assert(err, IsNil)
  205. err = dcp.load(newFile + ".cp")
  206. c.Assert(err, NotNil)
  207. eq, err = compareFiles(fileName, newFile)
  208. c.Assert(err, IsNil)
  209. c.Assert(eq, Equals, true)
  210. // Resumable download with checkpoint at a time. No error is expected in the download procedure.
  211. os.Remove(newFile)
  212. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10), Checkpoint(true, newFile+".cp"))
  213. c.Assert(err, IsNil)
  214. err = dcp.load(newFile + ".cp")
  215. c.Assert(err, NotNil)
  216. eq, err = compareFiles(fileName, newFile)
  217. c.Assert(err, IsNil)
  218. c.Assert(eq, Equals, true)
  219. err = s.bucket.DeleteObject(objectName)
  220. c.Assert(err, IsNil)
  221. }
  222. // TestDownloadOption options
  223. func (s *OssDownloadSuite) TestDownloadOption(c *C) {
  224. objectName := objectNamePrefix + "tdmo"
  225. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  226. newFile := "down-new-file-3.jpg"
  227. // Upload the file
  228. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  229. c.Assert(err, IsNil)
  230. meta, err := s.bucket.GetObjectDetailedMeta(objectName)
  231. c.Assert(err, IsNil)
  232. // IfMatch
  233. os.Remove(newFile)
  234. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag")))
  235. c.Assert(err, IsNil)
  236. eq, err := compareFiles(fileName, newFile)
  237. c.Assert(err, IsNil)
  238. c.Assert(eq, Equals, true)
  239. // IfNoneMatch
  240. os.Remove(newFile)
  241. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag")))
  242. c.Assert(err, NotNil)
  243. // IfMatch
  244. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag")))
  245. c.Assert(err, IsNil)
  246. eq, err = compareFiles(fileName, newFile)
  247. c.Assert(err, IsNil)
  248. c.Assert(eq, Equals, true)
  249. // IfNoneMatch
  250. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag")))
  251. c.Assert(err, NotNil)
  252. }
  253. // TestDownloadObjectChange tests the file is updated during the upload
  254. func (s *OssDownloadSuite) TestDownloadObjectChange(c *C) {
  255. objectName := objectNamePrefix + "tdloc"
  256. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  257. newFile := "down-new-file-4.jpg"
  258. // Upload a file
  259. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  260. c.Assert(err, IsNil)
  261. // Download with default checkpoint
  262. downloadPartHooker = DownErrorHooker
  263. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  264. c.Assert(err, NotNil)
  265. c.Assert(err.Error(), Equals, "ErrorHooker")
  266. downloadPartHooker = defaultDownloadPartHook
  267. err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  268. c.Assert(err, IsNil)
  269. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  270. c.Assert(err, IsNil)
  271. eq, err := compareFiles(fileName, newFile)
  272. c.Assert(err, IsNil)
  273. c.Assert(eq, Equals, true)
  274. }
  275. // TestDownloadNegative tests downloading negative
  276. func (s *OssDownloadSuite) TestDownloadNegative(c *C) {
  277. objectName := objectNamePrefix + "tdn"
  278. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  279. newFile := "down-new-file-3.jpg"
  280. // Upload a file
  281. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  282. c.Assert(err, IsNil)
  283. // Worker routine error
  284. downloadPartHooker = DownErrorHooker
  285. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2))
  286. c.Assert(err, NotNil)
  287. c.Assert(err.Error(), Equals, "ErrorHooker")
  288. downloadPartHooker = defaultDownloadPartHook
  289. // Local file does not exist
  290. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2))
  291. c.Assert(err, NotNil)
  292. // Invalid part size
  293. err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2))
  294. c.Assert(err, NotNil)
  295. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2))
  296. c.Assert(err, IsNil)
  297. err = s.bucket.DeleteObject(objectName)
  298. c.Assert(err, IsNil)
  299. // Local file does not exist
  300. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024)
  301. c.Assert(err, NotNil)
  302. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2))
  303. c.Assert(err, NotNil)
  304. // Invalid part size
  305. err = s.bucket.DownloadFile(objectName, newFile, -1)
  306. c.Assert(err, NotNil)
  307. err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2))
  308. c.Assert(err, NotNil)
  309. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100)
  310. c.Assert(err, NotNil)
  311. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2))
  312. c.Assert(err, NotNil)
  313. }
  314. // TestDownloadWithRange tests concurrent downloading with range specified and checkpoint enabled
  315. func (s *OssDownloadSuite) TestDownloadWithRange(c *C) {
  316. objectName := objectNamePrefix + "tdwr"
  317. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  318. newFile := "down-new-file-tdwr.jpg"
  319. newFileGet := "down-new-file-tdwr-2.jpg"
  320. // Upload a file
  321. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  322. c.Assert(err, IsNil)
  323. fileSize, err := getFileSize(fileName)
  324. c.Assert(err, IsNil)
  325. // Download with range, from 1024 to 4096
  326. os.Remove(newFile)
  327. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Range(1024, 4095))
  328. c.Assert(err, IsNil)
  329. // Check
  330. eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  331. c.Assert(err, IsNil)
  332. c.Assert(eq, Equals, true)
  333. os.Remove(newFileGet)
  334. err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095))
  335. c.Assert(err, IsNil)
  336. // Compare get and download
  337. eq, err = compareFiles(newFile, newFileGet)
  338. c.Assert(err, IsNil)
  339. c.Assert(eq, Equals, true)
  340. // Download with range, from 1024 to 4096
  341. os.Remove(newFile)
  342. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("1024-4095"))
  343. c.Assert(err, IsNil)
  344. // Check
  345. eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  346. c.Assert(err, IsNil)
  347. c.Assert(eq, Equals, true)
  348. os.Remove(newFileGet)
  349. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095"))
  350. c.Assert(err, IsNil)
  351. // Compare get and download
  352. eq, err = compareFiles(newFile, newFileGet)
  353. c.Assert(err, IsNil)
  354. c.Assert(eq, Equals, true)
  355. // Download with range, from 2048 to the end
  356. os.Remove(newFile)
  357. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), NormalizedRange("2048-"))
  358. c.Assert(err, IsNil)
  359. // Check
  360. eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
  361. c.Assert(err, IsNil)
  362. c.Assert(eq, Equals, true)
  363. os.Remove(newFileGet)
  364. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-"))
  365. c.Assert(err, IsNil)
  366. // Compare get and download
  367. eq, err = compareFiles(newFile, newFileGet)
  368. c.Assert(err, IsNil)
  369. c.Assert(eq, Equals, true)
  370. // Download with range, the last 4096
  371. os.Remove(newFile)
  372. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("-4096"))
  373. c.Assert(err, IsNil)
  374. // Check
  375. eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
  376. c.Assert(err, IsNil)
  377. c.Assert(eq, Equals, true)
  378. os.Remove(newFileGet)
  379. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096"))
  380. c.Assert(err, IsNil)
  381. // Compare get and download
  382. eq, err = compareFiles(newFile, newFileGet)
  383. c.Assert(err, IsNil)
  384. c.Assert(eq, Equals, true)
  385. err = s.bucket.DeleteObject(objectName)
  386. c.Assert(err, IsNil)
  387. }
  388. // TestDownloadWithCheckoutAndRange tests concurrent downloading with range specified and checkpoint enabled
  389. func (s *OssDownloadSuite) TestDownloadWithCheckoutAndRange(c *C) {
  390. objectName := objectNamePrefix + "tdwcr"
  391. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  392. newFile := "down-new-file-tdwcr.jpg"
  393. newFileGet := "down-new-file-tdwcr-2.jpg"
  394. // Upload a file
  395. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, fileName+".cp"))
  396. c.Assert(err, IsNil)
  397. fileSize, err := getFileSize(fileName)
  398. c.Assert(err, IsNil)
  399. // Download with range, from 1024 to 4096
  400. os.Remove(newFile)
  401. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Checkpoint(true, newFile+".cp"), Range(1024, 4095))
  402. c.Assert(err, IsNil)
  403. // Check
  404. eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  405. c.Assert(err, IsNil)
  406. c.Assert(eq, Equals, true)
  407. os.Remove(newFileGet)
  408. err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095))
  409. c.Assert(err, IsNil)
  410. // Compare get and download
  411. eq, err = compareFiles(newFile, newFileGet)
  412. c.Assert(err, IsNil)
  413. c.Assert(eq, Equals, true)
  414. // Download with range, from 1024 to 4096
  415. os.Remove(newFile)
  416. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("1024-4095"))
  417. c.Assert(err, IsNil)
  418. // Check
  419. eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  420. c.Assert(err, IsNil)
  421. c.Assert(eq, Equals, true)
  422. os.Remove(newFileGet)
  423. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095"))
  424. c.Assert(err, IsNil)
  425. // Compare get and download
  426. eq, err = compareFiles(newFile, newFileGet)
  427. c.Assert(err, IsNil)
  428. c.Assert(eq, Equals, true)
  429. // Download with range, from 2048 to the end
  430. os.Remove(newFile)
  431. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("2048-"))
  432. c.Assert(err, IsNil)
  433. // Check
  434. eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
  435. c.Assert(err, IsNil)
  436. c.Assert(eq, Equals, true)
  437. os.Remove(newFileGet)
  438. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-"))
  439. c.Assert(err, IsNil)
  440. // Compare get and download
  441. eq, err = compareFiles(newFile, newFileGet)
  442. c.Assert(err, IsNil)
  443. c.Assert(eq, Equals, true)
  444. // Download with range, the last 4096 bytes
  445. os.Remove(newFile)
  446. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("-4096"))
  447. c.Assert(err, IsNil)
  448. // Check
  449. eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
  450. c.Assert(err, IsNil)
  451. c.Assert(eq, Equals, true)
  452. os.Remove(newFileGet)
  453. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096"))
  454. c.Assert(err, IsNil)
  455. // Compare get and download
  456. eq, err = compareFiles(newFile, newFileGet)
  457. c.Assert(err, IsNil)
  458. c.Assert(eq, Equals, true)
  459. err = s.bucket.DeleteObject(objectName)
  460. c.Assert(err, IsNil)
  461. }
  462. // TestCombineCRCInDownloadParts tests combineCRCInParts
  463. func (s *OssDownloadSuite) TestCombineCRCInDownloadParts(c *C) {
  464. crc := combineCRCInParts(nil)
  465. c.Assert(crc == 0, Equals, true)
  466. crc = combineCRCInParts(make([]downloadPart, 0))
  467. c.Assert(crc == 0, Equals, true)
  468. parts := make([]downloadPart, 1)
  469. parts[0].CRC64 = 10278880121275185425
  470. crc = combineCRCInParts(parts)
  471. c.Assert(crc == 10278880121275185425, Equals, true)
  472. parts = make([]downloadPart, 2)
  473. parts[0].CRC64 = 6748440630437108969
  474. parts[0].Start = 0
  475. parts[0].End = 4
  476. parts[1].CRC64 = 10278880121275185425
  477. parts[1].Start = 5
  478. parts[1].End = 8
  479. crc = combineCRCInParts(parts)
  480. c.Assert(crc == 11051210869376104954, Equals, true)
  481. }
  482. func getFileSize(fileName string) (int64, error) {
  483. file, err := os.Open(fileName)
  484. if err != nil {
  485. return 0, err
  486. }
  487. defer file.Close()
  488. stat, err := file.Stat()
  489. if err != nil {
  490. return 0, err
  491. }
  492. return stat.Size(), nil
  493. }
  494. // compareFilesWithRange compares the content between fileL and fileR with specified range
  495. func compareFilesWithRange(fileL string, offsetL int64, fileR string, offsetR int64, size int64) (bool, error) {
  496. finL, err := os.Open(fileL)
  497. if err != nil {
  498. return false, err
  499. }
  500. defer finL.Close()
  501. finL.Seek(offsetL, os.SEEK_SET)
  502. finR, err := os.Open(fileR)
  503. if err != nil {
  504. return false, err
  505. }
  506. defer finR.Close()
  507. finR.Seek(offsetR, os.SEEK_SET)
  508. statL, err := finL.Stat()
  509. if err != nil {
  510. return false, err
  511. }
  512. statR, err := finR.Stat()
  513. if err != nil {
  514. return false, err
  515. }
  516. if (offsetL+size > statL.Size()) || (offsetR+size > statR.Size()) {
  517. return false, nil
  518. }
  519. part := statL.Size() - offsetL
  520. if part > 16*1024 {
  521. part = 16 * 1024
  522. }
  523. bufL := make([]byte, part)
  524. bufR := make([]byte, part)
  525. for readN := int64(0); readN < size; {
  526. n, _ := finL.Read(bufL)
  527. if 0 == n {
  528. break
  529. }
  530. n, _ = finR.Read(bufR)
  531. if 0 == n {
  532. break
  533. }
  534. tailer := part
  535. if tailer > size-readN {
  536. tailer = size - readN
  537. }
  538. readN += tailer
  539. if !bytes.Equal(bufL[0:tailer], bufR[0:tailer]) {
  540. return false, nil
  541. }
  542. }
  543. return true, nil
  544. }