download_test.go 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668
  1. package oss
  2. import (
  3. "bytes"
  4. "fmt"
  5. "os"
  6. "time"
  7. . "gopkg.in/check.v1"
  8. )
  9. type OssDownloadSuite struct {
  10. client *Client
  11. bucket *Bucket
  12. }
  13. var _ = Suite(&OssDownloadSuite{})
  14. // SetUpSuite runs once when the suite starts running
  15. func (s *OssDownloadSuite) SetUpSuite(c *C) {
  16. client, err := New(endpoint, accessID, accessKey)
  17. c.Assert(err, IsNil)
  18. s.client = client
  19. s.client.CreateBucket(bucketName)
  20. // time.Sleep(timeoutInOperation)
  21. bucket, err := s.client.Bucket(bucketName)
  22. c.Assert(err, IsNil)
  23. s.bucket = bucket
  24. testLogger.Println("test download started")
  25. }
  26. // TearDownSuite runs before each test or benchmark starts running
  27. func (s *OssDownloadSuite) TearDownSuite(c *C) {
  28. // Delete part
  29. lmur, err := s.bucket.ListMultipartUploads()
  30. c.Assert(err, IsNil)
  31. for _, upload := range lmur.Uploads {
  32. var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName,
  33. Key: upload.Key, UploadID: upload.UploadID}
  34. err = s.bucket.AbortMultipartUpload(imur)
  35. c.Assert(err, IsNil)
  36. }
  37. // Delete objects
  38. marker := Marker("")
  39. for{
  40. lor, err := s.bucket.ListObjects(marker)
  41. c.Assert(err, IsNil)
  42. for _, object := range lor.Objects {
  43. err = s.bucket.DeleteObject(object.Key)
  44. c.Assert(err, IsNil)
  45. }
  46. marker = Marker(lor.NextMarker)
  47. if !lor.IsTruncated {
  48. break
  49. }
  50. }
  51. // Delete bucket
  52. err = s.client.DeleteBucket(s.bucket.BucketName)
  53. c.Assert(err, IsNil)
  54. testLogger.Println("test download completed")
  55. }
  56. // SetUpTest runs after each test or benchmark runs
  57. func (s *OssDownloadSuite) SetUpTest(c *C) {
  58. err := removeTempFiles("../oss", ".jpg")
  59. c.Assert(err, IsNil)
  60. }
  61. // TearDownTest runs once after all tests or benchmarks have finished running
  62. func (s *OssDownloadSuite) TearDownTest(c *C) {
  63. err := removeTempFiles("../oss", ".jpg")
  64. c.Assert(err, IsNil)
  65. err = removeTempFiles("../oss", ".temp")
  66. c.Assert(err, IsNil)
  67. }
  68. // TestDownloadRoutineWithoutRecovery multipart downloads without checkpoint
  69. func (s *OssDownloadSuite) TestDownloadRoutineWithoutRecovery(c *C) {
  70. objectName := objectNamePrefix + "tdrwr"
  71. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  72. newFile := "down-new-file.jpg"
  73. // Upload a file
  74. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  75. c.Assert(err, IsNil)
  76. // Download the file by default
  77. err = s.bucket.DownloadFile(objectName, newFile, 100*1024)
  78. c.Assert(err, IsNil)
  79. // Check
  80. eq, err := compareFiles(fileName, newFile)
  81. c.Assert(err, IsNil)
  82. c.Assert(eq, Equals, true)
  83. // Use 2 coroutines to download the file and total parts count is 5
  84. os.Remove(newFile)
  85. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2))
  86. c.Assert(err, IsNil)
  87. // Check
  88. eq, err = compareFiles(fileName, newFile)
  89. c.Assert(err, IsNil)
  90. c.Assert(eq, Equals, true)
  91. // Use 5 coroutines to download the file and the total parts count is 5.
  92. os.Remove(newFile)
  93. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(5))
  94. c.Assert(err, IsNil)
  95. // Check
  96. eq, err = compareFiles(fileName, newFile)
  97. c.Assert(err, IsNil)
  98. c.Assert(eq, Equals, true)
  99. // Use 10 coroutines to download the file and the total parts count is 5.
  100. os.Remove(newFile)
  101. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10))
  102. c.Assert(err, IsNil)
  103. // Check
  104. eq, err = compareFiles(fileName, newFile)
  105. c.Assert(err, IsNil)
  106. c.Assert(eq, Equals, true)
  107. err = s.bucket.DeleteObject(objectName)
  108. c.Assert(err, IsNil)
  109. }
  110. // DownErrorHooker requests hook by downloadPart
  111. func DownErrorHooker(part downloadPart) error {
  112. if part.Index == 4 {
  113. time.Sleep(time.Second)
  114. return fmt.Errorf("ErrorHooker")
  115. }
  116. return nil
  117. }
  118. // TestDownloadRoutineWithRecovery multi-routine resumable download
  119. func (s *OssDownloadSuite) TestDownloadRoutineWithRecovery(c *C) {
  120. objectName := objectNamePrefix + "tdrtr"
  121. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  122. newFile := "down-new-file-2.jpg"
  123. // Upload a file
  124. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  125. c.Assert(err, IsNil)
  126. // Download a file with default checkpoint
  127. downloadPartHooker = DownErrorHooker
  128. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  129. c.Assert(err, NotNil)
  130. c.Assert(err.Error(), Equals, "ErrorHooker")
  131. downloadPartHooker = defaultDownloadPartHook
  132. // Check
  133. dcp := downloadCheckpoint{}
  134. err = dcp.load(newFile + ".cp")
  135. c.Assert(err, IsNil)
  136. c.Assert(dcp.Magic, Equals, downloadCpMagic)
  137. c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ=="))
  138. c.Assert(dcp.FilePath, Equals, newFile)
  139. c.Assert(dcp.ObjStat.Size, Equals, int64(482048))
  140. c.Assert(len(dcp.ObjStat.LastModified) > 0, Equals, true)
  141. c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"")
  142. c.Assert(dcp.Object, Equals, objectName)
  143. c.Assert(len(dcp.Parts), Equals, 5)
  144. c.Assert(len(dcp.todoParts()), Equals, 1)
  145. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  146. c.Assert(err, IsNil)
  147. //download success, checkpoint file has been deleted
  148. err = dcp.load(newFile + ".cp")
  149. c.Assert(err, NotNil)
  150. eq, err := compareFiles(fileName, newFile)
  151. c.Assert(err, IsNil)
  152. c.Assert(eq, Equals, true)
  153. // Resumable download with empty checkpoint file path
  154. downloadPartHooker = DownErrorHooker
  155. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, ""))
  156. c.Assert(err, NotNil)
  157. c.Assert(err.Error(), Equals, "ErrorHooker")
  158. downloadPartHooker = defaultDownloadPartHook
  159. dcp = downloadCheckpoint{}
  160. err = dcp.load(newFile + ".cp")
  161. c.Assert(err, NotNil)
  162. // Resumable download with checkpoint dir
  163. os.Remove(newFile)
  164. downloadPartHooker = DownErrorHooker
  165. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, CheckpointDir(true, "./"))
  166. c.Assert(err, NotNil)
  167. c.Assert(err.Error(), Equals, "ErrorHooker")
  168. downloadPartHooker = defaultDownloadPartHook
  169. // Check
  170. dcp = downloadCheckpoint{}
  171. cpConf := cpConfig{IsEnable: true, DirPath: "./"}
  172. cpFilePath := getDownloadCpFilePath(&cpConf, s.bucket.BucketName, objectName, newFile)
  173. err = dcp.load(cpFilePath)
  174. c.Assert(err, IsNil)
  175. c.Assert(dcp.Magic, Equals, downloadCpMagic)
  176. c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ=="))
  177. c.Assert(dcp.FilePath, Equals, newFile)
  178. c.Assert(dcp.ObjStat.Size, Equals, int64(482048))
  179. c.Assert(len(dcp.ObjStat.LastModified) > 0, Equals, true)
  180. c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"")
  181. c.Assert(dcp.Object, Equals, objectName)
  182. c.Assert(len(dcp.Parts), Equals, 5)
  183. c.Assert(len(dcp.todoParts()), Equals, 1)
  184. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, CheckpointDir(true, "./"))
  185. c.Assert(err, IsNil)
  186. //download success, checkpoint file has been deleted
  187. err = dcp.load(cpFilePath)
  188. c.Assert(err, NotNil)
  189. eq, err = compareFiles(fileName, newFile)
  190. c.Assert(err, IsNil)
  191. c.Assert(eq, Equals, true)
  192. // Resumable download with checkpoint at a time. No error is expected in the download procedure.
  193. os.Remove(newFile)
  194. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  195. c.Assert(err, IsNil)
  196. err = dcp.load(newFile + ".cp")
  197. c.Assert(err, NotNil)
  198. eq, err = compareFiles(fileName, newFile)
  199. c.Assert(err, IsNil)
  200. c.Assert(eq, Equals, true)
  201. // Resumable download with checkpoint at a time. No error is expected in the download procedure.
  202. os.Remove(newFile)
  203. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10), Checkpoint(true, newFile+".cp"))
  204. c.Assert(err, IsNil)
  205. err = dcp.load(newFile + ".cp")
  206. c.Assert(err, NotNil)
  207. eq, err = compareFiles(fileName, newFile)
  208. c.Assert(err, IsNil)
  209. c.Assert(eq, Equals, true)
  210. err = s.bucket.DeleteObject(objectName)
  211. c.Assert(err, IsNil)
  212. }
  213. // TestDownloadOption options
  214. func (s *OssDownloadSuite) TestDownloadOption(c *C) {
  215. objectName := objectNamePrefix + "tdmo"
  216. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  217. newFile := "down-new-file-3.jpg"
  218. // Upload the file
  219. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  220. c.Assert(err, IsNil)
  221. meta, err := s.bucket.GetObjectDetailedMeta(objectName)
  222. c.Assert(err, IsNil)
  223. // IfMatch
  224. os.Remove(newFile)
  225. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag")))
  226. c.Assert(err, IsNil)
  227. eq, err := compareFiles(fileName, newFile)
  228. c.Assert(err, IsNil)
  229. c.Assert(eq, Equals, true)
  230. // IfNoneMatch
  231. os.Remove(newFile)
  232. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag")))
  233. c.Assert(err, NotNil)
  234. // IfMatch
  235. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag")))
  236. c.Assert(err, IsNil)
  237. eq, err = compareFiles(fileName, newFile)
  238. c.Assert(err, IsNil)
  239. c.Assert(eq, Equals, true)
  240. // IfNoneMatch
  241. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag")))
  242. c.Assert(err, NotNil)
  243. }
  244. // TestDownloadObjectChange tests the file is updated during the upload
  245. func (s *OssDownloadSuite) TestDownloadObjectChange(c *C) {
  246. objectName := objectNamePrefix + "tdloc"
  247. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  248. newFile := "down-new-file-4.jpg"
  249. // Upload a file
  250. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  251. c.Assert(err, IsNil)
  252. // Download with default checkpoint
  253. downloadPartHooker = DownErrorHooker
  254. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  255. c.Assert(err, NotNil)
  256. c.Assert(err.Error(), Equals, "ErrorHooker")
  257. downloadPartHooker = defaultDownloadPartHook
  258. err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  259. c.Assert(err, IsNil)
  260. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  261. c.Assert(err, IsNil)
  262. eq, err := compareFiles(fileName, newFile)
  263. c.Assert(err, IsNil)
  264. c.Assert(eq, Equals, true)
  265. }
  266. // TestDownloadNegative tests downloading negative
  267. func (s *OssDownloadSuite) TestDownloadNegative(c *C) {
  268. objectName := objectNamePrefix + "tdn"
  269. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  270. newFile := "down-new-file-3.jpg"
  271. // Upload a file
  272. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  273. c.Assert(err, IsNil)
  274. // Worker routine error
  275. downloadPartHooker = DownErrorHooker
  276. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2))
  277. c.Assert(err, NotNil)
  278. c.Assert(err.Error(), Equals, "ErrorHooker")
  279. downloadPartHooker = defaultDownloadPartHook
  280. // Local file does not exist
  281. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2))
  282. c.Assert(err, NotNil)
  283. // Invalid part size
  284. err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2))
  285. c.Assert(err, NotNil)
  286. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2))
  287. c.Assert(err, IsNil)
  288. err = s.bucket.DeleteObject(objectName)
  289. c.Assert(err, IsNil)
  290. // Local file does not exist
  291. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024)
  292. c.Assert(err, NotNil)
  293. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2))
  294. c.Assert(err, NotNil)
  295. // Invalid part size
  296. err = s.bucket.DownloadFile(objectName, newFile, -1)
  297. c.Assert(err, NotNil)
  298. err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2))
  299. c.Assert(err, NotNil)
  300. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100)
  301. c.Assert(err, NotNil)
  302. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2))
  303. c.Assert(err, NotNil)
  304. }
  305. // TestDownloadWithRange tests concurrent downloading with range specified and checkpoint enabled
  306. func (s *OssDownloadSuite) TestDownloadWithRange(c *C) {
  307. objectName := objectNamePrefix + "tdwr"
  308. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  309. newFile := "down-new-file-tdwr.jpg"
  310. newFileGet := "down-new-file-tdwr-2.jpg"
  311. // Upload a file
  312. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  313. c.Assert(err, IsNil)
  314. fileSize, err := getFileSize(fileName)
  315. c.Assert(err, IsNil)
  316. // Download with range, from 1024 to 4096
  317. os.Remove(newFile)
  318. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Range(1024, 4095))
  319. c.Assert(err, IsNil)
  320. // Check
  321. eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  322. c.Assert(err, IsNil)
  323. c.Assert(eq, Equals, true)
  324. os.Remove(newFileGet)
  325. err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095))
  326. c.Assert(err, IsNil)
  327. // Compare get and download
  328. eq, err = compareFiles(newFile, newFileGet)
  329. c.Assert(err, IsNil)
  330. c.Assert(eq, Equals, true)
  331. // Download with range, from 1024 to 4096
  332. os.Remove(newFile)
  333. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("1024-4095"))
  334. c.Assert(err, IsNil)
  335. // Check
  336. eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  337. c.Assert(err, IsNil)
  338. c.Assert(eq, Equals, true)
  339. os.Remove(newFileGet)
  340. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095"))
  341. c.Assert(err, IsNil)
  342. // Compare get and download
  343. eq, err = compareFiles(newFile, newFileGet)
  344. c.Assert(err, IsNil)
  345. c.Assert(eq, Equals, true)
  346. // Download with range, from 2048 to the end
  347. os.Remove(newFile)
  348. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), NormalizedRange("2048-"))
  349. c.Assert(err, IsNil)
  350. // Check
  351. eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
  352. c.Assert(err, IsNil)
  353. c.Assert(eq, Equals, true)
  354. os.Remove(newFileGet)
  355. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-"))
  356. c.Assert(err, IsNil)
  357. // Compare get and download
  358. eq, err = compareFiles(newFile, newFileGet)
  359. c.Assert(err, IsNil)
  360. c.Assert(eq, Equals, true)
  361. // Download with range, the last 4096
  362. os.Remove(newFile)
  363. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("-4096"))
  364. c.Assert(err, IsNil)
  365. // Check
  366. eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
  367. c.Assert(err, IsNil)
  368. c.Assert(eq, Equals, true)
  369. os.Remove(newFileGet)
  370. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096"))
  371. c.Assert(err, IsNil)
  372. // Compare get and download
  373. eq, err = compareFiles(newFile, newFileGet)
  374. c.Assert(err, IsNil)
  375. c.Assert(eq, Equals, true)
  376. err = s.bucket.DeleteObject(objectName)
  377. c.Assert(err, IsNil)
  378. }
  379. // TestDownloadWithCheckoutAndRange tests concurrent downloading with range specified and checkpoint enabled
  380. func (s *OssDownloadSuite) TestDownloadWithCheckoutAndRange(c *C) {
  381. objectName := objectNamePrefix + "tdwcr"
  382. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  383. newFile := "down-new-file-tdwcr.jpg"
  384. newFileGet := "down-new-file-tdwcr-2.jpg"
  385. // Upload a file
  386. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, fileName+".cp"))
  387. c.Assert(err, IsNil)
  388. fileSize, err := getFileSize(fileName)
  389. c.Assert(err, IsNil)
  390. // Download with range, from 1024 to 4096
  391. os.Remove(newFile)
  392. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Checkpoint(true, newFile+".cp"), Range(1024, 4095))
  393. c.Assert(err, IsNil)
  394. // Check
  395. eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  396. c.Assert(err, IsNil)
  397. c.Assert(eq, Equals, true)
  398. os.Remove(newFileGet)
  399. err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095))
  400. c.Assert(err, IsNil)
  401. // Compare get and download
  402. eq, err = compareFiles(newFile, newFileGet)
  403. c.Assert(err, IsNil)
  404. c.Assert(eq, Equals, true)
  405. // Download with range, from 1024 to 4096
  406. os.Remove(newFile)
  407. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("1024-4095"))
  408. c.Assert(err, IsNil)
  409. // Check
  410. eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  411. c.Assert(err, IsNil)
  412. c.Assert(eq, Equals, true)
  413. os.Remove(newFileGet)
  414. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095"))
  415. c.Assert(err, IsNil)
  416. // Compare get and download
  417. eq, err = compareFiles(newFile, newFileGet)
  418. c.Assert(err, IsNil)
  419. c.Assert(eq, Equals, true)
  420. // Download with range, from 2048 to the end
  421. os.Remove(newFile)
  422. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("2048-"))
  423. c.Assert(err, IsNil)
  424. // Check
  425. eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
  426. c.Assert(err, IsNil)
  427. c.Assert(eq, Equals, true)
  428. os.Remove(newFileGet)
  429. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-"))
  430. c.Assert(err, IsNil)
  431. // Compare get and download
  432. eq, err = compareFiles(newFile, newFileGet)
  433. c.Assert(err, IsNil)
  434. c.Assert(eq, Equals, true)
  435. // Download with range, the last 4096 bytes
  436. os.Remove(newFile)
  437. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("-4096"))
  438. c.Assert(err, IsNil)
  439. // Check
  440. eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
  441. c.Assert(err, IsNil)
  442. c.Assert(eq, Equals, true)
  443. os.Remove(newFileGet)
  444. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096"))
  445. c.Assert(err, IsNil)
  446. // Compare get and download
  447. eq, err = compareFiles(newFile, newFileGet)
  448. c.Assert(err, IsNil)
  449. c.Assert(eq, Equals, true)
  450. err = s.bucket.DeleteObject(objectName)
  451. c.Assert(err, IsNil)
  452. }
  453. // TestCombineCRCInDownloadParts tests combineCRCInParts
  454. func (s *OssDownloadSuite) TestCombineCRCInDownloadParts(c *C) {
  455. crc := combineCRCInParts(nil)
  456. c.Assert(crc == 0, Equals, true)
  457. crc = combineCRCInParts(make([]downloadPart, 0))
  458. c.Assert(crc == 0, Equals, true)
  459. parts := make([]downloadPart, 1)
  460. parts[0].CRC64 = 10278880121275185425
  461. crc = combineCRCInParts(parts)
  462. c.Assert(crc == 10278880121275185425, Equals, true)
  463. parts = make([]downloadPart, 2)
  464. parts[0].CRC64 = 6748440630437108969
  465. parts[0].Start = 0
  466. parts[0].End = 4
  467. parts[1].CRC64 = 10278880121275185425
  468. parts[1].Start = 5
  469. parts[1].End = 8
  470. crc = combineCRCInParts(parts)
  471. c.Assert(crc == 11051210869376104954, Equals, true)
  472. }
  473. func getFileSize(fileName string) (int64, error) {
  474. file, err := os.Open(fileName)
  475. if err != nil {
  476. return 0, err
  477. }
  478. defer file.Close()
  479. stat, err := file.Stat()
  480. if err != nil {
  481. return 0, err
  482. }
  483. return stat.Size(), nil
  484. }
  485. // compareFilesWithRange compares the content between fileL and fileR with specified range
  486. func compareFilesWithRange(fileL string, offsetL int64, fileR string, offsetR int64, size int64) (bool, error) {
  487. finL, err := os.Open(fileL)
  488. if err != nil {
  489. return false, err
  490. }
  491. defer finL.Close()
  492. finL.Seek(offsetL, os.SEEK_SET)
  493. finR, err := os.Open(fileR)
  494. if err != nil {
  495. return false, err
  496. }
  497. defer finR.Close()
  498. finR.Seek(offsetR, os.SEEK_SET)
  499. statL, err := finL.Stat()
  500. if err != nil {
  501. return false, err
  502. }
  503. statR, err := finR.Stat()
  504. if err != nil {
  505. return false, err
  506. }
  507. if (offsetL+size > statL.Size()) || (offsetR+size > statR.Size()) {
  508. return false, nil
  509. }
  510. part := statL.Size() - offsetL
  511. if part > 16*1024 {
  512. part = 16 * 1024
  513. }
  514. bufL := make([]byte, part)
  515. bufR := make([]byte, part)
  516. for readN := int64(0); readN < size; {
  517. n, _ := finL.Read(bufL)
  518. if 0 == n {
  519. break
  520. }
  521. n, _ = finR.Read(bufR)
  522. if 0 == n {
  523. break
  524. }
  525. tailer := part
  526. if tailer > size-readN {
  527. tailer = size - readN
  528. }
  529. readN += tailer
  530. if !bytes.Equal(bufL[0:tailer], bufR[0:tailer]) {
  531. return false, nil
  532. }
  533. }
  534. return true, nil
  535. }