download_test.go 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675
  1. package oss
  2. import (
  3. "bytes"
  4. "fmt"
  5. "os"
  6. "time"
  7. . "gopkg.in/check.v1"
  8. )
  9. type OssDownloadSuite struct {
  10. client *Client
  11. bucket *Bucket
  12. }
  13. var _ = Suite(&OssDownloadSuite{})
  14. // SetUpSuite runs once when the suite starts running
  15. func (s *OssDownloadSuite) SetUpSuite(c *C) {
  16. client, err := New(endpoint, accessID, accessKey)
  17. c.Assert(err, IsNil)
  18. s.client = client
  19. s.client.CreateBucket(bucketName)
  20. bucket, err := s.client.Bucket(bucketName)
  21. c.Assert(err, IsNil)
  22. s.bucket = bucket
  23. testLogger.Println("test download started")
  24. }
  25. // TearDownSuite runs before each test or benchmark starts running
  26. func (s *OssDownloadSuite) TearDownSuite(c *C) {
  27. // Delete part
  28. keyMarker := KeyMarker("")
  29. uploadIDMarker := UploadIDMarker("")
  30. for {
  31. lmur, err := s.bucket.ListMultipartUploads(keyMarker, uploadIDMarker)
  32. c.Assert(err, IsNil)
  33. for _, upload := range lmur.Uploads {
  34. var imur = InitiateMultipartUploadResult{Bucket: s.bucket.BucketName,
  35. Key: upload.Key, UploadID: upload.UploadID}
  36. err = s.bucket.AbortMultipartUpload(imur)
  37. c.Assert(err, IsNil)
  38. }
  39. keyMarker = KeyMarker(lmur.NextKeyMarker)
  40. uploadIDMarker = UploadIDMarker(lmur.NextUploadIDMarker)
  41. if !lmur.IsTruncated {
  42. break
  43. }
  44. }
  45. // Delete objects
  46. marker := Marker("")
  47. for {
  48. lor, err := s.bucket.ListObjects(marker)
  49. c.Assert(err, IsNil)
  50. for _, object := range lor.Objects {
  51. err = s.bucket.DeleteObject(object.Key)
  52. c.Assert(err, IsNil)
  53. }
  54. marker = Marker(lor.NextMarker)
  55. if !lor.IsTruncated {
  56. break
  57. }
  58. }
  59. // Delete bucket
  60. err := s.client.DeleteBucket(s.bucket.BucketName)
  61. c.Assert(err, IsNil)
  62. testLogger.Println("test download completed")
  63. }
  64. // SetUpTest runs after each test or benchmark runs
  65. func (s *OssDownloadSuite) SetUpTest(c *C) {
  66. err := removeTempFiles("../oss", ".jpg")
  67. c.Assert(err, IsNil)
  68. }
  69. // TearDownTest runs once after all tests or benchmarks have finished running
  70. func (s *OssDownloadSuite) TearDownTest(c *C) {
  71. err := removeTempFiles("../oss", ".jpg")
  72. c.Assert(err, IsNil)
  73. err = removeTempFiles("../oss", ".temp")
  74. c.Assert(err, IsNil)
  75. }
  76. // TestDownloadRoutineWithoutRecovery multipart downloads without checkpoint
  77. func (s *OssDownloadSuite) TestDownloadRoutineWithoutRecovery(c *C) {
  78. objectName := objectNamePrefix + randStr(8)
  79. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  80. newFile := randStr(8) + ".jpg"
  81. // Upload a file
  82. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  83. c.Assert(err, IsNil)
  84. // Download the file by default
  85. err = s.bucket.DownloadFile(objectName, newFile, 100*1024)
  86. c.Assert(err, IsNil)
  87. // Check
  88. eq, err := compareFiles(fileName, newFile)
  89. c.Assert(err, IsNil)
  90. c.Assert(eq, Equals, true)
  91. // Use 2 coroutines to download the file and total parts count is 5
  92. os.Remove(newFile)
  93. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2))
  94. c.Assert(err, IsNil)
  95. // Check
  96. eq, err = compareFiles(fileName, newFile)
  97. c.Assert(err, IsNil)
  98. c.Assert(eq, Equals, true)
  99. // Use 5 coroutines to download the file and the total parts count is 5.
  100. os.Remove(newFile)
  101. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(5))
  102. c.Assert(err, IsNil)
  103. // Check
  104. eq, err = compareFiles(fileName, newFile)
  105. c.Assert(err, IsNil)
  106. c.Assert(eq, Equals, true)
  107. // Use 10 coroutines to download the file and the total parts count is 5.
  108. os.Remove(newFile)
  109. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10))
  110. c.Assert(err, IsNil)
  111. // Check
  112. eq, err = compareFiles(fileName, newFile)
  113. c.Assert(err, IsNil)
  114. c.Assert(eq, Equals, true)
  115. err = s.bucket.DeleteObject(objectName)
  116. c.Assert(err, IsNil)
  117. }
  118. // DownErrorHooker requests hook by downloadPart
  119. func DownErrorHooker(part downloadPart) error {
  120. if part.Index == 4 {
  121. time.Sleep(time.Second)
  122. return fmt.Errorf("ErrorHooker")
  123. }
  124. return nil
  125. }
  126. // TestDownloadRoutineWithRecovery multi-routine resumable download
  127. func (s *OssDownloadSuite) TestDownloadRoutineWithRecovery(c *C) {
  128. objectName := objectNamePrefix + randStr(8)
  129. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  130. newFile := randStr(8) + ".jpg"
  131. // Upload a file
  132. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  133. c.Assert(err, IsNil)
  134. // Download a file with default checkpoint
  135. downloadPartHooker = DownErrorHooker
  136. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  137. c.Assert(err, NotNil)
  138. c.Assert(err.Error(), Equals, "ErrorHooker")
  139. downloadPartHooker = defaultDownloadPartHook
  140. // Check
  141. dcp := downloadCheckpoint{}
  142. err = dcp.load(newFile + ".cp")
  143. c.Assert(err, IsNil)
  144. c.Assert(dcp.Magic, Equals, downloadCpMagic)
  145. c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ=="))
  146. c.Assert(dcp.FilePath, Equals, newFile)
  147. c.Assert(dcp.ObjStat.Size, Equals, int64(482048))
  148. c.Assert(len(dcp.ObjStat.LastModified) > 0, Equals, true)
  149. c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"")
  150. c.Assert(dcp.Object, Equals, objectName)
  151. c.Assert(len(dcp.Parts), Equals, 5)
  152. c.Assert(len(dcp.todoParts()), Equals, 1)
  153. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  154. c.Assert(err, IsNil)
  155. //download success, checkpoint file has been deleted
  156. err = dcp.load(newFile + ".cp")
  157. c.Assert(err, NotNil)
  158. eq, err := compareFiles(fileName, newFile)
  159. c.Assert(err, IsNil)
  160. c.Assert(eq, Equals, true)
  161. // Resumable download with empty checkpoint file path
  162. downloadPartHooker = DownErrorHooker
  163. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, ""))
  164. c.Assert(err, NotNil)
  165. c.Assert(err.Error(), Equals, "ErrorHooker")
  166. downloadPartHooker = defaultDownloadPartHook
  167. dcp = downloadCheckpoint{}
  168. err = dcp.load(newFile + ".cp")
  169. c.Assert(err, NotNil)
  170. // Resumable download with checkpoint dir
  171. os.Remove(newFile)
  172. downloadPartHooker = DownErrorHooker
  173. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, CheckpointDir(true, "./"))
  174. c.Assert(err, NotNil)
  175. c.Assert(err.Error(), Equals, "ErrorHooker")
  176. downloadPartHooker = defaultDownloadPartHook
  177. // Check
  178. dcp = downloadCheckpoint{}
  179. cpConf := cpConfig{IsEnable: true, DirPath: "./"}
  180. cpFilePath := getDownloadCpFilePath(&cpConf, s.bucket.BucketName, objectName, newFile)
  181. err = dcp.load(cpFilePath)
  182. c.Assert(err, IsNil)
  183. c.Assert(dcp.Magic, Equals, downloadCpMagic)
  184. c.Assert(len(dcp.MD5), Equals, len("LC34jZU5xK4hlxi3Qn3XGQ=="))
  185. c.Assert(dcp.FilePath, Equals, newFile)
  186. c.Assert(dcp.ObjStat.Size, Equals, int64(482048))
  187. c.Assert(len(dcp.ObjStat.LastModified) > 0, Equals, true)
  188. c.Assert(dcp.ObjStat.Etag, Equals, "\"2351E662233817A7AE974D8C5B0876DD-5\"")
  189. c.Assert(dcp.Object, Equals, objectName)
  190. c.Assert(len(dcp.Parts), Equals, 5)
  191. c.Assert(len(dcp.todoParts()), Equals, 1)
  192. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, CheckpointDir(true, "./"))
  193. c.Assert(err, IsNil)
  194. //download success, checkpoint file has been deleted
  195. err = dcp.load(cpFilePath)
  196. c.Assert(err, NotNil)
  197. eq, err = compareFiles(fileName, newFile)
  198. c.Assert(err, IsNil)
  199. c.Assert(eq, Equals, true)
  200. // Resumable download with checkpoint at a time. No error is expected in the download procedure.
  201. os.Remove(newFile)
  202. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  203. c.Assert(err, IsNil)
  204. err = dcp.load(newFile + ".cp")
  205. c.Assert(err, NotNil)
  206. eq, err = compareFiles(fileName, newFile)
  207. c.Assert(err, IsNil)
  208. c.Assert(eq, Equals, true)
  209. // Resumable download with checkpoint at a time. No error is expected in the download procedure.
  210. os.Remove(newFile)
  211. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(10), Checkpoint(true, newFile+".cp"))
  212. c.Assert(err, IsNil)
  213. err = dcp.load(newFile + ".cp")
  214. c.Assert(err, NotNil)
  215. eq, err = compareFiles(fileName, newFile)
  216. c.Assert(err, IsNil)
  217. c.Assert(eq, Equals, true)
  218. err = s.bucket.DeleteObject(objectName)
  219. c.Assert(err, IsNil)
  220. }
  221. // TestDownloadOption options
  222. func (s *OssDownloadSuite) TestDownloadOption(c *C) {
  223. objectName := objectNamePrefix + randStr(8)
  224. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  225. newFile := randStr(8) + ".jpg"
  226. // Upload the file
  227. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  228. c.Assert(err, IsNil)
  229. meta, err := s.bucket.GetObjectDetailedMeta(objectName)
  230. c.Assert(err, IsNil)
  231. // IfMatch
  232. os.Remove(newFile)
  233. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag")))
  234. c.Assert(err, IsNil)
  235. eq, err := compareFiles(fileName, newFile)
  236. c.Assert(err, IsNil)
  237. c.Assert(eq, Equals, true)
  238. // IfNoneMatch
  239. os.Remove(newFile)
  240. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag")))
  241. c.Assert(err, NotNil)
  242. // IfMatch
  243. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfMatch(meta.Get("Etag")))
  244. c.Assert(err, IsNil)
  245. eq, err = compareFiles(fileName, newFile)
  246. c.Assert(err, IsNil)
  247. c.Assert(eq, Equals, true)
  248. // IfNoneMatch
  249. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), IfNoneMatch(meta.Get("Etag")))
  250. c.Assert(err, NotNil)
  251. }
  252. // TestDownloadObjectChange tests the file is updated during the upload
  253. func (s *OssDownloadSuite) TestDownloadObjectChange(c *C) {
  254. objectName := objectNamePrefix + randStr(8)
  255. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  256. newFile := randStr(8) + ".jpg"
  257. // Upload a file
  258. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  259. c.Assert(err, IsNil)
  260. // Download with default checkpoint
  261. downloadPartHooker = DownErrorHooker
  262. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  263. c.Assert(err, NotNil)
  264. c.Assert(err.Error(), Equals, "ErrorHooker")
  265. downloadPartHooker = defaultDownloadPartHook
  266. err = s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  267. c.Assert(err, IsNil)
  268. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Checkpoint(true, newFile+".cp"))
  269. c.Assert(err, IsNil)
  270. eq, err := compareFiles(fileName, newFile)
  271. c.Assert(err, IsNil)
  272. c.Assert(eq, Equals, true)
  273. }
  274. // TestDownloadNegative tests downloading negative
  275. func (s *OssDownloadSuite) TestDownloadNegative(c *C) {
  276. objectName := objectNamePrefix + randStr(8)
  277. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  278. newFile := randStr(8) + ".jpg"
  279. // Upload a file
  280. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  281. c.Assert(err, IsNil)
  282. // Worker routine error
  283. downloadPartHooker = DownErrorHooker
  284. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(2))
  285. c.Assert(err, NotNil)
  286. c.Assert(err.Error(), Equals, "ErrorHooker")
  287. downloadPartHooker = defaultDownloadPartHook
  288. // Local file does not exist
  289. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2))
  290. c.Assert(err, NotNil)
  291. // Invalid part size
  292. err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2))
  293. c.Assert(err, NotNil)
  294. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2))
  295. c.Assert(err, IsNil)
  296. err = s.bucket.DeleteObject(objectName)
  297. c.Assert(err, IsNil)
  298. // Local file does not exist
  299. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024)
  300. c.Assert(err, NotNil)
  301. err = s.bucket.DownloadFile(objectName, "/tmp/", 100*1024, Routines(2))
  302. c.Assert(err, NotNil)
  303. // Invalid part size
  304. err = s.bucket.DownloadFile(objectName, newFile, -1)
  305. c.Assert(err, NotNil)
  306. err = s.bucket.DownloadFile(objectName, newFile, 0, Routines(2))
  307. c.Assert(err, NotNil)
  308. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100)
  309. c.Assert(err, NotNil)
  310. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024*1024*100, Routines(2))
  311. c.Assert(err, NotNil)
  312. }
  313. // TestDownloadWithRange tests concurrent downloading with range specified and checkpoint enabled
  314. func (s *OssDownloadSuite) TestDownloadWithRange(c *C) {
  315. objectName := objectNamePrefix + randStr(8)
  316. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  317. newFile := randStr(8) + ".jpg"
  318. newFileGet := randStr(8) + "-.jpg"
  319. // Upload a file
  320. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3))
  321. c.Assert(err, IsNil)
  322. fileSize, err := getFileSize(fileName)
  323. c.Assert(err, IsNil)
  324. // Download with range, from 1024 to 4096
  325. os.Remove(newFile)
  326. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Range(1024, 4095))
  327. c.Assert(err, IsNil)
  328. // Check
  329. eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  330. c.Assert(err, IsNil)
  331. c.Assert(eq, Equals, true)
  332. os.Remove(newFileGet)
  333. err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095))
  334. c.Assert(err, IsNil)
  335. // Compare get and download
  336. eq, err = compareFiles(newFile, newFileGet)
  337. c.Assert(err, IsNil)
  338. c.Assert(eq, Equals, true)
  339. // Download with range, from 1024 to 4096
  340. os.Remove(newFile)
  341. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("1024-4095"))
  342. c.Assert(err, IsNil)
  343. // Check
  344. eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  345. c.Assert(err, IsNil)
  346. c.Assert(eq, Equals, true)
  347. os.Remove(newFileGet)
  348. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095"))
  349. c.Assert(err, IsNil)
  350. // Compare get and download
  351. eq, err = compareFiles(newFile, newFileGet)
  352. c.Assert(err, IsNil)
  353. c.Assert(eq, Equals, true)
  354. // Download with range, from 2048 to the end
  355. os.Remove(newFile)
  356. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), NormalizedRange("2048-"))
  357. c.Assert(err, IsNil)
  358. // Check
  359. eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
  360. c.Assert(err, IsNil)
  361. c.Assert(eq, Equals, true)
  362. os.Remove(newFileGet)
  363. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-"))
  364. c.Assert(err, IsNil)
  365. // Compare get and download
  366. eq, err = compareFiles(newFile, newFileGet)
  367. c.Assert(err, IsNil)
  368. c.Assert(eq, Equals, true)
  369. // Download with range, the last 4096
  370. os.Remove(newFile)
  371. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), NormalizedRange("-4096"))
  372. c.Assert(err, IsNil)
  373. // Check
  374. eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
  375. c.Assert(err, IsNil)
  376. c.Assert(eq, Equals, true)
  377. os.Remove(newFileGet)
  378. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096"))
  379. c.Assert(err, IsNil)
  380. // Compare get and download
  381. eq, err = compareFiles(newFile, newFileGet)
  382. c.Assert(err, IsNil)
  383. c.Assert(eq, Equals, true)
  384. err = s.bucket.DeleteObject(objectName)
  385. c.Assert(err, IsNil)
  386. }
  387. // TestDownloadWithCheckoutAndRange tests concurrent downloading with range specified and checkpoint enabled
  388. func (s *OssDownloadSuite) TestDownloadWithCheckoutAndRange(c *C) {
  389. objectName := objectNamePrefix + randStr(8)
  390. fileName := "../sample/BingWallpaper-2015-11-07.jpg"
  391. newFile := randStr(8) + ".jpg"
  392. newFileGet := randStr(8) + "-get.jpg"
  393. // Upload a file
  394. err := s.bucket.UploadFile(objectName, fileName, 100*1024, Routines(3), Checkpoint(true, fileName+".cp"))
  395. c.Assert(err, IsNil)
  396. fileSize, err := getFileSize(fileName)
  397. c.Assert(err, IsNil)
  398. // Download with range, from 1024 to 4096
  399. os.Remove(newFile)
  400. err = s.bucket.DownloadFile(objectName, newFile, 100*1024, Routines(3), Checkpoint(true, newFile+".cp"), Range(1024, 4095))
  401. c.Assert(err, IsNil)
  402. // Check
  403. eq, err := compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  404. c.Assert(err, IsNil)
  405. c.Assert(eq, Equals, true)
  406. os.Remove(newFileGet)
  407. err = s.bucket.GetObjectToFile(objectName, newFileGet, Range(1024, 4095))
  408. c.Assert(err, IsNil)
  409. // Compare get and download
  410. eq, err = compareFiles(newFile, newFileGet)
  411. c.Assert(err, IsNil)
  412. c.Assert(eq, Equals, true)
  413. // Download with range, from 1024 to 4096
  414. os.Remove(newFile)
  415. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("1024-4095"))
  416. c.Assert(err, IsNil)
  417. // Check
  418. eq, err = compareFilesWithRange(fileName, 1024, newFile, 0, 3072)
  419. c.Assert(err, IsNil)
  420. c.Assert(eq, Equals, true)
  421. os.Remove(newFileGet)
  422. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("1024-4095"))
  423. c.Assert(err, IsNil)
  424. // Compare get and download
  425. eq, err = compareFiles(newFile, newFileGet)
  426. c.Assert(err, IsNil)
  427. c.Assert(eq, Equals, true)
  428. // Download with range, from 2048 to the end
  429. os.Remove(newFile)
  430. err = s.bucket.DownloadFile(objectName, newFile, 1024*1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("2048-"))
  431. c.Assert(err, IsNil)
  432. // Check
  433. eq, err = compareFilesWithRange(fileName, 2048, newFile, 0, fileSize-2048)
  434. c.Assert(err, IsNil)
  435. c.Assert(eq, Equals, true)
  436. os.Remove(newFileGet)
  437. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("2048-"))
  438. c.Assert(err, IsNil)
  439. // Compare get and download
  440. eq, err = compareFiles(newFile, newFileGet)
  441. c.Assert(err, IsNil)
  442. c.Assert(eq, Equals, true)
  443. // Download with range, the last 4096 bytes
  444. os.Remove(newFile)
  445. err = s.bucket.DownloadFile(objectName, newFile, 1024, Routines(3), Checkpoint(true, newFile+".cp"), NormalizedRange("-4096"))
  446. c.Assert(err, IsNil)
  447. // Check
  448. eq, err = compareFilesWithRange(fileName, fileSize-4096, newFile, 0, 4096)
  449. c.Assert(err, IsNil)
  450. c.Assert(eq, Equals, true)
  451. os.Remove(newFileGet)
  452. err = s.bucket.GetObjectToFile(objectName, newFileGet, NormalizedRange("-4096"))
  453. c.Assert(err, IsNil)
  454. // Compare get and download
  455. eq, err = compareFiles(newFile, newFileGet)
  456. c.Assert(err, IsNil)
  457. c.Assert(eq, Equals, true)
  458. err = s.bucket.DeleteObject(objectName)
  459. c.Assert(err, IsNil)
  460. }
  461. // TestCombineCRCInDownloadParts tests combineCRCInParts
  462. func (s *OssDownloadSuite) TestCombineCRCInDownloadParts(c *C) {
  463. crc := combineCRCInParts(nil)
  464. c.Assert(crc == 0, Equals, true)
  465. crc = combineCRCInParts(make([]downloadPart, 0))
  466. c.Assert(crc == 0, Equals, true)
  467. parts := make([]downloadPart, 1)
  468. parts[0].CRC64 = 10278880121275185425
  469. crc = combineCRCInParts(parts)
  470. c.Assert(crc == 10278880121275185425, Equals, true)
  471. parts = make([]downloadPart, 2)
  472. parts[0].CRC64 = 6748440630437108969
  473. parts[0].Start = 0
  474. parts[0].End = 4
  475. parts[1].CRC64 = 10278880121275185425
  476. parts[1].Start = 5
  477. parts[1].End = 8
  478. crc = combineCRCInParts(parts)
  479. c.Assert(crc == 11051210869376104954, Equals, true)
  480. }
  481. func getFileSize(fileName string) (int64, error) {
  482. file, err := os.Open(fileName)
  483. if err != nil {
  484. return 0, err
  485. }
  486. defer file.Close()
  487. stat, err := file.Stat()
  488. if err != nil {
  489. return 0, err
  490. }
  491. return stat.Size(), nil
  492. }
  493. // compareFilesWithRange compares the content between fileL and fileR with specified range
  494. func compareFilesWithRange(fileL string, offsetL int64, fileR string, offsetR int64, size int64) (bool, error) {
  495. finL, err := os.Open(fileL)
  496. if err != nil {
  497. return false, err
  498. }
  499. defer finL.Close()
  500. finL.Seek(offsetL, os.SEEK_SET)
  501. finR, err := os.Open(fileR)
  502. if err != nil {
  503. return false, err
  504. }
  505. defer finR.Close()
  506. finR.Seek(offsetR, os.SEEK_SET)
  507. statL, err := finL.Stat()
  508. if err != nil {
  509. return false, err
  510. }
  511. statR, err := finR.Stat()
  512. if err != nil {
  513. return false, err
  514. }
  515. if (offsetL+size > statL.Size()) || (offsetR+size > statR.Size()) {
  516. return false, nil
  517. }
  518. part := statL.Size() - offsetL
  519. if part > 16*1024 {
  520. part = 16 * 1024
  521. }
  522. bufL := make([]byte, part)
  523. bufR := make([]byte, part)
  524. for readN := int64(0); readN < size; {
  525. n, _ := finL.Read(bufL)
  526. if 0 == n {
  527. break
  528. }
  529. n, _ = finR.Read(bufR)
  530. if 0 == n {
  531. break
  532. }
  533. tailer := part
  534. if tailer > size-readN {
  535. tailer = size - readN
  536. }
  537. readN += tailer
  538. if !bytes.Equal(bufL[0:tailer], bufR[0:tailer]) {
  539. return false, nil
  540. }
  541. }
  542. return true, nil
  543. }