kvstore_test.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807
  1. // Copyright 2015 CoreOS, Inc.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package storage
  15. import (
  16. "crypto/rand"
  17. "encoding/binary"
  18. "math"
  19. "os"
  20. "reflect"
  21. "testing"
  22. "time"
  23. "github.com/coreos/etcd/pkg/testutil"
  24. "github.com/coreos/etcd/storage/backend"
  25. "github.com/coreos/etcd/storage/storagepb"
  26. )
  27. func TestStoreRev(t *testing.T) {
  28. s := newStore(tmpPath)
  29. defer os.Remove(tmpPath)
  30. for i := 0; i < 3; i++ {
  31. s.Put([]byte("foo"), []byte("bar"))
  32. if r := s.Rev(); r != int64(i+1) {
  33. t.Errorf("#%d: rev = %d, want %d", i, r, i+1)
  34. }
  35. }
  36. }
  37. func TestStorePut(t *testing.T) {
  38. tests := []struct {
  39. rev revision
  40. r indexGetResp
  41. wrev revision
  42. wkey []byte
  43. wkv storagepb.KeyValue
  44. wputrev revision
  45. }{
  46. {
  47. revision{1, 0},
  48. indexGetResp{revision{}, revision{}, 0, ErrRevisionNotFound},
  49. revision{1, 1},
  50. newTestKeyBytes(revision{2, 0}, false),
  51. storagepb.KeyValue{
  52. Key: []byte("foo"),
  53. Value: []byte("bar"),
  54. CreateRevision: 2,
  55. ModRevision: 2,
  56. Version: 1,
  57. },
  58. revision{2, 0},
  59. },
  60. {
  61. revision{1, 1},
  62. indexGetResp{revision{2, 0}, revision{2, 0}, 1, nil},
  63. revision{1, 2},
  64. newTestKeyBytes(revision{2, 1}, false),
  65. storagepb.KeyValue{
  66. Key: []byte("foo"),
  67. Value: []byte("bar"),
  68. CreateRevision: 2,
  69. ModRevision: 2,
  70. Version: 2,
  71. },
  72. revision{2, 1},
  73. },
  74. {
  75. revision{2, 0},
  76. indexGetResp{revision{2, 1}, revision{2, 0}, 2, nil},
  77. revision{2, 1},
  78. newTestKeyBytes(revision{3, 0}, false),
  79. storagepb.KeyValue{
  80. Key: []byte("foo"),
  81. Value: []byte("bar"),
  82. CreateRevision: 2,
  83. ModRevision: 3,
  84. Version: 3,
  85. },
  86. revision{3, 0},
  87. },
  88. }
  89. for i, tt := range tests {
  90. s, b, index := newFakeStore()
  91. s.currentRev = tt.rev
  92. s.tx = b.BatchTx()
  93. index.indexGetRespc <- tt.r
  94. s.put([]byte("foo"), []byte("bar"))
  95. data, err := tt.wkv.Marshal()
  96. if err != nil {
  97. t.Errorf("#%d: marshal err = %v, want nil", i, err)
  98. }
  99. wact := []testutil.Action{
  100. {"put", []interface{}{keyBucketName, tt.wkey, data}},
  101. }
  102. if g := b.tx.Action(); !reflect.DeepEqual(g, wact) {
  103. t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact)
  104. }
  105. wact = []testutil.Action{
  106. {"get", []interface{}{[]byte("foo"), tt.wputrev.main}},
  107. {"put", []interface{}{[]byte("foo"), tt.wputrev}},
  108. }
  109. if g := index.Action(); !reflect.DeepEqual(g, wact) {
  110. t.Errorf("#%d: index action = %+v, want %+v", i, g, wact)
  111. }
  112. if s.currentRev != tt.wrev {
  113. t.Errorf("#%d: rev = %+v, want %+v", i, s.currentRev, tt.wrev)
  114. }
  115. }
  116. }
  117. func TestStoreRange(t *testing.T) {
  118. key := newTestKeyBytes(revision{2, 0}, false)
  119. kv := storagepb.KeyValue{
  120. Key: []byte("foo"),
  121. Value: []byte("bar"),
  122. CreateRevision: 1,
  123. ModRevision: 2,
  124. Version: 1,
  125. }
  126. kvb, err := kv.Marshal()
  127. if err != nil {
  128. t.Fatal(err)
  129. }
  130. currev := revision{1, 1}
  131. wrev := int64(2)
  132. tests := []struct {
  133. idxr indexRangeResp
  134. r rangeResp
  135. }{
  136. {
  137. indexRangeResp{[][]byte{[]byte("foo")}, []revision{{2, 0}}},
  138. rangeResp{[][]byte{key}, [][]byte{kvb}},
  139. },
  140. {
  141. indexRangeResp{[][]byte{[]byte("foo"), []byte("foo1")}, []revision{{2, 0}, {3, 0}}},
  142. rangeResp{[][]byte{key}, [][]byte{kvb}},
  143. },
  144. }
  145. for i, tt := range tests {
  146. s, b, index := newFakeStore()
  147. s.currentRev = currev
  148. s.tx = b.BatchTx()
  149. b.tx.rangeRespc <- tt.r
  150. index.indexRangeRespc <- tt.idxr
  151. kvs, rev, err := s.rangeKeys([]byte("foo"), []byte("goo"), 1, 0)
  152. if err != nil {
  153. t.Errorf("#%d: err = %v, want nil", i, err)
  154. }
  155. if w := []storagepb.KeyValue{kv}; !reflect.DeepEqual(kvs, w) {
  156. t.Errorf("#%d: kvs = %+v, want %+v", i, kvs, w)
  157. }
  158. if rev != wrev {
  159. t.Errorf("#%d: rev = %d, want %d", i, rev, wrev)
  160. }
  161. wstart, wend := revBytesRange(tt.idxr.revs[0])
  162. wact := []testutil.Action{
  163. {"range", []interface{}{keyBucketName, wstart, wend, int64(0)}},
  164. }
  165. if g := b.tx.Action(); !reflect.DeepEqual(g, wact) {
  166. t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact)
  167. }
  168. wact = []testutil.Action{
  169. {"range", []interface{}{[]byte("foo"), []byte("goo"), wrev}},
  170. }
  171. if g := index.Action(); !reflect.DeepEqual(g, wact) {
  172. t.Errorf("#%d: index action = %+v, want %+v", i, g, wact)
  173. }
  174. if s.currentRev != currev {
  175. t.Errorf("#%d: current rev = %+v, want %+v", i, s.currentRev, currev)
  176. }
  177. }
  178. }
  179. func TestStoreDeleteRange(t *testing.T) {
  180. tests := []struct {
  181. rev revision
  182. r indexRangeResp
  183. wkey []byte
  184. wrev revision
  185. wrrev int64
  186. wdelrev revision
  187. }{
  188. {
  189. revision{2, 0},
  190. indexRangeResp{[][]byte{[]byte("foo")}, []revision{{2, 0}}},
  191. newTestKeyBytes(revision{3, 0}, true),
  192. revision{2, 1},
  193. 2,
  194. revision{3, 0},
  195. },
  196. {
  197. revision{2, 1},
  198. indexRangeResp{[][]byte{[]byte("foo")}, []revision{{2, 0}}},
  199. newTestKeyBytes(revision{3, 1}, true),
  200. revision{2, 2},
  201. 3,
  202. revision{3, 1},
  203. },
  204. }
  205. for i, tt := range tests {
  206. s, b, index := newFakeStore()
  207. s.currentRev = tt.rev
  208. s.tx = b.BatchTx()
  209. index.indexRangeRespc <- tt.r
  210. n := s.deleteRange([]byte("foo"), []byte("goo"))
  211. if n != 1 {
  212. t.Errorf("#%d: n = %d, want 1", i, n)
  213. }
  214. data, err := (&storagepb.KeyValue{
  215. Key: []byte("foo"),
  216. }).Marshal()
  217. if err != nil {
  218. t.Errorf("#%d: marshal err = %v, want nil", i, err)
  219. }
  220. wact := []testutil.Action{
  221. {"put", []interface{}{keyBucketName, tt.wkey, data}},
  222. }
  223. if g := b.tx.Action(); !reflect.DeepEqual(g, wact) {
  224. t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact)
  225. }
  226. wact = []testutil.Action{
  227. {"range", []interface{}{[]byte("foo"), []byte("goo"), tt.wrrev}},
  228. {"tombstone", []interface{}{[]byte("foo"), tt.wdelrev}},
  229. }
  230. if g := index.Action(); !reflect.DeepEqual(g, wact) {
  231. t.Errorf("#%d: index action = %+v, want %+v", i, g, wact)
  232. }
  233. if s.currentRev != tt.wrev {
  234. t.Errorf("#%d: rev = %+v, want %+v", i, s.currentRev, tt.wrev)
  235. }
  236. }
  237. }
  238. func TestStoreRangeHistory(t *testing.T) {
  239. key := newTestKeyBytes(revision{2, 0}, false)
  240. kv := storagepb.KeyValue{
  241. Key: []byte("foo"),
  242. Value: []byte("bar"),
  243. CreateRevision: 1,
  244. ModRevision: 2,
  245. Version: 1,
  246. }
  247. kvb, err := kv.Marshal()
  248. if err != nil {
  249. t.Fatal(err)
  250. }
  251. currev := revision{2, 0}
  252. tests := []struct {
  253. idxr indexRangeEventsResp
  254. r rangeResp
  255. }{
  256. {
  257. indexRangeEventsResp{[]revision{{2, 0}}},
  258. rangeResp{[][]byte{key}, [][]byte{kvb}},
  259. },
  260. {
  261. indexRangeEventsResp{[]revision{{2, 0}, {3, 0}}},
  262. rangeResp{[][]byte{key}, [][]byte{kvb}},
  263. },
  264. }
  265. for i, tt := range tests {
  266. s, b, index := newFakeStore()
  267. s.currentRev = currev
  268. index.indexRangeEventsRespc <- tt.idxr
  269. b.tx.rangeRespc <- tt.r
  270. keys, kvs, _, err := s.RangeHistory([]byte("foo"), []byte("goo"), 1, 1)
  271. if err != nil {
  272. t.Errorf("#%d: err = %v, want nil", i, err)
  273. }
  274. if w := [][]byte{key}; !reflect.DeepEqual(keys, w) {
  275. t.Errorf("#%d: keys = %+v, want %+v", i, keys, w)
  276. }
  277. if w := []storagepb.KeyValue{kv}; !reflect.DeepEqual(kvs, w) {
  278. t.Errorf("#%d: kvs = %+v, want %+v", i, kvs, w)
  279. }
  280. wact := []testutil.Action{
  281. {"rangeEvents", []interface{}{[]byte("foo"), []byte("goo"), int64(1)}},
  282. }
  283. if g := index.Action(); !reflect.DeepEqual(g, wact) {
  284. t.Errorf("#%d: index action = %+v, want %+v", i, g, wact)
  285. }
  286. wstart, wend := revBytesRange(tt.idxr.revs[0])
  287. wact = []testutil.Action{
  288. {"range", []interface{}{keyBucketName, wstart, wend, int64(0)}},
  289. }
  290. if g := b.tx.Action(); !reflect.DeepEqual(g, wact) {
  291. t.Errorf("#%d: tx action = %+v, want %+v", i, g, wact)
  292. }
  293. if s.currentRev != currev {
  294. t.Errorf("#%d: current rev = %+v, want %+v", i, s.currentRev, currev)
  295. }
  296. }
  297. }
  298. func TestStoreCompact(t *testing.T) {
  299. s, b, index := newFakeStore()
  300. s.currentRev = revision{3, 0}
  301. index.indexCompactRespc <- map[revision]struct{}{revision{1, 0}: {}}
  302. key1 := newTestKeyBytes(revision{1, 0}, false)
  303. key2 := newTestKeyBytes(revision{2, 0}, false)
  304. b.tx.rangeRespc <- rangeResp{[][]byte{key1, key2}, nil}
  305. s.Compact(3)
  306. s.wg.Wait()
  307. if s.compactMainRev != 3 {
  308. t.Errorf("compact main rev = %d, want 3", s.compactMainRev)
  309. }
  310. end := make([]byte, 8)
  311. binary.BigEndian.PutUint64(end, uint64(4))
  312. wact := []testutil.Action{
  313. {"put", []interface{}{metaBucketName, scheduledCompactKeyName, newTestRevBytes(revision{3, 0})}},
  314. {"range", []interface{}{keyBucketName, make([]byte, 17), end, int64(10000)}},
  315. {"delete", []interface{}{keyBucketName, key2}},
  316. {"put", []interface{}{metaBucketName, finishedCompactKeyName, newTestRevBytes(revision{3, 0})}},
  317. }
  318. if g := b.tx.Action(); !reflect.DeepEqual(g, wact) {
  319. t.Errorf("tx actions = %+v, want %+v", g, wact)
  320. }
  321. wact = []testutil.Action{
  322. {"compact", []interface{}{int64(3)}},
  323. }
  324. if g := index.Action(); !reflect.DeepEqual(g, wact) {
  325. t.Errorf("index action = %+v, want %+v", g, wact)
  326. }
  327. }
  328. func TestStoreRestore(t *testing.T) {
  329. s, b, index := newFakeStore()
  330. putkey := newTestKeyBytes(revision{3, 0}, false)
  331. putkv := storagepb.KeyValue{
  332. Key: []byte("foo"),
  333. Value: []byte("bar"),
  334. CreateRevision: 3,
  335. ModRevision: 3,
  336. Version: 1,
  337. }
  338. putkvb, err := putkv.Marshal()
  339. if err != nil {
  340. t.Fatal(err)
  341. }
  342. delkey := newTestKeyBytes(revision{4, 0}, true)
  343. delkv := storagepb.KeyValue{
  344. Key: []byte("foo"),
  345. }
  346. delkvb, err := delkv.Marshal()
  347. if err != nil {
  348. t.Fatal(err)
  349. }
  350. b.tx.rangeRespc <- rangeResp{[][]byte{finishedCompactKeyName}, [][]byte{newTestRevBytes(revision{2, 0})}}
  351. b.tx.rangeRespc <- rangeResp{[][]byte{putkey, delkey}, [][]byte{putkvb, delkvb}}
  352. b.tx.rangeRespc <- rangeResp{[][]byte{scheduledCompactKeyName}, [][]byte{newTestRevBytes(revision{2, 0})}}
  353. s.Restore()
  354. if s.compactMainRev != 2 {
  355. t.Errorf("compact rev = %d, want 4", s.compactMainRev)
  356. }
  357. wrev := revision{4, 0}
  358. if !reflect.DeepEqual(s.currentRev, wrev) {
  359. t.Errorf("current rev = %v, want %v", s.currentRev, wrev)
  360. }
  361. wact := []testutil.Action{
  362. {"range", []interface{}{metaBucketName, finishedCompactKeyName, []byte(nil), int64(0)}},
  363. {"range", []interface{}{keyBucketName, newTestRevBytes(revision{}), newTestRevBytes(revision{math.MaxInt64, math.MaxInt64}), int64(0)}},
  364. {"range", []interface{}{metaBucketName, scheduledCompactKeyName, []byte(nil), int64(0)}},
  365. }
  366. if g := b.tx.Action(); !reflect.DeepEqual(g, wact) {
  367. t.Errorf("tx actions = %+v, want %+v", g, wact)
  368. }
  369. wact = []testutil.Action{
  370. {"restore", []interface{}{[]byte("foo"), revision{3, 0}, revision{3, 0}, int64(1)}},
  371. {"tombstone", []interface{}{[]byte("foo"), revision{4, 0}}},
  372. }
  373. if g := index.Action(); !reflect.DeepEqual(g, wact) {
  374. t.Errorf("index action = %+v, want %+v", g, wact)
  375. }
  376. }
  377. // tests end parameter works well
  378. func TestStoreRangeHistoryEnd(t *testing.T) {
  379. s := newStore(tmpPath)
  380. defer cleanup(s, tmpPath)
  381. s.Put([]byte("foo"), []byte("bar"))
  382. s.Put([]byte("foo1"), []byte("bar1"))
  383. s.Put([]byte("foo2"), []byte("bar2"))
  384. keys := [][]byte{
  385. newTestKeyBytes(revision{1, 0}, false),
  386. newTestKeyBytes(revision{2, 0}, false),
  387. newTestKeyBytes(revision{3, 0}, false),
  388. }
  389. kvs := []storagepb.KeyValue{
  390. {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 1, Version: 1},
  391. {Key: []byte("foo1"), Value: []byte("bar1"), CreateRevision: 2, ModRevision: 2, Version: 1},
  392. {Key: []byte("foo2"), Value: []byte("bar2"), CreateRevision: 3, ModRevision: 3, Version: 1},
  393. }
  394. tests := []struct {
  395. key, end []byte
  396. wkeys [][]byte
  397. wkvs []storagepb.KeyValue
  398. }{
  399. // get no keys
  400. {
  401. []byte("doo"), []byte("foo"),
  402. nil, nil,
  403. },
  404. // get no keys when key == end
  405. {
  406. []byte("foo"), []byte("foo"),
  407. nil, nil,
  408. },
  409. // get no keys when ranging single key
  410. {
  411. []byte("doo"), nil,
  412. nil, nil,
  413. },
  414. // get all keys
  415. {
  416. []byte("foo"), []byte("foo3"),
  417. keys, kvs,
  418. },
  419. // get partial keys
  420. {
  421. []byte("foo"), []byte("foo1"),
  422. keys[:1], kvs[:1],
  423. },
  424. // get single key
  425. {
  426. []byte("foo"), nil,
  427. keys[:1], kvs[:1],
  428. },
  429. }
  430. for i, tt := range tests {
  431. keys, kvs, rev, err := s.RangeHistory(tt.key, tt.end, 0, 1)
  432. if err != nil {
  433. t.Fatal(err)
  434. }
  435. if rev != 4 {
  436. t.Errorf("#%d: rev = %d, want %d", i, rev, 4)
  437. }
  438. if !reflect.DeepEqual(keys, tt.wkeys) {
  439. t.Errorf("#%d: actions = %+v, want %+v", i, keys, tt.wkeys)
  440. }
  441. if !reflect.DeepEqual(kvs, tt.wkvs) {
  442. t.Errorf("#%d: kvs = %+v, want %+v", i, kvs, tt.wkvs)
  443. }
  444. }
  445. }
  446. func TestStoreRangeHistoryRev(t *testing.T) {
  447. s := newStore(tmpPath)
  448. defer cleanup(s, tmpPath)
  449. s.Put([]byte("foo"), []byte("bar"))
  450. s.DeleteRange([]byte("foo"), nil)
  451. s.Put([]byte("foo"), []byte("bar"))
  452. s.Put([]byte("unrelated"), []byte("unrelated"))
  453. keys := [][]byte{
  454. newTestKeyBytes(revision{1, 0}, false),
  455. newTestKeyBytes(revision{2, 0}, true),
  456. newTestKeyBytes(revision{3, 0}, false),
  457. }
  458. kvs := []storagepb.KeyValue{
  459. {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 1, Version: 1},
  460. {Key: []byte("foo")},
  461. {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 3, ModRevision: 3, Version: 1},
  462. }
  463. tests := []struct {
  464. start int64
  465. wkeys [][]byte
  466. wkvs []storagepb.KeyValue
  467. wnext int64
  468. }{
  469. {0, keys, kvs, 5},
  470. {1, keys, kvs, 5},
  471. {3, keys[2:], kvs[2:], 5},
  472. }
  473. for i, tt := range tests {
  474. keys, kvs, next, err := s.RangeHistory([]byte("foo"), nil, 0, tt.start)
  475. if err != nil {
  476. t.Fatal(err)
  477. }
  478. if !reflect.DeepEqual(keys, tt.wkeys) {
  479. t.Errorf("#%d: acts = %+v, want %+v", i, keys, tt.wkeys)
  480. }
  481. if !reflect.DeepEqual(kvs, tt.wkvs) {
  482. t.Errorf("#%d: kvs = %+v, want %+v", i, kvs, tt.wkvs)
  483. }
  484. if next != tt.wnext {
  485. t.Errorf("#%d: next = %d, want %d", i, next, tt.wnext)
  486. }
  487. }
  488. }
  489. func TestStoreRangeHistoryBad(t *testing.T) {
  490. s := newStore(tmpPath)
  491. defer cleanup(s, tmpPath)
  492. s.Put([]byte("foo"), []byte("bar"))
  493. s.Put([]byte("foo"), []byte("bar1"))
  494. s.Put([]byte("foo"), []byte("bar2"))
  495. if err := s.Compact(3); err != nil {
  496. t.Fatalf("compact error (%v)", err)
  497. }
  498. tests := []struct {
  499. rev int64
  500. werr error
  501. }{
  502. {1, ErrCompacted},
  503. {2, ErrCompacted},
  504. {3, ErrCompacted},
  505. {4, ErrFutureRev},
  506. {10, ErrFutureRev},
  507. }
  508. for i, tt := range tests {
  509. _, _, _, err := s.RangeHistory([]byte("foo"), nil, 0, tt.rev)
  510. if err != tt.werr {
  511. t.Errorf("#%d: error = %v, want %v", i, err, tt.werr)
  512. }
  513. }
  514. }
  515. func TestStoreRangeHistoryLimit(t *testing.T) {
  516. s := newStore(tmpPath)
  517. defer cleanup(s, tmpPath)
  518. s.Put([]byte("foo"), []byte("bar"))
  519. s.DeleteRange([]byte("foo"), nil)
  520. s.Put([]byte("foo"), []byte("bar"))
  521. keys := [][]byte{
  522. newTestKeyBytes(revision{1, 0}, false),
  523. newTestKeyBytes(revision{2, 0}, true),
  524. newTestKeyBytes(revision{3, 0}, false),
  525. }
  526. kvs := []storagepb.KeyValue{
  527. {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 1, ModRevision: 1, Version: 1},
  528. {Key: []byte("foo")},
  529. {Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 3, ModRevision: 3, Version: 1},
  530. }
  531. tests := []struct {
  532. limit int64
  533. wkeys [][]byte
  534. wkvs []storagepb.KeyValue
  535. }{
  536. // no limit
  537. {-1, keys, kvs},
  538. // no limit
  539. {0, keys, kvs},
  540. {1, keys[:1], kvs[:1]},
  541. {2, keys[:2], kvs[:2]},
  542. {3, keys, kvs},
  543. {100, keys, kvs},
  544. }
  545. for i, tt := range tests {
  546. keys, kvs, _, err := s.RangeHistory([]byte("foo"), nil, tt.limit, 1)
  547. if err != nil {
  548. t.Fatalf("#%d: range error (%v)", i, err)
  549. }
  550. if !reflect.DeepEqual(keys, tt.wkeys) {
  551. t.Errorf("#%d: acts = %+v, want %+v", i, keys, tt.wkeys)
  552. }
  553. if !reflect.DeepEqual(kvs, tt.wkvs) {
  554. t.Errorf("#%d: kvs = %+v, want %+v", i, kvs, tt.wkvs)
  555. }
  556. }
  557. }
  558. func TestRestoreContinueUnfinishedCompaction(t *testing.T) {
  559. s0 := newStore(tmpPath)
  560. defer os.Remove(tmpPath)
  561. s0.Put([]byte("foo"), []byte("bar"))
  562. s0.Put([]byte("foo"), []byte("bar1"))
  563. s0.Put([]byte("foo"), []byte("bar2"))
  564. // write scheduled compaction, but not do compaction
  565. rbytes := newRevBytes()
  566. revToBytes(revision{main: 2}, rbytes)
  567. tx := s0.b.BatchTx()
  568. tx.Lock()
  569. tx.UnsafePut(metaBucketName, scheduledCompactKeyName, rbytes)
  570. tx.Unlock()
  571. s0.Close()
  572. s1 := newStore(tmpPath)
  573. s1.Restore()
  574. // wait for scheduled compaction to be finished
  575. time.Sleep(100 * time.Millisecond)
  576. if _, _, err := s1.Range([]byte("foo"), nil, 0, 2); err != ErrCompacted {
  577. t.Errorf("range on compacted rev error = %v, want %v", err, ErrCompacted)
  578. }
  579. // check the key in backend is deleted
  580. revbytes := newRevBytes()
  581. // TODO: compact should delete main=2 key too
  582. revToBytes(revision{main: 1}, revbytes)
  583. tx = s1.b.BatchTx()
  584. tx.Lock()
  585. ks, _ := tx.UnsafeRange(keyBucketName, revbytes, nil, 0)
  586. if len(ks) != 0 {
  587. t.Errorf("key for rev %+v still exists, want deleted", bytesToRev(revbytes))
  588. }
  589. tx.Unlock()
  590. }
  591. func TestTxnBlockBackendForceCommit(t *testing.T) {
  592. s := newStore(tmpPath)
  593. defer os.Remove(tmpPath)
  594. id := s.TxnBegin()
  595. done := make(chan struct{})
  596. go func() {
  597. s.b.ForceCommit()
  598. done <- struct{}{}
  599. }()
  600. select {
  601. case <-done:
  602. t.Fatalf("failed to block ForceCommit")
  603. case <-time.After(100 * time.Millisecond):
  604. }
  605. s.TxnEnd(id)
  606. select {
  607. case <-done:
  608. case <-time.After(time.Second):
  609. t.Fatalf("failed to execute ForceCommit")
  610. }
  611. }
  612. func BenchmarkStorePut(b *testing.B) {
  613. s := newStore(tmpPath)
  614. defer os.Remove(tmpPath)
  615. // prepare keys
  616. keys := make([][]byte, b.N)
  617. for i := 0; i < b.N; i++ {
  618. keys[i] = make([]byte, 64)
  619. rand.Read(keys[i])
  620. }
  621. b.ResetTimer()
  622. for i := 0; i < b.N; i++ {
  623. s.Put(keys[i], []byte("foo"))
  624. }
  625. }
  626. func newTestRevBytes(rev revision) []byte {
  627. bytes := newRevBytes()
  628. revToBytes(rev, bytes)
  629. return bytes
  630. }
  631. func newTestKeyBytes(rev revision, tombstone bool) []byte {
  632. bytes := newRevBytes()
  633. revToBytes(rev, bytes)
  634. if tombstone {
  635. bytes = appendMarkTombstone(bytes)
  636. }
  637. return bytes
  638. }
  639. func newFakeStore() (*store, *fakeBackend, *fakeIndex) {
  640. b := &fakeBackend{&fakeBatchTx{rangeRespc: make(chan rangeResp, 5)}}
  641. index := &fakeIndex{
  642. indexGetRespc: make(chan indexGetResp, 1),
  643. indexRangeRespc: make(chan indexRangeResp, 1),
  644. indexRangeEventsRespc: make(chan indexRangeEventsResp, 1),
  645. indexCompactRespc: make(chan map[revision]struct{}, 1),
  646. }
  647. return &store{
  648. b: b,
  649. kvindex: index,
  650. currentRev: revision{},
  651. compactMainRev: -1,
  652. }, b, index
  653. }
  654. type rangeResp struct {
  655. keys [][]byte
  656. vals [][]byte
  657. }
  658. type fakeBatchTx struct {
  659. testutil.Recorder
  660. rangeRespc chan rangeResp
  661. }
  662. func (b *fakeBatchTx) Lock() {}
  663. func (b *fakeBatchTx) Unlock() {}
  664. func (b *fakeBatchTx) UnsafeCreateBucket(name []byte) {}
  665. func (b *fakeBatchTx) UnsafePut(bucketName []byte, key []byte, value []byte) {
  666. b.Recorder.Record(testutil.Action{Name: "put", Params: []interface{}{bucketName, key, value}})
  667. }
  668. func (b *fakeBatchTx) UnsafeRange(bucketName []byte, key, endKey []byte, limit int64) (keys [][]byte, vals [][]byte) {
  669. b.Recorder.Record(testutil.Action{Name: "range", Params: []interface{}{bucketName, key, endKey, limit}})
  670. r := <-b.rangeRespc
  671. return r.keys, r.vals
  672. }
  673. func (b *fakeBatchTx) UnsafeDelete(bucketName []byte, key []byte) {
  674. b.Recorder.Record(testutil.Action{Name: "delete", Params: []interface{}{bucketName, key}})
  675. }
  676. func (b *fakeBatchTx) Commit() {}
  677. func (b *fakeBatchTx) CommitAndStop() {}
  678. type fakeBackend struct {
  679. tx *fakeBatchTx
  680. }
  681. func (b *fakeBackend) BatchTx() backend.BatchTx { return b.tx }
  682. func (b *fakeBackend) Hash() (uint32, error) { return 0, nil }
  683. func (b *fakeBackend) Size() int64 { return 0 }
  684. func (b *fakeBackend) Snapshot() backend.Snapshot { return nil }
  685. func (b *fakeBackend) ForceCommit() {}
  686. func (b *fakeBackend) Close() error { return nil }
  687. type indexGetResp struct {
  688. rev revision
  689. created revision
  690. ver int64
  691. err error
  692. }
  693. type indexRangeResp struct {
  694. keys [][]byte
  695. revs []revision
  696. }
  697. type indexRangeEventsResp struct {
  698. revs []revision
  699. }
  700. type fakeIndex struct {
  701. testutil.Recorder
  702. indexGetRespc chan indexGetResp
  703. indexRangeRespc chan indexRangeResp
  704. indexRangeEventsRespc chan indexRangeEventsResp
  705. indexCompactRespc chan map[revision]struct{}
  706. }
  707. func (i *fakeIndex) Get(key []byte, atRev int64) (rev, created revision, ver int64, err error) {
  708. i.Recorder.Record(testutil.Action{Name: "get", Params: []interface{}{key, atRev}})
  709. r := <-i.indexGetRespc
  710. return r.rev, r.created, r.ver, r.err
  711. }
  712. func (i *fakeIndex) Range(key, end []byte, atRev int64) ([][]byte, []revision) {
  713. i.Recorder.Record(testutil.Action{Name: "range", Params: []interface{}{key, end, atRev}})
  714. r := <-i.indexRangeRespc
  715. return r.keys, r.revs
  716. }
  717. func (i *fakeIndex) Put(key []byte, rev revision) {
  718. i.Recorder.Record(testutil.Action{Name: "put", Params: []interface{}{key, rev}})
  719. }
  720. func (i *fakeIndex) Restore(key []byte, created, modified revision, ver int64) {
  721. i.Recorder.Record(testutil.Action{Name: "restore", Params: []interface{}{key, created, modified, ver}})
  722. }
  723. func (i *fakeIndex) Tombstone(key []byte, rev revision) error {
  724. i.Recorder.Record(testutil.Action{Name: "tombstone", Params: []interface{}{key, rev}})
  725. return nil
  726. }
  727. func (i *fakeIndex) RangeSince(key, end []byte, rev int64) []revision {
  728. i.Recorder.Record(testutil.Action{Name: "rangeEvents", Params: []interface{}{key, end, rev}})
  729. r := <-i.indexRangeEventsRespc
  730. return r.revs
  731. }
  732. func (i *fakeIndex) Compact(rev int64) map[revision]struct{} {
  733. i.Recorder.Record(testutil.Action{Name: "compact", Params: []interface{}{rev}})
  734. return <-i.indexCompactRespc
  735. }
  736. func (i *fakeIndex) Equal(b index) bool { return false }