PageRenderTime 34ms CodeModel.GetById 14ms RepoModel.GetById 1ms app.codeStats 0ms

/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go

https://gitlab.com/Blueprint-Marketing/syncthing
Go | 1886 lines | 1531 code | 280 blank | 75 comment | 300 complexity | fb5319d890e296182e753c1fed3105d9 MD5 | raw file
  1. // Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
  2. // All rights reserved.
  3. //
  4. // Use of this source code is governed by a BSD-style license that can be
  5. // found in the LICENSE file.
  6. package leveldb
  7. import (
  8. "fmt"
  9. "math/rand"
  10. "os"
  11. "path/filepath"
  12. "runtime"
  13. "strings"
  14. "sync"
  15. "sync/atomic"
  16. "testing"
  17. "time"
  18. "unsafe"
  19. "github.com/syndtr/goleveldb/leveldb/comparer"
  20. "github.com/syndtr/goleveldb/leveldb/filter"
  21. "github.com/syndtr/goleveldb/leveldb/iterator"
  22. "github.com/syndtr/goleveldb/leveldb/opt"
  23. "github.com/syndtr/goleveldb/leveldb/storage"
  24. "github.com/syndtr/goleveldb/leveldb/util"
  25. )
  26. func tkey(i int) []byte {
  27. return []byte(fmt.Sprintf("%016d", i))
  28. }
  29. func tval(seed, n int) []byte {
  30. r := rand.New(rand.NewSource(int64(seed)))
  31. return randomString(r, n)
  32. }
  33. type dbHarness struct {
  34. t *testing.T
  35. stor *testStorage
  36. db *DB
  37. o *opt.Options
  38. ro *opt.ReadOptions
  39. wo *opt.WriteOptions
  40. }
  41. func newDbHarnessWopt(t *testing.T, o *opt.Options) *dbHarness {
  42. h := new(dbHarness)
  43. h.init(t, o)
  44. return h
  45. }
  46. func newDbHarness(t *testing.T) *dbHarness {
  47. return newDbHarnessWopt(t, &opt.Options{})
  48. }
  49. func (h *dbHarness) init(t *testing.T, o *opt.Options) {
  50. h.t = t
  51. h.stor = newTestStorage(t)
  52. h.o = o
  53. h.ro = nil
  54. h.wo = nil
  55. if err := h.openDB0(); err != nil {
  56. // So that it will come after fatal message.
  57. defer h.stor.Close()
  58. h.t.Fatal("Open (init): got error: ", err)
  59. }
  60. }
  61. func (h *dbHarness) openDB0() (err error) {
  62. h.t.Log("opening DB")
  63. h.db, err = Open(h.stor, h.o)
  64. return
  65. }
  66. func (h *dbHarness) openDB() {
  67. if err := h.openDB0(); err != nil {
  68. h.t.Fatal("Open: got error: ", err)
  69. }
  70. }
  71. func (h *dbHarness) closeDB0() error {
  72. h.t.Log("closing DB")
  73. return h.db.Close()
  74. }
  75. func (h *dbHarness) closeDB() {
  76. if err := h.closeDB0(); err != nil {
  77. h.t.Error("Close: got error: ", err)
  78. }
  79. h.stor.CloseCheck()
  80. runtime.GC()
  81. }
  82. func (h *dbHarness) reopenDB() {
  83. h.closeDB()
  84. h.openDB()
  85. }
  86. func (h *dbHarness) close() {
  87. h.closeDB0()
  88. h.db = nil
  89. h.stor.Close()
  90. h.stor = nil
  91. runtime.GC()
  92. }
  93. func (h *dbHarness) openAssert(want bool) {
  94. db, err := Open(h.stor, h.o)
  95. if err != nil {
  96. if want {
  97. h.t.Error("Open: assert: got error: ", err)
  98. } else {
  99. h.t.Log("Open: assert: got error (expected): ", err)
  100. }
  101. } else {
  102. if !want {
  103. h.t.Error("Open: assert: expect error")
  104. }
  105. db.Close()
  106. }
  107. }
  108. func (h *dbHarness) write(batch *Batch) {
  109. if err := h.db.Write(batch, h.wo); err != nil {
  110. h.t.Error("Write: got error: ", err)
  111. }
  112. }
  113. func (h *dbHarness) put(key, value string) {
  114. if err := h.db.Put([]byte(key), []byte(value), h.wo); err != nil {
  115. h.t.Error("Put: got error: ", err)
  116. }
  117. }
  118. func (h *dbHarness) putMulti(n int, low, hi string) {
  119. for i := 0; i < n; i++ {
  120. h.put(low, "begin")
  121. h.put(hi, "end")
  122. h.compactMem()
  123. }
  124. }
  125. func (h *dbHarness) maxNextLevelOverlappingBytes(want uint64) {
  126. t := h.t
  127. db := h.db
  128. var res uint64
  129. v := db.s.version()
  130. for i, tt := range v.tables[1 : len(v.tables)-1] {
  131. level := i + 1
  132. next := v.tables[level+1]
  133. for _, t := range tt {
  134. r := next.getOverlaps(nil, db.s.icmp, t.imin.ukey(), t.imax.ukey(), false)
  135. sum := r.size()
  136. if sum > res {
  137. res = sum
  138. }
  139. }
  140. }
  141. v.release()
  142. if res > want {
  143. t.Errorf("next level overlapping bytes is more than %d, got=%d", want, res)
  144. }
  145. }
  146. func (h *dbHarness) delete(key string) {
  147. t := h.t
  148. db := h.db
  149. err := db.Delete([]byte(key), h.wo)
  150. if err != nil {
  151. t.Error("Delete: got error: ", err)
  152. }
  153. }
  154. func (h *dbHarness) assertNumKeys(want int) {
  155. iter := h.db.NewIterator(nil, h.ro)
  156. defer iter.Release()
  157. got := 0
  158. for iter.Next() {
  159. got++
  160. }
  161. if err := iter.Error(); err != nil {
  162. h.t.Error("assertNumKeys: ", err)
  163. }
  164. if want != got {
  165. h.t.Errorf("assertNumKeys: want=%d got=%d", want, got)
  166. }
  167. }
  168. func (h *dbHarness) getr(db Reader, key string, expectFound bool) (found bool, v []byte) {
  169. t := h.t
  170. v, err := db.Get([]byte(key), h.ro)
  171. switch err {
  172. case ErrNotFound:
  173. if expectFound {
  174. t.Errorf("Get: key '%s' not found, want found", key)
  175. }
  176. case nil:
  177. found = true
  178. if !expectFound {
  179. t.Errorf("Get: key '%s' found, want not found", key)
  180. }
  181. default:
  182. t.Error("Get: got error: ", err)
  183. }
  184. return
  185. }
  186. func (h *dbHarness) get(key string, expectFound bool) (found bool, v []byte) {
  187. return h.getr(h.db, key, expectFound)
  188. }
  189. func (h *dbHarness) getValr(db Reader, key, value string) {
  190. t := h.t
  191. found, r := h.getr(db, key, true)
  192. if !found {
  193. return
  194. }
  195. rval := string(r)
  196. if rval != value {
  197. t.Errorf("Get: invalid value, got '%s', want '%s'", rval, value)
  198. }
  199. }
  200. func (h *dbHarness) getVal(key, value string) {
  201. h.getValr(h.db, key, value)
  202. }
  203. func (h *dbHarness) allEntriesFor(key, want string) {
  204. t := h.t
  205. db := h.db
  206. s := db.s
  207. ikey := newIKey([]byte(key), kMaxSeq, tVal)
  208. iter := db.newRawIterator(nil, nil)
  209. if !iter.Seek(ikey) && iter.Error() != nil {
  210. t.Error("AllEntries: error during seek, err: ", iter.Error())
  211. return
  212. }
  213. res := "[ "
  214. first := true
  215. for iter.Valid() {
  216. rkey := iKey(iter.Key())
  217. if _, t, ok := rkey.parseNum(); ok {
  218. if s.icmp.uCompare(ikey.ukey(), rkey.ukey()) != 0 {
  219. break
  220. }
  221. if !first {
  222. res += ", "
  223. }
  224. first = false
  225. switch t {
  226. case tVal:
  227. res += string(iter.Value())
  228. case tDel:
  229. res += "DEL"
  230. }
  231. } else {
  232. if !first {
  233. res += ", "
  234. }
  235. first = false
  236. res += "CORRUPTED"
  237. }
  238. iter.Next()
  239. }
  240. if !first {
  241. res += " "
  242. }
  243. res += "]"
  244. if res != want {
  245. t.Errorf("AllEntries: assert failed for key %q, got=%q want=%q", key, res, want)
  246. }
  247. }
  248. // Return a string that contains all key,value pairs in order,
  249. // formatted like "(k1->v1)(k2->v2)".
  250. func (h *dbHarness) getKeyVal(want string) {
  251. t := h.t
  252. db := h.db
  253. s, err := db.GetSnapshot()
  254. if err != nil {
  255. t.Fatal("GetSnapshot: got error: ", err)
  256. }
  257. res := ""
  258. iter := s.NewIterator(nil, nil)
  259. for iter.Next() {
  260. res += fmt.Sprintf("(%s->%s)", string(iter.Key()), string(iter.Value()))
  261. }
  262. iter.Release()
  263. if res != want {
  264. t.Errorf("GetKeyVal: invalid key/value pair, got=%q want=%q", res, want)
  265. }
  266. s.Release()
  267. }
  268. func (h *dbHarness) waitCompaction() {
  269. t := h.t
  270. db := h.db
  271. if err := db.compSendIdle(db.tcompCmdC); err != nil {
  272. t.Error("compaction error: ", err)
  273. }
  274. }
  275. func (h *dbHarness) waitMemCompaction() {
  276. t := h.t
  277. db := h.db
  278. if err := db.compSendIdle(db.mcompCmdC); err != nil {
  279. t.Error("compaction error: ", err)
  280. }
  281. }
  282. func (h *dbHarness) compactMem() {
  283. t := h.t
  284. db := h.db
  285. db.writeLockC <- struct{}{}
  286. defer func() {
  287. <-db.writeLockC
  288. }()
  289. if _, err := db.rotateMem(0); err != nil {
  290. t.Error("compaction error: ", err)
  291. }
  292. if err := db.compSendIdle(db.mcompCmdC); err != nil {
  293. t.Error("compaction error: ", err)
  294. }
  295. if h.totalTables() == 0 {
  296. t.Error("zero tables after mem compaction")
  297. }
  298. }
  299. func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool) {
  300. t := h.t
  301. db := h.db
  302. var _min, _max []byte
  303. if min != "" {
  304. _min = []byte(min)
  305. }
  306. if max != "" {
  307. _max = []byte(max)
  308. }
  309. if err := db.compSendRange(db.tcompCmdC, level, _min, _max); err != nil {
  310. if wanterr {
  311. t.Log("CompactRangeAt: got error (expected): ", err)
  312. } else {
  313. t.Error("CompactRangeAt: got error: ", err)
  314. }
  315. } else if wanterr {
  316. t.Error("CompactRangeAt: expect error")
  317. }
  318. }
  319. func (h *dbHarness) compactRangeAt(level int, min, max string) {
  320. h.compactRangeAtErr(level, min, max, false)
  321. }
  322. func (h *dbHarness) compactRange(min, max string) {
  323. t := h.t
  324. db := h.db
  325. var r util.Range
  326. if min != "" {
  327. r.Start = []byte(min)
  328. }
  329. if max != "" {
  330. r.Limit = []byte(max)
  331. }
  332. if err := db.CompactRange(r); err != nil {
  333. t.Error("CompactRange: got error: ", err)
  334. }
  335. }
  336. func (h *dbHarness) sizeAssert(start, limit string, low, hi uint64) {
  337. t := h.t
  338. db := h.db
  339. s, err := db.SizeOf([]util.Range{
  340. {[]byte(start), []byte(limit)},
  341. })
  342. if err != nil {
  343. t.Error("SizeOf: got error: ", err)
  344. }
  345. if s.Sum() < low || s.Sum() > hi {
  346. t.Errorf("sizeof %q to %q not in range, want %d - %d, got %d",
  347. shorten(start), shorten(limit), low, hi, s.Sum())
  348. }
  349. }
  350. func (h *dbHarness) getSnapshot() (s *Snapshot) {
  351. s, err := h.db.GetSnapshot()
  352. if err != nil {
  353. h.t.Fatal("GetSnapshot: got error: ", err)
  354. }
  355. return
  356. }
  357. func (h *dbHarness) tablesPerLevel(want string) {
  358. res := ""
  359. nz := 0
  360. v := h.db.s.version()
  361. for level, tt := range v.tables {
  362. if level > 0 {
  363. res += ","
  364. }
  365. res += fmt.Sprint(len(tt))
  366. if len(tt) > 0 {
  367. nz = len(res)
  368. }
  369. }
  370. v.release()
  371. res = res[:nz]
  372. if res != want {
  373. h.t.Errorf("invalid tables len, want=%s, got=%s", want, res)
  374. }
  375. }
  376. func (h *dbHarness) totalTables() (n int) {
  377. v := h.db.s.version()
  378. for _, tt := range v.tables {
  379. n += len(tt)
  380. }
  381. v.release()
  382. return
  383. }
  384. type keyValue interface {
  385. Key() []byte
  386. Value() []byte
  387. }
  388. func testKeyVal(t *testing.T, kv keyValue, want string) {
  389. res := string(kv.Key()) + "->" + string(kv.Value())
  390. if res != want {
  391. t.Errorf("invalid key/value, want=%q, got=%q", want, res)
  392. }
  393. }
  394. func numKey(num int) string {
  395. return fmt.Sprintf("key%06d", num)
  396. }
  397. var _bloom_filter = filter.NewBloomFilter(10)
  398. func truno(t *testing.T, o *opt.Options, f func(h *dbHarness)) {
  399. for i := 0; i < 4; i++ {
  400. func() {
  401. switch i {
  402. case 0:
  403. case 1:
  404. if o == nil {
  405. o = &opt.Options{Filter: _bloom_filter}
  406. } else {
  407. old := o
  408. o = &opt.Options{}
  409. *o = *old
  410. o.Filter = _bloom_filter
  411. }
  412. case 2:
  413. if o == nil {
  414. o = &opt.Options{Compression: opt.NoCompression}
  415. } else {
  416. old := o
  417. o = &opt.Options{}
  418. *o = *old
  419. o.Compression = opt.NoCompression
  420. }
  421. }
  422. h := newDbHarnessWopt(t, o)
  423. defer h.close()
  424. switch i {
  425. case 3:
  426. h.reopenDB()
  427. }
  428. f(h)
  429. }()
  430. }
  431. }
  432. func trun(t *testing.T, f func(h *dbHarness)) {
  433. truno(t, nil, f)
  434. }
  435. func testAligned(t *testing.T, name string, offset uintptr) {
  436. if offset%8 != 0 {
  437. t.Errorf("field %s offset is not 64-bit aligned", name)
  438. }
  439. }
  440. func Test_FieldsAligned(t *testing.T) {
  441. p1 := new(DB)
  442. testAligned(t, "DB.seq", unsafe.Offsetof(p1.seq))
  443. p2 := new(session)
  444. testAligned(t, "session.stFileNum", unsafe.Offsetof(p2.stFileNum))
  445. testAligned(t, "session.stJournalNum", unsafe.Offsetof(p2.stJournalNum))
  446. testAligned(t, "session.stPrevJournalNum", unsafe.Offsetof(p2.stPrevJournalNum))
  447. testAligned(t, "session.stSeq", unsafe.Offsetof(p2.stSeq))
  448. }
  449. func TestDb_Locking(t *testing.T) {
  450. h := newDbHarness(t)
  451. defer h.stor.Close()
  452. h.openAssert(false)
  453. h.closeDB()
  454. h.openAssert(true)
  455. }
  456. func TestDb_Empty(t *testing.T) {
  457. trun(t, func(h *dbHarness) {
  458. h.get("foo", false)
  459. h.reopenDB()
  460. h.get("foo", false)
  461. })
  462. }
  463. func TestDb_ReadWrite(t *testing.T) {
  464. trun(t, func(h *dbHarness) {
  465. h.put("foo", "v1")
  466. h.getVal("foo", "v1")
  467. h.put("bar", "v2")
  468. h.put("foo", "v3")
  469. h.getVal("foo", "v3")
  470. h.getVal("bar", "v2")
  471. h.reopenDB()
  472. h.getVal("foo", "v3")
  473. h.getVal("bar", "v2")
  474. })
  475. }
  476. func TestDb_PutDeleteGet(t *testing.T) {
  477. trun(t, func(h *dbHarness) {
  478. h.put("foo", "v1")
  479. h.getVal("foo", "v1")
  480. h.put("foo", "v2")
  481. h.getVal("foo", "v2")
  482. h.delete("foo")
  483. h.get("foo", false)
  484. h.reopenDB()
  485. h.get("foo", false)
  486. })
  487. }
  488. func TestDb_EmptyBatch(t *testing.T) {
  489. h := newDbHarness(t)
  490. defer h.close()
  491. h.get("foo", false)
  492. err := h.db.Write(new(Batch), h.wo)
  493. if err != nil {
  494. t.Error("writing empty batch yield error: ", err)
  495. }
  496. h.get("foo", false)
  497. }
  498. func TestDb_GetFromFrozen(t *testing.T) {
  499. h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100100})
  500. defer h.close()
  501. h.put("foo", "v1")
  502. h.getVal("foo", "v1")
  503. h.stor.DelaySync(storage.TypeTable) // Block sync calls
  504. h.put("k1", strings.Repeat("x", 100000)) // Fill memtable
  505. h.put("k2", strings.Repeat("y", 100000)) // Trigger compaction
  506. for i := 0; h.db.getFrozenMem() == nil && i < 100; i++ {
  507. time.Sleep(10 * time.Microsecond)
  508. }
  509. if h.db.getFrozenMem() == nil {
  510. h.stor.ReleaseSync(storage.TypeTable)
  511. t.Fatal("No frozen mem")
  512. }
  513. h.getVal("foo", "v1")
  514. h.stor.ReleaseSync(storage.TypeTable) // Release sync calls
  515. h.reopenDB()
  516. h.getVal("foo", "v1")
  517. h.get("k1", true)
  518. h.get("k2", true)
  519. }
  520. func TestDb_GetFromTable(t *testing.T) {
  521. trun(t, func(h *dbHarness) {
  522. h.put("foo", "v1")
  523. h.compactMem()
  524. h.getVal("foo", "v1")
  525. })
  526. }
  527. func TestDb_GetSnapshot(t *testing.T) {
  528. trun(t, func(h *dbHarness) {
  529. bar := strings.Repeat("b", 200)
  530. h.put("foo", "v1")
  531. h.put(bar, "v1")
  532. snap, err := h.db.GetSnapshot()
  533. if err != nil {
  534. t.Fatal("GetSnapshot: got error: ", err)
  535. }
  536. h.put("foo", "v2")
  537. h.put(bar, "v2")
  538. h.getVal("foo", "v2")
  539. h.getVal(bar, "v2")
  540. h.getValr(snap, "foo", "v1")
  541. h.getValr(snap, bar, "v1")
  542. h.compactMem()
  543. h.getVal("foo", "v2")
  544. h.getVal(bar, "v2")
  545. h.getValr(snap, "foo", "v1")
  546. h.getValr(snap, bar, "v1")
  547. snap.Release()
  548. h.reopenDB()
  549. h.getVal("foo", "v2")
  550. h.getVal(bar, "v2")
  551. })
  552. }
  553. func TestDb_GetLevel0Ordering(t *testing.T) {
  554. trun(t, func(h *dbHarness) {
  555. for i := 0; i < 4; i++ {
  556. h.put("bar", fmt.Sprintf("b%d", i))
  557. h.put("foo", fmt.Sprintf("v%d", i))
  558. h.compactMem()
  559. }
  560. h.getVal("foo", "v3")
  561. h.getVal("bar", "b3")
  562. v := h.db.s.version()
  563. t0len := v.tLen(0)
  564. v.release()
  565. if t0len < 2 {
  566. t.Errorf("level-0 tables is less than 2, got %d", t0len)
  567. }
  568. h.reopenDB()
  569. h.getVal("foo", "v3")
  570. h.getVal("bar", "b3")
  571. })
  572. }
  573. func TestDb_GetOrderedByLevels(t *testing.T) {
  574. trun(t, func(h *dbHarness) {
  575. h.put("foo", "v1")
  576. h.compactMem()
  577. h.compactRange("a", "z")
  578. h.getVal("foo", "v1")
  579. h.put("foo", "v2")
  580. h.compactMem()
  581. h.getVal("foo", "v2")
  582. })
  583. }
  584. func TestDb_GetPicksCorrectFile(t *testing.T) {
  585. trun(t, func(h *dbHarness) {
  586. // Arrange to have multiple files in a non-level-0 level.
  587. h.put("a", "va")
  588. h.compactMem()
  589. h.compactRange("a", "b")
  590. h.put("x", "vx")
  591. h.compactMem()
  592. h.compactRange("x", "y")
  593. h.put("f", "vf")
  594. h.compactMem()
  595. h.compactRange("f", "g")
  596. h.getVal("a", "va")
  597. h.getVal("f", "vf")
  598. h.getVal("x", "vx")
  599. h.compactRange("", "")
  600. h.getVal("a", "va")
  601. h.getVal("f", "vf")
  602. h.getVal("x", "vx")
  603. })
  604. }
  605. func TestDb_GetEncountersEmptyLevel(t *testing.T) {
  606. trun(t, func(h *dbHarness) {
  607. // Arrange for the following to happen:
  608. // * sstable A in level 0
  609. // * nothing in level 1
  610. // * sstable B in level 2
  611. // Then do enough Get() calls to arrange for an automatic compaction
  612. // of sstable A. A bug would cause the compaction to be marked as
  613. // occuring at level 1 (instead of the correct level 0).
  614. // Step 1: First place sstables in levels 0 and 2
  615. for i := 0; ; i++ {
  616. if i >= 100 {
  617. t.Fatal("could not fill levels-0 and level-2")
  618. }
  619. v := h.db.s.version()
  620. if v.tLen(0) > 0 && v.tLen(2) > 0 {
  621. v.release()
  622. break
  623. }
  624. v.release()
  625. h.put("a", "begin")
  626. h.put("z", "end")
  627. h.compactMem()
  628. h.getVal("a", "begin")
  629. h.getVal("z", "end")
  630. }
  631. // Step 2: clear level 1 if necessary.
  632. h.compactRangeAt(1, "", "")
  633. h.tablesPerLevel("1,0,1")
  634. h.getVal("a", "begin")
  635. h.getVal("z", "end")
  636. // Step 3: read a bunch of times
  637. for i := 0; i < 200; i++ {
  638. h.get("missing", false)
  639. }
  640. // Step 4: Wait for compaction to finish
  641. h.waitCompaction()
  642. v := h.db.s.version()
  643. if v.tLen(0) > 0 {
  644. t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
  645. }
  646. v.release()
  647. h.getVal("a", "begin")
  648. h.getVal("z", "end")
  649. })
  650. }
  651. func TestDb_IterMultiWithDelete(t *testing.T) {
  652. trun(t, func(h *dbHarness) {
  653. h.put("a", "va")
  654. h.put("b", "vb")
  655. h.put("c", "vc")
  656. h.delete("b")
  657. h.get("b", false)
  658. iter := h.db.NewIterator(nil, nil)
  659. iter.Seek([]byte("c"))
  660. testKeyVal(t, iter, "c->vc")
  661. iter.Prev()
  662. testKeyVal(t, iter, "a->va")
  663. iter.Release()
  664. h.compactMem()
  665. iter = h.db.NewIterator(nil, nil)
  666. iter.Seek([]byte("c"))
  667. testKeyVal(t, iter, "c->vc")
  668. iter.Prev()
  669. testKeyVal(t, iter, "a->va")
  670. iter.Release()
  671. })
  672. }
  673. func TestDb_IteratorPinsRef(t *testing.T) {
  674. h := newDbHarness(t)
  675. defer h.close()
  676. h.put("foo", "hello")
  677. // Get iterator that will yield the current contents of the DB.
  678. iter := h.db.NewIterator(nil, nil)
  679. // Write to force compactions
  680. h.put("foo", "newvalue1")
  681. for i := 0; i < 100; i++ {
  682. h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
  683. }
  684. h.put("foo", "newvalue2")
  685. iter.First()
  686. testKeyVal(t, iter, "foo->hello")
  687. if iter.Next() {
  688. t.Errorf("expect eof")
  689. }
  690. iter.Release()
  691. }
  692. func TestDb_Recover(t *testing.T) {
  693. trun(t, func(h *dbHarness) {
  694. h.put("foo", "v1")
  695. h.put("baz", "v5")
  696. h.reopenDB()
  697. h.getVal("foo", "v1")
  698. h.getVal("foo", "v1")
  699. h.getVal("baz", "v5")
  700. h.put("bar", "v2")
  701. h.put("foo", "v3")
  702. h.reopenDB()
  703. h.getVal("foo", "v3")
  704. h.put("foo", "v4")
  705. h.getVal("foo", "v4")
  706. h.getVal("bar", "v2")
  707. h.getVal("baz", "v5")
  708. })
  709. }
  710. func TestDb_RecoverWithEmptyJournal(t *testing.T) {
  711. trun(t, func(h *dbHarness) {
  712. h.put("foo", "v1")
  713. h.put("foo", "v2")
  714. h.reopenDB()
  715. h.reopenDB()
  716. h.put("foo", "v3")
  717. h.reopenDB()
  718. h.getVal("foo", "v3")
  719. })
  720. }
  721. func TestDb_RecoverDuringMemtableCompaction(t *testing.T) {
  722. truno(t, &opt.Options{WriteBuffer: 1000000}, func(h *dbHarness) {
  723. h.stor.DelaySync(storage.TypeTable)
  724. h.put("big1", strings.Repeat("x", 10000000))
  725. h.put("big2", strings.Repeat("y", 1000))
  726. h.put("bar", "v2")
  727. h.stor.ReleaseSync(storage.TypeTable)
  728. h.reopenDB()
  729. h.getVal("bar", "v2")
  730. h.getVal("big1", strings.Repeat("x", 10000000))
  731. h.getVal("big2", strings.Repeat("y", 1000))
  732. })
  733. }
  734. func TestDb_MinorCompactionsHappen(t *testing.T) {
  735. h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 10000})
  736. defer h.close()
  737. n := 500
  738. key := func(i int) string {
  739. return fmt.Sprintf("key%06d", i)
  740. }
  741. for i := 0; i < n; i++ {
  742. h.put(key(i), key(i)+strings.Repeat("v", 1000))
  743. }
  744. for i := 0; i < n; i++ {
  745. h.getVal(key(i), key(i)+strings.Repeat("v", 1000))
  746. }
  747. h.reopenDB()
  748. for i := 0; i < n; i++ {
  749. h.getVal(key(i), key(i)+strings.Repeat("v", 1000))
  750. }
  751. }
  752. func TestDb_RecoverWithLargeJournal(t *testing.T) {
  753. h := newDbHarness(t)
  754. defer h.close()
  755. h.put("big1", strings.Repeat("1", 200000))
  756. h.put("big2", strings.Repeat("2", 200000))
  757. h.put("small3", strings.Repeat("3", 10))
  758. h.put("small4", strings.Repeat("4", 10))
  759. h.tablesPerLevel("")
  760. // Make sure that if we re-open with a small write buffer size that
  761. // we flush table files in the middle of a large journal file.
  762. h.o.WriteBuffer = 100000
  763. h.reopenDB()
  764. h.getVal("big1", strings.Repeat("1", 200000))
  765. h.getVal("big2", strings.Repeat("2", 200000))
  766. h.getVal("small3", strings.Repeat("3", 10))
  767. h.getVal("small4", strings.Repeat("4", 10))
  768. v := h.db.s.version()
  769. if v.tLen(0) <= 1 {
  770. t.Errorf("tables-0 less than one")
  771. }
  772. v.release()
  773. }
  774. func TestDb_CompactionsGenerateMultipleFiles(t *testing.T) {
  775. h := newDbHarnessWopt(t, &opt.Options{
  776. WriteBuffer: 10000000,
  777. Compression: opt.NoCompression,
  778. })
  779. defer h.close()
  780. v := h.db.s.version()
  781. if v.tLen(0) > 0 {
  782. t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
  783. }
  784. v.release()
  785. n := 80
  786. // Write 8MB (80 values, each 100K)
  787. for i := 0; i < n; i++ {
  788. h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
  789. }
  790. // Reopening moves updates to level-0
  791. h.reopenDB()
  792. h.compactRangeAt(0, "", "")
  793. v = h.db.s.version()
  794. if v.tLen(0) > 0 {
  795. t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
  796. }
  797. if v.tLen(1) <= 1 {
  798. t.Errorf("level-1 tables less than 1, got %d", v.tLen(1))
  799. }
  800. v.release()
  801. for i := 0; i < n; i++ {
  802. h.getVal(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
  803. }
  804. }
  805. func TestDb_RepeatedWritesToSameKey(t *testing.T) {
  806. h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100000})
  807. defer h.close()
  808. maxTables := kNumLevels + kL0_StopWritesTrigger
  809. value := strings.Repeat("v", 2*h.o.GetWriteBuffer())
  810. for i := 0; i < 5*maxTables; i++ {
  811. h.put("key", value)
  812. n := h.totalTables()
  813. if n > maxTables {
  814. t.Errorf("total tables exceed %d, got=%d, iter=%d", maxTables, n, i)
  815. }
  816. }
  817. }
  818. func TestDb_RepeatedWritesToSameKeyAfterReopen(t *testing.T) {
  819. h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100000})
  820. defer h.close()
  821. h.reopenDB()
  822. maxTables := kNumLevels + kL0_StopWritesTrigger
  823. value := strings.Repeat("v", 2*h.o.GetWriteBuffer())
  824. for i := 0; i < 5*maxTables; i++ {
  825. h.put("key", value)
  826. n := h.totalTables()
  827. if n > maxTables {
  828. t.Errorf("total tables exceed %d, got=%d, iter=%d", maxTables, n, i)
  829. }
  830. }
  831. }
  832. func TestDb_SparseMerge(t *testing.T) {
  833. h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression})
  834. defer h.close()
  835. h.putMulti(kNumLevels, "A", "Z")
  836. // Suppose there is:
  837. // small amount of data with prefix A
  838. // large amount of data with prefix B
  839. // small amount of data with prefix C
  840. // and that recent updates have made small changes to all three prefixes.
  841. // Check that we do not do a compaction that merges all of B in one shot.
  842. h.put("A", "va")
  843. value := strings.Repeat("x", 1000)
  844. for i := 0; i < 100000; i++ {
  845. h.put(fmt.Sprintf("B%010d", i), value)
  846. }
  847. h.put("C", "vc")
  848. h.compactMem()
  849. h.compactRangeAt(0, "", "")
  850. h.waitCompaction()
  851. // Make sparse update
  852. h.put("A", "va2")
  853. h.put("B100", "bvalue2")
  854. h.put("C", "vc2")
  855. h.compactMem()
  856. h.maxNextLevelOverlappingBytes(20 * 1048576)
  857. h.compactRangeAt(0, "", "")
  858. h.waitCompaction()
  859. h.maxNextLevelOverlappingBytes(20 * 1048576)
  860. h.compactRangeAt(1, "", "")
  861. h.waitCompaction()
  862. h.maxNextLevelOverlappingBytes(20 * 1048576)
  863. }
  864. func TestDb_SizeOf(t *testing.T) {
  865. h := newDbHarnessWopt(t, &opt.Options{
  866. Compression: opt.NoCompression,
  867. WriteBuffer: 10000000,
  868. })
  869. defer h.close()
  870. h.sizeAssert("", "xyz", 0, 0)
  871. h.reopenDB()
  872. h.sizeAssert("", "xyz", 0, 0)
  873. // Write 8MB (80 values, each 100K)
  874. n := 80
  875. s1 := 100000
  876. s2 := 105000
  877. for i := 0; i < n; i++ {
  878. h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), s1/10))
  879. }
  880. // 0 because SizeOf() does not account for memtable space
  881. h.sizeAssert("", numKey(50), 0, 0)
  882. for r := 0; r < 3; r++ {
  883. h.reopenDB()
  884. for cs := 0; cs < n; cs += 10 {
  885. for i := 0; i < n; i += 10 {
  886. h.sizeAssert("", numKey(i), uint64(s1*i), uint64(s2*i))
  887. h.sizeAssert("", numKey(i)+".suffix", uint64(s1*(i+1)), uint64(s2*(i+1)))
  888. h.sizeAssert(numKey(i), numKey(i+10), uint64(s1*10), uint64(s2*10))
  889. }
  890. h.sizeAssert("", numKey(50), uint64(s1*50), uint64(s2*50))
  891. h.sizeAssert("", numKey(50)+".suffix", uint64(s1*50), uint64(s2*50))
  892. h.compactRangeAt(0, numKey(cs), numKey(cs+9))
  893. }
  894. v := h.db.s.version()
  895. if v.tLen(0) != 0 {
  896. t.Errorf("level-0 tables was not zero, got %d", v.tLen(0))
  897. }
  898. if v.tLen(1) == 0 {
  899. t.Error("level-1 tables was zero")
  900. }
  901. v.release()
  902. }
  903. }
  904. func TestDb_SizeOf_MixOfSmallAndLarge(t *testing.T) {
  905. h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression})
  906. defer h.close()
  907. sizes := []uint64{
  908. 10000,
  909. 10000,
  910. 100000,
  911. 10000,
  912. 100000,
  913. 10000,
  914. 300000,
  915. 10000,
  916. }
  917. for i, n := range sizes {
  918. h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), int(n)/10))
  919. }
  920. for r := 0; r < 3; r++ {
  921. h.reopenDB()
  922. var x uint64
  923. for i, n := range sizes {
  924. y := x
  925. if i > 0 {
  926. y += 1000
  927. }
  928. h.sizeAssert("", numKey(i), x, y)
  929. x += n
  930. }
  931. h.sizeAssert(numKey(3), numKey(5), 110000, 111000)
  932. h.compactRangeAt(0, "", "")
  933. }
  934. }
  935. func TestDb_Snapshot(t *testing.T) {
  936. trun(t, func(h *dbHarness) {
  937. h.put("foo", "v1")
  938. s1 := h.getSnapshot()
  939. h.put("foo", "v2")
  940. s2 := h.getSnapshot()
  941. h.put("foo", "v3")
  942. s3 := h.getSnapshot()
  943. h.put("foo", "v4")
  944. h.getValr(s1, "foo", "v1")
  945. h.getValr(s2, "foo", "v2")
  946. h.getValr(s3, "foo", "v3")
  947. h.getVal("foo", "v4")
  948. s3.Release()
  949. h.getValr(s1, "foo", "v1")
  950. h.getValr(s2, "foo", "v2")
  951. h.getVal("foo", "v4")
  952. s1.Release()
  953. h.getValr(s2, "foo", "v2")
  954. h.getVal("foo", "v4")
  955. s2.Release()
  956. h.getVal("foo", "v4")
  957. })
  958. }
  959. func TestDb_HiddenValuesAreRemoved(t *testing.T) {
  960. trun(t, func(h *dbHarness) {
  961. s := h.db.s
  962. h.put("foo", "v1")
  963. h.compactMem()
  964. m := kMaxMemCompactLevel
  965. v := s.version()
  966. num := v.tLen(m)
  967. v.release()
  968. if num != 1 {
  969. t.Errorf("invalid level-%d len, want=1 got=%d", m, num)
  970. }
  971. // Place a table at level last-1 to prevent merging with preceding mutation
  972. h.put("a", "begin")
  973. h.put("z", "end")
  974. h.compactMem()
  975. v = s.version()
  976. if v.tLen(m) != 1 {
  977. t.Errorf("invalid level-%d len, want=1 got=%d", m, v.tLen(m))
  978. }
  979. if v.tLen(m-1) != 1 {
  980. t.Errorf("invalid level-%d len, want=1 got=%d", m-1, v.tLen(m-1))
  981. }
  982. v.release()
  983. h.delete("foo")
  984. h.put("foo", "v2")
  985. h.allEntriesFor("foo", "[ v2, DEL, v1 ]")
  986. h.compactMem()
  987. h.allEntriesFor("foo", "[ v2, DEL, v1 ]")
  988. h.compactRangeAt(m-2, "", "z")
  989. // DEL eliminated, but v1 remains because we aren't compacting that level
  990. // (DEL can be eliminated because v2 hides v1).
  991. h.allEntriesFor("foo", "[ v2, v1 ]")
  992. h.compactRangeAt(m-1, "", "")
  993. // Merging last-1 w/ last, so we are the base level for "foo", so
  994. // DEL is removed. (as is v1).
  995. h.allEntriesFor("foo", "[ v2 ]")
  996. })
  997. }
  998. func TestDb_DeletionMarkers2(t *testing.T) {
  999. h := newDbHarness(t)
  1000. defer h.close()
  1001. s := h.db.s
  1002. h.put("foo", "v1")
  1003. h.compactMem()
  1004. m := kMaxMemCompactLevel
  1005. v := s.version()
  1006. num := v.tLen(m)
  1007. v.release()
  1008. if num != 1 {
  1009. t.Errorf("invalid level-%d len, want=1 got=%d", m, num)
  1010. }
  1011. // Place a table at level last-1 to prevent merging with preceding mutation
  1012. h.put("a", "begin")
  1013. h.put("z", "end")
  1014. h.compactMem()
  1015. v = s.version()
  1016. if v.tLen(m) != 1 {
  1017. t.Errorf("invalid level-%d len, want=1 got=%d", m, v.tLen(m))
  1018. }
  1019. if v.tLen(m-1) != 1 {
  1020. t.Errorf("invalid level-%d len, want=1 got=%d", m-1, v.tLen(m-1))
  1021. }
  1022. v.release()
  1023. h.delete("foo")
  1024. h.allEntriesFor("foo", "[ DEL, v1 ]")
  1025. h.compactMem() // Moves to level last-2
  1026. h.allEntriesFor("foo", "[ DEL, v1 ]")
  1027. h.compactRangeAt(m-2, "", "")
  1028. // DEL kept: "last" file overlaps
  1029. h.allEntriesFor("foo", "[ DEL, v1 ]")
  1030. h.compactRangeAt(m-1, "", "")
  1031. // Merging last-1 w/ last, so we are the base level for "foo", so
  1032. // DEL is removed. (as is v1).
  1033. h.allEntriesFor("foo", "[ ]")
  1034. }
  1035. func TestDb_CompactionTableOpenError(t *testing.T) {
  1036. h := newDbHarnessWopt(t, &opt.Options{MaxOpenFiles: 0})
  1037. defer h.close()
  1038. im := 10
  1039. jm := 10
  1040. for r := 0; r < 2; r++ {
  1041. for i := 0; i < im; i++ {
  1042. for j := 0; j < jm; j++ {
  1043. h.put(fmt.Sprintf("k%d,%d", i, j), fmt.Sprintf("v%d,%d", i, j))
  1044. }
  1045. h.compactMem()
  1046. }
  1047. }
  1048. if n := h.totalTables(); n != im*2 {
  1049. t.Errorf("total tables is %d, want %d", n, im)
  1050. }
  1051. h.stor.SetOpenErr(storage.TypeTable)
  1052. go h.db.CompactRange(util.Range{})
  1053. if err := h.db.compSendIdle(h.db.tcompCmdC); err != nil {
  1054. t.Log("compaction error: ", err)
  1055. }
  1056. h.closeDB0()
  1057. h.openDB()
  1058. h.stor.SetOpenErr(0)
  1059. for i := 0; i < im; i++ {
  1060. for j := 0; j < jm; j++ {
  1061. h.getVal(fmt.Sprintf("k%d,%d", i, j), fmt.Sprintf("v%d,%d", i, j))
  1062. }
  1063. }
  1064. }
  1065. func TestDb_OverlapInLevel0(t *testing.T) {
  1066. trun(t, func(h *dbHarness) {
  1067. if kMaxMemCompactLevel != 2 {
  1068. t.Fatal("fix test to reflect the config")
  1069. }
  1070. // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
  1071. h.put("100", "v100")
  1072. h.put("999", "v999")
  1073. h.compactMem()
  1074. h.delete("100")
  1075. h.delete("999")
  1076. h.compactMem()
  1077. h.tablesPerLevel("0,1,1")
  1078. // Make files spanning the following ranges in level-0:
  1079. // files[0] 200 .. 900
  1080. // files[1] 300 .. 500
  1081. // Note that files are sorted by min key.
  1082. h.put("300", "v300")
  1083. h.put("500", "v500")
  1084. h.compactMem()
  1085. h.put("200", "v200")
  1086. h.put("600", "v600")
  1087. h.put("900", "v900")
  1088. h.compactMem()
  1089. h.tablesPerLevel("2,1,1")
  1090. // Compact away the placeholder files we created initially
  1091. h.compactRangeAt(1, "", "")
  1092. h.compactRangeAt(2, "", "")
  1093. h.tablesPerLevel("2")
  1094. // Do a memtable compaction. Before bug-fix, the compaction would
  1095. // not detect the overlap with level-0 files and would incorrectly place
  1096. // the deletion in a deeper level.
  1097. h.delete("600")
  1098. h.compactMem()
  1099. h.tablesPerLevel("3")
  1100. h.get("600", false)
  1101. })
  1102. }
  1103. func TestDb_L0_CompactionBug_Issue44_a(t *testing.T) {
  1104. h := newDbHarness(t)
  1105. defer h.close()
  1106. h.reopenDB()
  1107. h.put("b", "v")
  1108. h.reopenDB()
  1109. h.delete("b")
  1110. h.delete("a")
  1111. h.reopenDB()
  1112. h.delete("a")
  1113. h.reopenDB()
  1114. h.put("a", "v")
  1115. h.reopenDB()
  1116. h.reopenDB()
  1117. h.getKeyVal("(a->v)")
  1118. h.waitCompaction()
  1119. h.getKeyVal("(a->v)")
  1120. }
  1121. func TestDb_L0_CompactionBug_Issue44_b(t *testing.T) {
  1122. h := newDbHarness(t)
  1123. defer h.close()
  1124. h.reopenDB()
  1125. h.put("", "")
  1126. h.reopenDB()
  1127. h.delete("e")
  1128. h.put("", "")
  1129. h.reopenDB()
  1130. h.put("c", "cv")
  1131. h.reopenDB()
  1132. h.put("", "")
  1133. h.reopenDB()
  1134. h.put("", "")
  1135. h.waitCompaction()
  1136. h.reopenDB()
  1137. h.put("d", "dv")
  1138. h.reopenDB()
  1139. h.put("", "")
  1140. h.reopenDB()
  1141. h.delete("d")
  1142. h.delete("b")
  1143. h.reopenDB()
  1144. h.getKeyVal("(->)(c->cv)")
  1145. h.waitCompaction()
  1146. h.getKeyVal("(->)(c->cv)")
  1147. }
  1148. func TestDb_SingleEntryMemCompaction(t *testing.T) {
  1149. trun(t, func(h *dbHarness) {
  1150. for i := 0; i < 10; i++ {
  1151. h.put("big", strings.Repeat("v", opt.DefaultWriteBuffer))
  1152. h.compactMem()
  1153. h.put("key", strings.Repeat("v", opt.DefaultBlockSize))
  1154. h.compactMem()
  1155. h.put("k", "v")
  1156. h.compactMem()
  1157. h.put("", "")
  1158. h.compactMem()
  1159. h.put("verybig", strings.Repeat("v", opt.DefaultWriteBuffer*2))
  1160. h.compactMem()
  1161. }
  1162. })
  1163. }
  1164. func TestDb_ManifestWriteError(t *testing.T) {
  1165. for i := 0; i < 2; i++ {
  1166. func() {
  1167. h := newDbHarness(t)
  1168. defer h.close()
  1169. h.put("foo", "bar")
  1170. h.getVal("foo", "bar")
  1171. // Mem compaction (will succeed)
  1172. h.compactMem()
  1173. h.getVal("foo", "bar")
  1174. v := h.db.s.version()
  1175. if n := v.tLen(kMaxMemCompactLevel); n != 1 {
  1176. t.Errorf("invalid total tables, want=1 got=%d", n)
  1177. }
  1178. v.release()
  1179. if i == 0 {
  1180. h.stor.SetWriteErr(storage.TypeManifest)
  1181. } else {
  1182. h.stor.SetSyncErr(storage.TypeManifest)
  1183. }
  1184. // Merging compaction (will fail)
  1185. h.compactRangeAtErr(kMaxMemCompactLevel, "", "", true)
  1186. h.db.Close()
  1187. h.stor.SetWriteErr(0)
  1188. h.stor.SetSyncErr(0)
  1189. // Should not lose data
  1190. h.openDB()
  1191. h.getVal("foo", "bar")
  1192. }()
  1193. }
  1194. }
  1195. func assertErr(t *testing.T, err error, wanterr bool) {
  1196. if err != nil {
  1197. if wanterr {
  1198. t.Log("AssertErr: got error (expected): ", err)
  1199. } else {
  1200. t.Error("AssertErr: got error: ", err)
  1201. }
  1202. } else if wanterr {
  1203. t.Error("AssertErr: expect error")
  1204. }
  1205. }
  1206. func TestDb_ClosedIsClosed(t *testing.T) {
  1207. h := newDbHarness(t)
  1208. db := h.db
  1209. var iter, iter2 iterator.Iterator
  1210. var snap *Snapshot
  1211. func() {
  1212. defer h.close()
  1213. h.put("k", "v")
  1214. h.getVal("k", "v")
  1215. iter = db.NewIterator(nil, h.ro)
  1216. iter.Seek([]byte("k"))
  1217. testKeyVal(t, iter, "k->v")
  1218. var err error
  1219. snap, err = db.GetSnapshot()
  1220. if err != nil {
  1221. t.Fatal("GetSnapshot: got error: ", err)
  1222. }
  1223. h.getValr(snap, "k", "v")
  1224. iter2 = snap.NewIterator(nil, h.ro)
  1225. iter2.Seek([]byte("k"))
  1226. testKeyVal(t, iter2, "k->v")
  1227. h.put("foo", "v2")
  1228. h.delete("foo")
  1229. // closing DB
  1230. iter.Release()
  1231. iter2.Release()
  1232. }()
  1233. assertErr(t, db.Put([]byte("x"), []byte("y"), h.wo), true)
  1234. _, err := db.Get([]byte("k"), h.ro)
  1235. assertErr(t, err, true)
  1236. if iter.Valid() {
  1237. t.Errorf("iter.Valid should false")
  1238. }
  1239. assertErr(t, iter.Error(), false)
  1240. testKeyVal(t, iter, "->")
  1241. if iter.Seek([]byte("k")) {
  1242. t.Errorf("iter.Seek should false")
  1243. }
  1244. assertErr(t, iter.Error(), true)
  1245. assertErr(t, iter2.Error(), false)
  1246. _, err = snap.Get([]byte("k"), h.ro)
  1247. assertErr(t, err, true)
  1248. _, err = db.GetSnapshot()
  1249. assertErr(t, err, true)
  1250. iter3 := db.NewIterator(nil, h.ro)
  1251. assertErr(t, iter3.Error(), true)
  1252. iter3 = snap.NewIterator(nil, h.ro)
  1253. assertErr(t, iter3.Error(), true)
  1254. assertErr(t, db.Delete([]byte("k"), h.wo), true)
  1255. _, err = db.GetProperty("leveldb.stats")
  1256. assertErr(t, err, true)
  1257. _, err = db.SizeOf([]util.Range{{[]byte("a"), []byte("z")}})
  1258. assertErr(t, err, true)
  1259. assertErr(t, db.CompactRange(util.Range{}), true)
  1260. assertErr(t, db.Close(), true)
  1261. }
  1262. type numberComparer struct{}
  1263. func (numberComparer) num(x []byte) (n int) {
  1264. fmt.Sscan(string(x[1:len(x)-1]), &n)
  1265. return
  1266. }
  1267. func (numberComparer) Name() string {
  1268. return "test.NumberComparer"
  1269. }
  1270. func (p numberComparer) Compare(a, b []byte) int {
  1271. return p.num(a) - p.num(b)
  1272. }
  1273. func (numberComparer) Separator(dst, a, b []byte) []byte { return nil }
  1274. func (numberComparer) Successor(dst, b []byte) []byte { return nil }
  1275. func TestDb_CustomComparer(t *testing.T) {
  1276. h := newDbHarnessWopt(t, &opt.Options{
  1277. Comparer: numberComparer{},
  1278. WriteBuffer: 1000,
  1279. })
  1280. defer h.close()
  1281. h.put("[10]", "ten")
  1282. h.put("[0x14]", "twenty")
  1283. for i := 0; i < 2; i++ {
  1284. h.getVal("[10]", "ten")
  1285. h.getVal("[0xa]", "ten")
  1286. h.getVal("[20]", "twenty")
  1287. h.getVal("[0x14]", "twenty")
  1288. h.get("[15]", false)
  1289. h.get("[0xf]", false)
  1290. h.compactMem()
  1291. h.compactRange("[0]", "[9999]")
  1292. }
  1293. for n := 0; n < 2; n++ {
  1294. for i := 0; i < 100; i++ {
  1295. v := fmt.Sprintf("[%d]", i*10)
  1296. h.put(v, v)
  1297. }
  1298. h.compactMem()
  1299. h.compactRange("[0]", "[1000000]")
  1300. }
  1301. }
  1302. func TestDb_ManualCompaction(t *testing.T) {
  1303. h := newDbHarness(t)
  1304. defer h.close()
  1305. if kMaxMemCompactLevel != 2 {
  1306. t.Fatal("fix test to reflect the config")
  1307. }
  1308. h.putMulti(3, "p", "q")
  1309. h.tablesPerLevel("1,1,1")
  1310. // Compaction range falls before files
  1311. h.compactRange("", "c")
  1312. h.tablesPerLevel("1,1,1")
  1313. // Compaction range falls after files
  1314. h.compactRange("r", "z")
  1315. h.tablesPerLevel("1,1,1")
  1316. // Compaction range overlaps files
  1317. h.compactRange("p1", "p9")
  1318. h.tablesPerLevel("0,0,1")
  1319. // Populate a different range
  1320. h.putMulti(3, "c", "e")
  1321. h.tablesPerLevel("1,1,2")
  1322. // Compact just the new range
  1323. h.compactRange("b", "f")
  1324. h.tablesPerLevel("0,0,2")
  1325. // Compact all
  1326. h.putMulti(1, "a", "z")
  1327. h.tablesPerLevel("0,1,2")
  1328. h.compactRange("", "")
  1329. h.tablesPerLevel("0,0,1")
  1330. }
  1331. func TestDb_BloomFilter(t *testing.T) {
  1332. h := newDbHarnessWopt(t, &opt.Options{
  1333. BlockCache: opt.NoCache,
  1334. Filter: filter.NewBloomFilter(10),
  1335. })
  1336. defer h.close()
  1337. key := func(i int) string {
  1338. return fmt.Sprintf("key%06d", i)
  1339. }
  1340. n := 10000
  1341. // Populate multiple layers
  1342. for i := 0; i < n; i++ {
  1343. h.put(key(i), key(i))
  1344. }
  1345. h.compactMem()
  1346. h.compactRange("a", "z")
  1347. for i := 0; i < n; i += 100 {
  1348. h.put(key(i), key(i))
  1349. }
  1350. h.compactMem()
  1351. // Prevent auto compactions triggered by seeks
  1352. h.stor.DelaySync(storage.TypeTable)
  1353. // Lookup present keys. Should rarely read from small sstable.
  1354. h.stor.SetReadCounter(storage.TypeTable)
  1355. for i := 0; i < n; i++ {
  1356. h.getVal(key(i), key(i))
  1357. }
  1358. cnt := int(h.stor.ReadCounter())
  1359. t.Logf("lookup of %d present keys yield %d sstable I/O reads", n, cnt)
  1360. if min, max := n, n+2*n/100; cnt < min || cnt > max {
  1361. t.Errorf("num of sstable I/O reads of present keys not in range of %d - %d, got %d", min, max, cnt)
  1362. }
  1363. // Lookup missing keys. Should rarely read from either sstable.
  1364. h.stor.ResetReadCounter()
  1365. for i := 0; i < n; i++ {
  1366. h.get(key(i)+".missing", false)
  1367. }
  1368. cnt = int(h.stor.ReadCounter())
  1369. t.Logf("lookup of %d missing keys yield %d sstable I/O reads", n, cnt)
  1370. if max := 3 * n / 100; cnt > max {
  1371. t.Errorf("num of sstable I/O reads of missing keys was more than %d, got %d", max, cnt)
  1372. }
  1373. h.stor.ReleaseSync(storage.TypeTable)
  1374. }
  1375. func TestDb_Concurrent(t *testing.T) {
  1376. const n, secs, maxkey = 4, 2, 1000
  1377. runtime.GOMAXPROCS(n)
  1378. trun(t, func(h *dbHarness) {
  1379. var closeWg sync.WaitGroup
  1380. var stop uint32
  1381. var cnt [n]uint32
  1382. for i := 0; i < n; i++ {
  1383. closeWg.Add(1)
  1384. go func(i int) {
  1385. var put, get, found uint
  1386. defer func() {
  1387. t.Logf("goroutine %d stopped after %d ops, put=%d get=%d found=%d missing=%d",
  1388. i, cnt[i], put, get, found, get-found)
  1389. closeWg.Done()
  1390. }()
  1391. rnd := rand.New(rand.NewSource(int64(1000 + i)))
  1392. for atomic.LoadUint32(&stop) == 0 {
  1393. x := cnt[i]
  1394. k := rnd.Intn(maxkey)
  1395. kstr := fmt.Sprintf("%016d", k)
  1396. if (rnd.Int() % 2) > 0 {
  1397. put++
  1398. h.put(kstr, fmt.Sprintf("%d.%d.%-1000d", k, i, x))
  1399. } else {
  1400. get++
  1401. v, err := h.db.Get([]byte(kstr), h.ro)
  1402. if err == nil {
  1403. found++
  1404. rk, ri, rx := 0, -1, uint32(0)
  1405. fmt.Sscanf(string(v), "%d.%d.%d", &rk, &ri, &rx)
  1406. if rk != k {
  1407. t.Errorf("invalid key want=%d got=%d", k, rk)
  1408. }
  1409. if ri < 0 || ri >= n {
  1410. t.Error("invalid goroutine number: ", ri)
  1411. } else {
  1412. tx := atomic.LoadUint32(&(cnt[ri]))
  1413. if rx > tx {
  1414. t.Errorf("invalid seq number, %d > %d ", rx, tx)
  1415. }
  1416. }
  1417. } else if err != ErrNotFound {
  1418. t.Error("Get: got error: ", err)
  1419. return
  1420. }
  1421. }
  1422. atomic.AddUint32(&cnt[i], 1)
  1423. }
  1424. }(i)
  1425. }
  1426. time.Sleep(secs * time.Second)
  1427. atomic.StoreUint32(&stop, 1)
  1428. closeWg.Wait()
  1429. })
  1430. runtime.GOMAXPROCS(1)
  1431. }
  1432. func TestDb_Concurrent2(t *testing.T) {
  1433. const n, n2 = 4, 4000
  1434. runtime.GOMAXPROCS(n*2 + 2)
  1435. truno(t, &opt.Options{WriteBuffer: 30}, func(h *dbHarness) {
  1436. var closeWg sync.WaitGroup
  1437. var stop uint32
  1438. for i := 0; i < n; i++ {
  1439. closeWg.Add(1)
  1440. go func(i int) {
  1441. for k := 0; atomic.LoadUint32(&stop) == 0; k++ {
  1442. h.put(fmt.Sprintf("k%d", k), fmt.Sprintf("%d.%d.", k, i)+strings.Repeat("x", 10))
  1443. }
  1444. closeWg.Done()
  1445. }(i)
  1446. }
  1447. for i := 0; i < n; i++ {
  1448. closeWg.Add(1)
  1449. go func(i int) {
  1450. for k := 1000000; k < 0 || atomic.LoadUint32(&stop) == 0; k-- {
  1451. h.put(fmt.Sprintf("k%d", k), fmt.Sprintf("%d.%d.", k, i)+strings.Repeat("x", 10))
  1452. }
  1453. closeWg.Done()
  1454. }(i)
  1455. }
  1456. cmp := comparer.DefaultComparer
  1457. for i := 0; i < n2; i++ {
  1458. closeWg.Add(1)
  1459. go func(i int) {
  1460. it := h.db.NewIterator(nil, nil)
  1461. var pk []byte
  1462. for it.Next() {
  1463. kk := it.Key()
  1464. if cmp.Compare(kk, pk) <= 0 {
  1465. t.Errorf("iter %d: %q is successor of %q", i, pk, kk)
  1466. }
  1467. pk = append(pk[:0], kk...)
  1468. var k, vk, vi int
  1469. if n, err := fmt.Sscanf(string(it.Key()), "k%d", &k); err != nil {
  1470. t.Errorf("iter %d: Scanf error on key %q: %v", i, it.Key(), err)
  1471. } else if n < 1 {
  1472. t.Errorf("iter %d: Cannot parse key %q", i, it.Key())
  1473. }
  1474. if n, err := fmt.Sscanf(string(it.Value()), "%d.%d", &vk, &vi); err != nil {
  1475. t.Errorf("iter %d: Scanf error on value %q: %v", i, it.Value(), err)
  1476. } else if n < 2 {
  1477. t.Errorf("iter %d: Cannot parse value %q", i, it.Value())
  1478. }
  1479. if vk != k {
  1480. t.Errorf("iter %d: invalid value i=%d, want=%d got=%d", i, vi, k, vk)
  1481. }
  1482. }
  1483. if err := it.Error(); err != nil {
  1484. t.Errorf("iter %d: Got error: %v", i, err)
  1485. }
  1486. it.Release()
  1487. closeWg.Done()
  1488. }(i)
  1489. }
  1490. atomic.StoreUint32(&stop, 1)
  1491. closeWg.Wait()
  1492. })
  1493. runtime.GOMAXPROCS(1)
  1494. }
  1495. func TestDb_CreateReopenDbOnFile(t *testing.T) {
  1496. dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile-%d", os.Getuid()))
  1497. if err := os.RemoveAll(dbpath); err != nil {
  1498. t.Fatal("cannot remove old db: ", err)
  1499. }
  1500. defer os.RemoveAll(dbpath)
  1501. for i := 0; i < 3; i++ {
  1502. stor, err := storage.OpenFile(dbpath)
  1503. if err != nil {
  1504. t.Fatalf("(%d) cannot open storage: %s", i, err)
  1505. }
  1506. db, err := Open(stor, nil)
  1507. if err != nil {
  1508. t.Fatalf("(%d) cannot open db: %s", i, err)
  1509. }
  1510. if err := db.Put([]byte("foo"), []byte("bar"), nil); err != nil {
  1511. t.Fatalf("(%d) cannot write to db: %s", i, err)
  1512. }
  1513. if err := db.Close(); err != nil {
  1514. t.Fatalf("(%d) cannot close db: %s", i, err)
  1515. }
  1516. if err := stor.Close(); err != nil {
  1517. t.Fatalf("(%d) cannot close storage: %s", i, err)
  1518. }
  1519. }
  1520. }
  1521. func TestDb_CreateReopenDbOnFile2(t *testing.T) {
  1522. dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile2-%d", os.Getuid()))
  1523. if err := os.RemoveAll(dbpath); err != nil {
  1524. t.Fatal("cannot remove old db: ", err)
  1525. }
  1526. defer os.RemoveAll(dbpath)
  1527. for i := 0; i < 3; i++ {
  1528. db, err := OpenFile(dbpath, nil)
  1529. if err != nil {
  1530. t.Fatalf("(%d) cannot open db: %s", i, err)
  1531. }
  1532. if err := db.Put([]byte("foo"), []byte("bar"), nil); err != nil {
  1533. t.Fatalf("(%d) cannot write to db: %s", i, err)
  1534. }
  1535. if err := db.Close(); err != nil {
  1536. t.Fatalf("(%d) cannot close db: %s", i, err)
  1537. }
  1538. }
  1539. }
  1540. func TestDb_DeletionMarkersOnMemdb(t *testing.T) {
  1541. h := newDbHarness(t)
  1542. defer h.close()
  1543. h.put("foo", "v1")
  1544. h.compactMem()
  1545. h.delete("foo")
  1546. h.get("foo", false)
  1547. h.getKeyVal("")
  1548. }
  1549. func TestDb_LeveldbIssue178(t *testing.T) {
  1550. nKeys := (kMaxTableSize / 30) * 5
  1551. key1 := func(i int) string {
  1552. return fmt.Sprintf("my_key_%d", i)
  1553. }
  1554. key2 := func(i int) string {
  1555. return fmt.Sprintf("my_key_%d_xxx", i)
  1556. }
  1557. // Disable compression since it affects the creation of layers and the
  1558. // code below is trying to test against a very specific scenario.
  1559. h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression})
  1560. defer h.close()
  1561. // Create first key range.
  1562. batch := new(Batch)
  1563. for i := 0; i < nKeys; i++ {
  1564. batch.Put([]byte(key1(i)), []byte("value for range 1 key"))
  1565. }
  1566. h.write(batch)
  1567. // Create second key range.
  1568. batch.Reset()
  1569. for i := 0; i < nKeys; i++ {
  1570. batch.Put([]byte(key2(i)), []byte("value for range 2 key"))
  1571. }
  1572. h.write(batch)
  1573. // Delete second key range.
  1574. batch.Reset()
  1575. for i := 0; i < nKeys; i++ {
  1576. batch.Delete([]byte(key2(i)))
  1577. }
  1578. h.write(batch)
  1579. h.waitMemCompaction()
  1580. // Run manual compaction.
  1581. h.compactRange(key1(0), key1(nKeys-1))
  1582. // Checking the keys.
  1583. h.assertNumKeys(nKeys)
  1584. }
  1585. func TestDb_LeveldbIssue200(t *testing.T) {
  1586. h := newDbHarness(t)
  1587. defer h.close()
  1588. h.put("1", "b")
  1589. h.put("2", "c")
  1590. h.put("3", "d")
  1591. h.put("4", "e")
  1592. h.put("5", "f")
  1593. iter := h.db.NewIterator(nil, h.ro)
  1594. // Add an element that should not be reflected in the iterator.
  1595. h.put("25", "cd")
  1596. iter.Seek([]byte("5"))
  1597. assertBytes(t, []byte("5"), iter.Key())
  1598. iter.Prev()
  1599. assertBytes(t, []byte("4"), iter.Key())
  1600. iter.Prev()
  1601. assertBytes(t, []byte("3"), iter.Key())
  1602. iter.Next()
  1603. assertBytes(t, []byte("4"), iter.Key())
  1604. iter.Next()
  1605. assertBytes(t, []byte("5"), iter.Key())
  1606. }