PageRenderTime 31ms CodeModel.GetById 19ms RepoModel.GetById 0ms app.codeStats 1ms

/vendor/github.com/syndtr/goleveldb/leveldb/db_test.go

https://gitlab.com/0072016/syncthing
Go | 2213 lines | 1816 code | 321 blank | 76 comment | 381 complexity | 9a1b8178e99a6a3e6026266bd3d5fa0d MD5 | raw file
  1. // Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
  2. // All rights reserved.
  3. //
  4. // Use of this source code is governed by a BSD-style license that can be
  5. // found in the LICENSE file.
  6. package leveldb
  7. import (
  8. "bytes"
  9. "container/list"
  10. crand "crypto/rand"
  11. "encoding/binary"
  12. "fmt"
  13. "math/rand"
  14. "os"
  15. "path/filepath"
  16. "runtime"
  17. "strings"
  18. "sync"
  19. "sync/atomic"
  20. "testing"
  21. "time"
  22. "unsafe"
  23. "github.com/onsi/gomega"
  24. "github.com/syndtr/goleveldb/leveldb/comparer"
  25. "github.com/syndtr/goleveldb/leveldb/errors"
  26. "github.com/syndtr/goleveldb/leveldb/filter"
  27. "github.com/syndtr/goleveldb/leveldb/iterator"
  28. "github.com/syndtr/goleveldb/leveldb/opt"
  29. "github.com/syndtr/goleveldb/leveldb/storage"
  30. "github.com/syndtr/goleveldb/leveldb/testutil"
  31. "github.com/syndtr/goleveldb/leveldb/util"
  32. )
  33. func tkey(i int) []byte {
  34. return []byte(fmt.Sprintf("%016d", i))
  35. }
  36. func tval(seed, n int) []byte {
  37. r := rand.New(rand.NewSource(int64(seed)))
  38. return randomString(r, n)
  39. }
  40. func testingLogger(t *testing.T) func(log string) {
  41. return func(log string) {
  42. t.Log(log)
  43. }
  44. }
  45. func testingPreserveOnFailed(t *testing.T) func() (preserve bool, err error) {
  46. return func() (preserve bool, err error) {
  47. preserve = t.Failed()
  48. return
  49. }
  50. }
  51. type dbHarness struct {
  52. t *testing.T
  53. stor *testutil.Storage
  54. db *DB
  55. o *opt.Options
  56. ro *opt.ReadOptions
  57. wo *opt.WriteOptions
  58. }
  59. func newDbHarnessWopt(t *testing.T, o *opt.Options) *dbHarness {
  60. h := new(dbHarness)
  61. h.init(t, o)
  62. return h
  63. }
  64. func newDbHarness(t *testing.T) *dbHarness {
  65. return newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true})
  66. }
  67. func (h *dbHarness) init(t *testing.T, o *opt.Options) {
  68. gomega.RegisterTestingT(t)
  69. h.t = t
  70. h.stor = testutil.NewStorage()
  71. h.stor.OnLog(testingLogger(t))
  72. h.stor.OnClose(testingPreserveOnFailed(t))
  73. h.o = o
  74. h.ro = nil
  75. h.wo = nil
  76. if err := h.openDB0(); err != nil {
  77. // So that it will come after fatal message.
  78. defer h.stor.Close()
  79. h.t.Fatal("Open (init): got error: ", err)
  80. }
  81. }
  82. func (h *dbHarness) openDB0() (err error) {
  83. h.t.Log("opening DB")
  84. h.db, err = Open(h.stor, h.o)
  85. return
  86. }
  87. func (h *dbHarness) openDB() {
  88. if err := h.openDB0(); err != nil {
  89. h.t.Fatal("Open: got error: ", err)
  90. }
  91. }
  92. func (h *dbHarness) closeDB0() error {
  93. h.t.Log("closing DB")
  94. return h.db.Close()
  95. }
  96. func (h *dbHarness) closeDB() {
  97. if h.db != nil {
  98. if err := h.closeDB0(); err != nil {
  99. h.t.Error("Close: got error: ", err)
  100. }
  101. h.db = nil
  102. }
  103. h.stor.CloseCheck()
  104. runtime.GC()
  105. }
  106. func (h *dbHarness) reopenDB() {
  107. if h.db != nil {
  108. h.closeDB()
  109. }
  110. h.openDB()
  111. }
  112. func (h *dbHarness) close() {
  113. if h.db != nil {
  114. h.closeDB0()
  115. h.db = nil
  116. }
  117. h.stor.Close()
  118. h.stor = nil
  119. runtime.GC()
  120. }
  121. func (h *dbHarness) openAssert(want bool) {
  122. db, err := Open(h.stor, h.o)
  123. if err != nil {
  124. if want {
  125. h.t.Error("Open: assert: got error: ", err)
  126. } else {
  127. h.t.Log("Open: assert: got error (expected): ", err)
  128. }
  129. } else {
  130. if !want {
  131. h.t.Error("Open: assert: expect error")
  132. }
  133. db.Close()
  134. }
  135. }
  136. func (h *dbHarness) write(batch *Batch) {
  137. if err := h.db.Write(batch, h.wo); err != nil {
  138. h.t.Error("Write: got error: ", err)
  139. }
  140. }
  141. func (h *dbHarness) put(key, value string) {
  142. if err := h.db.Put([]byte(key), []byte(value), h.wo); err != nil {
  143. h.t.Error("Put: got error: ", err)
  144. }
  145. }
  146. func (h *dbHarness) putMulti(n int, low, hi string) {
  147. for i := 0; i < n; i++ {
  148. h.put(low, "begin")
  149. h.put(hi, "end")
  150. h.compactMem()
  151. }
  152. }
  153. func (h *dbHarness) maxNextLevelOverlappingBytes(want int64) {
  154. t := h.t
  155. db := h.db
  156. var (
  157. maxOverlaps int64
  158. maxLevel int
  159. )
  160. v := db.s.version()
  161. if len(v.levels) > 2 {
  162. for i, tt := range v.levels[1 : len(v.levels)-1] {
  163. level := i + 1
  164. next := v.levels[level+1]
  165. for _, t := range tt {
  166. r := next.getOverlaps(nil, db.s.icmp, t.imin.ukey(), t.imax.ukey(), false)
  167. sum := r.size()
  168. if sum > maxOverlaps {
  169. maxOverlaps = sum
  170. maxLevel = level
  171. }
  172. }
  173. }
  174. }
  175. v.release()
  176. if maxOverlaps > want {
  177. t.Errorf("next level most overlapping bytes is more than %d, got=%d level=%d", want, maxOverlaps, maxLevel)
  178. } else {
  179. t.Logf("next level most overlapping bytes is %d, level=%d want=%d", maxOverlaps, maxLevel, want)
  180. }
  181. }
  182. func (h *dbHarness) delete(key string) {
  183. t := h.t
  184. db := h.db
  185. err := db.Delete([]byte(key), h.wo)
  186. if err != nil {
  187. t.Error("Delete: got error: ", err)
  188. }
  189. }
  190. func (h *dbHarness) assertNumKeys(want int) {
  191. iter := h.db.NewIterator(nil, h.ro)
  192. defer iter.Release()
  193. got := 0
  194. for iter.Next() {
  195. got++
  196. }
  197. if err := iter.Error(); err != nil {
  198. h.t.Error("assertNumKeys: ", err)
  199. }
  200. if want != got {
  201. h.t.Errorf("assertNumKeys: want=%d got=%d", want, got)
  202. }
  203. }
  204. func (h *dbHarness) getr(db Reader, key string, expectFound bool) (found bool, v []byte) {
  205. t := h.t
  206. v, err := db.Get([]byte(key), h.ro)
  207. switch err {
  208. case ErrNotFound:
  209. if expectFound {
  210. t.Errorf("Get: key '%s' not found, want found", key)
  211. }
  212. case nil:
  213. found = true
  214. if !expectFound {
  215. t.Errorf("Get: key '%s' found, want not found", key)
  216. }
  217. default:
  218. t.Error("Get: got error: ", err)
  219. }
  220. return
  221. }
  222. func (h *dbHarness) get(key string, expectFound bool) (found bool, v []byte) {
  223. return h.getr(h.db, key, expectFound)
  224. }
  225. func (h *dbHarness) getValr(db Reader, key, value string) {
  226. t := h.t
  227. found, r := h.getr(db, key, true)
  228. if !found {
  229. return
  230. }
  231. rval := string(r)
  232. if rval != value {
  233. t.Errorf("Get: invalid value, got '%s', want '%s'", rval, value)
  234. }
  235. }
  236. func (h *dbHarness) getVal(key, value string) {
  237. h.getValr(h.db, key, value)
  238. }
  239. func (h *dbHarness) allEntriesFor(key, want string) {
  240. t := h.t
  241. db := h.db
  242. s := db.s
  243. ikey := makeInternalKey(nil, []byte(key), keyMaxSeq, keyTypeVal)
  244. iter := db.newRawIterator(nil, nil, nil, nil)
  245. if !iter.Seek(ikey) && iter.Error() != nil {
  246. t.Error("AllEntries: error during seek, err: ", iter.Error())
  247. return
  248. }
  249. res := "[ "
  250. first := true
  251. for iter.Valid() {
  252. if ukey, _, kt, kerr := parseInternalKey(iter.Key()); kerr == nil {
  253. if s.icmp.uCompare(ikey.ukey(), ukey) != 0 {
  254. break
  255. }
  256. if !first {
  257. res += ", "
  258. }
  259. first = false
  260. switch kt {
  261. case keyTypeVal:
  262. res += string(iter.Value())
  263. case keyTypeDel:
  264. res += "DEL"
  265. }
  266. } else {
  267. if !first {
  268. res += ", "
  269. }
  270. first = false
  271. res += "CORRUPTED"
  272. }
  273. iter.Next()
  274. }
  275. if !first {
  276. res += " "
  277. }
  278. res += "]"
  279. if res != want {
  280. t.Errorf("AllEntries: assert failed for key %q, got=%q want=%q", key, res, want)
  281. }
  282. }
  283. // Return a string that contains all key,value pairs in order,
  284. // formatted like "(k1->v1)(k2->v2)".
  285. func (h *dbHarness) getKeyVal(want string) {
  286. t := h.t
  287. db := h.db
  288. s, err := db.GetSnapshot()
  289. if err != nil {
  290. t.Fatal("GetSnapshot: got error: ", err)
  291. }
  292. res := ""
  293. iter := s.NewIterator(nil, nil)
  294. for iter.Next() {
  295. res += fmt.Sprintf("(%s->%s)", string(iter.Key()), string(iter.Value()))
  296. }
  297. iter.Release()
  298. if res != want {
  299. t.Errorf("GetKeyVal: invalid key/value pair, got=%q want=%q", res, want)
  300. }
  301. s.Release()
  302. }
  303. func (h *dbHarness) waitCompaction() {
  304. t := h.t
  305. db := h.db
  306. if err := db.compTriggerWait(db.tcompCmdC); err != nil {
  307. t.Error("compaction error: ", err)
  308. }
  309. }
  310. func (h *dbHarness) waitMemCompaction() {
  311. t := h.t
  312. db := h.db
  313. if err := db.compTriggerWait(db.mcompCmdC); err != nil {
  314. t.Error("compaction error: ", err)
  315. }
  316. }
  317. func (h *dbHarness) compactMem() {
  318. t := h.t
  319. db := h.db
  320. t.Log("starting memdb compaction")
  321. db.writeLockC <- struct{}{}
  322. defer func() {
  323. <-db.writeLockC
  324. }()
  325. if _, err := db.rotateMem(0, true); err != nil {
  326. t.Error("compaction error: ", err)
  327. }
  328. if h.totalTables() == 0 {
  329. t.Error("zero tables after mem compaction")
  330. }
  331. t.Log("memdb compaction done")
  332. }
  333. func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool) {
  334. t := h.t
  335. db := h.db
  336. var _min, _max []byte
  337. if min != "" {
  338. _min = []byte(min)
  339. }
  340. if max != "" {
  341. _max = []byte(max)
  342. }
  343. t.Logf("starting table range compaction: level=%d, min=%q, max=%q", level, min, max)
  344. if err := db.compTriggerRange(db.tcompCmdC, level, _min, _max); err != nil {
  345. if wanterr {
  346. t.Log("CompactRangeAt: got error (expected): ", err)
  347. } else {
  348. t.Error("CompactRangeAt: got error: ", err)
  349. }
  350. } else if wanterr {
  351. t.Error("CompactRangeAt: expect error")
  352. }
  353. t.Log("table range compaction done")
  354. }
  355. func (h *dbHarness) compactRangeAt(level int, min, max string) {
  356. h.compactRangeAtErr(level, min, max, false)
  357. }
  358. func (h *dbHarness) compactRange(min, max string) {
  359. t := h.t
  360. db := h.db
  361. t.Logf("starting DB range compaction: min=%q, max=%q", min, max)
  362. var r util.Range
  363. if min != "" {
  364. r.Start = []byte(min)
  365. }
  366. if max != "" {
  367. r.Limit = []byte(max)
  368. }
  369. if err := db.CompactRange(r); err != nil {
  370. t.Error("CompactRange: got error: ", err)
  371. }
  372. t.Log("DB range compaction done")
  373. }
  374. func (h *dbHarness) sizeOf(start, limit string) int64 {
  375. sz, err := h.db.SizeOf([]util.Range{
  376. {[]byte(start), []byte(limit)},
  377. })
  378. if err != nil {
  379. h.t.Error("SizeOf: got error: ", err)
  380. }
  381. return sz.Sum()
  382. }
  383. func (h *dbHarness) sizeAssert(start, limit string, low, hi int64) {
  384. sz := h.sizeOf(start, limit)
  385. if sz < low || sz > hi {
  386. h.t.Errorf("sizeOf %q to %q not in range, want %d - %d, got %d",
  387. shorten(start), shorten(limit), low, hi, sz)
  388. }
  389. }
  390. func (h *dbHarness) getSnapshot() (s *Snapshot) {
  391. s, err := h.db.GetSnapshot()
  392. if err != nil {
  393. h.t.Fatal("GetSnapshot: got error: ", err)
  394. }
  395. return
  396. }
  397. func (h *dbHarness) getTablesPerLevel() string {
  398. res := ""
  399. nz := 0
  400. v := h.db.s.version()
  401. for level, tables := range v.levels {
  402. if level > 0 {
  403. res += ","
  404. }
  405. res += fmt.Sprint(len(tables))
  406. if len(tables) > 0 {
  407. nz = len(res)
  408. }
  409. }
  410. v.release()
  411. return res[:nz]
  412. }
  413. func (h *dbHarness) tablesPerLevel(want string) {
  414. res := h.getTablesPerLevel()
  415. if res != want {
  416. h.t.Errorf("invalid tables len, want=%s, got=%s", want, res)
  417. }
  418. }
  419. func (h *dbHarness) totalTables() (n int) {
  420. v := h.db.s.version()
  421. for _, tables := range v.levels {
  422. n += len(tables)
  423. }
  424. v.release()
  425. return
  426. }
  427. type keyValue interface {
  428. Key() []byte
  429. Value() []byte
  430. }
  431. func testKeyVal(t *testing.T, kv keyValue, want string) {
  432. res := string(kv.Key()) + "->" + string(kv.Value())
  433. if res != want {
  434. t.Errorf("invalid key/value, want=%q, got=%q", want, res)
  435. }
  436. }
  437. func numKey(num int) string {
  438. return fmt.Sprintf("key%06d", num)
  439. }
  440. var testingBloomFilter = filter.NewBloomFilter(10)
  441. func truno(t *testing.T, o *opt.Options, f func(h *dbHarness)) {
  442. for i := 0; i < 4; i++ {
  443. func() {
  444. switch i {
  445. case 0:
  446. case 1:
  447. if o == nil {
  448. o = &opt.Options{
  449. DisableLargeBatchTransaction: true,
  450. Filter: testingBloomFilter,
  451. }
  452. } else {
  453. old := o
  454. o = &opt.Options{}
  455. *o = *old
  456. o.Filter = testingBloomFilter
  457. }
  458. case 2:
  459. if o == nil {
  460. o = &opt.Options{
  461. DisableLargeBatchTransaction: true,
  462. Compression: opt.NoCompression,
  463. }
  464. } else {
  465. old := o
  466. o = &opt.Options{}
  467. *o = *old
  468. o.Compression = opt.NoCompression
  469. }
  470. }
  471. h := newDbHarnessWopt(t, o)
  472. defer h.close()
  473. switch i {
  474. case 3:
  475. h.reopenDB()
  476. }
  477. f(h)
  478. }()
  479. }
  480. }
  481. func trun(t *testing.T, f func(h *dbHarness)) {
  482. truno(t, nil, f)
  483. }
  484. func testAligned(t *testing.T, name string, offset uintptr) {
  485. if offset%8 != 0 {
  486. t.Errorf("field %s offset is not 64-bit aligned", name)
  487. }
  488. }
  489. func Test_FieldsAligned(t *testing.T) {
  490. p1 := new(DB)
  491. testAligned(t, "DB.seq", unsafe.Offsetof(p1.seq))
  492. p2 := new(session)
  493. testAligned(t, "session.stNextFileNum", unsafe.Offsetof(p2.stNextFileNum))
  494. testAligned(t, "session.stJournalNum", unsafe.Offsetof(p2.stJournalNum))
  495. testAligned(t, "session.stPrevJournalNum", unsafe.Offsetof(p2.stPrevJournalNum))
  496. testAligned(t, "session.stSeqNum", unsafe.Offsetof(p2.stSeqNum))
  497. }
  498. func TestDB_Locking(t *testing.T) {
  499. h := newDbHarness(t)
  500. defer h.stor.Close()
  501. h.openAssert(false)
  502. h.closeDB()
  503. h.openAssert(true)
  504. }
  505. func TestDB_Empty(t *testing.T) {
  506. trun(t, func(h *dbHarness) {
  507. h.get("foo", false)
  508. h.reopenDB()
  509. h.get("foo", false)
  510. })
  511. }
  512. func TestDB_ReadWrite(t *testing.T) {
  513. trun(t, func(h *dbHarness) {
  514. h.put("foo", "v1")
  515. h.getVal("foo", "v1")
  516. h.put("bar", "v2")
  517. h.put("foo", "v3")
  518. h.getVal("foo", "v3")
  519. h.getVal("bar", "v2")
  520. h.reopenDB()
  521. h.getVal("foo", "v3")
  522. h.getVal("bar", "v2")
  523. })
  524. }
  525. func TestDB_PutDeleteGet(t *testing.T) {
  526. trun(t, func(h *dbHarness) {
  527. h.put("foo", "v1")
  528. h.getVal("foo", "v1")
  529. h.put("foo", "v2")
  530. h.getVal("foo", "v2")
  531. h.delete("foo")
  532. h.get("foo", false)
  533. h.reopenDB()
  534. h.get("foo", false)
  535. })
  536. }
  537. func TestDB_EmptyBatch(t *testing.T) {
  538. h := newDbHarness(t)
  539. defer h.close()
  540. h.get("foo", false)
  541. err := h.db.Write(new(Batch), h.wo)
  542. if err != nil {
  543. t.Error("writing empty batch yield error: ", err)
  544. }
  545. h.get("foo", false)
  546. }
  547. func TestDB_GetFromFrozen(t *testing.T) {
  548. h := newDbHarnessWopt(t, &opt.Options{
  549. DisableLargeBatchTransaction: true,
  550. WriteBuffer: 100100,
  551. })
  552. defer h.close()
  553. h.put("foo", "v1")
  554. h.getVal("foo", "v1")
  555. h.stor.Stall(testutil.ModeSync, storage.TypeTable) // Block sync calls
  556. h.put("k1", strings.Repeat("x", 100000)) // Fill memtable
  557. h.put("k2", strings.Repeat("y", 100000)) // Trigger compaction
  558. for i := 0; h.db.getFrozenMem() == nil && i < 100; i++ {
  559. time.Sleep(10 * time.Microsecond)
  560. }
  561. if h.db.getFrozenMem() == nil {
  562. h.stor.Release(testutil.ModeSync, storage.TypeTable)
  563. t.Fatal("No frozen mem")
  564. }
  565. h.getVal("foo", "v1")
  566. h.stor.Release(testutil.ModeSync, storage.TypeTable) // Release sync calls
  567. h.reopenDB()
  568. h.getVal("foo", "v1")
  569. h.get("k1", true)
  570. h.get("k2", true)
  571. }
  572. func TestDB_GetFromTable(t *testing.T) {
  573. trun(t, func(h *dbHarness) {
  574. h.put("foo", "v1")
  575. h.compactMem()
  576. h.getVal("foo", "v1")
  577. })
  578. }
  579. func TestDB_GetSnapshot(t *testing.T) {
  580. trun(t, func(h *dbHarness) {
  581. bar := strings.Repeat("b", 200)
  582. h.put("foo", "v1")
  583. h.put(bar, "v1")
  584. snap, err := h.db.GetSnapshot()
  585. if err != nil {
  586. t.Fatal("GetSnapshot: got error: ", err)
  587. }
  588. h.put("foo", "v2")
  589. h.put(bar, "v2")
  590. h.getVal("foo", "v2")
  591. h.getVal(bar, "v2")
  592. h.getValr(snap, "foo", "v1")
  593. h.getValr(snap, bar, "v1")
  594. h.compactMem()
  595. h.getVal("foo", "v2")
  596. h.getVal(bar, "v2")
  597. h.getValr(snap, "foo", "v1")
  598. h.getValr(snap, bar, "v1")
  599. snap.Release()
  600. h.reopenDB()
  601. h.getVal("foo", "v2")
  602. h.getVal(bar, "v2")
  603. })
  604. }
  605. func TestDB_GetLevel0Ordering(t *testing.T) {
  606. trun(t, func(h *dbHarness) {
  607. h.db.memdbMaxLevel = 2
  608. for i := 0; i < 4; i++ {
  609. h.put("bar", fmt.Sprintf("b%d", i))
  610. h.put("foo", fmt.Sprintf("v%d", i))
  611. h.compactMem()
  612. }
  613. h.getVal("foo", "v3")
  614. h.getVal("bar", "b3")
  615. v := h.db.s.version()
  616. t0len := v.tLen(0)
  617. v.release()
  618. if t0len < 2 {
  619. t.Errorf("level-0 tables is less than 2, got %d", t0len)
  620. }
  621. h.reopenDB()
  622. h.getVal("foo", "v3")
  623. h.getVal("bar", "b3")
  624. })
  625. }
  626. func TestDB_GetOrderedByLevels(t *testing.T) {
  627. trun(t, func(h *dbHarness) {
  628. h.put("foo", "v1")
  629. h.compactMem()
  630. h.compactRange("a", "z")
  631. h.getVal("foo", "v1")
  632. h.put("foo", "v2")
  633. h.compactMem()
  634. h.getVal("foo", "v2")
  635. })
  636. }
  637. func TestDB_GetPicksCorrectFile(t *testing.T) {
  638. trun(t, func(h *dbHarness) {
  639. // Arrange to have multiple files in a non-level-0 level.
  640. h.put("a", "va")
  641. h.compactMem()
  642. h.compactRange("a", "b")
  643. h.put("x", "vx")
  644. h.compactMem()
  645. h.compactRange("x", "y")
  646. h.put("f", "vf")
  647. h.compactMem()
  648. h.compactRange("f", "g")
  649. h.getVal("a", "va")
  650. h.getVal("f", "vf")
  651. h.getVal("x", "vx")
  652. h.compactRange("", "")
  653. h.getVal("a", "va")
  654. h.getVal("f", "vf")
  655. h.getVal("x", "vx")
  656. })
  657. }
  658. func TestDB_GetEncountersEmptyLevel(t *testing.T) {
  659. trun(t, func(h *dbHarness) {
  660. h.db.memdbMaxLevel = 2
  661. // Arrange for the following to happen:
  662. // * sstable A in level 0
  663. // * nothing in level 1
  664. // * sstable B in level 2
  665. // Then do enough Get() calls to arrange for an automatic compaction
  666. // of sstable A. A bug would cause the compaction to be marked as
  667. // occuring at level 1 (instead of the correct level 0).
  668. // Step 1: First place sstables in levels 0 and 2
  669. for i := 0; ; i++ {
  670. if i >= 100 {
  671. t.Fatal("could not fill levels-0 and level-2")
  672. }
  673. v := h.db.s.version()
  674. if v.tLen(0) > 0 && v.tLen(2) > 0 {
  675. v.release()
  676. break
  677. }
  678. v.release()
  679. h.put("a", "begin")
  680. h.put("z", "end")
  681. h.compactMem()
  682. h.getVal("a", "begin")
  683. h.getVal("z", "end")
  684. }
  685. // Step 2: clear level 1 if necessary.
  686. h.compactRangeAt(1, "", "")
  687. h.tablesPerLevel("1,0,1")
  688. h.getVal("a", "begin")
  689. h.getVal("z", "end")
  690. // Step 3: read a bunch of times
  691. for i := 0; i < 200; i++ {
  692. h.get("missing", false)
  693. }
  694. // Step 4: Wait for compaction to finish
  695. h.waitCompaction()
  696. v := h.db.s.version()
  697. if v.tLen(0) > 0 {
  698. t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
  699. }
  700. v.release()
  701. h.getVal("a", "begin")
  702. h.getVal("z", "end")
  703. })
  704. }
  705. func TestDB_IterMultiWithDelete(t *testing.T) {
  706. trun(t, func(h *dbHarness) {
  707. h.put("a", "va")
  708. h.put("b", "vb")
  709. h.put("c", "vc")
  710. h.delete("b")
  711. h.get("b", false)
  712. iter := h.db.NewIterator(nil, nil)
  713. iter.Seek([]byte("c"))
  714. testKeyVal(t, iter, "c->vc")
  715. iter.Prev()
  716. testKeyVal(t, iter, "a->va")
  717. iter.Release()
  718. h.compactMem()
  719. iter = h.db.NewIterator(nil, nil)
  720. iter.Seek([]byte("c"))
  721. testKeyVal(t, iter, "c->vc")
  722. iter.Prev()
  723. testKeyVal(t, iter, "a->va")
  724. iter.Release()
  725. })
  726. }
  727. func TestDB_IteratorPinsRef(t *testing.T) {
  728. h := newDbHarness(t)
  729. defer h.close()
  730. h.put("foo", "hello")
  731. // Get iterator that will yield the current contents of the DB.
  732. iter := h.db.NewIterator(nil, nil)
  733. // Write to force compactions
  734. h.put("foo", "newvalue1")
  735. for i := 0; i < 100; i++ {
  736. h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
  737. }
  738. h.put("foo", "newvalue2")
  739. iter.First()
  740. testKeyVal(t, iter, "foo->hello")
  741. if iter.Next() {
  742. t.Errorf("expect eof")
  743. }
  744. iter.Release()
  745. }
  746. func TestDB_Recover(t *testing.T) {
  747. trun(t, func(h *dbHarness) {
  748. h.put("foo", "v1")
  749. h.put("baz", "v5")
  750. h.reopenDB()
  751. h.getVal("foo", "v1")
  752. h.getVal("foo", "v1")
  753. h.getVal("baz", "v5")
  754. h.put("bar", "v2")
  755. h.put("foo", "v3")
  756. h.reopenDB()
  757. h.getVal("foo", "v3")
  758. h.put("foo", "v4")
  759. h.getVal("foo", "v4")
  760. h.getVal("bar", "v2")
  761. h.getVal("baz", "v5")
  762. })
  763. }
  764. func TestDB_RecoverWithEmptyJournal(t *testing.T) {
  765. trun(t, func(h *dbHarness) {
  766. h.put("foo", "v1")
  767. h.put("foo", "v2")
  768. h.reopenDB()
  769. h.reopenDB()
  770. h.put("foo", "v3")
  771. h.reopenDB()
  772. h.getVal("foo", "v3")
  773. })
  774. }
  775. func TestDB_RecoverDuringMemtableCompaction(t *testing.T) {
  776. truno(t, &opt.Options{DisableLargeBatchTransaction: true, WriteBuffer: 1000000}, func(h *dbHarness) {
  777. h.stor.Stall(testutil.ModeSync, storage.TypeTable)
  778. h.put("big1", strings.Repeat("x", 10000000))
  779. h.put("big2", strings.Repeat("y", 1000))
  780. h.put("bar", "v2")
  781. h.stor.Release(testutil.ModeSync, storage.TypeTable)
  782. h.reopenDB()
  783. h.getVal("bar", "v2")
  784. h.getVal("big1", strings.Repeat("x", 10000000))
  785. h.getVal("big2", strings.Repeat("y", 1000))
  786. })
  787. }
  788. func TestDB_MinorCompactionsHappen(t *testing.T) {
  789. h := newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true, WriteBuffer: 10000})
  790. defer h.close()
  791. n := 500
  792. key := func(i int) string {
  793. return fmt.Sprintf("key%06d", i)
  794. }
  795. for i := 0; i < n; i++ {
  796. h.put(key(i), key(i)+strings.Repeat("v", 1000))
  797. }
  798. for i := 0; i < n; i++ {
  799. h.getVal(key(i), key(i)+strings.Repeat("v", 1000))
  800. }
  801. h.reopenDB()
  802. for i := 0; i < n; i++ {
  803. h.getVal(key(i), key(i)+strings.Repeat("v", 1000))
  804. }
  805. }
  806. func TestDB_RecoverWithLargeJournal(t *testing.T) {
  807. h := newDbHarness(t)
  808. defer h.close()
  809. h.put("big1", strings.Repeat("1", 200000))
  810. h.put("big2", strings.Repeat("2", 200000))
  811. h.put("small3", strings.Repeat("3", 10))
  812. h.put("small4", strings.Repeat("4", 10))
  813. h.tablesPerLevel("")
  814. // Make sure that if we re-open with a small write buffer size that
  815. // we flush table files in the middle of a large journal file.
  816. h.o.WriteBuffer = 100000
  817. h.reopenDB()
  818. h.getVal("big1", strings.Repeat("1", 200000))
  819. h.getVal("big2", strings.Repeat("2", 200000))
  820. h.getVal("small3", strings.Repeat("3", 10))
  821. h.getVal("small4", strings.Repeat("4", 10))
  822. v := h.db.s.version()
  823. if v.tLen(0) <= 1 {
  824. t.Errorf("tables-0 less than one")
  825. }
  826. v.release()
  827. }
  828. func TestDB_CompactionsGenerateMultipleFiles(t *testing.T) {
  829. h := newDbHarnessWopt(t, &opt.Options{
  830. DisableLargeBatchTransaction: true,
  831. WriteBuffer: 10000000,
  832. Compression: opt.NoCompression,
  833. })
  834. defer h.close()
  835. v := h.db.s.version()
  836. if v.tLen(0) > 0 {
  837. t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
  838. }
  839. v.release()
  840. n := 80
  841. // Write 8MB (80 values, each 100K)
  842. for i := 0; i < n; i++ {
  843. h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
  844. }
  845. // Reopening moves updates to level-0
  846. h.reopenDB()
  847. h.compactRangeAt(0, "", "")
  848. v = h.db.s.version()
  849. if v.tLen(0) > 0 {
  850. t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
  851. }
  852. if v.tLen(1) <= 1 {
  853. t.Errorf("level-1 tables less than 1, got %d", v.tLen(1))
  854. }
  855. v.release()
  856. for i := 0; i < n; i++ {
  857. h.getVal(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
  858. }
  859. }
  860. func TestDB_RepeatedWritesToSameKey(t *testing.T) {
  861. h := newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true, WriteBuffer: 100000})
  862. defer h.close()
  863. maxTables := h.o.GetWriteL0PauseTrigger() + 7
  864. value := strings.Repeat("v", 2*h.o.GetWriteBuffer())
  865. for i := 0; i < 5*maxTables; i++ {
  866. h.put("key", value)
  867. n := h.totalTables()
  868. if n > maxTables {
  869. t.Errorf("total tables exceed %d, got=%d, iter=%d", maxTables, n, i)
  870. }
  871. }
  872. }
  873. func TestDB_RepeatedWritesToSameKeyAfterReopen(t *testing.T) {
  874. h := newDbHarnessWopt(t, &opt.Options{
  875. DisableLargeBatchTransaction: true,
  876. WriteBuffer: 100000,
  877. })
  878. defer h.close()
  879. h.reopenDB()
  880. maxTables := h.o.GetWriteL0PauseTrigger() + 7
  881. value := strings.Repeat("v", 2*h.o.GetWriteBuffer())
  882. for i := 0; i < 5*maxTables; i++ {
  883. h.put("key", value)
  884. n := h.totalTables()
  885. if n > maxTables {
  886. t.Errorf("total tables exceed %d, got=%d, iter=%d", maxTables, n, i)
  887. }
  888. }
  889. }
  890. func TestDB_SparseMerge(t *testing.T) {
  891. h := newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true, Compression: opt.NoCompression})
  892. defer h.close()
  893. h.putMulti(7, "A", "Z")
  894. // Suppose there is:
  895. // small amount of data with prefix A
  896. // large amount of data with prefix B
  897. // small amount of data with prefix C
  898. // and that recent updates have made small changes to all three prefixes.
  899. // Check that we do not do a compaction that merges all of B in one shot.
  900. h.put("A", "va")
  901. value := strings.Repeat("x", 1000)
  902. for i := 0; i < 100000; i++ {
  903. h.put(fmt.Sprintf("B%010d", i), value)
  904. }
  905. h.put("C", "vc")
  906. h.compactMem()
  907. h.compactRangeAt(0, "", "")
  908. h.waitCompaction()
  909. // Make sparse update
  910. h.put("A", "va2")
  911. h.put("B100", "bvalue2")
  912. h.put("C", "vc2")
  913. h.compactMem()
  914. h.waitCompaction()
  915. h.maxNextLevelOverlappingBytes(20 * 1048576)
  916. h.compactRangeAt(0, "", "")
  917. h.waitCompaction()
  918. h.maxNextLevelOverlappingBytes(20 * 1048576)
  919. h.compactRangeAt(1, "", "")
  920. h.waitCompaction()
  921. h.maxNextLevelOverlappingBytes(20 * 1048576)
  922. }
  923. func TestDB_SizeOf(t *testing.T) {
  924. h := newDbHarnessWopt(t, &opt.Options{
  925. DisableLargeBatchTransaction: true,
  926. Compression: opt.NoCompression,
  927. WriteBuffer: 10000000,
  928. })
  929. defer h.close()
  930. h.sizeAssert("", "xyz", 0, 0)
  931. h.reopenDB()
  932. h.sizeAssert("", "xyz", 0, 0)
  933. // Write 8MB (80 values, each 100K)
  934. n := 80
  935. s1 := 100000
  936. s2 := 105000
  937. for i := 0; i < n; i++ {
  938. h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), s1/10))
  939. }
  940. // 0 because SizeOf() does not account for memtable space
  941. h.sizeAssert("", numKey(50), 0, 0)
  942. for r := 0; r < 3; r++ {
  943. h.reopenDB()
  944. for cs := 0; cs < n; cs += 10 {
  945. for i := 0; i < n; i += 10 {
  946. h.sizeAssert("", numKey(i), int64(s1*i), int64(s2*i))
  947. h.sizeAssert("", numKey(i)+".suffix", int64(s1*(i+1)), int64(s2*(i+1)))
  948. h.sizeAssert(numKey(i), numKey(i+10), int64(s1*10), int64(s2*10))
  949. }
  950. h.sizeAssert("", numKey(50), int64(s1*50), int64(s2*50))
  951. h.sizeAssert("", numKey(50)+".suffix", int64(s1*50), int64(s2*50))
  952. h.compactRangeAt(0, numKey(cs), numKey(cs+9))
  953. }
  954. v := h.db.s.version()
  955. if v.tLen(0) != 0 {
  956. t.Errorf("level-0 tables was not zero, got %d", v.tLen(0))
  957. }
  958. if v.tLen(1) == 0 {
  959. t.Error("level-1 tables was zero")
  960. }
  961. v.release()
  962. }
  963. }
  964. func TestDB_SizeOf_MixOfSmallAndLarge(t *testing.T) {
  965. h := newDbHarnessWopt(t, &opt.Options{
  966. DisableLargeBatchTransaction: true,
  967. Compression: opt.NoCompression,
  968. })
  969. defer h.close()
  970. sizes := []int64{
  971. 10000,
  972. 10000,
  973. 100000,
  974. 10000,
  975. 100000,
  976. 10000,
  977. 300000,
  978. 10000,
  979. }
  980. for i, n := range sizes {
  981. h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), int(n)/10))
  982. }
  983. for r := 0; r < 3; r++ {
  984. h.reopenDB()
  985. var x int64
  986. for i, n := range sizes {
  987. y := x
  988. if i > 0 {
  989. y += 1000
  990. }
  991. h.sizeAssert("", numKey(i), x, y)
  992. x += n
  993. }
  994. h.sizeAssert(numKey(3), numKey(5), 110000, 111000)
  995. h.compactRangeAt(0, "", "")
  996. }
  997. }
  998. func TestDB_Snapshot(t *testing.T) {
  999. trun(t, func(h *dbHarness) {
  1000. h.put("foo", "v1")
  1001. s1 := h.getSnapshot()
  1002. h.put("foo", "v2")
  1003. s2 := h.getSnapshot()
  1004. h.put("foo", "v3")
  1005. s3 := h.getSnapshot()
  1006. h.put("foo", "v4")
  1007. h.getValr(s1, "foo", "v1")
  1008. h.getValr(s2, "foo", "v2")
  1009. h.getValr(s3, "foo", "v3")
  1010. h.getVal("foo", "v4")
  1011. s3.Release()
  1012. h.getValr(s1, "foo", "v1")
  1013. h.getValr(s2, "foo", "v2")
  1014. h.getVal("foo", "v4")
  1015. s1.Release()
  1016. h.getValr(s2, "foo", "v2")
  1017. h.getVal("foo", "v4")
  1018. s2.Release()
  1019. h.getVal("foo", "v4")
  1020. })
  1021. }
  1022. func TestDB_SnapshotList(t *testing.T) {
  1023. db := &DB{snapsList: list.New()}
  1024. e0a := db.acquireSnapshot()
  1025. e0b := db.acquireSnapshot()
  1026. db.seq = 1
  1027. e1 := db.acquireSnapshot()
  1028. db.seq = 2
  1029. e2 := db.acquireSnapshot()
  1030. if db.minSeq() != 0 {
  1031. t.Fatalf("invalid sequence number, got=%d", db.minSeq())
  1032. }
  1033. db.releaseSnapshot(e0a)
  1034. if db.minSeq() != 0 {
  1035. t.Fatalf("invalid sequence number, got=%d", db.minSeq())
  1036. }
  1037. db.releaseSnapshot(e2)
  1038. if db.minSeq() != 0 {
  1039. t.Fatalf("invalid sequence number, got=%d", db.minSeq())
  1040. }
  1041. db.releaseSnapshot(e0b)
  1042. if db.minSeq() != 1 {
  1043. t.Fatalf("invalid sequence number, got=%d", db.minSeq())
  1044. }
  1045. e2 = db.acquireSnapshot()
  1046. if db.minSeq() != 1 {
  1047. t.Fatalf("invalid sequence number, got=%d", db.minSeq())
  1048. }
  1049. db.releaseSnapshot(e1)
  1050. if db.minSeq() != 2 {
  1051. t.Fatalf("invalid sequence number, got=%d", db.minSeq())
  1052. }
  1053. db.releaseSnapshot(e2)
  1054. if db.minSeq() != 2 {
  1055. t.Fatalf("invalid sequence number, got=%d", db.minSeq())
  1056. }
  1057. }
  1058. func TestDB_HiddenValuesAreRemoved(t *testing.T) {
  1059. trun(t, func(h *dbHarness) {
  1060. s := h.db.s
  1061. m := 2
  1062. h.db.memdbMaxLevel = m
  1063. h.put("foo", "v1")
  1064. h.compactMem()
  1065. v := s.version()
  1066. num := v.tLen(m)
  1067. v.release()
  1068. if num != 1 {
  1069. t.Errorf("invalid level-%d len, want=1 got=%d", m, num)
  1070. }
  1071. // Place a table at level last-1 to prevent merging with preceding mutation
  1072. h.put("a", "begin")
  1073. h.put("z", "end")
  1074. h.compactMem()
  1075. v = s.version()
  1076. if v.tLen(m) != 1 {
  1077. t.Errorf("invalid level-%d len, want=1 got=%d", m, v.tLen(m))
  1078. }
  1079. if v.tLen(m-1) != 1 {
  1080. t.Errorf("invalid level-%d len, want=1 got=%d", m-1, v.tLen(m-1))
  1081. }
  1082. v.release()
  1083. h.delete("foo")
  1084. h.put("foo", "v2")
  1085. h.allEntriesFor("foo", "[ v2, DEL, v1 ]")
  1086. h.compactMem()
  1087. h.allEntriesFor("foo", "[ v2, DEL, v1 ]")
  1088. h.compactRangeAt(m-2, "", "z")
  1089. // DEL eliminated, but v1 remains because we aren't compacting that level
  1090. // (DEL can be eliminated because v2 hides v1).
  1091. h.allEntriesFor("foo", "[ v2, v1 ]")
  1092. h.compactRangeAt(m-1, "", "")
  1093. // Merging last-1 w/ last, so we are the base level for "foo", so
  1094. // DEL is removed. (as is v1).
  1095. h.allEntriesFor("foo", "[ v2 ]")
  1096. })
  1097. }
  1098. func TestDB_DeletionMarkers2(t *testing.T) {
  1099. h := newDbHarness(t)
  1100. defer h.close()
  1101. s := h.db.s
  1102. m := 2
  1103. h.db.memdbMaxLevel = m
  1104. h.put("foo", "v1")
  1105. h.compactMem()
  1106. v := s.version()
  1107. num := v.tLen(m)
  1108. v.release()
  1109. if num != 1 {
  1110. t.Errorf("invalid level-%d len, want=1 got=%d", m, num)
  1111. }
  1112. // Place a table at level last-1 to prevent merging with preceding mutation
  1113. h.put("a", "begin")
  1114. h.put("z", "end")
  1115. h.compactMem()
  1116. v = s.version()
  1117. if v.tLen(m) != 1 {
  1118. t.Errorf("invalid level-%d len, want=1 got=%d", m, v.tLen(m))
  1119. }
  1120. if v.tLen(m-1) != 1 {
  1121. t.Errorf("invalid level-%d len, want=1 got=%d", m-1, v.tLen(m-1))
  1122. }
  1123. v.release()
  1124. h.delete("foo")
  1125. h.allEntriesFor("foo", "[ DEL, v1 ]")
  1126. h.compactMem() // Moves to level last-2
  1127. h.allEntriesFor("foo", "[ DEL, v1 ]")
  1128. h.compactRangeAt(m-2, "", "")
  1129. // DEL kept: "last" file overlaps
  1130. h.allEntriesFor("foo", "[ DEL, v1 ]")
  1131. h.compactRangeAt(m-1, "", "")
  1132. // Merging last-1 w/ last, so we are the base level for "foo", so
  1133. // DEL is removed. (as is v1).
  1134. h.allEntriesFor("foo", "[ ]")
  1135. }
  1136. func TestDB_CompactionTableOpenError(t *testing.T) {
  1137. h := newDbHarnessWopt(t, &opt.Options{
  1138. DisableLargeBatchTransaction: true,
  1139. OpenFilesCacheCapacity: -1,
  1140. })
  1141. defer h.close()
  1142. h.db.memdbMaxLevel = 2
  1143. im := 10
  1144. jm := 10
  1145. for r := 0; r < 2; r++ {
  1146. for i := 0; i < im; i++ {
  1147. for j := 0; j < jm; j++ {
  1148. h.put(fmt.Sprintf("k%d,%d", i, j), fmt.Sprintf("v%d,%d", i, j))
  1149. }
  1150. h.compactMem()
  1151. }
  1152. }
  1153. if n := h.totalTables(); n != im*2 {
  1154. t.Errorf("total tables is %d, want %d", n, im*2)
  1155. }
  1156. h.stor.EmulateError(testutil.ModeOpen, storage.TypeTable, errors.New("open error during table compaction"))
  1157. go h.db.CompactRange(util.Range{})
  1158. if err := h.db.compTriggerWait(h.db.tcompCmdC); err != nil {
  1159. t.Log("compaction error: ", err)
  1160. }
  1161. h.closeDB0()
  1162. h.openDB()
  1163. h.stor.EmulateError(testutil.ModeOpen, storage.TypeTable, nil)
  1164. for i := 0; i < im; i++ {
  1165. for j := 0; j < jm; j++ {
  1166. h.getVal(fmt.Sprintf("k%d,%d", i, j), fmt.Sprintf("v%d,%d", i, j))
  1167. }
  1168. }
  1169. }
  1170. func TestDB_OverlapInLevel0(t *testing.T) {
  1171. trun(t, func(h *dbHarness) {
  1172. h.db.memdbMaxLevel = 2
  1173. // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
  1174. h.put("100", "v100")
  1175. h.put("999", "v999")
  1176. h.compactMem()
  1177. h.delete("100")
  1178. h.delete("999")
  1179. h.compactMem()
  1180. h.tablesPerLevel("0,1,1")
  1181. // Make files spanning the following ranges in level-0:
  1182. // files[0] 200 .. 900
  1183. // files[1] 300 .. 500
  1184. // Note that files are sorted by min key.
  1185. h.put("300", "v300")
  1186. h.put("500", "v500")
  1187. h.compactMem()
  1188. h.put("200", "v200")
  1189. h.put("600", "v600")
  1190. h.put("900", "v900")
  1191. h.compactMem()
  1192. h.tablesPerLevel("2,1,1")
  1193. // Compact away the placeholder files we created initially
  1194. h.compactRangeAt(1, "", "")
  1195. h.compactRangeAt(2, "", "")
  1196. h.tablesPerLevel("2")
  1197. // Do a memtable compaction. Before bug-fix, the compaction would
  1198. // not detect the overlap with level-0 files and would incorrectly place
  1199. // the deletion in a deeper level.
  1200. h.delete("600")
  1201. h.compactMem()
  1202. h.tablesPerLevel("3")
  1203. h.get("600", false)
  1204. })
  1205. }
  1206. func TestDB_L0_CompactionBug_Issue44_a(t *testing.T) {
  1207. h := newDbHarness(t)
  1208. defer h.close()
  1209. h.reopenDB()
  1210. h.put("b", "v")
  1211. h.reopenDB()
  1212. h.delete("b")
  1213. h.delete("a")
  1214. h.reopenDB()
  1215. h.delete("a")
  1216. h.reopenDB()
  1217. h.put("a", "v")
  1218. h.reopenDB()
  1219. h.reopenDB()
  1220. h.getKeyVal("(a->v)")
  1221. h.waitCompaction()
  1222. h.getKeyVal("(a->v)")
  1223. }
  1224. func TestDB_L0_CompactionBug_Issue44_b(t *testing.T) {
  1225. h := newDbHarness(t)
  1226. defer h.close()
  1227. h.reopenDB()
  1228. h.put("", "")
  1229. h.reopenDB()
  1230. h.delete("e")
  1231. h.put("", "")
  1232. h.reopenDB()
  1233. h.put("c", "cv")
  1234. h.reopenDB()
  1235. h.put("", "")
  1236. h.reopenDB()
  1237. h.put("", "")
  1238. h.waitCompaction()
  1239. h.reopenDB()
  1240. h.put("d", "dv")
  1241. h.reopenDB()
  1242. h.put("", "")
  1243. h.reopenDB()
  1244. h.delete("d")
  1245. h.delete("b")
  1246. h.reopenDB()
  1247. h.getKeyVal("(->)(c->cv)")
  1248. h.waitCompaction()
  1249. h.getKeyVal("(->)(c->cv)")
  1250. }
  1251. func TestDB_SingleEntryMemCompaction(t *testing.T) {
  1252. trun(t, func(h *dbHarness) {
  1253. for i := 0; i < 10; i++ {
  1254. h.put("big", strings.Repeat("v", opt.DefaultWriteBuffer))
  1255. h.compactMem()
  1256. h.put("key", strings.Repeat("v", opt.DefaultBlockSize))
  1257. h.compactMem()
  1258. h.put("k", "v")
  1259. h.compactMem()
  1260. h.put("", "")
  1261. h.compactMem()
  1262. h.put("verybig", strings.Repeat("v", opt.DefaultWriteBuffer*2))
  1263. h.compactMem()
  1264. }
  1265. })
  1266. }
  1267. func TestDB_ManifestWriteError(t *testing.T) {
  1268. for i := 0; i < 2; i++ {
  1269. func() {
  1270. h := newDbHarness(t)
  1271. defer h.close()
  1272. h.put("foo", "bar")
  1273. h.getVal("foo", "bar")
  1274. // Mem compaction (will succeed)
  1275. h.compactMem()
  1276. h.getVal("foo", "bar")
  1277. v := h.db.s.version()
  1278. if n := v.tLen(0); n != 1 {
  1279. t.Errorf("invalid total tables, want=1 got=%d", n)
  1280. }
  1281. v.release()
  1282. if i == 0 {
  1283. h.stor.EmulateError(testutil.ModeWrite, storage.TypeManifest, errors.New("manifest write error"))
  1284. } else {
  1285. h.stor.EmulateError(testutil.ModeSync, storage.TypeManifest, errors.New("manifest sync error"))
  1286. }
  1287. // Merging compaction (will fail)
  1288. h.compactRangeAtErr(0, "", "", true)
  1289. h.db.Close()
  1290. h.stor.EmulateError(testutil.ModeWrite, storage.TypeManifest, nil)
  1291. h.stor.EmulateError(testutil.ModeSync, storage.TypeManifest, nil)
  1292. // Should not lose data
  1293. h.openDB()
  1294. h.getVal("foo", "bar")
  1295. }()
  1296. }
  1297. }
  1298. func assertErr(t *testing.T, err error, wanterr bool) {
  1299. if err != nil {
  1300. if wanterr {
  1301. t.Log("AssertErr: got error (expected): ", err)
  1302. } else {
  1303. t.Error("AssertErr: got error: ", err)
  1304. }
  1305. } else if wanterr {
  1306. t.Error("AssertErr: expect error")
  1307. }
  1308. }
  1309. func TestDB_ClosedIsClosed(t *testing.T) {
  1310. h := newDbHarness(t)
  1311. db := h.db
  1312. var iter, iter2 iterator.Iterator
  1313. var snap *Snapshot
  1314. func() {
  1315. defer h.close()
  1316. h.put("k", "v")
  1317. h.getVal("k", "v")
  1318. iter = db.NewIterator(nil, h.ro)
  1319. iter.Seek([]byte("k"))
  1320. testKeyVal(t, iter, "k->v")
  1321. var err error
  1322. snap, err = db.GetSnapshot()
  1323. if err != nil {
  1324. t.Fatal("GetSnapshot: got error: ", err)
  1325. }
  1326. h.getValr(snap, "k", "v")
  1327. iter2 = snap.NewIterator(nil, h.ro)
  1328. iter2.Seek([]byte("k"))
  1329. testKeyVal(t, iter2, "k->v")
  1330. h.put("foo", "v2")
  1331. h.delete("foo")
  1332. // closing DB
  1333. iter.Release()
  1334. iter2.Release()
  1335. }()
  1336. assertErr(t, db.Put([]byte("x"), []byte("y"), h.wo), true)
  1337. _, err := db.Get([]byte("k"), h.ro)
  1338. assertErr(t, err, true)
  1339. if iter.Valid() {
  1340. t.Errorf("iter.Valid should false")
  1341. }
  1342. assertErr(t, iter.Error(), false)
  1343. testKeyVal(t, iter, "->")
  1344. if iter.Seek([]byte("k")) {
  1345. t.Errorf("iter.Seek should false")
  1346. }
  1347. assertErr(t, iter.Error(), true)
  1348. assertErr(t, iter2.Error(), false)
  1349. _, err = snap.Get([]byte("k"), h.ro)
  1350. assertErr(t, err, true)
  1351. _, err = db.GetSnapshot()
  1352. assertErr(t, err, true)
  1353. iter3 := db.NewIterator(nil, h.ro)
  1354. assertErr(t, iter3.Error(), true)
  1355. iter3 = snap.NewIterator(nil, h.ro)
  1356. assertErr(t, iter3.Error(), true)
  1357. assertErr(t, db.Delete([]byte("k"), h.wo), true)
  1358. _, err = db.GetProperty("leveldb.stats")
  1359. assertErr(t, err, true)
  1360. _, err = db.SizeOf([]util.Range{{[]byte("a"), []byte("z")}})
  1361. assertErr(t, err, true)
  1362. assertErr(t, db.CompactRange(util.Range{}), true)
  1363. assertErr(t, db.Close(), true)
  1364. }
  1365. type numberComparer struct{}
  1366. func (numberComparer) num(x []byte) (n int) {
  1367. fmt.Sscan(string(x[1:len(x)-1]), &n)
  1368. return
  1369. }
  1370. func (numberComparer) Name() string {
  1371. return "test.NumberComparer"
  1372. }
  1373. func (p numberComparer) Compare(a, b []byte) int {
  1374. return p.num(a) - p.num(b)
  1375. }
  1376. func (numberComparer) Separator(dst, a, b []byte) []byte { return nil }
  1377. func (numberComparer) Successor(dst, b []byte) []byte { return nil }
  1378. func TestDB_CustomComparer(t *testing.T) {
  1379. h := newDbHarnessWopt(t, &opt.Options{
  1380. DisableLargeBatchTransaction: true,
  1381. Comparer: numberComparer{},
  1382. WriteBuffer: 1000,
  1383. })
  1384. defer h.close()
  1385. h.put("[10]", "ten")
  1386. h.put("[0x14]", "twenty")
  1387. for i := 0; i < 2; i++ {
  1388. h.getVal("[10]", "ten")
  1389. h.getVal("[0xa]", "ten")
  1390. h.getVal("[20]", "twenty")
  1391. h.getVal("[0x14]", "twenty")
  1392. h.get("[15]", false)
  1393. h.get("[0xf]", false)
  1394. h.compactMem()
  1395. h.compactRange("[0]", "[9999]")
  1396. }
  1397. for n := 0; n < 2; n++ {
  1398. for i := 0; i < 100; i++ {
  1399. v := fmt.Sprintf("[%d]", i*10)
  1400. h.put(v, v)
  1401. }
  1402. h.compactMem()
  1403. h.compactRange("[0]", "[1000000]")
  1404. }
  1405. }
  1406. func TestDB_ManualCompaction(t *testing.T) {
  1407. h := newDbHarness(t)
  1408. defer h.close()
  1409. h.db.memdbMaxLevel = 2
  1410. h.putMulti(3, "p", "q")
  1411. h.tablesPerLevel("1,1,1")
  1412. // Compaction range falls before files
  1413. h.compactRange("", "c")
  1414. h.tablesPerLevel("1,1,1")
  1415. // Compaction range falls after files
  1416. h.compactRange("r", "z")
  1417. h.tablesPerLevel("1,1,1")
  1418. // Compaction range overlaps files
  1419. h.compactRange("p1", "p9")
  1420. h.tablesPerLevel("0,0,1")
  1421. // Populate a different range
  1422. h.putMulti(3, "c", "e")
  1423. h.tablesPerLevel("1,1,2")
  1424. // Compact just the new range
  1425. h.compactRange("b", "f")
  1426. h.tablesPerLevel("0,0,2")
  1427. // Compact all
  1428. h.putMulti(1, "a", "z")
  1429. h.tablesPerLevel("0,1,2")
  1430. h.compactRange("", "")
  1431. h.tablesPerLevel("0,0,1")
  1432. }
  1433. func TestDB_BloomFilter(t *testing.T) {
  1434. h := newDbHarnessWopt(t, &opt.Options{
  1435. DisableLargeBatchTransaction: true,
  1436. DisableBlockCache: true,
  1437. Filter: filter.NewBloomFilter(10),
  1438. })
  1439. defer h.close()
  1440. key := func(i int) string {
  1441. return fmt.Sprintf("key%06d", i)
  1442. }
  1443. const n = 10000
  1444. // Populate multiple layers
  1445. for i := 0; i < n; i++ {
  1446. h.put(key(i), key(i))
  1447. }
  1448. h.compactMem()
  1449. h.compactRange("a", "z")
  1450. for i := 0; i < n; i += 100 {
  1451. h.put(key(i), key(i))
  1452. }
  1453. h.compactMem()
  1454. // Prevent auto compactions triggered by seeks
  1455. h.stor.Stall(testutil.ModeSync, storage.TypeTable)
  1456. // Lookup present keys. Should rarely read from small sstable.
  1457. h.stor.ResetCounter(testutil.ModeRead, storage.TypeTable)
  1458. for i := 0; i < n; i++ {
  1459. h.getVal(key(i), key(i))
  1460. }
  1461. cnt, _ := h.stor.Counter(testutil.ModeRead, storage.TypeTable)
  1462. t.Logf("lookup of %d present keys yield %d sstable I/O reads", n, cnt)
  1463. if min, max := n, n+2*n/100; cnt < min || cnt > max {
  1464. t.Errorf("num of sstable I/O reads of present keys not in range of %d - %d, got %d", min, max, cnt)
  1465. }
  1466. // Lookup missing keys. Should rarely read from either sstable.
  1467. h.stor.ResetCounter(testutil.ModeRead, storage.TypeTable)
  1468. for i := 0; i < n; i++ {
  1469. h.get(key(i)+".missing", false)
  1470. }
  1471. cnt, _ = h.stor.Counter(testutil.ModeRead, storage.TypeTable)
  1472. t.Logf("lookup of %d missing keys yield %d sstable I/O reads", n, cnt)
  1473. if max := 3 * n / 100; cnt > max {
  1474. t.Errorf("num of sstable I/O reads of missing keys was more than %d, got %d", max, cnt)
  1475. }
  1476. h.stor.Release(testutil.ModeSync, storage.TypeTable)
  1477. }
  1478. func TestDB_Concurrent(t *testing.T) {
  1479. const n, secs, maxkey = 4, 6, 1000
  1480. h := newDbHarness(t)
  1481. defer h.close()
  1482. runtime.GOMAXPROCS(runtime.NumCPU())
  1483. var (
  1484. closeWg sync.WaitGroup
  1485. stop uint32
  1486. cnt [n]uint32
  1487. )
  1488. for i := 0; i < n; i++ {
  1489. closeWg.Add(1)
  1490. go func(i int) {
  1491. var put, get, found uint
  1492. defer func() {
  1493. t.Logf("goroutine %d stopped after %d ops, put=%d get=%d found=%d missing=%d",
  1494. i, cnt[i], put, get, found, get-found)
  1495. closeWg.Done()
  1496. }()
  1497. rnd := rand.New(rand.NewSource(int64(1000 + i)))
  1498. for atomic.LoadUint32(&stop) == 0 {
  1499. x := cnt[i]
  1500. k := rnd.Intn(maxkey)
  1501. kstr := fmt.Sprintf("%016d", k)
  1502. if (rnd.Int() % 2) > 0 {
  1503. put++
  1504. h.put(kstr, fmt.Sprintf("%d.%d.%-1000d", k, i, x))
  1505. } else {
  1506. get++
  1507. v, err := h.db.Get([]byte(kstr), h.ro)
  1508. if err == nil {
  1509. found++
  1510. rk, ri, rx := 0, -1, uint32(0)
  1511. fmt.Sscanf(string(v), "%d.%d.%d", &rk, &ri, &rx)
  1512. if rk != k {
  1513. t.Errorf("invalid key want=%d got=%d", k, rk)
  1514. }
  1515. if ri < 0 || ri >= n {
  1516. t.Error("invalid goroutine number: ", ri)
  1517. } else {
  1518. tx := atomic.LoadUint32(&(cnt[ri]))
  1519. if rx > tx {
  1520. t.Errorf("invalid seq number, %d > %d ", rx, tx)
  1521. }
  1522. }
  1523. } else if err != ErrNotFound {
  1524. t.Error("Get: got error: ", err)
  1525. return
  1526. }
  1527. }
  1528. atomic.AddUint32(&cnt[i], 1)
  1529. }
  1530. }(i)
  1531. }
  1532. time.Sleep(secs * time.Second)
  1533. atomic.StoreUint32(&stop, 1)
  1534. closeWg.Wait()
  1535. }
  1536. func TestDB_ConcurrentIterator(t *testing.T) {
  1537. const n, n2 = 4, 1000
  1538. h := newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true, WriteBuffer: 30})
  1539. defer h.close()
  1540. runtime.GOMAXPROCS(runtime.NumCPU())
  1541. var (
  1542. closeWg sync.WaitGroup
  1543. stop uint32
  1544. )
  1545. for i := 0; i < n; i++ {
  1546. closeWg.Add(1)
  1547. go func(i int) {
  1548. for k := 0; atomic.LoadUint32(&stop) == 0; k++ {
  1549. h.put(fmt.Sprintf("k%d", k), fmt.Sprintf("%d.%d.", k, i)+strings.Repeat("x", 10))
  1550. }
  1551. closeWg.Done()
  1552. }(i)
  1553. }
  1554. for i := 0; i < n; i++ {
  1555. closeWg.Add(1)
  1556. go func(i int) {
  1557. for k := 1000000; k < 0 || atomic.LoadUint32(&stop) == 0; k-- {
  1558. h.put(fmt.Sprintf("k%d", k), fmt.Sprintf("%d.%d.", k, i)+strings.Repeat("x", 10))
  1559. }
  1560. closeWg.Done()
  1561. }(i)
  1562. }
  1563. cmp := comparer.DefaultComparer
  1564. for i := 0; i < n2; i++ {
  1565. closeWg.Add(1)
  1566. go func(i int) {
  1567. it := h.db.NewIterator(nil, nil)
  1568. var pk []byte
  1569. for it.Next() {
  1570. kk := it.Key()
  1571. if cmp.Compare(kk, pk) <= 0 {
  1572. t.Errorf("iter %d: %q is successor of %q", i, pk, kk)
  1573. }
  1574. pk = append(pk[:0], kk...)
  1575. var k, vk, vi int
  1576. if n, err := fmt.Sscanf(string(it.Key()), "k%d", &k); err != nil {
  1577. t.Errorf("iter %d: Scanf error on key %q: %v", i, it.Key(), err)
  1578. } else if n < 1 {
  1579. t.Errorf("iter %d: Cannot parse key %q", i, it.Key())
  1580. }
  1581. if n, err := fmt.Sscanf(string(it.Value()), "%d.%d", &vk, &vi); err != nil {
  1582. t.Errorf("iter %d: Scanf error on value %q: %v", i, it.Value(), err)
  1583. } else if n < 2 {
  1584. t.Errorf("iter %d: Cannot parse value %q", i, it.Value())
  1585. }
  1586. if vk != k {
  1587. t.Errorf("iter %d: invalid value i=%d, want=%d got=%d", i, vi, k, vk)
  1588. }
  1589. }
  1590. if err := it.Error(); err != nil {
  1591. t.Errorf("iter %d: Got error: %v", i, err)
  1592. }
  1593. it.Release()
  1594. closeWg.Done()
  1595. }(i)
  1596. }
  1597. atomic.StoreUint32(&stop, 1)
  1598. closeWg.Wait()
  1599. }
  1600. func TestDB_ConcurrentWrite(t *testing.T) {
  1601. const n, niter = 10, 10000
  1602. h := newDbHarness(t)
  1603. defer h.close()
  1604. runtime.GOMAXPROCS(runtime.NumCPU())
  1605. var wg sync.WaitGroup
  1606. for i := 0; i < n; i++ {
  1607. wg.Add(1)
  1608. go func(i int) {
  1609. defer wg.Done()
  1610. for k := 0; k < niter; k++ {
  1611. kstr := fmt.Sprintf("%d.%d", i, k)
  1612. vstr := fmt.Sprintf("v%d", k)
  1613. h.put(kstr, vstr)
  1614. // Key should immediately available after put returns.
  1615. h.getVal(kstr, vstr)
  1616. }
  1617. }(i)
  1618. }
  1619. wg.Wait()
  1620. }
  1621. func TestDB_CreateReopenDbOnFile(t *testing.T) {
  1622. dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile-%d", os.Getuid()))
  1623. if err := os.RemoveAll(dbpath); err != nil {
  1624. t.Fatal("cannot remove old db: ", err)
  1625. }
  1626. defer os.RemoveAll(dbpath)
  1627. for i := 0; i < 3; i++ {
  1628. stor, err := storage.OpenFile(dbpath, false)
  1629. if err != nil {
  1630. t.Fatalf("(%d) cannot open storage: %s", i, err)
  1631. }
  1632. db, err := Open(stor, nil)
  1633. if err != nil {
  1634. t.Fatalf("(%d) cannot open db: %s", i, err)
  1635. }
  1636. if err := db.Put([]byte("foo"), []byte("bar"), nil); err != nil {
  1637. t.Fatalf("(%d) cannot write to db: %s", i, err)
  1638. }
  1639. if err := db.Close(); err != nil {
  1640. t.Fatalf("(%d) cannot close db: %s", i, err)
  1641. }
  1642. if err := stor.Close(); err != nil {
  1643. t.Fatalf("(%d) cannot close storage: %s", i, err)
  1644. }
  1645. }
  1646. }
  1647. func TestDB_CreateReopenDbOnFile2(t *testing.T) {
  1648. dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile2-%d", os.Getuid()))
  1649. if err := os.RemoveAll(dbpath); err != nil {
  1650. t.Fatal("cannot remove old db: ", err)
  1651. }
  1652. defer os.RemoveAll(dbpath)
  1653. for i := 0; i < 3; i++ {
  1654. db, err := OpenFile(dbpath, nil)
  1655. if err != nil {
  1656. t.Fatalf("(%d) cannot open db: %s", i, err)
  1657. }
  1658. if err := db.Put([]byte("foo"), []byte("bar"), nil); err != nil {
  1659. t.Fatalf("(%d) cannot write to db: %s", i, err)
  1660. }
  1661. if err := db.Close(); err != nil {
  1662. t.Fatalf("(%d) cannot close db: %s", i, err)
  1663. }
  1664. }
  1665. }
  1666. func TestDB_DeletionMarkersOnMemdb(t *testing.T) {
  1667. h := newDbHarness(t)
  1668. defer h.close()
  1669. h.put("foo", "v1")
  1670. h.compactMem()
  1671. h.delete("foo")
  1672. h.get("foo", false)
  1673. h.getKeyVal("")
  1674. }
  1675. func TestDB_LeveldbIssue178(t *testing.T) {
  1676. nKeys := (opt.DefaultCompactionTableSize / 30) * 5
  1677. key1 := func(i int) string {
  1678. return fmt.Sprintf("my_key_%d", i)
  1679. }
  1680. key2 := func(i int) string {
  1681. return fmt.Sprintf("my_key_%d_xxx", i)
  1682. }
  1683. // Disable compression since it affects the creation of layers and the
  1684. // code below is trying to test against a very specific scenario.
  1685. h := newDbHarnessWopt(t, &opt.Options{
  1686. DisableLargeBatchTransaction: true,
  1687. Compression: opt.NoCompression,
  1688. })
  1689. defer h.close()
  1690. // Create first key range.
  1691. batch := new(Batch)
  1692. for i := 0; i < nKeys; i++ {
  1693. batch.Put([]byte(key1(i)), []byte("value for range 1 key"))
  1694. }
  1695. h.write(batch)
  1696. // Create second key range.
  1697. batch.Reset()
  1698. for i := 0; i < nKeys; i++ {
  1699. batch.Put([]byte(key2(i)), []byte("value for range 2 key"))
  1700. }
  1701. h.write(batch)
  1702. // Delete second key range.
  1703. batch.Reset()
  1704. for i := 0; i < nKeys; i++ {
  1705. batch.Delete([]byte(key2(i)))
  1706. }
  1707. h.write(batch)
  1708. h.waitMemCompaction()
  1709. // Run manual compaction.
  1710. h.compactRange(key1(0), key1(nKeys-1))
  1711. // Checking the keys.
  1712. h.assertNumKeys(nKeys)
  1713. }
  1714. func TestDB_LeveldbIssue200(t *testing.T) {
  1715. h := newDbHarness(t)
  1716. defer h.close()
  1717. h.put("1", "b")
  1718. h.put("2", "c")
  1719. h.put("3", "d")
  1720. h.put("4", "e")
  1721. h.put("5", "f")
  1722. iter := h.db.NewIterator(nil, h.ro)
  1723. // Add an element that should not be reflected in the iterator.
  1724. h.put("25", "cd")
  1725. iter.Seek([]byte("5"))
  1726. assertBytes(t, []byte("5"), iter.Key())
  1727. iter.Prev()
  1728. assertBytes(t, []byte("4"), iter.Key())
  1729. iter.Prev()
  1730. assertBytes(t, []byte("3"), iter.Key())
  1731. iter.Next()
  1732. assertBytes(t, []byte("4"), iter.Key())
  1733. iter.Next()
  1734. assertBytes(t, []byte("5"), iter.Key())
  1735. }
  1736. func TestDB_GoleveldbIssue74(t *testing.T) {
  1737. h := newDbHarnessWopt(t, &opt.Options{
  1738. DisableLargeBatchTransaction: true,
  1739. WriteBuffer: 1 * opt.MiB,
  1740. })
  1741. defer h.close()
  1742. const n, dur = 10000, 5 * time.Second
  1743. runtime.GOMAXPROCS(runtime.NumCPU())
  1744. until := time.Now().Add(dur)
  1745. wg := new(sync.WaitGroup)
  1746. wg.Add(2)
  1747. var done uint32
  1748. go func() {
  1749. var i int
  1750. defer func() {
  1751. t.Logf("WRITER DONE #%d", i)
  1752. atomic.StoreUint32(&done, 1)
  1753. wg.Done()
  1754. }()
  1755. b := new(Batch)
  1756. for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
  1757. iv := fmt.Sprintf("VAL%010d", i)
  1758. for k := 0; k < n; k++ {
  1759. key := fmt.Sprintf("KEY%06d", k)
  1760. b.Put([]byte(key), []byte(key+iv))
  1761. b.Put([]byte(fmt.Sprintf("PTR%06d", k)), []byte(key))
  1762. }
  1763. h.write(b)
  1764. b.Reset()
  1765. snap := h.getSnapshot()
  1766. iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
  1767. var k int
  1768. for ; iter.Next(); k++ {
  1769. ptrKey := iter.Key()
  1770. key := iter.Value()
  1771. if _, err := snap.Get(ptrKey, nil); err != nil {
  1772. t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, ptrKey, err)
  1773. }
  1774. if value, err := snap.Get(key, nil); err != nil {
  1775. t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, key, err)
  1776. } else if string(value) != string(key)+iv {
  1777. t.Fatalf("WRITER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+iv, value)
  1778. }
  1779. b.Delete(key)
  1780. b.Delete(ptrKey)
  1781. }
  1782. h.write(b)
  1783. iter.Release()
  1784. snap.Release()
  1785. if k != n {
  1786. t.Fatalf("#%d %d != %d", i, k, n)
  1787. }
  1788. }
  1789. }()
  1790. go func() {
  1791. var i int
  1792. defer func() {
  1793. t.Logf("READER DONE #%d", i)
  1794. atomic.StoreUint32(&done, 1)
  1795. wg.Done()
  1796. }()
  1797. for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
  1798. snap := h.getSnapshot()
  1799. iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
  1800. var prevValue string
  1801. var k int
  1802. for ; iter.Next(); k++ {
  1803. ptrKey := iter.Key()
  1804. key := iter.Value()
  1805. if _, err := snap.Get(ptrKey, nil); err != nil {
  1806. t.Fatalf("READER #%d snapshot.Get %q: %v", i, ptrKey, err)
  1807. }
  1808. if value, err := snap.Get(key, nil); err != nil {
  1809. t.Fatalf("READER #%d snapshot.Get %q: %v", i, key, err)
  1810. } else if prevValue != "" && string(value) != string(key)+prevValue {
  1811. t.Fatalf("READER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+prevValue, value)
  1812. } else {
  1813. prevValue = string(value[len(key):])
  1814. }
  1815. }
  1816. iter.Release()
  1817. snap.Release()
  1818. if k > 0 && k != n {
  1819. t.Fatalf("#%d %d != %d", i, k, n)
  1820. }
  1821. }
  1822. }()
  1823. wg.Wait()
  1824. }
  1825. func TestDB_GetProperties(t *testing.T) {
  1826. h := newDbHarness(t)
  1827. defer h.close()
  1828. _, err := h.db.GetProperty("leveldb.num-files-at-level")
  1829. if err == nil {
  1830. t.Error("GetProperty() failed to detect missing level")
  1831. }
  1832. _, err = h.db.GetProperty("leveldb.num-files-at-level0")
  1833. if err != nil {
  1834. t.Error("got unexpected error", err)
  1835. }
  1836. _, err = h.db.GetProperty("leveldb.num-files-at-level0x")
  1837. if err == nil {
  1838. t.Error("GetProperty() failed to detect invalid level")
  1839. }
  1840. }
  1841. func TestDB_GoleveldbIssue72and83(t *testing.T) {
  1842. h := newDbHarnessWopt(t, &opt.Options{
  1843. DisableLargeBatchTransaction: true,
  1844. WriteBuffer: 1 * opt.MiB,
  1845. OpenFilesCacheCapacity: 3,
  1846. })
  1847. defer h.close()
  1848. const n, wn, dur = 10000, 100, 30 * time.Second
  1849. runtime.GOMAXPROCS(runtime.NumCPU())
  1850. randomData := func(prefix byte, i int) []byte {
  1851. data := make([]byte, 1+4+32+64+32)
  1852. _, err := crand.Reader.Read(data[1 : len(data)-8])
  1853. if err != nil {
  1854. panic(err)
  1855. }
  1856. data[0] = prefix
  1857. binary.LittleEndian.PutUint32(data[len(data)-8:], uint32(i))
  1858. binary.LittleEndian.PutUint32(data[len(data)-4:], util.NewCRC(data[:len(data)-4]).Value())
  1859. return data
  1860. }
  1861. keys := make([][]byte, n)
  1862. for i := range keys {
  1863. keys[i] = randomData(1, 0)
  1864. }
  1865. until := time.Now().Add(dur)
  1866. wg := new(sync.WaitGroup)
  1867. wg.Add(3)
  1868. var done uint32
  1869. go func() {
  1870. i := 0
  1871. defer func() {
  1872. t.Logf("WRITER DONE #%d", i)
  1873. wg.Done()
  1874. }()
  1875. b := new(Batch)
  1876. for ; i < wn && atomic.LoadUint32(&done) == 0; i++ {
  1877. b.Reset()
  1878. for _, k1 := range keys {
  1879. k2 := randomData(2, i)
  1880. b.Put(k2, randomData(42, i))
  1881. b.Put(k1, k2)
  1882. }
  1883. if err := h.db.Write(b, h.wo); err != nil {
  1884. atomic.StoreUint32(&done, 1)
  1885. t.Fatalf("WRITER #%d db.Write: %v", i, err)
  1886. }
  1887. }
  1888. }()
  1889. go func() {
  1890. var i int
  1891. defer func() {
  1892. t.Logf("READER0 DONE