storj/storage/boltdb/client_test.go

319 lines
7.9 KiB
Go
Raw Normal View History

2019-01-24 20:15:10 +00:00
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package boltdb
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"testing"
2018-12-21 10:54:20 +00:00
"github.com/zeebo/errs"
"golang.org/x/sync/errgroup"
"storj.io/common/testcontext"
"storj.io/storj/storage"
"storj.io/storj/storage/testsuite"
)
func TestSuite(t *testing.T) {
2018-09-11 14:57:12 +01:00
tempdir, err := ioutil.TempDir("", "storj-bolt")
2018-05-07 18:01:53 +01:00
if err != nil {
t.Fatal(err)
adds netstate pagination (#95) * adds netstate rpc server pagination, mocks pagination in test/util.go * updates ns client example, combines ns client and server test to netstate_test, adds pagination to bolt client * better organizes netstate test calls * wip breaking netstate test into smaller tests * wip modularizing netstate tests * adds some test panics * wip netstate test attempts * testing bug in netstate TestDeleteAuth * wip fixes global variable problem, still issues with list * wip fixes get request params and args * fixes bug in path when using MakePointers helper fn * updates mockdb list func, adds test, changes Limit to int * fixes merge conflicts * fixes broken tests from merge * remove unnecessary PointerEntry struct * removes error when Get returns nil value from boltdb * breaks boltdb client tests into smaller tests * renames AssertNoErr test helper to HandleErr * adds StartingKey and Limit parameters to redis list func, adds beginning of redis tests * adds helper func for mockdb List function * if no starting key provided for netstate List, the first value in storage will be used * adds basic pagination for redis List function, adds tests * adds list limit to call in overlay/server.go * streamlines/fixes some nits from review * removes use of obsolete EncryptedUnencryptedSize * uses MockKeyValueStore instead of redis instance in redis client test * changes test to expect nil returned for getting missing key * remove error from `KeyValueStore#Get` * fix bolt test * Merge pull request #1 from bryanchriswhite/nat-pagination remove error from `KeyValueStore#Get` * adds Get returning error back to KeyValueStore interface and affected clients * trying to appease travis: returns errors in Get calls in overlay/cache and cache_test * handles redis get error when no key found
2018-06-29 21:06:25 +01:00
}
defer func() { _ = os.RemoveAll(tempdir) }()
adds netstate pagination (#95) * adds netstate rpc server pagination, mocks pagination in test/util.go * updates ns client example, combines ns client and server test to netstate_test, adds pagination to bolt client * better organizes netstate test calls * wip breaking netstate test into smaller tests * wip modularizing netstate tests * adds some test panics * wip netstate test attempts * testing bug in netstate TestDeleteAuth * wip fixes global variable problem, still issues with list * wip fixes get request params and args * fixes bug in path when using MakePointers helper fn * updates mockdb list func, adds test, changes Limit to int * fixes merge conflicts * fixes broken tests from merge * remove unnecessary PointerEntry struct * removes error when Get returns nil value from boltdb * breaks boltdb client tests into smaller tests * renames AssertNoErr test helper to HandleErr * adds StartingKey and Limit parameters to redis list func, adds beginning of redis tests * adds helper func for mockdb List function * if no starting key provided for netstate List, the first value in storage will be used * adds basic pagination for redis List function, adds tests * adds list limit to call in overlay/server.go * streamlines/fixes some nits from review * removes use of obsolete EncryptedUnencryptedSize * uses MockKeyValueStore instead of redis instance in redis client test * changes test to expect nil returned for getting missing key * remove error from `KeyValueStore#Get` * fix bolt test * Merge pull request #1 from bryanchriswhite/nat-pagination remove error from `KeyValueStore#Get` * adds Get returning error back to KeyValueStore interface and affected clients * trying to appease travis: returns errors in Get calls in overlay/cache and cache_test * handles redis get error when no key found
2018-06-29 21:06:25 +01:00
dbname := filepath.Join(tempdir, "bolt.db")
store, err := New(dbname, "bucket")
if err != nil {
t.Fatalf("failed to create db: %v", err)
}
defer func() {
if err := store.Close(); err != nil {
t.Fatalf("failed to close db: %v", err)
}
}()
testsuite.RunTests(t, store)
adds netstate pagination (#95) * adds netstate rpc server pagination, mocks pagination in test/util.go * updates ns client example, combines ns client and server test to netstate_test, adds pagination to bolt client * better organizes netstate test calls * wip breaking netstate test into smaller tests * wip modularizing netstate tests * adds some test panics * wip netstate test attempts * testing bug in netstate TestDeleteAuth * wip fixes global variable problem, still issues with list * wip fixes get request params and args * fixes bug in path when using MakePointers helper fn * updates mockdb list func, adds test, changes Limit to int * fixes merge conflicts * fixes broken tests from merge * remove unnecessary PointerEntry struct * removes error when Get returns nil value from boltdb * breaks boltdb client tests into smaller tests * renames AssertNoErr test helper to HandleErr * adds StartingKey and Limit parameters to redis list func, adds beginning of redis tests * adds helper func for mockdb List function * if no starting key provided for netstate List, the first value in storage will be used * adds basic pagination for redis List function, adds tests * adds list limit to call in overlay/server.go * streamlines/fixes some nits from review * removes use of obsolete EncryptedUnencryptedSize * uses MockKeyValueStore instead of redis instance in redis client test * changes test to expect nil returned for getting missing key * remove error from `KeyValueStore#Get` * fix bolt test * Merge pull request #1 from bryanchriswhite/nat-pagination remove error from `KeyValueStore#Get` * adds Get returning error back to KeyValueStore interface and affected clients * trying to appease travis: returns errors in Get calls in overlay/cache and cache_test * handles redis get error when no key found
2018-06-29 21:06:25 +01:00
}
func BenchmarkSuite(b *testing.B) {
2018-09-11 14:57:12 +01:00
tempdir, err := ioutil.TempDir("", "storj-bolt")
adds netstate pagination (#95) * adds netstate rpc server pagination, mocks pagination in test/util.go * updates ns client example, combines ns client and server test to netstate_test, adds pagination to bolt client * better organizes netstate test calls * wip breaking netstate test into smaller tests * wip modularizing netstate tests * adds some test panics * wip netstate test attempts * testing bug in netstate TestDeleteAuth * wip fixes global variable problem, still issues with list * wip fixes get request params and args * fixes bug in path when using MakePointers helper fn * updates mockdb list func, adds test, changes Limit to int * fixes merge conflicts * fixes broken tests from merge * remove unnecessary PointerEntry struct * removes error when Get returns nil value from boltdb * breaks boltdb client tests into smaller tests * renames AssertNoErr test helper to HandleErr * adds StartingKey and Limit parameters to redis list func, adds beginning of redis tests * adds helper func for mockdb List function * if no starting key provided for netstate List, the first value in storage will be used * adds basic pagination for redis List function, adds tests * adds list limit to call in overlay/server.go * streamlines/fixes some nits from review * removes use of obsolete EncryptedUnencryptedSize * uses MockKeyValueStore instead of redis instance in redis client test * changes test to expect nil returned for getting missing key * remove error from `KeyValueStore#Get` * fix bolt test * Merge pull request #1 from bryanchriswhite/nat-pagination remove error from `KeyValueStore#Get` * adds Get returning error back to KeyValueStore interface and affected clients * trying to appease travis: returns errors in Get calls in overlay/cache and cache_test * handles redis get error when no key found
2018-06-29 21:06:25 +01:00
if err != nil {
b.Fatal(err)
}
defer func() { _ = os.RemoveAll(tempdir) }()
dbname := filepath.Join(tempdir, "bolt.db")
store, err := New(dbname, "bucket")
if err != nil {
b.Fatalf("failed to create db: %v", err)
}
defer func() {
if err := store.Close(); err != nil {
b.Fatalf("failed to close db: %v", err)
}
}()
testsuite.RunBenchmarks(b, store)
}
func TestSuiteShared(t *testing.T) {
tempdir, err := ioutil.TempDir("", "storj-bolt")
if err != nil {
t.Fatal(err)
}
defer func() { _ = os.RemoveAll(tempdir) }()
dbname := filepath.Join(tempdir, "bolt.db")
stores, err := NewShared(dbname, "alpha", "beta")
if err != nil {
t.Fatalf("failed to create db: %v", err)
}
defer func() {
for _, store := range stores {
if err := store.Close(); err != nil {
t.Fatalf("failed to close db: %v", err)
}
}
}()
for _, store := range stores {
testsuite.RunTests(t, store)
}
}
type boltLongBenchmarkStore struct {
*Client
dirPath string
}
func (store *boltLongBenchmarkStore) BulkImport(ctx context.Context, iter storage.Iterator) (err error) {
// turn off syncing during import
oldval := store.db.NoSync
store.db.NoSync = true
defer func() { store.db.NoSync = oldval }()
var item storage.ListItem
for iter.Next(ctx, &item) {
if err := store.Put(ctx, item.Key, item.Value); err != nil {
return fmt.Errorf("Failed to insert data (%q, %q): %v", item.Key, item.Value, err)
}
}
return store.db.Sync()
}
func (store *boltLongBenchmarkStore) BulkDeleteAll(ctx context.Context) error {
// do nothing here; everything will be cleaned up later after the test completes. it's not
// worth it to wait for BoltDB to remove every key, one by one, and we can't just
// os.RemoveAll() the whole test directory at this point because those files are still open
// and unremoveable on Windows.
return nil
}
var _ testsuite.BulkImporter = &boltLongBenchmarkStore{}
var _ testsuite.BulkCleaner = &boltLongBenchmarkStore{}
func BenchmarkSuiteLong(b *testing.B) {
tempdir, err := ioutil.TempDir("", "storj-bolt")
if err != nil {
b.Fatal(err)
}
defer func() {
if err := os.RemoveAll(tempdir); err != nil {
b.Fatal(err)
}
}()
dbname := filepath.Join(tempdir, "bolt.db")
store, err := New(dbname, "bucket")
if err != nil {
b.Fatalf("failed to create db: %v", err)
}
defer func() {
2018-12-21 10:54:20 +00:00
if err := errs.Combine(store.Close(), os.RemoveAll(tempdir)); err != nil {
b.Fatalf("failed to close db: %v", err)
}
}()
longStore := &boltLongBenchmarkStore{
Client: store,
dirPath: tempdir,
}
testsuite.BenchmarkPathOperationsInLargeDb(b, longStore)
}
func BenchmarkClientWrite(b *testing.B) {
// setup db
ctx := testcontext.New(b)
defer ctx.Cleanup()
dbfile := ctx.File("testbolt.db")
dbs, err := NewShared(dbfile, "kbuckets", "nodes")
if err != nil {
b.Fatalf("failed to create db: %v\n", err)
}
defer func() {
if err := dbs[0].Close(); err != nil {
b.Fatalf("failed to close db: %v\n", err)
}
if err := dbs[1].Close(); err != nil {
b.Fatalf("failed to close db: %v\n", err)
}
}()
kdb := dbs[0]
// benchmark test: execute 1000 Put operations where each call to `PutAndCommit` does the following:
// 1) create a BoltDB transaction (tx), 2) execute the db operation, 3) commit the tx which writes it to disk.
for n := 0; n < b.N; n++ {
var group errgroup.Group
for i := 0; i < 1000; i++ {
key := storage.Key(fmt.Sprintf("testkey%d", i))
value := storage.Value("testvalue")
group.Go(func() error {
return kdb.PutAndCommit(ctx, key, value)
})
}
if err := group.Wait(); err != nil {
b.Fatalf("PutAndCommit: %v", err)
}
}
}
func BenchmarkClientNoSyncWrite(b *testing.B) {
// setup db
ctx := testcontext.New(b)
defer ctx.Cleanup()
dbfile := ctx.File("testbolt.db")
dbs, err := NewShared(dbfile, "kbuckets", "nodes")
if err != nil {
b.Fatalf("failed to create db: %v\n", err)
}
defer func() {
if err := dbs[0].Close(); err != nil {
b.Fatalf("failed to close db: %v\n", err)
}
if err := dbs[1].Close(); err != nil {
b.Fatalf("failed to close db: %v\n", err)
}
}()
kdb := dbs[0]
// benchmark test: execute 1000 Put operations with fsync turned off.
// Each call to `PutAndCommit` does the following: 1) creates a BoltDB transaction (tx),
// 2) executes the db operation, and 3) commits the tx which does NOT write it to disk.
kdb.db.NoSync = true
for n := 0; n < b.N; n++ {
var group errgroup.Group
for i := 0; i < 1000; i++ {
key := storage.Key(fmt.Sprintf("testkey%d", i))
value := storage.Value("testvalue")
group.Go(func() error {
return kdb.PutAndCommit(ctx, key, value)
})
}
if err := group.Wait(); err != nil {
b.Fatalf("PutAndCommit: %v", err)
}
}
err = kdb.db.Sync()
if err != nil {
b.Fatalf("boltDB sync err: %v\n", err)
}
}
func BenchmarkClientBatchWrite(b *testing.B) {
// setup db
ctx := testcontext.New(b)
defer ctx.Cleanup()
dbfile := ctx.File("testbolt.db")
dbs, err := NewShared(dbfile, "kbuckets", "nodes")
if err != nil {
b.Fatalf("failed to create db: %v\n", err)
}
defer func() {
if err := dbs[0].Close(); err != nil {
b.Fatalf("failed to close db: %v\n", err)
}
if err := dbs[1].Close(); err != nil {
b.Fatalf("failed to close db: %v\n", err)
}
}()
kdb := dbs[0]
// benchmark test: batch 1000 Put operations.
// Each call to `Put` does the following: 1) adds the db operation to a queue in boltDB,
// 2) every 1000 operations or 10ms, whichever is first, BoltDB creates a single
// transaction for all operations currently in the batch, executes the operations,
// commits, and writes them to disk
for n := 0; n < b.N; n++ {
var group errgroup.Group
for i := 0; i < 1000; i++ {
key := storage.Key(fmt.Sprintf("testkey%d", i))
value := storage.Value("testvalue")
group.Go(func() error {
return kdb.Put(ctx, key, value)
})
}
if err := group.Wait(); err != nil {
b.Fatalf("Put: %v", err)
}
}
}
func BenchmarkClientBatchNoSyncWrite(b *testing.B) {
// setup db
ctx := testcontext.New(b)
defer ctx.Cleanup()
dbfile := ctx.File("testbolt.db")
dbs, err := NewShared(dbfile, "kbuckets", "nodes")
if err != nil {
b.Fatalf("failed to create db: %v\n", err)
}
defer func() {
if err := dbs[0].Close(); err != nil {
b.Fatalf("failed to close db: %v\n", err)
}
if err := dbs[1].Close(); err != nil {
b.Fatalf("failed to close db: %v\n", err)
}
}()
kdb := dbs[0]
// benchmark test: batch 1000 Put operations with fsync turned off.
// Each call to `Put` does the following: 1) adds the db operation to a queue in boltDB,
// 2) every 1000 operations or 2 ms, whichever is first, BoltDB creates a single
// transaction for all operations currently in the batch, executes the operations,
// commits, but does NOT write them to disk
kdb.db.NoSync = true
for n := 0; n < b.N; n++ {
var group errgroup.Group
for i := 0; i < 1000; i++ {
key := storage.Key(fmt.Sprintf("testkey%d", i))
value := storage.Value("testvalue")
group.Go(func() error {
return kdb.Put(ctx, key, value)
})
}
if err := group.Wait(); err != nil {
b.Fatalf("Put: %v", err)
}
err := kdb.db.Sync()
if err != nil {
b.Fatalf("boltDB sync err: %v\n", err)
}
}
}