e2c0dd437a
..although it ought to work for other storage.KeyValueStore needs as
well. it's just optimized to work pretty well for a largish hierarchy of
paths.
This includes the addition of "long benchmarks" for KeyValueStore
testing. These will only be run when -test-bench-long is added to the
test flags. In these benchmarks, a large corpus of paths matching a
natural ("real-life") hierarchy is read from paths.data.gz (which you
can get from https://github.com/storj/path-test-corpus) and imported
into a particular KeyValueStore. Recursive and non-recursive queries are
run on it to detect performance problems that arise only at scale.
This also includes alternate implementation of the postgreskv client,
which works in a less-bizarre way for non-recursive queries, but suffers
from poor performance in tests such as the long benchmarks. Once this
alternate impl is committed to the tree, we can remove it again; I just
want it to be available for future reference.
142 lines
3.2 KiB
Go
142 lines
3.2 KiB
Go
// Copyright (C) 2018 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package boltdb
|
|
|
|
import (
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"storj.io/storj/pkg/utils"
|
|
"storj.io/storj/storage"
|
|
"storj.io/storj/storage/testsuite"
|
|
)
|
|
|
|
func TestSuite(t *testing.T) {
|
|
tempdir, err := ioutil.TempDir("", "storj-bolt")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = os.RemoveAll(tempdir) }()
|
|
|
|
dbname := filepath.Join(tempdir, "bolt.db")
|
|
store, err := New(dbname, "bucket")
|
|
if err != nil {
|
|
t.Fatalf("failed to create db: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := store.Close(); err != nil {
|
|
t.Fatalf("failed to close db: %v", err)
|
|
}
|
|
}()
|
|
|
|
testsuite.RunTests(t, store)
|
|
}
|
|
|
|
func BenchmarkSuite(b *testing.B) {
|
|
tempdir, err := ioutil.TempDir("", "storj-bolt")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
defer func() { _ = os.RemoveAll(tempdir) }()
|
|
|
|
dbname := filepath.Join(tempdir, "bolt.db")
|
|
store, err := New(dbname, "bucket")
|
|
if err != nil {
|
|
b.Fatalf("failed to create db: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := store.Close(); err != nil {
|
|
b.Fatalf("failed to close db: %v", err)
|
|
}
|
|
}()
|
|
|
|
testsuite.RunBenchmarks(b, store)
|
|
}
|
|
|
|
func TestSuiteShared(t *testing.T) {
|
|
tempdir, err := ioutil.TempDir("", "storj-bolt")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer func() { _ = os.RemoveAll(tempdir) }()
|
|
|
|
dbname := filepath.Join(tempdir, "bolt.db")
|
|
stores, err := NewShared(dbname, "alpha", "beta")
|
|
if err != nil {
|
|
t.Fatalf("failed to create db: %v", err)
|
|
}
|
|
defer func() {
|
|
for _, store := range stores {
|
|
if err := store.Close(); err != nil {
|
|
t.Fatalf("failed to close db: %v", err)
|
|
}
|
|
}
|
|
}()
|
|
|
|
for _, store := range stores {
|
|
testsuite.RunTests(t, store)
|
|
}
|
|
}
|
|
|
|
type boltLongBenchmarkStore struct {
|
|
*Client
|
|
dirPath string
|
|
}
|
|
|
|
func (store *boltLongBenchmarkStore) BulkImport(iter storage.Iterator) (err error) {
|
|
// turn off syncing during import
|
|
oldval := store.db.NoSync
|
|
store.db.NoSync = true
|
|
defer func() { store.db.NoSync = oldval }()
|
|
|
|
var item storage.ListItem
|
|
for iter.Next(&item) {
|
|
if err := store.Put(item.Key, item.Value); err != nil {
|
|
return fmt.Errorf("Failed to insert data (%q, %q): %v", item.Key, item.Value, err)
|
|
}
|
|
}
|
|
|
|
return store.db.Sync()
|
|
}
|
|
|
|
func (store *boltLongBenchmarkStore) BulkDelete() error {
|
|
// do nothing here; everything will be cleaned up later after the test completes. it's not
|
|
// worth it to wait for BoltDB to remove every key, one by one, and we can't just
|
|
// os.RemoveAll() the whole test directory at this point because those files are still open
|
|
// and unremoveable on Windows.
|
|
return nil
|
|
}
|
|
|
|
func BenchmarkSuiteLong(b *testing.B) {
|
|
tempdir, err := ioutil.TempDir("", "storj-bolt")
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
defer func() {
|
|
if err := os.RemoveAll(tempdir); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}()
|
|
|
|
dbname := filepath.Join(tempdir, "bolt.db")
|
|
store, err := New(dbname, "bucket")
|
|
if err != nil {
|
|
b.Fatalf("failed to create db: %v", err)
|
|
}
|
|
defer func() {
|
|
if err := utils.CombineErrors(store.Close(), os.RemoveAll(tempdir)); err != nil {
|
|
b.Fatalf("failed to close db: %v", err)
|
|
}
|
|
}()
|
|
|
|
longStore := &boltLongBenchmarkStore{
|
|
Client: store,
|
|
dirPath: tempdir,
|
|
}
|
|
testsuite.BenchmarkPathOperationsInLargeDb(b, longStore)
|
|
}
|