JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storage/testsuite: pass ctx in to bulk setup methods

Browse Source 
to make them cancelable. Also,

* rename BulkDelete->BulkDeleteAll

this leaves room for a new method `BulkDelete(items storage.Items)` that
does a bulk deletion of a specified list of items, as opposed to
deleting _everything_. such a method would be used in the
`cleanupItems()` function found in utils.go, because when individual
deletes are fairly slow, that step takes way too long during tests.

* use BulkDelete method if available

nothing currently provides `BulkDelete(items storage.Items) error`,
but we made use of it with the Bigtable testing and code, and may make
use of it again when adding new kv backends.

* and eliminate the global context in test_iterate.go

Change-Id: I171c7a3818beffbad969b131e98b9bbe3f324bf2
This commit is contained in:
paul cannon 2019-09-24 13:06:22 -05:00 committed by Jennifer Li Johnson
parent 72d407559e
commit 94651921c3
14 changed files with 124 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
storage/boltdb/client_test.go
View File

@ -19,8 +19,6 @@ import (
"storj.io/storj/storage/testsuite"
)
var ctx = context.Background() // test context
func TestSuite(t *testing.T) {
tempdir, err := ioutil.TempDir("", "storj-bolt")
if err != nil {
@ -93,7 +91,7 @@ type boltLongBenchmarkStore struct {
dirPath string
}
func (store *boltLongBenchmarkStore) BulkImport(iter storage.Iterator) (err error) {
func (store *boltLongBenchmarkStore) BulkImport(ctx context.Context, iter storage.Iterator) (err error) {
// turn off syncing during import
oldval := store.db.NoSync
store.db.NoSync = true
@ -109,7 +107,7 @@ func (store *boltLongBenchmarkStore) BulkImport(iter storage.Iterator) (err erro
return store.db.Sync()
}
func (store *boltLongBenchmarkStore) BulkDelete() error {
func (store *boltLongBenchmarkStore) BulkDeleteAll(ctx context.Context) error {
// do nothing here; everything will be cleaned up later after the test completes. it's not
// worth it to wait for BoltDB to remove every key, one by one, and we can't just
// os.RemoveAll() the whole test directory at this point because those files are still open
@ -117,6 +115,9 @@ func (store *boltLongBenchmarkStore) BulkDelete() error {
return nil
}
var _ testsuite.BulkImporter = &boltLongBenchmarkStore{}
var _ testsuite.BulkCleaner = &boltLongBenchmarkStore{}
func BenchmarkSuiteLong(b *testing.B) {
tempdir, err := ioutil.TempDir("", "storj-bolt")
if err != nil {

10
storage/postgreskv/alternateclient_test.go
View File

@ -4,6 +4,7 @@
package postgreskv
import (
"context"
"flag"
"testing"
@ -56,14 +57,17 @@ type pgAltLongBenchmarkStore struct {
*AlternateClient
}
func (store *pgAltLongBenchmarkStore) BulkImport(iter storage.Iterator) error {
func (store *pgAltLongBenchmarkStore) BulkImport(ctx context.Context, iter storage.Iterator) error {
return bulkImport(store.pgConn, iter)
}
func (store *pgAltLongBenchmarkStore) BulkDelete() error {
return bulkDelete(store.pgConn)
func (store *pgAltLongBenchmarkStore) BulkDeleteAll(ctx context.Context) error {
return bulkDeleteAll(store.pgConn)
}
var _ testsuite.BulkImporter = &pgAltLongBenchmarkStore{}
var _ testsuite.BulkCleaner = &pgAltLongBenchmarkStore{}
func BenchmarkSuiteLongAlt(b *testing.B) {
store, cleanup := newTestAlternatePostgres(b)
defer cleanup()

6
storage/postgreskv/client_test.go
View File

@ -126,7 +126,7 @@ func bulkImport(db *sql.DB, iter storage.Iterator) (err error) {
return nil
}
func bulkDelete(db *sql.DB) error {
func bulkDeleteAll(db *sql.DB) error {
_, err := db.Exec("TRUNCATE pathdata")
if err != nil {
return errs.New("Failed to TRUNCATE pathdata table: %v", err)
@ -142,8 +142,8 @@ func (store *pgLongBenchmarkStore) BulkImport(iter storage.Iterator) error {
return bulkImport(store.pgConn, iter)
}
func (store *pgLongBenchmarkStore) BulkDelete() error {
return bulkDelete(store.pgConn)
func (store *pgLongBenchmarkStore) BulkDeleteAll() error {
return bulkDeleteAll(store.pgConn)
}
func BenchmarkSuiteLong(b *testing.B) {

5
storage/testsuite/bench.go
View File

@ -10,6 +10,7 @@ import (
"golang.org/x/sync/errgroup"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
@ -39,7 +40,9 @@ func RunBenchmarks(b *testing.B, store storage.KeyValueStore) {
}
}
defer cleanupItems(store, items)
ctx := testcontext.New(b)
defer ctx.Cleanup()
defer cleanupItems(b, ctx, store, items)
b.Run("Put", func(b *testing.B) {
b.SetBytes(int64(len(items)))

72
storage/testsuite/long_bench.go
View File

@ -20,6 +20,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/zeebo/errs"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
@ -166,29 +167,20 @@ func openTestData(tb testing.TB) *KVInputIterator {
return inputIter
}
// BulkImporter identifies KV storage facilities that can do bulk importing of items more
// efficiently than inserting one-by-one.
type BulkImporter interface {
BulkImport(storage.Iterator) error
}
// BulkCleaner identifies KV storage facilities that can delete all items efficiently.
type BulkCleaner interface {
BulkDelete() error
}
// BenchmarkPathOperationsInLargeDb runs the "long benchmarks" suite for KeyValueStore instances.
func BenchmarkPathOperationsInLargeDb(b *testing.B, store storage.KeyValueStore) {
if *longBenchmarksData == "" {
b.Skip("Long benchmarks not enabled.")
}
initStore(b, store)
ctx := testcontext.New(b)
defer ctx.Cleanup()
initStore(b, ctx, store)
doTest := func(name string, testFunc func(*testing.B, storage.KeyValueStore)) {
doTest := func(name string, testFunc func(*testing.B, *testcontext.Context, storage.KeyValueStore)) {
b.Run(name, func(bb *testing.B) {
for i := 0; i < bb.N; i++ {
testFunc(bb, store)
testFunc(bb, ctx, store)
}
})
}
@ -201,12 +193,12 @@ func BenchmarkPathOperationsInLargeDb(b *testing.B, store storage.KeyValueStore)
doTest("TopRecursiveStartAt", topRecursiveStartAt)
doTest("TopNonRecursive", topNonRecursive)
cleanupStore(b, store)
cleanupStore(b, ctx, store)
}
func importBigPathset(tb testing.TB, store storage.KeyValueStore) {
func importBigPathset(tb testing.TB, ctx *testcontext.Context, store storage.KeyValueStore) {
// make sure this is an empty db, or else refuse to run
if !isEmptyKVStore(tb, store) {
if !isEmptyKVStore(tb, ctx, store) {
tb.Fatal("Provided KeyValueStore is not empty. The long benchmarks are destructive. Not running!")
}
@ -220,7 +212,7 @@ func importBigPathset(tb testing.TB, store storage.KeyValueStore) {
importer, ok := store.(BulkImporter)
if ok {
tb.Log("Performing bulk import...")
err := importer.BulkImport(inputIter)
err := importer.BulkImport(ctx, inputIter)
if err != nil {
errStr := "Provided KeyValueStore failed to import data"
@ -249,7 +241,7 @@ func importBigPathset(tb testing.TB, store storage.KeyValueStore) {
}
}
func initStore(b *testing.B, store storage.KeyValueStore) {
func initStore(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
b.Helper()
if !*noInitDb {
@ -260,17 +252,17 @@ func initStore(b *testing.B, store storage.KeyValueStore) {
// so we'll at least log it.
b.StopTimer()
tStart := time.Now()
importBigPathset(b, store)
importBigPathset(b, ctx, store)
b.Logf("importing took %s", time.Since(tStart).String())
b.StartTimer()
}
}
func cleanupStore(b *testing.B, store storage.KeyValueStore) {
func cleanupStore(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
b.Helper()
if !*noCleanDb {
tStart := time.Now()
cleanupBigPathset(b, store)
cleanupBigPathset(b, ctx, store)
b.Logf("cleanup took %s", time.Since(tStart).String())
}
}
@ -283,7 +275,7 @@ type verifyOpts struct {
expectLastKey storage.Key
}
func benchAndVerifyIteration(b *testing.B, store storage.KeyValueStore, opts *verifyOpts) {
func benchAndVerifyIteration(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore, opts *verifyOpts) {
problems := 0
iteration := 0
@ -311,7 +303,7 @@ func benchAndVerifyIteration(b *testing.B, store storage.KeyValueStore, opts *ve
lookupSize := opts.batchSize
for iteration = 1; iteration <= opts.doIterations; iteration++ {
results, err := iterateItems(store, opts.iterateOpts, lookupSize)
results, err := iterateItems(ctx, store, opts.iterateOpts, lookupSize)
if err != nil {
fatalf("Failed to call iterateItems(): %v", err)
}
@ -377,7 +369,7 @@ func benchAndVerifyIteration(b *testing.B, store storage.KeyValueStore, opts *ve
}
}
func deepRecursive(b *testing.B, store storage.KeyValueStore) {
func deepRecursive(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
opts := &verifyOpts{
iterateOpts: storage.IterateOptions{
Prefix: storage.Key(largestLevel2Directory),
@ -397,10 +389,10 @@ func deepRecursive(b *testing.B, store storage.KeyValueStore) {
// where $1 = largestLevel2Directory, $2 = doIterations, and $3 = batchSize
opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/extrastate/firewood/renomination/cletch/herotheism/aluminiferous/nub")
benchAndVerifyIteration(b, store, opts)
benchAndVerifyIteration(b, ctx, store, opts)
}
func deepNonRecursive(b *testing.B, store storage.KeyValueStore) {
func deepNonRecursive(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
opts := &verifyOpts{
iterateOpts: storage.IterateOptions{
Prefix: storage.Key(largestLevel2Directory),
@ -422,10 +414,10 @@ func deepNonRecursive(b *testing.B, store storage.KeyValueStore) {
// where $1 is largestLevel2Directory
opts.expectLastKey = storage.Key("Peronosporales/hateless/xerophily/")
benchAndVerifyIteration(b, store, opts)
benchAndVerifyIteration(b, ctx, store, opts)
}
func shallowRecursive(b *testing.B, store storage.KeyValueStore) {
func shallowRecursive(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
opts := &verifyOpts{
iterateOpts: storage.IterateOptions{
Prefix: storage.Key(largestSingleDirectory),
@ -451,10 +443,10 @@ func shallowRecursive(b *testing.B, store storage.KeyValueStore) {
opts.doIterations = 74
opts.batchSize = 251
benchAndVerifyIteration(b, store, opts)
benchAndVerifyIteration(b, ctx, store, opts)
}
func shallowNonRecursive(b *testing.B, store storage.KeyValueStore) {
func shallowNonRecursive(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
opts := &verifyOpts{
iterateOpts: storage.IterateOptions{
Prefix: storage.Key(largestSingleDirectory),
@ -476,10 +468,10 @@ func shallowNonRecursive(b *testing.B, store storage.KeyValueStore) {
// where $1 = largestSingleDirectory
opts.expectLastKey = storage.Key("Peronosporales/hateless/tod/unricht/sniveling/Puyallup/élite")
benchAndVerifyIteration(b, store, opts)
benchAndVerifyIteration(b, ctx, store, opts)
}
func topRecursiveLimit(b *testing.B, store storage.KeyValueStore) {
func topRecursiveLimit(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
opts := &verifyOpts{
iterateOpts: storage.IterateOptions{
Recurse: true,
@ -498,10 +490,10 @@ func topRecursiveLimit(b *testing.B, store storage.KeyValueStore) {
// where $1 = expectCount
opts.expectLastKey = storage.Key("nonresuscitation/synchronically/bechern/hemangiomatosis")
benchAndVerifyIteration(b, store, opts)
benchAndVerifyIteration(b, ctx, store, opts)
}
func topRecursiveStartAt(b *testing.B, store storage.KeyValueStore) {
func topRecursiveStartAt(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
opts := &verifyOpts{
iterateOpts: storage.IterateOptions{
Recurse: true,
@ -523,10 +515,10 @@ func topRecursiveStartAt(b *testing.B, store storage.KeyValueStore) {
// where $1 = iterateOpts.First and $2 = expectCount
opts.expectLastKey = storage.Key("raptured/heathbird/histrionism/vermifugous/barefaced/beechdrops/lamber/phlegmatic/blended/Gershon/scallop/burglarproof/incompensated/allanite/alehouse/embroilment/lienotoxin/monotonically/cumbersomeness")
benchAndVerifyIteration(b, store, opts)
benchAndVerifyIteration(b, ctx, store, opts)
}
func topNonRecursive(b *testing.B, store storage.KeyValueStore) {
func topNonRecursive(b *testing.B, ctx *testcontext.Context, store storage.KeyValueStore) {
opts := &verifyOpts{
iterateOpts: storage.IterateOptions{
Recurse: false,
@ -545,10 +537,10 @@ func topNonRecursive(b *testing.B, store storage.KeyValueStore) {
// ) x order by fp desc limit 1;
opts.expectLastKey = storage.Key("vejoces")
benchAndVerifyIteration(b, store, opts)
benchAndVerifyIteration(b, ctx, store, opts)
}
func cleanupBigPathset(tb testing.TB, store storage.KeyValueStore) {
func cleanupBigPathset(tb testing.TB, ctx *testcontext.Context, store storage.KeyValueStore) {
if *noCleanDb {
tb.Skip("Instructed not to clean up this KeyValueStore after long benchmarks are complete.")
}
@ -556,7 +548,7 @@ func cleanupBigPathset(tb testing.TB, store storage.KeyValueStore) {
cleaner, ok := store.(BulkCleaner)
if ok {
tb.Log("Performing bulk cleanup...")
err := cleaner.BulkDelete()
err := cleaner.BulkDeleteAll(ctx)
if err != nil {
tb.Fatalf("Provided KeyValueStore failed to perform bulk delete: %v", err)

23
storage/testsuite/test.go
View File

@ -14,26 +14,29 @@ import (
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
// RunTests runs common storage.KeyValueStore tests
func RunTests(t *testing.T, store storage.KeyValueStore) {
// store = storelogger.NewTest(t, store)
ctx := testcontext.New(t)
defer ctx.Cleanup()
t.Run("CRUD", func(t *testing.T) { testCRUD(t, store) })
t.Run("Constraints", func(t *testing.T) { testConstraints(t, store) })
t.Run("Iterate", func(t *testing.T) { testIterate(t, store) })
t.Run("IterateAll", func(t *testing.T) { testIterateAll(t, store) })
t.Run("Prefix", func(t *testing.T) { testPrefix(t, store) })
t.Run("CRUD", func(t *testing.T) { testCRUD(t, ctx, store) })
t.Run("Constraints", func(t *testing.T) { testConstraints(t, ctx, store) })
t.Run("Iterate", func(t *testing.T) { testIterate(t, ctx, store) })
t.Run("IterateAll", func(t *testing.T) { testIterateAll(t, ctx, store) })
t.Run("Prefix", func(t *testing.T) { testPrefix(t, ctx, store) })
t.Run("List", func(t *testing.T) { testList(t, store) })
t.Run("ListV2", func(t *testing.T) { testListV2(t, store) })
t.Run("List", func(t *testing.T) { testList(t, ctx, store) })
t.Run("ListV2", func(t *testing.T) { testListV2(t, ctx, store) })
t.Run("Parallel", func(t *testing.T) { testParallel(t, store) })
t.Run("Parallel", func(t *testing.T) { testParallel(t, ctx, store) })
}
func testConstraints(t *testing.T, store storage.KeyValueStore) {
func testConstraints(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
var items storage.Items
for i := 0; i < storage.LookupLimit+5; i++ {
items = append(items, storage.ListItem{
@ -53,7 +56,7 @@ func testConstraints(t *testing.T, store storage.KeyValueStore) {
if err := group.Wait(); err != nil {
t.Fatalf("Put failed: %v", err)
}
defer cleanupItems(store, items)
defer cleanupItems(t, ctx, store, items)
t.Run("Put Empty", func(t *testing.T) {
var key storage.Key

5
storage/testsuite/test_crud.go
View File

@ -8,10 +8,11 @@ import (
"math/rand"
"testing"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
func testCRUD(t *testing.T, store storage.KeyValueStore) {
func testCRUD(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
// newItem("0", "", false), //TODO: broken
newItem("\x00", "\x00", false),
@ -25,7 +26,7 @@ func testCRUD(t *testing.T, store storage.KeyValueStore) {
newItem("öö", "üü", false),
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
defer cleanupItems(t, ctx, store, items)
t.Run("Put", func(t *testing.T) {
for _, item := range items {

11
storage/testsuite/test_iterate.go
View File

@ -4,16 +4,14 @@
package testsuite
import (
"context"
"math/rand"
"testing"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
var ctx = context.Background() // test context
func testIterate(t *testing.T, store storage.KeyValueStore) {
func testIterate(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
newItem("a", "a", false),
newItem("b/1", "b/1", false),
@ -27,12 +25,13 @@ func testIterate(t *testing.T, store storage.KeyValueStore) {
newItem("h", "h", false),
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
defer cleanupItems(t, ctx, store, items)
if err := storage.PutAll(ctx, store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
testIterations(t, store, []iterationTest{
testIterations(t, ctx, store, []iterationTest{
{"no limits",
storage.IterateOptions{}, storage.Items{
newItem("a", "a", false),

8
storage/testsuite/test_iterate_all.go
View File

@ -7,10 +7,11 @@ import (
"math/rand"
"testing"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
func testIterateAll(t *testing.T, store storage.KeyValueStore) {
func testIterateAll(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
newItem("a", "a", false),
newItem("b/1", "b/1", false),
@ -24,12 +25,13 @@ func testIterateAll(t *testing.T, store storage.KeyValueStore) {
newItem("h", "h", false),
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
defer cleanupItems(t, ctx, store, items)
if err := storage.PutAll(ctx, store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
testIterations(t, store, []iterationTest{
testIterations(t, ctx, store, []iterationTest{
{"no limits",
storage.IterateOptions{
Recurse: true,

5
storage/testsuite/test_list.go
View File

@ -10,10 +10,11 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
func testList(t *testing.T, store storage.KeyValueStore) {
func testList(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
newItem("path/0", "\x00\xFF\x00", false),
newItem("path/1", "\x01\xFF\x01", false),
@ -23,8 +24,8 @@ func testList(t *testing.T, store storage.KeyValueStore) {
newItem("path/5", "\x05\xFF\x05", false),
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(t, ctx, store, items)
defer cleanupItems(store, items)
if err := storage.PutAll(ctx, store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}

6
storage/testsuite/test_listv2.go
View File

@ -11,10 +11,11 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
func testListV2(t *testing.T, store storage.KeyValueStore) {
func testListV2(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
newItem("music/a-song1.mp3", "1", false),
newItem("music/a-song2.mp3", "2", false),
@ -25,7 +26,8 @@ func testListV2(t *testing.T, store storage.KeyValueStore) {
newItem("videos/movie.mkv", "7", false),
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
defer cleanupItems(t, ctx, store, items)
if err := storage.PutAll(ctx, store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}

5
storage/testsuite/test_parallel.go
View File

@ -9,17 +9,18 @@ import (
"strconv"
"testing"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
func testParallel(t *testing.T, store storage.KeyValueStore) {
func testParallel(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
newItem("a", "1", false),
newItem("b", "2", false),
newItem("c", "3", false),
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
defer cleanupItems(t, ctx, store, items)
for idx := range items {
i := idx

8
storage/testsuite/test_prefix.go
View File

@ -7,10 +7,11 @@ import (
"math/rand"
"testing"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
func testPrefix(t *testing.T, store storage.KeyValueStore) {
func testPrefix(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
newItem("x-a", "a", false),
newItem("x-b/1", "b/1", false),
@ -24,12 +25,13 @@ func testPrefix(t *testing.T, store storage.KeyValueStore) {
newItem("y-h", "h", false),
}
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(store, items)
defer cleanupItems(t, ctx, store, items)
if err := storage.PutAll(ctx, store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
testIterations(t, store, []iterationTest{
testIterations(t, ctx, store, []iterationTest{
{"prefix x dash b slash",
storage.IterateOptions{
Prefix: storage.Key("x-"), First: storage.Key("x-b"),

39
storage/testsuite/utils.go
View File

@ -10,6 +10,7 @@ import (
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"storj.io/storj/private/testcontext"
"storj.io/storj/storage"
)
@ -21,22 +22,46 @@ func newItem(key, value string, isPrefix bool) storage.ListItem {
}
}
func cleanupItems(store storage.KeyValueStore, items storage.Items) {
for _, item := range items {
_ = store.Delete(ctx, item.Key)
func cleanupItems(t testing.TB, ctx *testcontext.Context, store storage.KeyValueStore, items storage.Items) {
bulkDeleter, ok := store.(BulkDeleter)
if ok {
err := bulkDeleter.BulkDelete(ctx, items)
if err != nil {
t.Fatalf("could not do bulk cleanup of items: %v", err)
}
} else {
for _, item := range items {
_ = store.Delete(ctx, item.Key)
}
}
}
// BulkImporter identifies KV storage facilities that can do bulk importing of items more
// efficiently than inserting one-by-one.
type BulkImporter interface {
BulkImport(context.Context, storage.Iterator) error
}
// BulkDeleter identifies KV storage facilities that can delete multiple items efficiently.
type BulkDeleter interface {
BulkDelete(context.Context, storage.Items) error
}
// BulkCleaner identifies KV storage facilities that can delete all items efficiently.
type BulkCleaner interface {
BulkDeleteAll(ctx context.Context) error
}
type iterationTest struct {
Name string
Options storage.IterateOptions
Expected storage.Items
}
func testIterations(t *testing.T, store storage.KeyValueStore, tests []iterationTest) {
func testIterations(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore, tests []iterationTest) {
t.Helper()
for _, test := range tests {
items, err := iterateItems(store, test.Options, -1)
items, err := iterateItems(ctx, store, test.Options, -1)
if err != nil {
t.Errorf("%s: %v", test.Name, err)
continue
@ -47,7 +72,7 @@ func testIterations(t *testing.T, store storage.KeyValueStore, tests []iteration
}
}
func isEmptyKVStore(tb testing.TB, store storage.KeyValueStore) bool {
func isEmptyKVStore(tb testing.TB, ctx *testcontext.Context, store storage.KeyValueStore) bool {
tb.Helper()
keys, err := store.List(ctx, storage.Key(""), 1)
if err != nil {
@ -69,7 +94,7 @@ func (collect *collector) include(ctx context.Context, it storage.Iterator) erro
return nil
}
func iterateItems(store storage.KeyValueStore, opts storage.IterateOptions, limit int) (storage.Items, error) {
func iterateItems(ctx *testcontext.Context, store storage.KeyValueStore, opts storage.IterateOptions, limit int) (storage.Items, error) {
collect := &collector{Limit: limit}
err := store.Iterate(ctx, opts, collect.include)
if err != nil {