Metainfo buckets tests (#607)

This commit is contained in:
Kaloyan Raev 2018-11-12 15:23:19 +02:00 committed by GitHub
parent 5352778b3c
commit 9afa9b6b52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 386 additions and 13 deletions

View File

@ -21,7 +21,7 @@ func NewBuckets(store buckets.Store) *Buckets { return &Buckets{store} }
// CreateBucket creates a new bucket with the specified information
func (db *Buckets) CreateBucket(ctx context.Context, bucket string, info *storj.Bucket) (storj.Bucket, error) {
if bucket == "" {
return storj.Bucket{}, buckets.NoBucketError.New("")
return storj.Bucket{}, storj.ErrNoBucket.New("")
}
meta, err := db.store.Put(ctx, bucket)
@ -35,7 +35,7 @@ func (db *Buckets) CreateBucket(ctx context.Context, bucket string, info *storj.
// DeleteBucket deletes bucket
func (db *Buckets) DeleteBucket(ctx context.Context, bucket string) error {
if bucket == "" {
return buckets.NoBucketError.New("")
return storj.ErrNoBucket.New("")
}
return db.store.Delete(ctx, bucket)
@ -44,7 +44,7 @@ func (db *Buckets) DeleteBucket(ctx context.Context, bucket string) error {
// GetBucket gets bucket information
func (db *Buckets) GetBucket(ctx context.Context, bucket string) (storj.Bucket, error) {
if bucket == "" {
return storj.Bucket{}, buckets.NoBucketError.New("")
return storj.Bucket{}, storj.ErrNoBucket.New("")
}
meta, err := db.store.Get(ctx, bucket)
@ -75,6 +75,11 @@ func (db *Buckets) ListBuckets(ctx context.Context, options storj.BucketListOpti
return storj.BucketList{}, errClass.New("invalid direction %d", options.Direction)
}
// TODO: remove this hack-fix of specifying the last key
if options.Cursor == "" && (options.Direction == storj.Before || options.Direction == storj.Backward) {
endBefore = "\x7f\x7f\x7f\x7f\x7f\x7f\x7f"
}
items, more, err := db.store.List(ctx, startAfter, endBefore, options.Limit)
if err != nil {
return storj.BucketList{}, err

View File

@ -0,0 +1,364 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package kvmetainfo
import (
"context"
"flag"
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/vivint/infectious"
"storj.io/storj/internal/memory"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testplanet"
"storj.io/storj/pkg/eestream"
"storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/storage/buckets"
"storj.io/storj/pkg/storage/ec"
"storj.io/storj/pkg/storage/objects"
"storj.io/storj/pkg/storage/segments"
"storj.io/storj/pkg/storage/streams"
"storj.io/storj/pkg/storj"
"storj.io/storj/storage"
)
const (
TestAPIKey = "test-api-key"
TestEncKey = "test-encryption-key"
TestBucket = "testbucket"
)
func TestBucketsBasic(t *testing.T) {
runTest(t, func(ctx context.Context, bucketStore buckets.Store) {
buckets := NewBuckets(bucketStore)
// Create new bucket
bucket, err := buckets.CreateBucket(ctx, TestBucket, nil)
if assert.NoError(t, err) {
assert.Equal(t, TestBucket, bucket.Name)
}
// Check that bucket list include the new bucket
bucketList, err := buckets.ListBuckets(ctx, storj.BucketListOptions{Direction: storj.After})
if assert.NoError(t, err) {
assert.False(t, bucketList.More)
assert.Equal(t, 1, len(bucketList.Items))
assert.Equal(t, TestBucket, bucketList.Items[0].Name)
}
// Check that we can get the new bucket explicitly
bucket, err = buckets.GetBucket(ctx, TestBucket)
if assert.NoError(t, err) {
assert.Equal(t, TestBucket, bucket.Name)
}
// Delete the bucket
err = buckets.DeleteBucket(ctx, TestBucket)
assert.NoError(t, err)
// Check that the bucket list is empty
bucketList, err = buckets.ListBuckets(ctx, storj.BucketListOptions{Direction: storj.After})
if assert.NoError(t, err) {
assert.False(t, bucketList.More)
assert.Equal(t, 0, len(bucketList.Items))
}
// Check that the bucket cannot be get explicitly
bucket, err = buckets.GetBucket(ctx, TestBucket)
assert.True(t, storage.ErrKeyNotFound.Has(err))
})
}
func TestBucketsReadNewWayWriteOldWay(t *testing.T) {
runTest(t, func(ctx context.Context, bucketStore buckets.Store) {
buckets := NewBuckets(bucketStore)
// (Old API) Create new bucket
_, err := bucketStore.Put(ctx, TestBucket)
assert.NoError(t, err)
// (New API) Check that bucket list include the new bucket
bucketList, err := buckets.ListBuckets(ctx, storj.BucketListOptions{Direction: storj.After})
if assert.NoError(t, err) {
assert.False(t, bucketList.More)
assert.Equal(t, 1, len(bucketList.Items))
assert.Equal(t, TestBucket, bucketList.Items[0].Name)
}
// (New API) Check that we can get the new bucket explicitly
bucket, err := buckets.GetBucket(ctx, TestBucket)
if assert.NoError(t, err) {
assert.Equal(t, TestBucket, bucket.Name)
}
// (Old API) Delete the bucket
err = bucketStore.Delete(ctx, TestBucket)
assert.NoError(t, err)
// (New API) Check that the bucket list is empty
bucketList, err = buckets.ListBuckets(ctx, storj.BucketListOptions{Direction: storj.After})
if assert.NoError(t, err) {
assert.False(t, bucketList.More)
assert.Equal(t, 0, len(bucketList.Items))
}
// (New API) Check that the bucket cannot be get explicitly
bucket, err = buckets.GetBucket(ctx, TestBucket)
assert.True(t, storage.ErrKeyNotFound.Has(err))
})
}
func TestBucketsReadOldWayWriteNewWay(t *testing.T) {
runTest(t, func(ctx context.Context, bucketStore buckets.Store) {
buckets := NewBuckets(bucketStore)
// (New API) Create new bucket
bucket, err := buckets.CreateBucket(ctx, TestBucket, nil)
if assert.NoError(t, err) {
assert.Equal(t, TestBucket, bucket.Name)
}
// (Old API) Check that bucket list include the new bucket
items, more, err := bucketStore.List(ctx, "", "", 0)
if assert.NoError(t, err) {
assert.False(t, more)
assert.Equal(t, 1, len(items))
assert.Equal(t, TestBucket, items[0].Bucket)
}
// (Old API) Check that we can get the new bucket explicitly
_, err = bucketStore.Get(ctx, TestBucket)
assert.NoError(t, err)
// (New API) Delete the bucket
err = buckets.DeleteBucket(ctx, TestBucket)
assert.NoError(t, err)
// (Old API) Check that the bucket list is empty
items, more, err = bucketStore.List(ctx, "", "", 0)
if assert.NoError(t, err) {
assert.False(t, more)
assert.Equal(t, 0, len(items))
}
// (Old API) Check that the bucket cannot be get explicitly
_, err = bucketStore.Get(ctx, TestBucket)
assert.True(t, storage.ErrKeyNotFound.Has(err))
})
}
func TestErrNoBucket(t *testing.T) {
runTest(t, func(ctx context.Context, bucketStore buckets.Store) {
buckets := NewBuckets(bucketStore)
_, err := buckets.CreateBucket(ctx, "", nil)
assert.True(t, storj.ErrNoBucket.Has(err))
_, err = buckets.GetBucket(ctx, "")
assert.True(t, storj.ErrNoBucket.Has(err))
err = buckets.DeleteBucket(ctx, "")
assert.True(t, storj.ErrNoBucket.Has(err))
})
}
func TestListBucketsEmpty(t *testing.T) {
runTest(t, func(ctx context.Context, bucketStore buckets.Store) {
buckets := NewBuckets(bucketStore)
_, err := buckets.ListBuckets(ctx, storj.BucketListOptions{})
assert.EqualError(t, err, "kvmetainfo: invalid direction 0")
for _, direction := range []storj.ListDirection{
storj.Before,
storj.Backward,
storj.Forward,
storj.After,
} {
bucketList, err := buckets.ListBuckets(ctx, storj.BucketListOptions{Direction: direction})
if assert.NoError(t, err) {
assert.False(t, bucketList.More)
assert.Equal(t, 0, len(bucketList.Items))
}
}
})
}
func TestListBuckets(t *testing.T) {
runTest(t, func(ctx context.Context, bucketStore buckets.Store) {
buckets := NewBuckets(bucketStore)
bucketNames := []string{"a", "aa", "b", "bb", "c"}
for _, name := range bucketNames {
_, err := buckets.CreateBucket(ctx, name, nil)
if !assert.NoError(t, err) {
return
}
}
for i, tt := range []struct {
cursor string
dir storj.ListDirection
limit int
more bool
result []string
}{
{cursor: "", dir: storj.After, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "`", dir: storj.After, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "b", dir: storj.After, limit: 0, more: false, result: []string{"bb", "c"}},
{cursor: "c", dir: storj.After, limit: 0, more: false, result: []string{}},
{cursor: "ca", dir: storj.After, limit: 0, more: false, result: []string{}},
{cursor: "", dir: storj.After, limit: 1, more: true, result: []string{"a"}},
{cursor: "`", dir: storj.After, limit: 1, more: true, result: []string{"a"}},
{cursor: "aa", dir: storj.After, limit: 1, more: true, result: []string{"b"}},
{cursor: "c", dir: storj.After, limit: 1, more: false, result: []string{}},
{cursor: "ca", dir: storj.After, limit: 1, more: false, result: []string{}},
{cursor: "", dir: storj.After, limit: 2, more: true, result: []string{"a", "aa"}},
{cursor: "`", dir: storj.After, limit: 2, more: true, result: []string{"a", "aa"}},
{cursor: "aa", dir: storj.After, limit: 2, more: true, result: []string{"b", "bb"}},
{cursor: "bb", dir: storj.After, limit: 2, more: false, result: []string{"c"}},
{cursor: "c", dir: storj.After, limit: 2, more: false, result: []string{}},
{cursor: "ca", dir: storj.After, limit: 2, more: false, result: []string{}},
{cursor: "", dir: storj.Forward, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "`", dir: storj.Forward, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "b", dir: storj.Forward, limit: 0, more: false, result: []string{"b", "bb", "c"}},
{cursor: "c", dir: storj.Forward, limit: 0, more: false, result: []string{"c"}},
{cursor: "ca", dir: storj.Forward, limit: 0, more: false, result: []string{}},
{cursor: "", dir: storj.Forward, limit: 1, more: true, result: []string{"a"}},
{cursor: "`", dir: storj.Forward, limit: 1, more: true, result: []string{"a"}},
{cursor: "aa", dir: storj.Forward, limit: 1, more: true, result: []string{"aa"}},
{cursor: "c", dir: storj.Forward, limit: 1, more: false, result: []string{"c"}},
{cursor: "ca", dir: storj.Forward, limit: 1, more: false, result: []string{}},
{cursor: "", dir: storj.Forward, limit: 2, more: true, result: []string{"a", "aa"}},
{cursor: "`", dir: storj.Forward, limit: 2, more: true, result: []string{"a", "aa"}},
{cursor: "aa", dir: storj.Forward, limit: 2, more: true, result: []string{"aa", "b"}},
{cursor: "bb", dir: storj.Forward, limit: 2, more: false, result: []string{"bb", "c"}},
{cursor: "c", dir: storj.Forward, limit: 2, more: false, result: []string{"c"}},
{cursor: "ca", dir: storj.Forward, limit: 2, more: false, result: []string{}},
{cursor: "", dir: storj.Backward, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "`", dir: storj.Backward, limit: 0, more: false, result: []string{}},
{cursor: "b", dir: storj.Backward, limit: 0, more: false, result: []string{"a", "aa", "b"}},
{cursor: "c", dir: storj.Backward, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "ca", dir: storj.Backward, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "", dir: storj.Backward, limit: 1, more: true, result: []string{"c"}},
{cursor: "`", dir: storj.Backward, limit: 1, more: false, result: []string{}},
{cursor: "aa", dir: storj.Backward, limit: 1, more: true, result: []string{"aa"}},
{cursor: "c", dir: storj.Backward, limit: 1, more: true, result: []string{"c"}},
{cursor: "ca", dir: storj.Backward, limit: 1, more: true, result: []string{"c"}},
{cursor: "", dir: storj.Backward, limit: 2, more: true, result: []string{"bb", "c"}},
{cursor: "`", dir: storj.Backward, limit: 2, more: false, result: []string{}},
{cursor: "aa", dir: storj.Backward, limit: 2, more: false, result: []string{"a", "aa"}},
{cursor: "bb", dir: storj.Backward, limit: 2, more: true, result: []string{"b", "bb"}},
{cursor: "c", dir: storj.Backward, limit: 2, more: true, result: []string{"bb", "c"}},
{cursor: "ca", dir: storj.Backward, limit: 2, more: true, result: []string{"bb", "c"}},
{cursor: "", dir: storj.Before, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "`", dir: storj.Before, limit: 0, more: false, result: []string{}},
{cursor: "b", dir: storj.Before, limit: 0, more: false, result: []string{"a", "aa"}},
{cursor: "c", dir: storj.Before, limit: 0, more: false, result: []string{"a", "aa", "b", "bb"}},
{cursor: "ca", dir: storj.Before, limit: 0, more: false, result: []string{"a", "aa", "b", "bb", "c"}},
{cursor: "", dir: storj.Before, limit: 1, more: true, result: []string{"c"}},
{cursor: "`", dir: storj.Before, limit: 1, more: false, result: []string{}},
{cursor: "aa", dir: storj.Before, limit: 1, more: false, result: []string{"a"}},
{cursor: "c", dir: storj.Before, limit: 1, more: true, result: []string{"bb"}},
{cursor: "ca", dir: storj.Before, limit: 1, more: true, result: []string{"c"}},
{cursor: "", dir: storj.Before, limit: 2, more: true, result: []string{"bb", "c"}},
{cursor: "`", dir: storj.Before, limit: 2, more: false, result: []string{}},
{cursor: "aa", dir: storj.Before, limit: 2, more: false, result: []string{"a"}},
{cursor: "bb", dir: storj.Before, limit: 2, more: true, result: []string{"aa", "b"}},
{cursor: "c", dir: storj.Before, limit: 2, more: true, result: []string{"b", "bb"}},
{cursor: "ca", dir: storj.Before, limit: 2, more: true, result: []string{"bb", "c"}},
} {
errTag := fmt.Sprintf("%d. %+v", i, tt)
bucketList, err := buckets.ListBuckets(ctx, storj.BucketListOptions{
Cursor: tt.cursor,
Direction: tt.dir,
Limit: tt.limit,
})
if assert.NoError(t, err, errTag) {
assert.Equal(t, tt.more, bucketList.More, errTag)
assert.Equal(t, tt.result, getBucketNames(bucketList), errTag)
}
}
})
}
func getBucketNames(bucketList storj.BucketList) []string {
names := make([]string, len(bucketList.Items))
for i, item := range bucketList.Items {
names[i] = item.Name
}
return names
}
func runTest(t *testing.T, test func(context.Context, buckets.Store)) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
planet, err := testplanet.New(1, 4, 1)
if !assert.NoError(t, err) {
return
}
defer ctx.Check(planet.Shutdown)
planet.Start(context.Background())
bucketStore, err := newBucketStore(planet)
if !assert.NoError(t, err) {
return
}
test(ctx, bucketStore)
}
func newBucketStore(planet *testplanet.Planet) (buckets.Store, error) {
// TODO(kaloyan): We should have a better way for configuring the Satellite's API Key
err := flag.Set("pointer-db.auth.api-key", TestAPIKey)
if err != nil {
return nil, err
}
oc, err := overlay.NewOverlayClient(planet.Uplinks[0].Identity, planet.Uplinks[0].Addr())
if err != nil {
return nil, err
}
pdb, err := planet.Uplinks[0].DialPointerDB(planet.Satellites[0], TestAPIKey)
if err != nil {
return nil, err
}
ec := ecclient.NewClient(planet.Uplinks[0].Identity, 0)
fc, err := infectious.NewFEC(2, 4)
if err != nil {
return nil, err
}
rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, int(1*memory.KB)), 3, 4)
if err != nil {
return nil, err
}
segments := segments.NewSegmentStore(oc, ec, pdb, rs, int(8*memory.KB))
key := new(storj.Key)
copy(key[:], TestEncKey)
stream, err := streams.NewStreamStore(segments, int64(64*memory.MB), key, int(1*memory.KB), storj.AESGCM)
if err != nil {
return nil, err
}
obj := objects.NewStore(stream)
return buckets.NewStore(obj), nil
}

View File

@ -24,7 +24,7 @@ func keyBefore(cursor string) string {
}
before[len(before)-1]--
before = append(before, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF)
before = append(before, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f)
return string(before)
}

View File

@ -9,19 +9,16 @@ import (
"time"
minio "github.com/minio/minio/cmd"
"github.com/zeebo/errs"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storage/objects"
"storj.io/storj/pkg/storj"
"storj.io/storj/storage"
)
var mon = monkit.Package()
// NoBucketError is an error class for missing bucket name
var NoBucketError = errs.Class("no bucket specified")
// Store creates an interface for interacting with buckets
type Store interface {
Get(ctx context.Context, bucket string) (meta Meta, err error)
@ -55,7 +52,7 @@ func NewStore(obj objects.Store) Store {
// GetObjectStore returns an implementation of objects.Store
func (b *BucketStore) GetObjectStore(ctx context.Context, bucket string) (objects.Store, error) {
if bucket == "" {
return nil, NoBucketError.New("")
return nil, storj.ErrNoBucket.New("")
}
_, err := b.Get(ctx, bucket)
@ -77,7 +74,7 @@ func (b *BucketStore) Get(ctx context.Context, bucket string) (meta Meta, err er
defer mon.Task()(&ctx)(&err)
if bucket == "" {
return Meta{}, NoBucketError.New("")
return Meta{}, storj.ErrNoBucket.New("")
}
objMeta, err := b.o.Meta(ctx, bucket)
@ -92,7 +89,7 @@ func (b *BucketStore) Put(ctx context.Context, bucket string) (meta Meta, err er
defer mon.Task()(&ctx)(&err)
if bucket == "" {
return Meta{}, NoBucketError.New("")
return Meta{}, storj.ErrNoBucket.New("")
}
r := bytes.NewReader(nil)
@ -109,7 +106,7 @@ func (b *BucketStore) Delete(ctx context.Context, bucket string) (err error) {
defer mon.Task()(&ctx)(&err)
if bucket == "" {
return NoBucketError.New("")
return storj.ErrNoBucket.New("")
}
return b.o.Delete(ctx, bucket)

View File

@ -3,7 +3,14 @@
package storj
import "time"
import (
"time"
"github.com/zeebo/errs"
)
// ErrNoBucket is an error class for using empty bucket name
var ErrNoBucket = errs.Class("no bucket specified")
// Bucket contains information about a specific bucket
type Bucket struct {