diff --git a/Jenkinsfile.public b/Jenkinsfile.public index 060eee51c..ddfd7dda9 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -157,7 +157,7 @@ pipeline { STORJ_TEST_COCKROACH_ALT = 'cockroach://root@localhost:26260/testcockroach?sslmode=disable' STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable' STORJ_TEST_LOG_LEVEL = 'info' - COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storage/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}" + COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}" } steps { sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\'' diff --git a/Jenkinsfile.verify b/Jenkinsfile.verify index ac123b552..40be3d83a 100644 --- a/Jenkinsfile.verify +++ b/Jenkinsfile.verify @@ -155,7 +155,7 @@ pipeline { STORJ_TEST_COCKROACH_ALT = 'cockroach://root@localhost:26260/testcockroach?sslmode=disable' STORJ_TEST_POSTGRES = 'omit' STORJ_TEST_LOG_LEVEL = 'info' - COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storage/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}" + COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}" } steps { sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\'' diff --git a/certificate/authorization/db.go b/certificate/authorization/db.go index 4bfc6ada5..be7dec17a 100644 --- a/certificate/authorization/db.go +++ b/certificate/authorization/db.go @@ -13,9 +13,9 @@ import ( "storj.io/common/identity" "storj.io/common/peertls/extensions" "storj.io/private/dbutil" - "storj.io/storj/storage" - "storj.io/storj/storage/boltdb" - "storj.io/storj/storage/redis" + "storj.io/storj/private/kvstore" + "storj.io/storj/private/kvstore/boltdb" + "storj.io/storj/private/kvstore/redis" ) var ( @@ -38,7 +38,7 @@ var ( // DB stores authorizations which may be claimed in exchange for a // certificate signature. type DB struct { - db storage.KeyValueStore + db kvstore.Store } // DBConfig is the authorization db config. @@ -127,8 +127,8 @@ func (authDB *DB) Create(ctx context.Context, userID string, count int) (_ Group // Get retrieves authorizations by user ID. func (authDB *DB) Get(ctx context.Context, userID string) (_ Group, err error) { defer mon.Task()(&ctx, userID)(&err) - authsBytes, err := authDB.db.Get(ctx, storage.Key(userID)) - if storage.ErrKeyNotFound.Has(err) { + authsBytes, err := authDB.db.Get(ctx, kvstore.Key(userID)) + if kvstore.ErrKeyNotFound.Has(err) { return nil, ErrNotFound.New("userID: %s", userID) } if err != nil { @@ -150,7 +150,7 @@ func (authDB *DB) UserIDs(ctx context.Context) (userIDs []string, err error) { defer mon.Task()(&ctx)(&err) err = authDB.db.Range(ctx, - func(ctx context.Context, key storage.Key, _ storage.Value) error { + func(ctx context.Context, key kvstore.Key, _ kvstore.Value) error { userIDs = append(userIDs, key.String()) return nil }) @@ -163,7 +163,7 @@ func (authDB *DB) List(ctx context.Context) (auths Group, err error) { var errs errs.Group err = authDB.db.Range(ctx, - func(ctx context.Context, key storage.Key, value storage.Value) error { + func(ctx context.Context, key kvstore.Key, value kvstore.Value) error { var group Group err := group.Unmarshal(value) if err != nil { @@ -288,7 +288,7 @@ func (authDB *DB) put(ctx context.Context, userID string, auths Group) (err erro return ErrDBInternal.Wrap(err) } - if err := authDB.db.Put(ctx, storage.Key(userID), authsBytes); err != nil { + if err := authDB.db.Put(ctx, kvstore.Key(userID), authsBytes); err != nil { return ErrDBInternal.Wrap(err) } return nil diff --git a/certificate/authorization/db_test.go b/certificate/authorization/db_test.go index d250a5757..030891f67 100644 --- a/certificate/authorization/db_test.go +++ b/certificate/authorization/db_test.go @@ -20,7 +20,7 @@ import ( "storj.io/common/rpc/rpcpeer" "storj.io/common/testcontext" "storj.io/storj/certificate/certificatepb" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) func TestNewDB(t *testing.T) { @@ -86,7 +86,7 @@ func TestAuthorizationDB_Create(t *testing.T) { for _, c := range cases { testCase := c t.Run(c.testID, func(t *testing.T) { - emailKey := storage.Key(testCase.email) + emailKey := kvstore.Key(testCase.email) if testCase.startCount == 0 { _, err := authDB.db.Get(ctx, emailKey) @@ -174,7 +174,7 @@ func TestAuthorizationDB_Get(t *testing.T) { authsBytes, err := expectedAuths.Marshal() require.NoError(t, err) - err = authDB.db.Put(ctx, storage.Key("user@mail.test"), authsBytes) + err = authDB.db.Put(ctx, kvstore.Key("user@mail.test"), authsBytes) require.NoError(t, err) { diff --git a/storage/boltdb/client.go b/private/kvstore/boltdb/client.go similarity index 82% rename from storage/boltdb/client.go rename to private/kvstore/boltdb/client.go index ef59483aa..6660b4363 100644 --- a/storage/boltdb/client.go +++ b/private/kvstore/boltdb/client.go @@ -12,7 +12,7 @@ import ( "github.com/zeebo/errs" "go.etcd.io/bbolt" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) var mon = monkit.Package() @@ -87,11 +87,11 @@ func (client *Client) view(fn func(*bbolt.Bucket) error) error { // Ref: https://github.com/boltdb/bolt/blob/master/db.go#L160 // Note: when using this method, check if it needs to be executed asynchronously // since it blocks for the duration db.MaxBatchDelay. -func (client *Client) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) { +func (client *Client) Put(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) { defer mon.Task()(&ctx)(&err) start := time.Now() if key.IsZero() { - return storage.ErrEmptyKey.New("") + return kvstore.ErrEmptyKey.New("") } err = client.batch(func(bucket *bbolt.Bucket) error { @@ -102,10 +102,10 @@ func (client *Client) Put(ctx context.Context, key storage.Key, value storage.Va } // PutAndCommit adds a key/value to BoltDB and writes it to disk. -func (client *Client) PutAndCommit(ctx context.Context, key storage.Key, value storage.Value) (err error) { +func (client *Client) PutAndCommit(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) { defer mon.Task()(&ctx)(&err) if key.IsZero() { - return storage.ErrEmptyKey.New("") + return kvstore.ErrEmptyKey.New("") } return client.update(func(bucket *bbolt.Bucket) error { @@ -114,29 +114,29 @@ func (client *Client) PutAndCommit(ctx context.Context, key storage.Key, value s } // Get looks up the provided key from boltdb returning either an error or the result. -func (client *Client) Get(ctx context.Context, key storage.Key) (_ storage.Value, err error) { +func (client *Client) Get(ctx context.Context, key kvstore.Key) (_ kvstore.Value, err error) { defer mon.Task()(&ctx)(&err) if key.IsZero() { - return nil, storage.ErrEmptyKey.New("") + return nil, kvstore.ErrEmptyKey.New("") } - var value storage.Value + var value kvstore.Value err = client.view(func(bucket *bbolt.Bucket) error { data := bucket.Get([]byte(key)) if len(data) == 0 { - return storage.ErrKeyNotFound.New("%q", key) + return kvstore.ErrKeyNotFound.New("%q", key) } - value = storage.CloneValue(storage.Value(data)) + value = kvstore.CloneValue(kvstore.Value(data)) return nil }) return value, err } // Delete deletes a key/value pair from boltdb, for a given the key. -func (client *Client) Delete(ctx context.Context, key storage.Key) (err error) { +func (client *Client) Delete(ctx context.Context, key kvstore.Key) (err error) { defer mon.Task()(&ctx)(&err) if key.IsZero() { - return storage.ErrEmptyKey.New("") + return kvstore.ErrEmptyKey.New("") } return client.update(func(bucket *bbolt.Bucket) error { @@ -153,12 +153,12 @@ func (client *Client) Close() (err error) { } // Range iterates over all items in unspecified order. -func (client *Client) Range(ctx context.Context, fn func(context.Context, storage.Key, storage.Value) error) (err error) { +func (client *Client) Range(ctx context.Context, fn func(context.Context, kvstore.Key, kvstore.Value) error) (err error) { defer mon.Task()(&ctx)(&err) return client.view(func(bucket *bbolt.Bucket) error { return bucket.ForEach(func(k, v []byte) error { - return fn(ctx, storage.Key(k), storage.Value(v)) + return fn(ctx, kvstore.Key(k), kvstore.Value(v)) }) }) } diff --git a/storage/boltdb/client_test.go b/private/kvstore/boltdb/client_test.go similarity index 95% rename from storage/boltdb/client_test.go rename to private/kvstore/boltdb/client_test.go index 0a7f519fe..3ff85d046 100644 --- a/storage/boltdb/client_test.go +++ b/private/kvstore/boltdb/client_test.go @@ -8,7 +8,7 @@ import ( "path/filepath" "testing" - "storj.io/storj/storage/testsuite" + "storj.io/storj/private/kvstore/testsuite" ) func TestSuite(t *testing.T) { diff --git a/storage/common.go b/private/kvstore/common.go similarity index 85% rename from storage/common.go rename to private/kvstore/common.go index d63f64212..a01e23b74 100644 --- a/storage/common.go +++ b/private/kvstore/common.go @@ -1,7 +1,7 @@ // Copyright (C) 2019 Storj Labs, Inc. // See LICENSE for copying information. -package storage +package kvstore import ( "bytes" @@ -22,30 +22,30 @@ var ErrKeyNotFound = errs.Class("key not found") // ErrEmptyKey is returned when an empty key is used in Put or in CompareAndSwap. var ErrEmptyKey = errs.Class("empty key") -// Key is the type for the keys in a `KeyValueStore`. +// Key is the type for the keys in a `Store`. type Key []byte // Value is the type for the values in a `ValueValueStore`. type Value []byte -// Keys is the type for a slice of keys in a `KeyValueStore`. +// Keys is the type for a slice of keys in a `Store`. type Keys []Key -// Values is the type for a slice of Values in a `KeyValueStore`. +// Values is the type for a slice of Values in a `Store`. type Values []Value -// Items keeps all ListItem. -type Items []ListItem +// Items keeps all Item. +type Items []Item -// ListItem returns Key, Value, IsPrefix. -type ListItem struct { +// Item returns Key, Value, IsPrefix. +type Item struct { Key Key Value Value IsPrefix bool } -// KeyValueStore describes key/value stores like redis and boltdb. -type KeyValueStore interface { +// Store describes key/value stores like redis and boltdb. +type Store interface { // Put adds a value to store. Put(context.Context, Key, Value) error // Get gets a value to store. @@ -102,7 +102,7 @@ func (keys Keys) Strings() []string { return strs } -// GetKeys gets all the Keys in []ListItem and converts them to Keys. +// GetKeys gets all the Keys in []Item and converts them to Keys. func (items Items) GetKeys() Keys { if len(items) == 0 { return nil @@ -125,7 +125,7 @@ func (items Items) Less(i, k int) bool { return items[i].Less(items[k]) } func (items Items) Swap(i, k int) { items[i], items[k] = items[k], items[i] } // Less returns whether item should be sorted before b. -func (item ListItem) Less(b ListItem) bool { return item.Key.Less(b.Key) } +func (item Item) Less(b Item) bool { return item.Key.Less(b.Key) } // Less returns whether key should be sorted before b. func (key Key) Less(b Key) bool { return bytes.Compare([]byte(key), []byte(b)) < 0 } diff --git a/storage/redis/client.go b/private/kvstore/redis/client.go similarity index 81% rename from storage/redis/client.go rename to private/kvstore/redis/client.go index efe0185d2..a1f987766 100644 --- a/storage/redis/client.go +++ b/private/kvstore/redis/client.go @@ -14,7 +14,7 @@ import ( "github.com/spacemonkeygo/monkit/v3" "github.com/zeebo/errs" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) var ( @@ -75,28 +75,28 @@ func OpenClientFrom(ctx context.Context, address string) (*Client, error) { } // Get looks up the provided key from redis returning either an error or the result. -func (client *Client) Get(ctx context.Context, key storage.Key) (_ storage.Value, err error) { +func (client *Client) Get(ctx context.Context, key kvstore.Key) (_ kvstore.Value, err error) { defer mon.Task()(&ctx)(&err) if key.IsZero() { - return nil, storage.ErrEmptyKey.New("") + return nil, kvstore.ErrEmptyKey.New("") } return get(ctx, client.db, key) } // Put adds a value to the provided key in redis, returning an error on failure. -func (client *Client) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) { +func (client *Client) Put(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) { defer mon.Task()(&ctx)(&err) if key.IsZero() { - return storage.ErrEmptyKey.New("") + return kvstore.ErrEmptyKey.New("") } return put(ctx, client.db, key, value, client.TTL) } // IncrBy increments the value stored in key by the specified value. -func (client *Client) IncrBy(ctx context.Context, key storage.Key, value int64) (err error) { +func (client *Client) IncrBy(ctx context.Context, key kvstore.Key, value int64) (err error) { defer mon.Task()(&ctx)(&err) if key.IsZero() { - return storage.ErrEmptyKey.New("") + return kvstore.ErrEmptyKey.New("") } _, err = client.db.IncrBy(ctx, key.String(), value).Result() return err @@ -110,10 +110,10 @@ func (client *Client) Eval(ctx context.Context, script string, keys []string) (e } // Delete deletes a key/value pair from redis, for a given the key. -func (client *Client) Delete(ctx context.Context, key storage.Key) (err error) { +func (client *Client) Delete(ctx context.Context, key kvstore.Key) (err error) { defer mon.Task()(&ctx)(&err) if key.IsZero() { - return storage.ErrEmptyKey.New("") + return kvstore.ErrEmptyKey.New("") } return delete(ctx, client.db, key) } @@ -130,7 +130,7 @@ func (client *Client) Close() error { } // Range iterates over all items in unspecified order. -func (client *Client) Range(ctx context.Context, fn func(context.Context, storage.Key, storage.Value) error) (err error) { +func (client *Client) Range(ctx context.Context, fn func(context.Context, kvstore.Key, kvstore.Value) error) (err error) { defer mon.Task()(&ctx)(&err) it := client.db.Scan(ctx, 0, "", 0).Iterator() @@ -145,12 +145,12 @@ func (client *Client) Range(ctx context.Context, fn func(context.Context, storag } lastKey, lastOk = key, true - value, err := get(ctx, client.db, storage.Key(key)) + value, err := get(ctx, client.db, kvstore.Key(key)) if err != nil { return Error.Wrap(err) } - if err := fn(ctx, storage.Key(key), value); err != nil { + if err := fn(ctx, kvstore.Key(key), value); err != nil { return err } } @@ -158,11 +158,11 @@ func (client *Client) Range(ctx context.Context, fn func(context.Context, storag return Error.Wrap(it.Err()) } -func get(ctx context.Context, cmdable redis.Cmdable, key storage.Key) (_ storage.Value, err error) { +func get(ctx context.Context, cmdable redis.Cmdable, key kvstore.Key) (_ kvstore.Value, err error) { defer mon.Task()(&ctx)(&err) value, err := cmdable.Get(ctx, string(key)).Bytes() if errors.Is(err, redis.Nil) { - return nil, storage.ErrKeyNotFound.New("%q", key) + return nil, kvstore.ErrKeyNotFound.New("%q", key) } if err != nil && !errors.Is(err, redis.TxFailedErr) { return nil, Error.New("get error: %v", err) @@ -170,7 +170,7 @@ func get(ctx context.Context, cmdable redis.Cmdable, key storage.Key) (_ storage return value, errs.Wrap(err) } -func put(ctx context.Context, cmdable redis.Cmdable, key storage.Key, value storage.Value, ttl time.Duration) (err error) { +func put(ctx context.Context, cmdable redis.Cmdable, key kvstore.Key, value kvstore.Value, ttl time.Duration) (err error) { defer mon.Task()(&ctx)(&err) err = cmdable.Set(ctx, key.String(), []byte(value), ttl).Err() if err != nil && !errors.Is(err, redis.TxFailedErr) { @@ -179,7 +179,7 @@ func put(ctx context.Context, cmdable redis.Cmdable, key storage.Key, value stor return errs.Wrap(err) } -func delete(ctx context.Context, cmdable redis.Cmdable, key storage.Key) (err error) { +func delete(ctx context.Context, cmdable redis.Cmdable, key kvstore.Key) (err error) { defer mon.Task()(&ctx)(&err) err = cmdable.Del(ctx, key.String()).Err() if err != nil && !errors.Is(err, redis.TxFailedErr) { diff --git a/storage/redis/client_test.go b/private/kvstore/redis/client_test.go similarity index 95% rename from storage/redis/client_test.go rename to private/kvstore/redis/client_test.go index 23fabeea3..d6dffa50d 100644 --- a/storage/redis/client_test.go +++ b/private/kvstore/redis/client_test.go @@ -10,8 +10,8 @@ import ( "github.com/stretchr/testify/require" "storj.io/common/testcontext" + "storj.io/storj/private/kvstore/testsuite" "storj.io/storj/private/testredis" - "storj.io/storj/storage/testsuite" ) func TestSuite(t *testing.T) { diff --git a/storage/storelogger/logger.go b/private/kvstore/storelogger/logger.go similarity index 71% rename from storage/storelogger/logger.go rename to private/kvstore/storelogger/logger.go index 0acad0de1..13554a29b 100644 --- a/storage/storelogger/logger.go +++ b/private/kvstore/storelogger/logger.go @@ -11,52 +11,52 @@ import ( "github.com/spacemonkeygo/monkit/v3" "go.uber.org/zap" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) var mon = monkit.Package() var id int64 -// Logger implements a zap.Logger for storage.KeyValueStore. +// Logger implements a zap.Logger for kvstore.Store. type Logger struct { log *zap.Logger - store storage.KeyValueStore + store kvstore.Store } // New creates a new Logger with log and store. -func New(log *zap.Logger, store storage.KeyValueStore) *Logger { +func New(log *zap.Logger, store kvstore.Store) *Logger { loggerid := atomic.AddInt64(&id, 1) name := strconv.Itoa(int(loggerid)) return &Logger{log.Named(name), store} } // Put adds a value to store. -func (store *Logger) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) { +func (store *Logger) Put(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) { defer mon.Task()(&ctx)(&err) store.log.Debug("Put", zap.ByteString("key", key), zap.Int("value length", len(value)), zap.Binary("truncated value", truncate(value))) return store.store.Put(ctx, key, value) } // Get gets a value to store. -func (store *Logger) Get(ctx context.Context, key storage.Key) (_ storage.Value, err error) { +func (store *Logger) Get(ctx context.Context, key kvstore.Key) (_ kvstore.Value, err error) { defer mon.Task()(&ctx)(&err) store.log.Debug("Get", zap.ByteString("key", key)) return store.store.Get(ctx, key) } // Delete deletes key and the value. -func (store *Logger) Delete(ctx context.Context, key storage.Key) (err error) { +func (store *Logger) Delete(ctx context.Context, key kvstore.Key) (err error) { defer mon.Task()(&ctx)(&err) store.log.Debug("Delete", zap.ByteString("key", key)) return store.store.Delete(ctx, key) } // Range iterates over all items in unspecified order. -func (store *Logger) Range(ctx context.Context, fn func(context.Context, storage.Key, storage.Value) error) (err error) { +func (store *Logger) Range(ctx context.Context, fn func(context.Context, kvstore.Key, kvstore.Value) error) (err error) { defer mon.Task()(&ctx)(&err) store.log.Debug("Range") - return store.store.Range(ctx, func(ctx context.Context, key storage.Key, value storage.Value) error { + return store.store.Range(ctx, func(ctx context.Context, key kvstore.Key, value kvstore.Value) error { store.log.Debug(" ", zap.ByteString("key", key), zap.Int("value length", len(value)), @@ -72,7 +72,7 @@ func (store *Logger) Close() error { return store.store.Close() } -func truncate(v storage.Value) (t []byte) { +func truncate(v kvstore.Value) (t []byte) { if len(v)-1 < 10 { t = []byte(v) } else { diff --git a/storage/storelogger/logger_test.go b/private/kvstore/storelogger/logger_test.go similarity index 82% rename from storage/storelogger/logger_test.go rename to private/kvstore/storelogger/logger_test.go index 067fef625..a5e9f14d6 100644 --- a/storage/storelogger/logger_test.go +++ b/private/kvstore/storelogger/logger_test.go @@ -8,8 +8,8 @@ import ( "go.uber.org/zap" - "storj.io/storj/storage/teststore" - "storj.io/storj/storage/testsuite" + "storj.io/storj/private/kvstore/teststore" + "storj.io/storj/private/kvstore/testsuite" ) func TestSuite(t *testing.T) { diff --git a/storage/teststore/store.go b/private/kvstore/teststore/store.go similarity index 73% rename from storage/teststore/store.go rename to private/kvstore/teststore/store.go index e90ba12ea..cedd7cb85 100644 --- a/storage/teststore/store.go +++ b/private/kvstore/teststore/store.go @@ -11,7 +11,7 @@ import ( "github.com/spacemonkeygo/monkit/v3" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) var errInternal = errors.New("internal error") @@ -21,7 +21,7 @@ var mon = monkit.Package() type Client struct { mu sync.Mutex - Items []storage.ListItem + Items []kvstore.Item ForceError int CallCount struct { @@ -42,7 +42,7 @@ func New() *Client { return &Client{} } func (store *Client) MigrateToLatest(ctx context.Context) error { return nil } // indexOf finds index of key or where it could be inserted. -func (store *Client) indexOf(key storage.Key) (int, bool) { +func (store *Client) indexOf(key kvstore.Key) (int, bool) { i := sort.Search(len(store.Items), func(k int) bool { return !store.Items[k].Key.Less(key) }) @@ -67,7 +67,7 @@ func (store *Client) forcedError() bool { } // Put adds a value to store. -func (store *Client) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) { +func (store *Client) Put(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) { defer mon.Task()(&ctx)(&err) defer store.locked()() @@ -78,13 +78,13 @@ func (store *Client) Put(ctx context.Context, key storage.Key, value storage.Val } if key.IsZero() { - return storage.ErrEmptyKey.New("") + return kvstore.ErrEmptyKey.New("") } keyIndex, found := store.indexOf(key) if found { kv := &store.Items[keyIndex] - kv.Value = storage.CloneValue(value) + kv.Value = kvstore.CloneValue(value) return nil } @@ -93,7 +93,7 @@ func (store *Client) Put(ctx context.Context, key storage.Key, value storage.Val } // Get gets a value to store. -func (store *Client) Get(ctx context.Context, key storage.Key) (_ storage.Value, err error) { +func (store *Client) Get(ctx context.Context, key kvstore.Key) (_ kvstore.Value, err error) { defer mon.Task()(&ctx)(&err) defer store.locked()() @@ -104,19 +104,19 @@ func (store *Client) Get(ctx context.Context, key storage.Key) (_ storage.Value, } if key.IsZero() { - return nil, storage.ErrEmptyKey.New("") + return nil, kvstore.ErrEmptyKey.New("") } keyIndex, found := store.indexOf(key) if !found { - return nil, storage.ErrKeyNotFound.New("%q", key) + return nil, kvstore.ErrKeyNotFound.New("%q", key) } - return storage.CloneValue(store.Items[keyIndex].Value), nil + return kvstore.CloneValue(store.Items[keyIndex].Value), nil } // Delete deletes key and the value. -func (store *Client) Delete(ctx context.Context, key storage.Key) (err error) { +func (store *Client) Delete(ctx context.Context, key kvstore.Key) (err error) { defer mon.Task()(&ctx)(&err) defer store.locked()() @@ -128,12 +128,12 @@ func (store *Client) Delete(ctx context.Context, key storage.Key) (err error) { } if key.IsZero() { - return storage.ErrEmptyKey.New("") + return kvstore.ErrEmptyKey.New("") } keyIndex, found := store.indexOf(key) if !found { - return storage.ErrKeyNotFound.New("%q", key) + return kvstore.ErrKeyNotFound.New("%q", key) } store.delete(keyIndex) @@ -152,14 +152,14 @@ func (store *Client) Close() error { } // Range iterates over all items in unspecified order. -func (store *Client) Range(ctx context.Context, fn func(context.Context, storage.Key, storage.Value) error) error { +func (store *Client) Range(ctx context.Context, fn func(context.Context, kvstore.Key, kvstore.Value) error) error { store.mu.Lock() store.CallCount.Range++ if store.forcedError() { store.mu.Unlock() return errors.New("internal error") } - items := append([]storage.ListItem{}, store.Items...) + items := append([]kvstore.Item{}, store.Items...) store.mu.Unlock() for _, item := range items { @@ -170,12 +170,12 @@ func (store *Client) Range(ctx context.Context, fn func(context.Context, storage return nil } -func (store *Client) put(keyIndex int, key storage.Key, value storage.Value) { - store.Items = append(store.Items, storage.ListItem{}) +func (store *Client) put(keyIndex int, key kvstore.Key, value kvstore.Value) { + store.Items = append(store.Items, kvstore.Item{}) copy(store.Items[keyIndex+1:], store.Items[keyIndex:]) - store.Items[keyIndex] = storage.ListItem{ - Key: storage.CloneKey(key), - Value: storage.CloneValue(value), + store.Items[keyIndex] = kvstore.Item{ + Key: kvstore.CloneKey(key), + Value: kvstore.CloneValue(value), } } diff --git a/storage/teststore/store_test.go b/private/kvstore/teststore/store_test.go similarity index 85% rename from storage/teststore/store_test.go rename to private/kvstore/teststore/store_test.go index df2a54734..d676eb1d6 100644 --- a/storage/teststore/store_test.go +++ b/private/kvstore/teststore/store_test.go @@ -6,7 +6,7 @@ package teststore import ( "testing" - "storj.io/storj/storage/testsuite" + "storj.io/storj/private/kvstore/testsuite" ) func TestSuite(t *testing.T) { diff --git a/storage/testsuite/bench.go b/private/kvstore/testsuite/bench.go similarity index 83% rename from storage/testsuite/bench.go rename to private/kvstore/testsuite/bench.go index 807c1ca1c..c41e53310 100644 --- a/storage/testsuite/bench.go +++ b/private/kvstore/testsuite/bench.go @@ -11,11 +11,11 @@ import ( "golang.org/x/sync/errgroup" "storj.io/common/testcontext" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) -// RunBenchmarks runs common storage.KeyValueStore benchmarks. -func RunBenchmarks(b *testing.B, store storage.KeyValueStore) { +// RunBenchmarks runs common kvstore.Store benchmarks. +func RunBenchmarks(b *testing.B, store kvstore.Store) { var words = []string{ "alpha", "beta", "gamma", "delta", "iota", "kappa", "lambda", "mu", "άλφα", "βήτα", "γάμμα", "δέλτα", "έψιλον", "ζήτα", "ήτα", "θήτα", "ιώτα", "κάππα", "λάμδα", "μυ", @@ -28,15 +28,15 @@ func RunBenchmarks(b *testing.B, store storage.KeyValueStore) { words = words[:2] } - var items storage.Items + var items kvstore.Items k := 0 for _, a := range words { for _, b := range words { for _, c := range words { - items = append(items, storage.ListItem{ - Key: storage.Key(path.Join(a, b, c)), - Value: storage.Value(strconv.Itoa(k)), + items = append(items, kvstore.Item{ + Key: kvstore.Key(path.Join(a, b, c)), + Value: kvstore.Value(strconv.Itoa(k)), }) k++ } diff --git a/storage/testsuite/test.go b/private/kvstore/testsuite/test.go similarity index 76% rename from storage/testsuite/test.go rename to private/kvstore/testsuite/test.go index e7300b464..327da54b1 100644 --- a/storage/testsuite/test.go +++ b/private/kvstore/testsuite/test.go @@ -10,11 +10,11 @@ import ( "golang.org/x/sync/errgroup" "storj.io/common/testcontext" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) -// RunTests runs common storage.KeyValueStore tests. -func RunTests(t *testing.T, store storage.KeyValueStore) { +// RunTests runs common kvstore.Store tests. +func RunTests(t *testing.T, store kvstore.Store) { // store = storelogger.NewTest(t, store) ctx := testcontext.New(t) defer ctx.Cleanup() @@ -24,12 +24,12 @@ func RunTests(t *testing.T, store storage.KeyValueStore) { t.Run("Parallel", func(t *testing.T) { testParallel(t, ctx, store) }) } -func testConstraints(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) { - var items storage.Items +func testConstraints(t *testing.T, ctx *testcontext.Context, store kvstore.Store) { + var items kvstore.Items for i := 0; i < 10; i++ { - items = append(items, storage.ListItem{ - Key: storage.Key("test-" + strconv.Itoa(i)), - Value: storage.Value("xyz"), + items = append(items, kvstore.Item{ + Key: kvstore.Key("test-" + strconv.Itoa(i)), + Value: kvstore.Value("xyz"), }) } @@ -47,8 +47,8 @@ func testConstraints(t *testing.T, ctx *testcontext.Context, store storage.KeyVa defer cleanupItems(t, ctx, store, items) t.Run("Put Empty", func(t *testing.T) { - var key storage.Key - var val storage.Value + var key kvstore.Key + var val kvstore.Value defer func() { _ = store.Delete(ctx, key) }() err := store.Put(ctx, key, val) diff --git a/storage/testsuite/test_crud.go b/private/kvstore/testsuite/test_crud.go similarity index 93% rename from storage/testsuite/test_crud.go rename to private/kvstore/testsuite/test_crud.go index e8551078e..df5ad82df 100644 --- a/storage/testsuite/test_crud.go +++ b/private/kvstore/testsuite/test_crud.go @@ -9,11 +9,11 @@ import ( "testing" "storj.io/common/testcontext" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) -func testCRUD(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) { - items := storage.Items{ +func testCRUD(t *testing.T, ctx *testcontext.Context, store kvstore.Store) { + items := kvstore.Items{ // newItem("0", "", false), // TODO: broken newItem("\x00", "\x00", false), newItem("a/b", "\x01\x00", false), diff --git a/storage/testsuite/test_parallel.go b/private/kvstore/testsuite/test_parallel.go similarity index 93% rename from storage/testsuite/test_parallel.go rename to private/kvstore/testsuite/test_parallel.go index 9f1fe07b2..37cf5ee4f 100644 --- a/storage/testsuite/test_parallel.go +++ b/private/kvstore/testsuite/test_parallel.go @@ -10,11 +10,11 @@ import ( "testing" "storj.io/common/testcontext" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) -func testParallel(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) { - items := storage.Items{ +func testParallel(t *testing.T, ctx *testcontext.Context, store kvstore.Store) { + items := kvstore.Items{ newItem("a", "1", false), newItem("b", "2", false), newItem("c", "3", false), diff --git a/storage/testsuite/test_range.go b/private/kvstore/testsuite/test_range.go similarity index 65% rename from storage/testsuite/test_range.go rename to private/kvstore/testsuite/test_range.go index 3be9e36bb..d7d647e3a 100644 --- a/storage/testsuite/test_range.go +++ b/private/kvstore/testsuite/test_range.go @@ -13,16 +13,16 @@ import ( "github.com/stretchr/testify/require" "storj.io/common/testcontext" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) -func testRange(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) { - err := store.Range(ctx, func(ctx context.Context, key storage.Key, value storage.Value) error { +func testRange(t *testing.T, ctx *testcontext.Context, store kvstore.Store) { + err := store.Range(ctx, func(ctx context.Context, key kvstore.Key, value kvstore.Value) error { return errors.New("empty store") }) require.NoError(t, err) - items := storage.Items{ + items := kvstore.Items{ newItem("a", "a", false), newItem("b/1", "b/1", false), newItem("b/2", "b/2", false), @@ -37,13 +37,13 @@ func testRange(t *testing.T, ctx *testcontext.Context, store storage.KeyValueSto rand.Shuffle(len(items), items.Swap) defer cleanupItems(t, ctx, store, items) - if err := storage.PutAll(ctx, store, items...); err != nil { + if err := kvstore.PutAll(ctx, store, items...); err != nil { t.Fatalf("failed to setup: %v", err) } - var output storage.Items - err = store.Range(ctx, func(ctx context.Context, key storage.Key, value storage.Value) error { - output = append(output, storage.ListItem{ + var output kvstore.Items + err = store.Range(ctx, func(ctx context.Context, key kvstore.Key, value kvstore.Value) error { + output = append(output, kvstore.Item{ Key: append([]byte{}, key...), Value: append([]byte{}, value...), }) @@ -51,7 +51,7 @@ func testRange(t *testing.T, ctx *testcontext.Context, store storage.KeyValueSto }) require.NoError(t, err) - expected := storage.CloneItems(items) + expected := kvstore.CloneItems(items) sort.Sort(expected) sort.Sort(output) diff --git a/storage/testsuite/utils.go b/private/kvstore/testsuite/utils.go similarity index 58% rename from storage/testsuite/utils.go rename to private/kvstore/testsuite/utils.go index f2cbe08d6..219630cd8 100644 --- a/storage/testsuite/utils.go +++ b/private/kvstore/testsuite/utils.go @@ -7,18 +7,18 @@ import ( "testing" "storj.io/common/testcontext" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) -func newItem(key, value string, isPrefix bool) storage.ListItem { - return storage.ListItem{ - Key: storage.Key(key), - Value: storage.Value(value), +func newItem(key, value string, isPrefix bool) kvstore.Item { + return kvstore.Item{ + Key: kvstore.Key(key), + Value: kvstore.Value(value), IsPrefix: isPrefix, } } -func cleanupItems(t testing.TB, ctx *testcontext.Context, store storage.KeyValueStore, items storage.Items) { +func cleanupItems(t testing.TB, ctx *testcontext.Context, store kvstore.Store, items kvstore.Items) { for _, item := range items { _ = store.Delete(ctx, item.Key) } diff --git a/storage/util.go b/private/kvstore/util.go similarity index 84% rename from storage/util.go rename to private/kvstore/util.go index 143fe6bba..e94a65b5a 100644 --- a/storage/util.go +++ b/private/kvstore/util.go @@ -1,7 +1,7 @@ // Copyright (C) 2019 Storj Labs, Inc. // See LICENSE for copying information. -package storage +package kvstore import ( "context" @@ -15,8 +15,8 @@ func CloneKey(key Key) Key { return append(Key{}, key...) } func CloneValue(value Value) Value { return append(Value{}, value...) } // CloneItem creates a deep copy of item. -func CloneItem(item ListItem) ListItem { - return ListItem{ +func CloneItem(item Item) Item { + return Item{ Key: CloneKey(item.Key), Value: CloneValue(item.Value), IsPrefix: item.IsPrefix, @@ -33,7 +33,7 @@ func CloneItems(items Items) Items { } // PutAll adds multiple values to the store. -func PutAll(ctx context.Context, store KeyValueStore, items ...ListItem) (err error) { +func PutAll(ctx context.Context, store Store, items ...Item) (err error) { defer mon.Task()(&ctx)(&err) for _, item := range items { diff --git a/private/revocation/common.go b/private/revocation/common.go index a7ad0950e..b31bcf4bf 100644 --- a/private/revocation/common.go +++ b/private/revocation/common.go @@ -9,8 +9,8 @@ import ( "storj.io/common/peertls/extensions" "storj.io/common/peertls/tlsopts" "storj.io/private/dbutil" - "storj.io/storj/storage/boltdb" - "storj.io/storj/storage/redis" + "storj.io/storj/private/kvstore/boltdb" + "storj.io/storj/private/kvstore/redis" ) // OpenDBFromCfg is a convenience method to create a revocation DB diff --git a/private/revocation/db.go b/private/revocation/db.go index 96fa5d56a..e167d5a63 100644 --- a/private/revocation/db.go +++ b/private/revocation/db.go @@ -14,7 +14,7 @@ import ( "storj.io/common/identity" "storj.io/common/peertls" "storj.io/common/peertls/extensions" - "storj.io/storj/storage" + "storj.io/storj/private/kvstore" ) var ( @@ -28,7 +28,7 @@ var ( // (i.e. nodeID [CA certificate's public key hash] is the key, values is // the most recently seen revocation). type DB struct { - store storage.KeyValueStore + store kvstore.Store } // Get attempts to retrieve the most recent revocation for the given cert chain @@ -46,7 +46,7 @@ func (db *DB) Get(ctx context.Context, chain []*x509.Certificate) (_ *extensions } revBytes, err := db.store.Get(ctx, nodeID.Bytes()) - if err != nil && !storage.ErrKeyNotFound.Has(err) { + if err != nil && !kvstore.ErrKeyNotFound.Has(err) { return nil, extensions.ErrRevocationDB.Wrap(err) } if revBytes == nil { @@ -108,7 +108,7 @@ func (db *DB) List(ctx context.Context) (revs []*extensions.Revocation, err erro return nil, nil } - err = db.store.Range(ctx, func(ctx context.Context, key storage.Key, value storage.Value) error { + err = db.store.Range(ctx, func(ctx context.Context, key kvstore.Key, value kvstore.Value) error { rev := new(extensions.Revocation) if err := rev.Unmarshal(value); err != nil { return extensions.ErrRevocationDB.Wrap(err) @@ -121,7 +121,7 @@ func (db *DB) List(ctx context.Context) (revs []*extensions.Revocation, err erro } // TestGetStore returns the internal store for testing. -func (db *DB) TestGetStore() storage.KeyValueStore { +func (db *DB) TestGetStore() kvstore.Store { return db.store } diff --git a/private/revocation/db_test.go b/private/revocation/db_test.go index 350fd6883..700828c00 100644 --- a/private/revocation/db_test.go +++ b/private/revocation/db_test.go @@ -18,15 +18,15 @@ import ( "storj.io/common/peertls/testpeertls" "storj.io/common/storj" "storj.io/common/testcontext" + "storj.io/storj/private/kvstore" "storj.io/storj/private/testrevocation" - "storj.io/storj/storage" ) func TestRevocationDB_Get(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup() - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) { keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number) require.NoError(t, err) @@ -64,7 +64,7 @@ func TestRevocationDB_Put_success(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup() - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) { keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number) require.NoError(t, err) @@ -114,7 +114,7 @@ func TestRevocationDB_Put_error(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup() - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) { keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number) require.NoError(t, err) @@ -156,7 +156,7 @@ func TestRevocationDB_List(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup() - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) { keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number) require.NoError(t, err) keys2, chain2, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number) diff --git a/private/revocation/extensions_test.go b/private/revocation/extensions_test.go index b661a45e0..4c4aea2d4 100644 --- a/private/revocation/extensions_test.go +++ b/private/revocation/extensions_test.go @@ -18,15 +18,15 @@ import ( "storj.io/common/peertls/tlsopts" "storj.io/common/storj" "storj.io/common/testcontext" + "storj.io/storj/private/kvstore" "storj.io/storj/private/testrevocation" - "storj.io/storj/storage" ) func TestRevocationCheckHandler(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup() - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ kvstore.Store) { keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number) assert.NoError(t, err) @@ -68,7 +68,7 @@ func TestRevocationCheckHandler(t *testing.T) { } }) - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ kvstore.Store) { t.Log("new revocation DB") keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number) assert.NoError(t, err) @@ -123,7 +123,7 @@ func TestRevocationUpdateHandler(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup() - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ kvstore.Store) { keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number) assert.NoError(t, err) diff --git a/private/revocation/tlsopts_test.go b/private/revocation/tlsopts_test.go index a42d07608..e6f85dba1 100644 --- a/private/revocation/tlsopts_test.go +++ b/private/revocation/tlsopts_test.go @@ -21,9 +21,9 @@ import ( "storj.io/common/peertls/tlsopts" "storj.io/common/storj" "storj.io/common/testcontext" + "storj.io/storj/private/kvstore" "storj.io/storj/private/revocation" "storj.io/storj/private/testrevocation" - "storj.io/storj/storage" ) func TestNewOptions(t *testing.T) { @@ -150,7 +150,7 @@ func TestExtensionMap_HandleExtensions(t *testing.T) { err = rev.Verify(newRevokedLeafChain[peertls.CAIndex]) require.NoError(t, err) - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) { opts := &extensions.Options{ RevocationDB: revDB, PeerIDVersions: "*", @@ -183,7 +183,7 @@ func TestExtensionMap_HandleExtensions_error(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup() - testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) { + testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) { keys, chain, oldRevocation, err := testpeertls.NewRevokedLeafChain() assert.NoError(t, err) diff --git a/private/testrevocation/db.go b/private/testrevocation/db.go index 93dd1326e..b7dbb50ba 100644 --- a/private/testrevocation/db.go +++ b/private/testrevocation/db.go @@ -10,13 +10,13 @@ import ( "storj.io/common/peertls/extensions" "storj.io/common/testcontext" + "storj.io/storj/private/kvstore" "storj.io/storj/private/revocation" "storj.io/storj/private/testredis" - "storj.io/storj/storage" ) // RunDBs runs the passed test function with each type of revocation database. -func RunDBs(t *testing.T, test func(*testing.T, extensions.RevocationDB, storage.KeyValueStore)) { +func RunDBs(t *testing.T, test func(*testing.T, extensions.RevocationDB, kvstore.Store)) { t.Run("Redis", func(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup()