JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

private/kvstore: move storage package

Browse Source 
There's no reason it should be at the top-level.

Change-Id: I35b06e7baa0e425c6ff9a82964d0a1570d4eb6d0
This commit is contained in:
Egon Elbre 2023-04-06 16:54:41 +03:00
parent 48256c91b5
commit c1f1aacffe
26 changed files with 154 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile.public
View File

@ -157,7 +157,7 @@ pipeline {
STORJ_TEST_COCKROACH_ALT = 'cockroach://root@localhost:26260/testcockroach?sslmode=disable'
STORJ_TEST_POSTGRES = 'postgres://postgres@localhost/teststorj?sslmode=disable'
STORJ_TEST_LOG_LEVEL = 'info'
COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storage/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}"
COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}"
}
steps {
sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\''

2
Jenkinsfile.verify
View File

@ -155,7 +155,7 @@ pipeline {
STORJ_TEST_COCKROACH_ALT = 'cockroach://root@localhost:26260/testcockroach?sslmode=disable'
STORJ_TEST_POSTGRES = 'omit'
STORJ_TEST_LOG_LEVEL = 'info'
COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storage/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}"
COVERFLAGS = "${ env.BRANCH_NAME == 'main' ? '-coverprofile=.build/coverprofile -coverpkg=storj.io/storj/private/...,storj.io/storj/satellite/...,storj.io/storj/storagenode/...,storj.io/storj/versioncontrol/...' : ''}"
}
steps {
sh 'cockroach sql --insecure --host=localhost:26256 -e \'create database testcockroach;\''

18
certificate/authorization/db.go
View File

@ -13,9 +13,9 @@ import (
"storj.io/common/identity"
"storj.io/common/peertls/extensions"
"storj.io/private/dbutil"
"storj.io/storj/storage"
"storj.io/storj/storage/boltdb"
"storj.io/storj/storage/redis"
"storj.io/storj/private/kvstore"
"storj.io/storj/private/kvstore/boltdb"
"storj.io/storj/private/kvstore/redis"
)
var (
@ -38,7 +38,7 @@ var (
// DB stores authorizations which may be claimed in exchange for a
// certificate signature.
type DB struct {
db storage.KeyValueStore
db kvstore.Store
}
// DBConfig is the authorization db config.
@ -127,8 +127,8 @@ func (authDB *DB) Create(ctx context.Context, userID string, count int) (_ Group
// Get retrieves authorizations by user ID.
func (authDB *DB) Get(ctx context.Context, userID string) (_ Group, err error) {
defer mon.Task()(&ctx, userID)(&err)
authsBytes, err := authDB.db.Get(ctx, storage.Key(userID))
if storage.ErrKeyNotFound.Has(err) {
authsBytes, err := authDB.db.Get(ctx, kvstore.Key(userID))
if kvstore.ErrKeyNotFound.Has(err) {
return nil, ErrNotFound.New("userID: %s", userID)
}
if err != nil {
@ -150,7 +150,7 @@ func (authDB *DB) UserIDs(ctx context.Context) (userIDs []string, err error) {
defer mon.Task()(&ctx)(&err)
err = authDB.db.Range(ctx,
func(ctx context.Context, key storage.Key, _ storage.Value) error {
func(ctx context.Context, key kvstore.Key, _ kvstore.Value) error {
userIDs = append(userIDs, key.String())
return nil
})
@ -163,7 +163,7 @@ func (authDB *DB) List(ctx context.Context) (auths Group, err error) {
var errs errs.Group
err = authDB.db.Range(ctx,
func(ctx context.Context, key storage.Key, value storage.Value) error {
func(ctx context.Context, key kvstore.Key, value kvstore.Value) error {
var group Group
err := group.Unmarshal(value)
if err != nil {
@ -288,7 +288,7 @@ func (authDB *DB) put(ctx context.Context, userID string, auths Group) (err erro
return ErrDBInternal.Wrap(err)
}
if err := authDB.db.Put(ctx, storage.Key(userID), authsBytes); err != nil {
if err := authDB.db.Put(ctx, kvstore.Key(userID), authsBytes); err != nil {
return ErrDBInternal.Wrap(err)
}
return nil

6
certificate/authorization/db_test.go
View File

@ -20,7 +20,7 @@ import (
"storj.io/common/rpc/rpcpeer"
"storj.io/common/testcontext"
"storj.io/storj/certificate/certificatepb"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
func TestNewDB(t *testing.T) {
@ -86,7 +86,7 @@ func TestAuthorizationDB_Create(t *testing.T) {
for _, c := range cases {
testCase := c
t.Run(c.testID, func(t *testing.T) {
emailKey := storage.Key(testCase.email)
emailKey := kvstore.Key(testCase.email)
if testCase.startCount == 0 {
_, err := authDB.db.Get(ctx, emailKey)
@ -174,7 +174,7 @@ func TestAuthorizationDB_Get(t *testing.T) {
authsBytes, err := expectedAuths.Marshal()
require.NoError(t, err)
err = authDB.db.Put(ctx, storage.Key("user@mail.test"), authsBytes)
err = authDB.db.Put(ctx, kvstore.Key("user@mail.test"), authsBytes)
require.NoError(t, err)
{

28
storage/boltdb/client.go → private/kvstore/boltdb/client.go
View File

@ -12,7 +12,7 @@ import (
"github.com/zeebo/errs"
"go.etcd.io/bbolt"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
var mon = monkit.Package()
@ -87,11 +87,11 @@ func (client *Client) view(fn func(*bbolt.Bucket) error) error {
// Ref: https://github.com/boltdb/bolt/blob/master/db.go#L160
// Note: when using this method, check if it needs to be executed asynchronously
// since it blocks for the duration db.MaxBatchDelay.
func (client *Client) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) {
func (client *Client) Put(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) {
defer mon.Task()(&ctx)(&err)
start := time.Now()
if key.IsZero() {
return storage.ErrEmptyKey.New("")
return kvstore.ErrEmptyKey.New("")
}
err = client.batch(func(bucket *bbolt.Bucket) error {
@ -102,10 +102,10 @@ func (client *Client) Put(ctx context.Context, key storage.Key, value storage.Va
}
// PutAndCommit adds a key/value to BoltDB and writes it to disk.
func (client *Client) PutAndCommit(ctx context.Context, key storage.Key, value storage.Value) (err error) {
func (client *Client) PutAndCommit(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) {
defer mon.Task()(&ctx)(&err)
if key.IsZero() {
return storage.ErrEmptyKey.New("")
return kvstore.ErrEmptyKey.New("")
}
return client.update(func(bucket *bbolt.Bucket) error {
@ -114,29 +114,29 @@ func (client *Client) PutAndCommit(ctx context.Context, key storage.Key, value s
}
// Get looks up the provided key from boltdb returning either an error or the result.
func (client *Client) Get(ctx context.Context, key storage.Key) (_ storage.Value, err error) {
func (client *Client) Get(ctx context.Context, key kvstore.Key) (_ kvstore.Value, err error) {
defer mon.Task()(&ctx)(&err)
if key.IsZero() {
return nil, storage.ErrEmptyKey.New("")
return nil, kvstore.ErrEmptyKey.New("")
}
var value storage.Value
var value kvstore.Value
err = client.view(func(bucket *bbolt.Bucket) error {
data := bucket.Get([]byte(key))
if len(data) == 0 {
return storage.ErrKeyNotFound.New("%q", key)
return kvstore.ErrKeyNotFound.New("%q", key)
}
value = storage.CloneValue(storage.Value(data))
value = kvstore.CloneValue(kvstore.Value(data))
return nil
})
return value, err
}
// Delete deletes a key/value pair from boltdb, for a given the key.
func (client *Client) Delete(ctx context.Context, key storage.Key) (err error) {
func (client *Client) Delete(ctx context.Context, key kvstore.Key) (err error) {
defer mon.Task()(&ctx)(&err)
if key.IsZero() {
return storage.ErrEmptyKey.New("")
return kvstore.ErrEmptyKey.New("")
}
return client.update(func(bucket *bbolt.Bucket) error {
@ -153,12 +153,12 @@ func (client *Client) Close() (err error) {
}
// Range iterates over all items in unspecified order.
func (client *Client) Range(ctx context.Context, fn func(context.Context, storage.Key, storage.Value) error) (err error) {
func (client *Client) Range(ctx context.Context, fn func(context.Context, kvstore.Key, kvstore.Value) error) (err error) {
defer mon.Task()(&ctx)(&err)
return client.view(func(bucket *bbolt.Bucket) error {
return bucket.ForEach(func(k, v []byte) error {
return fn(ctx, storage.Key(k), storage.Value(v))
return fn(ctx, kvstore.Key(k), kvstore.Value(v))
})
})
}

2
storage/boltdb/client_test.go → private/kvstore/boltdb/client_test.go
View File

@ -8,7 +8,7 @@ import (
"path/filepath"
"testing"
"storj.io/storj/storage/testsuite"
"storj.io/storj/private/kvstore/testsuite"
)
func TestSuite(t *testing.T) {

24
storage/common.go → private/kvstore/common.go
View File

@ -1,7 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package storage
package kvstore
import (
"bytes"
@ -22,30 +22,30 @@ var ErrKeyNotFound = errs.Class("key not found")
// ErrEmptyKey is returned when an empty key is used in Put or in CompareAndSwap.
var ErrEmptyKey = errs.Class("empty key")
// Key is the type for the keys in a `KeyValueStore`.
// Key is the type for the keys in a `Store`.
type Key []byte
// Value is the type for the values in a `ValueValueStore`.
type Value []byte
// Keys is the type for a slice of keys in a `KeyValueStore`.
// Keys is the type for a slice of keys in a `Store`.
type Keys []Key
// Values is the type for a slice of Values in a `KeyValueStore`.
// Values is the type for a slice of Values in a `Store`.
type Values []Value
// Items keeps all ListItem.
type Items []ListItem
// Items keeps all Item.
type Items []Item
// ListItem returns Key, Value, IsPrefix.
type ListItem struct {
// Item returns Key, Value, IsPrefix.
type Item struct {
Key Key
Value Value
IsPrefix bool
}
// KeyValueStore describes key/value stores like redis and boltdb.
type KeyValueStore interface {
// Store describes key/value stores like redis and boltdb.
type Store interface {
// Put adds a value to store.
Put(context.Context, Key, Value) error
// Get gets a value to store.
@ -102,7 +102,7 @@ func (keys Keys) Strings() []string {
return strs
}
// GetKeys gets all the Keys in []ListItem and converts them to Keys.
// GetKeys gets all the Keys in []Item and converts them to Keys.
func (items Items) GetKeys() Keys {
if len(items) == 0 {
return nil
@ -125,7 +125,7 @@ func (items Items) Less(i, k int) bool { return items[i].Less(items[k]) }
func (items Items) Swap(i, k int) { items[i], items[k] = items[k], items[i] }
// Less returns whether item should be sorted before b.
func (item ListItem) Less(b ListItem) bool { return item.Key.Less(b.Key) }
func (item Item) Less(b Item) bool { return item.Key.Less(b.Key) }
// Less returns whether key should be sorted before b.
func (key Key) Less(b Key) bool { return bytes.Compare([]byte(key), []byte(b)) < 0 }

32
storage/redis/client.go → private/kvstore/redis/client.go
View File

@ -14,7 +14,7 @@ import (
"github.com/spacemonkeygo/monkit/v3"
"github.com/zeebo/errs"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
var (
@ -75,28 +75,28 @@ func OpenClientFrom(ctx context.Context, address string) (*Client, error) {
}
// Get looks up the provided key from redis returning either an error or the result.
func (client *Client) Get(ctx context.Context, key storage.Key) (_ storage.Value, err error) {
func (client *Client) Get(ctx context.Context, key kvstore.Key) (_ kvstore.Value, err error) {
defer mon.Task()(&ctx)(&err)
if key.IsZero() {
return nil, storage.ErrEmptyKey.New("")
return nil, kvstore.ErrEmptyKey.New("")
}
return get(ctx, client.db, key)
}
// Put adds a value to the provided key in redis, returning an error on failure.
func (client *Client) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) {
func (client *Client) Put(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) {
defer mon.Task()(&ctx)(&err)
if key.IsZero() {
return storage.ErrEmptyKey.New("")
return kvstore.ErrEmptyKey.New("")
}
return put(ctx, client.db, key, value, client.TTL)
}
// IncrBy increments the value stored in key by the specified value.
func (client *Client) IncrBy(ctx context.Context, key storage.Key, value int64) (err error) {
func (client *Client) IncrBy(ctx context.Context, key kvstore.Key, value int64) (err error) {
defer mon.Task()(&ctx)(&err)
if key.IsZero() {
return storage.ErrEmptyKey.New("")
return kvstore.ErrEmptyKey.New("")
}
_, err = client.db.IncrBy(ctx, key.String(), value).Result()
return err
@ -110,10 +110,10 @@ func (client *Client) Eval(ctx context.Context, script string, keys []string) (e
}
// Delete deletes a key/value pair from redis, for a given the key.
func (client *Client) Delete(ctx context.Context, key storage.Key) (err error) {
func (client *Client) Delete(ctx context.Context, key kvstore.Key) (err error) {
defer mon.Task()(&ctx)(&err)
if key.IsZero() {
return storage.ErrEmptyKey.New("")
return kvstore.ErrEmptyKey.New("")
}
return delete(ctx, client.db, key)
}
@ -130,7 +130,7 @@ func (client *Client) Close() error {
}
// Range iterates over all items in unspecified order.
func (client *Client) Range(ctx context.Context, fn func(context.Context, storage.Key, storage.Value) error) (err error) {
func (client *Client) Range(ctx context.Context, fn func(context.Context, kvstore.Key, kvstore.Value) error) (err error) {
defer mon.Task()(&ctx)(&err)
it := client.db.Scan(ctx, 0, "", 0).Iterator()
@ -145,12 +145,12 @@ func (client *Client) Range(ctx context.Context, fn func(context.Context, storag
}
lastKey, lastOk = key, true
value, err := get(ctx, client.db, storage.Key(key))
value, err := get(ctx, client.db, kvstore.Key(key))
if err != nil {
return Error.Wrap(err)
}
if err := fn(ctx, storage.Key(key), value); err != nil {
if err := fn(ctx, kvstore.Key(key), value); err != nil {
return err
}
}
@ -158,11 +158,11 @@ func (client *Client) Range(ctx context.Context, fn func(context.Context, storag
return Error.Wrap(it.Err())
}
func get(ctx context.Context, cmdable redis.Cmdable, key storage.Key) (_ storage.Value, err error) {
func get(ctx context.Context, cmdable redis.Cmdable, key kvstore.Key) (_ kvstore.Value, err error) {
defer mon.Task()(&ctx)(&err)
value, err := cmdable.Get(ctx, string(key)).Bytes()
if errors.Is(err, redis.Nil) {
return nil, storage.ErrKeyNotFound.New("%q", key)
return nil, kvstore.ErrKeyNotFound.New("%q", key)
}
if err != nil && !errors.Is(err, redis.TxFailedErr) {
return nil, Error.New("get error: %v", err)
@ -170,7 +170,7 @@ func get(ctx context.Context, cmdable redis.Cmdable, key storage.Key) (_ storage
return value, errs.Wrap(err)
}
func put(ctx context.Context, cmdable redis.Cmdable, key storage.Key, value storage.Value, ttl time.Duration) (err error) {
func put(ctx context.Context, cmdable redis.Cmdable, key kvstore.Key, value kvstore.Value, ttl time.Duration) (err error) {
defer mon.Task()(&ctx)(&err)
err = cmdable.Set(ctx, key.String(), []byte(value), ttl).Err()
if err != nil && !errors.Is(err, redis.TxFailedErr) {
@ -179,7 +179,7 @@ func put(ctx context.Context, cmdable redis.Cmdable, key storage.Key, value stor
return errs.Wrap(err)
}
func delete(ctx context.Context, cmdable redis.Cmdable, key storage.Key) (err error) {
func delete(ctx context.Context, cmdable redis.Cmdable, key kvstore.Key) (err error) {
defer mon.Task()(&ctx)(&err)
err = cmdable.Del(ctx, key.String()).Err()
if err != nil && !errors.Is(err, redis.TxFailedErr) {

2
storage/redis/client_test.go → private/kvstore/redis/client_test.go
View File

@ -10,8 +10,8 @@ import (
"github.com/stretchr/testify/require"
"storj.io/common/testcontext"
"storj.io/storj/private/kvstore/testsuite"
"storj.io/storj/private/testredis"
"storj.io/storj/storage/testsuite"
)
func TestSuite(t *testing.T) {

20
storage/storelogger/logger.go → private/kvstore/storelogger/logger.go
View File

@ -11,52 +11,52 @@ import (
"github.com/spacemonkeygo/monkit/v3"
"go.uber.org/zap"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
var mon = monkit.Package()
var id int64
// Logger implements a zap.Logger for storage.KeyValueStore.
// Logger implements a zap.Logger for kvstore.Store.
type Logger struct {
log *zap.Logger
store storage.KeyValueStore
store kvstore.Store
}
// New creates a new Logger with log and store.
func New(log *zap.Logger, store storage.KeyValueStore) *Logger {
func New(log *zap.Logger, store kvstore.Store) *Logger {
loggerid := atomic.AddInt64(&id, 1)
name := strconv.Itoa(int(loggerid))
return &Logger{log.Named(name), store}
}
// Put adds a value to store.
func (store *Logger) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) {
func (store *Logger) Put(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) {
defer mon.Task()(&ctx)(&err)
store.log.Debug("Put", zap.ByteString("key", key), zap.Int("value length", len(value)), zap.Binary("truncated value", truncate(value)))
return store.store.Put(ctx, key, value)
}
// Get gets a value to store.
func (store *Logger) Get(ctx context.Context, key storage.Key) (_ storage.Value, err error) {
func (store *Logger) Get(ctx context.Context, key kvstore.Key) (_ kvstore.Value, err error) {
defer mon.Task()(&ctx)(&err)
store.log.Debug("Get", zap.ByteString("key", key))
return store.store.Get(ctx, key)
}
// Delete deletes key and the value.
func (store *Logger) Delete(ctx context.Context, key storage.Key) (err error) {
func (store *Logger) Delete(ctx context.Context, key kvstore.Key) (err error) {
defer mon.Task()(&ctx)(&err)
store.log.Debug("Delete", zap.ByteString("key", key))
return store.store.Delete(ctx, key)
}
// Range iterates over all items in unspecified order.
func (store *Logger) Range(ctx context.Context, fn func(context.Context, storage.Key, storage.Value) error) (err error) {
func (store *Logger) Range(ctx context.Context, fn func(context.Context, kvstore.Key, kvstore.Value) error) (err error) {
defer mon.Task()(&ctx)(&err)
store.log.Debug("Range")
return store.store.Range(ctx, func(ctx context.Context, key storage.Key, value storage.Value) error {
return store.store.Range(ctx, func(ctx context.Context, key kvstore.Key, value kvstore.Value) error {
store.log.Debug(" ",
zap.ByteString("key", key),
zap.Int("value length", len(value)),
@ -72,7 +72,7 @@ func (store *Logger) Close() error {
return store.store.Close()
}
func truncate(v storage.Value) (t []byte) {
func truncate(v kvstore.Value) (t []byte) {
if len(v)-1 < 10 {
t = []byte(v)
} else {

4
storage/storelogger/logger_test.go → private/kvstore/storelogger/logger_test.go
View File

@ -8,8 +8,8 @@ import (
"go.uber.org/zap"
"storj.io/storj/storage/teststore"
"storj.io/storj/storage/testsuite"
"storj.io/storj/private/kvstore/teststore"
"storj.io/storj/private/kvstore/testsuite"
)
func TestSuite(t *testing.T) {

40
storage/teststore/store.go → private/kvstore/teststore/store.go
View File

@ -11,7 +11,7 @@ import (
"github.com/spacemonkeygo/monkit/v3"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
var errInternal = errors.New("internal error")
@ -21,7 +21,7 @@ var mon = monkit.Package()
type Client struct {
mu sync.Mutex
Items []storage.ListItem
Items []kvstore.Item
ForceError int
CallCount struct {
@ -42,7 +42,7 @@ func New() *Client { return &Client{} }
func (store *Client) MigrateToLatest(ctx context.Context) error { return nil }
// indexOf finds index of key or where it could be inserted.
func (store *Client) indexOf(key storage.Key) (int, bool) {
func (store *Client) indexOf(key kvstore.Key) (int, bool) {
i := sort.Search(len(store.Items), func(k int) bool {
return !store.Items[k].Key.Less(key)
})
@ -67,7 +67,7 @@ func (store *Client) forcedError() bool {
}
// Put adds a value to store.
func (store *Client) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) {
func (store *Client) Put(ctx context.Context, key kvstore.Key, value kvstore.Value) (err error) {
defer mon.Task()(&ctx)(&err)
defer store.locked()()
@ -78,13 +78,13 @@ func (store *Client) Put(ctx context.Context, key storage.Key, value storage.Val
}
if key.IsZero() {
return storage.ErrEmptyKey.New("")
return kvstore.ErrEmptyKey.New("")
}
keyIndex, found := store.indexOf(key)
if found {
kv := &store.Items[keyIndex]
kv.Value = storage.CloneValue(value)
kv.Value = kvstore.CloneValue(value)
return nil
}
@ -93,7 +93,7 @@ func (store *Client) Put(ctx context.Context, key storage.Key, value storage.Val
}
// Get gets a value to store.
func (store *Client) Get(ctx context.Context, key storage.Key) (_ storage.Value, err error) {
func (store *Client) Get(ctx context.Context, key kvstore.Key) (_ kvstore.Value, err error) {
defer mon.Task()(&ctx)(&err)
defer store.locked()()
@ -104,19 +104,19 @@ func (store *Client) Get(ctx context.Context, key storage.Key) (_ storage.Value,
}
if key.IsZero() {
return nil, storage.ErrEmptyKey.New("")
return nil, kvstore.ErrEmptyKey.New("")
}
keyIndex, found := store.indexOf(key)
if !found {
return nil, storage.ErrKeyNotFound.New("%q", key)
return nil, kvstore.ErrKeyNotFound.New("%q", key)
}
return storage.CloneValue(store.Items[keyIndex].Value), nil
return kvstore.CloneValue(store.Items[keyIndex].Value), nil
}
// Delete deletes key and the value.
func (store *Client) Delete(ctx context.Context, key storage.Key) (err error) {
func (store *Client) Delete(ctx context.Context, key kvstore.Key) (err error) {
defer mon.Task()(&ctx)(&err)
defer store.locked()()
@ -128,12 +128,12 @@ func (store *Client) Delete(ctx context.Context, key storage.Key) (err error) {
}
if key.IsZero() {
return storage.ErrEmptyKey.New("")
return kvstore.ErrEmptyKey.New("")
}
keyIndex, found := store.indexOf(key)
if !found {
return storage.ErrKeyNotFound.New("%q", key)
return kvstore.ErrKeyNotFound.New("%q", key)
}
store.delete(keyIndex)
@ -152,14 +152,14 @@ func (store *Client) Close() error {
}
// Range iterates over all items in unspecified order.
func (store *Client) Range(ctx context.Context, fn func(context.Context, storage.Key, storage.Value) error) error {
func (store *Client) Range(ctx context.Context, fn func(context.Context, kvstore.Key, kvstore.Value) error) error {
store.mu.Lock()
store.CallCount.Range++
if store.forcedError() {
store.mu.Unlock()
return errors.New("internal error")
}
items := append([]storage.ListItem{}, store.Items...)
items := append([]kvstore.Item{}, store.Items...)
store.mu.Unlock()
for _, item := range items {
@ -170,12 +170,12 @@ func (store *Client) Range(ctx context.Context, fn func(context.Context, storage
return nil
}
func (store *Client) put(keyIndex int, key storage.Key, value storage.Value) {
store.Items = append(store.Items, storage.ListItem{})
func (store *Client) put(keyIndex int, key kvstore.Key, value kvstore.Value) {
store.Items = append(store.Items, kvstore.Item{})
copy(store.Items[keyIndex+1:], store.Items[keyIndex:])
store.Items[keyIndex] = storage.ListItem{
Key: storage.CloneKey(key),
Value: storage.CloneValue(value),
store.Items[keyIndex] = kvstore.Item{
Key: kvstore.CloneKey(key),
Value: kvstore.CloneValue(value),
}
}

2
storage/teststore/store_test.go → private/kvstore/teststore/store_test.go
View File

@ -6,7 +6,7 @@ package teststore
import (
"testing"
"storj.io/storj/storage/testsuite"
"storj.io/storj/private/kvstore/testsuite"
)
func TestSuite(t *testing.T) {

14
storage/testsuite/bench.go → private/kvstore/testsuite/bench.go
View File

@ -11,11 +11,11 @@ import (
"golang.org/x/sync/errgroup"
"storj.io/common/testcontext"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
// RunBenchmarks runs common storage.KeyValueStore benchmarks.
func RunBenchmarks(b *testing.B, store storage.KeyValueStore) {
// RunBenchmarks runs common kvstore.Store benchmarks.
func RunBenchmarks(b *testing.B, store kvstore.Store) {
var words = []string{
"alpha", "beta", "gamma", "delta", "iota", "kappa", "lambda", "mu",
"άλφα", "βήτα", "γάμμα", "δέλτα", "έψιλον", "ζήτα", "ήτα", "θήτα", "ιώτα", "κάππα", "λάμδα", "μυ",
@ -28,15 +28,15 @@ func RunBenchmarks(b *testing.B, store storage.KeyValueStore) {
words = words[:2]
}
var items storage.Items
var items kvstore.Items
k := 0
for _, a := range words {
for _, b := range words {
for _, c := range words {
items = append(items, storage.ListItem{
Key: storage.Key(path.Join(a, b, c)),
Value: storage.Value(strconv.Itoa(k)),
items = append(items, kvstore.Item{
Key: kvstore.Key(path.Join(a, b, c)),
Value: kvstore.Value(strconv.Itoa(k)),
})
k++
}

20
storage/testsuite/test.go → private/kvstore/testsuite/test.go
View File

@ -10,11 +10,11 @@ import (
"golang.org/x/sync/errgroup"
"storj.io/common/testcontext"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
// RunTests runs common storage.KeyValueStore tests.
func RunTests(t *testing.T, store storage.KeyValueStore) {
// RunTests runs common kvstore.Store tests.
func RunTests(t *testing.T, store kvstore.Store) {
// store = storelogger.NewTest(t, store)
ctx := testcontext.New(t)
defer ctx.Cleanup()
@ -24,12 +24,12 @@ func RunTests(t *testing.T, store storage.KeyValueStore) {
t.Run("Parallel", func(t *testing.T) { testParallel(t, ctx, store) })
}
func testConstraints(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
var items storage.Items
func testConstraints(t *testing.T, ctx *testcontext.Context, store kvstore.Store) {
var items kvstore.Items
for i := 0; i < 10; i++ {
items = append(items, storage.ListItem{
Key: storage.Key("test-" + strconv.Itoa(i)),
Value: storage.Value("xyz"),
items = append(items, kvstore.Item{
Key: kvstore.Key("test-" + strconv.Itoa(i)),
Value: kvstore.Value("xyz"),
})
}
@ -47,8 +47,8 @@ func testConstraints(t *testing.T, ctx *testcontext.Context, store storage.KeyVa
defer cleanupItems(t, ctx, store, items)
t.Run("Put Empty", func(t *testing.T) {
var key storage.Key
var val storage.Value
var key kvstore.Key
var val kvstore.Value
defer func() { _ = store.Delete(ctx, key) }()
err := store.Put(ctx, key, val)

6
storage/testsuite/test_crud.go → private/kvstore/testsuite/test_crud.go
View File

@ -9,11 +9,11 @@ import (
"testing"
"storj.io/common/testcontext"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
func testCRUD(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
func testCRUD(t *testing.T, ctx *testcontext.Context, store kvstore.Store) {
items := kvstore.Items{
// newItem("0", "", false), // TODO: broken
newItem("\x00", "\x00", false),
newItem("a/b", "\x01\x00", false),

6
storage/testsuite/test_parallel.go → private/kvstore/testsuite/test_parallel.go
View File

@ -10,11 +10,11 @@ import (
"testing"
"storj.io/common/testcontext"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
func testParallel(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
items := storage.Items{
func testParallel(t *testing.T, ctx *testcontext.Context, store kvstore.Store) {
items := kvstore.Items{
newItem("a", "1", false),
newItem("b", "2", false),
newItem("c", "3", false),

18
storage/testsuite/test_range.go → private/kvstore/testsuite/test_range.go
View File

@ -13,16 +13,16 @@ import (
"github.com/stretchr/testify/require"
"storj.io/common/testcontext"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
func testRange(t *testing.T, ctx *testcontext.Context, store storage.KeyValueStore) {
err := store.Range(ctx, func(ctx context.Context, key storage.Key, value storage.Value) error {
func testRange(t *testing.T, ctx *testcontext.Context, store kvstore.Store) {
err := store.Range(ctx, func(ctx context.Context, key kvstore.Key, value kvstore.Value) error {
return errors.New("empty store")
})
require.NoError(t, err)
items := storage.Items{
items := kvstore.Items{
newItem("a", "a", false),
newItem("b/1", "b/1", false),
newItem("b/2", "b/2", false),
@ -37,13 +37,13 @@ func testRange(t *testing.T, ctx *testcontext.Context, store storage.KeyValueSto
rand.Shuffle(len(items), items.Swap)
defer cleanupItems(t, ctx, store, items)
if err := storage.PutAll(ctx, store, items...); err != nil {
if err := kvstore.PutAll(ctx, store, items...); err != nil {
t.Fatalf("failed to setup: %v", err)
}
var output storage.Items
err = store.Range(ctx, func(ctx context.Context, key storage.Key, value storage.Value) error {
output = append(output, storage.ListItem{
var output kvstore.Items
err = store.Range(ctx, func(ctx context.Context, key kvstore.Key, value kvstore.Value) error {
output = append(output, kvstore.Item{
Key: append([]byte{}, key...),
Value: append([]byte{}, value...),
})
@ -51,7 +51,7 @@ func testRange(t *testing.T, ctx *testcontext.Context, store storage.KeyValueSto
})
require.NoError(t, err)
expected := storage.CloneItems(items)
expected := kvstore.CloneItems(items)
sort.Sort(expected)
sort.Sort(output)

12
storage/testsuite/utils.go → private/kvstore/testsuite/utils.go
View File

@ -7,18 +7,18 @@ import (
"testing"
"storj.io/common/testcontext"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
func newItem(key, value string, isPrefix bool) storage.ListItem {
return storage.ListItem{
Key: storage.Key(key),
Value: storage.Value(value),
func newItem(key, value string, isPrefix bool) kvstore.Item {
return kvstore.Item{
Key: kvstore.Key(key),
Value: kvstore.Value(value),
IsPrefix: isPrefix,
}
}
func cleanupItems(t testing.TB, ctx *testcontext.Context, store storage.KeyValueStore, items storage.Items) {
func cleanupItems(t testing.TB, ctx *testcontext.Context, store kvstore.Store, items kvstore.Items) {
for _, item := range items {
_ = store.Delete(ctx, item.Key)
}

8
storage/util.go → private/kvstore/util.go
View File

@ -1,7 +1,7 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package storage
package kvstore
import (
"context"
@ -15,8 +15,8 @@ func CloneKey(key Key) Key { return append(Key{}, key...) }
func CloneValue(value Value) Value { return append(Value{}, value...) }
// CloneItem creates a deep copy of item.
func CloneItem(item ListItem) ListItem {
return ListItem{
func CloneItem(item Item) Item {
return Item{
Key: CloneKey(item.Key),
Value: CloneValue(item.Value),
IsPrefix: item.IsPrefix,
@ -33,7 +33,7 @@ func CloneItems(items Items) Items {
}
// PutAll adds multiple values to the store.
func PutAll(ctx context.Context, store KeyValueStore, items ...ListItem) (err error) {
func PutAll(ctx context.Context, store Store, items ...Item) (err error) {
defer mon.Task()(&ctx)(&err)
for _, item := range items {

4
private/revocation/common.go
View File

@ -9,8 +9,8 @@ import (
"storj.io/common/peertls/extensions"
"storj.io/common/peertls/tlsopts"
"storj.io/private/dbutil"
"storj.io/storj/storage/boltdb"
"storj.io/storj/storage/redis"
"storj.io/storj/private/kvstore/boltdb"
"storj.io/storj/private/kvstore/redis"
)
// OpenDBFromCfg is a convenience method to create a revocation DB

10
private/revocation/db.go
View File

@ -14,7 +14,7 @@ import (
"storj.io/common/identity"
"storj.io/common/peertls"
"storj.io/common/peertls/extensions"
"storj.io/storj/storage"
"storj.io/storj/private/kvstore"
)
var (
@ -28,7 +28,7 @@ var (
// (i.e. nodeID [CA certificate's public key hash] is the key, values is
// the most recently seen revocation).
type DB struct {
store storage.KeyValueStore
store kvstore.Store
}
// Get attempts to retrieve the most recent revocation for the given cert chain
@ -46,7 +46,7 @@ func (db *DB) Get(ctx context.Context, chain []*x509.Certificate) (_ *extensions
}
revBytes, err := db.store.Get(ctx, nodeID.Bytes())
if err != nil && !storage.ErrKeyNotFound.Has(err) {
if err != nil && !kvstore.ErrKeyNotFound.Has(err) {
return nil, extensions.ErrRevocationDB.Wrap(err)
}
if revBytes == nil {
@ -108,7 +108,7 @@ func (db *DB) List(ctx context.Context) (revs []*extensions.Revocation, err erro
return nil, nil
}
err = db.store.Range(ctx, func(ctx context.Context, key storage.Key, value storage.Value) error {
err = db.store.Range(ctx, func(ctx context.Context, key kvstore.Key, value kvstore.Value) error {
rev := new(extensions.Revocation)
if err := rev.Unmarshal(value); err != nil {
return extensions.ErrRevocationDB.Wrap(err)
@ -121,7 +121,7 @@ func (db *DB) List(ctx context.Context) (revs []*extensions.Revocation, err erro
}
// TestGetStore returns the internal store for testing.
func (db *DB) TestGetStore() storage.KeyValueStore {
func (db *DB) TestGetStore() kvstore.Store {
return db.store
}

10
private/revocation/db_test.go
View File

@ -18,15 +18,15 @@ import (
"storj.io/common/peertls/testpeertls"
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/storj/private/kvstore"
"storj.io/storj/private/testrevocation"
"storj.io/storj/storage"
)
func TestRevocationDB_Get(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
require.NoError(t, err)
@ -64,7 +64,7 @@ func TestRevocationDB_Put_success(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
require.NoError(t, err)
@ -114,7 +114,7 @@ func TestRevocationDB_Put_error(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
require.NoError(t, err)
@ -156,7 +156,7 @@ func TestRevocationDB_List(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
require.NoError(t, err)
keys2, chain2, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)

8
private/revocation/extensions_test.go
View File

@ -18,15 +18,15 @@ import (
"storj.io/common/peertls/tlsopts"
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/storj/private/kvstore"
"storj.io/storj/private/testrevocation"
"storj.io/storj/storage"
)
func TestRevocationCheckHandler(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ kvstore.Store) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
assert.NoError(t, err)
@ -68,7 +68,7 @@ func TestRevocationCheckHandler(t *testing.T) {
}
})
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ kvstore.Store) {
t.Log("new revocation DB")
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
assert.NoError(t, err)
@ -123,7 +123,7 @@ func TestRevocationUpdateHandler(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, _ kvstore.Store) {
keys, chain, err := testpeertls.NewCertChain(2, storj.LatestIDVersion().Number)
assert.NoError(t, err)

6
private/revocation/tlsopts_test.go
View File

@ -21,9 +21,9 @@ import (
"storj.io/common/peertls/tlsopts"
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/storj/private/kvstore"
"storj.io/storj/private/revocation"
"storj.io/storj/private/testrevocation"
"storj.io/storj/storage"
)
func TestNewOptions(t *testing.T) {
@ -150,7 +150,7 @@ func TestExtensionMap_HandleExtensions(t *testing.T) {
err = rev.Verify(newRevokedLeafChain[peertls.CAIndex])
require.NoError(t, err)
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) {
opts := &extensions.Options{
RevocationDB: revDB,
PeerIDVersions: "*",
@ -183,7 +183,7 @@ func TestExtensionMap_HandleExtensions_error(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db storage.KeyValueStore) {
testrevocation.RunDBs(t, func(t *testing.T, revDB extensions.RevocationDB, db kvstore.Store) {
keys, chain, oldRevocation, err := testpeertls.NewRevokedLeafChain()
assert.NoError(t, err)

4
private/testrevocation/db.go
View File

@ -10,13 +10,13 @@ import (
"storj.io/common/peertls/extensions"
"storj.io/common/testcontext"
"storj.io/storj/private/kvstore"
"storj.io/storj/private/revocation"
"storj.io/storj/private/testredis"
"storj.io/storj/storage"
)
// RunDBs runs the passed test function with each type of revocation database.
func RunDBs(t *testing.T, test func(*testing.T, extensions.RevocationDB, storage.KeyValueStore)) {
func RunDBs(t *testing.T, test func(*testing.T, extensions.RevocationDB, kvstore.Store)) {
t.Run("Redis", func(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()