JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove duplicated mock store implementations (#337)

Browse Source
This commit is contained in:
Egon Elbre 2018-09-11 10:27:12 +03:00 committed by GitHub
parent d0f87f0de1
commit 6ee6f0fdf5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 255 additions and 484 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -31,7 +31,7 @@ DOCKER_BUILD := docker build \
# gochecknoglobals # enable later
# dupl # needs tuning
# gocyclo # needs tuning
# lll # long lines, not relevant
# lll # long lines
# gotype, gotypex # already done by compiling
# safesql # no sql
# interfacer # not that useful

160
internal/test/storage.go
View File

@ -1,160 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package test
import (
"bytes"
"sort"
"github.com/zeebo/errs"
"storj.io/storj/storage"
)
// KvStore is an in-memory, crappy key/value store type for testing
type KvStore map[string]storage.Value
// Empty checks if there are any keys in the store
func (k *KvStore) Empty() bool {
return len(*k) == 0
}
// MockKeyValueStore is a `KeyValueStore` type used for testing (see storj.io/storj/storage/common.go)
type MockKeyValueStore struct {
Data KvStore
GetCalled int
GetAllCalled int
PutCalled int
ListCalled int
ReverseListCalled int
DeleteCalled int
CloseCalled int
PingCalled int
IterateCalled int
}
var (
// ErrMissingKey is the error class returned if a key is not in the mock store
ErrMissingKey = errs.Class("missing")
// ErrForced is the error class returned when the forced error flag is passed
// to mock an error
ErrForced = errs.Class("error forced by using 'error' key in mock")
)
// Get looks up the provided key from the MockKeyValueStore returning either an error or the result.
func (store *MockKeyValueStore) Get(key storage.Key) (storage.Value, error) {
store.GetCalled++
if key.String() == "error" {
return nil, nil
}
v, ok := store.Data[key.String()]
if !ok {
return storage.Value{}, nil
}
return v, nil
}
// Put adds a value to the provided key in the MockKeyValueStore, returning an error on failure.
func (store *MockKeyValueStore) Put(key storage.Key, value storage.Value) error {
store.PutCalled++
store.Data[key.String()] = value
return nil
}
// Delete deletes a key/value pair from the MockKeyValueStore, for a given the key
func (store *MockKeyValueStore) Delete(key storage.Key) error {
store.DeleteCalled++
delete(store.Data, key.String())
return nil
}
// List returns either a list of keys for which the MockKeyValueStore has values or an error.
func (store *MockKeyValueStore) List(first storage.Key, limit int) (storage.Keys, error) {
store.ListCalled++
return storage.ListKeys(store, first, limit)
}
// GetAll is a noop to adhere to the interface
func (store *MockKeyValueStore) GetAll(keys storage.Keys) (values storage.Values, err error) {
store.GetAllCalled++
result := storage.Values{}
for _, v := range keys {
result = append(result, store.Data[v.String()])
}
return result, nil
}
func (store *MockKeyValueStore) allPrefixedItems(prefix, first, last storage.Key) storage.Items {
var all storage.Items
for key, value := range store.Data {
if !bytes.HasPrefix([]byte(key), prefix) {
continue
}
if first != nil && storage.Key(key).Less(first) {
continue
}
if last != nil && last.Less(storage.Key(key)) {
continue
}
all = append(all, storage.ListItem{
Key: storage.Key(key),
Value: value,
IsPrefix: false,
})
}
sort.Sort(all)
return all
}
// ReverseList returns either a list of keys for which the MockKeyValueStore has values or an error.
func (store *MockKeyValueStore) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
return storage.ReverseListKeys(store, first, limit)
}
// Iterate iterates over items based on opts
func (store *MockKeyValueStore) Iterate(opts storage.IterateOptions, fn func(storage.Iterator) error) error {
store.IterateCalled++
var items storage.Items
if !opts.Reverse {
items = store.allPrefixedItems(opts.Prefix, opts.First, nil)
} else {
items = store.allPrefixedItems(opts.Prefix, nil, opts.First)
}
if !opts.Recurse {
items = storage.SortAndCollapse(items, opts.Prefix)
}
if opts.Reverse {
items = storage.ReverseItems(items)
}
return fn(&storage.StaticIterator{
Items: items,
})
}
// Close closes the client
func (store *MockKeyValueStore) Close() error {
store.CloseCalled++
return nil
}
// Ping is called by some redis client code
func (store *MockKeyValueStore) Ping() error {
store.PingCalled++
return nil
}
// NewMockKeyValueStore returns a mocked `KeyValueStore` implementation for testing
func NewMockKeyValueStore(d KvStore) *MockKeyValueStore {
return &MockKeyValueStore{
Data: d,
}
}

32
internal/test/utils.go
View File

@ -1,32 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package test
import (
"testing"
pb "github.com/gogo/protobuf/proto"
"github.com/stretchr/testify/assert"
"storj.io/storj/pkg/kademlia"
proto "storj.io/storj/protos/overlay"
"storj.io/storj/storage"
)
// NewNodeAddressValue provides a convient way to create a storage.Value for testing purposes
func NewNodeAddressValue(t *testing.T, address string) storage.Value {
na := &proto.NodeAddress{Transport: proto.NodeTransport_TCP, Address: address}
d, err := pb.Marshal(na)
assert.NoError(t, err)
return d
}
// NewNodeID returns the string representation of a dht node ID
func NewNodeID(t *testing.T) string {
id, err := kademlia.NewID()
assert.NoError(t, err)
return id.String()
}

16
pkg/node/node_test.go
View File

@ -11,7 +11,7 @@ import (
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"storj.io/storj/internal/test"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/provider"
proto "storj.io/storj/protos/overlay"
)
@ -26,9 +26,9 @@ func TestLookup(t *testing.T) {
expectedErr error
}{
{
self: proto.Node{Id: test.NewNodeID(t), Address: &proto.NodeAddress{Address: ":7070"}},
self: proto.Node{Id: NewNodeID(t), Address: &proto.NodeAddress{Address: ":7070"}},
to: proto.Node{}, // filled after server has been started
find: proto.Node{Id: test.NewNodeID(t), Address: &proto.NodeAddress{Address: ":9090"}},
find: proto.Node{Id: NewNodeID(t), Address: &proto.NodeAddress{Address: ":9090"}},
expectedErr: nil,
},
}
@ -37,7 +37,7 @@ func TestLookup(t *testing.T) {
lis, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
v.to = proto.Node{Id: test.NewNodeID(t), Address: &proto.NodeAddress{Address: lis.Addr().String()}}
v.to = proto.Node{Id: NewNodeID(t), Address: &proto.NodeAddress{Address: lis.Addr().String()}}
srv, mock, err := newTestServer(ctx)
assert.NoError(t, err)
@ -89,3 +89,11 @@ func (mn *mockNodeServer) Query(ctx context.Context, req *proto.QueryRequest) (*
mn.queryCalled++
return &proto.QueryResponse{}, nil
}
// NewNodeID returns the string representation of a dht node ID
func NewNodeID(t *testing.T) string {
id, err := kademlia.NewID()
assert.NoError(t, err)
return id.String()
}

231
pkg/overlay/cache_test.go
View File

@ -16,7 +16,6 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap/zaptest"
"storj.io/storj/internal/test"
"storj.io/storj/pkg/dht"
"storj.io/storj/pkg/kademlia"
"storj.io/storj/protos/overlay"
@ -25,6 +24,7 @@ import (
"storj.io/storj/storage/redis"
"storj.io/storj/storage/redis/redisserver"
"storj.io/storj/storage/storelogger"
"storj.io/storj/storage/teststore"
)
var (
@ -108,7 +108,7 @@ var (
key string
expectedResponses responses
expectedErrors errors
data test.KvStore
data []storage.ListItem
}{
{
testID: "valid Get",
@ -127,16 +127,18 @@ var (
bolt: nil,
_redis: nil,
},
data: test.KvStore{"foo": func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}()},
},
{
data: []storage.ListItem{{
Key: storage.Key("foo"),
Value: func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
}},
}, {
testID: "forced get error",
expectedTimesCalled: 1,
key: "error",
@ -153,14 +155,17 @@ var (
bolt: nil,
_redis: nil,
},
data: test.KvStore{"error": func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}()},
data: []storage.ListItem{{
Key: storage.Key("error"),
Value: func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
}},
},
{
testID: "get missing key",
@ -171,20 +176,22 @@ var (
bolt: nil,
_redis: nil,
},
// TODO(bryanchriswhite): compare actual errors
expectedErrors: errors{
mock: nil,
mock: &storage.ErrKeyNotFound,
bolt: &storage.ErrKeyNotFound,
_redis: &storage.ErrKeyNotFound,
},
data: test.KvStore{"foo": func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}()},
data: []storage.ListItem{{
Key: storage.Key("foo"),
Value: func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
}},
},
}
getAllCases = []struct {
@ -193,7 +200,7 @@ var (
keys []string
expectedResponses responsesB
expectedErrors errors
data test.KvStore
data []storage.ListItem
}{
{testID: "valid GetAll",
expectedTimesCalled: 1,
@ -212,15 +219,18 @@ var (
bolt: nil,
_redis: nil,
},
data: test.KvStore{
"key1": func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
data: []storage.ListItem{
{
Key: storage.Key("key1"),
Value: func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
},
},
},
{testID: "valid GetAll",
@ -241,23 +251,28 @@ var (
bolt: nil,
_redis: nil,
},
data: test.KvStore{
"key1": func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
"key2": func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9998"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
data: []storage.ListItem{
{
Key: storage.Key("key1"),
Value: func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
}, {
Key: storage.Key("key2"),
Value: func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9998"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
},
},
},
{testID: "mix of valid and nil nodes returned",
@ -277,15 +292,18 @@ var (
bolt: nil,
_redis: nil,
},
data: test.KvStore{
"key1": func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
data: []storage.ListItem{
{
Key: storage.Key("key1"),
Value: func() storage.Value {
na := &overlay.Node{Address: &overlay.NodeAddress{Transport: overlay.NodeTransport_TCP, Address: "127.0.0.1:9999"}}
d, err := proto.Marshal(na)
if err != nil {
panic(err)
}
return d
}(),
},
},
},
{testID: "empty string keys",
@ -328,7 +346,7 @@ var (
key string
value overlay.Node
expectedErrors errors
data test.KvStore
data []storage.ListItem
}{
{
testID: "valid Put",
@ -340,7 +358,7 @@ var (
bolt: nil,
_redis: nil,
},
data: test.KvStore{},
data: []storage.ListItem{},
},
}
@ -348,31 +366,31 @@ var (
testID string
expectedTimesCalled int
expectedErr error
data test.KvStore
data []storage.ListItem
}{
{
testID: "valid update",
expectedTimesCalled: 1,
expectedErr: nil,
data: test.KvStore{},
data: []storage.ListItem{},
},
}
)
func redisTestClient(t *testing.T, addr string, data test.KvStore) storage.KeyValueStore {
func redisTestClient(t *testing.T, addr string, items []storage.ListItem) storage.KeyValueStore {
client, err := redis.NewClient(addr, "", 1)
if err != nil {
t.Fatal(err)
}
if !(data.Empty()) {
populateStorage(t, client, data)
if err := storage.PutAll(client, items...); err != nil {
t.Fatal(err)
}
return client
}
func boltTestClient(t *testing.T, data test.KvStore) (_ storage.KeyValueStore, _ func()) {
func boltTestClient(t *testing.T, items []storage.ListItem) (_ storage.KeyValueStore, _ func()) {
boltPath, err := filepath.Abs("test_bolt.db")
assert.NoError(t, err)
@ -384,20 +402,13 @@ func boltTestClient(t *testing.T, data test.KvStore) (_ storage.KeyValueStore, _
assert.NoError(t, os.Remove(boltPath))
}
if !(data.Empty()) {
populateStorage(t, client, data)
if err := storage.PutAll(client, items...); err != nil {
t.Fatal(err)
}
return storelogger.New(zaptest.NewLogger(t), client), cleanup
}
func populateStorage(t *testing.T, client storage.KeyValueStore, data test.KvStore) {
for k, v := range data {
err := client.Put(storage.Key(k), v)
assert.NoError(t, err)
}
}
func TestRedisGet(t *testing.T) {
redisAddr, cleanup, err := redisserver.Start()
if err != nil {
@ -436,6 +447,7 @@ func TestRedisGetAll(t *testing.T) {
}
func assertErrClass(t *testing.T, class *errs.Class, err error) {
t.Helper()
if class != nil {
assert.True(t, class.Has(err))
} else {
@ -520,15 +532,25 @@ func TestMockGet(t *testing.T) {
for _, c := range getCases {
t.Run(c.testID, func(t *testing.T) {
db := test.NewMockKeyValueStore(c.data)
db := teststore.New()
if err := storage.PutAll(db, c.data...); err != nil {
t.Fatal(err)
}
oc := Cache{DB: db}
assert.Equal(t, 0, db.GetCalled)
if c.key == "error" {
db.ForceError = 1
}
assert.Equal(t, 0, db.CallCount.Get)
resp, err := oc.Get(ctx, c.key)
assertErrClass(t, c.expectedErrors[mock], err)
if c.key == "error" {
assert.Error(t, err)
} else {
assertErrClass(t, c.expectedErrors[mock], err)
}
assert.Equal(t, c.expectedResponses[mock], resp)
assert.Equal(t, c.expectedTimesCalled, db.GetCalled)
assert.Equal(t, c.expectedTimesCalled, db.CallCount.Get)
})
}
}
@ -537,15 +559,18 @@ func TestMockGetAll(t *testing.T) {
for _, c := range getAllCases {
t.Run(c.testID, func(t *testing.T) {
db := test.NewMockKeyValueStore(c.data)
db := teststore.New()
if err := storage.PutAll(db, c.data...); err != nil {
t.Fatal(err)
}
oc := Cache{DB: db}
assert.Equal(t, 0, db.GetAllCalled)
assert.Equal(t, 0, db.CallCount.GetAll)
resp, err := oc.GetAll(ctx, c.keys)
assertErrClass(t, c.expectedErrors[mock], err)
assert.Equal(t, c.expectedResponses[mock], resp)
assert.Equal(t, c.expectedTimesCalled, db.GetAllCalled)
assert.Equal(t, c.expectedTimesCalled, db.CallCount.GetAll)
})
}
}
@ -553,19 +578,22 @@ func TestMockGetAll(t *testing.T) {
func TestMockPut(t *testing.T) {
for _, c := range putCases {
t.Run(c.testID, func(t *testing.T) {
db := teststore.New()
if err := storage.PutAll(db, c.data...); err != nil {
t.Fatal(err)
}
db.CallCount.Put = 0
db := test.NewMockKeyValueStore(c.data)
oc := Cache{DB: db}
assert.Equal(t, 0, db.PutCalled)
err := oc.Put(c.key, c.value)
assertErrClass(t, c.expectedErrors[mock], err)
assert.Equal(t, c.expectedTimesCalled, db.PutCalled)
assert.Equal(t, c.expectedTimesCalled, db.CallCount.Put)
v, err := db.Get(storage.Key(c.key))
assert.NoError(t, err)
v := db.Data[c.key]
na := &overlay.Node{}
assert.NoError(t, proto.Unmarshal(v, na))
assert.True(t, proto.Equal(na, &c.value))
})
@ -576,10 +604,15 @@ func TestRefresh(t *testing.T) {
t.Skip()
for _, c := range refreshCases {
t.Run(c.testID, func(t *testing.T) {
dhts, b := bootstrapTestNetwork(t, "127.0.0.1", "3000")
dhts, b := bootstrapTestNetwork(t, "127.0.0.1", "0")
ctx := context.Background()
db := test.NewMockKeyValueStore(c.data)
dht := newTestKademlia(t, "127.0.0.1", "2999", dhts[rand.Intn(testNetSize)], b)
db := teststore.New()
if err := storage.PutAll(db, c.data...); err != nil {
t.Fatal(err)
}
dht := newTestKademlia(t, "127.0.0.1", "0", dhts[rand.Intn(testNetSize)], b)
_cache := &Cache{
DB: db,

19
pkg/overlay/client_test.go
View File

@ -5,7 +5,6 @@ package overlay
import (
"context"
"fmt"
"net"
"testing"
@ -17,6 +16,7 @@ import (
"storj.io/storj/pkg/kademlia"
"storj.io/storj/pkg/provider"
proto "storj.io/storj/protos/overlay"
"storj.io/storj/storage/redis/redisserver"
)
type mockNodeID struct {
@ -141,7 +141,7 @@ func TestBulkLookup(t *testing.T) {
},
}
for _, v := range cases {
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
lis, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
srv, mock, err := newTestServer(ctx)
@ -166,10 +166,16 @@ func TestBulkLookup(t *testing.T) {
}
}
func TestBulkLookupV2(t *testing.T) {
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
redisAddr, cleanup, err := redisserver.Start()
if err != nil {
t.Fatal(err)
}
defer cleanup()
lis, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
srv, s, err := newServer(ctx)
srv, s, err := newServer(ctx, redisAddr)
assert.NoError(t, err)
go srv.Serve(lis)
@ -246,7 +252,7 @@ func TestBulkLookupV2(t *testing.T) {
}
}
func newServer(ctx context.Context) (*grpc.Server, *Server, error) {
func newServer(ctx context.Context, redisAddr string) (*grpc.Server, *Server, error) {
ca, err := provider.NewCA(ctx, 12, 4)
if err != nil {
return nil, nil, err
@ -261,7 +267,7 @@ func newServer(ctx context.Context) (*grpc.Server, *Server, error) {
}
grpcServer := grpc.NewServer(identOpt)
cache, err := NewRedisOverlayCache("127.0.0.1:6379", "", 1, nil)
cache, err := NewRedisOverlayCache(redisAddr, "", 1, nil)
if err != nil {
return nil, nil, err
}
@ -271,6 +277,7 @@ func newServer(ctx context.Context) (*grpc.Server, *Server, error) {
return grpcServer, s, nil
}
func newTestServer(ctx context.Context) (*grpc.Server, *mockOverlayServer, error) {
ca, err := provider.NewCA(ctx, 12, 4)
if err != nil {

29
pkg/overlay/overlay_test.go
View File

@ -5,16 +5,15 @@ package overlay
import (
"context"
"fmt"
"net"
"testing"
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"storj.io/storj/internal/test"
"storj.io/storj/pkg/kademlia"
proto "storj.io/storj/protos/overlay" // naming proto to avoid confusion with this package
"storj.io/storj/storage"
)
func TestFindStorageNodes(t *testing.T) {
@ -26,7 +25,15 @@ func TestFindStorageNodes(t *testing.T) {
id2, err := kademlia.NewID()
assert.NoError(t, err)
srv := NewMockServer(test.KvStore{id.String(): NewNodeAddressValue(t, "127.0.0.1:9090"), id2.String(): NewNodeAddressValue(t, "127.0.0.1:9090")})
srv := NewMockServer([]storage.ListItem{
{
Key: storage.Key(id.String()),
Value: storage.Value(NewNodeAddressValue(t, "127.0.0.1:9090")),
}, {
Key: storage.Key(id2.String()),
Value: storage.Value(NewNodeAddressValue(t, "127.0.0.1:9090")),
},
})
assert.NotNil(t, srv)
go srv.Serve(lis)
@ -51,7 +58,12 @@ func TestOverlayLookup(t *testing.T) {
assert.NoError(t, err)
srv := NewMockServer(test.KvStore{id.String(): NewNodeAddressValue(t, "127.0.0.1:9090")})
srv := NewMockServer([]storage.ListItem{
{
Key: storage.Key(id.String()),
Value: storage.Value(NewNodeAddressValue(t, "127.0.0.1:9090")),
},
})
go srv.Serve(lis)
defer srv.Stop()
@ -65,7 +77,7 @@ func TestOverlayLookup(t *testing.T) {
}
func TestOverlayBulkLookup(t *testing.T) {
lis, err := net.Listen("tcp", fmt.Sprintf(":%d", 0))
lis, err := net.Listen("tcp", "127.0.0.1:0")
assert.NoError(t, err)
id, err := kademlia.NewID()
@ -73,7 +85,12 @@ func TestOverlayBulkLookup(t *testing.T) {
id2, err := kademlia.NewID()
assert.NoError(t, err)
srv := NewMockServer(test.KvStore{id.String(): NewNodeAddressValue(t, "127.0.0.1:9090")})
srv := NewMockServer([]storage.ListItem{
{
Key: storage.Key(id.String()),
Value: storage.Value(NewNodeAddressValue(t, "127.0.0.1:9090")),
},
})
go srv.Serve(lis)
defer srv.Stop()

8
pkg/overlay/utils_test.go
View File

@ -12,14 +12,14 @@ import (
"google.golang.org/grpc"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/internal/test"
"storj.io/storj/pkg/kademlia"
proto "storj.io/storj/protos/overlay"
"storj.io/storj/storage"
"storj.io/storj/storage/teststore"
)
// NewMockServer provides a mock grpc server for testing
func NewMockServer(kv test.KvStore) *grpc.Server {
func NewMockServer(items []storage.ListItem) *grpc.Server {
grpcServer := grpc.NewServer()
registry := monkit.Default
@ -27,10 +27,12 @@ func NewMockServer(kv test.KvStore) *grpc.Server {
k := kademlia.NewMockKademlia()
c := &Cache{
DB: test.NewMockKeyValueStore(kv),
DB: teststore.New(),
DHT: k,
}
_ = storage.PutAll(c.DB, items...)
s := Server{
dht: k,
cache: c,

134
pkg/pointerdb/kvstore_mock_test.go
View File

@ -1,134 +0,0 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: storj.io/storj/storage (interfaces: KeyValueStore)
// Package pointerdb is a generated GoMock package.
package pointerdb
import (
gomock "github.com/golang/mock/gomock"
reflect "reflect"
storage "storj.io/storj/storage"
)
// MockKeyValueStore is a mock of KeyValueStore interface
type MockKeyValueStore struct {
ctrl *gomock.Controller
recorder *MockKeyValueStoreMockRecorder
}
// MockKeyValueStoreMockRecorder is the mock recorder for MockKeyValueStore
type MockKeyValueStoreMockRecorder struct {
mock *MockKeyValueStore
}
// NewMockKeyValueStore creates a new mock instance
func NewMockKeyValueStore(ctrl *gomock.Controller) *MockKeyValueStore {
mock := &MockKeyValueStore{ctrl: ctrl}
mock.recorder = &MockKeyValueStoreMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockKeyValueStore) EXPECT() *MockKeyValueStoreMockRecorder {
return m.recorder
}
// Close mocks base method
func (m *MockKeyValueStore) Close() error {
ret := m.ctrl.Call(m, "Close")
ret0, _ := ret[0].(error)
return ret0
}
// Close indicates an expected call of Close
func (mr *MockKeyValueStoreMockRecorder) Close() *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockKeyValueStore)(nil).Close))
}
// Delete mocks base method
func (m *MockKeyValueStore) Delete(arg0 storage.Key) error {
ret := m.ctrl.Call(m, "Delete", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// Delete indicates an expected call of Delete
func (mr *MockKeyValueStoreMockRecorder) Delete(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockKeyValueStore)(nil).Delete), arg0)
}
// Get mocks base method
func (m *MockKeyValueStore) Get(arg0 storage.Key) (storage.Value, error) {
ret := m.ctrl.Call(m, "Get", arg0)
ret0, _ := ret[0].(storage.Value)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Get indicates an expected call of Get
func (mr *MockKeyValueStoreMockRecorder) Get(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockKeyValueStore)(nil).Get), arg0)
}
// GetAll mocks base method
func (m *MockKeyValueStore) GetAll(arg0 storage.Keys) (storage.Values, error) {
ret := m.ctrl.Call(m, "GetAll", arg0)
ret0, _ := ret[0].(storage.Values)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// GetAll indicates an expected call of GetAll
func (mr *MockKeyValueStoreMockRecorder) GetAll(arg0 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockKeyValueStore)(nil).GetAll), arg0)
}
// Iterate mocks base method
func (m *MockKeyValueStore) Iterate(arg0 storage.IterateOptions, arg1 func(storage.Iterator) error) error {
ret := m.ctrl.Call(m, "Iterate", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// Iterate indicates an expected call of Iterate
func (mr *MockKeyValueStoreMockRecorder) Iterate(arg0, arg1 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Iterate", reflect.TypeOf((*MockKeyValueStore)(nil).Iterate), arg0, arg1)
}
// List mocks base method
func (m *MockKeyValueStore) List(arg0 storage.Key, arg1 int) (storage.Keys, error) {
ret := m.ctrl.Call(m, "List", arg0, arg1)
ret0, _ := ret[0].(storage.Keys)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// List indicates an expected call of List
func (mr *MockKeyValueStoreMockRecorder) List(arg0, arg1 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockKeyValueStore)(nil).List), arg0, arg1)
}
// Put mocks base method
func (m *MockKeyValueStore) Put(arg0 storage.Key, arg1 storage.Value) error {
ret := m.ctrl.Call(m, "Put", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
}
// Put indicates an expected call of Put
func (mr *MockKeyValueStoreMockRecorder) Put(arg0, arg1 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Put", reflect.TypeOf((*MockKeyValueStore)(nil).Put), arg0, arg1)
}
// ReverseList mocks base method
func (m *MockKeyValueStore) ReverseList(arg0 storage.Key, arg1 int) (storage.Keys, error) {
ret := m.ctrl.Call(m, "ReverseList", arg0, arg1)
ret0, _ := ret[0].(storage.Keys)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// ReverseList indicates an expected call of ReverseList
func (mr *MockKeyValueStoreMockRecorder) ReverseList(arg0, arg1 interface{}) *gomock.Call {
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReverseList", reflect.TypeOf((*MockKeyValueStore)(nil).ReverseList), arg0, arg1)
}

54
pkg/pointerdb/pointerdb_test.go
View File

@ -13,7 +13,6 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/golang/mock/gomock"
"github.com/golang/protobuf/proto"
"github.com/golang/protobuf/ptypes"
"github.com/google/go-cmp/cmp"
@ -26,16 +25,11 @@ import (
"storj.io/storj/storage/teststore"
)
//go:generate mockgen -destination kvstore_mock_test.go -package pointerdb storj.io/storj/storage KeyValueStore
var (
ctx = context.Background()
)
func TestServicePut(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
for i, tt := range []struct {
apiKey []byte
err error
@ -43,18 +37,18 @@ func TestServicePut(t *testing.T) {
}{
{nil, nil, ""},
{[]byte("wrong key"), nil, status.Errorf(codes.Unauthenticated, "Invalid API credential").Error()},
{nil, errors.New("put error"), status.Errorf(codes.Internal, "put error").Error()},
{nil, errors.New("put error"), status.Errorf(codes.Internal, "internal error").Error()},
} {
errTag := fmt.Sprintf("Test case #%d", i)
db := NewMockKeyValueStore(ctrl)
db := teststore.New()
s := Server{DB: db, logger: zap.NewNop()}
path := "a/b/c"
pr := pb.Pointer{}
if tt.err != nil || tt.errString == "" {
db.EXPECT().Put(storage.Key([]byte(path)), gomock.Any()).Return(tt.err)
if tt.err != nil {
db.ForceError++
}
req := pb.PutRequest{Path: path, Pointer: &pr, APIKey: tt.apiKey}
@ -69,9 +63,6 @@ func TestServicePut(t *testing.T) {
}
func TestServiceGet(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
for i, tt := range []struct {
apiKey []byte
err error
@ -79,20 +70,23 @@ func TestServiceGet(t *testing.T) {
}{
{nil, nil, ""},
{[]byte("wrong key"), nil, status.Errorf(codes.Unauthenticated, "Invalid API credential").Error()},
{nil, errors.New("get error"), status.Errorf(codes.Internal, "get error").Error()},
{nil, errors.New("get error"), status.Errorf(codes.Internal, "internal error").Error()},
} {
errTag := fmt.Sprintf("Test case #%d", i)
db := NewMockKeyValueStore(ctrl)
db := teststore.New()
s := Server{DB: db, logger: zap.NewNop()}
path := "a/b/c"
pr := pb.Pointer{}
prBytes, err := proto.Marshal(&pr)
pr := &pb.Pointer{Size: 123}
prBytes, err := proto.Marshal(pr)
assert.NoError(t, err, errTag)
if tt.err != nil || tt.errString == "" {
db.EXPECT().Get(storage.Key([]byte(path))).Return(prBytes, tt.err)
_ = db.Put(storage.Key(path), storage.Value(prBytes))
if tt.err != nil {
db.ForceError++
}
req := pb.GetRequest{Path: path, APIKey: tt.apiKey}
@ -102,18 +96,15 @@ func TestServiceGet(t *testing.T) {
assert.EqualError(t, err, tt.errString, errTag)
} else {
assert.NoError(t, err, errTag)
respPr := pb.Pointer{}
err := proto.Unmarshal(resp.GetPointer(), &respPr)
respPr := &pb.Pointer{}
err := proto.Unmarshal(resp.GetPointer(), respPr)
assert.NoError(t, err, errTag)
assert.Equal(t, pr, respPr, errTag)
assert.True(t, proto.Equal(pr, respPr), errTag)
}
}
}
func TestServiceDelete(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
for i, tt := range []struct {
apiKey []byte
err error
@ -121,17 +112,18 @@ func TestServiceDelete(t *testing.T) {
}{
{nil, nil, ""},
{[]byte("wrong key"), nil, status.Errorf(codes.Unauthenticated, "Invalid API credential").Error()},
{nil, errors.New("delete error"), status.Errorf(codes.Internal, "delete error").Error()},
{nil, errors.New("delete error"), status.Errorf(codes.Internal, "internal error").Error()},
} {
errTag := fmt.Sprintf("Test case #%d", i)
db := NewMockKeyValueStore(ctrl)
s := Server{DB: db, logger: zap.NewNop()}
path := "a/b/c"
if tt.err != nil || tt.errString == "" {
db.EXPECT().Delete(storage.Key([]byte(path))).Return(tt.err)
db := teststore.New()
_ = db.Put(storage.Key(path), storage.Value("hello"))
s := Server{DB: db, logger: zap.NewNop()}
if tt.err != nil {
db.ForceError++
}
req := pb.DeleteRequest{Path: path, APIKey: tt.apiKey}

54
storage/teststore/store.go
View File

@ -11,14 +11,13 @@ import (
"storj.io/storj/storage"
)
var (
// ErrNotExist is returned when looked item does not exist
ErrNotExist = errors.New("does not exist")
)
var errInternal = errors.New("internal error")
// Client implements in-memory key value store
type Client struct {
Items []storage.ListItem
Items []storage.ListItem
ForceError int
CallCount struct {
Get int
Put int
@ -48,10 +47,22 @@ func (store *Client) indexOf(key storage.Key) (int, bool) {
return i, store.Items[i].Key.Equal(key)
}
func (store *Client) forcedError() bool {
if store.ForceError > 0 {
store.ForceError--
return true
}
return false
}
// Put adds a value to store
func (store *Client) Put(key storage.Key, value storage.Value) error {
store.version++
store.CallCount.Put++
if store.forcedError() {
return errInternal
}
if key.IsZero() {
return storage.ErrEmptyKey
}
@ -77,9 +88,13 @@ func (store *Client) Put(key storage.Key, value storage.Value) error {
func (store *Client) Get(key storage.Key) (storage.Value, error) {
store.CallCount.Get++
if store.forcedError() {
return nil, errors.New("internal error")
}
keyIndex, found := store.indexOf(key)
if !found {
return nil, ErrNotExist
return nil, storage.ErrKeyNotFound.New(key.String())
}
return storage.CloneValue(store.Items[keyIndex].Value), nil
@ -92,11 +107,16 @@ func (store *Client) GetAll(keys storage.Keys) (storage.Values, error) {
return nil, storage.ErrLimitExceeded
}
if store.forcedError() {
return nil, errors.New("internal error")
}
values := storage.Values{}
for _, key := range keys {
keyIndex, found := store.indexOf(key)
if !found {
return nil, ErrNotExist
values = append(values, nil)
continue
}
values = append(values, storage.CloneValue(store.Items[keyIndex].Value))
}
@ -107,9 +127,14 @@ func (store *Client) GetAll(keys storage.Keys) (storage.Values, error) {
func (store *Client) Delete(key storage.Key) error {
store.version++
store.CallCount.Delete++
if store.forcedError() {
return errInternal
}
keyIndex, found := store.indexOf(key)
if !found {
return ErrNotExist
return storage.ErrKeyNotFound.New(key.String())
}
copy(store.Items[keyIndex:], store.Items[keyIndex+1:])
@ -120,24 +145,37 @@ func (store *Client) Delete(key storage.Key) error {
// List lists all keys starting from start and upto limit items
func (store *Client) List(first storage.Key, limit int) (storage.Keys, error) {
store.CallCount.List++
if store.forcedError() {
return nil, errors.New("internal error")
}
return storage.ListKeys(store, first, limit)
}
// ReverseList lists all keys in revers order
func (store *Client) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
store.CallCount.ReverseList++
if store.forcedError() {
return nil, errors.New("internal error")
}
return storage.ReverseListKeys(store, first, limit)
}
// Close closes the store
func (store *Client) Close() error {
store.CallCount.Close++
if store.forcedError() {
return errInternal
}
return nil
}
// Iterate iterates over items based on opts
func (store *Client) Iterate(opts storage.IterateOptions, fn func(storage.Iterator) error) error {
store.CallCount.Iterate++
if store.forcedError() {
return errInternal
}
var cursor advancer
if !opts.Reverse {
cursor = &forward{newCursor(store)}