473 lines
13 KiB
Go
473 lines
13 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package pointerdb_test
|
|
|
|
import (
|
|
"context"
|
|
"crypto/rand"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"errors"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/gogo/protobuf/proto"
|
|
"github.com/golang/protobuf/ptypes"
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/stretchr/testify/assert"
|
|
"go.uber.org/zap"
|
|
"go.uber.org/zap/zaptest"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/credentials"
|
|
"google.golang.org/grpc/peer"
|
|
"google.golang.org/grpc/status"
|
|
|
|
"storj.io/storj/internal/testidentity"
|
|
"storj.io/storj/internal/testplanet"
|
|
"storj.io/storj/pkg/auth"
|
|
"storj.io/storj/pkg/auth/signing"
|
|
"storj.io/storj/pkg/pb"
|
|
"storj.io/storj/pkg/pointerdb"
|
|
"storj.io/storj/pkg/storage/meta"
|
|
"storj.io/storj/pkg/storj"
|
|
"storj.io/storj/satellite/console"
|
|
"storj.io/storj/satellite/satellitedb"
|
|
"storj.io/storj/storage"
|
|
"storj.io/storj/storage/teststore"
|
|
)
|
|
|
|
// mockAPIKeys is mock for api keys store of pointerdb
|
|
type mockAPIKeys struct {
|
|
info console.APIKeyInfo
|
|
err error
|
|
}
|
|
|
|
var (
|
|
identities = testplanet.NewPregeneratedIdentities()
|
|
)
|
|
|
|
// GetByKey return api key info for given key
|
|
func (keys *mockAPIKeys) GetByKey(ctx context.Context, key console.APIKey) (*console.APIKeyInfo, error) {
|
|
return &keys.info, keys.err
|
|
}
|
|
|
|
func TestServicePut(t *testing.T) {
|
|
validAPIKey := console.APIKey{}
|
|
apiKeys := &mockAPIKeys{}
|
|
|
|
for i, tt := range []struct {
|
|
apiKey []byte
|
|
numOfValidPieces int
|
|
numOfInvalidPieces int
|
|
err error
|
|
errString string
|
|
}{
|
|
{[]byte(validAPIKey.String()), 8, 0, nil, ""},
|
|
{[]byte(validAPIKey.String()), 6, 0, nil, ""},
|
|
{[]byte(validAPIKey.String()), 3, 0, nil, "pointerdb error: Number of valid pieces is lower then success threshold: 3 < 6"},
|
|
|
|
{[]byte(validAPIKey.String()), 6, 2, nil, ""},
|
|
{[]byte(validAPIKey.String()), 3, 5, nil, "pointerdb error: Number of valid pieces is lower then success threshold: 3 < 6"},
|
|
|
|
{[]byte("wrong key"), 1, 0, nil, status.Errorf(codes.Unauthenticated, "Invalid API credential").Error()},
|
|
{nil, 8, 0, errors.New("put error"), status.Errorf(codes.Internal, "internal error").Error()},
|
|
} {
|
|
ctx := context.Background()
|
|
ctx = auth.WithAPIKey(ctx, tt.apiKey)
|
|
|
|
errTag := fmt.Sprintf("Test case #%d", i)
|
|
|
|
log := zaptest.NewLogger(t)
|
|
db := teststore.New()
|
|
service := pointerdb.NewService(log, db)
|
|
s := pointerdb.NewServer(log, service, nil, nil, pointerdb.Config{}, nil, apiKeys)
|
|
|
|
path := "a/b/c"
|
|
pointer := makePointer(ctx, t, tt.numOfValidPieces, tt.numOfInvalidPieces)
|
|
|
|
if tt.err != nil {
|
|
db.ForceError++
|
|
}
|
|
|
|
req := pb.PutRequest{Path: path, Pointer: pointer}
|
|
_, err := s.Put(ctx, &req)
|
|
|
|
if err != nil {
|
|
assert.EqualError(t, err, tt.errString, errTag)
|
|
} else {
|
|
assert.NoError(t, err, errTag)
|
|
}
|
|
}
|
|
}
|
|
|
|
func makePointer(ctx context.Context, t *testing.T, numOfValidPieces, numOfInvalidPieces int) *pb.Pointer {
|
|
pieces := make([]*pb.RemotePiece, numOfValidPieces+numOfInvalidPieces)
|
|
for i := 0; i < numOfValidPieces; i++ {
|
|
identity, err := identities.NewIdentity()
|
|
assert.NoError(t, err)
|
|
pieces[i] = &pb.RemotePiece{
|
|
PieceNum: int32(i),
|
|
NodeId: identity.ID,
|
|
Hash: &pb.PieceHash{Hash: make([]byte, 32)},
|
|
}
|
|
|
|
_, err = rand.Read(pieces[i].Hash.Hash)
|
|
assert.NoError(t, err)
|
|
signer := signing.SignerFromFullIdentity(identity)
|
|
pieces[i].Hash, err = signing.SignPieceHash(signer, pieces[i].Hash)
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
// public key did not match expected signer
|
|
for i := numOfValidPieces; i < len(pieces); i++ {
|
|
identity, err := identities.NewIdentity()
|
|
assert.NoError(t, err)
|
|
pieces[i] = &pb.RemotePiece{
|
|
PieceNum: int32(i),
|
|
NodeId: storj.NodeID{byte(i)},
|
|
Hash: &pb.PieceHash{Hash: make([]byte, 32)},
|
|
}
|
|
|
|
_, err = rand.Read(pieces[i].Hash.Hash)
|
|
assert.NoError(t, err)
|
|
signer := signing.SignerFromFullIdentity(identity)
|
|
pieces[i].Hash, err = signing.SignPieceHash(signer, pieces[i].Hash)
|
|
assert.NoError(t, err)
|
|
}
|
|
|
|
pointer := &pb.Pointer{
|
|
Type: pb.Pointer_REMOTE,
|
|
Remote: &pb.RemoteSegment{
|
|
Redundancy: &pb.RedundancyScheme{
|
|
MinReq: 2,
|
|
RepairThreshold: 4,
|
|
SuccessThreshold: 6,
|
|
Total: 8,
|
|
},
|
|
RemotePieces: pieces,
|
|
},
|
|
}
|
|
|
|
return pointer
|
|
}
|
|
|
|
func TestServiceGet(t *testing.T) {
|
|
ctx := context.Background()
|
|
ca, err := testidentity.NewTestCA(ctx)
|
|
assert.NoError(t, err)
|
|
identity, err := ca.NewIdentity()
|
|
assert.NoError(t, err)
|
|
|
|
peerCertificates := make([]*x509.Certificate, 2)
|
|
peerCertificates[0] = identity.Leaf
|
|
peerCertificates[1] = identity.CA
|
|
|
|
info := credentials.TLSInfo{State: tls.ConnectionState{PeerCertificates: peerCertificates}}
|
|
|
|
validAPIKey := console.APIKey{}
|
|
apiKeys := &mockAPIKeys{}
|
|
// creating in-memory db and opening connection
|
|
satdb, err := satellitedb.NewInMemory(zaptest.NewLogger(t))
|
|
defer func() {
|
|
err = satdb.Close()
|
|
assert.NoError(t, err)
|
|
}()
|
|
err = satdb.CreateTables()
|
|
assert.NoError(t, err)
|
|
|
|
for i, tt := range []struct {
|
|
apiKey []byte
|
|
err error
|
|
errString string
|
|
}{
|
|
{[]byte(validAPIKey.String()), nil, ""},
|
|
{[]byte("wrong key"), nil, status.Errorf(codes.Unauthenticated, "Invalid API credential").Error()},
|
|
{nil, errors.New("get error"), status.Errorf(codes.Internal, "internal error").Error()},
|
|
} {
|
|
ctx = auth.WithAPIKey(ctx, tt.apiKey)
|
|
ctx = peer.NewContext(ctx, &peer.Peer{AuthInfo: info})
|
|
|
|
errTag := fmt.Sprintf("Test case #%d", i)
|
|
|
|
db := teststore.New()
|
|
service := pointerdb.NewService(zap.NewNop(), db)
|
|
allocation := pointerdb.NewAllocationSigner(identity, 45, satdb.CertDB())
|
|
|
|
s := pointerdb.NewServer(zap.NewNop(), service, allocation, nil, pointerdb.Config{}, identity, apiKeys)
|
|
|
|
path := "a/b/c"
|
|
|
|
pr := &pb.Pointer{SegmentSize: 123}
|
|
prBytes, err := proto.Marshal(pr)
|
|
assert.NoError(t, err, errTag)
|
|
|
|
_ = db.Put(storage.Key(storj.JoinPaths(apiKeys.info.ProjectID.String(), path)), storage.Value(prBytes))
|
|
|
|
if tt.err != nil {
|
|
db.ForceError++
|
|
}
|
|
|
|
req := pb.GetRequest{Path: path}
|
|
resp, err := s.Get(ctx, &req)
|
|
|
|
if err != nil {
|
|
assert.EqualError(t, err, tt.errString, errTag)
|
|
} else {
|
|
assert.NoError(t, err, errTag)
|
|
assert.NoError(t, err, errTag)
|
|
assert.True(t, pb.Equal(pr, resp.Pointer), errTag)
|
|
|
|
assert.NotNil(t, resp.GetPba())
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestServiceDelete(t *testing.T) {
|
|
validAPIKey := console.APIKey{}
|
|
apiKeys := &mockAPIKeys{}
|
|
|
|
for i, tt := range []struct {
|
|
apiKey []byte
|
|
err error
|
|
errString string
|
|
}{
|
|
{[]byte(validAPIKey.String()), nil, ""},
|
|
{[]byte("wrong key"), nil, status.Errorf(codes.Unauthenticated, "Invalid API credential").Error()},
|
|
{nil, errors.New("delete error"), status.Errorf(codes.Internal, "internal error").Error()},
|
|
} {
|
|
ctx := context.Background()
|
|
ctx = auth.WithAPIKey(ctx, tt.apiKey)
|
|
|
|
errTag := fmt.Sprintf("Test case #%d", i)
|
|
|
|
path := "a/b/c"
|
|
|
|
db := teststore.New()
|
|
_ = db.Put(storage.Key(storj.JoinPaths(apiKeys.info.ProjectID.String(), path)), storage.Value("hello"))
|
|
service := pointerdb.NewService(zap.NewNop(), db)
|
|
s := pointerdb.NewServer(zap.NewNop(), service, nil, nil, pointerdb.Config{}, nil, apiKeys)
|
|
|
|
if tt.err != nil {
|
|
db.ForceError++
|
|
}
|
|
|
|
req := pb.DeleteRequest{Path: path}
|
|
_, err := s.Delete(ctx, &req)
|
|
|
|
if err != nil {
|
|
assert.EqualError(t, err, tt.errString, errTag)
|
|
} else {
|
|
assert.NoError(t, err, errTag)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestServiceList(t *testing.T) {
|
|
validAPIKey := console.APIKey{}
|
|
apiKeys := &mockAPIKeys{}
|
|
|
|
db := teststore.New()
|
|
service := pointerdb.NewService(zap.NewNop(), db)
|
|
server := pointerdb.NewServer(zap.NewNop(), service, nil, nil, pointerdb.Config{}, nil, apiKeys)
|
|
|
|
pointer := &pb.Pointer{}
|
|
pointer.CreationDate = ptypes.TimestampNow()
|
|
|
|
pointerBytes, err := proto.Marshal(pointer)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
pointerValue := storage.Value(pointerBytes)
|
|
|
|
items := []storage.ListItem{
|
|
{Key: storage.Key("sample.😶"), Value: pointerValue},
|
|
{Key: storage.Key("müsic"), Value: pointerValue},
|
|
{Key: storage.Key("müsic/söng1.mp3"), Value: pointerValue},
|
|
{Key: storage.Key("müsic/söng2.mp3"), Value: pointerValue},
|
|
{Key: storage.Key("müsic/album/söng3.mp3"), Value: pointerValue},
|
|
{Key: storage.Key("müsic/söng4.mp3"), Value: pointerValue},
|
|
{Key: storage.Key("ビデオ/movie.mkv"), Value: pointerValue},
|
|
}
|
|
|
|
for i := range items {
|
|
items[i].Key = storage.Key(storj.JoinPaths(apiKeys.info.ProjectID.String(), items[i].Key.String()))
|
|
}
|
|
|
|
err = storage.PutAll(db, items...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
type Test struct {
|
|
APIKey string
|
|
Request pb.ListRequest
|
|
Expected *pb.ListResponse
|
|
Error func(i int, err error)
|
|
}
|
|
|
|
// TODO: ZZZ temporarily disabled until endpoint and service split
|
|
// errorWithCode := func(code codes.Code) func(i int, err error) {
|
|
// t.Helper()
|
|
// return func(i int, err error) {
|
|
// t.Helper()
|
|
// if status.Code(err) != code {
|
|
// t.Fatalf("%d: should fail with %v, got: %v", i, code, err)
|
|
// }
|
|
// }
|
|
// }
|
|
|
|
tests := []Test{
|
|
{
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Recursive: true},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "müsic"},
|
|
{Path: "müsic/album/söng3.mp3"},
|
|
{Path: "müsic/söng1.mp3"},
|
|
{Path: "müsic/söng2.mp3"},
|
|
{Path: "müsic/söng4.mp3"},
|
|
{Path: "sample.😶"},
|
|
{Path: "ビデオ/movie.mkv"},
|
|
},
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Recursive: true, MetaFlags: meta.All},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "müsic", Pointer: pointer},
|
|
{Path: "müsic/album/söng3.mp3", Pointer: pointer},
|
|
{Path: "müsic/söng1.mp3", Pointer: pointer},
|
|
{Path: "müsic/söng2.mp3", Pointer: pointer},
|
|
{Path: "müsic/söng4.mp3", Pointer: pointer},
|
|
{Path: "sample.😶", Pointer: pointer},
|
|
{Path: "ビデオ/movie.mkv", Pointer: pointer},
|
|
},
|
|
},
|
|
},
|
|
// { // TODO: ZZZ temporarily disabled until endpoint and service split
|
|
// APIKey: "wrong key",
|
|
// Request: pb.ListRequest{Recursive: true, MetaFlags: meta.All}, //, APIKey: []byte("wrong key")},
|
|
// Error: errorWithCode(codes.Unauthenticated),
|
|
// },
|
|
{
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Recursive: true, Limit: 3},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "müsic"},
|
|
{Path: "müsic/album/söng3.mp3"},
|
|
{Path: "müsic/söng1.mp3"},
|
|
},
|
|
More: true,
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{MetaFlags: meta.All},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "müsic", Pointer: pointer},
|
|
{Path: "müsic/", IsPrefix: true},
|
|
{Path: "sample.😶", Pointer: pointer},
|
|
{Path: "ビデオ/", IsPrefix: true},
|
|
},
|
|
More: false,
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{EndBefore: "ビデオ"},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "müsic"},
|
|
{Path: "müsic/", IsPrefix: true},
|
|
{Path: "sample.😶"},
|
|
},
|
|
More: false,
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Recursive: true, Prefix: "müsic/"},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "album/söng3.mp3"},
|
|
{Path: "söng1.mp3"},
|
|
{Path: "söng2.mp3"},
|
|
{Path: "söng4.mp3"},
|
|
},
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Recursive: true, Prefix: "müsic/", StartAfter: "album/söng3.mp3"},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "söng1.mp3"},
|
|
{Path: "söng2.mp3"},
|
|
{Path: "söng4.mp3"},
|
|
},
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Prefix: "müsic/"},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "album/", IsPrefix: true},
|
|
{Path: "söng1.mp3"},
|
|
{Path: "söng2.mp3"},
|
|
{Path: "söng4.mp3"},
|
|
},
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Prefix: "müsic/", StartAfter: "söng1.mp3"},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "söng2.mp3"},
|
|
{Path: "söng4.mp3"},
|
|
},
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Prefix: "müsic/", EndBefore: "söng4.mp3"},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
{Path: "album/", IsPrefix: true},
|
|
{Path: "söng1.mp3"},
|
|
{Path: "söng2.mp3"},
|
|
},
|
|
},
|
|
}, {
|
|
APIKey: validAPIKey.String(),
|
|
Request: pb.ListRequest{Prefix: "müs", Recursive: true, EndBefore: "ic/söng4.mp3", Limit: 1},
|
|
Expected: &pb.ListResponse{
|
|
Items: []*pb.ListResponse_Item{
|
|
// {Path: "ic/söng2.mp3"},
|
|
},
|
|
// More: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
// TODO:
|
|
// pb.ListRequest{Prefix: "müsic/", StartAfter: "söng1.mp3", EndBefore: "söng4.mp3"},
|
|
// failing database
|
|
for i, test := range tests {
|
|
ctx := context.Background()
|
|
ctx = auth.WithAPIKey(ctx, []byte(test.APIKey))
|
|
|
|
resp, err := server.List(ctx, &test.Request)
|
|
if test.Error == nil {
|
|
if err != nil {
|
|
t.Fatalf("%d: failed %v", i, err)
|
|
}
|
|
} else {
|
|
test.Error(i, err)
|
|
}
|
|
|
|
if diff := cmp.Diff(test.Expected, resp, cmp.Comparer(pb.Equal)); diff != "" {
|
|
t.Errorf("%d: (-want +got) %v\n%s", i, test.Request.String(), diff)
|
|
}
|
|
}
|
|
}
|