99640225fd
The old paths.Path type is now replaced with the new storj.Path.
storj.Path is simply an alias to the built-in string type. As such it can be used just as any string, which simplifies a lot working with paths. No more conversions paths.New and path.String().
As an alias storj.Path does not define any methods. However, any functions applying to strings (like those from the strings package) gracefully apply to storj.Path too. In addition we have a few more functions defined:
storj.SplitPath
storj.JoinPaths
encryption.EncryptPath
encryption.DecryptPath
encryption.DerivePathKey
encryption.DeriveContentKey
All code in master is migrated to the new storj.Path type.
The Path example is also updated and is good for reference: /pkg/encryption/examples_test.go
This PR also resolve a nonce misuse issue in path encryption: https://storjlabs.atlassian.net/browse/V3-545
460 lines
12 KiB
Go
460 lines
12 KiB
Go
// Copyright (C) 2018 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package segments
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/golang/mock/gomock"
|
|
"github.com/golang/protobuf/ptypes"
|
|
"github.com/stretchr/testify/assert"
|
|
|
|
"storj.io/storj/pkg/eestream"
|
|
mock_eestream "storj.io/storj/pkg/eestream/mocks"
|
|
mock_overlay "storj.io/storj/pkg/overlay/mocks"
|
|
"storj.io/storj/pkg/pb"
|
|
pdb "storj.io/storj/pkg/pointerdb/pdbclient"
|
|
mock_pointerdb "storj.io/storj/pkg/pointerdb/pdbclient/mocks"
|
|
mock_ecclient "storj.io/storj/pkg/storage/ec/mocks"
|
|
"storj.io/storj/pkg/storage/meta"
|
|
"storj.io/storj/pkg/storj"
|
|
)
|
|
|
|
var (
|
|
ctx = context.Background()
|
|
)
|
|
|
|
func TestNewSegmentStore(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mock_eestream.NewMockErasureScheme(ctrl),
|
|
}
|
|
|
|
ss := NewSegmentStore(mockOC, mockEC, mockPDB, rs, 10)
|
|
assert.NotNil(t, ss)
|
|
}
|
|
|
|
func TestSegmentStoreMeta(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mock_eestream.NewMockErasureScheme(ctrl),
|
|
}
|
|
|
|
ss := segmentStore{mockOC, mockEC, mockPDB, rs, 10}
|
|
assert.NotNil(t, ss)
|
|
|
|
var mExp time.Time
|
|
pExp, err := ptypes.TimestampProto(mExp)
|
|
assert.NoError(t, err)
|
|
|
|
for _, tt := range []struct {
|
|
pathInput string
|
|
returnPointer *pb.Pointer
|
|
returnMeta Meta
|
|
}{
|
|
{"path/1/2/3", &pb.Pointer{CreationDate: pExp, ExpirationDate: pExp}, Meta{Modified: mExp, Expiration: mExp}},
|
|
} {
|
|
calls := []*gomock.Call{
|
|
mockPDB.EXPECT().Get(
|
|
gomock.Any(), gomock.Any(),
|
|
).Return(tt.returnPointer, nil),
|
|
}
|
|
gomock.InOrder(calls...)
|
|
|
|
m, err := ss.Meta(ctx, tt.pathInput)
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, m, tt.returnMeta)
|
|
}
|
|
}
|
|
|
|
func TestSegmentStorePutRemote(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
for _, tt := range []struct {
|
|
name string
|
|
pathInput string
|
|
mdInput []byte
|
|
thresholdSize int
|
|
expiration time.Time
|
|
readerContent string
|
|
}{
|
|
{"test remote put", "path/1", []byte("abcdefghijklmnopqrstuvwxyz"), 2, time.Unix(0, 0).UTC(), "readerreaderreader"},
|
|
} {
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
mockES := mock_eestream.NewMockErasureScheme(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mockES,
|
|
}
|
|
|
|
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
|
|
assert.NotNil(t, ss)
|
|
|
|
calls := []*gomock.Call{
|
|
mockES.EXPECT().TotalCount().Return(1),
|
|
mockOC.EXPECT().Choose(
|
|
gomock.Any(), gomock.Any(),
|
|
).Return([]*pb.Node{
|
|
{Id: "im-a-node"},
|
|
}, nil),
|
|
mockPDB.EXPECT().SignedMessage(),
|
|
mockEC.EXPECT().Put(
|
|
gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(),
|
|
),
|
|
mockES.EXPECT().RequiredCount().Return(1),
|
|
mockES.EXPECT().TotalCount().Return(1),
|
|
mockES.EXPECT().ErasureShareSize().Return(1),
|
|
mockPDB.EXPECT().Put(
|
|
gomock.Any(), gomock.Any(), gomock.Any(),
|
|
).Return(nil),
|
|
mockPDB.EXPECT().Get(
|
|
gomock.Any(), gomock.Any(),
|
|
),
|
|
}
|
|
gomock.InOrder(calls...)
|
|
|
|
_, err := ss.Put(ctx, strings.NewReader(tt.readerContent), tt.expiration, func() (storj.Path, []byte, error) {
|
|
return tt.pathInput, tt.mdInput, nil
|
|
})
|
|
assert.NoError(t, err, tt.name)
|
|
}
|
|
}
|
|
|
|
func TestSegmentStorePutInline(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
for _, tt := range []struct {
|
|
name string
|
|
pathInput string
|
|
mdInput []byte
|
|
thresholdSize int
|
|
expiration time.Time
|
|
readerContent string
|
|
}{
|
|
{"test inline put", "path/1", []byte("111"), 1000, time.Unix(0, 0).UTC(), "readerreaderreader"},
|
|
} {
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
mockES := mock_eestream.NewMockErasureScheme(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mockES,
|
|
}
|
|
|
|
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
|
|
assert.NotNil(t, ss)
|
|
|
|
calls := []*gomock.Call{
|
|
mockPDB.EXPECT().Put(
|
|
gomock.Any(), gomock.Any(), gomock.Any(),
|
|
).Return(nil),
|
|
mockPDB.EXPECT().Get(
|
|
gomock.Any(), gomock.Any(),
|
|
),
|
|
}
|
|
gomock.InOrder(calls...)
|
|
|
|
_, err := ss.Put(ctx, strings.NewReader(tt.readerContent), tt.expiration, func() (storj.Path, []byte, error) {
|
|
return tt.pathInput, tt.mdInput, nil
|
|
})
|
|
assert.NoError(t, err, tt.name)
|
|
}
|
|
}
|
|
|
|
func TestSegmentStoreGetInline(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
ti := time.Unix(0, 0).UTC()
|
|
someTime, err := ptypes.TimestampProto(ti)
|
|
assert.NoError(t, err)
|
|
|
|
for _, tt := range []struct {
|
|
pathInput string
|
|
thresholdSize int
|
|
pointerType pb.Pointer_DataType
|
|
inlineContent []byte
|
|
size int64
|
|
metadata []byte
|
|
}{
|
|
{"path/1/2/3", 10, pb.Pointer_INLINE, []byte("000"), int64(3), []byte("metadata")},
|
|
} {
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
mockES := mock_eestream.NewMockErasureScheme(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mockES,
|
|
}
|
|
|
|
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
|
|
assert.NotNil(t, ss)
|
|
|
|
calls := []*gomock.Call{
|
|
mockPDB.EXPECT().Get(
|
|
gomock.Any(), gomock.Any(),
|
|
).Return(&pb.Pointer{
|
|
Type: tt.pointerType,
|
|
InlineSegment: tt.inlineContent,
|
|
CreationDate: someTime,
|
|
ExpirationDate: someTime,
|
|
Size: tt.size,
|
|
Metadata: tt.metadata,
|
|
}, nil),
|
|
}
|
|
gomock.InOrder(calls...)
|
|
|
|
_, _, err := ss.Get(ctx, tt.pathInput)
|
|
assert.NoError(t, err)
|
|
}
|
|
}
|
|
|
|
func TestSegmentStoreGetRemote(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
ti := time.Unix(0, 0).UTC()
|
|
someTime, err := ptypes.TimestampProto(ti)
|
|
assert.NoError(t, err)
|
|
|
|
for _, tt := range []struct {
|
|
pathInput string
|
|
thresholdSize int
|
|
pointerType pb.Pointer_DataType
|
|
size int64
|
|
metadata []byte
|
|
}{
|
|
{"path/1/2/3", 10, pb.Pointer_REMOTE, int64(3), []byte("metadata")},
|
|
} {
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
mockES := mock_eestream.NewMockErasureScheme(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mockES,
|
|
}
|
|
|
|
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
|
|
assert.NotNil(t, ss)
|
|
|
|
calls := []*gomock.Call{
|
|
mockPDB.EXPECT().Get(
|
|
gomock.Any(), gomock.Any(),
|
|
).Return(&pb.Pointer{
|
|
Type: tt.pointerType,
|
|
Remote: &pb.RemoteSegment{
|
|
Redundancy: &pb.RedundancyScheme{
|
|
Type: pb.RedundancyScheme_RS,
|
|
MinReq: 1,
|
|
Total: 2,
|
|
RepairThreshold: 1,
|
|
SuccessThreshold: 2,
|
|
},
|
|
PieceId: "here's my piece id",
|
|
RemotePieces: []*pb.RemotePiece{},
|
|
},
|
|
CreationDate: someTime,
|
|
ExpirationDate: someTime,
|
|
Size: tt.size,
|
|
Metadata: tt.metadata,
|
|
}, nil),
|
|
mockOC.EXPECT().BulkLookup(gomock.Any(), gomock.Any()),
|
|
mockPDB.EXPECT().SignedMessage(),
|
|
mockEC.EXPECT().Get(
|
|
gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(),
|
|
),
|
|
}
|
|
gomock.InOrder(calls...)
|
|
|
|
_, _, err := ss.Get(ctx, tt.pathInput)
|
|
assert.NoError(t, err)
|
|
}
|
|
}
|
|
|
|
func TestSegmentStoreDeleteInline(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
ti := time.Unix(0, 0).UTC()
|
|
someTime, err := ptypes.TimestampProto(ti)
|
|
assert.NoError(t, err)
|
|
|
|
for _, tt := range []struct {
|
|
pathInput string
|
|
thresholdSize int
|
|
pointerType pb.Pointer_DataType
|
|
inlineContent []byte
|
|
size int64
|
|
metadata []byte
|
|
}{
|
|
{"path/1/2/3", 10, pb.Pointer_INLINE, []byte("000"), int64(3), []byte("metadata")},
|
|
} {
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
mockES := mock_eestream.NewMockErasureScheme(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mockES,
|
|
}
|
|
|
|
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
|
|
assert.NotNil(t, ss)
|
|
|
|
calls := []*gomock.Call{
|
|
mockPDB.EXPECT().Get(
|
|
gomock.Any(), gomock.Any(),
|
|
).Return(&pb.Pointer{
|
|
Type: tt.pointerType,
|
|
InlineSegment: tt.inlineContent,
|
|
CreationDate: someTime,
|
|
ExpirationDate: someTime,
|
|
Size: tt.size,
|
|
Metadata: tt.metadata,
|
|
}, nil),
|
|
mockPDB.EXPECT().Delete(
|
|
gomock.Any(), gomock.Any(),
|
|
),
|
|
}
|
|
gomock.InOrder(calls...)
|
|
|
|
err := ss.Delete(ctx, tt.pathInput)
|
|
assert.NoError(t, err)
|
|
}
|
|
}
|
|
|
|
func TestSegmentStoreDeleteRemote(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
ti := time.Unix(0, 0).UTC()
|
|
someTime, err := ptypes.TimestampProto(ti)
|
|
assert.NoError(t, err)
|
|
|
|
for _, tt := range []struct {
|
|
pathInput string
|
|
thresholdSize int
|
|
pointerType pb.Pointer_DataType
|
|
size int64
|
|
metadata []byte
|
|
}{
|
|
{"path/1/2/3", 10, pb.Pointer_REMOTE, int64(3), []byte("metadata")},
|
|
} {
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
mockES := mock_eestream.NewMockErasureScheme(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mockES,
|
|
}
|
|
|
|
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
|
|
assert.NotNil(t, ss)
|
|
|
|
calls := []*gomock.Call{
|
|
mockPDB.EXPECT().Get(
|
|
gomock.Any(), gomock.Any(),
|
|
).Return(&pb.Pointer{
|
|
Type: tt.pointerType,
|
|
Remote: &pb.RemoteSegment{
|
|
Redundancy: &pb.RedundancyScheme{
|
|
Type: pb.RedundancyScheme_RS,
|
|
MinReq: 1,
|
|
Total: 2,
|
|
RepairThreshold: 1,
|
|
SuccessThreshold: 2,
|
|
},
|
|
PieceId: "here's my piece id",
|
|
RemotePieces: []*pb.RemotePiece{},
|
|
},
|
|
CreationDate: someTime,
|
|
ExpirationDate: someTime,
|
|
Size: tt.size,
|
|
Metadata: tt.metadata,
|
|
}, nil),
|
|
mockOC.EXPECT().BulkLookup(gomock.Any(), gomock.Any()),
|
|
mockPDB.EXPECT().SignedMessage(),
|
|
mockEC.EXPECT().Delete(
|
|
gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(),
|
|
),
|
|
mockPDB.EXPECT().Delete(
|
|
gomock.Any(), gomock.Any(),
|
|
),
|
|
}
|
|
gomock.InOrder(calls...)
|
|
|
|
err := ss.Delete(ctx, tt.pathInput)
|
|
assert.NoError(t, err)
|
|
}
|
|
}
|
|
|
|
func TestSegmentStoreList(t *testing.T) {
|
|
ctrl := gomock.NewController(t)
|
|
defer ctrl.Finish()
|
|
|
|
for _, tt := range []struct {
|
|
prefix string
|
|
startAfter string
|
|
thresholdSize int
|
|
itemPath string
|
|
inlineContent []byte
|
|
metadata []byte
|
|
}{
|
|
{"bucket1", "s0/path/1", 10, "s0/path/1", []byte("inline"), []byte("metadata")},
|
|
} {
|
|
mockOC := mock_overlay.NewMockClient(ctrl)
|
|
mockEC := mock_ecclient.NewMockClient(ctrl)
|
|
mockPDB := mock_pointerdb.NewMockClient(ctrl)
|
|
mockES := mock_eestream.NewMockErasureScheme(ctrl)
|
|
rs := eestream.RedundancyStrategy{
|
|
ErasureScheme: mockES,
|
|
}
|
|
|
|
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
|
|
assert.NotNil(t, ss)
|
|
|
|
ti := time.Unix(0, 0).UTC()
|
|
someTime, err := ptypes.TimestampProto(ti)
|
|
assert.NoError(t, err)
|
|
|
|
calls := []*gomock.Call{
|
|
mockPDB.EXPECT().List(
|
|
gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(),
|
|
gomock.Any(), gomock.Any(), gomock.Any(),
|
|
).Return([]pdb.ListItem{
|
|
{
|
|
Path: tt.itemPath,
|
|
Pointer: &pb.Pointer{
|
|
Type: pb.Pointer_INLINE,
|
|
InlineSegment: tt.inlineContent,
|
|
CreationDate: someTime,
|
|
ExpirationDate: someTime,
|
|
Size: int64(4),
|
|
Metadata: tt.metadata,
|
|
},
|
|
},
|
|
}, true, nil),
|
|
}
|
|
gomock.InOrder(calls...)
|
|
|
|
_, _, err = ss.List(ctx, tt.prefix, tt.startAfter, "", false, 10, meta.Modified)
|
|
assert.NoError(t, err)
|
|
}
|
|
}
|