285 lines
8.9 KiB
Go
285 lines
8.9 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package segments_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"fmt"
|
|
io "io"
|
|
"io/ioutil"
|
|
"strconv"
|
|
"testing"
|
|
time "time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"github.com/vivint/infectious"
|
|
|
|
"storj.io/storj/internal/memory"
|
|
"storj.io/storj/internal/testcontext"
|
|
"storj.io/storj/internal/testplanet"
|
|
"storj.io/storj/pkg/eestream"
|
|
"storj.io/storj/pkg/macaroon"
|
|
"storj.io/storj/pkg/pb"
|
|
ecclient "storj.io/storj/pkg/storage/ec"
|
|
"storj.io/storj/pkg/storage/meta"
|
|
"storj.io/storj/pkg/storage/segments"
|
|
storj "storj.io/storj/pkg/storj"
|
|
"storj.io/storj/satellite/console"
|
|
"storj.io/storj/storage"
|
|
)
|
|
|
|
func TestSegmentStoreMeta(t *testing.T) {
|
|
for i, tt := range []struct {
|
|
path string
|
|
data []byte
|
|
metadata []byte
|
|
expiration time.Time
|
|
err string
|
|
}{
|
|
{"l/path/1/2/3", []byte("content"), []byte("metadata"), time.Now().UTC().Add(time.Hour * 12), ""},
|
|
{"l/not-exists-path/1/2/3", []byte{}, []byte{}, time.Now(), "key not found"},
|
|
{"", []byte{}, []byte{}, time.Now(), "invalid segment component"},
|
|
} {
|
|
test := tt
|
|
t.Run("#"+strconv.Itoa(i), func(t *testing.T) {
|
|
runTest(t, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet, segmentStore segments.Store) {
|
|
expectedSize := int64(len(test.data))
|
|
reader := bytes.NewReader(test.data)
|
|
|
|
beforeModified := time.Now()
|
|
if test.err == "" {
|
|
meta, err := segmentStore.Put(ctx, reader, test.expiration, func() (storj.Path, []byte, error) {
|
|
return test.path, test.metadata, nil
|
|
})
|
|
require.NoError(t, err)
|
|
assert.Equal(t, expectedSize, meta.Size)
|
|
assert.Equal(t, test.metadata, meta.Data)
|
|
assert.Equal(t, test.expiration, meta.Expiration)
|
|
assert.True(t, meta.Modified.After(beforeModified))
|
|
}
|
|
|
|
meta, err := segmentStore.Meta(ctx, test.path)
|
|
if test.err == "" {
|
|
require.NoError(t, err)
|
|
assert.Equal(t, expectedSize, meta.Size)
|
|
assert.Equal(t, test.metadata, meta.Data)
|
|
assert.Equal(t, test.expiration, meta.Expiration)
|
|
assert.True(t, meta.Modified.After(beforeModified))
|
|
} else {
|
|
require.Contains(t, err.Error(), test.err)
|
|
}
|
|
})
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSegmentStorePutGet(t *testing.T) {
|
|
for _, tt := range []struct {
|
|
name string
|
|
path string
|
|
metadata []byte
|
|
expiration time.Time
|
|
content []byte
|
|
}{
|
|
{"test inline put/get", "l/path/1", []byte("metadata-intline"), time.Time{}, createTestData(t, 2*memory.KiB.Int64())},
|
|
{"test remote put/get", "s0/test-bucket/mypath/1", []byte("metadata-remote"), time.Time{}, createTestData(t, 100*memory.KiB.Int64())},
|
|
} {
|
|
test := tt
|
|
runTest(t, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet, segmentStore segments.Store) {
|
|
metadata, err := segmentStore.Put(ctx, bytes.NewReader(test.content), test.expiration, func() (storj.Path, []byte, error) {
|
|
return test.path, test.metadata, nil
|
|
})
|
|
require.NoError(t, err, test.name)
|
|
require.Equal(t, test.metadata, metadata.Data)
|
|
|
|
rr, metadata, err := segmentStore.Get(ctx, test.path)
|
|
require.NoError(t, err, test.name)
|
|
require.Equal(t, test.metadata, metadata.Data)
|
|
|
|
reader, err := rr.Range(ctx, 0, rr.Size())
|
|
require.NoError(t, err, test.name)
|
|
content, err := ioutil.ReadAll(reader)
|
|
require.NoError(t, err, test.name)
|
|
require.Equal(t, test.content, content)
|
|
|
|
require.NoError(t, reader.Close(), test.name)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSegmentStoreDelete(t *testing.T) {
|
|
for _, tt := range []struct {
|
|
name string
|
|
path string
|
|
metadata []byte
|
|
expiration time.Time
|
|
content []byte
|
|
}{
|
|
{"test inline delete", "l/path/1", []byte("metadata"), time.Time{}, createTestData(t, 2*memory.KiB.Int64())},
|
|
{"test remote delete", "s0/test-bucket/mypath/1", []byte("metadata"), time.Time{}, createTestData(t, 100*memory.KiB.Int64())},
|
|
} {
|
|
test := tt
|
|
runTest(t, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet, segmentStore segments.Store) {
|
|
_, err := segmentStore.Put(ctx, bytes.NewReader(test.content), test.expiration, func() (storj.Path, []byte, error) {
|
|
return test.path, test.metadata, nil
|
|
})
|
|
require.NoError(t, err, test.name)
|
|
|
|
_, _, err = segmentStore.Get(ctx, test.path)
|
|
require.NoError(t, err, test.name)
|
|
|
|
// delete existing
|
|
err = segmentStore.Delete(ctx, test.path)
|
|
require.NoError(t, err, test.name)
|
|
|
|
_, _, err = segmentStore.Get(ctx, test.path)
|
|
require.Error(t, err, test.name)
|
|
require.True(t, storage.ErrKeyNotFound.Has(err))
|
|
|
|
// delete non existing
|
|
err = segmentStore.Delete(ctx, test.path)
|
|
require.Error(t, err, test.name)
|
|
require.True(t, storage.ErrKeyNotFound.Has(err))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSegmentStoreList(t *testing.T) {
|
|
runTest(t, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet, segmentStore segments.Store) {
|
|
expiration := time.Now().Add(24 * time.Hour * 10)
|
|
|
|
segments := []struct {
|
|
path string
|
|
content []byte
|
|
}{
|
|
{"l/aaaa/afile1", []byte("content")},
|
|
{"l/aaaa/bfile2", []byte("content")},
|
|
{"l/bbbb/afile1", []byte("content")},
|
|
{"l/bbbb/bfile2", []byte("content")},
|
|
{"l/bbbb/bfolder/file1", []byte("content")},
|
|
}
|
|
for _, seg := range segments {
|
|
segment := seg
|
|
_, err := segmentStore.Put(ctx, bytes.NewReader(segment.content), expiration, func() (storj.Path, []byte, error) {
|
|
return segment.path, []byte{}, nil
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
// should list all
|
|
items, more, err := segmentStore.List(ctx, "l", "", "", true, 10, meta.None)
|
|
require.NoError(t, err)
|
|
require.False(t, more)
|
|
require.Equal(t, len(segments), len(items))
|
|
|
|
// should list first two and more = true
|
|
items, more, err = segmentStore.List(ctx, "l", "", "", true, 2, meta.None)
|
|
require.NoError(t, err)
|
|
require.True(t, more)
|
|
require.Equal(t, 2, len(items))
|
|
|
|
// should list only prefixes
|
|
items, more, err = segmentStore.List(ctx, "l", "", "", false, 10, meta.None)
|
|
require.NoError(t, err)
|
|
require.False(t, more)
|
|
require.Equal(t, 2, len(items))
|
|
|
|
// should list only BBBB bucket
|
|
items, more, err = segmentStore.List(ctx, "l/bbbb", "", "", false, 10, meta.None)
|
|
require.NoError(t, err)
|
|
require.False(t, more)
|
|
require.Equal(t, 3, len(items))
|
|
|
|
// should list only BBBB bucket after afile1
|
|
items, more, err = segmentStore.List(ctx, "l/bbbb", "afile1", "", false, 10, meta.None)
|
|
require.NoError(t, err)
|
|
require.False(t, more)
|
|
require.Equal(t, 2, len(items))
|
|
|
|
// should list nothing
|
|
items, more, err = segmentStore.List(ctx, "l/cccc", "", "", true, 10, meta.None)
|
|
require.NoError(t, err)
|
|
require.False(t, more)
|
|
require.Equal(t, 0, len(items))
|
|
})
|
|
}
|
|
|
|
func TestCalcNeededNodes(t *testing.T) {
|
|
for i, tt := range []struct {
|
|
k, m, o, n int32
|
|
needed int32
|
|
}{
|
|
{k: 0, m: 0, o: 0, n: 0, needed: 0},
|
|
{k: 1, m: 1, o: 1, n: 1, needed: 1},
|
|
{k: 1, m: 1, o: 2, n: 2, needed: 2},
|
|
{k: 1, m: 2, o: 2, n: 2, needed: 2},
|
|
{k: 2, m: 3, o: 4, n: 4, needed: 3},
|
|
{k: 2, m: 4, o: 6, n: 8, needed: 3},
|
|
{k: 20, m: 30, o: 40, n: 50, needed: 25},
|
|
{k: 29, m: 35, o: 80, n: 95, needed: 34},
|
|
} {
|
|
tag := fmt.Sprintf("#%d. %+v", i, tt)
|
|
|
|
rs := pb.RedundancyScheme{
|
|
MinReq: tt.k,
|
|
RepairThreshold: tt.m,
|
|
SuccessThreshold: tt.o,
|
|
Total: tt.n,
|
|
}
|
|
|
|
assert.Equal(t, tt.needed, segments.CalcNeededNodes(&rs), tag)
|
|
}
|
|
}
|
|
|
|
func createTestData(t *testing.T, size int64) []byte {
|
|
data, err := ioutil.ReadAll(io.LimitReader(rand.Reader, size))
|
|
require.NoError(t, err)
|
|
return data
|
|
}
|
|
|
|
func runTest(t *testing.T, test func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet, segmentStore segments.Store)) {
|
|
testplanet.Run(t, testplanet.Config{
|
|
SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1,
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
// TODO move apikey creation to testplanet
|
|
project, err := planet.Satellites[0].DB.Console().Projects().Insert(context.Background(), &console.Project{
|
|
Name: "testProject",
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
apiKey, err := macaroon.NewAPIKey([]byte("testSecret"))
|
|
require.NoError(t, err)
|
|
|
|
apiKeyInfo := console.APIKeyInfo{
|
|
ProjectID: project.ID,
|
|
Name: "testKey",
|
|
Secret: []byte("testSecret"),
|
|
}
|
|
|
|
// add api key to db
|
|
_, err = planet.Satellites[0].DB.Console().APIKeys().Create(context.Background(), apiKey.Head(), apiKeyInfo)
|
|
require.NoError(t, err)
|
|
|
|
TestAPIKey := apiKey.Serialize()
|
|
|
|
metainfo, err := planet.Uplinks[0].DialMetainfo(context.Background(), planet.Satellites[0], TestAPIKey)
|
|
require.NoError(t, err)
|
|
|
|
ec := ecclient.NewClient(planet.Uplinks[0].Transport, 0)
|
|
fc, err := infectious.NewFEC(2, 4)
|
|
require.NoError(t, err)
|
|
|
|
rs, err := eestream.NewRedundancyStrategy(eestream.NewRSScheme(fc, 1*memory.KiB.Int()), 0, 0)
|
|
require.NoError(t, err)
|
|
|
|
segmentStore := segments.NewSegmentStore(metainfo, ec, rs, 4*memory.KiB.Int(), 8*memory.MiB.Int64())
|
|
assert.NotNil(t, segmentStore)
|
|
|
|
test(t, ctx, planet, segmentStore)
|
|
})
|
|
}
|