storj/pkg/metainfo/kvmetainfo/objects_test.go

676 lines
19 KiB
Go
Raw Normal View History

2018-11-15 15:31:33 +00:00
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package kvmetainfo
import (
"context"
2018-11-21 14:35:53 +00:00
"crypto/rand"
2018-11-15 15:31:33 +00:00
"fmt"
2018-11-26 07:39:05 +00:00
"io"
2018-11-15 15:31:33 +00:00
"testing"
"time"
"github.com/stretchr/testify/assert"
2018-11-21 14:35:53 +00:00
"storj.io/storj/internal/memory"
2018-11-15 15:31:33 +00:00
"storj.io/storj/pkg/storj"
2018-11-26 07:39:05 +00:00
"storj.io/storj/pkg/stream"
2018-11-15 15:31:33 +00:00
)
2018-11-30 13:50:52 +00:00
const TestFile = "test-file"
func TestCreateObject(t *testing.T) {
customRS := storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
RequiredShares: 29,
RepairShares: 35,
OptimalShares: 80,
TotalShares: 95,
ShareSize: 2 * memory.KB.Int32(),
}
customES := storj.EncryptionScheme{
Cipher: storj.Unencrypted,
BlockSize: 1 * memory.KB.Int32(),
}
2018-11-15 15:31:33 +00:00
runTest(t, func(ctx context.Context, db *DB) {
bucket, err := db.CreateBucket(ctx, TestBucket, nil)
if !assert.NoError(t, err) {
return
}
2018-11-30 13:50:52 +00:00
for i, tt := range []struct {
create *storj.CreateObject
expectedRS storj.RedundancyScheme
expectedES storj.EncryptionScheme
}{
{
create: nil,
expectedRS: defaultRS,
expectedES: defaultES,
}, {
create: &storj.CreateObject{RedundancyScheme: customRS, EncryptionScheme: customES},
expectedRS: customRS,
expectedES: customES,
}, {
create: &storj.CreateObject{RedundancyScheme: customRS},
expectedRS: customRS,
expectedES: storj.EncryptionScheme{Cipher: defaultES.Cipher, BlockSize: customRS.ShareSize},
}, {
create: &storj.CreateObject{EncryptionScheme: customES},
expectedRS: defaultRS,
expectedES: customES,
},
} {
errTag := fmt.Sprintf("%d. %+v", i, tt)
obj, err := db.CreateObject(ctx, bucket.Name, TestFile, tt.create)
if !assert.NoError(t, err) {
return
}
info := obj.Info()
assert.Equal(t, TestBucket, info.Bucket.Name, errTag)
assert.Equal(t, storj.AESGCM, info.Bucket.PathCipher, errTag)
2018-11-30 13:50:52 +00:00
assert.Equal(t, TestFile, info.Path, errTag)
assert.EqualValues(t, 0, info.Size, errTag)
assert.Equal(t, tt.expectedRS, info.RedundancyScheme, errTag)
assert.Equal(t, tt.expectedES, info.EncryptionScheme, errTag)
2018-11-15 15:31:33 +00:00
}
2018-11-30 13:50:52 +00:00
})
}
2018-11-15 15:31:33 +00:00
2018-11-30 13:50:52 +00:00
func TestGetObject(t *testing.T) {
runTest(t, func(ctx context.Context, db *DB) {
bucket, err := db.CreateBucket(ctx, TestBucket, nil)
2018-11-15 15:31:33 +00:00
if !assert.NoError(t, err) {
return
}
2018-11-30 13:50:52 +00:00
upload(ctx, t, db, bucket, TestFile, nil)
2018-11-15 15:31:33 +00:00
_, err = db.GetObject(ctx, "", "")
assert.True(t, storj.ErrNoBucket.Has(err))
_, err = db.GetObject(ctx, bucket.Name, "")
2018-11-30 13:50:52 +00:00
assert.True(t, storj.ErrNoPath.Has(err))
2018-11-15 15:31:33 +00:00
2018-11-30 13:50:52 +00:00
_, err = db.GetObject(ctx, "non-existing-bucket", TestFile)
assert.True(t, storj.ErrBucketNotFound.Has(err))
2018-11-15 15:31:33 +00:00
_, err = db.GetObject(ctx, bucket.Name, "non-existing-file")
assert.True(t, storj.ErrObjectNotFound.Has(err))
2018-11-15 15:31:33 +00:00
2018-11-30 13:50:52 +00:00
object, err := db.GetObject(ctx, bucket.Name, TestFile)
2018-11-15 15:31:33 +00:00
if assert.NoError(t, err) {
2018-11-30 13:50:52 +00:00
assert.Equal(t, TestFile, object.Path)
assert.Equal(t, TestBucket, object.Bucket.Name)
assert.Equal(t, storj.AESGCM, object.Bucket.PathCipher)
2018-11-15 15:31:33 +00:00
}
})
}
func TestGetObjectStream(t *testing.T) {
runTest(t, func(ctx context.Context, db *DB) {
2018-11-21 14:35:53 +00:00
// we wait a second for all the nodes to complete bootstrapping off the satellite
time.Sleep(2 * time.Second)
data := make([]byte, 32*memory.KB)
2018-11-30 13:50:52 +00:00
_, err := rand.Read(data)
2018-11-21 14:35:53 +00:00
if !assert.NoError(t, err) {
return
}
2018-11-30 13:50:52 +00:00
bucket, err := db.CreateBucket(ctx, TestBucket, nil)
2018-11-15 15:31:33 +00:00
if !assert.NoError(t, err) {
return
}
2018-11-30 13:50:52 +00:00
upload(ctx, t, db, bucket, "empty-file", nil)
upload(ctx, t, db, bucket, "small-file", []byte("test"))
upload(ctx, t, db, bucket, "large-file", data)
2018-11-15 15:31:33 +00:00
_, err = db.GetObjectStream(ctx, "", "")
assert.True(t, storj.ErrNoBucket.Has(err))
_, err = db.GetObjectStream(ctx, bucket.Name, "")
2018-11-30 13:50:52 +00:00
assert.True(t, storj.ErrNoPath.Has(err))
2018-11-15 15:31:33 +00:00
2018-11-21 14:35:53 +00:00
_, err = db.GetObjectStream(ctx, "non-existing-bucket", "small-file")
assert.True(t, storj.ErrBucketNotFound.Has(err))
2018-11-15 15:31:33 +00:00
_, err = db.GetObjectStream(ctx, bucket.Name, "non-existing-file")
assert.True(t, storj.ErrObjectNotFound.Has(err))
2018-11-15 15:31:33 +00:00
assertStream(ctx, t, db, bucket, "empty-file", 0, []byte{})
assertStream(ctx, t, db, bucket, "small-file", 4, []byte("test"))
assertStream(ctx, t, db, bucket, "large-file", int64(32*memory.KB), data)
2018-11-15 15:31:33 +00:00
})
}
2018-11-30 13:50:52 +00:00
func upload(ctx context.Context, t *testing.T, db *DB, bucket storj.Bucket, path storj.Path, data []byte) {
obj, err := db.CreateObject(ctx, bucket.Name, path, nil)
if !assert.NoError(t, err) {
return
}
str, err := obj.CreateStream(ctx)
if !assert.NoError(t, err) {
return
}
upload := stream.NewUpload(ctx, str, db.streams)
2018-11-30 13:50:52 +00:00
_, err = upload.Write(data)
if !assert.NoError(t, err) {
return
}
err = upload.Close()
if !assert.NoError(t, err) {
return
}
err = obj.Commit(ctx)
if !assert.NoError(t, err) {
return
}
}
func assertStream(ctx context.Context, t *testing.T, db *DB, bucket storj.Bucket, path storj.Path, size int64, content []byte) {
readOnly, err := db.GetObjectStream(ctx, bucket.Name, path)
if !assert.NoError(t, err) {
return
}
2018-11-26 07:39:05 +00:00
assert.Equal(t, path, readOnly.Info().Path)
assert.Equal(t, TestBucket, readOnly.Info().Bucket.Name)
assert.Equal(t, storj.AESGCM, readOnly.Info().Bucket.PathCipher)
2018-11-15 15:31:33 +00:00
2018-11-26 07:39:05 +00:00
segments, more, err := readOnly.Segments(ctx, 0, 0)
2018-11-15 15:31:33 +00:00
if !assert.NoError(t, err) {
2018-11-21 14:35:53 +00:00
return
2018-11-15 15:31:33 +00:00
}
assert.False(t, more)
if !assert.Equal(t, 1, len(segments)) {
2018-11-21 14:35:53 +00:00
return
2018-11-15 15:31:33 +00:00
}
2018-11-21 14:35:53 +00:00
2018-11-15 15:31:33 +00:00
assert.EqualValues(t, 0, segments[0].Index)
assert.EqualValues(t, len(content), segments[0].Size)
2018-11-21 14:35:53 +00:00
if segments[0].Size > int64(4*memory.KB) {
assertRemoteSegment(t, segments[0])
} else {
assertInlineSegment(t, segments[0], content)
}
2018-11-26 07:39:05 +00:00
download := stream.NewDownload(ctx, readOnly, db.streams)
2018-11-26 07:39:05 +00:00
defer func() {
err = download.Close()
assert.NoError(t, err)
}()
data := make([]byte, len(content))
n, err := io.ReadFull(download, data)
if !assert.NoError(t, err) {
return
}
assert.Equal(t, len(content), n)
assert.Equal(t, content, data)
2018-11-21 14:35:53 +00:00
}
2018-11-15 15:31:33 +00:00
2018-11-21 14:35:53 +00:00
func assertInlineSegment(t *testing.T, segment storj.Segment, content []byte) {
assert.Equal(t, content, segment.Inline)
assert.Nil(t, segment.PieceID)
assert.Equal(t, 0, len(segment.Pieces))
}
func assertRemoteSegment(t *testing.T, segment storj.Segment) {
assert.Nil(t, segment.Inline)
assert.NotNil(t, segment.PieceID)
assert.NotEqual(t, 0, len(segment.Pieces))
// check that piece numbers and nodes are unique
nums := make(map[byte]struct{})
nodes := make(map[string]struct{})
for _, piece := range segment.Pieces {
if _, ok := nums[piece.Number]; ok {
t.Fatalf("piece number %d is not unique", piece.Number)
}
nums[piece.Number] = struct{}{}
2018-11-15 15:31:33 +00:00
id := piece.Location.String()
2018-11-21 14:35:53 +00:00
if _, ok := nodes[id]; ok {
t.Fatalf("node id %s is not unique", id)
}
nodes[id] = struct{}{}
}
2018-11-15 15:31:33 +00:00
}
func TestDeleteObject(t *testing.T) {
runTest(t, func(ctx context.Context, db *DB) {
bucket, err := db.CreateBucket(ctx, TestBucket, nil)
if !assert.NoError(t, err) {
return
}
2018-11-30 13:50:52 +00:00
upload(ctx, t, db, bucket, TestFile, nil)
2018-11-15 15:31:33 +00:00
err = db.DeleteObject(ctx, "", "")
assert.True(t, storj.ErrNoBucket.Has(err))
err = db.DeleteObject(ctx, bucket.Name, "")
assert.True(t, storj.ErrNoPath.Has(err))
2018-11-15 15:31:33 +00:00
2018-11-30 13:50:52 +00:00
err = db.DeleteObject(ctx, "non-existing-bucket", TestFile)
assert.True(t, storj.ErrBucketNotFound.Has(err))
2018-11-15 15:31:33 +00:00
err = db.DeleteObject(ctx, bucket.Name, "non-existing-file")
assert.True(t, storj.ErrObjectNotFound.Has(err))
2018-11-15 15:31:33 +00:00
2018-11-30 13:50:52 +00:00
err = db.DeleteObject(ctx, bucket.Name, TestFile)
2018-11-15 15:31:33 +00:00
assert.NoError(t, err)
})
}
func TestListObjectsEmpty(t *testing.T) {
runTest(t, func(ctx context.Context, db *DB) {
bucket, err := db.CreateBucket(ctx, TestBucket, nil)
if !assert.NoError(t, err) {
return
}
_, err = db.ListObjects(ctx, "", storj.ListOptions{})
assert.True(t, storj.ErrNoBucket.Has(err))
_, err = db.ListObjects(ctx, bucket.Name, storj.ListOptions{})
assert.EqualError(t, err, "kvmetainfo: invalid direction 0")
for _, direction := range []storj.ListDirection{
storj.Before,
storj.Backward,
storj.Forward,
storj.After,
} {
list, err := db.ListObjects(ctx, bucket.Name, storj.ListOptions{Direction: direction})
if assert.NoError(t, err) {
assert.False(t, list.More)
assert.Equal(t, 0, len(list.Items))
}
}
})
}
func TestListObjects(t *testing.T) {
runTest(t, func(ctx context.Context, db *DB) {
bucket, err := db.CreateBucket(ctx, TestBucket, &storj.Bucket{PathCipher: storj.Unencrypted})
if !assert.NoError(t, err) {
return
}
filePaths := []string{
"a", "aa", "b", "bb", "c",
"a/xa", "a/xaa", "a/xb", "a/xbb", "a/xc",
"b/ya", "b/yaa", "b/yb", "b/ybb", "b/yc",
}
2018-11-30 13:50:52 +00:00
2018-11-15 15:31:33 +00:00
for _, path := range filePaths {
2018-11-30 13:50:52 +00:00
upload(ctx, t, db, bucket, path, nil)
2018-11-15 15:31:33 +00:00
}
otherBucket, err := db.CreateBucket(ctx, "otherbucket", nil)
if !assert.NoError(t, err) {
return
}
2018-11-30 13:50:52 +00:00
upload(ctx, t, db, otherBucket, "file-in-other-bucket", nil)
2018-11-15 15:31:33 +00:00
for i, tt := range []struct {
options storj.ListOptions
more bool
result []string
}{
{
options: options("", "", storj.After, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "`", storj.After, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "b", storj.After, 0),
result: []string{"b/", "bb", "c"},
}, {
options: options("", "c", storj.After, 0),
result: []string{},
}, {
options: options("", "ca", storj.After, 0),
result: []string{},
}, {
options: options("", "", storj.After, 1),
more: true,
result: []string{"a"},
}, {
options: options("", "`", storj.After, 1),
more: true,
result: []string{"a"},
}, {
options: options("", "aa", storj.After, 1),
more: true,
result: []string{"b"},
}, {
options: options("", "c", storj.After, 1),
result: []string{},
}, {
options: options("", "ca", storj.After, 1),
result: []string{},
}, {
options: options("", "", storj.After, 2),
more: true,
result: []string{"a", "a/"},
}, {
options: options("", "`", storj.After, 2),
2018-11-15 15:31:33 +00:00
more: true,
result: []string{"a", "a/"},
}, {
options: options("", "aa", storj.After, 2),
more: true,
result: []string{"b", "b/"},
}, {
options: options("", "bb", storj.After, 2),
result: []string{"c"},
}, {
options: options("", "c", storj.After, 2),
result: []string{},
}, {
options: options("", "ca", storj.After, 2),
result: []string{},
}, {
options: optionsRecursive("", "", storj.After, 0),
result: []string{"a", "a/xa", "a/xaa", "a/xb", "a/xbb", "a/xc", "aa", "b", "b/ya", "b/yaa", "b/yb", "b/ybb", "b/yc", "bb", "c"},
}, {
options: options("a", "", storj.After, 0),
result: []string{"xa", "xaa", "xb", "xbb", "xc"},
}, {
options: options("a/", "", storj.After, 0),
result: []string{"xa", "xaa", "xb", "xbb", "xc"},
}, {
options: options("a/", "xb", storj.After, 0),
result: []string{"xbb", "xc"},
}, {
options: optionsRecursive("", "a/xbb", storj.After, 5),
more: true,
result: []string{"a/xc", "aa", "b", "b/ya", "b/yaa"},
}, {
options: options("a/", "xaa", storj.After, 2),
more: true,
result: []string{"xb", "xbb"},
}, {
options: options("", "", storj.Forward, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "`", storj.Forward, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "b", storj.Forward, 0),
result: []string{"b", "b/", "bb", "c"},
}, {
options: options("", "c", storj.Forward, 0),
result: []string{"c"},
}, {
options: options("", "ca", storj.Forward, 0),
result: []string{},
}, {
options: options("", "", storj.Forward, 1),
more: true,
result: []string{"a"},
}, {
options: options("", "`", storj.Forward, 1),
more: true,
result: []string{"a"},
}, {
options: options("", "aa", storj.Forward, 1),
more: true,
result: []string{"aa"},
}, {
options: options("", "c", storj.Forward, 1),
result: []string{"c"},
}, {
options: options("", "ca", storj.Forward, 1),
result: []string{},
}, {
options: options("", "", storj.Forward, 2),
more: true,
result: []string{"a", "a/"},
}, {
options: options("", "`", storj.Forward, 2),
more: true,
result: []string{"a", "a/"},
}, {
options: options("", "aa", storj.Forward, 2),
more: true,
result: []string{"aa", "b"},
}, {
options: options("", "bb", storj.Forward, 2),
result: []string{"bb", "c"},
}, {
options: options("", "c", storj.Forward, 2),
result: []string{"c"},
}, {
options: options("", "ca", storj.Forward, 2),
result: []string{},
}, {
options: optionsRecursive("", "", storj.Forward, 0),
result: []string{"a", "a/xa", "a/xaa", "a/xb", "a/xbb", "a/xc", "aa", "b", "b/ya", "b/yaa", "b/yb", "b/ybb", "b/yc", "bb", "c"},
}, {
options: options("a", "", storj.Forward, 0),
result: []string{"xa", "xaa", "xb", "xbb", "xc"},
}, {
options: options("a/", "", storj.Forward, 0),
result: []string{"xa", "xaa", "xb", "xbb", "xc"},
}, {
options: options("a/", "xb", storj.Forward, 0),
result: []string{"xb", "xbb", "xc"},
}, {
options: optionsRecursive("", "a/xbb", storj.Forward, 5),
more: true,
result: []string{"a/xbb", "a/xc", "aa", "b", "b/ya"},
}, {
options: options("a/", "xaa", storj.Forward, 2),
more: true,
result: []string{"xaa", "xb"},
}, {
options: options("", "", storj.Backward, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "`", storj.Backward, 0),
2018-11-15 15:31:33 +00:00
result: []string{},
}, {
options: options("", "b", storj.Backward, 0),
result: []string{"a", "a/", "aa", "b"},
}, {
options: options("", "c", storj.Backward, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "ca", storj.Backward, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "", storj.Backward, 1),
more: true,
result: []string{"c"},
}, {
options: options("", "`", storj.Backward, 1),
2018-11-15 15:31:33 +00:00
result: []string{},
}, {
options: options("", "aa", storj.Backward, 1),
more: true,
result: []string{"aa"},
}, {
options: options("", "c", storj.Backward, 1),
more: true,
result: []string{"c"},
}, {
options: options("", "ca", storj.Backward, 1),
more: true,
result: []string{"c"},
}, {
options: options("", "", storj.Backward, 2),
more: true,
result: []string{"bb", "c"},
}, {
options: options("", "`", storj.Backward, 2),
result: []string{},
}, {
options: options("", "a/", storj.Backward, 2),
result: []string{"a"},
}, {
options: options("", "bb", storj.Backward, 2),
more: true,
result: []string{"b/", "bb"},
}, {
options: options("", "c", storj.Backward, 2),
more: true,
result: []string{"bb", "c"},
}, {
options: options("", "ca", storj.Backward, 2),
more: true,
result: []string{"bb", "c"},
}, {
options: optionsRecursive("", "", storj.Backward, 0),
result: []string{"a", "a/xa", "a/xaa", "a/xb", "a/xbb", "a/xc", "aa", "b", "b/ya", "b/yaa", "b/yb", "b/ybb", "b/yc", "bb", "c"},
}, {
options: options("a", "", storj.Backward, 0),
result: []string{"xa", "xaa", "xb", "xbb", "xc"},
}, {
options: options("a/", "", storj.Backward, 0),
result: []string{"xa", "xaa", "xb", "xbb", "xc"},
}, {
options: options("a/", "xb", storj.Backward, 0),
result: []string{"xa", "xaa", "xb"},
}, {
options: optionsRecursive("", "b/yaa", storj.Backward, 5),
more: true,
result: []string{"a/xc", "aa", "b", "b/ya", "b/yaa"},
}, {
options: options("a/", "xbb", storj.Backward, 2),
more: true,
result: []string{"xb", "xbb"},
}, {
options: options("", "", storj.Before, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "`", storj.Before, 0),
result: []string{},
}, {
options: options("", "a", storj.Before, 0),
result: []string{},
}, {
options: options("", "b", storj.Before, 0),
result: []string{"a", "a/", "aa"},
}, {
options: options("", "c", storj.Before, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb"},
}, {
options: options("", "ca", storj.Before, 0),
result: []string{"a", "a/", "aa", "b", "b/", "bb", "c"},
}, {
options: options("", "", storj.Before, 1),
more: true,
result: []string{"c"},
}, {
options: options("", "`", storj.Before, 1),
result: []string{},
}, {
options: options("", "a/", storj.Before, 1),
result: []string{"a"},
}, {
options: options("", "c", storj.Before, 1),
more: true,
result: []string{"bb"},
}, {
options: options("", "ca", storj.Before, 1),
more: true,
result: []string{"c"},
}, {
options: options("", "", storj.Before, 2),
more: true,
result: []string{"bb", "c"},
}, {
options: options("", "`", storj.Before, 2),
result: []string{},
}, {
options: options("", "a/", storj.Before, 2),
result: []string{"a"},
}, {
options: options("", "bb", storj.Before, 2),
more: true,
result: []string{"b", "b/"},
}, {
options: options("", "c", storj.Before, 2),
more: true,
result: []string{"b/", "bb"},
}, {
options: options("", "ca", storj.Before, 2),
more: true,
result: []string{"bb", "c"},
}, {
options: optionsRecursive("", "", storj.Before, 0),
result: []string{"a", "a/xa", "a/xaa", "a/xb", "a/xbb", "a/xc", "aa", "b", "b/ya", "b/yaa", "b/yb", "b/ybb", "b/yc", "bb", "c"},
}, {
options: options("a", "", storj.Before, 0),
result: []string{"xa", "xaa", "xb", "xbb", "xc"},
}, {
options: options("a/", "", storj.Before, 0),
result: []string{"xa", "xaa", "xb", "xbb", "xc"},
}, {
options: options("a/", "xb", storj.Before, 0),
result: []string{"xa", "xaa"},
}, {
options: optionsRecursive("", "b/yaa", storj.Before, 5),
more: true,
result: []string{"a/xbb", "a/xc", "aa", "b", "b/ya"},
}, {
options: options("a/", "xbb", storj.Before, 2),
more: true,
result: []string{"xaa", "xb"},
},
} {
errTag := fmt.Sprintf("%d. %+v", i, tt)
list, err := db.ListObjects(ctx, bucket.Name, tt.options)
if assert.NoError(t, err, errTag) {
assert.Equal(t, tt.more, list.More, errTag)
for i, item := range list.Items {
assert.Equal(t, tt.result[i], item.Path, errTag)
assert.Equal(t, TestBucket, item.Bucket.Name, errTag)
assert.Equal(t, storj.Unencrypted, item.Bucket.PathCipher, errTag)
}
2018-11-15 15:31:33 +00:00
}
}
})
}
func options(prefix, cursor string, direction storj.ListDirection, limit int) storj.ListOptions {
return storj.ListOptions{
Prefix: prefix,
Cursor: cursor,
Direction: direction,
Limit: limit,
}
}
func optionsRecursive(prefix, cursor string, direction storj.ListDirection, limit int) storj.ListOptions {
return storj.ListOptions{
Prefix: prefix,
Cursor: cursor,
Direction: direction,
Limit: limit,
Recursive: true,
}
}