JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor Path type (#522)

Browse Source 
The old paths.Path type is now replaced with the new storj.Path.

storj.Path is simply an alias to the built-in string type. As such it can be used just as any string, which simplifies a lot working with paths. No more conversions paths.New and path.String().

As an alias storj.Path does not define any methods. However, any functions applying to strings (like those from the strings package) gracefully apply to storj.Path too. In addition we have a few more functions defined:

    storj.SplitPath
    storj.JoinPaths
    encryption.EncryptPath
    encryption.DecryptPath
    encryption.DerivePathKey
    encryption.DeriveContentKey

All code in master is migrated to the new storj.Path type.

The Path example is also updated and is good for reference: /pkg/encryption/examples_test.go

This PR also resolve a nonce misuse issue in path encryption: https://storjlabs.atlassian.net/browse/V3-545
This commit is contained in:
Kaloyan Raev 2018-10-25 23:28:16 +03:00 committed by GitHub
parent f7828e73ea
commit 99640225fd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 691 additions and 914 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
cmd/uplink/cmd/cp.go
View File

@ -16,7 +16,6 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/storage/buckets"
"storj.io/storj/pkg/storage/objects"
@ -88,7 +87,7 @@ func upload(ctx context.Context, bs buckets.Store, src fpath.FPath, dst fpath.FP
meta := objects.SerializableMeta{}
expTime := time.Time{}
_, err = o.Put(ctx, paths.New(dst.Path()), r, meta, expTime)
_, err = o.Put(ctx, dst.Path(), r, meta, expTime)
if err != nil {
return err
}
@ -117,7 +116,7 @@ func download(ctx context.Context, bs buckets.Store, src fpath.FPath, dst fpath.
return err
}
rr, _, err := o.Get(ctx, paths.New(src.Path()))
rr, _, err := o.Get(ctx, src.Path())
if err != nil {
return err
}
@ -181,7 +180,7 @@ func copy(ctx context.Context, bs buckets.Store, src fpath.FPath, dst fpath.FPat
return err
}
rr, _, err := o.Get(ctx, paths.New(src.Path()))
rr, _, err := o.Get(ctx, src.Path())
if err != nil {
return err
}
@ -214,7 +213,7 @@ func copy(ctx context.Context, bs buckets.Store, src fpath.FPath, dst fpath.FPat
dst = dst.Join(src.Base())
}
_, err = o.Put(ctx, paths.New(dst.Path()), r, meta, expTime)
_, err = o.Put(ctx, dst.Path(), r, meta, expTime)
if err != nil {
return err
}

11
cmd/uplink/cmd/ls.go
View File

@ -11,7 +11,6 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/storage/buckets"
"storj.io/storj/pkg/storage/meta"
@ -45,7 +44,7 @@ func list(cmd *cobra.Command, args []string) error {
}
if src.IsLocal() {
return fmt.Errorf("No bucket specified. Please use format sj://bucket/")
return fmt.Errorf("No bucket specified, use format sj://bucket/")
}
return listFiles(ctx, bs, src, false)
@ -94,21 +93,21 @@ func listFiles(ctx context.Context, bs buckets.Store, prefix fpath.FPath, prepen
return err
}
startAfter := paths.New("")
startAfter := ""
for {
items, more, err := o.List(ctx, paths.New(prefix.Path()), startAfter, nil, *recursiveFlag, 0, meta.Modified|meta.Size)
items, more, err := o.List(ctx, prefix.Path(), startAfter, "", *recursiveFlag, 0, meta.Modified|meta.Size)
if err != nil {
return err
}
for _, object := range items {
path := object.Path.String()
path := object.Path
if prependBucket {
path = fmt.Sprintf("%s/%s", prefix.Bucket(), path)
}
if object.IsPrefix {
fmt.Println("PRE", path+"/")
fmt.Println("PRE", path)
} else {
fmt.Printf("%v %v %12v %v\n", "OBJ", formatTime(object.Meta.Modified), object.Meta.Size, path)
}

15
cmd/uplink/cmd/mount.go
View File

@ -18,7 +18,6 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/process"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storage/objects"
@ -98,14 +97,14 @@ func (sf *storjFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse
return &fuse.Attr{Mode: fuse.S_IFDIR | 0755}, fuse.OK
}
metadata, err := sf.store.Meta(sf.ctx, paths.New(name))
metadata, err := sf.store.Meta(sf.ctx, name)
if err != nil && !storage.ErrKeyNotFound.Has(err) {
return nil, fuse.EIO
}
// file not found so maybe it's a prefix/directory
if err != nil {
items, _, err := sf.store.List(sf.ctx, paths.New(name), nil, nil, false, 1, meta.None)
items, _, err := sf.store.List(sf.ctx, name, "", "", false, 1, meta.None)
if err != nil && !storage.ErrKeyNotFound.Has(err) {
return nil, fuse.EIO
}
@ -147,16 +146,16 @@ func (sf *storjFs) Open(name string, flags uint32, context *fuse.Context) (file
func (sf *storjFs) listFiles(ctx context.Context, name string, store objects.Store) (c []fuse.DirEntry, err error) {
var entries []fuse.DirEntry
startAfter := paths.New("")
startAfter := ""
for {
items, more, err := store.List(ctx, paths.New(name), startAfter, nil, false, 0, meta.Modified)
items, more, err := store.List(ctx, name, startAfter, "", false, 0, meta.Modified)
if err != nil {
return nil, err
}
for _, object := range items {
path := object.Path.String()
path := object.Path
mode := fuse.S_IFREG
if object.IsPrefix {
@ -176,7 +175,7 @@ func (sf *storjFs) listFiles(ctx context.Context, name string, store objects.Sto
}
func (sf *storjFs) Unlink(name string, context *fuse.Context) (code fuse.Status) {
err := sf.store.Delete(sf.ctx, paths.New(name))
err := sf.store.Delete(sf.ctx, name)
if err != nil {
if storage.ErrKeyNotFound.Has(err) {
return fuse.ENOENT
@ -222,7 +221,7 @@ func (f *storjFile) Read(buf []byte, off int64) (res fuse.ReadResult, code fuse.
func (f *storjFile) getReader(off int64) (io.ReadCloser, error) {
if f.reader == nil {
ranger, _, err := f.store.Get(f.ctx, paths.New(f.name))
ranger, _, err := f.store.Get(f.ctx, f.name)
if err != nil {
return nil, err
}

2
cmd/uplink/cmd/rb.go
View File

@ -60,7 +60,7 @@ func deleteBucket(cmd *cobra.Command, args []string) error {
return err
}
items, _, err := o.List(ctx, nil, nil, nil, true, 1, meta.None)
items, _, err := o.List(ctx, "", "", "", true, 1, meta.None)
if err != nil {
return err
}

5
cmd/uplink/cmd/rm.go
View File

@ -9,7 +9,6 @@ import (
"github.com/spf13/cobra"
"storj.io/storj/internal/fpath"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/process"
)
@ -34,7 +33,7 @@ func delete(cmd *cobra.Command, args []string) error {
}
if dst.IsLocal() {
return fmt.Errorf("No bucket specified. Please use format sj://bucket/")
return fmt.Errorf("No bucket specified, use format sj://bucket/")
}
bs, err := cfg.BucketStore(ctx)
@ -47,7 +46,7 @@ func delete(cmd *cobra.Command, args []string) error {
return err
}
err = o.Delete(ctx, paths.New(dst.Path()))
err = o.Delete(ctx, dst.Path())
if err != nil {
return err
}

10
examples/pointerdb-client/main.go
View File

@ -14,7 +14,6 @@ import (
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
p "storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/pointerdb/pdbclient"
"storj.io/storj/pkg/provider"
@ -59,7 +58,7 @@ func main() {
ctx := context.Background()
// Example parameters to pass into API calls
var path = p.New("fold1/fold2/fold3/file.txt")
var path = "fold1/fold2/fold3/file.txt"
pointer := &pb.Pointer{
Type: pb.Pointer_INLINE,
InlineSegment: []byte("popcorn"),
@ -75,7 +74,7 @@ func main() {
}
// Example Put2
err = client.Put(ctx, p.New("fold1/fold2"), pointer)
err = client.Put(ctx, "fold1/fold2", pointer)
if err != nil || status.Code(err) == codes.Internal {
logger.Error("couldn't put pointer in db", zap.Error(err))
@ -95,15 +94,14 @@ func main() {
}
// Example List with pagination
prefix := p.New("fold1")
items, more, err := client.List(ctx, prefix, nil, nil, true, 1, meta.None)
items, more, err := client.List(ctx, "fold1", "", "", true, 1, meta.None)
if err != nil || status.Code(err) == codes.Internal {
logger.Error("failed to list file paths", zap.Error(err))
} else {
var stringList []string
for _, item := range items {
stringList = append(stringList, item.Path.String())
stringList = append(stringList, item.Path)
}
logger.Debug("Success: listed paths: " + strings.Join(stringList, ", ") + "; more: " + fmt.Sprintf("%t", more))
}

16
pkg/audit/cursor.go
View File

@ -12,10 +12,10 @@ import (
"github.com/vivint/infectious"
"storj.io/storj/pkg/eestream"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/pointerdb/pdbclient"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storj"
)
// Stripe keeps track of a stripe's index and its parent segment
@ -27,7 +27,7 @@ type Stripe struct {
// Cursor keeps track of audit location in pointer db
type Cursor struct {
pointers pdbclient.Client
lastPath *paths.Path
lastPath storj.Path
mutex sync.Mutex
}
@ -42,13 +42,13 @@ func (cursor *Cursor) NextStripe(ctx context.Context) (stripe *Stripe, err error
defer cursor.mutex.Unlock()
var pointerItems []pdbclient.ListItem
var path paths.Path
var path storj.Path
var more bool
if cursor.lastPath == nil {
pointerItems, more, err = cursor.pointers.List(ctx, nil, nil, nil, true, 0, meta.None)
if cursor.lastPath == "" {
pointerItems, more, err = cursor.pointers.List(ctx, "", "", "", true, 0, meta.None)
} else {
pointerItems, more, err = cursor.pointers.List(ctx, nil, *cursor.lastPath, nil, true, 0, meta.None)
pointerItems, more, err = cursor.pointers.List(ctx, "", cursor.lastPath, "", true, 0, meta.None)
}
if err != nil {
@ -66,9 +66,9 @@ func (cursor *Cursor) NextStripe(ctx context.Context) (stripe *Stripe, err error
// keep track of last path listed
if !more {
cursor.lastPath = nil
cursor.lastPath = ""
} else {
cursor.lastPath = &pointerItems[len(pointerItems)-1].Path
cursor.lastPath = pointerItems[len(pointerItems)-1].Path
}
// get pointer info

34
pkg/audit/cursor_test.go
View File

@ -18,11 +18,11 @@ import (
"storj.io/storj/pkg/auth"
"storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/pointerdb"
"storj.io/storj/pkg/pointerdb/pdbclient"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storj"
"storj.io/storj/storage/redis/redisserver"
"storj.io/storj/storage/teststore"
)
@ -60,7 +60,7 @@ func (pbd *pointerDBWrapper) Delete(ctx context.Context, in *pb.DeleteRequest, o
func TestAuditSegment(t *testing.T) {
type pathCount struct {
path paths.Path
path storj.Path
count int
}
@ -69,47 +69,47 @@ func TestAuditSegment(t *testing.T) {
// list api call, default is 0 == 1000 listing
tests := []struct {
bm string
path paths.Path
path storj.Path
}{
{
bm: "success-1",
path: paths.New("folder1/file1"),
path: "folder1/file1",
},
{
bm: "success-2",
path: paths.New("foodFolder1/file1/file2"),
path: "foodFolder1/file1/file2",
},
{
bm: "success-3",
path: paths.New("foodFolder1/file1/file2/foodFolder2/file3"),
path: "foodFolder1/file1/file2/foodFolder2/file3",
},
{
bm: "success-4",
path: paths.New("projectFolder/project1.txt/"),
path: "projectFolder/project1.txt/",
},
{
bm: "success-5",
path: paths.New("newProjectFolder/project2.txt"),
path: "newProjectFolder/project2.txt",
},
{
bm: "success-6",
path: paths.New("Pictures/image1.png"),
path: "Pictures/image1.png",
},
{
bm: "success-7",
path: paths.New("Pictures/Nature/mountains.png"),
path: "Pictures/Nature/mountains.png",
},
{
bm: "success-8",
path: paths.New("Pictures/City/streets.png"),
path: "Pictures/City/streets.png",
},
{
bm: "success-9",
path: paths.New("Pictures/Animals/Dogs/dogs.png"),
path: "Pictures/Animals/Dogs/dogs.png",
},
{
bm: "success-10",
path: paths.New("Nada/ビデオ/😶"),
path: "Nada/ビデオ/😶",
},
}
@ -147,7 +147,7 @@ func TestAuditSegment(t *testing.T) {
putRequest := makePutRequest(tt.path)
// create putreq. object
req := &pb.PutRequest{Path: tt.path.String(), Pointer: putRequest.Pointer}
req := &pb.PutRequest{Path: tt.path, Pointer: putRequest.Pointer}
// put pointer into db
_, err := pdbw.Put(ctx, req)
@ -180,7 +180,7 @@ func TestAuditSegment(t *testing.T) {
// test to see how random paths are
t.Run("probabilisticTest", func(t *testing.T) {
list, _, err := pointers.List(ctx, nil, nil, nil, true, 10, meta.None)
list, _, err := pointers.List(ctx, "", "", "", true, 10, meta.None)
if err != nil {
t.Error(ErrNoList)
}
@ -237,14 +237,14 @@ func TestAuditSegment(t *testing.T) {
})
}
func makePutRequest(path paths.Path) pb.PutRequest {
func makePutRequest(path storj.Path) pb.PutRequest {
var rps []*pb.RemotePiece
rps = append(rps, &pb.RemotePiece{
PieceNum: 1,
NodeId: "testId",
})
pr := pb.PutRequest{
Path: path.String(),
Path: path,
Pointer: &pb.Pointer{
Type: pb.Pointer_REMOTE,
Remote: &pb.RemoteSegment{

17
pkg/encryption/encryption.go
View File

@ -4,6 +4,9 @@
package encryption
import (
"crypto/hmac"
"crypto/sha512"
"storj.io/storj/pkg/storj"
)
@ -98,3 +101,17 @@ func DecryptKey(keyToDecrypt storj.EncryptedPrivateKey, cipher storj.Cipher, key
return &decryptedKey, nil
}
// DeriveKey derives new key from the given key and message using HMAC-SHA512
func DeriveKey(key *storj.Key, message string) (*storj.Key, error) {
mac := hmac.New(sha512.New, key[:])
_, err := mac.Write([]byte(message))
if err != nil {
return nil, Error.Wrap(err)
}
derived := new(storj.Key)
copy(derived[:], mac.Sum(nil))
return derived, nil
}

62
pkg/encryption/examples_test.go Normal file
View File

@ -0,0 +1,62 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package encryption_test
import (
"encoding/hex"
"fmt"
"storj.io/storj/pkg/encryption"
"storj.io/storj/pkg/storj"
)
func ExampleEncryptPath() {
var path = "fold1/fold2/fold3/file.txt"
// seed
seed := new(storj.Key)
for i := range seed {
seed[i] = byte(i)
}
fmt.Printf("root key (%d bytes): %s\n", len(seed), hex.EncodeToString(seed[:]))
// use the seed for encrypting the path
encryptedPath, err := encryption.EncryptPath(path, seed)
if err != nil {
panic(err)
}
fmt.Println("path to encrypt:", path)
fmt.Println("encrypted path: ", encryptedPath)
// decrypting the path
decryptedPath, err := encryption.DecryptPath(encryptedPath, seed)
if err != nil {
panic(err)
}
fmt.Println("decrypted path: ", decryptedPath)
// handling of shared path
sharedPath := storj.JoinPaths(storj.SplitPath(encryptedPath)[2:]...)
fmt.Println("shared path: ", sharedPath)
derivedKey, err := encryption.DerivePathKey(decryptedPath, seed, 2)
if err != nil {
panic(err)
}
fmt.Printf("derived key (%d bytes): %s\n", len(derivedKey), hex.EncodeToString(derivedKey[:]))
decryptedPath, err = encryption.DecryptPath(sharedPath, derivedKey)
if err != nil {
panic(err)
}
fmt.Println("decrypted path: ", decryptedPath)
// Output:
// root key (32 bytes): 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f
// path to encrypt: fold1/fold2/fold3/file.txt
// encrypted path: urxuYzqG_ZlJfBhkGaz87WvvnCZaYD7qf1_ZN_Pd91n5/IyncDwLhWPv4F7EaoUivwICnUeJMWlUnMATL4faaoH2s/_1gitX6uPd3etc3RgoD9R1waT5MPKrlrY32ehz_vqlOv/6qO4DU5AHFabE2r7hmAauvnomvtNByuO-FCw4ch_xaVR3SPE
// decrypted path: fold1/fold2/fold3/file.txt
// shared path: _1gitX6uPd3etc3RgoD9R1waT5MPKrlrY32ehz_vqlOv/6qO4DU5AHFabE2r7hmAauvnomvtNByuO-FCw4ch_xaVR3SPE
// derived key (32 bytes): 909db5ccf2b645e3352ee8212305596ed514d9f84d5acd21d93b4527d2a0c7e1
// decrypted path: fold3/file.txt
}

143
pkg/encryption/path.go Normal file
View File

@ -0,0 +1,143 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package encryption
import (
"crypto/hmac"
"crypto/sha512"
"encoding/base64"
"storj.io/storj/pkg/storj"
)
// EncryptPath encrypts path with the given key
func EncryptPath(path storj.Path, key *storj.Key) (encrypted storj.Path, err error) {
// do not encrypt empty paths
if len(path) == 0 {
return path, nil
}
comps := storj.SplitPath(path)
for i, comp := range comps {
comps[i], err = encryptPathComponent(comp, key)
if err != nil {
return "", err
}
key, err = DeriveKey(key, "path:"+comp)
if err != nil {
return "", err
}
}
return storj.JoinPaths(comps...), nil
}
// DecryptPath decrypts path with the given key
func DecryptPath(path storj.Path, key *storj.Key) (decrypted storj.Path, err error) {
comps := storj.SplitPath(path)
for i, comp := range comps {
comps[i], err = decryptPathComponent(comp, key)
if err != nil {
return "", err
}
key, err = DeriveKey(key, "path:"+comps[i])
if err != nil {
return "", err
}
}
return storj.JoinPaths(comps...), nil
}
// DerivePathKey derives the key for the given depth from the given root key.
// This method must be called on an unencrypted path.
func DerivePathKey(path storj.Path, key *storj.Key, depth int) (derivedKey *storj.Key, err error) {
if depth < 0 {
return nil, Error.New("negative depth")
}
// do not derive key from empty path
if len(path) == 0 {
return key, nil
}
comps := storj.SplitPath(path)
if depth > len(comps) {
return nil, Error.New("depth greater than path length")
}
derivedKey = key
for i := 0; i < depth; i++ {
derivedKey, err = DeriveKey(derivedKey, "path:"+comps[i])
if err != nil {
return nil, err
}
}
return derivedKey, nil
}
// DeriveContentKey derives the key for the encrypted object data using the root key.
// This method must be called on an unencrypted path.
func DeriveContentKey(path storj.Path, key *storj.Key) (derivedKey *storj.Key, err error) {
comps := storj.SplitPath(path)
if len(comps) == 0 {
return nil, Error.New("path is empty")
}
derivedKey, err = DerivePathKey(path, key, len(comps))
if err != nil {
return nil, err
}
derivedKey, err = DeriveKey(derivedKey, "content")
if err != nil {
return nil, err
}
return derivedKey, nil
}
func encryptPathComponent(comp string, key *storj.Key) (string, error) {
// derive the key for the current path component
derivedKey, err := DeriveKey(key, "path:"+comp)
if err != nil {
return "", err
}
// use the derived key to derive the nonce
mac := hmac.New(sha512.New, derivedKey[:])
_, err = mac.Write([]byte("nonce"))
if err != nil {
return "", Error.Wrap(err)
}
nonce := new(AESGCMNonce)
copy(nonce[:], mac.Sum(nil))
// encrypt the path components with the parent's key and the derived nonce
cipherText, err := EncryptAESGCM([]byte(comp), key, nonce)
if err != nil {
return "", Error.Wrap(err)
}
// keep the nonce together with the cipher text
return base64.RawURLEncoding.EncodeToString(append(nonce[:], cipherText...)), nil
}
func decryptPathComponent(comp string, key *storj.Key) (string, error) {
if comp == "" {
return "", nil
}
data, err := base64.RawURLEncoding.DecodeString(comp)
if err != nil {
return "", Error.Wrap(err)
}
// extract the nonce from the cipher text
nonce := new(AESGCMNonce)
copy(nonce[:], data[:AESGCMNonceSize])
decrypted, err := DecryptAESGCM(data[AESGCMNonceSize:], key, nonce)
if err != nil {
return "", Error.Wrap(err)
}
return string(decrypted), nil
}

87
pkg/encryption/path_test.go Normal file
View File

@ -0,0 +1,87 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package encryption
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"storj.io/storj/pkg/storj"
)
func TestEncryption(t *testing.T) {
for i, path := range []storj.Path{
"",
"/",
"//",
"file.txt",
"file.txt/",
"fold1/file.txt",
"fold1/fold2/file.txt",
"/fold1/fold2/fold3/file.txt",
} {
errTag := fmt.Sprintf("Test case #%d", i)
key := new(storj.Key)
copy(key[:], randData(storj.KeySize))
encrypted, err := EncryptPath(path, key)
if !assert.NoError(t, err, errTag) {
continue
}
decrypted, err := DecryptPath(encrypted, key)
if !assert.NoError(t, err, errTag) {
continue
}
assert.Equal(t, path, decrypted, errTag)
}
}
func TestDeriveKey(t *testing.T) {
for i, tt := range []struct {
path storj.Path
depth int
errString string
}{
{"fold1/fold2/fold3/file.txt", -1, "encryption error: negative depth"},
{"fold1/fold2/fold3/file.txt", 0, ""},
{"fold1/fold2/fold3/file.txt", 1, ""},
{"fold1/fold2/fold3/file.txt", 2, ""},
{"fold1/fold2/fold3/file.txt", 3, ""},
{"fold1/fold2/fold3/file.txt", 4, ""},
{"fold1/fold2/fold3/file.txt", 5, "encryption error: depth greater than path length"},
} {
errTag := fmt.Sprintf("Test case #%d", i)
key := new(storj.Key)
copy(key[:], randData(storj.KeySize))
encrypted, err := EncryptPath(tt.path, key)
if !assert.NoError(t, err, errTag) {
continue
}
derivedKey, err := DerivePathKey(tt.path, key, tt.depth)
if tt.errString != "" {
assert.EqualError(t, err, tt.errString, errTag)
continue
}
if !assert.NoError(t, err, errTag) {
continue
}
shared := storj.JoinPaths(storj.SplitPath(encrypted)[tt.depth:]...)
decrypted, err := DecryptPath(shared, derivedKey)
if !assert.NoError(t, err, errTag) {
continue
}
expected := storj.JoinPaths(storj.SplitPath(tt.path)[tt.depth:]...)
assert.Equal(t, expected, decrypted, errTag)
}
}

7
pkg/miniogw/config.go
View File

@ -155,10 +155,15 @@ func (c Config) GetBucketStore(ctx context.Context, identity *provider.FullIdent
err = Error.New("EncryptionBlockSize must be a multiple of ErasureShareSize * RS MinThreshold")
return nil, err
}
stream, err := streams.NewStreamStore(segments, c.SegmentSize, c.EncKey, c.EncBlockSize, storj.Cipher(c.EncType))
key := new(storj.Key)
copy(key[:], c.EncKey)
stream, err := streams.NewStreamStore(segments, c.SegmentSize, key, c.EncBlockSize, storj.Cipher(c.EncType))
if err != nil {
return nil, err
}
obj := objects.NewStore(stream)
return buckets.NewStore(obj), nil

47
pkg/miniogw/gateway-storj.go
View File

@ -6,6 +6,7 @@ package miniogw
import (
"context"
"io"
"strings"
"time"
minio "github.com/minio/minio/cmd"
@ -14,11 +15,11 @@ import (
"github.com/zeebo/errs"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/buckets"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storage/objects"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/utils"
"storj.io/storj/storage"
)
@ -74,7 +75,7 @@ func (s *storjObjects) DeleteBucket(ctx context.Context, bucket string) (err err
if err != nil {
return err
}
items, _, err := o.List(ctx, nil, nil, nil, true, 1, meta.None)
items, _, err := o.List(ctx, "", "", "", true, 1, meta.None)
if err != nil {
return err
}
@ -90,7 +91,7 @@ func (s *storjObjects) DeleteObject(ctx context.Context, bucket, object string)
if err != nil {
return err
}
err = o.Delete(ctx, paths.New(object))
err = o.Delete(ctx, object)
if storage.ErrKeyNotFound.Has(err) {
err = minio.ObjectNotFound{Bucket: bucket, Object: object}
}
@ -119,7 +120,7 @@ func (s *storjObjects) getObject(ctx context.Context, bucket, object string) (rr
return nil, err
}
rr, _, err = o.Get(ctx, paths.New(object))
rr, _, err = o.Get(ctx, object)
return rr, err
}
@ -155,7 +156,7 @@ func (s *storjObjects) GetObjectInfo(ctx context.Context, bucket,
if err != nil {
return minio.ObjectInfo{}, err
}
m, err := o.Meta(ctx, paths.New(object))
m, err := o.Meta(ctx, object)
if err != nil {
if storage.ErrKeyNotFound.Has(err) {
return objInfo, minio.ObjectNotFound{
@ -208,7 +209,7 @@ func (s *storjObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
return minio.ListObjectsInfo{}, Error.New("delimiter %s not supported", delimiter)
}
startAfter := paths.New(marker)
startAfter := marker
recursive := delimiter == ""
var objects []minio.ObjectInfo
@ -217,24 +218,24 @@ func (s *storjObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
if err != nil {
return minio.ListObjectsInfo{}, err
}
items, more, err := o.List(ctx, paths.New(prefix), startAfter, nil, recursive, maxKeys, meta.All)
items, more, err := o.List(ctx, prefix, startAfter, "", recursive, maxKeys, meta.All)
if err != nil {
return result, err
}
if len(items) > 0 {
for _, item := range items {
path := item.Path
if recursive {
path = path.Prepend(prefix)
if recursive && prefix != "" {
path = storj.JoinPaths(strings.TrimSuffix(prefix, "/"), path)
}
if item.IsPrefix {
prefixes = append(prefixes, path.String()+"/")
prefixes = append(prefixes, path)
continue
}
objects = append(objects, minio.ObjectInfo{
Bucket: bucket,
IsDir: false,
Name: path.String(),
Name: path,
ModTime: item.Meta.Modified,
Size: item.Meta.Size,
ContentType: item.Meta.ContentType,
@ -251,7 +252,7 @@ func (s *storjObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
Prefixes: prefixes,
}
if more {
result.NextMarker = startAfter.String()
result.NextMarker = startAfter
}
return result, err
@ -268,12 +269,12 @@ func (s *storjObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
recursive := delimiter == ""
var nextContinuationToken string
var startAfterPath paths.Path
var startAfterPath storj.Path
if continuationToken != "" {
startAfterPath = paths.New(continuationToken)
startAfterPath = continuationToken
}
if startAfterPath == nil && startAfter != "" {
startAfterPath = paths.New(startAfter)
if startAfterPath == "" && startAfter != "" {
startAfterPath = startAfter
}
var objects []minio.ObjectInfo
@ -282,7 +283,7 @@ func (s *storjObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
if err != nil {
return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, err
}
items, more, err := o.List(ctx, paths.New(prefix), startAfterPath, nil, recursive, maxKeys, meta.All)
items, more, err := o.List(ctx, prefix, startAfterPath, "", recursive, maxKeys, meta.All)
if err != nil {
return result, err
}
@ -290,17 +291,17 @@ func (s *storjObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
if len(items) > 0 {
for _, item := range items {
path := item.Path
if recursive {
path = path.Prepend(prefix)
if recursive && prefix != "" {
path = storj.JoinPaths(strings.TrimSuffix(prefix, "/"), path)
}
if item.IsPrefix {
prefixes = append(prefixes, path.String()+"/")
prefixes = append(prefixes, path)
continue
}
objects = append(objects, minio.ObjectInfo{
Bucket: bucket,
IsDir: false,
Name: path.String(),
Name: path,
ModTime: item.Meta.Modified,
Size: item.Meta.Size,
ContentType: item.Meta.ContentType,
@ -309,7 +310,7 @@ func (s *storjObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
})
}
nextContinuationToken = items[len(items)-1].Path.String() + "\x00"
nextContinuationToken = items[len(items)-1].Path + "\x00"
}
result = minio.ListObjectsV2Info{
@ -380,7 +381,7 @@ func (s *storjObjects) putObject(ctx context.Context, bucket, object string, r i
if err != nil {
return minio.ObjectInfo{}, err
}
m, err := o.Put(ctx, paths.New(object), r, meta, expTime)
m, err := o.Put(ctx, object, r, meta, expTime)
return minio.ObjectInfo{
Name: object,
Bucket: bucket,

37
pkg/miniogw/gateway-storj_test.go
View File

@ -9,7 +9,6 @@ import (
"errors"
"fmt"
"io"
"path"
"testing"
"time"
@ -18,7 +17,6 @@ import (
"github.com/minio/minio/pkg/hash"
"github.com/stretchr/testify/assert"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/buckets"
mock_buckets "storj.io/storj/pkg/storage/buckets/mocks"
@ -98,11 +96,11 @@ func TestCopyObject(t *testing.T) {
// if o.Get returns an error, only expect GetObjectStore once, do not expect Put
if example.errString != "some Get err" {
mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil).Times(2)
mockOS.EXPECT().Get(gomock.Any(), paths.New(example.srcObject)).Return(rr, meta, example.getErr)
mockOS.EXPECT().Put(gomock.Any(), paths.New(example.destObject), r, serMeta, time.Time{}).Return(meta, example.putErr)
mockOS.EXPECT().Get(gomock.Any(), example.srcObject).Return(rr, meta, example.getErr)
mockOS.EXPECT().Put(gomock.Any(), example.destObject, r, serMeta, time.Time{}).Return(meta, example.putErr)
} else {
mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil)
mockOS.EXPECT().Get(gomock.Any(), paths.New(example.srcObject)).Return(rr, meta, example.getErr)
mockOS.EXPECT().Get(gomock.Any(), example.srcObject).Return(rr, meta, example.getErr)
}
objInfo, err := storjObj.CopyObject(ctx, example.bucket, example.srcObject, example.bucket, example.destObject, srcInfo)
@ -158,7 +156,7 @@ func TestGetObject(t *testing.T) {
rr := ranger.ByteRanger([]byte(example.data))
mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil)
mockOS.EXPECT().Get(gomock.Any(), paths.New(example.object)).Return(rr, meta, example.err)
mockOS.EXPECT().Get(gomock.Any(), example.object).Return(rr, meta, example.err)
var buf bytes.Buffer
iowriter := io.Writer(&buf)
@ -195,7 +193,7 @@ func TestDeleteObject(t *testing.T) {
errTag := fmt.Sprintf("Test case #%d", i)
mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil)
mockOS.EXPECT().Delete(gomock.Any(), paths.New(example.object)).Return(example.err)
mockOS.EXPECT().Delete(gomock.Any(), example.object).Return(example.err)
err := storjObj.DeleteObject(ctx, example.bucket, example.object)
assert.NoError(t, err, errTag)
@ -256,7 +254,7 @@ func TestPutObject(t *testing.T) {
}
mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil)
mockOS.EXPECT().Put(gomock.Any(), paths.New(example.object), data, serMeta, time.Time{}).Return(meta, example.err)
mockOS.EXPECT().Put(gomock.Any(), example.object, data, serMeta, time.Time{}).Return(meta, example.err)
objInfo, err := storjObj.PutObject(ctx, example.bucket, example.object, data, metadata)
if err != nil {
@ -314,7 +312,7 @@ func TestGetObjectInfo(t *testing.T) {
errTag := fmt.Sprintf("Test case #%d", i)
mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil)
mockOS.EXPECT().Meta(gomock.Any(), paths.New(example.object)).Return(meta, example.err)
mockOS.EXPECT().Meta(gomock.Any(), example.object).Return(meta, example.err)
objInfo, err := storjObj.GetObjectInfo(ctx, example.bucket, example.object)
if err != nil {
@ -351,8 +349,8 @@ func TestListObjects(t *testing.T) {
maxKeys := 123
items := []objects.ListItem{
{Path: paths.New("test-file-1.txt")},
{Path: paths.New("test-file-2.txt")},
{Path: "test-file-1.txt"},
{Path: "test-file-2.txt"},
}
for i, example := range []struct {
@ -368,30 +366,29 @@ func TestListObjects(t *testing.T) {
{
more: false, startAfter: "", nextMarker: "", delimiter: "", recursive: true,
objInfos: []minio.ObjectInfo{
{Bucket: bucket, Name: path.Join("test-prefix/test-file-1.txt")},
{Bucket: bucket, Name: path.Join("test-prefix/test-file-2.txt")},
{Bucket: bucket, Name: "test-prefix/test-file-1.txt"},
{Bucket: bucket, Name: "test-prefix/test-file-2.txt"},
}, err: nil, errString: "",
},
{
more: true, startAfter: "test-start-after", nextMarker: "test-file-2.txt", delimiter: "/", recursive: false,
objInfos: []minio.ObjectInfo{
{Bucket: bucket, Name: path.Join("test-file-1.txt")},
{Bucket: bucket, Name: path.Join("test-file-2.txt")},
{Bucket: bucket, Name: "test-file-1.txt"},
{Bucket: bucket, Name: "test-file-2.txt"},
}, err: nil, errString: "",
},
{
more: false, startAfter: "", nextMarker: "", delimiter: "", recursive: true,
objInfos: []minio.ObjectInfo{
{Bucket: bucket, Name: path.Join("test-prefix/test-file-1.txt")},
{Bucket: bucket, Name: path.Join("test-prefix/test-file-2.txt")},
{Bucket: bucket, Name: "test-prefix/test-file-1.txt"},
{Bucket: bucket, Name: "test-prefix/test-file-2.txt"},
}, err: Error.New("error"), errString: "Storj Gateway error: error",
},
} {
errTag := fmt.Sprintf("Test case #%d", i)
mockBS.EXPECT().GetObjectStore(gomock.Any(), bucket).Return(mockOS, nil)
mockOS.EXPECT().List(gomock.Any(), paths.New(prefix), paths.New(example.startAfter),
nil, example.recursive, maxKeys, meta.All).Return(items, example.more, example.err)
mockOS.EXPECT().List(gomock.Any(), prefix, example.startAfter, "", example.recursive, maxKeys, meta.All).Return(items, example.more, example.err)
listInfo, err := storjObj.ListObjects(ctx, bucket, prefix, example.startAfter, example.delimiter, maxKeys)
@ -422,7 +419,7 @@ func TestDeleteBucket(t *testing.T) {
storjObj := storjObjects{storj: &b}
itemsInBucket := make([]objects.ListItem, 1)
itemsInBucket[0] = objects.ListItem{Path: paths.New("path1"), Meta: objects.Meta{}}
itemsInBucket[0] = objects.ListItem{Path: "path1", Meta: objects.Meta{}}
exp := time.Unix(0, 0).UTC()
var noItemsInBucket []objects.ListItem

3
pkg/miniogw/multipart.go
View File

@ -15,7 +15,6 @@ import (
minio "github.com/minio/minio/cmd"
"github.com/minio/minio/pkg/hash"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storage/objects"
)
@ -49,7 +48,7 @@ func (s *storjObjects) NewMultipartUpload(ctx context.Context, bucket, object st
UserDefined: metadata,
}
result, err := objectStore.Put(ctx, paths.New(object), upload.Stream, serMetaInfo, expTime)
result, err := objectStore.Put(ctx, object, upload.Stream, serMetaInfo, expTime)
uploads.RemoveByID(upload.ID)
if err != nil {

11
pkg/miniogw/objstore_mock_test.go
View File

@ -15,7 +15,6 @@ import (
gomock "github.com/golang/mock/gomock"
paths "storj.io/storj/pkg/paths"
ranger "storj.io/storj/pkg/ranger"
objects "storj.io/storj/pkg/storage/objects"
)
@ -44,7 +43,7 @@ func (m *MockStore) EXPECT() *MockStoreMockRecorder {
}
// Delete mocks base method
func (m *MockStore) Delete(arg0 context.Context, arg1 paths.Path) error {
func (m *MockStore) Delete(arg0 context.Context, arg1 string) error {
ret := m.ctrl.Call(m, "Delete", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
@ -56,7 +55,7 @@ func (mr *MockStoreMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call {
}
// Get mocks base method
func (m *MockStore) Get(arg0 context.Context, arg1 paths.Path) (ranger.Ranger, objects.Meta, error) {
func (m *MockStore) Get(arg0 context.Context, arg1 string) (ranger.Ranger, objects.Meta, error) {
ret := m.ctrl.Call(m, "Get", arg0, arg1)
ret0, _ := ret[0].(ranger.Ranger)
ret1, _ := ret[1].(objects.Meta)
@ -70,7 +69,7 @@ func (mr *MockStoreMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call {
}
// List mocks base method
func (m *MockStore) List(arg0 context.Context, arg1, arg2, arg3 paths.Path, arg4 bool, arg5 int, arg6 uint32) ([]objects.ListItem, bool, error) {
func (m *MockStore) List(arg0 context.Context, arg1, arg2, arg3 string, arg4 bool, arg5 int, arg6 uint32) ([]objects.ListItem, bool, error) {
ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3, arg4, arg5, arg6)
ret0, _ := ret[0].([]objects.ListItem)
ret1, _ := ret[1].(bool)
@ -84,7 +83,7 @@ func (mr *MockStoreMockRecorder) List(arg0, arg1, arg2, arg3, arg4, arg5, arg6 i
}
// Meta mocks base method
func (m *MockStore) Meta(arg0 context.Context, arg1 paths.Path) (objects.Meta, error) {
func (m *MockStore) Meta(arg0 context.Context, arg1 string) (objects.Meta, error) {
ret := m.ctrl.Call(m, "Meta", arg0, arg1)
ret0, _ := ret[0].(objects.Meta)
ret1, _ := ret[1].(error)
@ -97,7 +96,7 @@ func (mr *MockStoreMockRecorder) Meta(arg0, arg1 interface{}) *gomock.Call {
}
// Put mocks base method
func (m *MockStore) Put(arg0 context.Context, arg1 paths.Path, arg2 io.Reader, arg3 objects.SerializableMeta, arg4 time.Time) (objects.Meta, error) {
func (m *MockStore) Put(arg0 context.Context, arg1 string, arg2 io.Reader, arg3 objects.SerializableMeta, arg4 time.Time) (objects.Meta, error) {
ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2, arg3, arg4)
ret0, _ := ret[0].(objects.Meta)
ret1, _ := ret[1].(error)

11
pkg/paths/common.go
View File

@ -1,11 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package paths
import (
"github.com/zeebo/errs"
)
// Error is the errs class of standard path errors
var Error = errs.Class("paths error")

66
pkg/paths/examples_test.go
View File

@ -1,66 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package paths_test
import (
"encoding/hex"
"fmt"
"storj.io/storj/pkg/paths"
)
func ExamplePath_Encrypt() {
var path = paths.New("fold1/fold2/fold3/file.txt")
// seed
seed := make([]byte, 64)
for i := range seed {
seed[i] = byte(i)
}
fmt.Printf("root key (%d bytes): %s\n", len(seed), hex.EncodeToString(seed))
// use the seed for encrypting the path
encryptedPath, err := path.Encrypt(seed)
if err != nil {
panic(err)
}
fmt.Println("path to encrypt:", path)
fmt.Println("encrypted path: ", encryptedPath)
// decrypting the path
decryptedPath, err := encryptedPath.Decrypt(seed)
if err != nil {
panic(err)
}
fmt.Println("decrypted path: ", decryptedPath)
// handling of shared path
sharedPath := encryptedPath[2:]
fmt.Println("shared path: ", encryptedPath[2:])
derivedKey, err := decryptedPath.DeriveKey(seed, 2)
if err != nil {
panic(err)
}
fmt.Printf("derived key (%d bytes): %s\n", len(derivedKey), hex.EncodeToString(derivedKey))
decryptedPath, err = sharedPath.Decrypt(derivedKey)
if err != nil {
panic(err)
}
fmt.Println("decrypted path: ", decryptedPath)
// implement Bytes() function
var pathBytes = path.Bytes()
fmt.Println("path in Bytes is: ", pathBytes)
// Output:
// root key (64 bytes): 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f
// path to encrypt: fold1/fold2/fold3/file.txt
// encrypted path: 1ziIKg8Mw9_ywBqSOm78gQ9aQVbrQ/1FWdS4WUzuWXBrNML_FzGH8O2usos/1TKa8xYYCrUBEvEGj7YENdwViZOYh/17JWesXCBVU9Nx8IdnwuNUuwqgGFKL_jH
// decrypted path: fold1/fold2/fold3/file.txt
// shared path: 1TKa8xYYCrUBEvEGj7YENdwViZOYh/17JWesXCBVU9Nx8IdnwuNUuwqgGFKL_jH
// derived key (64 bytes): 2592f0694bc72a2719d09b7192b9b8f9e2517fda71419314d93a7c96844f28763ed3b829f3c9a09e812b402a66b1b0a832ae3cdd7435b7b496ca95eec108c629
// decrypted path: fold3/file.txt
// path in Bytes is: [102 111 108 100 49 47 102 111 108 100 50 47 102 111 108 100 51 47 102 105 108 101 46 116 120 116]
}

209
pkg/paths/paths.go
View File

@ -1,209 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package paths
import (
"crypto/aes"
"crypto/cipher"
"crypto/hmac"
"crypto/sha512"
"encoding/base64"
"path"
"strings"
)
// Path is a unique identifier for an object stored in the Storj network
type Path []string
// New creates new Path from the given path segments
func New(segs ...string) Path {
s := path.Join(segs...)
s = strings.Trim(s, "/")
p := strings.Split(s, "/")
if len(p) == 1 && p[0] == "" {
// Avoid building a path with a single empty segment
return []string{}
}
return p
}
// String returns the string representation of the path
func (p Path) String() string {
return path.Join([]string(p)...)
}
// Bytes serializes the current path to []byte
func (p Path) Bytes() []byte {
return []byte(p.String())
}
// Prepend creates new Path from the current path with the given segments prepended
func (p Path) Prepend(segs ...string) Path {
return New(append(segs, []string(p)...)...)
}
// Append creates new Path from the current path with the given segments appended
func (p Path) Append(segs ...string) Path {
return New(append(p, segs...)...)
}
// HasPrefix tests whether the current path begins with prefix.
func (p Path) HasPrefix(prefix Path) bool {
if len(prefix) > len(p) {
return false
}
for i := 0; i < len(prefix); i++ {
if p[i] != prefix[i] {
return false
}
}
return true
}
// Encrypt creates new Path by encrypting the current path with the given key
func (p Path) Encrypt(key []byte) (encrypted Path, err error) {
encrypted = make([]string, len(p))
for i, seg := range p {
encrypted[i], err = encrypt(seg, key)
if err != nil {
return nil, err
}
key, err = deriveSecret(key, seg)
if err != nil {
return nil, err
}
}
return encrypted, nil
}
// Decrypt creates new Path by decrypting the current path with the given key
func (p Path) Decrypt(key []byte) (decrypted Path, err error) {
decrypted = make([]string, len(p))
for i, seg := range p {
decrypted[i], err = decrypt(seg, key)
if err != nil {
return nil, err
}
key, err = deriveSecret(key, decrypted[i])
if err != nil {
return nil, err
}
}
return decrypted, nil
}
// DeriveKey derives the key for the given depth from the given root key
// This method must be called on an unencrypted path.
func (p Path) DeriveKey(key []byte, depth int) (derivedKey []byte, err error) {
if depth < 0 {
return nil, Error.New("negative depth")
}
if depth > len(p) {
return nil, Error.New("depth greater than path length")
}
derivedKey = key
for i := 0; i < depth; i++ {
derivedKey, err = deriveSecret(derivedKey, p[i])
if err != nil {
return nil, err
}
}
return derivedKey, nil
}
// DeriveContentKey derives the key for the encrypted object data using the root key
// This method must be called on an unencrypted path.
func (p Path) DeriveContentKey(key []byte) (derivedKey *[32]byte, err error) {
if len(p) == 0 {
return nil, Error.New("path is empty")
}
d, err := p.DeriveKey(key, len(p)-1)
if err != nil {
return nil, err
}
d, err = deriveSecret(d, "content")
if err != nil {
return nil, err
}
derivedKey = new([32]byte)
copy((*derivedKey)[:], d[:32])
return derivedKey, nil
}
func encrypt(text string, secret []byte) (cipherText string, err error) {
key, nonce, err := getAESGCMKeyAndNonce(secret)
if err != nil {
return "", Error.Wrap(err)
}
block, err := aes.NewCipher(key)
if err != nil {
return "", Error.Wrap(err)
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", Error.Wrap(err)
}
data := aesgcm.Seal(nil, nonce, []byte(text), nil)
cipherText = base64.RawURLEncoding.EncodeToString(data)
// prepend version number to the cipher text
return "1" + cipherText, nil
}
func decrypt(cipherText string, secret []byte) (text string, err error) {
if cipherText == "" {
return "", Error.New("empty cipher text")
}
// check the version number, only "1" is supported for now
if cipherText[0] != '1' {
return "", Error.New("invalid version number")
}
data, err := base64.RawURLEncoding.DecodeString(cipherText[1:])
if err != nil {
return "", Error.Wrap(err)
}
key, nonce, err := getAESGCMKeyAndNonce(secret)
if err != nil {
return "", Error.Wrap(err)
}
block, err := aes.NewCipher(key)
if err != nil {
return "", Error.Wrap(err)
}
aesgcm, err := cipher.NewGCM(block)
if err != nil {
return "", Error.Wrap(err)
}
decrypted, err := aesgcm.Open(nil, nonce, data, nil)
if err != nil {
return "", Error.Wrap(err)
}
return string(decrypted), nil
}
func getAESGCMKeyAndNonce(secret []byte) (key, nonce []byte, err error) {
mac := hmac.New(sha512.New, secret)
_, err = mac.Write([]byte("enc"))
if err != nil {
return nil, nil, Error.Wrap(err)
}
key = mac.Sum(nil)[:32]
mac.Reset()
_, err = mac.Write([]byte("nonce"))
if err != nil {
return nil, nil, Error.Wrap(err)
}
nonce = mac.Sum(nil)[:12]
return key, nonce, nil
}
func deriveSecret(secret []byte, child string) (derived []byte, err error) {
mac := hmac.New(sha512.New, secret)
_, err = mac.Write([]byte(child))
if err != nil {
return nil, Error.Wrap(err)
}
return mac.Sum(nil), nil
}

255
pkg/paths/paths_test.go
View File

@ -1,255 +0,0 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package paths
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestNew(t *testing.T) {
for i, tt := range []struct {
path string
expected Path
}{
{"", []string{}},
{"/", []string{}},
{"a", []string{"a"}},
{"/a/", []string{"a"}},
{"a/b/c/d", []string{"a", "b", "c", "d"}},
{"///a//b////c/d///", []string{"a", "b", "c", "d"}},
} {
errTag := fmt.Sprintf("Test case #%d", i)
p := New(tt.path)
assert.Equal(t, tt.expected, p, errTag)
}
}
func TestNewWithSegments(t *testing.T) {
for i, tt := range []struct {
segs []string
expected Path
}{
{nil, []string{}},
{[]string{""}, []string{}},
{[]string{"", ""}, []string{}},
{[]string{"/"}, []string{}},
{[]string{"a"}, []string{"a"}},
{[]string{"/a/"}, []string{"a"}},
{[]string{"", "a", "", "b", "c", "d", ""}, []string{"a", "b", "c", "d"}},
{[]string{"a", "b", "c", "d"}, []string{"a", "b", "c", "d"}},
{[]string{"/a", "b/", "/c/", "d"}, []string{"a", "b", "c", "d"}},
{[]string{"a/b", "c", "d"}, []string{"a", "b", "c", "d"}},
{[]string{"a/b", "c/d"}, []string{"a", "b", "c", "d"}},
{[]string{"//a/b", "c///d//"}, []string{"a", "b", "c", "d"}},
{[]string{"a/b/c/d"}, []string{"a", "b", "c", "d"}},
} {
errTag := fmt.Sprintf("Test case #%d", i)
p := New(tt.segs...)
assert.Equal(t, tt.expected, p, errTag)
}
}
func TestString(t *testing.T) {
for i, tt := range []struct {
path Path
expected string
}{
{nil, ""},
{[]string{}, ""},
{[]string{""}, ""},
{[]string{"a"}, "a"},
{[]string{"a", "b"}, "a/b"},
{[]string{"a", "b", "c", "d"}, "a/b/c/d"},
} {
errTag := fmt.Sprintf("Test case #%d", i)
s := tt.path.String()
assert.Equal(t, tt.expected, s, errTag)
}
}
func TestBytes(t *testing.T) {
for i, tt := range []struct {
path Path
expected []byte
}{
{nil, []byte{}},
{[]string{}, []byte{}},
{[]string{""}, []byte{}},
{[]string{"a/b"}, []byte{97, 47, 98}},
{[]string{"a/b/c"}, []byte{97, 47, 98, 47, 99}},
{[]string{"a/b/c/d/e/f"}, []byte{97, 47, 98, 47, 99, 47, 100, 47, 101, 47, 102}},
} {
errTag := fmt.Sprintf("Test case #%d", i)
b := tt.path.Bytes()
assert.Equal(t, tt.expected, b, errTag)
}
}
func TestPrepend(t *testing.T) {
for i, tt := range []struct {
prefix string
path string
expected Path
}{
{"", "", []string{}},
{"prefix", "", []string{"prefix"}},
{"", "my/path", []string{"my", "path"}},
{"prefix", "my/path", []string{"prefix", "my", "path"}},
{"p1/p2/p3", "my/path", []string{"p1", "p2", "p3", "my", "path"}},
} {
errTag := fmt.Sprintf("Test case #%d", i)
p := New(tt.path).Prepend(tt.prefix)
assert.Equal(t, tt.expected, p, errTag)
}
}
func TestPrependWithSegments(t *testing.T) {
for i, tt := range []struct {
segs []string
path string
expected Path
}{
{nil, "", []string{}},
{[]string{""}, "", []string{}},
{[]string{"prefix"}, "", []string{"prefix"}},
{[]string{""}, "my/path", []string{"my", "path"}},
{[]string{"prefix"}, "my/path", []string{"prefix", "my", "path"}},
{[]string{"p1/p2/p3"}, "my/path", []string{"p1", "p2", "p3", "my", "path"}},
{[]string{"p1", "p2/p3"}, "my/path", []string{"p1", "p2", "p3", "my", "path"}},
{[]string{"p1", "p2", "p3"}, "my/path", []string{"p1", "p2", "p3", "my", "path"}},
{[]string{"", "p1", "", "", "p2", "p3", ""}, "my/path", []string{"p1", "p2", "p3", "my", "path"}},
} {
errTag := fmt.Sprintf("Test case #%d", i)
p := New(tt.path).Prepend(tt.segs...)
assert.Equal(t, tt.expected, p, errTag)
}
}
func TestAppend(t *testing.T) {
for i, tt := range []struct {
path string
suffix string
expected Path
}{
{"", "", []string{}},
{"", "suffix", []string{"suffix"}},
{"my/path", "", []string{"my", "path"}},
{"my/path", "suffix", []string{"my", "path", "suffix"}},
{"my/path", "s1/s2/s3", []string{"my", "path", "s1", "s2", "s3"}},
} {
errTag := fmt.Sprintf("Test case #%d", i)
p := New(tt.path).Append(tt.suffix)
assert.Equal(t, tt.expected, p, errTag)
}
}
func TestAppendWithSegments(t *testing.T) {
for i, tt := range []struct {
path string
segs []string
expected Path
}{
{"", nil, []string{}},
{"", []string{""}, []string{}},
{"", []string{"suffix"}, []string{"suffix"}},
{"my/path", []string{""}, []string{"my", "path"}},
{"my/path", []string{"suffix"}, []string{"my", "path", "suffix"}},
{"my/path", []string{"s1/s2/s3"}, []string{"my", "path", "s1", "s2", "s3"}},
{"my/path", []string{"s1", "s2/s3"}, []string{"my", "path", "s1", "s2", "s3"}},
{"my/path", []string{"s1", "s2", "s3"}, []string{"my", "path", "s1", "s2", "s3"}},
{"my/path", []string{"", "s1", "", "", "s2", "s3", ""}, []string{"my", "path", "s1", "s2", "s3"}},
} {
errTag := fmt.Sprintf("Test case #%d", i)
p := New(tt.path).Append(tt.segs...)
assert.Equal(t, tt.expected, p, errTag)
}
}
func TestHasPrefix(t *testing.T) {
for i, tt := range []struct {
path string
prefix string
expected bool
}{
{"", "", true},
{"my/path", "", true},
{"", "prefix", false},
{"prefix/path", "prefix", true},
{"prefix/path", "prefix/path", true},
{"prefix/path", "prefix/path/more", false},
{"my/path/s1/s2/s3", "my/path", true},
{"my/path/s1/s2/s3", "s1/s2/s3", false},
} {
errTag := fmt.Sprintf("Test case #%d", i)
p := New(tt.path).HasPrefix(New(tt.prefix))
assert.Equal(t, tt.expected, p, errTag)
}
}
func TestEncryption(t *testing.T) {
for i, segs := range []Path{
nil, // empty path
[]string{}, // empty path
[]string{""}, // empty path segment
[]string{"file.txt"},
[]string{"fold1", "file.txt"},
[]string{"fold1", "fold2", "file.txt"},
[]string{"fold1", "fold2", "fold3", "file.txt"},
} {
errTag := fmt.Sprintf("Test case #%d", i)
path := New(segs...)
key := []byte("my secret")
encrypted, err := path.Encrypt(key)
if !assert.NoError(t, err, errTag) {
continue
}
decrypted, err := encrypted.Decrypt(key)
if !assert.NoError(t, err, errTag) {
continue
}
assert.Equal(t, path, decrypted, errTag)
}
}
func TestDeriveKey(t *testing.T) {
for i, tt := range []struct {
path Path
depth int
errString string
}{
{[]string{"fold1", "fold2", "fold3", "file.txt"}, -1,
"paths error: negative depth"},
{[]string{"fold1", "fold2", "fold3", "file.txt"}, 0, ""},
{[]string{"fold1", "fold2", "fold3", "file.txt"}, 1, ""},
{[]string{"fold1", "fold2", "fold3", "file.txt"}, 2, ""},
{[]string{"fold1", "fold2", "fold3", "file.txt"}, 3, ""},
{[]string{"fold1", "fold2", "fold3", "file.txt"}, 4, ""},
{[]string{"fold1", "fold2", "fold3", "file.txt"}, 5,
"paths error: depth greater than path length"},
} {
errTag := fmt.Sprintf("Test case #%d", i)
key := []byte("my secret")
encrypted, err := tt.path.Encrypt(key)
if !assert.NoError(t, err, errTag) {
continue
}
derivedKey, err := tt.path.DeriveKey(key, tt.depth)
if tt.errString != "" {
assert.EqualError(t, err, tt.errString, errTag)
continue
}
if !assert.NoError(t, err, errTag) {
continue
}
shared := encrypted[tt.depth:]
decrypted, err := shared.Decrypt(derivedKey)
if !assert.NoError(t, err, errTag) {
continue
}
assert.Equal(t, tt.path[tt.depth:], decrypted, errTag)
}
}

38
pkg/pointerdb/pdbclient/client.go
View File

@ -17,9 +17,9 @@ import (
"storj.io/storj/pkg/auth"
"storj.io/storj/pkg/auth/grpcauth"
p "storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/provider"
"storj.io/storj/pkg/storj"
"storj.io/storj/storage"
)
@ -44,19 +44,17 @@ var _ Client = (*PointerDB)(nil)
// ListItem is a single item in a listing
type ListItem struct {
Path p.Path
Path storj.Path
Pointer *pb.Pointer
IsPrefix bool
}
// Client services offerred for the interface
type Client interface {
Put(ctx context.Context, path p.Path, pointer *pb.Pointer) error
Get(ctx context.Context, path p.Path) (*pb.Pointer, error)
List(ctx context.Context, prefix, startAfter, endBefore p.Path,
recursive bool, limit int, metaFlags uint32) (
items []ListItem, more bool, err error)
Delete(ctx context.Context, path p.Path) error
Put(ctx context.Context, path storj.Path, pointer *pb.Pointer) error
Get(ctx context.Context, path storj.Path) (*pb.Pointer, error)
List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error)
Delete(ctx context.Context, path storj.Path) error
SignedMessage() (*pb.SignedMessage, error)
}
@ -93,19 +91,19 @@ func clientConnection(serverAddr string, opts ...grpc.DialOption) (pb.PointerDBC
}
// Put is the interface to make a PUT request, needs Pointer and APIKey
func (pdb *PointerDB) Put(ctx context.Context, path p.Path, pointer *pb.Pointer) (err error) {
func (pdb *PointerDB) Put(ctx context.Context, path storj.Path, pointer *pb.Pointer) (err error) {
defer mon.Task()(&ctx)(&err)
_, err = pdb.grpcClient.Put(ctx, &pb.PutRequest{Path: path.String(), Pointer: pointer})
_, err = pdb.grpcClient.Put(ctx, &pb.PutRequest{Path: path, Pointer: pointer})
return err
}
// Get is the interface to make a GET request, needs PATH and APIKey
func (pdb *PointerDB) Get(ctx context.Context, path p.Path) (pointer *pb.Pointer, err error) {
func (pdb *PointerDB) Get(ctx context.Context, path storj.Path) (pointer *pb.Pointer, err error) {
defer mon.Task()(&ctx)(&err)
res, err := pdb.grpcClient.Get(ctx, &pb.GetRequest{Path: path.String()})
res, err := pdb.grpcClient.Get(ctx, &pb.GetRequest{Path: path})
if err != nil {
if status.Code(err) == codes.NotFound {
return nil, storage.ErrKeyNotFound.Wrap(err)
@ -117,15 +115,13 @@ func (pdb *PointerDB) Get(ctx context.Context, path p.Path) (pointer *pb.Pointer
}
// List is the interface to make a LIST request, needs StartingPathKey, Limit, and APIKey
func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore p.Path,
recursive bool, limit int, metaFlags uint32) (
items []ListItem, more bool, err error) {
func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)
res, err := pdb.grpcClient.List(ctx, &pb.ListRequest{
Prefix: prefix.String(),
StartAfter: startAfter.String(),
EndBefore: endBefore.String(),
Prefix: prefix,
StartAfter: startAfter,
EndBefore: endBefore,
Recursive: recursive,
Limit: int32(limit),
MetaFlags: metaFlags,
@ -138,7 +134,7 @@ func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore p.
items = make([]ListItem, len(list))
for i, itm := range list {
items[i] = ListItem{
Path: p.New(itm.GetPath()),
Path: itm.GetPath(),
Pointer: itm.GetPointer(),
IsPrefix: itm.IsPrefix,
}
@ -148,10 +144,10 @@ func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore p.
}
// Delete is the interface to make a Delete request, needs Path and APIKey
func (pdb *PointerDB) Delete(ctx context.Context, path p.Path) (err error) {
func (pdb *PointerDB) Delete(ctx context.Context, path storj.Path) (err error) {
defer mon.Task()(&ctx)(&err)
_, err = pdb.grpcClient.Delete(ctx, &pb.DeleteRequest{Path: path.String()})
_, err = pdb.grpcClient.Delete(ctx, &pb.DeleteRequest{Path: path})
return err
}

60
pkg/pointerdb/pdbclient/client_test.go
View File

@ -25,10 +25,10 @@ import (
"google.golang.org/grpc/peer"
"storj.io/storj/pkg/auth"
p "storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/provider"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storj"
)
const (
@ -54,7 +54,7 @@ func TestNewPointerDBClient(t *testing.T) {
assert.NotNil(t, pdb.grpcClient)
}
func makePointer(path p.Path) pb.PutRequest {
func makePointer(path storj.Path) pb.PutRequest {
// rps is an example slice of RemotePieces to add to this
// REMOTE pointer type.
var rps []*pb.RemotePiece
@ -63,7 +63,7 @@ func makePointer(path p.Path) pb.PutRequest {
NodeId: "testId",
})
pr := pb.PutRequest{
Path: path.String(),
Path: path,
Pointer: &pb.Pointer{
Type: pb.Pointer_REMOTE,
Remote: &pb.RemoteSegment{
@ -89,15 +89,15 @@ func TestPut(t *testing.T) {
for i, tt := range []struct {
APIKey []byte
path p.Path
path storj.Path
err error
errString string
}{
{[]byte("abc123"), p.New("file1/file2"), nil, ""},
{[]byte("wrong key"), p.New("file1/file2"), ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), p.New(""), ErrNoFileGiven, noPathGiven},
{[]byte("wrong key"), p.New(""), ErrUnauthenticated, unauthenticated},
{[]byte(""), p.New(""), ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), "file1/file2", nil, ""},
{[]byte("wrong key"), "file1/file2", ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), "", ErrNoFileGiven, noPathGiven},
{[]byte("wrong key"), "", ErrUnauthenticated, unauthenticated},
{[]byte(""), "", ErrUnauthenticated, unauthenticated},
} {
ctx := context.Background()
ctx = auth.WithAPIKey(ctx, tt.APIKey)
@ -127,21 +127,21 @@ func TestGet(t *testing.T) {
for i, tt := range []struct {
APIKey []byte
path p.Path
path storj.Path
err error
errString string
}{
{[]byte("wrong key"), p.New("file1/file2"), ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), p.New(""), ErrNoFileGiven, noPathGiven},
{[]byte("wrong key"), p.New(""), ErrUnauthenticated, unauthenticated},
{[]byte(""), p.New(""), ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), p.New("file1/file2"), nil, ""},
{[]byte("wrong key"), "file1/file2", ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), "", ErrNoFileGiven, noPathGiven},
{[]byte("wrong key"), "", ErrUnauthenticated, unauthenticated},
{[]byte(""), "", ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), "file1/file2", nil, ""},
} {
ctx := context.Background()
ctx = auth.WithAPIKey(ctx, tt.APIKey)
getPointer := makePointer(tt.path)
getRequest := pb.GetRequest{Path: tt.path.String()}
getRequest := pb.GetRequest{Path: tt.path}
data, err := proto.Marshal(getPointer.Pointer)
if err != nil {
@ -239,8 +239,7 @@ func TestList(t *testing.T) {
gc.EXPECT().List(gomock.Any(), &listRequest).Return(&listResponse, tt.err)
items, more, err := pdb.List(ctx, p.New(tt.prefix), p.New(tt.startAfter),
p.New(tt.endBefore), tt.recursive, tt.limit, tt.metaFlags)
items, more, err := pdb.List(ctx, tt.prefix, tt.startAfter, tt.endBefore, tt.recursive, tt.limit, tt.metaFlags)
if err != nil {
assert.EqualError(t, err, tt.errString, errTag)
@ -253,13 +252,10 @@ func TestList(t *testing.T) {
assert.Equal(t, len(tt.items), len(items))
for i := 0; i < len(items); i++ {
assert.Equal(t, tt.items[i].GetPath(), items[i].Path.String())
assert.Equal(t, tt.items[i].GetPointer().GetSize(),
items[i].Pointer.GetSize())
assert.Equal(t, tt.items[i].GetPointer().GetCreationDate(),
items[i].Pointer.GetCreationDate())
assert.Equal(t, tt.items[i].GetPointer().GetExpirationDate(),
items[i].Pointer.GetExpirationDate())
assert.Equal(t, tt.items[i].GetPath(), items[i].Path)
assert.Equal(t, tt.items[i].GetPointer().GetSize(), items[i].Pointer.GetSize())
assert.Equal(t, tt.items[i].GetPointer().GetCreationDate(), items[i].Pointer.GetCreationDate())
assert.Equal(t, tt.items[i].GetPointer().GetExpirationDate(), items[i].Pointer.GetExpirationDate())
}
}
}
@ -271,20 +267,20 @@ func TestDelete(t *testing.T) {
for i, tt := range []struct {
APIKey []byte
path p.Path
path storj.Path
err error
errString string
}{
{[]byte("wrong key"), p.New("file1/file2"), ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), p.New(""), ErrNoFileGiven, noPathGiven},
{[]byte("wrong key"), p.New(""), ErrUnauthenticated, unauthenticated},
{[]byte(""), p.New(""), ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), p.New("file1/file2"), nil, ""},
{[]byte("wrong key"), "file1/file2", ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), "", ErrNoFileGiven, noPathGiven},
{[]byte("wrong key"), "", ErrUnauthenticated, unauthenticated},
{[]byte(""), "", ErrUnauthenticated, unauthenticated},
{[]byte("abc123"), "file1/file2", nil, ""},
} {
ctx := context.Background()
ctx = auth.WithAPIKey(ctx, tt.APIKey)
deleteRequest := pb.DeleteRequest{Path: tt.path.String()}
deleteRequest := pb.DeleteRequest{Path: tt.path}
errTag := fmt.Sprintf("Test case #%d", i)
gc := NewMockPointerDBClient(ctrl)

13
pkg/pointerdb/pdbclient/mocks/mock_client.go
View File

@ -6,11 +6,8 @@ package mock_pointerdb
import (
context "context"
"reflect"
gomock "github.com/golang/mock/gomock"
paths "storj.io/storj/pkg/paths"
reflect "reflect"
pb "storj.io/storj/pkg/pb"
pdbclient "storj.io/storj/pkg/pointerdb/pdbclient"
)
@ -39,7 +36,7 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder {
}
// Delete mocks base method
func (m *MockClient) Delete(arg0 context.Context, arg1 paths.Path) error {
func (m *MockClient) Delete(arg0 context.Context, arg1 string) error {
ret := m.ctrl.Call(m, "Delete", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
@ -51,7 +48,7 @@ func (mr *MockClientMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call {
}
// Get mocks base method
func (m *MockClient) Get(arg0 context.Context, arg1 paths.Path) (*pb.Pointer, error) {
func (m *MockClient) Get(arg0 context.Context, arg1 string) (*pb.Pointer, error) {
ret := m.ctrl.Call(m, "Get", arg0, arg1)
ret0, _ := ret[0].(*pb.Pointer)
ret1, _ := ret[1].(error)
@ -64,7 +61,7 @@ func (mr *MockClientMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call {
}
// List mocks base method
func (m *MockClient) List(arg0 context.Context, arg1, arg2, arg3 paths.Path, arg4 bool, arg5 int, arg6 uint32) ([]pdbclient.ListItem, bool, error) {
func (m *MockClient) List(arg0 context.Context, arg1, arg2, arg3 string, arg4 bool, arg5 int, arg6 uint32) ([]pdbclient.ListItem, bool, error) {
ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3, arg4, arg5, arg6)
ret0, _ := ret[0].([]pdbclient.ListItem)
ret1, _ := ret[1].(bool)
@ -78,7 +75,7 @@ func (mr *MockClientMockRecorder) List(arg0, arg1, arg2, arg3, arg4, arg5, arg6
}
// Put mocks base method
func (m *MockClient) Put(arg0 context.Context, arg1 paths.Path, arg2 *pb.Pointer) error {
func (m *MockClient) Put(arg0 context.Context, arg1 string, arg2 *pb.Pointer) error {
ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0

19
pkg/pointerdb/pointerdb_test.go
View File

@ -18,7 +18,6 @@ import (
"google.golang.org/grpc/status"
"storj.io/storj/pkg/auth"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/storage"
@ -144,10 +143,6 @@ func TestServiceList(t *testing.T) {
db := teststore.New()
server := Server{DB: db, logger: zap.NewNop()}
key := func(s string) storage.Key {
return storage.Key(paths.New(s).Bytes())
}
pointer := &pb.Pointer{}
pointer.CreationDate = ptypes.TimestampNow()
@ -158,13 +153,13 @@ func TestServiceList(t *testing.T) {
pointerValue := storage.Value(pointerBytes)
err = storage.PutAll(db, []storage.ListItem{
{Key: key("sample.😶"), Value: pointerValue},
{Key: key("müsic"), Value: pointerValue},
{Key: key("müsic/söng1.mp3"), Value: pointerValue},
{Key: key("müsic/söng2.mp3"), Value: pointerValue},
{Key: key("müsic/album/söng3.mp3"), Value: pointerValue},
{Key: key("müsic/söng4.mp3"), Value: pointerValue},
{Key: key("ビデオ/movie.mkv"), Value: pointerValue},
{Key: storage.Key("sample.😶"), Value: pointerValue},
{Key: storage.Key("müsic"), Value: pointerValue},
{Key: storage.Key("müsic/söng1.mp3"), Value: pointerValue},
{Key: storage.Key("müsic/söng2.mp3"), Value: pointerValue},
{Key: storage.Key("müsic/album/söng3.mp3"), Value: pointerValue},
{Key: storage.Key("müsic/söng4.mp3"), Value: pointerValue},
{Key: storage.Key("ビデオ/movie.mkv"), Value: pointerValue},
}...)
if err != nil {
t.Fatal(err)

27
pkg/storage/buckets/prefixed.go
View File

@ -8,9 +8,9 @@ import (
"io"
"time"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/objects"
"storj.io/storj/pkg/storj"
)
type prefixedObjStore struct {
@ -18,55 +18,50 @@ type prefixedObjStore struct {
prefix string
}
func (o *prefixedObjStore) Meta(ctx context.Context, path paths.Path) (meta objects.Meta,
err error) {
func (o *prefixedObjStore) Meta(ctx context.Context, path storj.Path) (meta objects.Meta, err error) {
defer mon.Task()(&ctx)(&err)
if len(path) == 0 {
return objects.Meta{}, objects.NoPathError.New("")
}
m, err := o.o.Meta(ctx, path.Prepend(o.prefix))
m, err := o.o.Meta(ctx, storj.JoinPaths(o.prefix, path))
return m, err
}
func (o *prefixedObjStore) Get(ctx context.Context, path paths.Path) (
rr ranger.Ranger, meta objects.Meta, err error) {
func (o *prefixedObjStore) Get(ctx context.Context, path storj.Path) (rr ranger.Ranger, meta objects.Meta, err error) {
defer mon.Task()(&ctx)(&err)
if len(path) == 0 {
return nil, objects.Meta{}, objects.NoPathError.New("")
}
rr, m, err := o.o.Get(ctx, path.Prepend(o.prefix))
rr, m, err := o.o.Get(ctx, storj.JoinPaths(o.prefix, path))
return rr, m, err
}
func (o *prefixedObjStore) Put(ctx context.Context, path paths.Path, data io.Reader,
metadata objects.SerializableMeta, expiration time.Time) (meta objects.Meta, err error) {
func (o *prefixedObjStore) Put(ctx context.Context, path storj.Path, data io.Reader, metadata objects.SerializableMeta, expiration time.Time) (meta objects.Meta, err error) {
defer mon.Task()(&ctx)(&err)
if len(path) == 0 {
return objects.Meta{}, objects.NoPathError.New("")
}
m, err := o.o.Put(ctx, path.Prepend(o.prefix), data, metadata, expiration)
m, err := o.o.Put(ctx, storj.JoinPaths(o.prefix, path), data, metadata, expiration)
return m, err
}
func (o *prefixedObjStore) Delete(ctx context.Context, path paths.Path) (err error) {
func (o *prefixedObjStore) Delete(ctx context.Context, path storj.Path) (err error) {
defer mon.Task()(&ctx)(&err)
if len(path) == 0 {
return objects.NoPathError.New("")
}
return o.o.Delete(ctx, path.Prepend(o.prefix))
return o.o.Delete(ctx, storj.JoinPaths(o.prefix, path))
}
func (o *prefixedObjStore) List(ctx context.Context, prefix, startAfter,
endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (
items []objects.ListItem, more bool, err error) {
func (o *prefixedObjStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []objects.ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)
return o.o.List(ctx, prefix.Prepend(o.prefix), startAfter, endBefore, recursive, limit, metaFlags)
return o.o.List(ctx, storj.JoinPaths(o.prefix, prefix), startAfter, endBefore, recursive, limit, metaFlags)
}

14
pkg/storage/buckets/store.go
View File

@ -12,7 +12,6 @@ import (
"github.com/zeebo/errs"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storage/objects"
"storj.io/storj/storage"
@ -81,8 +80,7 @@ func (b *BucketStore) Get(ctx context.Context, bucket string) (meta Meta, err er
return Meta{}, NoBucketError.New("")
}
p := paths.New(bucket)
objMeta, err := b.o.Meta(ctx, p)
objMeta, err := b.o.Meta(ctx, bucket)
if err != nil {
return Meta{}, err
}
@ -97,10 +95,9 @@ func (b *BucketStore) Put(ctx context.Context, bucket string) (meta Meta, err er
return Meta{}, NoBucketError.New("")
}
p := paths.New(bucket)
r := bytes.NewReader(nil)
var exp time.Time
m, err := b.o.Put(ctx, p, r, objects.SerializableMeta{}, exp)
m, err := b.o.Put(ctx, bucket, r, objects.SerializableMeta{}, exp)
if err != nil {
return Meta{}, err
}
@ -115,15 +112,14 @@ func (b *BucketStore) Delete(ctx context.Context, bucket string) (err error) {
return NoBucketError.New("")
}
p := paths.New(bucket)
return b.o.Delete(ctx, p)
return b.o.Delete(ctx, bucket)
}
// List calls objects store List
func (b *BucketStore) List(ctx context.Context, startAfter, endBefore string, limit int) (items []ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)
objItems, more, err := b.o.List(ctx, nil, paths.New(startAfter), paths.New(endBefore), false, limit, meta.Modified)
objItems, more, err := b.o.List(ctx, "", startAfter, endBefore, false, limit, meta.Modified)
if err != nil {
return items, more, err
}
@ -134,7 +130,7 @@ func (b *BucketStore) List(ctx context.Context, startAfter, endBefore string, li
continue
}
items = append(items, ListItem{
Bucket: itm.Path.String(),
Bucket: itm.Path,
Meta: convertMeta(itm.Meta),
})
}

34
pkg/storage/objects/store.go
View File

@ -8,14 +8,14 @@ import (
"io"
"time"
"github.com/gogo/protobuf/proto"
"github.com/golang/protobuf/proto"
"github.com/zeebo/errs"
"go.uber.org/zap"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/streams"
"storj.io/storj/pkg/storj"
)
var mon = monkit.Package()
@ -34,22 +34,18 @@ type Meta struct {
// ListItem is a single item in a listing
type ListItem struct {
Path paths.Path
Path storj.Path
Meta Meta
IsPrefix bool
}
// Store for objects
type Store interface {
Meta(ctx context.Context, path paths.Path) (meta Meta, err error)
Get(ctx context.Context, path paths.Path) (rr ranger.Ranger,
meta Meta, err error)
Put(ctx context.Context, path paths.Path, data io.Reader,
metadata SerializableMeta, expiration time.Time) (meta Meta, err error)
Delete(ctx context.Context, path paths.Path) (err error)
List(ctx context.Context, prefix, startAfter, endBefore paths.Path,
recursive bool, limit int, metaFlags uint32) (items []ListItem,
more bool, err error)
Meta(ctx context.Context, path storj.Path) (meta Meta, err error)
Get(ctx context.Context, path storj.Path) (rr ranger.Ranger, meta Meta, err error)
Put(ctx context.Context, path storj.Path, data io.Reader, metadata SerializableMeta, expiration time.Time) (meta Meta, err error)
Delete(ctx context.Context, path storj.Path) (err error)
List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error)
}
type objStore struct {
@ -61,8 +57,7 @@ func NewStore(store streams.Store) Store {
return &objStore{s: store}
}
func (o *objStore) Meta(ctx context.Context, path paths.Path) (meta Meta,
err error) {
func (o *objStore) Meta(ctx context.Context, path storj.Path) (meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
if len(path) == 0 {
@ -73,7 +68,7 @@ func (o *objStore) Meta(ctx context.Context, path paths.Path) (meta Meta,
return convertMeta(m), err
}
func (o *objStore) Get(ctx context.Context, path paths.Path) (
func (o *objStore) Get(ctx context.Context, path storj.Path) (
rr ranger.Ranger, meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
@ -85,8 +80,7 @@ func (o *objStore) Get(ctx context.Context, path paths.Path) (
return rr, convertMeta(m), err
}
func (o *objStore) Put(ctx context.Context, path paths.Path, data io.Reader,
metadata SerializableMeta, expiration time.Time) (meta Meta, err error) {
func (o *objStore) Put(ctx context.Context, path storj.Path, data io.Reader, metadata SerializableMeta, expiration time.Time) (meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
if len(path) == 0 {
@ -96,7 +90,6 @@ func (o *objStore) Put(ctx context.Context, path paths.Path, data io.Reader,
// TODO(kaloyan): autodetect content type
// if metadata.GetContentType() == "" {}
// TODO(kaloyan): encrypt metadata.UserDefined before serializing
b, err := proto.Marshal(&metadata)
if err != nil {
return Meta{}, err
@ -105,7 +98,7 @@ func (o *objStore) Put(ctx context.Context, path paths.Path, data io.Reader,
return convertMeta(m), err
}
func (o *objStore) Delete(ctx context.Context, path paths.Path) (err error) {
func (o *objStore) Delete(ctx context.Context, path storj.Path) (err error) {
defer mon.Task()(&ctx)(&err)
if len(path) == 0 {
@ -115,8 +108,7 @@ func (o *objStore) Delete(ctx context.Context, path paths.Path) (err error) {
return o.s.Delete(ctx, path)
}
func (o *objStore) List(ctx context.Context, prefix, startAfter,
endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (
func (o *objStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (
items []ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)

11
pkg/storage/segments/mock_store.go
View File

@ -12,7 +12,6 @@ import (
gomock "github.com/golang/mock/gomock"
paths "storj.io/storj/pkg/paths"
ranger "storj.io/storj/pkg/ranger"
)
@ -40,7 +39,7 @@ func (m *MockStore) EXPECT() *MockStoreMockRecorder {
}
// Delete mocks base method
func (m *MockStore) Delete(arg0 context.Context, arg1 paths.Path) error {
func (m *MockStore) Delete(arg0 context.Context, arg1 string) error {
ret := m.ctrl.Call(m, "Delete", arg0, arg1)
ret0, _ := ret[0].(error)
return ret0
@ -52,7 +51,7 @@ func (mr *MockStoreMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call {
}
// Get mocks base method
func (m *MockStore) Get(arg0 context.Context, arg1 paths.Path) (ranger.Ranger, Meta, error) {
func (m *MockStore) Get(arg0 context.Context, arg1 string) (ranger.Ranger, Meta, error) {
ret := m.ctrl.Call(m, "Get", arg0, arg1)
ret0, _ := ret[0].(ranger.Ranger)
ret1, _ := ret[1].(Meta)
@ -66,7 +65,7 @@ func (mr *MockStoreMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call {
}
// List mocks base method
func (m *MockStore) List(arg0 context.Context, arg1, arg2, arg3 paths.Path, arg4 bool, arg5 int, arg6 uint32) ([]ListItem, bool, error) {
func (m *MockStore) List(arg0 context.Context, arg1, arg2, arg3 string, arg4 bool, arg5 int, arg6 uint32) ([]ListItem, bool, error) {
ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3, arg4, arg5, arg6)
ret0, _ := ret[0].([]ListItem)
ret1, _ := ret[1].(bool)
@ -80,7 +79,7 @@ func (mr *MockStoreMockRecorder) List(arg0, arg1, arg2, arg3, arg4, arg5, arg6 i
}
// Meta mocks base method
func (m *MockStore) Meta(arg0 context.Context, arg1 paths.Path) (Meta, error) {
func (m *MockStore) Meta(arg0 context.Context, arg1 string) (Meta, error) {
ret := m.ctrl.Call(m, "Meta", arg0, arg1)
ret0, _ := ret[0].(Meta)
ret1, _ := ret[1].(error)
@ -93,7 +92,7 @@ func (mr *MockStoreMockRecorder) Meta(arg0, arg1 interface{}) *gomock.Call {
}
// Put mocks base method
func (m *MockStore) Put(arg0 context.Context, arg1 io.Reader, arg2 time.Time, arg3 func() (paths.Path, []byte, error)) (Meta, error) {
func (m *MockStore) Put(arg0 context.Context, arg1 io.Reader, arg2 time.Time, arg3 func() (string, []byte, error)) (Meta, error) {
ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2, arg3)
ret0, _ := ret[0].(Meta)
ret1, _ := ret[1].(error)

29
pkg/storage/segments/store.go
View File

@ -18,12 +18,12 @@ import (
"storj.io/storj/pkg/eestream"
"storj.io/storj/pkg/node"
"storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/piecestore/rpc/client"
"storj.io/storj/pkg/pointerdb/pdbclient"
"storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/ec"
"storj.io/storj/pkg/storj"
)
var (
@ -40,18 +40,18 @@ type Meta struct {
// ListItem is a single item in a listing
type ListItem struct {
Path paths.Path
Path storj.Path
Meta Meta
IsPrefix bool
}
// Store for segments
type Store interface {
Meta(ctx context.Context, path paths.Path) (meta Meta, err error)
Get(ctx context.Context, path paths.Path) (rr ranger.Ranger, meta Meta, err error)
Put(ctx context.Context, data io.Reader, expiration time.Time, segmentInfo func() (paths.Path, []byte, error)) (meta Meta, err error)
Delete(ctx context.Context, path paths.Path) (err error)
List(ctx context.Context, prefix, startAfter, endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error)
Meta(ctx context.Context, path storj.Path) (meta Meta, err error)
Get(ctx context.Context, path storj.Path) (rr ranger.Ranger, meta Meta, err error)
Put(ctx context.Context, data io.Reader, expiration time.Time, segmentInfo func() (storj.Path, []byte, error)) (meta Meta, err error)
Delete(ctx context.Context, path storj.Path) (err error)
List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error)
}
type segmentStore struct {
@ -69,8 +69,7 @@ func NewSegmentStore(oc overlay.Client, ec ecclient.Client,
}
// Meta retrieves the metadata of the segment
func (s *segmentStore) Meta(ctx context.Context, path paths.Path) (meta Meta,
err error) {
func (s *segmentStore) Meta(ctx context.Context, path storj.Path) (meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
pr, err := s.pdb.Get(ctx, path)
@ -82,7 +81,7 @@ func (s *segmentStore) Meta(ctx context.Context, path paths.Path) (meta Meta,
}
// Put uploads a segment to an erasure code client
func (s *segmentStore) Put(ctx context.Context, data io.Reader, expiration time.Time, segmentInfo func() (paths.Path, []byte, error)) (meta Meta, err error) {
func (s *segmentStore) Put(ctx context.Context, data io.Reader, expiration time.Time, segmentInfo func() (storj.Path, []byte, error)) (meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
exp, err := ptypes.TimestampProto(expiration)
@ -96,7 +95,7 @@ func (s *segmentStore) Put(ctx context.Context, data io.Reader, expiration time.
return Meta{}, err
}
var path paths.Path
var path storj.Path
var pointer *pb.Pointer
if !remoteSized {
p, metadata, err := segmentInfo()
@ -193,7 +192,7 @@ func (s *segmentStore) makeRemotePointer(nodes []*pb.Node, pieceID client.PieceI
}
// Get retrieves a segment using erasure code, overlay, and pointerdb clients
func (s *segmentStore) Get(ctx context.Context, path paths.Path) (
func (s *segmentStore) Get(ctx context.Context, path storj.Path) (
rr ranger.Ranger, meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
@ -240,7 +239,7 @@ func makeErasureScheme(rs *pb.RedundancyScheme) (eestream.ErasureScheme, error)
}
// Delete tells piece stores to delete a segment and deletes pointer from pointerdb
func (s *segmentStore) Delete(ctx context.Context, path paths.Path) (err error) {
func (s *segmentStore) Delete(ctx context.Context, path storj.Path) (err error) {
defer mon.Task()(&ctx)(&err)
pr, err := s.pdb.Get(ctx, path)
@ -293,9 +292,7 @@ func (s *segmentStore) lookupNodes(ctx context.Context, seg *pb.RemoteSegment) (
}
// List retrieves paths to segments and their metadata stored in the pointerdb
func (s *segmentStore) List(ctx context.Context, prefix, startAfter,
endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (
items []ListItem, more bool, err error) {
func (s *segmentStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)
pdbItems, more, err := s.pdb.List(ctx, prefix, startAfter, endBefore, recursive, limit, metaFlags)

57
pkg/storage/segments/store_test.go
View File

@ -16,11 +16,12 @@ import (
"storj.io/storj/pkg/eestream"
mock_eestream "storj.io/storj/pkg/eestream/mocks"
mock_overlay "storj.io/storj/pkg/overlay/mocks"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
pdb "storj.io/storj/pkg/pointerdb/pdbclient"
mock_pointerdb "storj.io/storj/pkg/pointerdb/pdbclient/mocks"
mock_ecclient "storj.io/storj/pkg/storage/ec/mocks"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storj"
)
var (
@ -67,8 +68,6 @@ func TestSegmentStoreMeta(t *testing.T) {
}{
{"path/1/2/3", &pb.Pointer{CreationDate: pExp, ExpirationDate: pExp}, Meta{Modified: mExp, Expiration: mExp}},
} {
p := paths.New(tt.pathInput)
calls := []*gomock.Call{
mockPDB.EXPECT().Get(
gomock.Any(), gomock.Any(),
@ -76,7 +75,7 @@ func TestSegmentStoreMeta(t *testing.T) {
}
gomock.InOrder(calls...)
m, err := ss.Meta(ctx, p)
m, err := ss.Meta(ctx, tt.pathInput)
assert.NoError(t, err)
assert.Equal(t, m, tt.returnMeta)
}
@ -107,9 +106,6 @@ func TestSegmentStorePutRemote(t *testing.T) {
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
assert.NotNil(t, ss)
p := paths.New(tt.pathInput)
r := strings.NewReader(tt.readerContent)
calls := []*gomock.Call{
mockES.EXPECT().TotalCount().Return(1),
mockOC.EXPECT().Choose(
@ -133,8 +129,8 @@ func TestSegmentStorePutRemote(t *testing.T) {
}
gomock.InOrder(calls...)
_, err := ss.Put(ctx, r, tt.expiration, func() (paths.Path, []byte, error) {
return p, tt.mdInput, nil
_, err := ss.Put(ctx, strings.NewReader(tt.readerContent), tt.expiration, func() (storj.Path, []byte, error) {
return tt.pathInput, tt.mdInput, nil
})
assert.NoError(t, err, tt.name)
}
@ -165,9 +161,6 @@ func TestSegmentStorePutInline(t *testing.T) {
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
assert.NotNil(t, ss)
p := paths.New(tt.pathInput)
r := strings.NewReader(tt.readerContent)
calls := []*gomock.Call{
mockPDB.EXPECT().Put(
gomock.Any(), gomock.Any(), gomock.Any(),
@ -178,8 +171,8 @@ func TestSegmentStorePutInline(t *testing.T) {
}
gomock.InOrder(calls...)
_, err := ss.Put(ctx, r, tt.expiration, func() (paths.Path, []byte, error) {
return p, tt.mdInput, nil
_, err := ss.Put(ctx, strings.NewReader(tt.readerContent), tt.expiration, func() (storj.Path, []byte, error) {
return tt.pathInput, tt.mdInput, nil
})
assert.NoError(t, err, tt.name)
}
@ -214,8 +207,6 @@ func TestSegmentStoreGetInline(t *testing.T) {
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
assert.NotNil(t, ss)
p := paths.New(tt.pathInput)
calls := []*gomock.Call{
mockPDB.EXPECT().Get(
gomock.Any(), gomock.Any(),
@ -230,7 +221,7 @@ func TestSegmentStoreGetInline(t *testing.T) {
}
gomock.InOrder(calls...)
_, _, err := ss.Get(ctx, p)
_, _, err := ss.Get(ctx, tt.pathInput)
assert.NoError(t, err)
}
}
@ -263,8 +254,6 @@ func TestSegmentStoreGetRemote(t *testing.T) {
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
assert.NotNil(t, ss)
p := paths.New(tt.pathInput)
calls := []*gomock.Call{
mockPDB.EXPECT().Get(
gomock.Any(), gomock.Any(),
@ -294,7 +283,7 @@ func TestSegmentStoreGetRemote(t *testing.T) {
}
gomock.InOrder(calls...)
_, _, err := ss.Get(ctx, p)
_, _, err := ss.Get(ctx, tt.pathInput)
assert.NoError(t, err)
}
}
@ -328,8 +317,6 @@ func TestSegmentStoreDeleteInline(t *testing.T) {
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
assert.NotNil(t, ss)
p := paths.New(tt.pathInput)
calls := []*gomock.Call{
mockPDB.EXPECT().Get(
gomock.Any(), gomock.Any(),
@ -347,7 +334,7 @@ func TestSegmentStoreDeleteInline(t *testing.T) {
}
gomock.InOrder(calls...)
err := ss.Delete(ctx, p)
err := ss.Delete(ctx, tt.pathInput)
assert.NoError(t, err)
}
}
@ -380,8 +367,6 @@ func TestSegmentStoreDeleteRemote(t *testing.T) {
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
assert.NotNil(t, ss)
p := paths.New(tt.pathInput)
calls := []*gomock.Call{
mockPDB.EXPECT().Get(
gomock.Any(), gomock.Any(),
@ -414,7 +399,7 @@ func TestSegmentStoreDeleteRemote(t *testing.T) {
}
gomock.InOrder(calls...)
err := ss.Delete(ctx, p)
err := ss.Delete(ctx, tt.pathInput)
assert.NoError(t, err)
}
}
@ -424,12 +409,12 @@ func TestSegmentStoreList(t *testing.T) {
defer ctrl.Finish()
for _, tt := range []struct {
prefixInput string
startAfterInput string
thresholdSize int
itemPath string
inlineContent []byte
metadata []byte
prefix string
startAfter string
thresholdSize int
itemPath string
inlineContent []byte
metadata []byte
}{
{"bucket1", "s0/path/1", 10, "s0/path/1", []byte("inline"), []byte("metadata")},
} {
@ -444,10 +429,6 @@ func TestSegmentStoreList(t *testing.T) {
ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize}
assert.NotNil(t, ss)
prefix := paths.New(tt.prefixInput)
startAfter := paths.New(tt.startAfterInput)
listedPath := paths.New(tt.itemPath)
ti := time.Unix(0, 0).UTC()
someTime, err := ptypes.TimestampProto(ti)
assert.NoError(t, err)
@ -458,7 +439,7 @@ func TestSegmentStoreList(t *testing.T) {
gomock.Any(), gomock.Any(), gomock.Any(),
).Return([]pdb.ListItem{
{
Path: listedPath,
Path: tt.itemPath,
Pointer: &pb.Pointer{
Type: pb.Pointer_INLINE,
InlineSegment: tt.inlineContent,
@ -472,7 +453,7 @@ func TestSegmentStoreList(t *testing.T) {
}
gomock.InOrder(calls...)
_, _, err = ss.List(ctx, prefix, startAfter, nil, false, 10, uint32(1))
_, _, err = ss.List(ctx, tt.prefix, tt.startAfter, "", false, 10, meta.Modified)
assert.NoError(t, err)
}
}

150
pkg/storage/streams/store.go
View File

@ -10,18 +10,18 @@ import (
"fmt"
"io"
"io/ioutil"
"strings"
"time"
proto "github.com/gogo/protobuf/proto"
"github.com/golang/protobuf/proto"
"github.com/zeebo/errs"
"go.uber.org/zap"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/eestream"
"storj.io/storj/pkg/encryption"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
ranger "storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storage/segments"
"storj.io/storj/pkg/storj"
@ -56,28 +56,28 @@ func convertMeta(lastSegmentMeta segments.Meta) (Meta, error) {
// Store interface methods for streams to satisfy to be a store
type Store interface {
Meta(ctx context.Context, path paths.Path) (Meta, error)
Get(ctx context.Context, path paths.Path) (ranger.Ranger, Meta, error)
Put(ctx context.Context, path paths.Path, data io.Reader, metadata []byte, expiration time.Time) (Meta, error)
Delete(ctx context.Context, path paths.Path) error
List(ctx context.Context, prefix, startAfter, endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error)
Meta(ctx context.Context, path storj.Path) (Meta, error)
Get(ctx context.Context, path storj.Path) (ranger.Ranger, Meta, error)
Put(ctx context.Context, path storj.Path, data io.Reader, metadata []byte, expiration time.Time) (Meta, error)
Delete(ctx context.Context, path storj.Path) error
List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error)
}
// streamStore is a store for streams
type streamStore struct {
segments segments.Store
segmentSize int64
rootKey []byte
rootKey *storj.Key
encBlockSize int
cipher storj.Cipher
}
// NewStreamStore stuff
func NewStreamStore(segments segments.Store, segmentSize int64, rootKey string, encBlockSize int, cipher storj.Cipher) (Store, error) {
func NewStreamStore(segments segments.Store, segmentSize int64, rootKey *storj.Key, encBlockSize int, cipher storj.Cipher) (Store, error) {
if segmentSize <= 0 {
return nil, errs.New("segment size must be larger than 0")
}
if rootKey == "" {
if rootKey == nil {
return nil, errs.New("encryption key must not be empty")
}
if encBlockSize <= 0 {
@ -87,7 +87,7 @@ func NewStreamStore(segments segments.Store, segmentSize int64, rootKey string,
return &streamStore{
segments: segments,
segmentSize: segmentSize,
rootKey: []byte(rootKey),
rootKey: rootKey,
encBlockSize: encBlockSize,
cipher: cipher,
}, nil
@ -97,7 +97,7 @@ func NewStreamStore(segments segments.Store, segmentSize int64, rootKey string,
// store the first piece at s0/<path>, second piece at s1/<path>, and the
// *last* piece at l/<path>. Store the given metadata, along with the number
// of segments, in a new protobuf, in the metadata of l/<path>.
func (s *streamStore) Put(ctx context.Context, path paths.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, err error) {
func (s *streamStore) Put(ctx context.Context, path storj.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, err error) {
defer mon.Task()(&ctx)(&err)
// previously file uploaded?
err = s.Delete(ctx, path)
@ -115,7 +115,7 @@ func (s *streamStore) Put(ctx context.Context, path paths.Path, data io.Reader,
return m, err
}
func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, lastSegment int64, err error) {
func (s *streamStore) upload(ctx context.Context, path storj.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, lastSegment int64, err error) {
defer mon.Task()(&ctx)(&err)
var currentSegment int64
@ -130,7 +130,7 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade
}
}()
derivedKey, err := path.DeriveContentKey(s.rootKey)
derivedKey, err := encryption.DeriveContentKey(path, s.rootKey)
if err != nil {
return Meta{}, currentSegment, err
}
@ -166,7 +166,7 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade
return Meta{}, currentSegment, err
}
encryptedKey, err := encryption.EncryptKey(&contentKey, s.cipher, (*storj.Key)(derivedKey), &keyNonce)
encryptedKey, err := encryption.EncryptKey(&contentKey, s.cipher, derivedKey, &keyNonce)
if err != nil {
return Meta{}, currentSegment, err
}
@ -194,10 +194,10 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade
transformedReader = bytes.NewReader(cipherData)
}
putMeta, err = s.segments.Put(ctx, transformedReader, expiration, func() (paths.Path, []byte, error) {
putMeta, err = s.segments.Put(ctx, transformedReader, expiration, func() (storj.Path, []byte, error) {
encPath, err := encryptAfterBucket(path, s.rootKey)
if err != nil {
return nil, nil, err
return "", nil, err
}
if !eofReader.isEOF() {
@ -212,13 +212,13 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade
KeyNonce: keyNonce[:],
})
if err != nil {
return nil, nil, err
return "", nil, err
}
return segmentPath, segmentMeta, nil
}
lastSegmentPath := encPath.Prepend("l")
lastSegmentPath := storj.JoinPaths("l", encPath)
streamInfo, err := proto.Marshal(&pb.StreamInfo{
NumberOfSegments: currentSegment + 1,
@ -227,13 +227,13 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade
Metadata: metadata,
})
if err != nil {
return nil, nil, err
return "", nil, err
}
// encrypt metadata with the content encryption key and zero nonce
encryptedStreamInfo, err := encryption.Encrypt(streamInfo, s.cipher, &contentKey, &storj.Nonce{})
if err != nil {
return nil, nil, err
return "", nil, err
}
streamMeta := pb.StreamMeta{
@ -251,7 +251,7 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade
lastSegmentMeta, err := proto.Marshal(&streamMeta)
if err != nil {
return nil, nil, err
return "", nil, err
}
return lastSegmentPath, lastSegmentMeta, nil
@ -279,14 +279,14 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade
}
// getSegmentPath returns the unique path for a particular segment
func getSegmentPath(p paths.Path, segNum int64) paths.Path {
return p.Prepend(fmt.Sprintf("s%d", segNum))
func getSegmentPath(path storj.Path, segNum int64) storj.Path {
return storj.JoinPaths(fmt.Sprintf("s%d", segNum), path)
}
// Get returns a ranger that knows what the overall size is (from l/<path>)
// and then returns the appropriate data from segments s0/<path>, s1/<path>,
// ..., l/<path>.
func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Ranger, meta Meta, err error) {
func (s *streamStore) Get(ctx context.Context, path storj.Path) (rr ranger.Ranger, meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
encPath, err := encryptAfterBucket(path, s.rootKey)
@ -294,7 +294,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range
return nil, Meta{}, err
}
lastSegmentRanger, lastSegmentMeta, err := s.segments.Get(ctx, encPath.Prepend("l"))
lastSegmentRanger, lastSegmentMeta, err := s.segments.Get(ctx, storj.JoinPaths("l", encPath))
if err != nil {
return nil, Meta{}, err
}
@ -316,7 +316,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range
return nil, Meta{}, err
}
derivedKey, err := path.DeriveContentKey(s.rootKey)
derivedKey, err := encryption.DeriveContentKey(path, s.rootKey)
if err != nil {
return nil, Meta{}, err
}
@ -334,7 +334,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range
segments: s.segments,
path: currentPath,
size: size,
derivedKey: (*storj.Key)(derivedKey),
derivedKey: derivedKey,
startingNonce: &contentNonce,
encBlockSize: int(streamMeta.EncryptionBlockSize),
cipher: storj.Cipher(streamMeta.EncryptionType),
@ -353,7 +353,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range
lastSegmentRanger,
stream.LastSegmentSize,
storj.Cipher(streamMeta.EncryptionType),
(*storj.Key)(derivedKey),
derivedKey,
encryptedKey,
keyNonce,
&contentNonce,
@ -376,7 +376,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range
}
// Meta implements Store.Meta
func (s *streamStore) Meta(ctx context.Context, path paths.Path) (meta Meta, err error) {
func (s *streamStore) Meta(ctx context.Context, path storj.Path) (meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
encPath, err := encryptAfterBucket(path, s.rootKey)
@ -384,7 +384,7 @@ func (s *streamStore) Meta(ctx context.Context, path paths.Path) (meta Meta, err
return Meta{}, err
}
lastSegmentMeta, err := s.segments.Meta(ctx, encPath.Prepend("l"))
lastSegmentMeta, err := s.segments.Meta(ctx, storj.JoinPaths("l", encPath))
if err != nil {
return Meta{}, err
}
@ -404,14 +404,14 @@ func (s *streamStore) Meta(ctx context.Context, path paths.Path) (meta Meta, err
}
// Delete all the segments, with the last one last
func (s *streamStore) Delete(ctx context.Context, path paths.Path) (err error) {
func (s *streamStore) Delete(ctx context.Context, path storj.Path) (err error) {
defer mon.Task()(&ctx)(&err)
encPath, err := encryptAfterBucket(path, s.rootKey)
if err != nil {
return err
}
lastSegmentMeta, err := s.segments.Meta(ctx, encPath.Prepend("l"))
lastSegmentMeta, err := s.segments.Meta(ctx, storj.JoinPaths("l", encPath))
if err != nil {
return err
}
@ -439,18 +439,18 @@ func (s *streamStore) Delete(ctx context.Context, path paths.Path) (err error) {
}
}
return s.segments.Delete(ctx, encPath.Prepend("l"))
return s.segments.Delete(ctx, storj.JoinPaths("l", encPath))
}
// ListItem is a single item in a listing
type ListItem struct {
Path paths.Path
Path storj.Path
Meta Meta
IsPrefix bool
}
// List all the paths inside l/, stripping off the l/ prefix
func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) {
func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)
if metaFlags&meta.Size != 0 {
@ -459,12 +459,14 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa
metaFlags |= meta.UserDefined
}
prefix = strings.TrimSuffix(prefix, "/")
encPrefix, err := encryptAfterBucket(prefix, s.rootKey)
if err != nil {
return nil, false, err
}
prefixKey, err := prefix.DeriveKey(s.rootKey, len(prefix))
prefixKey, err := encryption.DerivePathKey(prefix, s.rootKey, len(storj.SplitPath(prefix)))
if err != nil {
return nil, false, err
}
@ -479,7 +481,7 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa
return nil, false, err
}
segments, more, err := s.segments.List(ctx, encPrefix.Prepend("l"), encStartAfter, encEndBefore, recursive, limit, metaFlags)
segments, more, err := s.segments.List(ctx, storj.JoinPaths("l", encPrefix), encStartAfter, encEndBefore, recursive, limit, metaFlags)
if err != nil {
return nil, false, err
}
@ -491,7 +493,7 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa
return nil, false, err
}
streamInfo, err := decryptStreamInfo(ctx, item.Meta, path.Prepend(prefix...), s.rootKey)
streamInfo, err := decryptStreamInfo(ctx, item.Meta, storj.JoinPaths(prefix, path), s.rootKey)
if err != nil {
return nil, false, err
}
@ -509,25 +511,25 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa
}
// encryptMarker is a helper method for encrypting startAfter and endBefore markers
func (s *streamStore) encryptMarker(marker paths.Path, prefixKey []byte) (paths.Path, error) {
if bytes.Equal(s.rootKey, prefixKey) { // empty prefix
func (s *streamStore) encryptMarker(marker storj.Path, prefixKey *storj.Key) (storj.Path, error) {
if bytes.Equal(s.rootKey[:], prefixKey[:]) { // empty prefix
return encryptAfterBucket(marker, s.rootKey)
}
return marker.Encrypt(prefixKey)
return encryption.EncryptPath(marker, prefixKey)
}
// decryptMarker is a helper method for decrypting listed path markers
func (s *streamStore) decryptMarker(marker paths.Path, prefixKey []byte) (paths.Path, error) {
if bytes.Equal(s.rootKey, prefixKey) { // empty prefix
func (s *streamStore) decryptMarker(marker storj.Path, prefixKey *storj.Key) (storj.Path, error) {
if bytes.Equal(s.rootKey[:], prefixKey[:]) { // empty prefix
return decryptAfterBucket(marker, s.rootKey)
}
return marker.Decrypt(prefixKey)
return encryption.DecryptPath(marker, prefixKey)
}
type lazySegmentRanger struct {
ranger ranger.Ranger
segments segments.Store
path paths.Path
path storj.Path
size int64
derivedKey *storj.Key
startingNonce *storj.Nonce
@ -598,46 +600,46 @@ func decryptRanger(ctx context.Context, rr ranger.Ranger, decryptedSize int64, c
}
// encryptAfterBucket encrypts a path without encrypting its first element
func encryptAfterBucket(p paths.Path, key []byte) (encrypted paths.Path, err error) {
if len(p) <= 1 {
return p, nil
func encryptAfterBucket(path storj.Path, key *storj.Key) (encrypted storj.Path, err error) {
comps := storj.SplitPath(path)
if len(comps) <= 1 {
return path, nil
}
bucket := p[0]
toEncrypt := p[1:]
// derive a key from the bucket so the same path in different buckets is encrypted differently
bucketKey, err := p.DeriveKey(key, 1)
encrypted, err = encryption.EncryptPath(path, key)
if err != nil {
return nil, err
return "", err
}
encPath, err := toEncrypt.Encrypt(bucketKey)
if err != nil {
return nil, err
}
return encPath.Prepend(bucket), nil
// replace the first path component with the unencrypted bucket name
return storj.JoinPaths(comps[0], storj.JoinPaths(storj.SplitPath(encrypted)[1:]...)), nil
}
// decryptAfterBucket decrypts a path without modifying its first element
func decryptAfterBucket(p paths.Path, key []byte) (decrypted paths.Path, err error) {
if len(p) <= 1 {
return p, nil
func decryptAfterBucket(path storj.Path, key *storj.Key) (decrypted storj.Path, err error) {
comps := storj.SplitPath(path)
if len(comps) <= 1 {
return path, nil
}
bucket := p[0]
toDecrypt := p[1:]
bucketKey, err := p.DeriveKey(key, 1)
bucket := comps[0]
toDecrypt := storj.JoinPaths(comps[1:]...)
bucketKey, err := encryption.DerivePathKey(path, key, 1)
if err != nil {
return nil, err
return "", err
}
decPath, err := toDecrypt.Decrypt(bucketKey)
decPath, err := encryption.DecryptPath(toDecrypt, bucketKey)
if err != nil {
return nil, err
return "", err
}
return decPath.Prepend(bucket), nil
return storj.JoinPaths(bucket, decPath), nil
}
// CancelHandler handles clean up of segments on receiving CTRL+C
func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path paths.Path) {
func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path storj.Path) {
for i := int64(0); i < totalSegments; i++ {
encPath, err := encryptAfterBucket(path, s.rootKey)
if err != nil {
@ -663,21 +665,21 @@ func getEncryptedKeyAndNonce(m *pb.SegmentMeta) (storj.EncryptedPrivateKey, *sto
return m.EncryptedKey, &nonce
}
func decryptStreamInfo(ctx context.Context, item segments.Meta, path paths.Path, rootKey []byte) (streamInfo []byte, err error) {
func decryptStreamInfo(ctx context.Context, item segments.Meta, path storj.Path, rootKey *storj.Key) (streamInfo []byte, err error) {
streamMeta := pb.StreamMeta{}
err = proto.Unmarshal(item.Data, &streamMeta)
if err != nil {
return nil, err
}
derivedKey, err := path.DeriveContentKey(rootKey)
derivedKey, err := encryption.DeriveContentKey(path, rootKey)
if err != nil {
return nil, err
}
cipher := storj.Cipher(streamMeta.EncryptionType)
encryptedKey, keyNonce := getEncryptedKeyAndNonce(streamMeta.LastSegmentMeta)
contentKey, err := encryption.DecryptKey(encryptedKey, cipher, (*storj.Key)(derivedKey), keyNonce)
contentKey, err := encryption.DecryptKey(encryptedKey, cipher, derivedKey, keyNonce)
if err != nil {
return nil, err
}

28
pkg/storage/streams/store_test.go
View File

@ -11,14 +11,14 @@ import (
"testing"
"time"
proto "github.com/gogo/protobuf/proto"
"github.com/golang/mock/gomock"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
ranger "storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/segments"
"storj.io/storj/pkg/storj"
)
var (
@ -91,12 +91,12 @@ func TestStreamStoreMeta(t *testing.T) {
Meta(gomock.Any(), gomock.Any()).
Return(test.segmentMeta, test.segmentError)
streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0)
if err != nil {
t.Fatal(err)
}
meta, err := streamStore.Meta(ctx, paths.New(test.path))
meta, err := streamStore.Meta(ctx, test.path)
if err != nil {
t.Fatal(err)
}
@ -146,7 +146,7 @@ func TestStreamStorePut(t *testing.T) {
mockSegmentStore.EXPECT().
Put(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
Return(test.segmentMeta, test.segmentError).
Do(func(ctx context.Context, data io.Reader, expiration time.Time, info func() (paths.Path, []byte, error)) {
Do(func(ctx context.Context, data io.Reader, expiration time.Time, info func() (storj.Path, []byte, error)) {
for {
buf := make([]byte, 4)
_, err := data.Read(buf)
@ -163,12 +163,12 @@ func TestStreamStorePut(t *testing.T) {
Delete(gomock.Any(), gomock.Any()).
Return(test.segmentError)
streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0)
if err != nil {
t.Fatal(err)
}
meta, err := streamStore.Put(ctx, paths.New(test.path), test.data, test.metadata, test.expiration)
meta, err := streamStore.Put(ctx, test.path, test.data, test.metadata, test.expiration)
if err != nil {
t.Fatal(err)
}
@ -263,12 +263,12 @@ func TestStreamStoreGet(t *testing.T) {
gomock.InOrder(calls...)
streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0)
if err != nil {
t.Fatal(err)
}
ranger, meta, err := streamStore.Get(ctx, paths.New(test.path))
ranger, meta, err := streamStore.Get(ctx, test.path)
if err != nil {
t.Fatal(err)
}
@ -312,12 +312,12 @@ func TestStreamStoreDelete(t *testing.T) {
Delete(gomock.Any(), gomock.Any()).
Return(test.segmentError)
streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0)
if err != nil {
t.Fatal(err)
}
err = streamStore.Delete(ctx, paths.New(test.path))
err = streamStore.Delete(ctx, test.path)
if err != nil {
t.Fatal(err)
}
@ -356,12 +356,12 @@ func TestStreamStoreList(t *testing.T) {
List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
Return(test.segments, test.segmentMore, test.segmentError)
streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0)
if err != nil {
t.Fatal(err)
}
items, more, err := streamStore.List(ctx, paths.New(test.prefix), paths.New(test.startAfter), paths.New(test.endBefore), test.recursive, test.limit, test.metaFlags)
items, more, err := streamStore.List(ctx, test.prefix, test.startAfter, test.endBefore, test.recursive, test.limit, test.metaFlags)
if err != nil {
t.Fatal(err)
}

14
pkg/storj/path.go
View File

@ -3,5 +3,19 @@
package storj
import (
"strings"
)
// Path represents a object path
type Path = string
// SplitPath splits path into a slice of path components
func SplitPath(path Path) []string {
return strings.Split(path, "/")
}
// JoinPaths concatenates paths to a new single path
func JoinPaths(paths ...Path) Path {
return strings.Join(paths, "/")
}

54
pkg/storj/path_test.go Normal file
View File

@ -0,0 +1,54 @@
// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package storj
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
)
func TestSplitPath(t *testing.T) {
for i, tt := range []struct {
path string
comps []string
}{
{"", []string{""}},
{"/", []string{"", ""}},
{"//", []string{"", "", ""}},
{" ", []string{" "}},
{"a", []string{"a"}},
{"/a/", []string{"", "a", ""}},
{"a/b/c/d", []string{"a", "b", "c", "d"}},
{"///a//b////c/d///", []string{"", "", "", "a", "", "b", "", "", "", "c", "d", "", "", ""}},
} {
errTag := fmt.Sprintf("Test case #%d", i)
assert.Equal(t, tt.comps, SplitPath(tt.path), errTag)
}
}
func TestJoinPaths(t *testing.T) {
for i, tt := range []struct {
comps []string
path string
}{
{[]string{}, ""},
{[]string{""}, ""},
{[]string{"", ""}, "/"},
{[]string{"/", ""}, "//"},
{[]string{"/", "/"}, "///"},
{[]string{"", "", ""}, "//"},
{[]string{" "}, " "},
{[]string{"a"}, "a"},
{[]string{"", "a", ""}, "/a/"},
{[]string{"a", "b", "c", "d"}, "a/b/c/d"},
{[]string{"a/b", "c/d"}, "a/b/c/d"},
{[]string{"a/b/", "c/d"}, "a/b//c/d"},
{[]string{"", "", "", "a", "", "b", "", "", "", "c", "d", "", "", ""}, "///a//b////c/d///"},
} {
errTag := fmt.Sprintf("Test case #%d", i)
assert.Equal(t, tt.path, JoinPaths(tt.comps...), errTag)
}
}