diff --git a/cmd/uplink/cmd/cp.go b/cmd/uplink/cmd/cp.go index d1ab2a1f8..a45e7f3d0 100644 --- a/cmd/uplink/cmd/cp.go +++ b/cmd/uplink/cmd/cp.go @@ -16,7 +16,6 @@ import ( "github.com/spf13/cobra" "storj.io/storj/internal/fpath" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/process" "storj.io/storj/pkg/storage/buckets" "storj.io/storj/pkg/storage/objects" @@ -88,7 +87,7 @@ func upload(ctx context.Context, bs buckets.Store, src fpath.FPath, dst fpath.FP meta := objects.SerializableMeta{} expTime := time.Time{} - _, err = o.Put(ctx, paths.New(dst.Path()), r, meta, expTime) + _, err = o.Put(ctx, dst.Path(), r, meta, expTime) if err != nil { return err } @@ -117,7 +116,7 @@ func download(ctx context.Context, bs buckets.Store, src fpath.FPath, dst fpath. return err } - rr, _, err := o.Get(ctx, paths.New(src.Path())) + rr, _, err := o.Get(ctx, src.Path()) if err != nil { return err } @@ -181,7 +180,7 @@ func copy(ctx context.Context, bs buckets.Store, src fpath.FPath, dst fpath.FPat return err } - rr, _, err := o.Get(ctx, paths.New(src.Path())) + rr, _, err := o.Get(ctx, src.Path()) if err != nil { return err } @@ -214,7 +213,7 @@ func copy(ctx context.Context, bs buckets.Store, src fpath.FPath, dst fpath.FPat dst = dst.Join(src.Base()) } - _, err = o.Put(ctx, paths.New(dst.Path()), r, meta, expTime) + _, err = o.Put(ctx, dst.Path(), r, meta, expTime) if err != nil { return err } diff --git a/cmd/uplink/cmd/ls.go b/cmd/uplink/cmd/ls.go index e415d4538..44e0874d1 100644 --- a/cmd/uplink/cmd/ls.go +++ b/cmd/uplink/cmd/ls.go @@ -11,7 +11,6 @@ import ( "github.com/spf13/cobra" "storj.io/storj/internal/fpath" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/process" "storj.io/storj/pkg/storage/buckets" "storj.io/storj/pkg/storage/meta" @@ -45,7 +44,7 @@ func list(cmd *cobra.Command, args []string) error { } if src.IsLocal() { - return fmt.Errorf("No bucket specified. Please use format sj://bucket/") + return fmt.Errorf("No bucket specified, use format sj://bucket/") } return listFiles(ctx, bs, src, false) @@ -94,21 +93,21 @@ func listFiles(ctx context.Context, bs buckets.Store, prefix fpath.FPath, prepen return err } - startAfter := paths.New("") + startAfter := "" for { - items, more, err := o.List(ctx, paths.New(prefix.Path()), startAfter, nil, *recursiveFlag, 0, meta.Modified|meta.Size) + items, more, err := o.List(ctx, prefix.Path(), startAfter, "", *recursiveFlag, 0, meta.Modified|meta.Size) if err != nil { return err } for _, object := range items { - path := object.Path.String() + path := object.Path if prependBucket { path = fmt.Sprintf("%s/%s", prefix.Bucket(), path) } if object.IsPrefix { - fmt.Println("PRE", path+"/") + fmt.Println("PRE", path) } else { fmt.Printf("%v %v %12v %v\n", "OBJ", formatTime(object.Meta.Modified), object.Meta.Size, path) } diff --git a/cmd/uplink/cmd/mount.go b/cmd/uplink/cmd/mount.go index 6fa22f8cc..62c20a449 100644 --- a/cmd/uplink/cmd/mount.go +++ b/cmd/uplink/cmd/mount.go @@ -18,7 +18,6 @@ import ( "github.com/spf13/cobra" "storj.io/storj/internal/fpath" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/process" "storj.io/storj/pkg/storage/meta" "storj.io/storj/pkg/storage/objects" @@ -98,14 +97,14 @@ func (sf *storjFs) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse return &fuse.Attr{Mode: fuse.S_IFDIR | 0755}, fuse.OK } - metadata, err := sf.store.Meta(sf.ctx, paths.New(name)) + metadata, err := sf.store.Meta(sf.ctx, name) if err != nil && !storage.ErrKeyNotFound.Has(err) { return nil, fuse.EIO } // file not found so maybe it's a prefix/directory if err != nil { - items, _, err := sf.store.List(sf.ctx, paths.New(name), nil, nil, false, 1, meta.None) + items, _, err := sf.store.List(sf.ctx, name, "", "", false, 1, meta.None) if err != nil && !storage.ErrKeyNotFound.Has(err) { return nil, fuse.EIO } @@ -147,16 +146,16 @@ func (sf *storjFs) Open(name string, flags uint32, context *fuse.Context) (file func (sf *storjFs) listFiles(ctx context.Context, name string, store objects.Store) (c []fuse.DirEntry, err error) { var entries []fuse.DirEntry - startAfter := paths.New("") + startAfter := "" for { - items, more, err := store.List(ctx, paths.New(name), startAfter, nil, false, 0, meta.Modified) + items, more, err := store.List(ctx, name, startAfter, "", false, 0, meta.Modified) if err != nil { return nil, err } for _, object := range items { - path := object.Path.String() + path := object.Path mode := fuse.S_IFREG if object.IsPrefix { @@ -176,7 +175,7 @@ func (sf *storjFs) listFiles(ctx context.Context, name string, store objects.Sto } func (sf *storjFs) Unlink(name string, context *fuse.Context) (code fuse.Status) { - err := sf.store.Delete(sf.ctx, paths.New(name)) + err := sf.store.Delete(sf.ctx, name) if err != nil { if storage.ErrKeyNotFound.Has(err) { return fuse.ENOENT @@ -222,7 +221,7 @@ func (f *storjFile) Read(buf []byte, off int64) (res fuse.ReadResult, code fuse. func (f *storjFile) getReader(off int64) (io.ReadCloser, error) { if f.reader == nil { - ranger, _, err := f.store.Get(f.ctx, paths.New(f.name)) + ranger, _, err := f.store.Get(f.ctx, f.name) if err != nil { return nil, err } diff --git a/cmd/uplink/cmd/rb.go b/cmd/uplink/cmd/rb.go index 2e59482f0..638990ee8 100644 --- a/cmd/uplink/cmd/rb.go +++ b/cmd/uplink/cmd/rb.go @@ -60,7 +60,7 @@ func deleteBucket(cmd *cobra.Command, args []string) error { return err } - items, _, err := o.List(ctx, nil, nil, nil, true, 1, meta.None) + items, _, err := o.List(ctx, "", "", "", true, 1, meta.None) if err != nil { return err } diff --git a/cmd/uplink/cmd/rm.go b/cmd/uplink/cmd/rm.go index bc7761c9b..f0a499814 100644 --- a/cmd/uplink/cmd/rm.go +++ b/cmd/uplink/cmd/rm.go @@ -9,7 +9,6 @@ import ( "github.com/spf13/cobra" "storj.io/storj/internal/fpath" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/process" ) @@ -34,7 +33,7 @@ func delete(cmd *cobra.Command, args []string) error { } if dst.IsLocal() { - return fmt.Errorf("No bucket specified. Please use format sj://bucket/") + return fmt.Errorf("No bucket specified, use format sj://bucket/") } bs, err := cfg.BucketStore(ctx) @@ -47,7 +46,7 @@ func delete(cmd *cobra.Command, args []string) error { return err } - err = o.Delete(ctx, paths.New(dst.Path())) + err = o.Delete(ctx, dst.Path()) if err != nil { return err } diff --git a/examples/pointerdb-client/main.go b/examples/pointerdb-client/main.go index 6fb8e3c22..03017b592 100644 --- a/examples/pointerdb-client/main.go +++ b/examples/pointerdb-client/main.go @@ -14,7 +14,6 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - p "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/pointerdb/pdbclient" "storj.io/storj/pkg/provider" @@ -59,7 +58,7 @@ func main() { ctx := context.Background() // Example parameters to pass into API calls - var path = p.New("fold1/fold2/fold3/file.txt") + var path = "fold1/fold2/fold3/file.txt" pointer := &pb.Pointer{ Type: pb.Pointer_INLINE, InlineSegment: []byte("popcorn"), @@ -75,7 +74,7 @@ func main() { } // Example Put2 - err = client.Put(ctx, p.New("fold1/fold2"), pointer) + err = client.Put(ctx, "fold1/fold2", pointer) if err != nil || status.Code(err) == codes.Internal { logger.Error("couldn't put pointer in db", zap.Error(err)) @@ -95,15 +94,14 @@ func main() { } // Example List with pagination - prefix := p.New("fold1") - items, more, err := client.List(ctx, prefix, nil, nil, true, 1, meta.None) + items, more, err := client.List(ctx, "fold1", "", "", true, 1, meta.None) if err != nil || status.Code(err) == codes.Internal { logger.Error("failed to list file paths", zap.Error(err)) } else { var stringList []string for _, item := range items { - stringList = append(stringList, item.Path.String()) + stringList = append(stringList, item.Path) } logger.Debug("Success: listed paths: " + strings.Join(stringList, ", ") + "; more: " + fmt.Sprintf("%t", more)) } diff --git a/pkg/audit/cursor.go b/pkg/audit/cursor.go index ffa172428..73a2191fc 100644 --- a/pkg/audit/cursor.go +++ b/pkg/audit/cursor.go @@ -12,10 +12,10 @@ import ( "github.com/vivint/infectious" "storj.io/storj/pkg/eestream" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/pointerdb/pdbclient" "storj.io/storj/pkg/storage/meta" + "storj.io/storj/pkg/storj" ) // Stripe keeps track of a stripe's index and its parent segment @@ -27,7 +27,7 @@ type Stripe struct { // Cursor keeps track of audit location in pointer db type Cursor struct { pointers pdbclient.Client - lastPath *paths.Path + lastPath storj.Path mutex sync.Mutex } @@ -42,13 +42,13 @@ func (cursor *Cursor) NextStripe(ctx context.Context) (stripe *Stripe, err error defer cursor.mutex.Unlock() var pointerItems []pdbclient.ListItem - var path paths.Path + var path storj.Path var more bool - if cursor.lastPath == nil { - pointerItems, more, err = cursor.pointers.List(ctx, nil, nil, nil, true, 0, meta.None) + if cursor.lastPath == "" { + pointerItems, more, err = cursor.pointers.List(ctx, "", "", "", true, 0, meta.None) } else { - pointerItems, more, err = cursor.pointers.List(ctx, nil, *cursor.lastPath, nil, true, 0, meta.None) + pointerItems, more, err = cursor.pointers.List(ctx, "", cursor.lastPath, "", true, 0, meta.None) } if err != nil { @@ -66,9 +66,9 @@ func (cursor *Cursor) NextStripe(ctx context.Context) (stripe *Stripe, err error // keep track of last path listed if !more { - cursor.lastPath = nil + cursor.lastPath = "" } else { - cursor.lastPath = &pointerItems[len(pointerItems)-1].Path + cursor.lastPath = pointerItems[len(pointerItems)-1].Path } // get pointer info diff --git a/pkg/audit/cursor_test.go b/pkg/audit/cursor_test.go index a6edeb9ba..8f3297416 100644 --- a/pkg/audit/cursor_test.go +++ b/pkg/audit/cursor_test.go @@ -18,11 +18,11 @@ import ( "storj.io/storj/pkg/auth" "storj.io/storj/pkg/overlay" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/pointerdb" "storj.io/storj/pkg/pointerdb/pdbclient" "storj.io/storj/pkg/storage/meta" + "storj.io/storj/pkg/storj" "storj.io/storj/storage/redis/redisserver" "storj.io/storj/storage/teststore" ) @@ -60,7 +60,7 @@ func (pbd *pointerDBWrapper) Delete(ctx context.Context, in *pb.DeleteRequest, o func TestAuditSegment(t *testing.T) { type pathCount struct { - path paths.Path + path storj.Path count int } @@ -69,47 +69,47 @@ func TestAuditSegment(t *testing.T) { // list api call, default is 0 == 1000 listing tests := []struct { bm string - path paths.Path + path storj.Path }{ { bm: "success-1", - path: paths.New("folder1/file1"), + path: "folder1/file1", }, { bm: "success-2", - path: paths.New("foodFolder1/file1/file2"), + path: "foodFolder1/file1/file2", }, { bm: "success-3", - path: paths.New("foodFolder1/file1/file2/foodFolder2/file3"), + path: "foodFolder1/file1/file2/foodFolder2/file3", }, { bm: "success-4", - path: paths.New("projectFolder/project1.txt/"), + path: "projectFolder/project1.txt/", }, { bm: "success-5", - path: paths.New("newProjectFolder/project2.txt"), + path: "newProjectFolder/project2.txt", }, { bm: "success-6", - path: paths.New("Pictures/image1.png"), + path: "Pictures/image1.png", }, { bm: "success-7", - path: paths.New("Pictures/Nature/mountains.png"), + path: "Pictures/Nature/mountains.png", }, { bm: "success-8", - path: paths.New("Pictures/City/streets.png"), + path: "Pictures/City/streets.png", }, { bm: "success-9", - path: paths.New("Pictures/Animals/Dogs/dogs.png"), + path: "Pictures/Animals/Dogs/dogs.png", }, { bm: "success-10", - path: paths.New("Nada/ビデオ/😶"), + path: "Nada/ビデオ/😶", }, } @@ -147,7 +147,7 @@ func TestAuditSegment(t *testing.T) { putRequest := makePutRequest(tt.path) // create putreq. object - req := &pb.PutRequest{Path: tt.path.String(), Pointer: putRequest.Pointer} + req := &pb.PutRequest{Path: tt.path, Pointer: putRequest.Pointer} // put pointer into db _, err := pdbw.Put(ctx, req) @@ -180,7 +180,7 @@ func TestAuditSegment(t *testing.T) { // test to see how random paths are t.Run("probabilisticTest", func(t *testing.T) { - list, _, err := pointers.List(ctx, nil, nil, nil, true, 10, meta.None) + list, _, err := pointers.List(ctx, "", "", "", true, 10, meta.None) if err != nil { t.Error(ErrNoList) } @@ -237,14 +237,14 @@ func TestAuditSegment(t *testing.T) { }) } -func makePutRequest(path paths.Path) pb.PutRequest { +func makePutRequest(path storj.Path) pb.PutRequest { var rps []*pb.RemotePiece rps = append(rps, &pb.RemotePiece{ PieceNum: 1, NodeId: "testId", }) pr := pb.PutRequest{ - Path: path.String(), + Path: path, Pointer: &pb.Pointer{ Type: pb.Pointer_REMOTE, Remote: &pb.RemoteSegment{ diff --git a/pkg/encryption/encryption.go b/pkg/encryption/encryption.go index a48ec8284..b5317de9f 100644 --- a/pkg/encryption/encryption.go +++ b/pkg/encryption/encryption.go @@ -4,6 +4,9 @@ package encryption import ( + "crypto/hmac" + "crypto/sha512" + "storj.io/storj/pkg/storj" ) @@ -98,3 +101,17 @@ func DecryptKey(keyToDecrypt storj.EncryptedPrivateKey, cipher storj.Cipher, key return &decryptedKey, nil } + +// DeriveKey derives new key from the given key and message using HMAC-SHA512 +func DeriveKey(key *storj.Key, message string) (*storj.Key, error) { + mac := hmac.New(sha512.New, key[:]) + _, err := mac.Write([]byte(message)) + if err != nil { + return nil, Error.Wrap(err) + } + + derived := new(storj.Key) + copy(derived[:], mac.Sum(nil)) + + return derived, nil +} diff --git a/pkg/encryption/examples_test.go b/pkg/encryption/examples_test.go new file mode 100644 index 000000000..d57237a87 --- /dev/null +++ b/pkg/encryption/examples_test.go @@ -0,0 +1,62 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package encryption_test + +import ( + "encoding/hex" + "fmt" + + "storj.io/storj/pkg/encryption" + "storj.io/storj/pkg/storj" +) + +func ExampleEncryptPath() { + var path = "fold1/fold2/fold3/file.txt" + + // seed + seed := new(storj.Key) + for i := range seed { + seed[i] = byte(i) + } + fmt.Printf("root key (%d bytes): %s\n", len(seed), hex.EncodeToString(seed[:])) + + // use the seed for encrypting the path + encryptedPath, err := encryption.EncryptPath(path, seed) + if err != nil { + panic(err) + } + fmt.Println("path to encrypt:", path) + fmt.Println("encrypted path: ", encryptedPath) + + // decrypting the path + decryptedPath, err := encryption.DecryptPath(encryptedPath, seed) + if err != nil { + panic(err) + } + fmt.Println("decrypted path: ", decryptedPath) + + // handling of shared path + sharedPath := storj.JoinPaths(storj.SplitPath(encryptedPath)[2:]...) + fmt.Println("shared path: ", sharedPath) + derivedKey, err := encryption.DerivePathKey(decryptedPath, seed, 2) + if err != nil { + panic(err) + } + + fmt.Printf("derived key (%d bytes): %s\n", len(derivedKey), hex.EncodeToString(derivedKey[:])) + decryptedPath, err = encryption.DecryptPath(sharedPath, derivedKey) + if err != nil { + panic(err) + } + fmt.Println("decrypted path: ", decryptedPath) + + // Output: + // root key (32 bytes): 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f + // path to encrypt: fold1/fold2/fold3/file.txt + // encrypted path: urxuYzqG_ZlJfBhkGaz87WvvnCZaYD7qf1_ZN_Pd91n5/IyncDwLhWPv4F7EaoUivwICnUeJMWlUnMATL4faaoH2s/_1gitX6uPd3etc3RgoD9R1waT5MPKrlrY32ehz_vqlOv/6qO4DU5AHFabE2r7hmAauvnomvtNByuO-FCw4ch_xaVR3SPE + // decrypted path: fold1/fold2/fold3/file.txt + // shared path: _1gitX6uPd3etc3RgoD9R1waT5MPKrlrY32ehz_vqlOv/6qO4DU5AHFabE2r7hmAauvnomvtNByuO-FCw4ch_xaVR3SPE + // derived key (32 bytes): 909db5ccf2b645e3352ee8212305596ed514d9f84d5acd21d93b4527d2a0c7e1 + // decrypted path: fold3/file.txt +} diff --git a/pkg/encryption/path.go b/pkg/encryption/path.go new file mode 100644 index 000000000..40f035c7b --- /dev/null +++ b/pkg/encryption/path.go @@ -0,0 +1,143 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package encryption + +import ( + "crypto/hmac" + "crypto/sha512" + "encoding/base64" + + "storj.io/storj/pkg/storj" +) + +// EncryptPath encrypts path with the given key +func EncryptPath(path storj.Path, key *storj.Key) (encrypted storj.Path, err error) { + // do not encrypt empty paths + if len(path) == 0 { + return path, nil + } + + comps := storj.SplitPath(path) + for i, comp := range comps { + comps[i], err = encryptPathComponent(comp, key) + if err != nil { + return "", err + } + key, err = DeriveKey(key, "path:"+comp) + if err != nil { + return "", err + } + } + return storj.JoinPaths(comps...), nil +} + +// DecryptPath decrypts path with the given key +func DecryptPath(path storj.Path, key *storj.Key) (decrypted storj.Path, err error) { + comps := storj.SplitPath(path) + for i, comp := range comps { + comps[i], err = decryptPathComponent(comp, key) + if err != nil { + return "", err + } + key, err = DeriveKey(key, "path:"+comps[i]) + if err != nil { + return "", err + } + } + return storj.JoinPaths(comps...), nil +} + +// DerivePathKey derives the key for the given depth from the given root key. +// This method must be called on an unencrypted path. +func DerivePathKey(path storj.Path, key *storj.Key, depth int) (derivedKey *storj.Key, err error) { + if depth < 0 { + return nil, Error.New("negative depth") + } + + // do not derive key from empty path + if len(path) == 0 { + return key, nil + } + + comps := storj.SplitPath(path) + if depth > len(comps) { + return nil, Error.New("depth greater than path length") + } + + derivedKey = key + for i := 0; i < depth; i++ { + derivedKey, err = DeriveKey(derivedKey, "path:"+comps[i]) + if err != nil { + return nil, err + } + } + return derivedKey, nil +} + +// DeriveContentKey derives the key for the encrypted object data using the root key. +// This method must be called on an unencrypted path. +func DeriveContentKey(path storj.Path, key *storj.Key) (derivedKey *storj.Key, err error) { + comps := storj.SplitPath(path) + if len(comps) == 0 { + return nil, Error.New("path is empty") + } + derivedKey, err = DerivePathKey(path, key, len(comps)) + if err != nil { + return nil, err + } + derivedKey, err = DeriveKey(derivedKey, "content") + if err != nil { + return nil, err + } + return derivedKey, nil +} + +func encryptPathComponent(comp string, key *storj.Key) (string, error) { + // derive the key for the current path component + derivedKey, err := DeriveKey(key, "path:"+comp) + if err != nil { + return "", err + } + + // use the derived key to derive the nonce + mac := hmac.New(sha512.New, derivedKey[:]) + _, err = mac.Write([]byte("nonce")) + if err != nil { + return "", Error.Wrap(err) + } + + nonce := new(AESGCMNonce) + copy(nonce[:], mac.Sum(nil)) + + // encrypt the path components with the parent's key and the derived nonce + cipherText, err := EncryptAESGCM([]byte(comp), key, nonce) + if err != nil { + return "", Error.Wrap(err) + } + + // keep the nonce together with the cipher text + return base64.RawURLEncoding.EncodeToString(append(nonce[:], cipherText...)), nil +} + +func decryptPathComponent(comp string, key *storj.Key) (string, error) { + if comp == "" { + return "", nil + } + + data, err := base64.RawURLEncoding.DecodeString(comp) + if err != nil { + return "", Error.Wrap(err) + } + + // extract the nonce from the cipher text + nonce := new(AESGCMNonce) + copy(nonce[:], data[:AESGCMNonceSize]) + + decrypted, err := DecryptAESGCM(data[AESGCMNonceSize:], key, nonce) + if err != nil { + return "", Error.Wrap(err) + } + + return string(decrypted), nil +} diff --git a/pkg/encryption/path_test.go b/pkg/encryption/path_test.go new file mode 100644 index 000000000..e1c1c4fe0 --- /dev/null +++ b/pkg/encryption/path_test.go @@ -0,0 +1,87 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package encryption + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "storj.io/storj/pkg/storj" +) + +func TestEncryption(t *testing.T) { + for i, path := range []storj.Path{ + "", + "/", + "//", + "file.txt", + "file.txt/", + "fold1/file.txt", + "fold1/fold2/file.txt", + "/fold1/fold2/fold3/file.txt", + } { + errTag := fmt.Sprintf("Test case #%d", i) + + key := new(storj.Key) + copy(key[:], randData(storj.KeySize)) + + encrypted, err := EncryptPath(path, key) + if !assert.NoError(t, err, errTag) { + continue + } + + decrypted, err := DecryptPath(encrypted, key) + if !assert.NoError(t, err, errTag) { + continue + } + + assert.Equal(t, path, decrypted, errTag) + } +} + +func TestDeriveKey(t *testing.T) { + for i, tt := range []struct { + path storj.Path + depth int + errString string + }{ + {"fold1/fold2/fold3/file.txt", -1, "encryption error: negative depth"}, + {"fold1/fold2/fold3/file.txt", 0, ""}, + {"fold1/fold2/fold3/file.txt", 1, ""}, + {"fold1/fold2/fold3/file.txt", 2, ""}, + {"fold1/fold2/fold3/file.txt", 3, ""}, + {"fold1/fold2/fold3/file.txt", 4, ""}, + {"fold1/fold2/fold3/file.txt", 5, "encryption error: depth greater than path length"}, + } { + errTag := fmt.Sprintf("Test case #%d", i) + + key := new(storj.Key) + copy(key[:], randData(storj.KeySize)) + + encrypted, err := EncryptPath(tt.path, key) + if !assert.NoError(t, err, errTag) { + continue + } + + derivedKey, err := DerivePathKey(tt.path, key, tt.depth) + if tt.errString != "" { + assert.EqualError(t, err, tt.errString, errTag) + continue + } + if !assert.NoError(t, err, errTag) { + continue + } + + shared := storj.JoinPaths(storj.SplitPath(encrypted)[tt.depth:]...) + decrypted, err := DecryptPath(shared, derivedKey) + if !assert.NoError(t, err, errTag) { + continue + } + + expected := storj.JoinPaths(storj.SplitPath(tt.path)[tt.depth:]...) + assert.Equal(t, expected, decrypted, errTag) + } +} diff --git a/pkg/miniogw/config.go b/pkg/miniogw/config.go index fe8bc075a..7f25cf507 100644 --- a/pkg/miniogw/config.go +++ b/pkg/miniogw/config.go @@ -155,10 +155,15 @@ func (c Config) GetBucketStore(ctx context.Context, identity *provider.FullIdent err = Error.New("EncryptionBlockSize must be a multiple of ErasureShareSize * RS MinThreshold") return nil, err } - stream, err := streams.NewStreamStore(segments, c.SegmentSize, c.EncKey, c.EncBlockSize, storj.Cipher(c.EncType)) + + key := new(storj.Key) + copy(key[:], c.EncKey) + + stream, err := streams.NewStreamStore(segments, c.SegmentSize, key, c.EncBlockSize, storj.Cipher(c.EncType)) if err != nil { return nil, err } + obj := objects.NewStore(stream) return buckets.NewStore(obj), nil diff --git a/pkg/miniogw/gateway-storj.go b/pkg/miniogw/gateway-storj.go index 7ad478090..2bcc82c41 100644 --- a/pkg/miniogw/gateway-storj.go +++ b/pkg/miniogw/gateway-storj.go @@ -6,6 +6,7 @@ package miniogw import ( "context" "io" + "strings" "time" minio "github.com/minio/minio/cmd" @@ -14,11 +15,11 @@ import ( "github.com/zeebo/errs" monkit "gopkg.in/spacemonkeygo/monkit.v2" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/buckets" "storj.io/storj/pkg/storage/meta" "storj.io/storj/pkg/storage/objects" + "storj.io/storj/pkg/storj" "storj.io/storj/pkg/utils" "storj.io/storj/storage" ) @@ -74,7 +75,7 @@ func (s *storjObjects) DeleteBucket(ctx context.Context, bucket string) (err err if err != nil { return err } - items, _, err := o.List(ctx, nil, nil, nil, true, 1, meta.None) + items, _, err := o.List(ctx, "", "", "", true, 1, meta.None) if err != nil { return err } @@ -90,7 +91,7 @@ func (s *storjObjects) DeleteObject(ctx context.Context, bucket, object string) if err != nil { return err } - err = o.Delete(ctx, paths.New(object)) + err = o.Delete(ctx, object) if storage.ErrKeyNotFound.Has(err) { err = minio.ObjectNotFound{Bucket: bucket, Object: object} } @@ -119,7 +120,7 @@ func (s *storjObjects) getObject(ctx context.Context, bucket, object string) (rr return nil, err } - rr, _, err = o.Get(ctx, paths.New(object)) + rr, _, err = o.Get(ctx, object) return rr, err } @@ -155,7 +156,7 @@ func (s *storjObjects) GetObjectInfo(ctx context.Context, bucket, if err != nil { return minio.ObjectInfo{}, err } - m, err := o.Meta(ctx, paths.New(object)) + m, err := o.Meta(ctx, object) if err != nil { if storage.ErrKeyNotFound.Has(err) { return objInfo, minio.ObjectNotFound{ @@ -208,7 +209,7 @@ func (s *storjObjects) ListObjects(ctx context.Context, bucket, prefix, marker, return minio.ListObjectsInfo{}, Error.New("delimiter %s not supported", delimiter) } - startAfter := paths.New(marker) + startAfter := marker recursive := delimiter == "" var objects []minio.ObjectInfo @@ -217,24 +218,24 @@ func (s *storjObjects) ListObjects(ctx context.Context, bucket, prefix, marker, if err != nil { return minio.ListObjectsInfo{}, err } - items, more, err := o.List(ctx, paths.New(prefix), startAfter, nil, recursive, maxKeys, meta.All) + items, more, err := o.List(ctx, prefix, startAfter, "", recursive, maxKeys, meta.All) if err != nil { return result, err } if len(items) > 0 { for _, item := range items { path := item.Path - if recursive { - path = path.Prepend(prefix) + if recursive && prefix != "" { + path = storj.JoinPaths(strings.TrimSuffix(prefix, "/"), path) } if item.IsPrefix { - prefixes = append(prefixes, path.String()+"/") + prefixes = append(prefixes, path) continue } objects = append(objects, minio.ObjectInfo{ Bucket: bucket, IsDir: false, - Name: path.String(), + Name: path, ModTime: item.Meta.Modified, Size: item.Meta.Size, ContentType: item.Meta.ContentType, @@ -251,7 +252,7 @@ func (s *storjObjects) ListObjects(ctx context.Context, bucket, prefix, marker, Prefixes: prefixes, } if more { - result.NextMarker = startAfter.String() + result.NextMarker = startAfter } return result, err @@ -268,12 +269,12 @@ func (s *storjObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin recursive := delimiter == "" var nextContinuationToken string - var startAfterPath paths.Path + var startAfterPath storj.Path if continuationToken != "" { - startAfterPath = paths.New(continuationToken) + startAfterPath = continuationToken } - if startAfterPath == nil && startAfter != "" { - startAfterPath = paths.New(startAfter) + if startAfterPath == "" && startAfter != "" { + startAfterPath = startAfter } var objects []minio.ObjectInfo @@ -282,7 +283,7 @@ func (s *storjObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin if err != nil { return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, err } - items, more, err := o.List(ctx, paths.New(prefix), startAfterPath, nil, recursive, maxKeys, meta.All) + items, more, err := o.List(ctx, prefix, startAfterPath, "", recursive, maxKeys, meta.All) if err != nil { return result, err } @@ -290,17 +291,17 @@ func (s *storjObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin if len(items) > 0 { for _, item := range items { path := item.Path - if recursive { - path = path.Prepend(prefix) + if recursive && prefix != "" { + path = storj.JoinPaths(strings.TrimSuffix(prefix, "/"), path) } if item.IsPrefix { - prefixes = append(prefixes, path.String()+"/") + prefixes = append(prefixes, path) continue } objects = append(objects, minio.ObjectInfo{ Bucket: bucket, IsDir: false, - Name: path.String(), + Name: path, ModTime: item.Meta.Modified, Size: item.Meta.Size, ContentType: item.Meta.ContentType, @@ -309,7 +310,7 @@ func (s *storjObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin }) } - nextContinuationToken = items[len(items)-1].Path.String() + "\x00" + nextContinuationToken = items[len(items)-1].Path + "\x00" } result = minio.ListObjectsV2Info{ @@ -380,7 +381,7 @@ func (s *storjObjects) putObject(ctx context.Context, bucket, object string, r i if err != nil { return minio.ObjectInfo{}, err } - m, err := o.Put(ctx, paths.New(object), r, meta, expTime) + m, err := o.Put(ctx, object, r, meta, expTime) return minio.ObjectInfo{ Name: object, Bucket: bucket, diff --git a/pkg/miniogw/gateway-storj_test.go b/pkg/miniogw/gateway-storj_test.go index a1196c3ba..4f13da359 100644 --- a/pkg/miniogw/gateway-storj_test.go +++ b/pkg/miniogw/gateway-storj_test.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "io" - "path" "testing" "time" @@ -18,7 +17,6 @@ import ( "github.com/minio/minio/pkg/hash" "github.com/stretchr/testify/assert" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/buckets" mock_buckets "storj.io/storj/pkg/storage/buckets/mocks" @@ -98,11 +96,11 @@ func TestCopyObject(t *testing.T) { // if o.Get returns an error, only expect GetObjectStore once, do not expect Put if example.errString != "some Get err" { mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil).Times(2) - mockOS.EXPECT().Get(gomock.Any(), paths.New(example.srcObject)).Return(rr, meta, example.getErr) - mockOS.EXPECT().Put(gomock.Any(), paths.New(example.destObject), r, serMeta, time.Time{}).Return(meta, example.putErr) + mockOS.EXPECT().Get(gomock.Any(), example.srcObject).Return(rr, meta, example.getErr) + mockOS.EXPECT().Put(gomock.Any(), example.destObject, r, serMeta, time.Time{}).Return(meta, example.putErr) } else { mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil) - mockOS.EXPECT().Get(gomock.Any(), paths.New(example.srcObject)).Return(rr, meta, example.getErr) + mockOS.EXPECT().Get(gomock.Any(), example.srcObject).Return(rr, meta, example.getErr) } objInfo, err := storjObj.CopyObject(ctx, example.bucket, example.srcObject, example.bucket, example.destObject, srcInfo) @@ -158,7 +156,7 @@ func TestGetObject(t *testing.T) { rr := ranger.ByteRanger([]byte(example.data)) mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil) - mockOS.EXPECT().Get(gomock.Any(), paths.New(example.object)).Return(rr, meta, example.err) + mockOS.EXPECT().Get(gomock.Any(), example.object).Return(rr, meta, example.err) var buf bytes.Buffer iowriter := io.Writer(&buf) @@ -195,7 +193,7 @@ func TestDeleteObject(t *testing.T) { errTag := fmt.Sprintf("Test case #%d", i) mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil) - mockOS.EXPECT().Delete(gomock.Any(), paths.New(example.object)).Return(example.err) + mockOS.EXPECT().Delete(gomock.Any(), example.object).Return(example.err) err := storjObj.DeleteObject(ctx, example.bucket, example.object) assert.NoError(t, err, errTag) @@ -256,7 +254,7 @@ func TestPutObject(t *testing.T) { } mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil) - mockOS.EXPECT().Put(gomock.Any(), paths.New(example.object), data, serMeta, time.Time{}).Return(meta, example.err) + mockOS.EXPECT().Put(gomock.Any(), example.object, data, serMeta, time.Time{}).Return(meta, example.err) objInfo, err := storjObj.PutObject(ctx, example.bucket, example.object, data, metadata) if err != nil { @@ -314,7 +312,7 @@ func TestGetObjectInfo(t *testing.T) { errTag := fmt.Sprintf("Test case #%d", i) mockBS.EXPECT().GetObjectStore(gomock.Any(), example.bucket).Return(mockOS, nil) - mockOS.EXPECT().Meta(gomock.Any(), paths.New(example.object)).Return(meta, example.err) + mockOS.EXPECT().Meta(gomock.Any(), example.object).Return(meta, example.err) objInfo, err := storjObj.GetObjectInfo(ctx, example.bucket, example.object) if err != nil { @@ -351,8 +349,8 @@ func TestListObjects(t *testing.T) { maxKeys := 123 items := []objects.ListItem{ - {Path: paths.New("test-file-1.txt")}, - {Path: paths.New("test-file-2.txt")}, + {Path: "test-file-1.txt"}, + {Path: "test-file-2.txt"}, } for i, example := range []struct { @@ -368,30 +366,29 @@ func TestListObjects(t *testing.T) { { more: false, startAfter: "", nextMarker: "", delimiter: "", recursive: true, objInfos: []minio.ObjectInfo{ - {Bucket: bucket, Name: path.Join("test-prefix/test-file-1.txt")}, - {Bucket: bucket, Name: path.Join("test-prefix/test-file-2.txt")}, + {Bucket: bucket, Name: "test-prefix/test-file-1.txt"}, + {Bucket: bucket, Name: "test-prefix/test-file-2.txt"}, }, err: nil, errString: "", }, { more: true, startAfter: "test-start-after", nextMarker: "test-file-2.txt", delimiter: "/", recursive: false, objInfos: []minio.ObjectInfo{ - {Bucket: bucket, Name: path.Join("test-file-1.txt")}, - {Bucket: bucket, Name: path.Join("test-file-2.txt")}, + {Bucket: bucket, Name: "test-file-1.txt"}, + {Bucket: bucket, Name: "test-file-2.txt"}, }, err: nil, errString: "", }, { more: false, startAfter: "", nextMarker: "", delimiter: "", recursive: true, objInfos: []minio.ObjectInfo{ - {Bucket: bucket, Name: path.Join("test-prefix/test-file-1.txt")}, - {Bucket: bucket, Name: path.Join("test-prefix/test-file-2.txt")}, + {Bucket: bucket, Name: "test-prefix/test-file-1.txt"}, + {Bucket: bucket, Name: "test-prefix/test-file-2.txt"}, }, err: Error.New("error"), errString: "Storj Gateway error: error", }, } { errTag := fmt.Sprintf("Test case #%d", i) mockBS.EXPECT().GetObjectStore(gomock.Any(), bucket).Return(mockOS, nil) - mockOS.EXPECT().List(gomock.Any(), paths.New(prefix), paths.New(example.startAfter), - nil, example.recursive, maxKeys, meta.All).Return(items, example.more, example.err) + mockOS.EXPECT().List(gomock.Any(), prefix, example.startAfter, "", example.recursive, maxKeys, meta.All).Return(items, example.more, example.err) listInfo, err := storjObj.ListObjects(ctx, bucket, prefix, example.startAfter, example.delimiter, maxKeys) @@ -422,7 +419,7 @@ func TestDeleteBucket(t *testing.T) { storjObj := storjObjects{storj: &b} itemsInBucket := make([]objects.ListItem, 1) - itemsInBucket[0] = objects.ListItem{Path: paths.New("path1"), Meta: objects.Meta{}} + itemsInBucket[0] = objects.ListItem{Path: "path1", Meta: objects.Meta{}} exp := time.Unix(0, 0).UTC() var noItemsInBucket []objects.ListItem diff --git a/pkg/miniogw/multipart.go b/pkg/miniogw/multipart.go index 87662e0d7..21355c3a6 100644 --- a/pkg/miniogw/multipart.go +++ b/pkg/miniogw/multipart.go @@ -15,7 +15,6 @@ import ( minio "github.com/minio/minio/cmd" "github.com/minio/minio/pkg/hash" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/storage/objects" ) @@ -49,7 +48,7 @@ func (s *storjObjects) NewMultipartUpload(ctx context.Context, bucket, object st UserDefined: metadata, } - result, err := objectStore.Put(ctx, paths.New(object), upload.Stream, serMetaInfo, expTime) + result, err := objectStore.Put(ctx, object, upload.Stream, serMetaInfo, expTime) uploads.RemoveByID(upload.ID) if err != nil { diff --git a/pkg/miniogw/objstore_mock_test.go b/pkg/miniogw/objstore_mock_test.go index f8590b499..f79ba92b5 100644 --- a/pkg/miniogw/objstore_mock_test.go +++ b/pkg/miniogw/objstore_mock_test.go @@ -15,7 +15,6 @@ import ( gomock "github.com/golang/mock/gomock" - paths "storj.io/storj/pkg/paths" ranger "storj.io/storj/pkg/ranger" objects "storj.io/storj/pkg/storage/objects" ) @@ -44,7 +43,7 @@ func (m *MockStore) EXPECT() *MockStoreMockRecorder { } // Delete mocks base method -func (m *MockStore) Delete(arg0 context.Context, arg1 paths.Path) error { +func (m *MockStore) Delete(arg0 context.Context, arg1 string) error { ret := m.ctrl.Call(m, "Delete", arg0, arg1) ret0, _ := ret[0].(error) return ret0 @@ -56,7 +55,7 @@ func (mr *MockStoreMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { } // Get mocks base method -func (m *MockStore) Get(arg0 context.Context, arg1 paths.Path) (ranger.Ranger, objects.Meta, error) { +func (m *MockStore) Get(arg0 context.Context, arg1 string) (ranger.Ranger, objects.Meta, error) { ret := m.ctrl.Call(m, "Get", arg0, arg1) ret0, _ := ret[0].(ranger.Ranger) ret1, _ := ret[1].(objects.Meta) @@ -70,7 +69,7 @@ func (mr *MockStoreMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { } // List mocks base method -func (m *MockStore) List(arg0 context.Context, arg1, arg2, arg3 paths.Path, arg4 bool, arg5 int, arg6 uint32) ([]objects.ListItem, bool, error) { +func (m *MockStore) List(arg0 context.Context, arg1, arg2, arg3 string, arg4 bool, arg5 int, arg6 uint32) ([]objects.ListItem, bool, error) { ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3, arg4, arg5, arg6) ret0, _ := ret[0].([]objects.ListItem) ret1, _ := ret[1].(bool) @@ -84,7 +83,7 @@ func (mr *MockStoreMockRecorder) List(arg0, arg1, arg2, arg3, arg4, arg5, arg6 i } // Meta mocks base method -func (m *MockStore) Meta(arg0 context.Context, arg1 paths.Path) (objects.Meta, error) { +func (m *MockStore) Meta(arg0 context.Context, arg1 string) (objects.Meta, error) { ret := m.ctrl.Call(m, "Meta", arg0, arg1) ret0, _ := ret[0].(objects.Meta) ret1, _ := ret[1].(error) @@ -97,7 +96,7 @@ func (mr *MockStoreMockRecorder) Meta(arg0, arg1 interface{}) *gomock.Call { } // Put mocks base method -func (m *MockStore) Put(arg0 context.Context, arg1 paths.Path, arg2 io.Reader, arg3 objects.SerializableMeta, arg4 time.Time) (objects.Meta, error) { +func (m *MockStore) Put(arg0 context.Context, arg1 string, arg2 io.Reader, arg3 objects.SerializableMeta, arg4 time.Time) (objects.Meta, error) { ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(objects.Meta) ret1, _ := ret[1].(error) diff --git a/pkg/paths/common.go b/pkg/paths/common.go deleted file mode 100644 index 9096a1f85..000000000 --- a/pkg/paths/common.go +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (C) 2018 Storj Labs, Inc. -// See LICENSE for copying information. - -package paths - -import ( - "github.com/zeebo/errs" -) - -// Error is the errs class of standard path errors -var Error = errs.Class("paths error") diff --git a/pkg/paths/examples_test.go b/pkg/paths/examples_test.go deleted file mode 100644 index fa1c2032d..000000000 --- a/pkg/paths/examples_test.go +++ /dev/null @@ -1,66 +0,0 @@ -// Copyright (C) 2018 Storj Labs, Inc. -// See LICENSE for copying information. - -package paths_test - -import ( - "encoding/hex" - "fmt" - - "storj.io/storj/pkg/paths" -) - -func ExamplePath_Encrypt() { - var path = paths.New("fold1/fold2/fold3/file.txt") - - // seed - seed := make([]byte, 64) - for i := range seed { - seed[i] = byte(i) - } - fmt.Printf("root key (%d bytes): %s\n", len(seed), hex.EncodeToString(seed)) - - // use the seed for encrypting the path - encryptedPath, err := path.Encrypt(seed) - if err != nil { - panic(err) - } - fmt.Println("path to encrypt:", path) - fmt.Println("encrypted path: ", encryptedPath) - - // decrypting the path - decryptedPath, err := encryptedPath.Decrypt(seed) - if err != nil { - panic(err) - } - fmt.Println("decrypted path: ", decryptedPath) - - // handling of shared path - sharedPath := encryptedPath[2:] - fmt.Println("shared path: ", encryptedPath[2:]) - derivedKey, err := decryptedPath.DeriveKey(seed, 2) - if err != nil { - panic(err) - } - - fmt.Printf("derived key (%d bytes): %s\n", len(derivedKey), hex.EncodeToString(derivedKey)) - decryptedPath, err = sharedPath.Decrypt(derivedKey) - if err != nil { - panic(err) - } - fmt.Println("decrypted path: ", decryptedPath) - - // implement Bytes() function - var pathBytes = path.Bytes() - fmt.Println("path in Bytes is: ", pathBytes) - - // Output: - // root key (64 bytes): 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f - // path to encrypt: fold1/fold2/fold3/file.txt - // encrypted path: 1ziIKg8Mw9_ywBqSOm78gQ9aQVbrQ/1FWdS4WUzuWXBrNML_FzGH8O2usos/1TKa8xYYCrUBEvEGj7YENdwViZOYh/17JWesXCBVU9Nx8IdnwuNUuwqgGFKL_jH - // decrypted path: fold1/fold2/fold3/file.txt - // shared path: 1TKa8xYYCrUBEvEGj7YENdwViZOYh/17JWesXCBVU9Nx8IdnwuNUuwqgGFKL_jH - // derived key (64 bytes): 2592f0694bc72a2719d09b7192b9b8f9e2517fda71419314d93a7c96844f28763ed3b829f3c9a09e812b402a66b1b0a832ae3cdd7435b7b496ca95eec108c629 - // decrypted path: fold3/file.txt - // path in Bytes is: [102 111 108 100 49 47 102 111 108 100 50 47 102 111 108 100 51 47 102 105 108 101 46 116 120 116] -} diff --git a/pkg/paths/paths.go b/pkg/paths/paths.go deleted file mode 100644 index f74431a78..000000000 --- a/pkg/paths/paths.go +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright (C) 2018 Storj Labs, Inc. -// See LICENSE for copying information. - -package paths - -import ( - "crypto/aes" - "crypto/cipher" - "crypto/hmac" - "crypto/sha512" - "encoding/base64" - "path" - "strings" -) - -// Path is a unique identifier for an object stored in the Storj network -type Path []string - -// New creates new Path from the given path segments -func New(segs ...string) Path { - s := path.Join(segs...) - s = strings.Trim(s, "/") - p := strings.Split(s, "/") - if len(p) == 1 && p[0] == "" { - // Avoid building a path with a single empty segment - return []string{} - } - return p -} - -// String returns the string representation of the path -func (p Path) String() string { - return path.Join([]string(p)...) -} - -// Bytes serializes the current path to []byte -func (p Path) Bytes() []byte { - return []byte(p.String()) -} - -// Prepend creates new Path from the current path with the given segments prepended -func (p Path) Prepend(segs ...string) Path { - return New(append(segs, []string(p)...)...) -} - -// Append creates new Path from the current path with the given segments appended -func (p Path) Append(segs ...string) Path { - return New(append(p, segs...)...) -} - -// HasPrefix tests whether the current path begins with prefix. -func (p Path) HasPrefix(prefix Path) bool { - if len(prefix) > len(p) { - return false - } - for i := 0; i < len(prefix); i++ { - if p[i] != prefix[i] { - return false - } - } - return true -} - -// Encrypt creates new Path by encrypting the current path with the given key -func (p Path) Encrypt(key []byte) (encrypted Path, err error) { - encrypted = make([]string, len(p)) - for i, seg := range p { - encrypted[i], err = encrypt(seg, key) - if err != nil { - return nil, err - } - key, err = deriveSecret(key, seg) - if err != nil { - return nil, err - } - } - return encrypted, nil -} - -// Decrypt creates new Path by decrypting the current path with the given key -func (p Path) Decrypt(key []byte) (decrypted Path, err error) { - decrypted = make([]string, len(p)) - for i, seg := range p { - decrypted[i], err = decrypt(seg, key) - if err != nil { - return nil, err - } - key, err = deriveSecret(key, decrypted[i]) - if err != nil { - return nil, err - } - } - return decrypted, nil -} - -// DeriveKey derives the key for the given depth from the given root key -// This method must be called on an unencrypted path. -func (p Path) DeriveKey(key []byte, depth int) (derivedKey []byte, err error) { - if depth < 0 { - return nil, Error.New("negative depth") - } - if depth > len(p) { - return nil, Error.New("depth greater than path length") - } - - derivedKey = key - for i := 0; i < depth; i++ { - derivedKey, err = deriveSecret(derivedKey, p[i]) - if err != nil { - return nil, err - } - } - return derivedKey, nil -} - -// DeriveContentKey derives the key for the encrypted object data using the root key -// This method must be called on an unencrypted path. -func (p Path) DeriveContentKey(key []byte) (derivedKey *[32]byte, err error) { - if len(p) == 0 { - return nil, Error.New("path is empty") - } - d, err := p.DeriveKey(key, len(p)-1) - if err != nil { - return nil, err - } - d, err = deriveSecret(d, "content") - if err != nil { - return nil, err - } - - derivedKey = new([32]byte) - copy((*derivedKey)[:], d[:32]) - return derivedKey, nil -} - -func encrypt(text string, secret []byte) (cipherText string, err error) { - key, nonce, err := getAESGCMKeyAndNonce(secret) - if err != nil { - return "", Error.Wrap(err) - } - block, err := aes.NewCipher(key) - if err != nil { - return "", Error.Wrap(err) - } - aesgcm, err := cipher.NewGCM(block) - if err != nil { - return "", Error.Wrap(err) - } - data := aesgcm.Seal(nil, nonce, []byte(text), nil) - cipherText = base64.RawURLEncoding.EncodeToString(data) - // prepend version number to the cipher text - return "1" + cipherText, nil -} - -func decrypt(cipherText string, secret []byte) (text string, err error) { - if cipherText == "" { - return "", Error.New("empty cipher text") - } - // check the version number, only "1" is supported for now - if cipherText[0] != '1' { - return "", Error.New("invalid version number") - } - data, err := base64.RawURLEncoding.DecodeString(cipherText[1:]) - if err != nil { - return "", Error.Wrap(err) - } - key, nonce, err := getAESGCMKeyAndNonce(secret) - if err != nil { - return "", Error.Wrap(err) - } - block, err := aes.NewCipher(key) - if err != nil { - return "", Error.Wrap(err) - } - aesgcm, err := cipher.NewGCM(block) - if err != nil { - return "", Error.Wrap(err) - } - decrypted, err := aesgcm.Open(nil, nonce, data, nil) - if err != nil { - return "", Error.Wrap(err) - } - return string(decrypted), nil -} - -func getAESGCMKeyAndNonce(secret []byte) (key, nonce []byte, err error) { - mac := hmac.New(sha512.New, secret) - _, err = mac.Write([]byte("enc")) - if err != nil { - return nil, nil, Error.Wrap(err) - } - key = mac.Sum(nil)[:32] - mac.Reset() - _, err = mac.Write([]byte("nonce")) - if err != nil { - return nil, nil, Error.Wrap(err) - } - nonce = mac.Sum(nil)[:12] - return key, nonce, nil -} - -func deriveSecret(secret []byte, child string) (derived []byte, err error) { - mac := hmac.New(sha512.New, secret) - _, err = mac.Write([]byte(child)) - if err != nil { - return nil, Error.Wrap(err) - } - return mac.Sum(nil), nil -} diff --git a/pkg/paths/paths_test.go b/pkg/paths/paths_test.go deleted file mode 100644 index e4f0dbffd..000000000 --- a/pkg/paths/paths_test.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright (C) 2018 Storj Labs, Inc. -// See LICENSE for copying information. - -package paths - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNew(t *testing.T) { - for i, tt := range []struct { - path string - expected Path - }{ - {"", []string{}}, - {"/", []string{}}, - {"a", []string{"a"}}, - {"/a/", []string{"a"}}, - {"a/b/c/d", []string{"a", "b", "c", "d"}}, - {"///a//b////c/d///", []string{"a", "b", "c", "d"}}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - p := New(tt.path) - assert.Equal(t, tt.expected, p, errTag) - } -} - -func TestNewWithSegments(t *testing.T) { - for i, tt := range []struct { - segs []string - expected Path - }{ - {nil, []string{}}, - {[]string{""}, []string{}}, - {[]string{"", ""}, []string{}}, - {[]string{"/"}, []string{}}, - {[]string{"a"}, []string{"a"}}, - {[]string{"/a/"}, []string{"a"}}, - {[]string{"", "a", "", "b", "c", "d", ""}, []string{"a", "b", "c", "d"}}, - {[]string{"a", "b", "c", "d"}, []string{"a", "b", "c", "d"}}, - {[]string{"/a", "b/", "/c/", "d"}, []string{"a", "b", "c", "d"}}, - {[]string{"a/b", "c", "d"}, []string{"a", "b", "c", "d"}}, - {[]string{"a/b", "c/d"}, []string{"a", "b", "c", "d"}}, - {[]string{"//a/b", "c///d//"}, []string{"a", "b", "c", "d"}}, - {[]string{"a/b/c/d"}, []string{"a", "b", "c", "d"}}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - p := New(tt.segs...) - assert.Equal(t, tt.expected, p, errTag) - } -} - -func TestString(t *testing.T) { - for i, tt := range []struct { - path Path - expected string - }{ - {nil, ""}, - {[]string{}, ""}, - {[]string{""}, ""}, - {[]string{"a"}, "a"}, - {[]string{"a", "b"}, "a/b"}, - {[]string{"a", "b", "c", "d"}, "a/b/c/d"}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - s := tt.path.String() - assert.Equal(t, tt.expected, s, errTag) - } -} - -func TestBytes(t *testing.T) { - for i, tt := range []struct { - path Path - expected []byte - }{ - {nil, []byte{}}, - {[]string{}, []byte{}}, - {[]string{""}, []byte{}}, - {[]string{"a/b"}, []byte{97, 47, 98}}, - {[]string{"a/b/c"}, []byte{97, 47, 98, 47, 99}}, - {[]string{"a/b/c/d/e/f"}, []byte{97, 47, 98, 47, 99, 47, 100, 47, 101, 47, 102}}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - b := tt.path.Bytes() - assert.Equal(t, tt.expected, b, errTag) - } -} - -func TestPrepend(t *testing.T) { - for i, tt := range []struct { - prefix string - path string - expected Path - }{ - {"", "", []string{}}, - {"prefix", "", []string{"prefix"}}, - {"", "my/path", []string{"my", "path"}}, - {"prefix", "my/path", []string{"prefix", "my", "path"}}, - {"p1/p2/p3", "my/path", []string{"p1", "p2", "p3", "my", "path"}}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - p := New(tt.path).Prepend(tt.prefix) - assert.Equal(t, tt.expected, p, errTag) - } -} - -func TestPrependWithSegments(t *testing.T) { - for i, tt := range []struct { - segs []string - path string - expected Path - }{ - {nil, "", []string{}}, - {[]string{""}, "", []string{}}, - {[]string{"prefix"}, "", []string{"prefix"}}, - {[]string{""}, "my/path", []string{"my", "path"}}, - {[]string{"prefix"}, "my/path", []string{"prefix", "my", "path"}}, - {[]string{"p1/p2/p3"}, "my/path", []string{"p1", "p2", "p3", "my", "path"}}, - {[]string{"p1", "p2/p3"}, "my/path", []string{"p1", "p2", "p3", "my", "path"}}, - {[]string{"p1", "p2", "p3"}, "my/path", []string{"p1", "p2", "p3", "my", "path"}}, - {[]string{"", "p1", "", "", "p2", "p3", ""}, "my/path", []string{"p1", "p2", "p3", "my", "path"}}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - p := New(tt.path).Prepend(tt.segs...) - assert.Equal(t, tt.expected, p, errTag) - } -} - -func TestAppend(t *testing.T) { - for i, tt := range []struct { - path string - suffix string - expected Path - }{ - {"", "", []string{}}, - {"", "suffix", []string{"suffix"}}, - {"my/path", "", []string{"my", "path"}}, - {"my/path", "suffix", []string{"my", "path", "suffix"}}, - {"my/path", "s1/s2/s3", []string{"my", "path", "s1", "s2", "s3"}}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - p := New(tt.path).Append(tt.suffix) - assert.Equal(t, tt.expected, p, errTag) - } -} - -func TestAppendWithSegments(t *testing.T) { - for i, tt := range []struct { - path string - segs []string - expected Path - }{ - {"", nil, []string{}}, - {"", []string{""}, []string{}}, - {"", []string{"suffix"}, []string{"suffix"}}, - {"my/path", []string{""}, []string{"my", "path"}}, - {"my/path", []string{"suffix"}, []string{"my", "path", "suffix"}}, - {"my/path", []string{"s1/s2/s3"}, []string{"my", "path", "s1", "s2", "s3"}}, - {"my/path", []string{"s1", "s2/s3"}, []string{"my", "path", "s1", "s2", "s3"}}, - {"my/path", []string{"s1", "s2", "s3"}, []string{"my", "path", "s1", "s2", "s3"}}, - {"my/path", []string{"", "s1", "", "", "s2", "s3", ""}, []string{"my", "path", "s1", "s2", "s3"}}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - p := New(tt.path).Append(tt.segs...) - assert.Equal(t, tt.expected, p, errTag) - } -} - -func TestHasPrefix(t *testing.T) { - for i, tt := range []struct { - path string - prefix string - expected bool - }{ - {"", "", true}, - {"my/path", "", true}, - {"", "prefix", false}, - {"prefix/path", "prefix", true}, - {"prefix/path", "prefix/path", true}, - {"prefix/path", "prefix/path/more", false}, - {"my/path/s1/s2/s3", "my/path", true}, - {"my/path/s1/s2/s3", "s1/s2/s3", false}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - p := New(tt.path).HasPrefix(New(tt.prefix)) - assert.Equal(t, tt.expected, p, errTag) - } -} - -func TestEncryption(t *testing.T) { - for i, segs := range []Path{ - nil, // empty path - []string{}, // empty path - []string{""}, // empty path segment - []string{"file.txt"}, - []string{"fold1", "file.txt"}, - []string{"fold1", "fold2", "file.txt"}, - []string{"fold1", "fold2", "fold3", "file.txt"}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - path := New(segs...) - key := []byte("my secret") - encrypted, err := path.Encrypt(key) - if !assert.NoError(t, err, errTag) { - continue - } - decrypted, err := encrypted.Decrypt(key) - if !assert.NoError(t, err, errTag) { - continue - } - assert.Equal(t, path, decrypted, errTag) - } -} - -func TestDeriveKey(t *testing.T) { - for i, tt := range []struct { - path Path - depth int - errString string - }{ - {[]string{"fold1", "fold2", "fold3", "file.txt"}, -1, - "paths error: negative depth"}, - {[]string{"fold1", "fold2", "fold3", "file.txt"}, 0, ""}, - {[]string{"fold1", "fold2", "fold3", "file.txt"}, 1, ""}, - {[]string{"fold1", "fold2", "fold3", "file.txt"}, 2, ""}, - {[]string{"fold1", "fold2", "fold3", "file.txt"}, 3, ""}, - {[]string{"fold1", "fold2", "fold3", "file.txt"}, 4, ""}, - {[]string{"fold1", "fold2", "fold3", "file.txt"}, 5, - "paths error: depth greater than path length"}, - } { - errTag := fmt.Sprintf("Test case #%d", i) - key := []byte("my secret") - encrypted, err := tt.path.Encrypt(key) - if !assert.NoError(t, err, errTag) { - continue - } - derivedKey, err := tt.path.DeriveKey(key, tt.depth) - if tt.errString != "" { - assert.EqualError(t, err, tt.errString, errTag) - continue - } - if !assert.NoError(t, err, errTag) { - continue - } - shared := encrypted[tt.depth:] - decrypted, err := shared.Decrypt(derivedKey) - if !assert.NoError(t, err, errTag) { - continue - } - assert.Equal(t, tt.path[tt.depth:], decrypted, errTag) - } -} diff --git a/pkg/pointerdb/pdbclient/client.go b/pkg/pointerdb/pdbclient/client.go index 4b242f67f..6cfad5627 100644 --- a/pkg/pointerdb/pdbclient/client.go +++ b/pkg/pointerdb/pdbclient/client.go @@ -17,9 +17,9 @@ import ( "storj.io/storj/pkg/auth" "storj.io/storj/pkg/auth/grpcauth" - p "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/provider" + "storj.io/storj/pkg/storj" "storj.io/storj/storage" ) @@ -44,19 +44,17 @@ var _ Client = (*PointerDB)(nil) // ListItem is a single item in a listing type ListItem struct { - Path p.Path + Path storj.Path Pointer *pb.Pointer IsPrefix bool } // Client services offerred for the interface type Client interface { - Put(ctx context.Context, path p.Path, pointer *pb.Pointer) error - Get(ctx context.Context, path p.Path) (*pb.Pointer, error) - List(ctx context.Context, prefix, startAfter, endBefore p.Path, - recursive bool, limit int, metaFlags uint32) ( - items []ListItem, more bool, err error) - Delete(ctx context.Context, path p.Path) error + Put(ctx context.Context, path storj.Path, pointer *pb.Pointer) error + Get(ctx context.Context, path storj.Path) (*pb.Pointer, error) + List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) + Delete(ctx context.Context, path storj.Path) error SignedMessage() (*pb.SignedMessage, error) } @@ -93,19 +91,19 @@ func clientConnection(serverAddr string, opts ...grpc.DialOption) (pb.PointerDBC } // Put is the interface to make a PUT request, needs Pointer and APIKey -func (pdb *PointerDB) Put(ctx context.Context, path p.Path, pointer *pb.Pointer) (err error) { +func (pdb *PointerDB) Put(ctx context.Context, path storj.Path, pointer *pb.Pointer) (err error) { defer mon.Task()(&ctx)(&err) - _, err = pdb.grpcClient.Put(ctx, &pb.PutRequest{Path: path.String(), Pointer: pointer}) + _, err = pdb.grpcClient.Put(ctx, &pb.PutRequest{Path: path, Pointer: pointer}) return err } // Get is the interface to make a GET request, needs PATH and APIKey -func (pdb *PointerDB) Get(ctx context.Context, path p.Path) (pointer *pb.Pointer, err error) { +func (pdb *PointerDB) Get(ctx context.Context, path storj.Path) (pointer *pb.Pointer, err error) { defer mon.Task()(&ctx)(&err) - res, err := pdb.grpcClient.Get(ctx, &pb.GetRequest{Path: path.String()}) + res, err := pdb.grpcClient.Get(ctx, &pb.GetRequest{Path: path}) if err != nil { if status.Code(err) == codes.NotFound { return nil, storage.ErrKeyNotFound.Wrap(err) @@ -117,15 +115,13 @@ func (pdb *PointerDB) Get(ctx context.Context, path p.Path) (pointer *pb.Pointer } // List is the interface to make a LIST request, needs StartingPathKey, Limit, and APIKey -func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore p.Path, - recursive bool, limit int, metaFlags uint32) ( - items []ListItem, more bool, err error) { +func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) { defer mon.Task()(&ctx)(&err) res, err := pdb.grpcClient.List(ctx, &pb.ListRequest{ - Prefix: prefix.String(), - StartAfter: startAfter.String(), - EndBefore: endBefore.String(), + Prefix: prefix, + StartAfter: startAfter, + EndBefore: endBefore, Recursive: recursive, Limit: int32(limit), MetaFlags: metaFlags, @@ -138,7 +134,7 @@ func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore p. items = make([]ListItem, len(list)) for i, itm := range list { items[i] = ListItem{ - Path: p.New(itm.GetPath()), + Path: itm.GetPath(), Pointer: itm.GetPointer(), IsPrefix: itm.IsPrefix, } @@ -148,10 +144,10 @@ func (pdb *PointerDB) List(ctx context.Context, prefix, startAfter, endBefore p. } // Delete is the interface to make a Delete request, needs Path and APIKey -func (pdb *PointerDB) Delete(ctx context.Context, path p.Path) (err error) { +func (pdb *PointerDB) Delete(ctx context.Context, path storj.Path) (err error) { defer mon.Task()(&ctx)(&err) - _, err = pdb.grpcClient.Delete(ctx, &pb.DeleteRequest{Path: path.String()}) + _, err = pdb.grpcClient.Delete(ctx, &pb.DeleteRequest{Path: path}) return err } diff --git a/pkg/pointerdb/pdbclient/client_test.go b/pkg/pointerdb/pdbclient/client_test.go index 372b1ae88..e6be3dd72 100644 --- a/pkg/pointerdb/pdbclient/client_test.go +++ b/pkg/pointerdb/pdbclient/client_test.go @@ -25,10 +25,10 @@ import ( "google.golang.org/grpc/peer" "storj.io/storj/pkg/auth" - p "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/provider" "storj.io/storj/pkg/storage/meta" + "storj.io/storj/pkg/storj" ) const ( @@ -54,7 +54,7 @@ func TestNewPointerDBClient(t *testing.T) { assert.NotNil(t, pdb.grpcClient) } -func makePointer(path p.Path) pb.PutRequest { +func makePointer(path storj.Path) pb.PutRequest { // rps is an example slice of RemotePieces to add to this // REMOTE pointer type. var rps []*pb.RemotePiece @@ -63,7 +63,7 @@ func makePointer(path p.Path) pb.PutRequest { NodeId: "testId", }) pr := pb.PutRequest{ - Path: path.String(), + Path: path, Pointer: &pb.Pointer{ Type: pb.Pointer_REMOTE, Remote: &pb.RemoteSegment{ @@ -89,15 +89,15 @@ func TestPut(t *testing.T) { for i, tt := range []struct { APIKey []byte - path p.Path + path storj.Path err error errString string }{ - {[]byte("abc123"), p.New("file1/file2"), nil, ""}, - {[]byte("wrong key"), p.New("file1/file2"), ErrUnauthenticated, unauthenticated}, - {[]byte("abc123"), p.New(""), ErrNoFileGiven, noPathGiven}, - {[]byte("wrong key"), p.New(""), ErrUnauthenticated, unauthenticated}, - {[]byte(""), p.New(""), ErrUnauthenticated, unauthenticated}, + {[]byte("abc123"), "file1/file2", nil, ""}, + {[]byte("wrong key"), "file1/file2", ErrUnauthenticated, unauthenticated}, + {[]byte("abc123"), "", ErrNoFileGiven, noPathGiven}, + {[]byte("wrong key"), "", ErrUnauthenticated, unauthenticated}, + {[]byte(""), "", ErrUnauthenticated, unauthenticated}, } { ctx := context.Background() ctx = auth.WithAPIKey(ctx, tt.APIKey) @@ -127,21 +127,21 @@ func TestGet(t *testing.T) { for i, tt := range []struct { APIKey []byte - path p.Path + path storj.Path err error errString string }{ - {[]byte("wrong key"), p.New("file1/file2"), ErrUnauthenticated, unauthenticated}, - {[]byte("abc123"), p.New(""), ErrNoFileGiven, noPathGiven}, - {[]byte("wrong key"), p.New(""), ErrUnauthenticated, unauthenticated}, - {[]byte(""), p.New(""), ErrUnauthenticated, unauthenticated}, - {[]byte("abc123"), p.New("file1/file2"), nil, ""}, + {[]byte("wrong key"), "file1/file2", ErrUnauthenticated, unauthenticated}, + {[]byte("abc123"), "", ErrNoFileGiven, noPathGiven}, + {[]byte("wrong key"), "", ErrUnauthenticated, unauthenticated}, + {[]byte(""), "", ErrUnauthenticated, unauthenticated}, + {[]byte("abc123"), "file1/file2", nil, ""}, } { ctx := context.Background() ctx = auth.WithAPIKey(ctx, tt.APIKey) getPointer := makePointer(tt.path) - getRequest := pb.GetRequest{Path: tt.path.String()} + getRequest := pb.GetRequest{Path: tt.path} data, err := proto.Marshal(getPointer.Pointer) if err != nil { @@ -239,8 +239,7 @@ func TestList(t *testing.T) { gc.EXPECT().List(gomock.Any(), &listRequest).Return(&listResponse, tt.err) - items, more, err := pdb.List(ctx, p.New(tt.prefix), p.New(tt.startAfter), - p.New(tt.endBefore), tt.recursive, tt.limit, tt.metaFlags) + items, more, err := pdb.List(ctx, tt.prefix, tt.startAfter, tt.endBefore, tt.recursive, tt.limit, tt.metaFlags) if err != nil { assert.EqualError(t, err, tt.errString, errTag) @@ -253,13 +252,10 @@ func TestList(t *testing.T) { assert.Equal(t, len(tt.items), len(items)) for i := 0; i < len(items); i++ { - assert.Equal(t, tt.items[i].GetPath(), items[i].Path.String()) - assert.Equal(t, tt.items[i].GetPointer().GetSize(), - items[i].Pointer.GetSize()) - assert.Equal(t, tt.items[i].GetPointer().GetCreationDate(), - items[i].Pointer.GetCreationDate()) - assert.Equal(t, tt.items[i].GetPointer().GetExpirationDate(), - items[i].Pointer.GetExpirationDate()) + assert.Equal(t, tt.items[i].GetPath(), items[i].Path) + assert.Equal(t, tt.items[i].GetPointer().GetSize(), items[i].Pointer.GetSize()) + assert.Equal(t, tt.items[i].GetPointer().GetCreationDate(), items[i].Pointer.GetCreationDate()) + assert.Equal(t, tt.items[i].GetPointer().GetExpirationDate(), items[i].Pointer.GetExpirationDate()) } } } @@ -271,20 +267,20 @@ func TestDelete(t *testing.T) { for i, tt := range []struct { APIKey []byte - path p.Path + path storj.Path err error errString string }{ - {[]byte("wrong key"), p.New("file1/file2"), ErrUnauthenticated, unauthenticated}, - {[]byte("abc123"), p.New(""), ErrNoFileGiven, noPathGiven}, - {[]byte("wrong key"), p.New(""), ErrUnauthenticated, unauthenticated}, - {[]byte(""), p.New(""), ErrUnauthenticated, unauthenticated}, - {[]byte("abc123"), p.New("file1/file2"), nil, ""}, + {[]byte("wrong key"), "file1/file2", ErrUnauthenticated, unauthenticated}, + {[]byte("abc123"), "", ErrNoFileGiven, noPathGiven}, + {[]byte("wrong key"), "", ErrUnauthenticated, unauthenticated}, + {[]byte(""), "", ErrUnauthenticated, unauthenticated}, + {[]byte("abc123"), "file1/file2", nil, ""}, } { ctx := context.Background() ctx = auth.WithAPIKey(ctx, tt.APIKey) - deleteRequest := pb.DeleteRequest{Path: tt.path.String()} + deleteRequest := pb.DeleteRequest{Path: tt.path} errTag := fmt.Sprintf("Test case #%d", i) gc := NewMockPointerDBClient(ctrl) diff --git a/pkg/pointerdb/pdbclient/mocks/mock_client.go b/pkg/pointerdb/pdbclient/mocks/mock_client.go index 8cd9b1327..ca196e7ab 100644 --- a/pkg/pointerdb/pdbclient/mocks/mock_client.go +++ b/pkg/pointerdb/pdbclient/mocks/mock_client.go @@ -6,11 +6,8 @@ package mock_pointerdb import ( context "context" - "reflect" - gomock "github.com/golang/mock/gomock" - - paths "storj.io/storj/pkg/paths" + reflect "reflect" pb "storj.io/storj/pkg/pb" pdbclient "storj.io/storj/pkg/pointerdb/pdbclient" ) @@ -39,7 +36,7 @@ func (m *MockClient) EXPECT() *MockClientMockRecorder { } // Delete mocks base method -func (m *MockClient) Delete(arg0 context.Context, arg1 paths.Path) error { +func (m *MockClient) Delete(arg0 context.Context, arg1 string) error { ret := m.ctrl.Call(m, "Delete", arg0, arg1) ret0, _ := ret[0].(error) return ret0 @@ -51,7 +48,7 @@ func (mr *MockClientMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { } // Get mocks base method -func (m *MockClient) Get(arg0 context.Context, arg1 paths.Path) (*pb.Pointer, error) { +func (m *MockClient) Get(arg0 context.Context, arg1 string) (*pb.Pointer, error) { ret := m.ctrl.Call(m, "Get", arg0, arg1) ret0, _ := ret[0].(*pb.Pointer) ret1, _ := ret[1].(error) @@ -64,7 +61,7 @@ func (mr *MockClientMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { } // List mocks base method -func (m *MockClient) List(arg0 context.Context, arg1, arg2, arg3 paths.Path, arg4 bool, arg5 int, arg6 uint32) ([]pdbclient.ListItem, bool, error) { +func (m *MockClient) List(arg0 context.Context, arg1, arg2, arg3 string, arg4 bool, arg5 int, arg6 uint32) ([]pdbclient.ListItem, bool, error) { ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3, arg4, arg5, arg6) ret0, _ := ret[0].([]pdbclient.ListItem) ret1, _ := ret[1].(bool) @@ -78,7 +75,7 @@ func (mr *MockClientMockRecorder) List(arg0, arg1, arg2, arg3, arg4, arg5, arg6 } // Put mocks base method -func (m *MockClient) Put(arg0 context.Context, arg1 paths.Path, arg2 *pb.Pointer) error { +func (m *MockClient) Put(arg0 context.Context, arg1 string, arg2 *pb.Pointer) error { ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2) ret0, _ := ret[0].(error) return ret0 diff --git a/pkg/pointerdb/pointerdb_test.go b/pkg/pointerdb/pointerdb_test.go index 8d6b1ce81..8ca429a5f 100644 --- a/pkg/pointerdb/pointerdb_test.go +++ b/pkg/pointerdb/pointerdb_test.go @@ -18,7 +18,6 @@ import ( "google.golang.org/grpc/status" "storj.io/storj/pkg/auth" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/storage/meta" "storj.io/storj/storage" @@ -144,10 +143,6 @@ func TestServiceList(t *testing.T) { db := teststore.New() server := Server{DB: db, logger: zap.NewNop()} - key := func(s string) storage.Key { - return storage.Key(paths.New(s).Bytes()) - } - pointer := &pb.Pointer{} pointer.CreationDate = ptypes.TimestampNow() @@ -158,13 +153,13 @@ func TestServiceList(t *testing.T) { pointerValue := storage.Value(pointerBytes) err = storage.PutAll(db, []storage.ListItem{ - {Key: key("sample.😶"), Value: pointerValue}, - {Key: key("müsic"), Value: pointerValue}, - {Key: key("müsic/söng1.mp3"), Value: pointerValue}, - {Key: key("müsic/söng2.mp3"), Value: pointerValue}, - {Key: key("müsic/album/söng3.mp3"), Value: pointerValue}, - {Key: key("müsic/söng4.mp3"), Value: pointerValue}, - {Key: key("ビデオ/movie.mkv"), Value: pointerValue}, + {Key: storage.Key("sample.😶"), Value: pointerValue}, + {Key: storage.Key("müsic"), Value: pointerValue}, + {Key: storage.Key("müsic/söng1.mp3"), Value: pointerValue}, + {Key: storage.Key("müsic/söng2.mp3"), Value: pointerValue}, + {Key: storage.Key("müsic/album/söng3.mp3"), Value: pointerValue}, + {Key: storage.Key("müsic/söng4.mp3"), Value: pointerValue}, + {Key: storage.Key("ビデオ/movie.mkv"), Value: pointerValue}, }...) if err != nil { t.Fatal(err) diff --git a/pkg/storage/buckets/prefixed.go b/pkg/storage/buckets/prefixed.go index ae53fdffa..2e9712a4e 100644 --- a/pkg/storage/buckets/prefixed.go +++ b/pkg/storage/buckets/prefixed.go @@ -8,9 +8,9 @@ import ( "io" "time" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/objects" + "storj.io/storj/pkg/storj" ) type prefixedObjStore struct { @@ -18,55 +18,50 @@ type prefixedObjStore struct { prefix string } -func (o *prefixedObjStore) Meta(ctx context.Context, path paths.Path) (meta objects.Meta, - err error) { +func (o *prefixedObjStore) Meta(ctx context.Context, path storj.Path) (meta objects.Meta, err error) { defer mon.Task()(&ctx)(&err) if len(path) == 0 { return objects.Meta{}, objects.NoPathError.New("") } - m, err := o.o.Meta(ctx, path.Prepend(o.prefix)) + m, err := o.o.Meta(ctx, storj.JoinPaths(o.prefix, path)) return m, err } -func (o *prefixedObjStore) Get(ctx context.Context, path paths.Path) ( - rr ranger.Ranger, meta objects.Meta, err error) { +func (o *prefixedObjStore) Get(ctx context.Context, path storj.Path) (rr ranger.Ranger, meta objects.Meta, err error) { defer mon.Task()(&ctx)(&err) if len(path) == 0 { return nil, objects.Meta{}, objects.NoPathError.New("") } - rr, m, err := o.o.Get(ctx, path.Prepend(o.prefix)) + rr, m, err := o.o.Get(ctx, storj.JoinPaths(o.prefix, path)) return rr, m, err } -func (o *prefixedObjStore) Put(ctx context.Context, path paths.Path, data io.Reader, - metadata objects.SerializableMeta, expiration time.Time) (meta objects.Meta, err error) { +func (o *prefixedObjStore) Put(ctx context.Context, path storj.Path, data io.Reader, metadata objects.SerializableMeta, expiration time.Time) (meta objects.Meta, err error) { defer mon.Task()(&ctx)(&err) if len(path) == 0 { return objects.Meta{}, objects.NoPathError.New("") } - m, err := o.o.Put(ctx, path.Prepend(o.prefix), data, metadata, expiration) + m, err := o.o.Put(ctx, storj.JoinPaths(o.prefix, path), data, metadata, expiration) return m, err } -func (o *prefixedObjStore) Delete(ctx context.Context, path paths.Path) (err error) { +func (o *prefixedObjStore) Delete(ctx context.Context, path storj.Path) (err error) { defer mon.Task()(&ctx)(&err) if len(path) == 0 { return objects.NoPathError.New("") } - return o.o.Delete(ctx, path.Prepend(o.prefix)) + return o.o.Delete(ctx, storj.JoinPaths(o.prefix, path)) } -func (o *prefixedObjStore) List(ctx context.Context, prefix, startAfter, - endBefore paths.Path, recursive bool, limit int, metaFlags uint32) ( - items []objects.ListItem, more bool, err error) { +func (o *prefixedObjStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []objects.ListItem, more bool, err error) { defer mon.Task()(&ctx)(&err) - return o.o.List(ctx, prefix.Prepend(o.prefix), startAfter, endBefore, recursive, limit, metaFlags) + return o.o.List(ctx, storj.JoinPaths(o.prefix, prefix), startAfter, endBefore, recursive, limit, metaFlags) } diff --git a/pkg/storage/buckets/store.go b/pkg/storage/buckets/store.go index 817d8cb58..9f8453749 100644 --- a/pkg/storage/buckets/store.go +++ b/pkg/storage/buckets/store.go @@ -12,7 +12,6 @@ import ( "github.com/zeebo/errs" monkit "gopkg.in/spacemonkeygo/monkit.v2" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/storage/meta" "storj.io/storj/pkg/storage/objects" "storj.io/storj/storage" @@ -81,8 +80,7 @@ func (b *BucketStore) Get(ctx context.Context, bucket string) (meta Meta, err er return Meta{}, NoBucketError.New("") } - p := paths.New(bucket) - objMeta, err := b.o.Meta(ctx, p) + objMeta, err := b.o.Meta(ctx, bucket) if err != nil { return Meta{}, err } @@ -97,10 +95,9 @@ func (b *BucketStore) Put(ctx context.Context, bucket string) (meta Meta, err er return Meta{}, NoBucketError.New("") } - p := paths.New(bucket) r := bytes.NewReader(nil) var exp time.Time - m, err := b.o.Put(ctx, p, r, objects.SerializableMeta{}, exp) + m, err := b.o.Put(ctx, bucket, r, objects.SerializableMeta{}, exp) if err != nil { return Meta{}, err } @@ -115,15 +112,14 @@ func (b *BucketStore) Delete(ctx context.Context, bucket string) (err error) { return NoBucketError.New("") } - p := paths.New(bucket) - return b.o.Delete(ctx, p) + return b.o.Delete(ctx, bucket) } // List calls objects store List func (b *BucketStore) List(ctx context.Context, startAfter, endBefore string, limit int) (items []ListItem, more bool, err error) { defer mon.Task()(&ctx)(&err) - objItems, more, err := b.o.List(ctx, nil, paths.New(startAfter), paths.New(endBefore), false, limit, meta.Modified) + objItems, more, err := b.o.List(ctx, "", startAfter, endBefore, false, limit, meta.Modified) if err != nil { return items, more, err } @@ -134,7 +130,7 @@ func (b *BucketStore) List(ctx context.Context, startAfter, endBefore string, li continue } items = append(items, ListItem{ - Bucket: itm.Path.String(), + Bucket: itm.Path, Meta: convertMeta(itm.Meta), }) } diff --git a/pkg/storage/objects/store.go b/pkg/storage/objects/store.go index a5a7d640c..85c400be7 100644 --- a/pkg/storage/objects/store.go +++ b/pkg/storage/objects/store.go @@ -8,14 +8,14 @@ import ( "io" "time" - "github.com/gogo/protobuf/proto" + "github.com/golang/protobuf/proto" "github.com/zeebo/errs" "go.uber.org/zap" monkit "gopkg.in/spacemonkeygo/monkit.v2" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/streams" + "storj.io/storj/pkg/storj" ) var mon = monkit.Package() @@ -34,22 +34,18 @@ type Meta struct { // ListItem is a single item in a listing type ListItem struct { - Path paths.Path + Path storj.Path Meta Meta IsPrefix bool } // Store for objects type Store interface { - Meta(ctx context.Context, path paths.Path) (meta Meta, err error) - Get(ctx context.Context, path paths.Path) (rr ranger.Ranger, - meta Meta, err error) - Put(ctx context.Context, path paths.Path, data io.Reader, - metadata SerializableMeta, expiration time.Time) (meta Meta, err error) - Delete(ctx context.Context, path paths.Path) (err error) - List(ctx context.Context, prefix, startAfter, endBefore paths.Path, - recursive bool, limit int, metaFlags uint32) (items []ListItem, - more bool, err error) + Meta(ctx context.Context, path storj.Path) (meta Meta, err error) + Get(ctx context.Context, path storj.Path) (rr ranger.Ranger, meta Meta, err error) + Put(ctx context.Context, path storj.Path, data io.Reader, metadata SerializableMeta, expiration time.Time) (meta Meta, err error) + Delete(ctx context.Context, path storj.Path) (err error) + List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) } type objStore struct { @@ -61,8 +57,7 @@ func NewStore(store streams.Store) Store { return &objStore{s: store} } -func (o *objStore) Meta(ctx context.Context, path paths.Path) (meta Meta, - err error) { +func (o *objStore) Meta(ctx context.Context, path storj.Path) (meta Meta, err error) { defer mon.Task()(&ctx)(&err) if len(path) == 0 { @@ -73,7 +68,7 @@ func (o *objStore) Meta(ctx context.Context, path paths.Path) (meta Meta, return convertMeta(m), err } -func (o *objStore) Get(ctx context.Context, path paths.Path) ( +func (o *objStore) Get(ctx context.Context, path storj.Path) ( rr ranger.Ranger, meta Meta, err error) { defer mon.Task()(&ctx)(&err) @@ -85,8 +80,7 @@ func (o *objStore) Get(ctx context.Context, path paths.Path) ( return rr, convertMeta(m), err } -func (o *objStore) Put(ctx context.Context, path paths.Path, data io.Reader, - metadata SerializableMeta, expiration time.Time) (meta Meta, err error) { +func (o *objStore) Put(ctx context.Context, path storj.Path, data io.Reader, metadata SerializableMeta, expiration time.Time) (meta Meta, err error) { defer mon.Task()(&ctx)(&err) if len(path) == 0 { @@ -96,7 +90,6 @@ func (o *objStore) Put(ctx context.Context, path paths.Path, data io.Reader, // TODO(kaloyan): autodetect content type // if metadata.GetContentType() == "" {} - // TODO(kaloyan): encrypt metadata.UserDefined before serializing b, err := proto.Marshal(&metadata) if err != nil { return Meta{}, err @@ -105,7 +98,7 @@ func (o *objStore) Put(ctx context.Context, path paths.Path, data io.Reader, return convertMeta(m), err } -func (o *objStore) Delete(ctx context.Context, path paths.Path) (err error) { +func (o *objStore) Delete(ctx context.Context, path storj.Path) (err error) { defer mon.Task()(&ctx)(&err) if len(path) == 0 { @@ -115,8 +108,7 @@ func (o *objStore) Delete(ctx context.Context, path paths.Path) (err error) { return o.s.Delete(ctx, path) } -func (o *objStore) List(ctx context.Context, prefix, startAfter, - endBefore paths.Path, recursive bool, limit int, metaFlags uint32) ( +func (o *objStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) ( items []ListItem, more bool, err error) { defer mon.Task()(&ctx)(&err) diff --git a/pkg/storage/segments/mock_store.go b/pkg/storage/segments/mock_store.go index ea0b50747..7dcf286b6 100644 --- a/pkg/storage/segments/mock_store.go +++ b/pkg/storage/segments/mock_store.go @@ -12,7 +12,6 @@ import ( gomock "github.com/golang/mock/gomock" - paths "storj.io/storj/pkg/paths" ranger "storj.io/storj/pkg/ranger" ) @@ -40,7 +39,7 @@ func (m *MockStore) EXPECT() *MockStoreMockRecorder { } // Delete mocks base method -func (m *MockStore) Delete(arg0 context.Context, arg1 paths.Path) error { +func (m *MockStore) Delete(arg0 context.Context, arg1 string) error { ret := m.ctrl.Call(m, "Delete", arg0, arg1) ret0, _ := ret[0].(error) return ret0 @@ -52,7 +51,7 @@ func (mr *MockStoreMockRecorder) Delete(arg0, arg1 interface{}) *gomock.Call { } // Get mocks base method -func (m *MockStore) Get(arg0 context.Context, arg1 paths.Path) (ranger.Ranger, Meta, error) { +func (m *MockStore) Get(arg0 context.Context, arg1 string) (ranger.Ranger, Meta, error) { ret := m.ctrl.Call(m, "Get", arg0, arg1) ret0, _ := ret[0].(ranger.Ranger) ret1, _ := ret[1].(Meta) @@ -66,7 +65,7 @@ func (mr *MockStoreMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { } // List mocks base method -func (m *MockStore) List(arg0 context.Context, arg1, arg2, arg3 paths.Path, arg4 bool, arg5 int, arg6 uint32) ([]ListItem, bool, error) { +func (m *MockStore) List(arg0 context.Context, arg1, arg2, arg3 string, arg4 bool, arg5 int, arg6 uint32) ([]ListItem, bool, error) { ret := m.ctrl.Call(m, "List", arg0, arg1, arg2, arg3, arg4, arg5, arg6) ret0, _ := ret[0].([]ListItem) ret1, _ := ret[1].(bool) @@ -80,7 +79,7 @@ func (mr *MockStoreMockRecorder) List(arg0, arg1, arg2, arg3, arg4, arg5, arg6 i } // Meta mocks base method -func (m *MockStore) Meta(arg0 context.Context, arg1 paths.Path) (Meta, error) { +func (m *MockStore) Meta(arg0 context.Context, arg1 string) (Meta, error) { ret := m.ctrl.Call(m, "Meta", arg0, arg1) ret0, _ := ret[0].(Meta) ret1, _ := ret[1].(error) @@ -93,7 +92,7 @@ func (mr *MockStoreMockRecorder) Meta(arg0, arg1 interface{}) *gomock.Call { } // Put mocks base method -func (m *MockStore) Put(arg0 context.Context, arg1 io.Reader, arg2 time.Time, arg3 func() (paths.Path, []byte, error)) (Meta, error) { +func (m *MockStore) Put(arg0 context.Context, arg1 io.Reader, arg2 time.Time, arg3 func() (string, []byte, error)) (Meta, error) { ret := m.ctrl.Call(m, "Put", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(Meta) ret1, _ := ret[1].(error) diff --git a/pkg/storage/segments/store.go b/pkg/storage/segments/store.go index 840d86489..06ec985d2 100644 --- a/pkg/storage/segments/store.go +++ b/pkg/storage/segments/store.go @@ -18,12 +18,12 @@ import ( "storj.io/storj/pkg/eestream" "storj.io/storj/pkg/node" "storj.io/storj/pkg/overlay" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/piecestore/rpc/client" "storj.io/storj/pkg/pointerdb/pdbclient" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/ec" + "storj.io/storj/pkg/storj" ) var ( @@ -40,18 +40,18 @@ type Meta struct { // ListItem is a single item in a listing type ListItem struct { - Path paths.Path + Path storj.Path Meta Meta IsPrefix bool } // Store for segments type Store interface { - Meta(ctx context.Context, path paths.Path) (meta Meta, err error) - Get(ctx context.Context, path paths.Path) (rr ranger.Ranger, meta Meta, err error) - Put(ctx context.Context, data io.Reader, expiration time.Time, segmentInfo func() (paths.Path, []byte, error)) (meta Meta, err error) - Delete(ctx context.Context, path paths.Path) (err error) - List(ctx context.Context, prefix, startAfter, endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) + Meta(ctx context.Context, path storj.Path) (meta Meta, err error) + Get(ctx context.Context, path storj.Path) (rr ranger.Ranger, meta Meta, err error) + Put(ctx context.Context, data io.Reader, expiration time.Time, segmentInfo func() (storj.Path, []byte, error)) (meta Meta, err error) + Delete(ctx context.Context, path storj.Path) (err error) + List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) } type segmentStore struct { @@ -69,8 +69,7 @@ func NewSegmentStore(oc overlay.Client, ec ecclient.Client, } // Meta retrieves the metadata of the segment -func (s *segmentStore) Meta(ctx context.Context, path paths.Path) (meta Meta, - err error) { +func (s *segmentStore) Meta(ctx context.Context, path storj.Path) (meta Meta, err error) { defer mon.Task()(&ctx)(&err) pr, err := s.pdb.Get(ctx, path) @@ -82,7 +81,7 @@ func (s *segmentStore) Meta(ctx context.Context, path paths.Path) (meta Meta, } // Put uploads a segment to an erasure code client -func (s *segmentStore) Put(ctx context.Context, data io.Reader, expiration time.Time, segmentInfo func() (paths.Path, []byte, error)) (meta Meta, err error) { +func (s *segmentStore) Put(ctx context.Context, data io.Reader, expiration time.Time, segmentInfo func() (storj.Path, []byte, error)) (meta Meta, err error) { defer mon.Task()(&ctx)(&err) exp, err := ptypes.TimestampProto(expiration) @@ -96,7 +95,7 @@ func (s *segmentStore) Put(ctx context.Context, data io.Reader, expiration time. return Meta{}, err } - var path paths.Path + var path storj.Path var pointer *pb.Pointer if !remoteSized { p, metadata, err := segmentInfo() @@ -193,7 +192,7 @@ func (s *segmentStore) makeRemotePointer(nodes []*pb.Node, pieceID client.PieceI } // Get retrieves a segment using erasure code, overlay, and pointerdb clients -func (s *segmentStore) Get(ctx context.Context, path paths.Path) ( +func (s *segmentStore) Get(ctx context.Context, path storj.Path) ( rr ranger.Ranger, meta Meta, err error) { defer mon.Task()(&ctx)(&err) @@ -240,7 +239,7 @@ func makeErasureScheme(rs *pb.RedundancyScheme) (eestream.ErasureScheme, error) } // Delete tells piece stores to delete a segment and deletes pointer from pointerdb -func (s *segmentStore) Delete(ctx context.Context, path paths.Path) (err error) { +func (s *segmentStore) Delete(ctx context.Context, path storj.Path) (err error) { defer mon.Task()(&ctx)(&err) pr, err := s.pdb.Get(ctx, path) @@ -293,9 +292,7 @@ func (s *segmentStore) lookupNodes(ctx context.Context, seg *pb.RemoteSegment) ( } // List retrieves paths to segments and their metadata stored in the pointerdb -func (s *segmentStore) List(ctx context.Context, prefix, startAfter, - endBefore paths.Path, recursive bool, limit int, metaFlags uint32) ( - items []ListItem, more bool, err error) { +func (s *segmentStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) { defer mon.Task()(&ctx)(&err) pdbItems, more, err := s.pdb.List(ctx, prefix, startAfter, endBefore, recursive, limit, metaFlags) diff --git a/pkg/storage/segments/store_test.go b/pkg/storage/segments/store_test.go index 0f37f5396..b62f50e44 100644 --- a/pkg/storage/segments/store_test.go +++ b/pkg/storage/segments/store_test.go @@ -16,11 +16,12 @@ import ( "storj.io/storj/pkg/eestream" mock_eestream "storj.io/storj/pkg/eestream/mocks" mock_overlay "storj.io/storj/pkg/overlay/mocks" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" pdb "storj.io/storj/pkg/pointerdb/pdbclient" mock_pointerdb "storj.io/storj/pkg/pointerdb/pdbclient/mocks" mock_ecclient "storj.io/storj/pkg/storage/ec/mocks" + "storj.io/storj/pkg/storage/meta" + "storj.io/storj/pkg/storj" ) var ( @@ -67,8 +68,6 @@ func TestSegmentStoreMeta(t *testing.T) { }{ {"path/1/2/3", &pb.Pointer{CreationDate: pExp, ExpirationDate: pExp}, Meta{Modified: mExp, Expiration: mExp}}, } { - p := paths.New(tt.pathInput) - calls := []*gomock.Call{ mockPDB.EXPECT().Get( gomock.Any(), gomock.Any(), @@ -76,7 +75,7 @@ func TestSegmentStoreMeta(t *testing.T) { } gomock.InOrder(calls...) - m, err := ss.Meta(ctx, p) + m, err := ss.Meta(ctx, tt.pathInput) assert.NoError(t, err) assert.Equal(t, m, tt.returnMeta) } @@ -107,9 +106,6 @@ func TestSegmentStorePutRemote(t *testing.T) { ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize} assert.NotNil(t, ss) - p := paths.New(tt.pathInput) - r := strings.NewReader(tt.readerContent) - calls := []*gomock.Call{ mockES.EXPECT().TotalCount().Return(1), mockOC.EXPECT().Choose( @@ -133,8 +129,8 @@ func TestSegmentStorePutRemote(t *testing.T) { } gomock.InOrder(calls...) - _, err := ss.Put(ctx, r, tt.expiration, func() (paths.Path, []byte, error) { - return p, tt.mdInput, nil + _, err := ss.Put(ctx, strings.NewReader(tt.readerContent), tt.expiration, func() (storj.Path, []byte, error) { + return tt.pathInput, tt.mdInput, nil }) assert.NoError(t, err, tt.name) } @@ -165,9 +161,6 @@ func TestSegmentStorePutInline(t *testing.T) { ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize} assert.NotNil(t, ss) - p := paths.New(tt.pathInput) - r := strings.NewReader(tt.readerContent) - calls := []*gomock.Call{ mockPDB.EXPECT().Put( gomock.Any(), gomock.Any(), gomock.Any(), @@ -178,8 +171,8 @@ func TestSegmentStorePutInline(t *testing.T) { } gomock.InOrder(calls...) - _, err := ss.Put(ctx, r, tt.expiration, func() (paths.Path, []byte, error) { - return p, tt.mdInput, nil + _, err := ss.Put(ctx, strings.NewReader(tt.readerContent), tt.expiration, func() (storj.Path, []byte, error) { + return tt.pathInput, tt.mdInput, nil }) assert.NoError(t, err, tt.name) } @@ -214,8 +207,6 @@ func TestSegmentStoreGetInline(t *testing.T) { ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize} assert.NotNil(t, ss) - p := paths.New(tt.pathInput) - calls := []*gomock.Call{ mockPDB.EXPECT().Get( gomock.Any(), gomock.Any(), @@ -230,7 +221,7 @@ func TestSegmentStoreGetInline(t *testing.T) { } gomock.InOrder(calls...) - _, _, err := ss.Get(ctx, p) + _, _, err := ss.Get(ctx, tt.pathInput) assert.NoError(t, err) } } @@ -263,8 +254,6 @@ func TestSegmentStoreGetRemote(t *testing.T) { ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize} assert.NotNil(t, ss) - p := paths.New(tt.pathInput) - calls := []*gomock.Call{ mockPDB.EXPECT().Get( gomock.Any(), gomock.Any(), @@ -294,7 +283,7 @@ func TestSegmentStoreGetRemote(t *testing.T) { } gomock.InOrder(calls...) - _, _, err := ss.Get(ctx, p) + _, _, err := ss.Get(ctx, tt.pathInput) assert.NoError(t, err) } } @@ -328,8 +317,6 @@ func TestSegmentStoreDeleteInline(t *testing.T) { ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize} assert.NotNil(t, ss) - p := paths.New(tt.pathInput) - calls := []*gomock.Call{ mockPDB.EXPECT().Get( gomock.Any(), gomock.Any(), @@ -347,7 +334,7 @@ func TestSegmentStoreDeleteInline(t *testing.T) { } gomock.InOrder(calls...) - err := ss.Delete(ctx, p) + err := ss.Delete(ctx, tt.pathInput) assert.NoError(t, err) } } @@ -380,8 +367,6 @@ func TestSegmentStoreDeleteRemote(t *testing.T) { ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize} assert.NotNil(t, ss) - p := paths.New(tt.pathInput) - calls := []*gomock.Call{ mockPDB.EXPECT().Get( gomock.Any(), gomock.Any(), @@ -414,7 +399,7 @@ func TestSegmentStoreDeleteRemote(t *testing.T) { } gomock.InOrder(calls...) - err := ss.Delete(ctx, p) + err := ss.Delete(ctx, tt.pathInput) assert.NoError(t, err) } } @@ -424,12 +409,12 @@ func TestSegmentStoreList(t *testing.T) { defer ctrl.Finish() for _, tt := range []struct { - prefixInput string - startAfterInput string - thresholdSize int - itemPath string - inlineContent []byte - metadata []byte + prefix string + startAfter string + thresholdSize int + itemPath string + inlineContent []byte + metadata []byte }{ {"bucket1", "s0/path/1", 10, "s0/path/1", []byte("inline"), []byte("metadata")}, } { @@ -444,10 +429,6 @@ func TestSegmentStoreList(t *testing.T) { ss := segmentStore{mockOC, mockEC, mockPDB, rs, tt.thresholdSize} assert.NotNil(t, ss) - prefix := paths.New(tt.prefixInput) - startAfter := paths.New(tt.startAfterInput) - listedPath := paths.New(tt.itemPath) - ti := time.Unix(0, 0).UTC() someTime, err := ptypes.TimestampProto(ti) assert.NoError(t, err) @@ -458,7 +439,7 @@ func TestSegmentStoreList(t *testing.T) { gomock.Any(), gomock.Any(), gomock.Any(), ).Return([]pdb.ListItem{ { - Path: listedPath, + Path: tt.itemPath, Pointer: &pb.Pointer{ Type: pb.Pointer_INLINE, InlineSegment: tt.inlineContent, @@ -472,7 +453,7 @@ func TestSegmentStoreList(t *testing.T) { } gomock.InOrder(calls...) - _, _, err = ss.List(ctx, prefix, startAfter, nil, false, 10, uint32(1)) + _, _, err = ss.List(ctx, tt.prefix, tt.startAfter, "", false, 10, meta.Modified) assert.NoError(t, err) } } diff --git a/pkg/storage/streams/store.go b/pkg/storage/streams/store.go index 0046a8bd7..ea1202151 100644 --- a/pkg/storage/streams/store.go +++ b/pkg/storage/streams/store.go @@ -10,18 +10,18 @@ import ( "fmt" "io" "io/ioutil" + "strings" "time" - proto "github.com/gogo/protobuf/proto" + "github.com/golang/protobuf/proto" "github.com/zeebo/errs" "go.uber.org/zap" monkit "gopkg.in/spacemonkeygo/monkit.v2" "storj.io/storj/pkg/eestream" "storj.io/storj/pkg/encryption" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" - ranger "storj.io/storj/pkg/ranger" + "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/meta" "storj.io/storj/pkg/storage/segments" "storj.io/storj/pkg/storj" @@ -56,28 +56,28 @@ func convertMeta(lastSegmentMeta segments.Meta) (Meta, error) { // Store interface methods for streams to satisfy to be a store type Store interface { - Meta(ctx context.Context, path paths.Path) (Meta, error) - Get(ctx context.Context, path paths.Path) (ranger.Ranger, Meta, error) - Put(ctx context.Context, path paths.Path, data io.Reader, metadata []byte, expiration time.Time) (Meta, error) - Delete(ctx context.Context, path paths.Path) error - List(ctx context.Context, prefix, startAfter, endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) + Meta(ctx context.Context, path storj.Path) (Meta, error) + Get(ctx context.Context, path storj.Path) (ranger.Ranger, Meta, error) + Put(ctx context.Context, path storj.Path, data io.Reader, metadata []byte, expiration time.Time) (Meta, error) + Delete(ctx context.Context, path storj.Path) error + List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) } // streamStore is a store for streams type streamStore struct { segments segments.Store segmentSize int64 - rootKey []byte + rootKey *storj.Key encBlockSize int cipher storj.Cipher } // NewStreamStore stuff -func NewStreamStore(segments segments.Store, segmentSize int64, rootKey string, encBlockSize int, cipher storj.Cipher) (Store, error) { +func NewStreamStore(segments segments.Store, segmentSize int64, rootKey *storj.Key, encBlockSize int, cipher storj.Cipher) (Store, error) { if segmentSize <= 0 { return nil, errs.New("segment size must be larger than 0") } - if rootKey == "" { + if rootKey == nil { return nil, errs.New("encryption key must not be empty") } if encBlockSize <= 0 { @@ -87,7 +87,7 @@ func NewStreamStore(segments segments.Store, segmentSize int64, rootKey string, return &streamStore{ segments: segments, segmentSize: segmentSize, - rootKey: []byte(rootKey), + rootKey: rootKey, encBlockSize: encBlockSize, cipher: cipher, }, nil @@ -97,7 +97,7 @@ func NewStreamStore(segments segments.Store, segmentSize int64, rootKey string, // store the first piece at s0/, second piece at s1/, and the // *last* piece at l/. Store the given metadata, along with the number // of segments, in a new protobuf, in the metadata of l/. -func (s *streamStore) Put(ctx context.Context, path paths.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, err error) { +func (s *streamStore) Put(ctx context.Context, path storj.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, err error) { defer mon.Task()(&ctx)(&err) // previously file uploaded? err = s.Delete(ctx, path) @@ -115,7 +115,7 @@ func (s *streamStore) Put(ctx context.Context, path paths.Path, data io.Reader, return m, err } -func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, lastSegment int64, err error) { +func (s *streamStore) upload(ctx context.Context, path storj.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, lastSegment int64, err error) { defer mon.Task()(&ctx)(&err) var currentSegment int64 @@ -130,7 +130,7 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade } }() - derivedKey, err := path.DeriveContentKey(s.rootKey) + derivedKey, err := encryption.DeriveContentKey(path, s.rootKey) if err != nil { return Meta{}, currentSegment, err } @@ -166,7 +166,7 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade return Meta{}, currentSegment, err } - encryptedKey, err := encryption.EncryptKey(&contentKey, s.cipher, (*storj.Key)(derivedKey), &keyNonce) + encryptedKey, err := encryption.EncryptKey(&contentKey, s.cipher, derivedKey, &keyNonce) if err != nil { return Meta{}, currentSegment, err } @@ -194,10 +194,10 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade transformedReader = bytes.NewReader(cipherData) } - putMeta, err = s.segments.Put(ctx, transformedReader, expiration, func() (paths.Path, []byte, error) { + putMeta, err = s.segments.Put(ctx, transformedReader, expiration, func() (storj.Path, []byte, error) { encPath, err := encryptAfterBucket(path, s.rootKey) if err != nil { - return nil, nil, err + return "", nil, err } if !eofReader.isEOF() { @@ -212,13 +212,13 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade KeyNonce: keyNonce[:], }) if err != nil { - return nil, nil, err + return "", nil, err } return segmentPath, segmentMeta, nil } - lastSegmentPath := encPath.Prepend("l") + lastSegmentPath := storj.JoinPaths("l", encPath) streamInfo, err := proto.Marshal(&pb.StreamInfo{ NumberOfSegments: currentSegment + 1, @@ -227,13 +227,13 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade Metadata: metadata, }) if err != nil { - return nil, nil, err + return "", nil, err } // encrypt metadata with the content encryption key and zero nonce encryptedStreamInfo, err := encryption.Encrypt(streamInfo, s.cipher, &contentKey, &storj.Nonce{}) if err != nil { - return nil, nil, err + return "", nil, err } streamMeta := pb.StreamMeta{ @@ -251,7 +251,7 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade lastSegmentMeta, err := proto.Marshal(&streamMeta) if err != nil { - return nil, nil, err + return "", nil, err } return lastSegmentPath, lastSegmentMeta, nil @@ -279,14 +279,14 @@ func (s *streamStore) upload(ctx context.Context, path paths.Path, data io.Reade } // getSegmentPath returns the unique path for a particular segment -func getSegmentPath(p paths.Path, segNum int64) paths.Path { - return p.Prepend(fmt.Sprintf("s%d", segNum)) +func getSegmentPath(path storj.Path, segNum int64) storj.Path { + return storj.JoinPaths(fmt.Sprintf("s%d", segNum), path) } // Get returns a ranger that knows what the overall size is (from l/) // and then returns the appropriate data from segments s0/, s1/, // ..., l/. -func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Ranger, meta Meta, err error) { +func (s *streamStore) Get(ctx context.Context, path storj.Path) (rr ranger.Ranger, meta Meta, err error) { defer mon.Task()(&ctx)(&err) encPath, err := encryptAfterBucket(path, s.rootKey) @@ -294,7 +294,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range return nil, Meta{}, err } - lastSegmentRanger, lastSegmentMeta, err := s.segments.Get(ctx, encPath.Prepend("l")) + lastSegmentRanger, lastSegmentMeta, err := s.segments.Get(ctx, storj.JoinPaths("l", encPath)) if err != nil { return nil, Meta{}, err } @@ -316,7 +316,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range return nil, Meta{}, err } - derivedKey, err := path.DeriveContentKey(s.rootKey) + derivedKey, err := encryption.DeriveContentKey(path, s.rootKey) if err != nil { return nil, Meta{}, err } @@ -334,7 +334,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range segments: s.segments, path: currentPath, size: size, - derivedKey: (*storj.Key)(derivedKey), + derivedKey: derivedKey, startingNonce: &contentNonce, encBlockSize: int(streamMeta.EncryptionBlockSize), cipher: storj.Cipher(streamMeta.EncryptionType), @@ -353,7 +353,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range lastSegmentRanger, stream.LastSegmentSize, storj.Cipher(streamMeta.EncryptionType), - (*storj.Key)(derivedKey), + derivedKey, encryptedKey, keyNonce, &contentNonce, @@ -376,7 +376,7 @@ func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Range } // Meta implements Store.Meta -func (s *streamStore) Meta(ctx context.Context, path paths.Path) (meta Meta, err error) { +func (s *streamStore) Meta(ctx context.Context, path storj.Path) (meta Meta, err error) { defer mon.Task()(&ctx)(&err) encPath, err := encryptAfterBucket(path, s.rootKey) @@ -384,7 +384,7 @@ func (s *streamStore) Meta(ctx context.Context, path paths.Path) (meta Meta, err return Meta{}, err } - lastSegmentMeta, err := s.segments.Meta(ctx, encPath.Prepend("l")) + lastSegmentMeta, err := s.segments.Meta(ctx, storj.JoinPaths("l", encPath)) if err != nil { return Meta{}, err } @@ -404,14 +404,14 @@ func (s *streamStore) Meta(ctx context.Context, path paths.Path) (meta Meta, err } // Delete all the segments, with the last one last -func (s *streamStore) Delete(ctx context.Context, path paths.Path) (err error) { +func (s *streamStore) Delete(ctx context.Context, path storj.Path) (err error) { defer mon.Task()(&ctx)(&err) encPath, err := encryptAfterBucket(path, s.rootKey) if err != nil { return err } - lastSegmentMeta, err := s.segments.Meta(ctx, encPath.Prepend("l")) + lastSegmentMeta, err := s.segments.Meta(ctx, storj.JoinPaths("l", encPath)) if err != nil { return err } @@ -439,18 +439,18 @@ func (s *streamStore) Delete(ctx context.Context, path paths.Path) (err error) { } } - return s.segments.Delete(ctx, encPath.Prepend("l")) + return s.segments.Delete(ctx, storj.JoinPaths("l", encPath)) } // ListItem is a single item in a listing type ListItem struct { - Path paths.Path + Path storj.Path Meta Meta IsPrefix bool } // List all the paths inside l/, stripping off the l/ prefix -func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) { +func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) { defer mon.Task()(&ctx)(&err) if metaFlags&meta.Size != 0 { @@ -459,12 +459,14 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa metaFlags |= meta.UserDefined } + prefix = strings.TrimSuffix(prefix, "/") + encPrefix, err := encryptAfterBucket(prefix, s.rootKey) if err != nil { return nil, false, err } - prefixKey, err := prefix.DeriveKey(s.rootKey, len(prefix)) + prefixKey, err := encryption.DerivePathKey(prefix, s.rootKey, len(storj.SplitPath(prefix))) if err != nil { return nil, false, err } @@ -479,7 +481,7 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa return nil, false, err } - segments, more, err := s.segments.List(ctx, encPrefix.Prepend("l"), encStartAfter, encEndBefore, recursive, limit, metaFlags) + segments, more, err := s.segments.List(ctx, storj.JoinPaths("l", encPrefix), encStartAfter, encEndBefore, recursive, limit, metaFlags) if err != nil { return nil, false, err } @@ -491,7 +493,7 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa return nil, false, err } - streamInfo, err := decryptStreamInfo(ctx, item.Meta, path.Prepend(prefix...), s.rootKey) + streamInfo, err := decryptStreamInfo(ctx, item.Meta, storj.JoinPaths(prefix, path), s.rootKey) if err != nil { return nil, false, err } @@ -509,25 +511,25 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore pa } // encryptMarker is a helper method for encrypting startAfter and endBefore markers -func (s *streamStore) encryptMarker(marker paths.Path, prefixKey []byte) (paths.Path, error) { - if bytes.Equal(s.rootKey, prefixKey) { // empty prefix +func (s *streamStore) encryptMarker(marker storj.Path, prefixKey *storj.Key) (storj.Path, error) { + if bytes.Equal(s.rootKey[:], prefixKey[:]) { // empty prefix return encryptAfterBucket(marker, s.rootKey) } - return marker.Encrypt(prefixKey) + return encryption.EncryptPath(marker, prefixKey) } // decryptMarker is a helper method for decrypting listed path markers -func (s *streamStore) decryptMarker(marker paths.Path, prefixKey []byte) (paths.Path, error) { - if bytes.Equal(s.rootKey, prefixKey) { // empty prefix +func (s *streamStore) decryptMarker(marker storj.Path, prefixKey *storj.Key) (storj.Path, error) { + if bytes.Equal(s.rootKey[:], prefixKey[:]) { // empty prefix return decryptAfterBucket(marker, s.rootKey) } - return marker.Decrypt(prefixKey) + return encryption.DecryptPath(marker, prefixKey) } type lazySegmentRanger struct { ranger ranger.Ranger segments segments.Store - path paths.Path + path storj.Path size int64 derivedKey *storj.Key startingNonce *storj.Nonce @@ -598,46 +600,46 @@ func decryptRanger(ctx context.Context, rr ranger.Ranger, decryptedSize int64, c } // encryptAfterBucket encrypts a path without encrypting its first element -func encryptAfterBucket(p paths.Path, key []byte) (encrypted paths.Path, err error) { - if len(p) <= 1 { - return p, nil +func encryptAfterBucket(path storj.Path, key *storj.Key) (encrypted storj.Path, err error) { + comps := storj.SplitPath(path) + if len(comps) <= 1 { + return path, nil } - bucket := p[0] - toEncrypt := p[1:] - // derive a key from the bucket so the same path in different buckets is encrypted differently - bucketKey, err := p.DeriveKey(key, 1) + encrypted, err = encryption.EncryptPath(path, key) if err != nil { - return nil, err + return "", err } - encPath, err := toEncrypt.Encrypt(bucketKey) - if err != nil { - return nil, err - } - return encPath.Prepend(bucket), nil + + // replace the first path component with the unencrypted bucket name + return storj.JoinPaths(comps[0], storj.JoinPaths(storj.SplitPath(encrypted)[1:]...)), nil } // decryptAfterBucket decrypts a path without modifying its first element -func decryptAfterBucket(p paths.Path, key []byte) (decrypted paths.Path, err error) { - if len(p) <= 1 { - return p, nil +func decryptAfterBucket(path storj.Path, key *storj.Key) (decrypted storj.Path, err error) { + comps := storj.SplitPath(path) + if len(comps) <= 1 { + return path, nil } - bucket := p[0] - toDecrypt := p[1:] - bucketKey, err := p.DeriveKey(key, 1) + bucket := comps[0] + toDecrypt := storj.JoinPaths(comps[1:]...) + + bucketKey, err := encryption.DerivePathKey(path, key, 1) if err != nil { - return nil, err + return "", err } - decPath, err := toDecrypt.Decrypt(bucketKey) + + decPath, err := encryption.DecryptPath(toDecrypt, bucketKey) if err != nil { - return nil, err + return "", err } - return decPath.Prepend(bucket), nil + + return storj.JoinPaths(bucket, decPath), nil } // CancelHandler handles clean up of segments on receiving CTRL+C -func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path paths.Path) { +func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path storj.Path) { for i := int64(0); i < totalSegments; i++ { encPath, err := encryptAfterBucket(path, s.rootKey) if err != nil { @@ -663,21 +665,21 @@ func getEncryptedKeyAndNonce(m *pb.SegmentMeta) (storj.EncryptedPrivateKey, *sto return m.EncryptedKey, &nonce } -func decryptStreamInfo(ctx context.Context, item segments.Meta, path paths.Path, rootKey []byte) (streamInfo []byte, err error) { +func decryptStreamInfo(ctx context.Context, item segments.Meta, path storj.Path, rootKey *storj.Key) (streamInfo []byte, err error) { streamMeta := pb.StreamMeta{} err = proto.Unmarshal(item.Data, &streamMeta) if err != nil { return nil, err } - derivedKey, err := path.DeriveContentKey(rootKey) + derivedKey, err := encryption.DeriveContentKey(path, rootKey) if err != nil { return nil, err } cipher := storj.Cipher(streamMeta.EncryptionType) encryptedKey, keyNonce := getEncryptedKeyAndNonce(streamMeta.LastSegmentMeta) - contentKey, err := encryption.DecryptKey(encryptedKey, cipher, (*storj.Key)(derivedKey), keyNonce) + contentKey, err := encryption.DecryptKey(encryptedKey, cipher, derivedKey, keyNonce) if err != nil { return nil, err } diff --git a/pkg/storage/streams/store_test.go b/pkg/storage/streams/store_test.go index ed9bcdbf3..d54b095c0 100644 --- a/pkg/storage/streams/store_test.go +++ b/pkg/storage/streams/store_test.go @@ -11,14 +11,14 @@ import ( "testing" "time" - proto "github.com/gogo/protobuf/proto" "github.com/golang/mock/gomock" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" - "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" - ranger "storj.io/storj/pkg/ranger" + "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/segments" + "storj.io/storj/pkg/storj" ) var ( @@ -91,12 +91,12 @@ func TestStreamStoreMeta(t *testing.T) { Meta(gomock.Any(), gomock.Any()). Return(test.segmentMeta, test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0) + streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) if err != nil { t.Fatal(err) } - meta, err := streamStore.Meta(ctx, paths.New(test.path)) + meta, err := streamStore.Meta(ctx, test.path) if err != nil { t.Fatal(err) } @@ -146,7 +146,7 @@ func TestStreamStorePut(t *testing.T) { mockSegmentStore.EXPECT(). Put(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(test.segmentMeta, test.segmentError). - Do(func(ctx context.Context, data io.Reader, expiration time.Time, info func() (paths.Path, []byte, error)) { + Do(func(ctx context.Context, data io.Reader, expiration time.Time, info func() (storj.Path, []byte, error)) { for { buf := make([]byte, 4) _, err := data.Read(buf) @@ -163,12 +163,12 @@ func TestStreamStorePut(t *testing.T) { Delete(gomock.Any(), gomock.Any()). Return(test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0) + streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) if err != nil { t.Fatal(err) } - meta, err := streamStore.Put(ctx, paths.New(test.path), test.data, test.metadata, test.expiration) + meta, err := streamStore.Put(ctx, test.path, test.data, test.metadata, test.expiration) if err != nil { t.Fatal(err) } @@ -263,12 +263,12 @@ func TestStreamStoreGet(t *testing.T) { gomock.InOrder(calls...) - streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0) + streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) if err != nil { t.Fatal(err) } - ranger, meta, err := streamStore.Get(ctx, paths.New(test.path)) + ranger, meta, err := streamStore.Get(ctx, test.path) if err != nil { t.Fatal(err) } @@ -312,12 +312,12 @@ func TestStreamStoreDelete(t *testing.T) { Delete(gomock.Any(), gomock.Any()). Return(test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0) + streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) if err != nil { t.Fatal(err) } - err = streamStore.Delete(ctx, paths.New(test.path)) + err = streamStore.Delete(ctx, test.path) if err != nil { t.Fatal(err) } @@ -356,12 +356,12 @@ func TestStreamStoreList(t *testing.T) { List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(test.segments, test.segmentMore, test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, 10, "key", 10, 0) + streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) if err != nil { t.Fatal(err) } - items, more, err := streamStore.List(ctx, paths.New(test.prefix), paths.New(test.startAfter), paths.New(test.endBefore), test.recursive, test.limit, test.metaFlags) + items, more, err := streamStore.List(ctx, test.prefix, test.startAfter, test.endBefore, test.recursive, test.limit, test.metaFlags) if err != nil { t.Fatal(err) } diff --git a/pkg/storj/path.go b/pkg/storj/path.go index 4e7cd2618..3e62c7025 100644 --- a/pkg/storj/path.go +++ b/pkg/storj/path.go @@ -3,5 +3,19 @@ package storj +import ( + "strings" +) + // Path represents a object path type Path = string + +// SplitPath splits path into a slice of path components +func SplitPath(path Path) []string { + return strings.Split(path, "/") +} + +// JoinPaths concatenates paths to a new single path +func JoinPaths(paths ...Path) Path { + return strings.Join(paths, "/") +} diff --git a/pkg/storj/path_test.go b/pkg/storj/path_test.go new file mode 100644 index 000000000..62c47373f --- /dev/null +++ b/pkg/storj/path_test.go @@ -0,0 +1,54 @@ +// Copyright (C) 2018 Storj Labs, Inc. +// See LICENSE for copying information. + +package storj + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSplitPath(t *testing.T) { + for i, tt := range []struct { + path string + comps []string + }{ + {"", []string{""}}, + {"/", []string{"", ""}}, + {"//", []string{"", "", ""}}, + {" ", []string{" "}}, + {"a", []string{"a"}}, + {"/a/", []string{"", "a", ""}}, + {"a/b/c/d", []string{"a", "b", "c", "d"}}, + {"///a//b////c/d///", []string{"", "", "", "a", "", "b", "", "", "", "c", "d", "", "", ""}}, + } { + errTag := fmt.Sprintf("Test case #%d", i) + assert.Equal(t, tt.comps, SplitPath(tt.path), errTag) + } +} + +func TestJoinPaths(t *testing.T) { + for i, tt := range []struct { + comps []string + path string + }{ + {[]string{}, ""}, + {[]string{""}, ""}, + {[]string{"", ""}, "/"}, + {[]string{"/", ""}, "//"}, + {[]string{"/", "/"}, "///"}, + {[]string{"", "", ""}, "//"}, + {[]string{" "}, " "}, + {[]string{"a"}, "a"}, + {[]string{"", "a", ""}, "/a/"}, + {[]string{"a", "b", "c", "d"}, "a/b/c/d"}, + {[]string{"a/b", "c/d"}, "a/b/c/d"}, + {[]string{"a/b/", "c/d"}, "a/b//c/d"}, + {[]string{"", "", "", "a", "", "b", "", "", "", "c", "d", "", "", ""}, "///a//b////c/d///"}, + } { + errTag := fmt.Sprintf("Test case #%d", i) + assert.Equal(t, tt.path, JoinPaths(tt.comps...), errTag) + } +}