diff --git a/lib/uplink/project.go b/lib/uplink/project.go index f309d663e..a8c1134e9 100644 --- a/lib/uplink/project.go +++ b/lib/uplink/project.go @@ -182,7 +182,11 @@ func (p *Project) OpenBucket(ctx context.Context, bucketName string, access *Enc } segmentStore := segments.NewSegmentStore(p.metainfo, ec, rs, p.maxInlineSize.Int(), maxEncryptedSegmentSize) - streamStore, err := streams.NewStreamStore(segmentStore, cfg.Volatile.SegmentsSize.Int64(), &access.Key, int(encryptionScheme.BlockSize), encryptionScheme.Cipher, p.maxInlineSize.Int()) + // TODO(jeff): this is where we would load scope information in. + encStore := encryption.NewStore() + encStore.SetDefaultKey(&access.Key) + + streamStore, err := streams.NewStreamStore(segmentStore, cfg.Volatile.SegmentsSize.Int64(), encStore, int(encryptionScheme.BlockSize), encryptionScheme.Cipher, p.maxInlineSize.Int()) if err != nil { return nil, err } @@ -192,7 +196,7 @@ func (p *Project) OpenBucket(ctx context.Context, bucketName string, access *Enc Name: bucketInfo.Name, Created: bucketInfo.Created, bucket: bucketInfo, - metainfo: kvmetainfo.New(p.project, p.metainfo, streamStore, segmentStore, &access.Key), + metainfo: kvmetainfo.New(p.project, p.metainfo, streamStore, segmentStore, encStore), streams: streamStore, }, nil } diff --git a/pkg/encryption/path_new.go b/pkg/encryption/path_new.go new file mode 100644 index 000000000..bf995bd39 --- /dev/null +++ b/pkg/encryption/path_new.go @@ -0,0 +1,300 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package encryption + +import ( + "crypto/hmac" + "crypto/sha512" + "encoding/base64" + "strings" + + "github.com/zeebo/errs" + + "storj.io/storj/pkg/paths" + "storj.io/storj/pkg/storj" +) + +// StoreEncryptPath encrypts the path using the provided cipher and looking up +// keys from the provided store and bucket. +func StoreEncryptPath(bucket string, path paths.Unencrypted, cipher storj.Cipher, store *Store) ( + encPath paths.Encrypted, err error) { + + // Invalid paths map to invalid paths + if !path.Valid() { + return paths.Encrypted{}, nil + } + + if cipher == storj.Unencrypted { + return paths.NewEncrypted(path.Raw()), nil + } + + _, consumed, base := store.LookupUnencrypted(bucket, path) + if base == nil { + return paths.Encrypted{}, errs.New("unable to find encryption base for: %s/%q", bucket, path) + } + + remaining, ok := path.Consume(consumed) + if !ok { + return paths.Encrypted{}, errs.New("unable to encrypt bucket path: %s/%q", bucket, path) + } + + // if we didn't consume any path, we're at the root of the bucket, and so we have + // to fold the bucket name into the key. + key := &base.Key + if !consumed.Valid() { + key, err = derivePathKeyComponent(key, bucket) + if err != nil { + return paths.Encrypted{}, errs.Wrap(err) + } + } + + encrypted, err := EncryptPathRaw(remaining.Raw(), cipher, key) + if err != nil { + return paths.Encrypted{}, errs.Wrap(err) + } + + var builder strings.Builder + builder.WriteString(base.Encrypted.Raw()) + + if len(encrypted) > 0 { + if builder.Len() > 0 { + builder.WriteByte('/') + } + builder.WriteString(encrypted) + } + + return paths.NewEncrypted(builder.String()), nil +} + +// EncryptPathRaw encrypts the path using the provided key directly. EncryptPath should be +// preferred if possible. +func EncryptPathRaw(raw string, cipher storj.Cipher, key *storj.Key) (string, error) { + if cipher == storj.Unencrypted { + return raw, nil + } + + var builder strings.Builder + for iter, i := paths.NewIterator(raw), 0; !iter.Done(); i++ { + component := iter.Next() + encComponent, err := storeEncryptPathComponent(component, cipher, key) + if err != nil { + return "", errs.Wrap(err) + } + key, err = derivePathKeyComponent(key, component) + if err != nil { + return "", errs.Wrap(err) + } + if i > 0 { + builder.WriteByte('/') + } + builder.WriteString(encComponent) + } + return builder.String(), nil +} + +// StoreDecryptPath decrypts the path using the provided cipher and looking up +// keys from the provided store and bucket. +func StoreDecryptPath(bucket string, path paths.Encrypted, cipher storj.Cipher, store *Store) ( + unencPath paths.Unencrypted, err error) { + + // Invalid paths map to invalid paths + if !path.Valid() { + return paths.Unencrypted{}, nil + } + + if cipher == storj.Unencrypted { + return paths.NewUnencrypted(path.Raw()), nil + } + + _, consumed, base := store.LookupEncrypted(bucket, path) + if base == nil { + return paths.Unencrypted{}, errs.New("unable to find decryption base for: %q", path) + } + + remaining, ok := path.Consume(consumed) + if !ok { + return paths.Unencrypted{}, errs.New("unable to decrypt bucket path: %q", path) + } + + // if we didn't consume any path, we're at the root of the bucket, and so we have + // to fold the bucket name into the key. + key := &base.Key + if !consumed.Valid() { + key, err = derivePathKeyComponent(key, bucket) + if err != nil { + return paths.Unencrypted{}, errs.Wrap(err) + } + } + + decrypted, err := DecryptPathRaw(remaining.Raw(), cipher, key) + if err != nil { + return paths.Unencrypted{}, errs.Wrap(err) + } + + var builder strings.Builder + builder.WriteString(base.Unencrypted.Raw()) + + if len(decrypted) > 0 { + if builder.Len() > 0 { + builder.WriteByte('/') + } + builder.WriteString(decrypted) + } + + return paths.NewUnencrypted(builder.String()), nil +} + +// DecryptPathRaw decrypts the path using the provided key directly. DecryptPath should be +// preferred if possible. +func DecryptPathRaw(raw string, cipher storj.Cipher, key *storj.Key) (string, error) { + if cipher == storj.Unencrypted { + return raw, nil + } + + var builder strings.Builder + for iter, i := paths.NewIterator(raw), 0; !iter.Done(); i++ { + component := iter.Next() + unencComponent, err := storeDecryptPathComponent(component, cipher, key) + if err != nil { + return "", errs.Wrap(err) + } + key, err = derivePathKeyComponent(key, unencComponent) + if err != nil { + return "", errs.Wrap(err) + } + if i > 0 { + builder.WriteByte('/') + } + builder.WriteString(unencComponent) + } + return builder.String(), nil +} + +// StoreDeriveContentKey returns the content key for the passed in path by looking up +// the appropriate base key from the store and bucket and deriving the rest. +func StoreDeriveContentKey(bucket string, path paths.Unencrypted, store *Store) (key *storj.Key, err error) { + key, err = StoreDerivePathKey(bucket, path, store) + if err != nil { + return nil, errs.Wrap(err) + } + key, err = DeriveKey(key, "content") + return key, errs.Wrap(err) +} + +// StoreDerivePathKey returns the path key for the passed in path by looking up the +// appropriate base key from the store and bucket and deriving the rest. +func StoreDerivePathKey(bucket string, path paths.Unencrypted, store *Store) (key *storj.Key, err error) { + _, consumed, base := store.LookupUnencrypted(bucket, path) + if base == nil { + return nil, errs.New("unable to find encryption base for: %s/%q", bucket, path) + } + + // If asking for the key at the bucket, do that and return. + if !path.Valid() { + key, err = derivePathKeyComponent(&base.Key, bucket) + if err != nil { + return nil, errs.Wrap(err) + } + return key, nil + } + + remaining, ok := path.Consume(consumed) + if !ok { + return nil, errs.New("unable to derive path key for: %s/%q", bucket, path) + } + + // if we didn't consume any path, we're at the root of the bucket, and so we have + // to fold the bucket name into the key. + key = &base.Key + if !consumed.Valid() { + key, err = derivePathKeyComponent(key, bucket) + if err != nil { + return nil, errs.Wrap(err) + } + } + + for iter := remaining.Iterator(); !iter.Done(); { + key, err = derivePathKeyComponent(key, iter.Next()) + if err != nil { + return nil, errs.Wrap(err) + } + } + return key, nil +} + +// derivePathKeyComponent derives a new key from the provided one using the component. It +// should be preferred over DeriveKey when adding path components as it performs the +// necessary transformation to the component. +func derivePathKeyComponent(key *storj.Key, component string) (*storj.Key, error) { + return DeriveKey(key, "path:"+component) +} + +// storeEncryptPathComponent encrypts a single path component with the provided cipher and key. +func storeEncryptPathComponent(comp string, cipher storj.Cipher, key *storj.Key) (string, error) { + if comp == "" { + return "", nil + } + + // derive the key for the next path component. this is so that + // every encrypted component has a unique nonce. + derivedKey, err := derivePathKeyComponent(key, comp) + if err != nil { + return "", err + } + + // use the derived key to derive the nonce + mac := hmac.New(sha512.New, derivedKey[:]) + _, err = mac.Write([]byte("nonce")) + if err != nil { + return "", Error.Wrap(err) + } + + nonce := new(storj.Nonce) + copy(nonce[:], mac.Sum(nil)) + + // encrypt the path components with the parent's key and the derived nonce + cipherText, err := Encrypt([]byte(comp), cipher, key, nonce) + if err != nil { + return "", Error.Wrap(err) + } + + nonceSize := storj.NonceSize + if cipher == storj.AESGCM { + nonceSize = AESGCMNonceSize + } + + // keep the nonce together with the cipher text + return base64.RawURLEncoding.EncodeToString(append(nonce[:nonceSize], cipherText...)), nil +} + +// storeDecryptPathComponent decrypts a single path component with the provided cipher and key. +func storeDecryptPathComponent(comp string, cipher storj.Cipher, key *storj.Key) (string, error) { + if comp == "" { + return "", nil + } + + data, err := base64.RawURLEncoding.DecodeString(comp) + if err != nil { + return "", Error.Wrap(err) + } + + nonceSize := storj.NonceSize + if cipher == storj.AESGCM { + nonceSize = AESGCMNonceSize + } + if len(data) < nonceSize || nonceSize < 0 { + return "", errs.New("component did not contain enough nonce bytes") + } + + // extract the nonce from the cipher text + nonce := new(storj.Nonce) + copy(nonce[:], data[:nonceSize]) + + decrypted, err := Decrypt(data[nonceSize:], cipher, key, nonce) + if err != nil { + return "", Error.Wrap(err) + } + + return string(decrypted), nil +} diff --git a/pkg/encryption/path_new_test.go b/pkg/encryption/path_new_test.go new file mode 100644 index 000000000..c2debcb01 --- /dev/null +++ b/pkg/encryption/path_new_test.go @@ -0,0 +1,56 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package encryption + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "storj.io/storj/pkg/paths" + "storj.io/storj/pkg/storj" +) + +func newStore(key storj.Key) *Store { + store := NewStore() + if err := store.Add("bucket", paths.Unencrypted{}, paths.Encrypted{}, key); err != nil { + panic(err) + } + return store +} + +func TestStoreEncryption(t *testing.T) { + forAllCiphers(func(cipher storj.Cipher) { + for i, rawPath := range []string{ + "", + "/", + "//", + "file.txt", + "file.txt/", + "fold1/file.txt", + "fold1/fold2/file.txt", + "/fold1/fold2/fold3/file.txt", + } { + errTag := fmt.Sprintf("test:%d path:%q cipher:%v", i, rawPath, cipher) + + var key storj.Key + copy(key[:], randData(storj.KeySize)) + store := newStore(key) + path := paths.NewUnencrypted(rawPath) + + encPath, err := StoreEncryptPath("bucket", path, cipher, store) + if !assert.NoError(t, err, errTag) { + continue + } + + decPath, err := StoreDecryptPath("bucket", encPath, cipher, store) + if !assert.NoError(t, err, errTag) { + continue + } + + assert.Equal(t, rawPath, decPath.Raw(), errTag) + } + }) +} diff --git a/pkg/encryption/store.go b/pkg/encryption/store.go new file mode 100644 index 000000000..20fb88011 --- /dev/null +++ b/pkg/encryption/store.go @@ -0,0 +1,215 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package encryption + +import ( + "github.com/zeebo/errs" + + "storj.io/storj/pkg/paths" + "storj.io/storj/pkg/storj" +) + +// The Store allows one to find the matching most encrypted key and path for +// some unencrypted path. It also reports a mapping of encrypted to unencrypted paths +// at the searched for unencrypted path. +// +// For example, if the Store contains the mappings +// +// b1, u1/u2/u3 => +// b1, u1/u2/u3/u4 => +// b1, u1/u5 => +// b1, u6 => +// b1, u6/u7/u8 => +// b2, u1 => +// +// Then the following lookups have outputs +// +// b1, u1 => <{e2:u2, e5:u5}, u1, nil> +// b1, u1/u2/u3 => <{e4:u4}, u1/u2/u3, > +// b1, u1/u2/u3/u6 => <{}, u1/u2/u3/, > +// b1, u1/u2/u3/u4 => <{}, u1/u2/u3/u4, > +// b1, u6/u7 => <{e8:u8}, u6/, > +// b2, u1 => <{}, u1, > +type Store struct { + roots map[string]*node + defaultKey *storj.Key +} + +// node is a node in the Store graph. It may contain an encryption key and encrypted path, +// a list of children nodes, and data to ensure a bijection between encrypted and unencrypted +// path entries. +type node struct { + unenc map[string]*node // unenc => node + unencMap map[string]string // unenc => enc + enc map[string]*node // enc => node + encMap map[string]string // enc => unenc + base *Base +} + +// Base represents a key with which to derive further keys at some encrypted/unencrypted path. +type Base struct { + Unencrypted paths.Unencrypted + Encrypted paths.Encrypted + Key storj.Key +} + +// clone returns a copy of the Base. The implementation can be simple because the +// types of its fields do not contain any references. +func (b *Base) clone() *Base { + if b == nil { + return nil + } + bc := *b + return &bc +} + +// NewStore constructs a Store. +func NewStore() *Store { + return &Store{roots: make(map[string]*node)} +} + +// newNode constructs a node. +func newNode() *node { + return &node{ + unenc: make(map[string]*node), + unencMap: make(map[string]string), + enc: make(map[string]*node), + encMap: make(map[string]string), + } +} + +// SetDefaultKey adds a default key to be returned for any lookup that does not match a bucket. +func (s *Store) SetDefaultKey(defaultKey *storj.Key) { + s.defaultKey = defaultKey +} + +// Add creates a mapping from the unencrypted path to the encrypted path and key. +func (s *Store) Add(bucket string, unenc paths.Unencrypted, enc paths.Encrypted, key storj.Key) error { + root, ok := s.roots[bucket] + if !ok { + root = newNode() + } + + // Perform the addition starting at the root node. + if err := root.add(unenc.Iterator(), enc.Iterator(), &Base{ + Unencrypted: unenc, + Encrypted: enc, + Key: key, + }); err != nil { + return err + } + + // only update the root for the bucket if the add was successful. + s.roots[bucket] = root + return nil +} + +// add places the paths and base into the node tree structure. +func (n *node) add(unenc, enc paths.Iterator, base *Base) error { + if unenc.Done() != enc.Done() { + return errs.New("encrypted and unencrypted paths had different number of components") + } + + // If we're done walking the paths, this node must have the provided base. + if unenc.Done() { + n.base = base + return nil + } + + // Walk to the next parts and ensure they're consistent with previous additions. + unencPart, encPart := unenc.Next(), enc.Next() + if exUnencPart, ok := n.encMap[encPart]; ok && exUnencPart != unencPart { + return errs.New("conflicting encrypted parts for unencrypted path") + } + if exEncPart, ok := n.unencMap[unencPart]; ok && exEncPart != encPart { + return errs.New("conflicting encrypted parts for unencrypted path") + } + + // Look up the child node. Since we're sure the unenc and enc mappings are + // consistent, we can look it up in one map and unconditionally insert it + // into both maps if necessary. + child, ok := n.unenc[unencPart] + if !ok { + child = newNode() + } + + // Recurse to the next node in the tree. + if err := child.add(unenc, enc, base); err != nil { + return err + } + + // Only add to the maps if the child add was successful. + n.unencMap[unencPart] = encPart + n.encMap[encPart] = unencPart + n.unenc[unencPart] = child + n.enc[encPart] = child + return nil +} + +// LookupUnencrypted finds the matching most unencrypted path added to the Store, reports how +// much of the path matched, any known unencrypted paths at the requested path, and if a key +// and encrypted path exists for some prefix of the unencrypted path. +func (s *Store) LookupUnencrypted(bucket string, path paths.Unencrypted) ( + revealed map[string]string, consumed paths.Unencrypted, base *Base) { + + root, ok := s.roots[bucket] + if ok { + var rawConsumed string + revealed, rawConsumed, base = root.lookup(path.Iterator(), "", nil, true) + consumed = paths.NewUnencrypted(rawConsumed) + } + if base == nil && s.defaultKey != nil { + return nil, paths.Unencrypted{}, &Base{Key: *s.defaultKey} + } + return revealed, consumed, base.clone() +} + +// LookupEncrypted finds the matching most encrypted path added to the Store, reports how +// much of the path matched, any known encrypted paths at the requested path, and if a key +// an encrypted path exists for some prefix of the encrypted path. +func (s *Store) LookupEncrypted(bucket string, path paths.Encrypted) ( + revealed map[string]string, consumed paths.Encrypted, base *Base) { + + root, ok := s.roots[bucket] + if ok { + var rawConsumed string + revealed, rawConsumed, base = root.lookup(path.Iterator(), "", nil, false) + consumed = paths.NewEncrypted(rawConsumed) + } + if base == nil && s.defaultKey != nil { + return nil, paths.Encrypted{}, &Base{Key: *s.defaultKey} + } + return revealed, consumed, base.clone() +} + +// lookup searches for the path in the node tree structure. +func (n *node) lookup(path paths.Iterator, bestConsumed string, bestBase *Base, unenc bool) ( + map[string]string, string, *Base) { + + // Keep track of the best match so far. + if n.base != nil || bestBase == nil { + bestBase, bestConsumed = n.base, path.Consumed() + } + + // Pick the tree we're walking down based on the unenc bool. + revealed, children := n.unencMap, n.enc + if unenc { + revealed, children = n.encMap, n.unenc + } + + // If we're done walking the path, then return our best match along with the + // revealed paths at this node. + if path.Done() { + return revealed, bestConsumed, bestBase + } + + // Walk to the next node in the tree. If there is no node, then report our best match. + child, ok := children[path.Next()] + if !ok { + return nil, bestConsumed, bestBase + } + + // Recurse to the next node in the tree. + return child.lookup(path, bestConsumed, bestBase, unenc) +} diff --git a/pkg/encryption/store_test.go b/pkg/encryption/store_test.go new file mode 100644 index 000000000..04518a648 --- /dev/null +++ b/pkg/encryption/store_test.go @@ -0,0 +1,128 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package encryption + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "storj.io/storj/pkg/paths" + "storj.io/storj/pkg/storj" +) + +func printLookup(revealed map[string]string, consumed interface{ Raw() string }, base *Base) { + if base == nil { + fmt.Printf("<%q, %q, nil>\n", revealed, consumed.Raw()) + } else { + fmt.Printf("<%q, %q, <%q, %q, %q>>\n", + revealed, consumed, base.Unencrypted, base.Encrypted, base.Key[:2]) + } +} + +func toKey(val string) (out storj.Key) { + copy(out[:], val) + return out +} + +func abortIfError(err error) { + if err != nil { + panic(fmt.Sprintf("%+v", err)) + } +} + +func ExampleStore() { + s := NewStore() + ep := paths.NewEncrypted + up := paths.NewUnencrypted + + // Add a fairly complicated tree to the store. + abortIfError(s.Add("b1", up("u1/u2/u3"), ep("e1/e2/e3"), toKey("k3"))) + abortIfError(s.Add("b1", up("u1/u2/u3/u4"), ep("e1/e2/e3/e4"), toKey("k4"))) + abortIfError(s.Add("b1", up("u1/u5"), ep("e1/e5"), toKey("k5"))) + abortIfError(s.Add("b1", up("u6"), ep("e6"), toKey("k6"))) + abortIfError(s.Add("b1", up("u6/u7/u8"), ep("e6/e7/e8"), toKey("k8"))) + abortIfError(s.Add("b2", up("u1"), ep("e1'"), toKey("k1"))) + abortIfError(s.Add("b3", paths.Unencrypted{}, paths.Encrypted{}, toKey("m1"))) + + // Look up some complicated queries by the unencrypted path. + printLookup(s.LookupUnencrypted("b1", up("u1"))) + printLookup(s.LookupUnencrypted("b1", up("u1/u2/u3"))) + printLookup(s.LookupUnencrypted("b1", up("u1/u2/u3/u6"))) + printLookup(s.LookupUnencrypted("b1", up("u1/u2/u3/u4"))) + printLookup(s.LookupUnencrypted("b1", up("u6/u7"))) + printLookup(s.LookupUnencrypted("b2", up("u1"))) + printLookup(s.LookupUnencrypted("b3", paths.Unencrypted{})) + printLookup(s.LookupUnencrypted("b3", up("z1"))) + + fmt.Println() + + // Look up some complicated queries by the encrypted path. + printLookup(s.LookupEncrypted("b1", ep("e1"))) + printLookup(s.LookupEncrypted("b1", ep("e1/e2/e3"))) + printLookup(s.LookupEncrypted("b1", ep("e1/e2/e3/e6"))) + printLookup(s.LookupEncrypted("b1", ep("e1/e2/e3/e4"))) + printLookup(s.LookupEncrypted("b1", ep("e6/e7"))) + printLookup(s.LookupEncrypted("b2", ep("e1'"))) + printLookup(s.LookupEncrypted("b3", paths.Encrypted{})) + printLookup(s.LookupEncrypted("b3", ep("z1"))) + + // output: + // + // + // > + // > + // > + // > + // > + // > + // > + // + // + // > + // > + // > + // > + // > + // > + // > +} + +func TestStoreErrors(t *testing.T) { + s := NewStore() + ep := paths.NewEncrypted + up := paths.NewUnencrypted + + // Too many encrypted parts + require.Error(t, s.Add("b1", up("u1"), ep("e1/e2/e3"), storj.Key{})) + + // Too many unencrypted parts + require.Error(t, s.Add("b1", up("u1/u2/u3"), ep("e1"), storj.Key{})) + + // Mismatches + require.NoError(t, s.Add("b1", up("u1"), ep("e1"), storj.Key{})) + require.Error(t, s.Add("b1", up("u2"), ep("e1"), storj.Key{})) + require.Error(t, s.Add("b1", up("u1"), ep("f1"), storj.Key{})) +} + +func TestStoreErrorState(t *testing.T) { + s := NewStore() + ep := paths.NewEncrypted + up := paths.NewUnencrypted + + // Do an empty lookup. + revealed1, consumed1, base1 := s.LookupUnencrypted("b1", up("u1/u2")) + + // Attempt to do an addition that fails. + require.Error(t, s.Add("b1", up("u1/u2"), ep("e1/e2/e3"), storj.Key{})) + + // Ensure that we get the same results as before + revealed2, consumed2, base2 := s.LookupUnencrypted("b1", up("u1/u2")) + + assert.Equal(t, revealed1, revealed2) + assert.Equal(t, consumed1, consumed2) + assert.Equal(t, base1, base2) +} diff --git a/pkg/metainfo/kvmetainfo/buckets_test.go b/pkg/metainfo/kvmetainfo/buckets_test.go index b07f1256a..15a3749f8 100644 --- a/pkg/metainfo/kvmetainfo/buckets_test.go +++ b/pkg/metainfo/kvmetainfo/buckets_test.go @@ -18,6 +18,7 @@ import ( "storj.io/storj/internal/testcontext" "storj.io/storj/internal/testplanet" "storj.io/storj/pkg/eestream" + "storj.io/storj/pkg/encryption" "storj.io/storj/pkg/macaroon" "storj.io/storj/pkg/metainfo/kvmetainfo" ecclient "storj.io/storj/pkg/storage/ec" @@ -32,7 +33,7 @@ const ( ) func TestBucketsBasic(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { // Create new bucket bucket, err := db.CreateBucket(ctx, TestBucket, nil) if assert.NoError(t, err) { @@ -72,7 +73,7 @@ func TestBucketsBasic(t *testing.T) { } func TestBucketsReadWrite(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { // Create new bucket bucket, err := db.CreateBucket(ctx, TestBucket, nil) if assert.NoError(t, err) { @@ -112,7 +113,7 @@ func TestBucketsReadWrite(t *testing.T) { } func TestErrNoBucket(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { _, err := db.CreateBucket(ctx, "", nil) assert.True(t, storj.ErrNoBucket.Has(err)) @@ -125,7 +126,7 @@ func TestErrNoBucket(t *testing.T) { } func TestBucketCreateCipher(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { forAllCiphers(func(cipher storj.Cipher) { bucket, err := db.CreateBucket(ctx, "test", &storj.Bucket{PathCipher: cipher}) if assert.NoError(t, err) { @@ -144,7 +145,7 @@ func TestBucketCreateCipher(t *testing.T) { } func TestListBucketsEmpty(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { _, err := db.ListBuckets(ctx, storj.BucketListOptions{}) assert.EqualError(t, err, "kvmetainfo: invalid direction 0") @@ -164,7 +165,7 @@ func TestListBucketsEmpty(t *testing.T) { } func TestListBuckets(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { bucketNames := []string{"a00", "aa0", "b00", "bb0", "c00"} for _, name := range bucketNames { @@ -270,14 +271,14 @@ func getBucketNames(bucketList storj.BucketList) []string { return names } -func runTest(t *testing.T, test func(context.Context, *testplanet.Planet, *kvmetainfo.DB, streams.Store)) { +func runTest(t *testing.T, test func(*testing.T, context.Context, *testplanet.Planet, *kvmetainfo.DB, streams.Store)) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { db, streams, err := newMetainfoParts(planet) require.NoError(t, err) - test(ctx, planet, db, streams) + test(t, ctx, planet, db, streams) }) } @@ -329,15 +330,18 @@ func newMetainfoParts(planet *testplanet.Planet) (*kvmetainfo.DB, streams.Store, key := new(storj.Key) copy(key[:], TestEncKey) + encStore := encryption.NewStore() + encStore.SetDefaultKey(key) + const stripesPerBlock = 2 blockSize := stripesPerBlock * rs.StripeSize() inlineThreshold := 8 * memory.KiB.Int() - streams, err := streams.NewStreamStore(segments, 64*memory.MiB.Int64(), key, blockSize, storj.AESGCM, inlineThreshold) + streams, err := streams.NewStreamStore(segments, 64*memory.MiB.Int64(), encStore, blockSize, storj.AESGCM, inlineThreshold) if err != nil { return nil, nil, err } proj := kvmetainfo.NewProject(streams, int32(blockSize), rs, 64*memory.MiB.Int64()) - return kvmetainfo.New(proj, metainfo, streams, segments, key), streams, nil + return kvmetainfo.New(proj, metainfo, streams, segments, encStore), streams, nil } func forAllCiphers(test func(cipher storj.Cipher)) { diff --git a/pkg/metainfo/kvmetainfo/metainfo.go b/pkg/metainfo/kvmetainfo/metainfo.go index 824153a0b..46ef42401 100644 --- a/pkg/metainfo/kvmetainfo/metainfo.go +++ b/pkg/metainfo/kvmetainfo/metainfo.go @@ -10,6 +10,7 @@ import ( monkit "gopkg.in/spacemonkeygo/monkit.v2" "storj.io/storj/internal/memory" + "storj.io/storj/pkg/encryption" "storj.io/storj/pkg/storage/segments" "storj.io/storj/pkg/storage/streams" "storj.io/storj/pkg/storj" @@ -34,17 +35,17 @@ type DB struct { streams streams.Store segments segments.Store - rootKey *storj.Key + encStore *encryption.Store } // New creates a new metainfo database -func New(project *Project, metainfo metainfo.Client, streams streams.Store, segments segments.Store, rootKey *storj.Key) *DB { +func New(project *Project, metainfo metainfo.Client, streams streams.Store, segments segments.Store, encStore *encryption.Store) *DB { return &DB{ project: project, metainfo: metainfo, streams: streams, segments: segments, - rootKey: rootKey, + encStore: encStore, } } diff --git a/pkg/metainfo/kvmetainfo/objects.go b/pkg/metainfo/kvmetainfo/objects.go index 44a00d6a8..972d6f0b1 100644 --- a/pkg/metainfo/kvmetainfo/objects.go +++ b/pkg/metainfo/kvmetainfo/objects.go @@ -15,6 +15,7 @@ import ( "storj.io/storj/internal/memory" "storj.io/storj/pkg/encryption" + "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/storage/meta" "storj.io/storj/pkg/storage/objects" @@ -24,11 +25,6 @@ import ( "storj.io/storj/storage" ) -const ( - // commitedPrefix is prefix where completed object info is stored - committedPrefix = "l/" -) - // DefaultRS default values for RedundancyScheme var DefaultRS = storj.RedundancyScheme{ Algorithm: storj.ReedSolomon, @@ -50,7 +46,7 @@ var DefaultES = storj.EncryptionScheme{ func (db *DB) GetObject(ctx context.Context, bucket string, path storj.Path) (info storj.Object, err error) { defer mon.Task()(&ctx)(&err) - _, info, err = db.getInfo(ctx, committedPrefix, bucket, path) + _, info, err = db.getInfo(ctx, bucket, path) return info, err } @@ -59,21 +55,22 @@ func (db *DB) GetObject(ctx context.Context, bucket string, path storj.Path) (in func (db *DB) GetObjectStream(ctx context.Context, bucket string, path storj.Path) (stream storj.ReadOnlyStream, err error) { defer mon.Task()(&ctx)(&err) - meta, info, err := db.getInfo(ctx, committedPrefix, bucket, path) + meta, info, err := db.getInfo(ctx, bucket, path) if err != nil { return nil, err } - streamKey, err := encryption.DeriveContentKey(meta.fullpath, db.rootKey) + streamKey, err := encryption.StoreDeriveContentKey(bucket, meta.fullpath.UnencryptedPath(), db.encStore) if err != nil { return nil, err } return &readonlyStream{ - db: db, - info: info, - encryptedPath: meta.encryptedPath, - streamKey: streamKey, + db: db, + info: info, + bucket: meta.bucket, + encPath: meta.encPath.Raw(), + streamKey: streamKey, }, nil } @@ -222,16 +219,18 @@ func (db *DB) ListObjects(ctx context.Context, bucket string, options storj.List } type object struct { - fullpath string - encryptedPath string + fullpath streams.Path + bucket string + encPath paths.Encrypted lastSegmentMeta segments.Meta streamInfo pb.StreamInfo streamMeta pb.StreamMeta } -func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path storj.Path) (obj object, info storj.Object, err error) { +func (db *DB) getInfo(ctx context.Context, bucket string, path storj.Path) (obj object, info storj.Object, err error) { defer mon.Task()(&ctx)(&err) + // TODO: we shouldn't need to go load the bucket metadata every time we get object info bucketInfo, err := db.GetBucket(ctx, bucket) if err != nil { return object{}, storj.Object{}, err @@ -241,14 +240,14 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st return object{}, storj.Object{}, storj.ErrNoPath.New("") } - fullpath := bucket + "/" + path + fullpath := streams.CreatePath(bucket, paths.NewUnencrypted(path)) - encryptedPath, err := streams.EncryptAfterBucket(ctx, fullpath, bucketInfo.PathCipher, db.rootKey) + encPath, err := encryption.StoreEncryptPath(bucket, paths.NewUnencrypted(path), bucketInfo.PathCipher, db.encStore) if err != nil { return object{}, storj.Object{}, err } - pointer, err := db.metainfo.SegmentInfo(ctx, bucket, storj.JoinPaths(storj.SplitPath(encryptedPath)[1:]...), -1) + pointer, err := db.metainfo.SegmentInfo(ctx, bucket, encPath.Raw(), -1) if err != nil { if storage.ErrKeyNotFound.Has(err) { err = storj.ErrObjectNotFound.Wrap(err) @@ -278,7 +277,7 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st Data: pointer.GetMetadata(), } - streamInfoData, streamMeta, err := streams.DecryptStreamInfo(ctx, lastSegmentMeta.Data, fullpath, db.rootKey) + streamInfoData, streamMeta, err := streams.TypedDecryptStreamInfo(ctx, lastSegmentMeta.Data, fullpath, db.encStore) if err != nil { return object{}, storj.Object{}, err } @@ -296,7 +295,8 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st return object{ fullpath: fullpath, - encryptedPath: encryptedPath, + bucket: bucket, + encPath: encPath, lastSegmentMeta: lastSegmentMeta, streamInfo: streamInfo, streamMeta: streamMeta, @@ -418,7 +418,7 @@ func (object *mutableObject) DeleteStream(ctx context.Context) (err error) { func (object *mutableObject) Commit(ctx context.Context) (err error) { defer mon.Task()(&ctx)(&err) - _, info, err := object.db.getInfo(ctx, committedPrefix, object.info.Bucket.Name, object.info.Path) + _, info, err := object.db.getInfo(ctx, object.info.Bucket.Name, object.info.Path) object.info = info return err } diff --git a/pkg/metainfo/kvmetainfo/objects_test.go b/pkg/metainfo/kvmetainfo/objects_test.go index a6ce30f16..448939fd3 100644 --- a/pkg/metainfo/kvmetainfo/objects_test.go +++ b/pkg/metainfo/kvmetainfo/objects_test.go @@ -39,7 +39,7 @@ func TestCreateObject(t *testing.T) { BlockSize: stripesPerBlock * customRS.StripeSize(), } - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { bucket, err := db.CreateBucket(ctx, TestBucket, nil) require.NoError(t, err) @@ -87,7 +87,7 @@ func TestCreateObject(t *testing.T) { } func TestGetObject(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { bucket, err := db.CreateBucket(ctx, TestBucket, nil) require.NoError(t, err) upload(ctx, t, db, streams, bucket, TestFile, nil) @@ -114,7 +114,7 @@ func TestGetObject(t *testing.T) { } func TestGetObjectStream(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { data := make([]byte, 32*memory.KiB) _, err := rand.Read(data) require.NoError(t, err) @@ -243,7 +243,7 @@ func assertRemoteSegment(t *testing.T, segment storj.Segment) { } func TestDeleteObject(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { bucket, err := db.CreateBucket(ctx, TestBucket, nil) if !assert.NoError(t, err) { return @@ -269,7 +269,7 @@ func TestDeleteObject(t *testing.T) { } func TestListObjectsEmpty(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { bucket, err := db.CreateBucket(ctx, TestBucket, nil) require.NoError(t, err) @@ -295,7 +295,7 @@ func TestListObjectsEmpty(t *testing.T) { } func TestListObjects(t *testing.T) { - runTest(t, func(ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { + runTest(t, func(t *testing.T, ctx context.Context, planet *testplanet.Planet, db *kvmetainfo.DB, streams streams.Store) { bucket, err := db.CreateBucket(ctx, TestBucket, &storj.Bucket{PathCipher: storj.Unencrypted}) require.NoError(t, err) diff --git a/pkg/metainfo/kvmetainfo/stream.go b/pkg/metainfo/kvmetainfo/stream.go index dd722630c..ac29b6497 100644 --- a/pkg/metainfo/kvmetainfo/stream.go +++ b/pkg/metainfo/kvmetainfo/stream.go @@ -19,9 +19,10 @@ var _ storj.ReadOnlyStream = (*readonlyStream)(nil) type readonlyStream struct { db *DB - info storj.Object - encryptedPath storj.Path - streamKey *storj.Key // lazySegmentReader derivedKey + info storj.Object + bucket string + encPath storj.Path + streamKey *storj.Key // lazySegmentReader derivedKey } func (stream *readonlyStream) Info() storj.Object { return stream.info } @@ -44,10 +45,9 @@ func (stream *readonlyStream) segment(ctx context.Context, index int64) (segment Index: index, } - var segmentPath storj.Path isLastSegment := segment.Index+1 == stream.info.SegmentCount if !isLastSegment { - segmentPath = getSegmentPath(stream.encryptedPath, index) + segmentPath := getSegmentPath(storj.JoinPaths(stream.bucket, stream.encPath), index) _, meta, err := stream.db.segments.Get(ctx, segmentPath) if err != nil { return segment, err @@ -79,15 +79,11 @@ func (stream *readonlyStream) segment(ctx context.Context, index int64) (segment return segment, err } - pathComponents := storj.SplitPath(stream.encryptedPath) - bucket := pathComponents[0] - segmentPath = storj.JoinPaths(pathComponents[1:]...) - if isLastSegment { index = -1 } - pointer, err := stream.db.metainfo.SegmentInfo(ctx, bucket, segmentPath, index) + pointer, err := stream.db.metainfo.SegmentInfo(ctx, stream.bucket, stream.encPath, index) if err != nil { return segment, err } diff --git a/pkg/metainfo/kvmetainfo/temputils.go b/pkg/metainfo/kvmetainfo/temputils.go index f98a77f32..926f5f32b 100644 --- a/pkg/metainfo/kvmetainfo/temputils.go +++ b/pkg/metainfo/kvmetainfo/temputils.go @@ -9,6 +9,7 @@ import ( "storj.io/storj/internal/memory" "storj.io/storj/pkg/eestream" + "storj.io/storj/pkg/encryption" "storj.io/storj/pkg/storage/segments" "storj.io/storj/pkg/storage/streams" "storj.io/storj/pkg/storj" @@ -34,10 +35,13 @@ func SetupProject(m metainfo.Client) (*Project, error) { maxBucketMetaSize := 10 * memory.MiB segment := segments.NewSegmentStore(m, nil, rs, maxBucketMetaSize.Int(), maxBucketMetaSize.Int64()) - // volatile warning: we're setting an encryption key of all zeros when one isn't provided. - // TODO: fix before the final alpha network wipe - encryptionKey := new(storj.Key) - strms, err := streams.NewStreamStore(segment, maxBucketMetaSize.Int64(), encryptionKey, memory.KiB.Int(), storj.AESGCM, maxBucketMetaSize.Int()) + // volatile warning: we're setting an encryption key of all zeros for bucket + // metadata, when really the bucket metadata should be stored in a different + // system altogether. + // TODO: https://storjlabs.atlassian.net/browse/V3-1967 + encStore := encryption.NewStore() + encStore.SetDefaultKey(new(storj.Key)) + strms, err := streams.NewStreamStore(segment, maxBucketMetaSize.Int64(), encStore, memory.KiB.Int(), storj.AESGCM, maxBucketMetaSize.Int()) if err != nil { return nil, Error.New("failed to create streams: %v", err) } diff --git a/pkg/miniogw/gateway_test.go b/pkg/miniogw/gateway_test.go index 7d774e09e..213cb7f35 100644 --- a/pkg/miniogw/gateway_test.go +++ b/pkg/miniogw/gateway_test.go @@ -23,6 +23,7 @@ import ( "storj.io/storj/internal/testplanet" libuplink "storj.io/storj/lib/uplink" "storj.io/storj/pkg/eestream" + "storj.io/storj/pkg/encryption" "storj.io/storj/pkg/macaroon" "storj.io/storj/pkg/metainfo/kvmetainfo" "storj.io/storj/pkg/pb" @@ -704,9 +705,12 @@ func initEnv(ctx context.Context, planet *testplanet.Planet) (minio.ObjectLayer, encKey := new(storj.Key) copy(encKey[:], TestEncKey) + encStore := encryption.NewStore() + encStore.SetDefaultKey(encKey) + blockSize := rs.StripeSize() inlineThreshold := 4 * memory.KiB.Int() - strms, err := streams.NewStreamStore(segments, 64*memory.MiB.Int64(), encKey, blockSize, storj.AESGCM, inlineThreshold) + strms, err := streams.NewStreamStore(segments, 64*memory.MiB.Int64(), encStore, blockSize, storj.AESGCM, inlineThreshold) if err != nil { return nil, nil, nil, err } @@ -715,7 +719,7 @@ func initEnv(ctx context.Context, planet *testplanet.Planet) (minio.ObjectLayer, if err != nil { return nil, nil, nil, err } - kvm := kvmetainfo.New(p, m, strms, segments, encKey) + kvm := kvmetainfo.New(p, m, strms, segments, encStore) cfg := libuplink.Config{} cfg.Volatile.TLS = struct { diff --git a/pkg/paths/path.go b/pkg/paths/path.go new file mode 100644 index 000000000..6d60dcde7 --- /dev/null +++ b/pkg/paths/path.go @@ -0,0 +1,153 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package paths + +import ( + "strings" +) + +// +// To avoid confusion about when paths are encrypted, unencrypted, empty or +// non existent, we create some wrapper types so that the compiler will complain +// if someone attempts to use one in the wrong context. +// + +// Unencrypted is an opaque type representing an unencrypted path. +type Unencrypted struct { + raw string +} + +// Encrypted is an opaque type representing an encrypted path. +type Encrypted struct { + raw string +} + +// +// unencrypted paths +// + +// NewUnencrypted takes a raw unencrypted path and returns it wrapped. +func NewUnencrypted(raw string) Unencrypted { + return Unencrypted{raw: raw} +} + +// Valid returns if the unencrypted path is valid, which is the same as not being empty. +func (path Unencrypted) Valid() bool { + return path.raw != "" +} + +// Raw returns the original raw path for the Unencrypted. +func (path Unencrypted) Raw() string { + return path.raw +} + +// String returns a human readable form of the Unencrypted. +func (path Unencrypted) String() string { + return path.Raw() +} + +// Consume attempts to remove the prefix from the Unencrypted path and +// reports a boolean indicating if it was able to do so. +func (path Unencrypted) Consume(prefix Unencrypted) (Unencrypted, bool) { + if len(path.raw) >= len(prefix.raw) && path.raw[:len(prefix.raw)] == prefix.raw { + return NewUnencrypted(path.raw[len(prefix.raw):]), true + } + return Unencrypted{}, false +} + +// Iterator returns an iterator over the components of the Unencrypted. +func (path Unencrypted) Iterator() Iterator { + return NewIterator(path.raw) +} + +// Less returns true if 'path' should be sorted earlier than 'other' +func (path Unencrypted) Less(other Unencrypted) bool { + return path.raw < other.raw +} + +// +// encrypted path +// + +// NewEncrypted takes a raw encrypted path and returns it wrapped. +func NewEncrypted(raw string) Encrypted { + return Encrypted{raw: raw} +} + +// Valid returns if the encrypted path is valid, which is the same as not being empty. +func (path Encrypted) Valid() bool { + return path.raw != "" +} + +// Raw returns the original path for the Encrypted. +func (path Encrypted) Raw() string { + return path.raw +} + +// String returns a human readable form of the Encrypted. +func (path Encrypted) String() string { + return path.Raw() +} + +// Consume attempts to remove the prefix from the Encrypted path and +// reports a boolean indicating if it was able to do so. +func (path Encrypted) Consume(prefix Encrypted) (Encrypted, bool) { + if len(path.raw) >= len(prefix.raw) && path.raw[:len(prefix.raw)] == prefix.raw { + return NewEncrypted(path.raw[len(prefix.raw):]), true + } + return Encrypted{}, false +} + +// Iterator returns an iterator over the components of the Encrypted. +func (path Encrypted) Iterator() Iterator { + return NewIterator(path.raw) +} + +// Less returns true if 'path' should be sorted earlier than 'other' +func (path Encrypted) Less(other Encrypted) bool { + return path.raw < other.raw +} + +// +// path component iteration +// + +// Iterator allows one to efficiently iterate over components of a path. +type Iterator struct { + raw string + consumed int + lastEmpty bool +} + +// NewIterator returns an Iterator for components of the provided raw path. +func NewIterator(raw string) Iterator { + return Iterator{raw: raw, lastEmpty: raw != ""} +} + +// Consumed reports how much of the path has been consumed (if any). +func (pi Iterator) Consumed() string { return pi.raw[:pi.consumed] } + +// Remaining reports how much of the path is remaining. +func (pi Iterator) Remaining() string { return pi.raw[pi.consumed:] } + +// Done reports if the path has been fully consumed. +func (pi Iterator) Done() bool { return len(pi.raw) == pi.consumed && !pi.lastEmpty } + +// Next returns the first component of the path, consuming it. +func (pi *Iterator) Next() string { + if pi.Done() { + return "" + } + + rem := pi.Remaining() + index := strings.IndexByte(rem, '/') + if index == -1 { + pi.consumed += len(rem) + pi.lastEmpty = false + return rem + } + pi.consumed += index + 1 + pi.lastEmpty = index == len(rem)-1 + return rem[:index] +} diff --git a/pkg/paths/path_test.go b/pkg/paths/path_test.go new file mode 100644 index 000000000..e957c282a --- /dev/null +++ b/pkg/paths/path_test.go @@ -0,0 +1,96 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package paths + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestUnencrypted(t *testing.T) { + it := NewUnencrypted("foo").Iterator() + assert.Equal(t, it.Consumed(), "") + assert.False(t, it.Done()) + assert.Equal(t, it.Next(), "foo") + assert.Equal(t, it.Consumed(), "foo") + assert.True(t, it.Done()) + + it = NewUnencrypted("").Iterator() + assert.Equal(t, it.Consumed(), "") + assert.True(t, it.Done()) + assert.Equal(t, it.Next(), "") + assert.Equal(t, it.Consumed(), "") + + it = NewUnencrypted("foo/").Iterator() + assert.Equal(t, it.Consumed(), "") + assert.False(t, it.Done()) + assert.Equal(t, it.Next(), "foo") + assert.Equal(t, it.Consumed(), "foo/") + assert.False(t, it.Done()) + assert.Equal(t, it.Next(), "") + assert.Equal(t, it.Consumed(), "foo/") + assert.True(t, it.Done()) + + it = Unencrypted{}.Iterator() + assert.Equal(t, it.Consumed(), "") + assert.True(t, it.Done()) + assert.Equal(t, it.Next(), "") + assert.Equal(t, it.Consumed(), "") +} + +func TestEncrypted(t *testing.T) { + it := NewEncrypted("foo").Iterator() + assert.Equal(t, it.Consumed(), "") + assert.False(t, it.Done()) + assert.Equal(t, it.Next(), "foo") + assert.Equal(t, it.Consumed(), "foo") + assert.True(t, it.Done()) + + it = NewEncrypted("").Iterator() + assert.Equal(t, it.Consumed(), "") + assert.True(t, it.Done()) + assert.Equal(t, it.Next(), "") + assert.Equal(t, it.Consumed(), "") + + it = NewEncrypted("foo/").Iterator() + assert.Equal(t, it.Consumed(), "") + assert.False(t, it.Done()) + assert.Equal(t, it.Next(), "foo") + assert.Equal(t, it.Consumed(), "foo/") + assert.False(t, it.Done()) + assert.Equal(t, it.Next(), "") + assert.Equal(t, it.Consumed(), "foo/") + assert.True(t, it.Done()) + + it = Encrypted{}.Iterator() + assert.Equal(t, it.Consumed(), "") + assert.True(t, it.Done()) + assert.Equal(t, it.Next(), "") + assert.Equal(t, it.Consumed(), "") +} + +func TestIterator(t *testing.T) { + for i, tt := range []struct { + path string + comps []string + }{ + {"", []string{}}, + {"/", []string{"", ""}}, + {"//", []string{"", "", ""}}, + {" ", []string{" "}}, + {"a", []string{"a"}}, + {"/a/", []string{"", "a", ""}}, + {"a/b/c/d", []string{"a", "b", "c", "d"}}, + {"///a//b////c/d///", []string{"", "", "", "a", "", "b", "", "", "", "c", "d", "", "", ""}}, + } { + errTag := fmt.Sprintf("Test case #%d", i) + iter, got := NewIterator(tt.path), make([]string, 0, len(tt.comps)) + for !iter.Done() { + got = append(got, iter.Next()) + } + assert.Equal(t, tt.comps, got, errTag) + } +} diff --git a/pkg/storage/streams/path.go b/pkg/storage/streams/path.go new file mode 100644 index 000000000..fe3638395 --- /dev/null +++ b/pkg/storage/streams/path.go @@ -0,0 +1,60 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package streams + +import ( + "bytes" + "context" + + "storj.io/storj/pkg/paths" +) + +// Path is a representation of an object path within a bucket +type Path struct { + bucket string + unencPath paths.Unencrypted + raw []byte +} + +// Bucket returns the bucket part of the path. +func (p Path) Bucket() string { return p.bucket } + +// UnencryptedPath returns the unencrypted path part of the path. +func (p Path) UnencryptedPath() paths.Unencrypted { return p.unencPath } + +// Raw returns the raw data in the path. +func (p Path) Raw() []byte { return append([]byte(nil), p.raw...) } + +// String returns the string form of the raw data in the path. +func (p Path) String() string { return string(p.raw) } + +// ParsePath returns a new Path with the given raw bytes. +func ParsePath(ctx context.Context, raw []byte) (path Path, err error) { + defer mon.Task()(&ctx)(&err) + + // A path must contain a bucket and maybe an unencrypted path. + parts := bytes.SplitN(raw, []byte("/"), 2) + + path.raw = raw + path.bucket = string(parts[0]) + if len(parts) > 1 { + path.unencPath = paths.NewUnencrypted(string(parts[1])) + } + + return path, nil +} + +// CreatePath will create a Path for the provided information. +func CreatePath(bucket string, unencPath paths.Unencrypted) (path Path) { + path.bucket = bucket + path.unencPath = unencPath + + path.raw = append(path.raw, bucket...) + if unencPath.Valid() { + path.raw = append(path.raw, '/') + path.raw = append(path.raw, unencPath.Raw()...) + } + + return path +} diff --git a/pkg/storage/streams/shim.go b/pkg/storage/streams/shim.go new file mode 100644 index 000000000..4806e0051 --- /dev/null +++ b/pkg/storage/streams/shim.go @@ -0,0 +1,130 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package streams + +import ( + "context" + "io" + "time" + + "storj.io/storj/pkg/encryption" + "storj.io/storj/pkg/pb" + "storj.io/storj/pkg/ranger" + "storj.io/storj/pkg/storage/segments" + "storj.io/storj/pkg/storj" +) + +// Store interface methods for streams to satisfy to be a store +type Store interface { + Meta(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (Meta, error) + Get(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (ranger.Ranger, Meta, error) + Put(ctx context.Context, path storj.Path, pathCipher storj.Cipher, data io.Reader, metadata []byte, expiration time.Time) (Meta, error) + Delete(ctx context.Context, path storj.Path, pathCipher storj.Cipher) error + List(ctx context.Context, prefix, startAfter, endBefore storj.Path, pathCipher storj.Cipher, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) +} + +type shimStore struct { + store typedStore +} + +// NewStreamStore constructs a Store. +func NewStreamStore(segments segments.Store, segmentSize int64, encStore *encryption.Store, encBlockSize int, cipher storj.Cipher, inlineThreshold int) (Store, error) { + typedStore, err := newTypedStreamStore(segments, segmentSize, encStore, encBlockSize, cipher, inlineThreshold) + if err != nil { + return nil, err + } + return &shimStore{store: typedStore}, nil +} + +// Meta parses the passed in path and dispatches to the typed store. +func (s *shimStore) Meta(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (_ Meta, err error) { + defer mon.Task()(&ctx)(&err) + + streamsPath, err := ParsePath(ctx, []byte(path)) + if err != nil { + return Meta{}, err + } + return s.store.Meta(ctx, streamsPath, pathCipher) +} + +// Get parses the passed in path and dispatches to the typed store. +func (s *shimStore) Get(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (_ ranger.Ranger, _ Meta, err error) { + defer mon.Task()(&ctx)(&err) + + streamsPath, err := ParsePath(ctx, []byte(path)) + if err != nil { + return nil, Meta{}, err + } + return s.store.Get(ctx, streamsPath, pathCipher) +} + +// Put parses the passed in path and dispatches to the typed store. +func (s *shimStore) Put(ctx context.Context, path storj.Path, pathCipher storj.Cipher, data io.Reader, metadata []byte, expiration time.Time) (_ Meta, err error) { + defer mon.Task()(&ctx)(&err) + + streamsPath, err := ParsePath(ctx, []byte(path)) + if err != nil { + return Meta{}, err + } + return s.store.Put(ctx, streamsPath, pathCipher, data, metadata, expiration) +} + +// Delete parses the passed in path and dispatches to the typed store. +func (s *shimStore) Delete(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (err error) { + defer mon.Task()(&ctx)(&err) + + streamsPath, err := ParsePath(ctx, []byte(path)) + if err != nil { + return err + } + return s.store.Delete(ctx, streamsPath, pathCipher) +} + +// List parses the passed in path and dispatches to the typed store. +func (s *shimStore) List(ctx context.Context, prefix storj.Path, startAfter storj.Path, endBefore storj.Path, pathCipher storj.Cipher, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) { + defer mon.Task()(&ctx)(&err) + + // TODO: list is maybe wrong? + streamsPrefix, err := ParsePath(ctx, []byte(prefix)) + if err != nil { + return nil, false, err + } + return s.store.List(ctx, streamsPrefix, startAfter, endBefore, pathCipher, recursive, limit, metaFlags) +} + +// EncryptAfterBucket encrypts a path without encrypting its first element. This is a legacy function +// that should no longer be needed after the typed path refactoring. +func EncryptAfterBucket(ctx context.Context, path storj.Path, cipher storj.Cipher, key *storj.Key) (encrypted storj.Path, err error) { + defer mon.Task()(&ctx)(&err) + + comps := storj.SplitPath(path) + if len(comps) <= 1 { + return path, nil + } + + encrypted, err = encryption.EncryptPath(path, cipher, key) + if err != nil { + return "", err + } + + // replace the first path component with the unencrypted bucket name + return storj.JoinPaths(comps[0], storj.JoinPaths(storj.SplitPath(encrypted)[1:]...)), nil +} + +// DecryptStreamInfo decrypts stream info. This is a legacy function that should no longer +// be needed after the typed path refactoring. +func DecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path storj.Path, rootKey *storj.Key) ( + streamInfo []byte, streamMeta pb.StreamMeta, err error) { + defer mon.Task()(&ctx)(&err) + + streamsPath, err := ParsePath(ctx, []byte(path)) + if err != nil { + return nil, pb.StreamMeta{}, err + } + + store := encryption.NewStore() + store.SetDefaultKey(rootKey) + + return TypedDecryptStreamInfo(ctx, streamMetaBytes, streamsPath, store) +} diff --git a/pkg/storage/streams/store.go b/pkg/storage/streams/store.go index 9ddb5a946..cbbfdcef2 100644 --- a/pkg/storage/streams/store.go +++ b/pkg/storage/streams/store.go @@ -7,10 +7,9 @@ import ( "bytes" "context" "crypto/rand" - "fmt" "io" "io/ioutil" - "strings" + "strconv" "time" "github.com/gogo/protobuf/proto" @@ -20,6 +19,7 @@ import ( "storj.io/storj/pkg/eestream" "storj.io/storj/pkg/encryption" + "storj.io/storj/pkg/paths" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/meta" @@ -49,32 +49,30 @@ func convertMeta(lastSegmentMeta segments.Meta, stream pb.StreamInfo, streamMeta } // Store interface methods for streams to satisfy to be a store -type Store interface { - Meta(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (Meta, error) - Get(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (ranger.Ranger, Meta, error) - Put(ctx context.Context, path storj.Path, pathCipher storj.Cipher, data io.Reader, metadata []byte, expiration time.Time) (Meta, error) - Delete(ctx context.Context, path storj.Path, pathCipher storj.Cipher) error - List(ctx context.Context, prefix, startAfter, endBefore storj.Path, pathCipher storj.Cipher, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) +type typedStore interface { + Meta(ctx context.Context, path Path, pathCipher storj.Cipher) (Meta, error) + Get(ctx context.Context, path Path, pathCipher storj.Cipher) (ranger.Ranger, Meta, error) + Put(ctx context.Context, path Path, pathCipher storj.Cipher, data io.Reader, metadata []byte, expiration time.Time) (Meta, error) + Delete(ctx context.Context, path Path, pathCipher storj.Cipher) error + List(ctx context.Context, prefix Path, startAfter, endBefore string, pathCipher storj.Cipher, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) } -// streamStore is a store for streams +// streamStore is a store for streams. It implements typedStore as part of an ongoing migration +// to use typed paths. See the shim for the store that the rest of the world interacts with. type streamStore struct { segments segments.Store segmentSize int64 - rootKey *storj.Key + encStore *encryption.Store encBlockSize int cipher storj.Cipher inlineThreshold int } -// NewStreamStore stuff -func NewStreamStore(segments segments.Store, segmentSize int64, rootKey *storj.Key, encBlockSize int, cipher storj.Cipher, inlineThreshold int) (Store, error) { +// newTypedStreamStore constructs a typedStore backed by a streamStore. +func newTypedStreamStore(segments segments.Store, segmentSize int64, encStore *encryption.Store, encBlockSize int, cipher storj.Cipher, inlineThreshold int) (typedStore, error) { if segmentSize <= 0 { return nil, errs.New("segment size must be larger than 0") } - if rootKey == nil { - return nil, errs.New("encryption key must not be empty") - } if encBlockSize <= 0 { return nil, errs.New("encryption block size must be larger than 0") } @@ -82,7 +80,7 @@ func NewStreamStore(segments segments.Store, segmentSize int64, rootKey *storj.K return &streamStore{ segments: segments, segmentSize: segmentSize, - rootKey: rootKey, + encStore: encStore, encBlockSize: encBlockSize, cipher: cipher, inlineThreshold: inlineThreshold, @@ -93,13 +91,14 @@ func NewStreamStore(segments segments.Store, segmentSize int64, rootKey *storj.K // store the first piece at s0/, second piece at s1/, and the // *last* piece at l/. Store the given metadata, along with the number // of segments, in a new protobuf, in the metadata of l/. -func (s *streamStore) Put(ctx context.Context, path storj.Path, pathCipher storj.Cipher, data io.Reader, metadata []byte, expiration time.Time) (m Meta, err error) { +func (s *streamStore) Put(ctx context.Context, path Path, pathCipher storj.Cipher, data io.Reader, metadata []byte, expiration time.Time) (m Meta, err error) { defer mon.Task()(&ctx)(&err) + // previously file uploaded? err = s.Delete(ctx, path, pathCipher) if err != nil && !storage.ErrKeyNotFound.Has(err) { - //something wrong happened checking for an existing - //file with the same name + // something wrong happened checking for an existing + // file with the same name return Meta{}, err } @@ -111,7 +110,7 @@ func (s *streamStore) Put(ctx context.Context, path storj.Path, pathCipher storj return m, err } -func (s *streamStore) upload(ctx context.Context, path storj.Path, pathCipher storj.Cipher, data io.Reader, metadata []byte, expiration time.Time) (m Meta, lastSegment int64, err error) { +func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Cipher, data io.Reader, metadata []byte, expiration time.Time) (m Meta, lastSegment int64, err error) { defer mon.Task()(&ctx)(&err) var currentSegment int64 @@ -126,7 +125,11 @@ func (s *streamStore) upload(ctx context.Context, path storj.Path, pathCipher st } }() - derivedKey, err := encryption.DeriveContentKey(path, s.rootKey) + derivedKey, err := encryption.StoreDeriveContentKey(path.Bucket(), path.UnencryptedPath(), s.encStore) + if err != nil { + return Meta{}, currentSegment, err + } + encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) if err != nil { return Meta{}, currentSegment, err } @@ -192,13 +195,11 @@ func (s *streamStore) upload(ctx context.Context, path storj.Path, pathCipher st } putMeta, err = s.segments.Put(ctx, transformedReader, expiration, func() (storj.Path, []byte, error) { - encPath, err := EncryptAfterBucket(ctx, path, pathCipher, s.rootKey) - if err != nil { - return "", nil, err - } - if !eofReader.isEOF() { - segmentPath := getSegmentPath(encPath, currentSegment) + segmentPath, err := createSegmentPath(ctx, currentSegment, path.Bucket(), encPath) + if err != nil { + return "", nil, err + } if s.cipher == storj.Unencrypted { return segmentPath, nil, nil @@ -215,7 +216,10 @@ func (s *streamStore) upload(ctx context.Context, path storj.Path, pathCipher st return segmentPath, segmentMeta, nil } - lastSegmentPath := storj.JoinPaths("l", encPath) + lastSegmentPath, err := createSegmentPath(ctx, -1, path.Bucket(), encPath) + if err != nil { + return "", nil, err + } streamInfo, err := proto.Marshal(&pb.StreamInfo{ NumberOfSegments: currentSegment + 1, @@ -275,28 +279,28 @@ func (s *streamStore) upload(ctx context.Context, path storj.Path, pathCipher st return resultMeta, currentSegment, nil } -// getSegmentPath returns the unique path for a particular segment -func getSegmentPath(path storj.Path, segNum int64) storj.Path { - return storj.JoinPaths(fmt.Sprintf("s%d", segNum), path) -} - // Get returns a ranger that knows what the overall size is (from l/) // and then returns the appropriate data from segments s0/, s1/, // ..., l/. -func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (rr ranger.Ranger, meta Meta, err error) { +func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.Cipher) (rr ranger.Ranger, meta Meta, err error) { defer mon.Task()(&ctx)(&err) - encPath, err := EncryptAfterBucket(ctx, path, pathCipher, s.rootKey) + encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) if err != nil { return nil, Meta{}, err } - lastSegmentRanger, lastSegmentMeta, err := s.segments.Get(ctx, storj.JoinPaths("l", encPath)) + segmentPath, err := createSegmentPath(ctx, -1, path.Bucket(), encPath) if err != nil { return nil, Meta{}, err } - streamInfo, streamMeta, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey) + lastSegmentRanger, lastSegmentMeta, err := s.segments.Get(ctx, segmentPath) + if err != nil { + return nil, Meta{}, err + } + + streamInfo, streamMeta, err := TypedDecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.encStore) if err != nil { return nil, Meta{}, err } @@ -307,30 +311,33 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj return nil, Meta{}, err } - derivedKey, err := encryption.DeriveContentKey(path, s.rootKey) + derivedKey, err := encryption.StoreDeriveContentKey(path.Bucket(), path.UnencryptedPath(), s.encStore) if err != nil { return nil, Meta{}, err } var rangers []ranger.Ranger for i := int64(0); i < stream.NumberOfSegments-1; i++ { - currentPath := getSegmentPath(encPath, i) - size := stream.SegmentsSize - var contentNonce storj.Nonce - _, err := encryption.Increment(&contentNonce, i+1) + currentPath, err := createSegmentPath(ctx, i, path.Bucket(), encPath) if err != nil { return nil, Meta{}, err } - rr := &lazySegmentRanger{ + + var contentNonce storj.Nonce + _, err = encryption.Increment(&contentNonce, i+1) + if err != nil { + return nil, Meta{}, err + } + + rangers = append(rangers, &lazySegmentRanger{ segments: s.segments, path: currentPath, - size: size, + size: stream.SegmentsSize, derivedKey: derivedKey, startingNonce: &contentNonce, encBlockSize: int(streamMeta.EncryptionBlockSize), cipher: storj.Cipher(streamMeta.EncryptionType), - } - rangers = append(rangers, rr) + }) } var contentNonce storj.Nonce @@ -338,6 +345,7 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj if err != nil { return nil, Meta{}, err } + encryptedKey, keyNonce := getEncryptedKeyAndNonce(streamMeta.LastSegmentMeta) decryptedLastSegmentRanger, err := decryptRanger( ctx, @@ -361,23 +369,29 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj } // Meta implements Store.Meta -func (s *streamStore) Meta(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (meta Meta, err error) { +func (s *streamStore) Meta(ctx context.Context, path Path, pathCipher storj.Cipher) (meta Meta, err error) { defer mon.Task()(&ctx)(&err) - encPath, err := EncryptAfterBucket(ctx, path, pathCipher, s.rootKey) + encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) if err != nil { return Meta{}, err } - lastSegmentMeta, err := s.segments.Meta(ctx, storj.JoinPaths("l", encPath)) + segmentPath, err := createSegmentPath(ctx, -1, path.Bucket(), encPath) if err != nil { return Meta{}, err } - streamInfo, streamMeta, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey) + lastSegmentMeta, err := s.segments.Meta(ctx, segmentPath) if err != nil { return Meta{}, err } + + streamInfo, streamMeta, err := TypedDecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.encStore) + if err != nil { + return Meta{}, err + } + var stream pb.StreamInfo if err := proto.Unmarshal(streamInfo, &stream); err != nil { return Meta{}, err @@ -387,19 +401,25 @@ func (s *streamStore) Meta(ctx context.Context, path storj.Path, pathCipher stor } // Delete all the segments, with the last one last -func (s *streamStore) Delete(ctx context.Context, path storj.Path, pathCipher storj.Cipher) (err error) { +func (s *streamStore) Delete(ctx context.Context, path Path, pathCipher storj.Cipher) (err error) { defer mon.Task()(&ctx)(&err) - encPath, err := EncryptAfterBucket(ctx, path, pathCipher, s.rootKey) - if err != nil { - return err - } - lastSegmentMeta, err := s.segments.Meta(ctx, storj.JoinPaths("l", encPath)) + encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) if err != nil { return err } - streamInfo, _, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey) + lastSegmentPath, err := createSegmentPath(ctx, -1, path.Bucket(), encPath) + if err != nil { + return err + } + + lastSegmentMeta, err := s.segments.Meta(ctx, lastSegmentPath) + if err != nil { + return err + } + + streamInfo, _, err := TypedDecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.encStore) if err != nil { return err } @@ -409,29 +429,29 @@ func (s *streamStore) Delete(ctx context.Context, path storj.Path, pathCipher st } for i := 0; i < int(stream.NumberOfSegments-1); i++ { - encPath, err = EncryptAfterBucket(ctx, path, pathCipher, s.rootKey) + currentPath, err := createSegmentPath(ctx, int64(i), path.Bucket(), encPath) if err != nil { return err } - currentPath := getSegmentPath(encPath, int64(i)) - err := s.segments.Delete(ctx, currentPath) + + err = s.segments.Delete(ctx, currentPath) if err != nil { return err } } - return s.segments.Delete(ctx, storj.JoinPaths("l", encPath)) + return s.segments.Delete(ctx, lastSegmentPath) } // ListItem is a single item in a listing type ListItem struct { - Path storj.Path + Path string Meta Meta IsPrefix bool } // List all the paths inside l/, stripping off the l/ prefix -func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore storj.Path, pathCipher storj.Cipher, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) { +func (s *streamStore) List(ctx context.Context, prefix Path, startAfter, endBefore string, pathCipher storj.Cipher, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) { defer mon.Task()(&ctx)(&err) if metaFlags&meta.Size != 0 { @@ -440,74 +460,88 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore st metaFlags |= meta.UserDefined } - prefix = strings.TrimSuffix(prefix, "/") - - encPrefix, err := EncryptAfterBucket(ctx, prefix, pathCipher, s.rootKey) + prefixKey, err := encryption.StoreDerivePathKey(prefix.Bucket(), prefix.UnencryptedPath(), s.encStore) if err != nil { return nil, false, err } - prefixKey, err := encryption.DerivePathKey(prefix, s.rootKey, len(storj.SplitPath(prefix))) + encPrefix, err := encryption.StoreEncryptPath(prefix.Bucket(), prefix.UnencryptedPath(), pathCipher, s.encStore) if err != nil { return nil, false, err } - encStartAfter, err := s.encryptMarker(ctx, startAfter, pathCipher, prefixKey) + // We have to encrypt startAfter and endBefore but only if they don't contain a bucket. + // They contain a bucket if and only if the prefix has no bucket. This is why they are raw + // strings instead of a typed string: it's either a bucket or an unencrypted path component + // and that isn't known at compile time. + needsEncryption := prefix.Bucket() != "" + if needsEncryption { + startAfter, err = encryption.EncryptPathRaw(startAfter, pathCipher, prefixKey) + if err != nil { + return nil, false, err + } + endBefore, err = encryption.EncryptPathRaw(endBefore, pathCipher, prefixKey) + if err != nil { + return nil, false, err + } + } + + segmentPrefix, err := createSegmentPath(ctx, -1, prefix.Bucket(), encPrefix) if err != nil { return nil, false, err } - encEndBefore, err := s.encryptMarker(ctx, endBefore, pathCipher, prefixKey) - if err != nil { - return nil, false, err - } - - segments, more, err := s.segments.List(ctx, storj.JoinPaths("l", encPrefix), encStartAfter, encEndBefore, recursive, limit, metaFlags) + segments, more, err := s.segments.List(ctx, segmentPrefix, startAfter, endBefore, recursive, limit, metaFlags) if err != nil { return nil, false, err } items = make([]ListItem, len(segments)) for i, item := range segments { - path, err := s.decryptMarker(ctx, item.Path, pathCipher, prefixKey) + var path Path + var itemPath string + + if needsEncryption { + itemPath, err = encryption.DecryptPathRaw(item.Path, pathCipher, prefixKey) + if err != nil { + return nil, false, err + } + + // TODO(jeff): this shouldn't be necessary if we handled trailing slashes + // appropriately. there's some issues with list. + fullPath := prefix.UnencryptedPath().Raw() + if len(fullPath) > 0 && fullPath[len(fullPath)-1] != '/' { + fullPath += "/" + } + fullPath += itemPath + + path = CreatePath(prefix.Bucket(), paths.NewUnencrypted(fullPath)) + } else { + itemPath = item.Path + path = CreatePath(item.Path, paths.Unencrypted{}) + } + + streamInfo, streamMeta, err := TypedDecryptStreamInfo(ctx, item.Meta.Data, path, s.encStore) if err != nil { return nil, false, err } - streamInfo, streamMeta, err := DecryptStreamInfo(ctx, item.Meta.Data, storj.JoinPaths(prefix, path), s.rootKey) - if err != nil { - return nil, false, err - } var stream pb.StreamInfo if err := proto.Unmarshal(streamInfo, &stream); err != nil { return nil, false, err } newMeta := convertMeta(item.Meta, stream, streamMeta) - items[i] = ListItem{Path: path, Meta: newMeta, IsPrefix: item.IsPrefix} + items[i] = ListItem{ + Path: itemPath, + Meta: newMeta, + IsPrefix: item.IsPrefix, + } } return items, more, nil } -// encryptMarker is a helper method for encrypting startAfter and endBefore markers -func (s *streamStore) encryptMarker(ctx context.Context, marker storj.Path, pathCipher storj.Cipher, prefixKey *storj.Key) (_ storj.Path, err error) { - defer mon.Task()(&ctx)(&err) - if bytes.Equal(s.rootKey[:], prefixKey[:]) { // empty prefix - return EncryptAfterBucket(ctx, marker, pathCipher, s.rootKey) - } - return encryption.EncryptPath(marker, pathCipher, prefixKey) -} - -// decryptMarker is a helper method for decrypting listed path markers -func (s *streamStore) decryptMarker(ctx context.Context, marker storj.Path, pathCipher storj.Cipher, prefixKey *storj.Key) (_ storj.Path, err error) { - defer mon.Task()(&ctx)(&err) - if bytes.Equal(s.rootKey[:], prefixKey[:]) { // empty prefix - return DecryptAfterBucket(ctx, marker, pathCipher, s.rootKey) - } - return encryption.DecryptPath(marker, pathCipher, prefixKey) -} - type lazySegmentRanger struct { ranger ranger.Ranger segments segments.Store @@ -584,60 +618,27 @@ func decryptRanger(ctx context.Context, rr ranger.Ranger, decryptedSize int64, c return eestream.Unpad(rd, int(rd.Size()-decryptedSize)) } -// EncryptAfterBucket encrypts a path without encrypting its first element -func EncryptAfterBucket(ctx context.Context, path storj.Path, cipher storj.Cipher, key *storj.Key) (encrypted storj.Path, err error) { - defer mon.Task()(&ctx)(&err) - comps := storj.SplitPath(path) - if len(comps) <= 1 { - return path, nil - } - - encrypted, err = encryption.EncryptPath(path, cipher, key) - if err != nil { - return "", err - } - - // replace the first path component with the unencrypted bucket name - return storj.JoinPaths(comps[0], storj.JoinPaths(storj.SplitPath(encrypted)[1:]...)), nil -} - -// DecryptAfterBucket decrypts a path without modifying its first element -func DecryptAfterBucket(ctx context.Context, path storj.Path, cipher storj.Cipher, key *storj.Key) (decrypted storj.Path, err error) { - defer mon.Task()(&ctx)(&err) - comps := storj.SplitPath(path) - if len(comps) <= 1 { - return path, nil - } - - bucket := comps[0] - toDecrypt := storj.JoinPaths(comps[1:]...) - - bucketKey, err := encryption.DerivePathKey(path, key, 1) - if err != nil { - return "", err - } - - decPath, err := encryption.DecryptPath(toDecrypt, cipher, bucketKey) - if err != nil { - return "", err - } - - return storj.JoinPaths(bucket, decPath), nil -} - // CancelHandler handles clean up of segments on receiving CTRL+C -func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path storj.Path, pathCipher storj.Cipher) { +func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path Path, pathCipher storj.Cipher) { defer mon.Task()(&ctx)(nil) + + encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) + if err != nil { + zap.S().Warnf("Failed deleting segments: %v", err) + return + } + for i := int64(0); i < totalSegments; i++ { - encPath, err := EncryptAfterBucket(ctx, path, pathCipher, s.rootKey) + currentPath, err := createSegmentPath(ctx, i, path.Bucket(), encPath) if err != nil { - zap.S().Warnf("Failed deleting a segment due to encryption path %v %v", i, err) + zap.S().Warnf("Failed deleting segment %d: %v", i, err) + continue } - currentPath := getSegmentPath(encPath, i) err = s.segments.Delete(ctx, currentPath) if err != nil { - zap.S().Warnf("Failed deleting a segment %v %v", currentPath, err) + zap.S().Warnf("Failed deleting segment %v: %v", currentPath, err) + continue } } } @@ -653,16 +654,17 @@ func getEncryptedKeyAndNonce(m *pb.SegmentMeta) (storj.EncryptedPrivateKey, *sto return m.EncryptedKey, &nonce } -// DecryptStreamInfo decrypts stream info -func DecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path storj.Path, rootKey *storj.Key) ( +// TypedDecryptStreamInfo decrypts stream info +func TypedDecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path Path, encStore *encryption.Store) ( streamInfo []byte, streamMeta pb.StreamMeta, err error) { defer mon.Task()(&ctx)(&err) + err = proto.Unmarshal(streamMetaBytes, &streamMeta) if err != nil { return nil, pb.StreamMeta{}, err } - derivedKey, err := encryption.DeriveContentKey(path, rootKey) + derivedKey, err := encryption.StoreDeriveContentKey(path.Bucket(), path.UnencryptedPath(), encStore) if err != nil { return nil, pb.StreamMeta{}, err } @@ -678,3 +680,33 @@ func DecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path storj.P streamInfo, err = encryption.Decrypt(streamMeta.EncryptedStreamInfo, cipher, contentKey, &storj.Nonce{}) return streamInfo, streamMeta, err } + +// createSegmentPath will create a storj.Path that the segment store expects. +func createSegmentPath(ctx context.Context, segmentIndex int64, bucket string, encPath paths.Encrypted) (path storj.Path, err error) { + defer mon.Task()(&ctx)(&err) + + if segmentIndex < -1 { + return "", errs.New("invalid segment index") + } + + var raw []byte + if segmentIndex > -1 { + raw = append(raw, 's') + raw = append(raw, strconv.FormatInt(segmentIndex, 10)...) + } else { + raw = append(raw, 'l') + } + raw = append(raw, '/') + + if len(bucket) > 0 { + raw = append(raw, bucket...) + raw = append(raw, '/') + + if encPath.Valid() { + raw = append(raw, encPath.Raw()...) + raw = append(raw, '/') + } + } + + return storj.Path(raw[:len(raw)-1]), nil +} diff --git a/pkg/storage/streams/store_test.go b/pkg/storage/streams/store_test.go index 712f62e7f..1d442678e 100644 --- a/pkg/storage/streams/store_test.go +++ b/pkg/storage/streams/store_test.go @@ -15,6 +15,7 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "storj.io/storj/pkg/encryption" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/ranger" "storj.io/storj/pkg/storage/segments" @@ -25,6 +26,12 @@ var ( ctx = context.Background() ) +func newStore() *encryption.Store { + store := encryption.NewStore() + store.SetDefaultKey(new(storj.Key)) + return store +} + func TestStreamStoreMeta(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() @@ -89,7 +96,7 @@ func TestStreamStoreMeta(t *testing.T) { Meta(gomock.Any(), gomock.Any()). Return(test.segmentMeta, test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, storj.AESGCM, 4) + streamStore, err := NewStreamStore(mockSegmentStore, 10, newStore(), 10, storj.AESGCM, 4) if err != nil { t.Fatal(err) } @@ -169,7 +176,7 @@ func TestStreamStorePut(t *testing.T) { Delete(gomock.Any(), gomock.Any()). Return(test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, segSize, new(storj.Key), encBlockSize, dataCipher, inlineSize) + streamStore, err := NewStreamStore(mockSegmentStore, segSize, newStore(), encBlockSize, dataCipher, inlineSize) if err != nil { t.Fatal(err) } @@ -279,7 +286,7 @@ func TestStreamStoreGet(t *testing.T) { gomock.InOrder(calls...) - streamStore, err := NewStreamStore(mockSegmentStore, segSize, new(storj.Key), encBlockSize, dataCipher, inlineSize) + streamStore, err := NewStreamStore(mockSegmentStore, segSize, newStore(), encBlockSize, dataCipher, inlineSize) if err != nil { t.Fatal(err) } @@ -328,7 +335,7 @@ func TestStreamStoreDelete(t *testing.T) { Delete(gomock.Any(), gomock.Any()). Return(test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0, 0) + streamStore, err := NewStreamStore(mockSegmentStore, 10, newStore(), 10, 0, 0) if err != nil { t.Fatal(err) } @@ -372,7 +379,7 @@ func TestStreamStoreList(t *testing.T) { List(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(test.segments, test.segmentMore, test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0, 0) + streamStore, err := NewStreamStore(mockSegmentStore, 10, newStore(), 10, 0, 0) if err != nil { t.Fatal(err) } diff --git a/uplink/config.go b/uplink/config.go index 6e71e8531..e2b79485e 100644 --- a/uplink/config.go +++ b/uplink/config.go @@ -130,14 +130,16 @@ func (c Config) GetMetainfo(ctx context.Context, identity *identity.FullIdentity return nil, nil, Error.Wrap(err) } - strms, err := streams.NewStreamStore(segment, c.Client.SegmentSize.Int64(), key, + encStore := encryption.NewStore() + encStore.SetDefaultKey(key) + strms, err := streams.NewStreamStore(segment, c.Client.SegmentSize.Int64(), encStore, int(blockSize), storj.Cipher(c.Enc.DataType), c.Client.MaxInlineSize.Int(), ) if err != nil { return nil, nil, Error.New("failed to create stream store: %v", err) } - return kvmetainfo.New(project, m, strms, segment, key), strms, nil + return kvmetainfo.New(project, m, strms, segment, encStore), strms, nil } // GetRedundancyScheme returns the configured redundancy scheme for new uploads