refactor some store encryption stuff (#2434)

This commit is contained in:
Jeff Wendling 2019-07-05 08:36:35 +00:00 committed by Kaloyan Raev
parent 537c6021d5
commit 8b07df37f5
10 changed files with 238 additions and 537 deletions

View File

@ -88,11 +88,11 @@ func (s *EncryptionAccess) Restrict(apiKey APIKey, restrictions ...EncryptionRes
unencPath := paths.NewUnencrypted(res.PathPrefix) unencPath := paths.NewUnencrypted(res.PathPrefix)
cipher := storj.EncAESGCM // TODO(jeff): pick the right path cipher cipher := storj.EncAESGCM // TODO(jeff): pick the right path cipher
encPath, err := encryption.StoreEncryptPath(res.Bucket, unencPath, cipher, s.store) encPath, err := encryption.EncryptPath(res.Bucket, unencPath, cipher, s.store)
if err != nil { if err != nil {
return APIKey{}, nil, err return APIKey{}, nil, err
} }
derivedKey, err := encryption.StoreDerivePathKey(res.Bucket, unencPath, s.store) derivedKey, err := encryption.DerivePathKey(res.Bucket, unencPath, s.store)
if err != nil { if err != nil {
return APIKey{}, nil, err return APIKey{}, nil, err
} }

View File

@ -16,8 +16,9 @@ import (
"storj.io/storj/internal/testplanet" "storj.io/storj/internal/testplanet"
"storj.io/storj/internal/testrand" "storj.io/storj/internal/testrand"
"storj.io/storj/pkg/audit" "storj.io/storj/pkg/audit"
"storj.io/storj/pkg/encryption"
"storj.io/storj/pkg/overlay" "storj.io/storj/pkg/overlay"
"storj.io/storj/pkg/storage/streams" "storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storj" "storj.io/storj/pkg/storj"
"storj.io/storj/satellite" "storj.io/storj/satellite"
) )
@ -136,10 +137,11 @@ func TestDisqualifiedNodesGetNoDownload(t *testing.T) {
encParameters := upl.GetConfig(satellite).GetEncryptionParameters() encParameters := upl.GetConfig(satellite).GetEncryptionParameters()
cipherSuite := encParameters.CipherSuite cipherSuite := encParameters.CipherSuite
encryptedAfterBucket, err := streams.EncryptAfterBucket(ctx, "testbucket/test/path", cipherSuite, &storj.Key{}) store := encryption.NewStore()
store.SetDefaultKey(new(storj.Key))
encryptedPath, err := encryption.EncryptPath("testbucket", paths.NewUnencrypted("test/path"), cipherSuite, store)
require.NoError(t, err) require.NoError(t, err)
lastSegPath := storj.JoinPaths(projects[0].ID.String(), "l", "testbucket", encryptedPath.Raw())
lastSegPath := storj.JoinPaths(projects[0].ID.String(), "l", encryptedAfterBucket)
pointer, err := satellite.Metainfo.Service.Get(ctx, lastSegPath) pointer, err := satellite.Metainfo.Service.Get(ctx, lastSegPath)
require.NoError(t, err) require.NoError(t, err)

View File

@ -8,11 +8,12 @@ import (
"fmt" "fmt"
"storj.io/storj/pkg/encryption" "storj.io/storj/pkg/encryption"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storj" "storj.io/storj/pkg/storj"
) )
func ExampleEncryptPath() { func ExampleEncryptPath() {
var path = "fold1/fold2/fold3/file.txt" path := paths.NewUnencrypted("fold1/fold2/fold3/file.txt")
// seed // seed
seed := new(storj.Key) seed := new(storj.Key)
@ -21,8 +22,11 @@ func ExampleEncryptPath() {
} }
fmt.Printf("root key (%d bytes): %s\n", len(seed), hex.EncodeToString(seed[:])) fmt.Printf("root key (%d bytes): %s\n", len(seed), hex.EncodeToString(seed[:]))
store := encryption.NewStore()
store.SetDefaultKey(seed)
// use the seed for encrypting the path // use the seed for encrypting the path
encryptedPath, err := encryption.EncryptPath(path, storj.EncAESGCM, seed) encryptedPath, err := encryption.EncryptPath("bucket", path, storj.EncAESGCM, store)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -30,22 +34,7 @@ func ExampleEncryptPath() {
fmt.Println("encrypted path: ", encryptedPath) fmt.Println("encrypted path: ", encryptedPath)
// decrypting the path // decrypting the path
decryptedPath, err := encryption.DecryptPath(encryptedPath, storj.EncAESGCM, seed) decryptedPath, err := encryption.DecryptPath("bucket", encryptedPath, storj.EncAESGCM, store)
if err != nil {
panic(err)
}
fmt.Println("decrypted path: ", decryptedPath)
// handling of shared path
sharedPath := storj.JoinPaths(storj.SplitPath(encryptedPath)[2:]...)
fmt.Println("shared path: ", sharedPath)
derivedKey, err := encryption.DerivePathKey(decryptedPath, seed, 2)
if err != nil {
panic(err)
}
fmt.Printf("derived key (%d bytes): %s\n", len(derivedKey), hex.EncodeToString(derivedKey[:]))
decryptedPath, err = encryption.DecryptPath(sharedPath, storj.EncAESGCM, derivedKey)
if err != nil { if err != nil {
panic(err) panic(err)
} }
@ -54,9 +43,6 @@ func ExampleEncryptPath() {
// Output: // Output:
// root key (32 bytes): 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f // root key (32 bytes): 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f
// path to encrypt: fold1/fold2/fold3/file.txt // path to encrypt: fold1/fold2/fold3/file.txt
// encrypted path: urxuYzqG_ZlJfBhkGaz87WvvnCZaYD7qf1_ZN_Pd91n5/IyncDwLhWPv4F7EaoUivwICnUeJMWlUnMATL4faaoH2s/_1gitX6uPd3etc3RgoD9R1waT5MPKrlrY32ehz_vqlOv/6qO4DU5AHFabE2r7hmAauvnomvtNByuO-FCw4ch_xaVR3SPE // encrypted path: OHzjTiBUvLmgQouCAYdu74MlqDl791aOka_EBzlAb_rR/RT0pG5y4lHFVRi1sHtwjZ1B7DeVbRvpyMfO6atfOefSC/rXJX6O9Pk4rGtnlLUIUoc9Gz0y6N-xemdNyAasbo3dQm/qiEo3IYUlA989mKFE7WB98GHJK88AI98hhUgwv39ePexslzg
// decrypted path: fold1/fold2/fold3/file.txt // decrypted path: fold1/fold2/fold3/file.txt
// shared path: _1gitX6uPd3etc3RgoD9R1waT5MPKrlrY32ehz_vqlOv/6qO4DU5AHFabE2r7hmAauvnomvtNByuO-FCw4ch_xaVR3SPE
// derived key (32 bytes): 909db5ccf2b645e3352ee8212305596ed514d9f84d5acd21d93b4527d2a0c7e1
// decrypted path: fold3/file.txt
} }

View File

@ -7,103 +7,234 @@ import (
"crypto/hmac" "crypto/hmac"
"crypto/sha512" "crypto/sha512"
"encoding/base64" "encoding/base64"
"strings"
"github.com/zeebo/errs"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storj" "storj.io/storj/pkg/storj"
) )
// EncryptPath encrypts path with the given key // EncryptPath encrypts the path using the provided cipher and looking up
func EncryptPath(path storj.Path, cipher storj.CipherSuite, key *storj.Key) (encrypted storj.Path, err error) { // keys from the provided store and bucket.
// do not encrypt empty paths func EncryptPath(bucket string, path paths.Unencrypted, cipher storj.CipherSuite, store *Store) (
if len(path) == 0 { encPath paths.Encrypted, err error) {
return path, nil
// Invalid paths map to invalid paths
if !path.Valid() {
return paths.Encrypted{}, nil
} }
if cipher == storj.EncNull { if cipher == storj.EncNull {
return path, nil return paths.NewEncrypted(path.Raw()), nil
} }
comps := storj.SplitPath(path) _, consumed, base := store.LookupUnencrypted(bucket, path)
for i, comp := range comps { if base == nil {
comps[i], err = encryptPathComponent(comp, cipher, key) return paths.Encrypted{}, errs.New("unable to find encryption base for: %s/%q", bucket, path)
}
remaining, ok := path.Consume(consumed)
if !ok {
return paths.Encrypted{}, errs.New("unable to encrypt bucket path: %s/%q", bucket, path)
}
// if we didn't consume any path, we're at the root of the bucket, and so we have
// to fold the bucket name into the key.
key := &base.Key
if !consumed.Valid() {
key, err = derivePathKeyComponent(key, bucket)
if err != nil { if err != nil {
return "", err return paths.Encrypted{}, errs.Wrap(err)
}
key, err = DeriveKey(key, "path:"+comp)
if err != nil {
return "", err
} }
} }
return storj.JoinPaths(comps...), nil
encrypted, err := EncryptPathRaw(remaining.Raw(), cipher, key)
if err != nil {
return paths.Encrypted{}, errs.Wrap(err)
}
var builder strings.Builder
builder.WriteString(base.Encrypted.Raw())
if len(encrypted) > 0 {
if builder.Len() > 0 {
builder.WriteByte('/')
}
builder.WriteString(encrypted)
}
return paths.NewEncrypted(builder.String()), nil
} }
// DecryptPath decrypts path with the given key // EncryptPathRaw encrypts the path using the provided key directly. EncryptPath should be
func DecryptPath(path storj.Path, cipher storj.CipherSuite, key *storj.Key) (decrypted storj.Path, err error) { // preferred if possible.
func EncryptPathRaw(raw string, cipher storj.CipherSuite, key *storj.Key) (string, error) {
if cipher == storj.EncNull { if cipher == storj.EncNull {
return path, nil return raw, nil
} }
comps := storj.SplitPath(path) var builder strings.Builder
for i, comp := range comps { for iter, i := paths.NewIterator(raw), 0; !iter.Done(); i++ {
comps[i], err = decryptPathComponent(comp, cipher, key) component := iter.Next()
encComponent, err := encryptPathComponent(component, cipher, key)
if err != nil { if err != nil {
return "", err return "", errs.Wrap(err)
} }
key, err = DeriveKey(key, "path:"+comps[i]) key, err = derivePathKeyComponent(key, component)
if err != nil { if err != nil {
return "", err return "", errs.Wrap(err)
} }
if i > 0 {
builder.WriteByte('/')
}
builder.WriteString(encComponent)
} }
return storj.JoinPaths(comps...), nil return builder.String(), nil
} }
// DerivePathKey derives the key for the given depth from the given root key. // DecryptPath decrypts the path using the provided cipher and looking up
// This method must be called on an unencrypted path. // keys from the provided store and bucket.
func DerivePathKey(path storj.Path, key *storj.Key, depth int) (derivedKey *storj.Key, err error) { func DecryptPath(bucket string, path paths.Encrypted, cipher storj.CipherSuite, store *Store) (
if depth < 0 { unencPath paths.Unencrypted, err error) {
return nil, Error.New("negative depth")
// Invalid paths map to invalid paths
if !path.Valid() {
return paths.Unencrypted{}, nil
} }
// do not derive key from empty path if cipher == storj.EncNull {
if len(path) == 0 { return paths.NewUnencrypted(path.Raw()), nil
}
_, consumed, base := store.LookupEncrypted(bucket, path)
if base == nil {
return paths.Unencrypted{}, errs.New("unable to find decryption base for: %q", path)
}
remaining, ok := path.Consume(consumed)
if !ok {
return paths.Unencrypted{}, errs.New("unable to decrypt bucket path: %q", path)
}
// if we didn't consume any path, we're at the root of the bucket, and so we have
// to fold the bucket name into the key.
key := &base.Key
if !consumed.Valid() {
key, err = derivePathKeyComponent(key, bucket)
if err != nil {
return paths.Unencrypted{}, errs.Wrap(err)
}
}
decrypted, err := DecryptPathRaw(remaining.Raw(), cipher, key)
if err != nil {
return paths.Unencrypted{}, errs.Wrap(err)
}
var builder strings.Builder
builder.WriteString(base.Unencrypted.Raw())
if len(decrypted) > 0 {
if builder.Len() > 0 {
builder.WriteByte('/')
}
builder.WriteString(decrypted)
}
return paths.NewUnencrypted(builder.String()), nil
}
// DecryptPathRaw decrypts the path using the provided key directly. DecryptPath should be
// preferred if possible.
func DecryptPathRaw(raw string, cipher storj.CipherSuite, key *storj.Key) (string, error) {
if cipher == storj.EncNull {
return raw, nil
}
var builder strings.Builder
for iter, i := paths.NewIterator(raw), 0; !iter.Done(); i++ {
component := iter.Next()
unencComponent, err := decryptPathComponent(component, cipher, key)
if err != nil {
return "", errs.Wrap(err)
}
key, err = derivePathKeyComponent(key, unencComponent)
if err != nil {
return "", errs.Wrap(err)
}
if i > 0 {
builder.WriteByte('/')
}
builder.WriteString(unencComponent)
}
return builder.String(), nil
}
// DeriveContentKey returns the content key for the passed in path by looking up
// the appropriate base key from the store and bucket and deriving the rest.
func DeriveContentKey(bucket string, path paths.Unencrypted, store *Store) (key *storj.Key, err error) {
key, err = DerivePathKey(bucket, path, store)
if err != nil {
return nil, errs.Wrap(err)
}
key, err = DeriveKey(key, "content")
return key, errs.Wrap(err)
}
// DerivePathKey returns the path key for the passed in path by looking up the
// appropriate base key from the store and bucket and deriving the rest.
func DerivePathKey(bucket string, path paths.Unencrypted, store *Store) (key *storj.Key, err error) {
_, consumed, base := store.LookupUnencrypted(bucket, path)
if base == nil {
return nil, errs.New("unable to find encryption base for: %s/%q", bucket, path)
}
// If asking for the key at the bucket, do that and return.
if !path.Valid() {
key, err = derivePathKeyComponent(&base.Key, bucket)
if err != nil {
return nil, errs.Wrap(err)
}
return key, nil return key, nil
} }
comps := storj.SplitPath(path) remaining, ok := path.Consume(consumed)
if depth > len(comps) { if !ok {
return nil, Error.New("depth greater than path length") return nil, errs.New("unable to derive path key for: %s/%q", bucket, path)
} }
derivedKey = key // if we didn't consume any path, we're at the root of the bucket, and so we have
for i := 0; i < depth; i++ { // to fold the bucket name into the key.
derivedKey, err = DeriveKey(derivedKey, "path:"+comps[i]) key = &base.Key
if !consumed.Valid() {
key, err = derivePathKeyComponent(key, bucket)
if err != nil { if err != nil {
return nil, err return nil, errs.Wrap(err)
} }
} }
return derivedKey, nil
for iter := remaining.Iterator(); !iter.Done(); {
key, err = derivePathKeyComponent(key, iter.Next())
if err != nil {
return nil, errs.Wrap(err)
}
}
return key, nil
} }
// DeriveContentKey derives the key for the encrypted object data using the root key. // derivePathKeyComponent derives a new key from the provided one using the component. It
// This method must be called on an unencrypted path. // should be preferred over DeriveKey when adding path components as it performs the
func DeriveContentKey(path storj.Path, key *storj.Key) (derivedKey *storj.Key, err error) { // necessary transformation to the component.
comps := storj.SplitPath(path) func derivePathKeyComponent(key *storj.Key, component string) (*storj.Key, error) {
if len(comps) == 0 { return DeriveKey(key, "path:"+component)
return nil, Error.New("path is empty")
}
derivedKey, err = DerivePathKey(path, key, len(comps))
if err != nil {
return nil, err
}
derivedKey, err = DeriveKey(derivedKey, "content")
if err != nil {
return nil, err
}
return derivedKey, nil
} }
// encryptPathComponent encrypts a single path component with the provided cipher and key.
func encryptPathComponent(comp string, cipher storj.CipherSuite, key *storj.Key) (string, error) { func encryptPathComponent(comp string, cipher storj.CipherSuite, key *storj.Key) (string, error) {
// derive the key for the current path component // derive the key for the next path component. this is so that
derivedKey, err := DeriveKey(key, "path:"+comp) // every encrypted component has a unique nonce.
derivedKey, err := derivePathKeyComponent(key, comp)
if err != nil { if err != nil {
return "", err return "", err
} }
@ -133,6 +264,7 @@ func encryptPathComponent(comp string, cipher storj.CipherSuite, key *storj.Key)
return base64.RawURLEncoding.EncodeToString(append(nonce[:nonceSize], cipherText...)), nil return base64.RawURLEncoding.EncodeToString(append(nonce[:nonceSize], cipherText...)), nil
} }
// decryptPathComponent decrypts a single path component with the provided cipher and key.
func decryptPathComponent(comp string, cipher storj.CipherSuite, key *storj.Key) (string, error) { func decryptPathComponent(comp string, cipher storj.CipherSuite, key *storj.Key) (string, error) {
if comp == "" { if comp == "" {
return "", nil return "", nil
@ -147,6 +279,9 @@ func decryptPathComponent(comp string, cipher storj.CipherSuite, key *storj.Key)
if cipher == storj.EncAESGCM { if cipher == storj.EncAESGCM {
nonceSize = AESGCMNonceSize nonceSize = AESGCMNonceSize
} }
if len(data) < nonceSize || nonceSize < 0 {
return "", errs.New("component did not contain enough nonce bytes")
}
// extract the nonce from the cipher text // extract the nonce from the cipher text
nonce := new(storj.Nonce) nonce := new(storj.Nonce)

View File

@ -1,300 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package encryption
import (
"crypto/hmac"
"crypto/sha512"
"encoding/base64"
"strings"
"github.com/zeebo/errs"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storj"
)
// StoreEncryptPath encrypts the path using the provided cipher and looking up
// keys from the provided store and bucket.
func StoreEncryptPath(bucket string, path paths.Unencrypted, cipher storj.CipherSuite, store *Store) (
encPath paths.Encrypted, err error) {
// Invalid paths map to invalid paths
if !path.Valid() {
return paths.Encrypted{}, nil
}
if cipher == storj.EncNull {
return paths.NewEncrypted(path.Raw()), nil
}
_, consumed, base := store.LookupUnencrypted(bucket, path)
if base == nil {
return paths.Encrypted{}, errs.New("unable to find encryption base for: %s/%q", bucket, path)
}
remaining, ok := path.Consume(consumed)
if !ok {
return paths.Encrypted{}, errs.New("unable to encrypt bucket path: %s/%q", bucket, path)
}
// if we didn't consume any path, we're at the root of the bucket, and so we have
// to fold the bucket name into the key.
key := &base.Key
if !consumed.Valid() {
key, err = derivePathKeyComponent(key, bucket)
if err != nil {
return paths.Encrypted{}, errs.Wrap(err)
}
}
encrypted, err := EncryptPathRaw(remaining.Raw(), cipher, key)
if err != nil {
return paths.Encrypted{}, errs.Wrap(err)
}
var builder strings.Builder
builder.WriteString(base.Encrypted.Raw())
if len(encrypted) > 0 {
if builder.Len() > 0 {
builder.WriteByte('/')
}
builder.WriteString(encrypted)
}
return paths.NewEncrypted(builder.String()), nil
}
// EncryptPathRaw encrypts the path using the provided key directly. EncryptPath should be
// preferred if possible.
func EncryptPathRaw(raw string, cipher storj.CipherSuite, key *storj.Key) (string, error) {
if cipher == storj.EncNull {
return raw, nil
}
var builder strings.Builder
for iter, i := paths.NewIterator(raw), 0; !iter.Done(); i++ {
component := iter.Next()
encComponent, err := storeEncryptPathComponent(component, cipher, key)
if err != nil {
return "", errs.Wrap(err)
}
key, err = derivePathKeyComponent(key, component)
if err != nil {
return "", errs.Wrap(err)
}
if i > 0 {
builder.WriteByte('/')
}
builder.WriteString(encComponent)
}
return builder.String(), nil
}
// StoreDecryptPath decrypts the path using the provided cipher and looking up
// keys from the provided store and bucket.
func StoreDecryptPath(bucket string, path paths.Encrypted, cipher storj.CipherSuite, store *Store) (
unencPath paths.Unencrypted, err error) {
// Invalid paths map to invalid paths
if !path.Valid() {
return paths.Unencrypted{}, nil
}
if cipher == storj.EncNull {
return paths.NewUnencrypted(path.Raw()), nil
}
_, consumed, base := store.LookupEncrypted(bucket, path)
if base == nil {
return paths.Unencrypted{}, errs.New("unable to find decryption base for: %q", path)
}
remaining, ok := path.Consume(consumed)
if !ok {
return paths.Unencrypted{}, errs.New("unable to decrypt bucket path: %q", path)
}
// if we didn't consume any path, we're at the root of the bucket, and so we have
// to fold the bucket name into the key.
key := &base.Key
if !consumed.Valid() {
key, err = derivePathKeyComponent(key, bucket)
if err != nil {
return paths.Unencrypted{}, errs.Wrap(err)
}
}
decrypted, err := DecryptPathRaw(remaining.Raw(), cipher, key)
if err != nil {
return paths.Unencrypted{}, errs.Wrap(err)
}
var builder strings.Builder
builder.WriteString(base.Unencrypted.Raw())
if len(decrypted) > 0 {
if builder.Len() > 0 {
builder.WriteByte('/')
}
builder.WriteString(decrypted)
}
return paths.NewUnencrypted(builder.String()), nil
}
// DecryptPathRaw decrypts the path using the provided key directly. DecryptPath should be
// preferred if possible.
func DecryptPathRaw(raw string, cipher storj.CipherSuite, key *storj.Key) (string, error) {
if cipher == storj.EncNull {
return raw, nil
}
var builder strings.Builder
for iter, i := paths.NewIterator(raw), 0; !iter.Done(); i++ {
component := iter.Next()
unencComponent, err := storeDecryptPathComponent(component, cipher, key)
if err != nil {
return "", errs.Wrap(err)
}
key, err = derivePathKeyComponent(key, unencComponent)
if err != nil {
return "", errs.Wrap(err)
}
if i > 0 {
builder.WriteByte('/')
}
builder.WriteString(unencComponent)
}
return builder.String(), nil
}
// StoreDeriveContentKey returns the content key for the passed in path by looking up
// the appropriate base key from the store and bucket and deriving the rest.
func StoreDeriveContentKey(bucket string, path paths.Unencrypted, store *Store) (key *storj.Key, err error) {
key, err = StoreDerivePathKey(bucket, path, store)
if err != nil {
return nil, errs.Wrap(err)
}
key, err = DeriveKey(key, "content")
return key, errs.Wrap(err)
}
// StoreDerivePathKey returns the path key for the passed in path by looking up the
// appropriate base key from the store and bucket and deriving the rest.
func StoreDerivePathKey(bucket string, path paths.Unencrypted, store *Store) (key *storj.Key, err error) {
_, consumed, base := store.LookupUnencrypted(bucket, path)
if base == nil {
return nil, errs.New("unable to find encryption base for: %s/%q", bucket, path)
}
// If asking for the key at the bucket, do that and return.
if !path.Valid() {
key, err = derivePathKeyComponent(&base.Key, bucket)
if err != nil {
return nil, errs.Wrap(err)
}
return key, nil
}
remaining, ok := path.Consume(consumed)
if !ok {
return nil, errs.New("unable to derive path key for: %s/%q", bucket, path)
}
// if we didn't consume any path, we're at the root of the bucket, and so we have
// to fold the bucket name into the key.
key = &base.Key
if !consumed.Valid() {
key, err = derivePathKeyComponent(key, bucket)
if err != nil {
return nil, errs.Wrap(err)
}
}
for iter := remaining.Iterator(); !iter.Done(); {
key, err = derivePathKeyComponent(key, iter.Next())
if err != nil {
return nil, errs.Wrap(err)
}
}
return key, nil
}
// derivePathKeyComponent derives a new key from the provided one using the component. It
// should be preferred over DeriveKey when adding path components as it performs the
// necessary transformation to the component.
func derivePathKeyComponent(key *storj.Key, component string) (*storj.Key, error) {
return DeriveKey(key, "path:"+component)
}
// storeEncryptPathComponent encrypts a single path component with the provided cipher and key.
func storeEncryptPathComponent(comp string, cipher storj.CipherSuite, key *storj.Key) (string, error) {
if comp == "" {
return "", nil
}
// derive the key for the next path component. this is so that
// every encrypted component has a unique nonce.
derivedKey, err := derivePathKeyComponent(key, comp)
if err != nil {
return "", err
}
// use the derived key to derive the nonce
mac := hmac.New(sha512.New, derivedKey[:])
_, err = mac.Write([]byte("nonce"))
if err != nil {
return "", Error.Wrap(err)
}
nonce := new(storj.Nonce)
copy(nonce[:], mac.Sum(nil))
// encrypt the path components with the parent's key and the derived nonce
cipherText, err := Encrypt([]byte(comp), cipher, key, nonce)
if err != nil {
return "", Error.Wrap(err)
}
nonceSize := storj.NonceSize
if cipher == storj.EncAESGCM {
nonceSize = AESGCMNonceSize
}
// keep the nonce together with the cipher text
return base64.RawURLEncoding.EncodeToString(append(nonce[:nonceSize], cipherText...)), nil
}
// storeDecryptPathComponent decrypts a single path component with the provided cipher and key.
func storeDecryptPathComponent(comp string, cipher storj.CipherSuite, key *storj.Key) (string, error) {
if comp == "" {
return "", nil
}
data, err := base64.RawURLEncoding.DecodeString(comp)
if err != nil {
return "", Error.Wrap(err)
}
nonceSize := storj.NonceSize
if cipher == storj.EncAESGCM {
nonceSize = AESGCMNonceSize
}
if len(data) < nonceSize || nonceSize < 0 {
return "", errs.New("component did not contain enough nonce bytes")
}
// extract the nonce from the cipher text
nonce := new(storj.Nonce)
copy(nonce[:], data[:nonceSize])
decrypted, err := Decrypt(data[nonceSize:], cipher, key, nonce)
if err != nil {
return "", Error.Wrap(err)
}
return string(decrypted), nil
}

View File

@ -1,55 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package encryption
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"storj.io/storj/internal/testrand"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storj"
)
func newStore(key storj.Key) *Store {
store := NewStore()
if err := store.Add("bucket", paths.Unencrypted{}, paths.Encrypted{}, key); err != nil {
panic(err)
}
return store
}
func TestStoreEncryption(t *testing.T) {
forAllCiphers(func(cipher storj.CipherSuite) {
for i, rawPath := range []string{
"",
"/",
"//",
"file.txt",
"file.txt/",
"fold1/file.txt",
"fold1/fold2/file.txt",
"/fold1/fold2/fold3/file.txt",
} {
errTag := fmt.Sprintf("test:%d path:%q cipher:%v", i, rawPath, cipher)
store := newStore(testrand.Key())
path := paths.NewUnencrypted(rawPath)
encPath, err := StoreEncryptPath("bucket", path, cipher, store)
if !assert.NoError(t, err, errTag) {
continue
}
decPath, err := StoreDecryptPath("bucket", encPath, cipher, store)
if !assert.NoError(t, err, errTag) {
continue
}
assert.Equal(t, rawPath, decPath.Raw(), errTag)
}
})
}

View File

@ -10,12 +10,21 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"storj.io/storj/internal/testrand" "storj.io/storj/internal/testrand"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/storj" "storj.io/storj/pkg/storj"
) )
func TestEncryption(t *testing.T) { func newStore(key storj.Key) *Store {
store := NewStore()
if err := store.Add("bucket", paths.Unencrypted{}, paths.Encrypted{}, key); err != nil {
panic(err)
}
return store
}
func TestStoreEncryption(t *testing.T) {
forAllCiphers(func(cipher storj.CipherSuite) { forAllCiphers(func(cipher storj.CipherSuite) {
for i, path := range []storj.Path{ for i, rawPath := range []string{
"", "",
"/", "/",
"//", "//",
@ -25,66 +34,22 @@ func TestEncryption(t *testing.T) {
"fold1/fold2/file.txt", "fold1/fold2/file.txt",
"/fold1/fold2/fold3/file.txt", "/fold1/fold2/fold3/file.txt",
} { } {
errTag := fmt.Sprintf("%d. %+v", i, path) errTag := fmt.Sprintf("test:%d path:%q cipher:%v", i, rawPath, cipher)
key := testrand.Key() store := newStore(testrand.Key())
path := paths.NewUnencrypted(rawPath)
encrypted, err := EncryptPath(path, cipher, &key) encPath, err := EncryptPath("bucket", path, cipher, store)
if !assert.NoError(t, err, errTag) { if !assert.NoError(t, err, errTag) {
continue continue
} }
decrypted, err := DecryptPath(encrypted, cipher, &key) decPath, err := DecryptPath("bucket", encPath, cipher, store)
if !assert.NoError(t, err, errTag) { if !assert.NoError(t, err, errTag) {
continue continue
} }
assert.Equal(t, path, decrypted, errTag) assert.Equal(t, rawPath, decPath.Raw(), errTag)
}
})
}
func TestDeriveKey(t *testing.T) {
forAllCiphers(func(cipher storj.CipherSuite) {
for i, tt := range []struct {
path storj.Path
depth int
errString string
}{
{"fold1/fold2/fold3/file.txt", -1, "encryption error: negative depth"},
{"fold1/fold2/fold3/file.txt", 0, ""},
{"fold1/fold2/fold3/file.txt", 1, ""},
{"fold1/fold2/fold3/file.txt", 2, ""},
{"fold1/fold2/fold3/file.txt", 3, ""},
{"fold1/fold2/fold3/file.txt", 4, ""},
{"fold1/fold2/fold3/file.txt", 5, "encryption error: depth greater than path length"},
} {
errTag := fmt.Sprintf("%d. %+v", i, tt)
key := testrand.Key()
encrypted, err := EncryptPath(tt.path, cipher, &key)
if !assert.NoError(t, err, errTag) {
continue
}
derivedKey, err := DerivePathKey(tt.path, &key, tt.depth)
if tt.errString != "" {
assert.EqualError(t, err, tt.errString, errTag)
continue
}
if !assert.NoError(t, err, errTag) {
continue
}
shared := storj.JoinPaths(storj.SplitPath(encrypted)[tt.depth:]...)
decrypted, err := DecryptPath(shared, cipher, derivedKey)
if !assert.NoError(t, err, errTag) {
continue
}
expected := storj.JoinPaths(storj.SplitPath(tt.path)[tt.depth:]...)
assert.Equal(t, expected, decrypted, errTag)
} }
}) })
} }

View File

@ -60,7 +60,7 @@ func (db *DB) GetObjectStream(ctx context.Context, bucket string, path storj.Pat
return nil, err return nil, err
} }
streamKey, err := encryption.StoreDeriveContentKey(bucket, meta.fullpath.UnencryptedPath(), db.encStore) streamKey, err := encryption.DeriveContentKey(bucket, meta.fullpath.UnencryptedPath(), db.encStore)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -242,7 +242,7 @@ func (db *DB) getInfo(ctx context.Context, bucket string, path storj.Path) (obj
fullpath := streams.CreatePath(bucket, paths.NewUnencrypted(path)) fullpath := streams.CreatePath(bucket, paths.NewUnencrypted(path))
encPath, err := encryption.StoreEncryptPath(bucket, paths.NewUnencrypted(path), bucketInfo.PathCipher, db.encStore) encPath, err := encryption.EncryptPath(bucket, paths.NewUnencrypted(path), bucketInfo.PathCipher, db.encStore)
if err != nil { if err != nil {
return object{}, storj.Object{}, err return object{}, storj.Object{}, err
} }

View File

@ -9,7 +9,6 @@ import (
"time" "time"
"storj.io/storj/pkg/encryption" "storj.io/storj/pkg/encryption"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/ranger" "storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/segments" "storj.io/storj/pkg/storage/segments"
"storj.io/storj/pkg/storj" "storj.io/storj/pkg/storj"
@ -71,34 +70,3 @@ func (s *shimStore) List(ctx context.Context, prefix storj.Path, startAfter stor
return s.store.List(ctx, ParsePath(prefix), startAfter, endBefore, pathCipher, recursive, limit, metaFlags) return s.store.List(ctx, ParsePath(prefix), startAfter, endBefore, pathCipher, recursive, limit, metaFlags)
} }
// EncryptAfterBucket encrypts a path without encrypting its first element. This is a legacy function
// that should no longer be needed after the typed path refactoring.
func EncryptAfterBucket(ctx context.Context, path storj.Path, cipher storj.CipherSuite, key *storj.Key) (encrypted storj.Path, err error) {
defer mon.Task()(&ctx)(&err)
comps := storj.SplitPath(path)
if len(comps) <= 1 {
return path, nil
}
encrypted, err = encryption.EncryptPath(path, cipher, key)
if err != nil {
return "", err
}
// replace the first path component with the unencrypted bucket name
return storj.JoinPaths(comps[0], storj.JoinPaths(storj.SplitPath(encrypted)[1:]...)), nil
}
// DecryptStreamInfo decrypts stream info. This is a legacy function that should no longer
// be needed after the typed path refactoring.
func DecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path storj.Path, rootKey *storj.Key) (
streamInfo []byte, streamMeta pb.StreamMeta, err error) {
defer mon.Task()(&ctx)(&err)
store := encryption.NewStore()
store.SetDefaultKey(rootKey)
return TypedDecryptStreamInfo(ctx, streamMetaBytes, ParsePath(path), store)
}

View File

@ -125,11 +125,11 @@ func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Ci
} }
}() }()
derivedKey, err := encryption.StoreDeriveContentKey(path.Bucket(), path.UnencryptedPath(), s.encStore) derivedKey, err := encryption.DeriveContentKey(path.Bucket(), path.UnencryptedPath(), s.encStore)
if err != nil { if err != nil {
return Meta{}, currentSegment, err return Meta{}, currentSegment, err
} }
encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) encPath, err := encryption.EncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore)
if err != nil { if err != nil {
return Meta{}, currentSegment, err return Meta{}, currentSegment, err
} }
@ -285,7 +285,7 @@ func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Ci
func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.CipherSuite) (rr ranger.Ranger, meta Meta, err error) { func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.CipherSuite) (rr ranger.Ranger, meta Meta, err error) {
defer mon.Task()(&ctx)(&err) defer mon.Task()(&ctx)(&err)
encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) encPath, err := encryption.EncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore)
if err != nil { if err != nil {
return nil, Meta{}, err return nil, Meta{}, err
} }
@ -311,7 +311,7 @@ func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.Ciphe
return nil, Meta{}, err return nil, Meta{}, err
} }
derivedKey, err := encryption.StoreDeriveContentKey(path.Bucket(), path.UnencryptedPath(), s.encStore) derivedKey, err := encryption.DeriveContentKey(path.Bucket(), path.UnencryptedPath(), s.encStore)
if err != nil { if err != nil {
return nil, Meta{}, err return nil, Meta{}, err
} }
@ -372,7 +372,7 @@ func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.Ciphe
func (s *streamStore) Meta(ctx context.Context, path Path, pathCipher storj.CipherSuite) (meta Meta, err error) { func (s *streamStore) Meta(ctx context.Context, path Path, pathCipher storj.CipherSuite) (meta Meta, err error) {
defer mon.Task()(&ctx)(&err) defer mon.Task()(&ctx)(&err)
encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) encPath, err := encryption.EncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore)
if err != nil { if err != nil {
return Meta{}, err return Meta{}, err
} }
@ -404,7 +404,7 @@ func (s *streamStore) Meta(ctx context.Context, path Path, pathCipher storj.Ciph
func (s *streamStore) Delete(ctx context.Context, path Path, pathCipher storj.CipherSuite) (err error) { func (s *streamStore) Delete(ctx context.Context, path Path, pathCipher storj.CipherSuite) (err error) {
defer mon.Task()(&ctx)(&err) defer mon.Task()(&ctx)(&err)
encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) encPath, err := encryption.EncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore)
if err != nil { if err != nil {
return err return err
} }
@ -460,12 +460,12 @@ func (s *streamStore) List(ctx context.Context, prefix Path, startAfter, endBefo
metaFlags |= meta.UserDefined metaFlags |= meta.UserDefined
} }
prefixKey, err := encryption.StoreDerivePathKey(prefix.Bucket(), prefix.UnencryptedPath(), s.encStore) prefixKey, err := encryption.DerivePathKey(prefix.Bucket(), prefix.UnencryptedPath(), s.encStore)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
encPrefix, err := encryption.StoreEncryptPath(prefix.Bucket(), prefix.UnencryptedPath(), pathCipher, s.encStore) encPrefix, err := encryption.EncryptPath(prefix.Bucket(), prefix.UnencryptedPath(), pathCipher, s.encStore)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
@ -622,7 +622,7 @@ func decryptRanger(ctx context.Context, rr ranger.Ranger, decryptedSize int64, c
func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path Path, pathCipher storj.CipherSuite) { func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path Path, pathCipher storj.CipherSuite) {
defer mon.Task()(&ctx)(nil) defer mon.Task()(&ctx)(nil)
encPath, err := encryption.StoreEncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore) encPath, err := encryption.EncryptPath(path.Bucket(), path.UnencryptedPath(), pathCipher, s.encStore)
if err != nil { if err != nil {
zap.S().Warnf("Failed deleting segments: %v", err) zap.S().Warnf("Failed deleting segments: %v", err)
return return
@ -664,7 +664,7 @@ func TypedDecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path Pa
return nil, pb.StreamMeta{}, err return nil, pb.StreamMeta{}, err
} }
derivedKey, err := encryption.StoreDeriveContentKey(path.Bucket(), path.UnencryptedPath(), encStore) derivedKey, err := encryption.DeriveContentKey(path.Bucket(), path.UnencryptedPath(), encStore)
if err != nil { if err != nil {
return nil, pb.StreamMeta{}, err return nil, pb.StreamMeta{}, err
} }