storj/pkg/storage/streams/store.go

450 lines
12 KiB
Go
Raw Normal View History

// Copyright (C) 2018 Storj Labs, Inc.
// See LICENSE for copying information.
package streams
import (
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
"bytes"
"context"
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
"crypto/rand"
"fmt"
"io"
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
"io/ioutil"
"time"
proto "github.com/gogo/protobuf/proto"
"github.com/zeebo/errs"
"go.uber.org/zap"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
"storj.io/storj/pkg/eestream"
"storj.io/storj/pkg/paths"
"storj.io/storj/pkg/pb"
ranger "storj.io/storj/pkg/ranger"
"storj.io/storj/pkg/storage/meta"
"storj.io/storj/pkg/storage/segments"
"storj.io/storj/storage"
)
var mon = monkit.Package()
// Meta info about a segment
type Meta struct {
Modified time.Time
Expiration time.Time
Size int64
Data []byte
}
// convertMeta converts segment metadata to stream metadata
func convertMeta(lastSegmentMeta segments.Meta) (Meta, error) {
msi := pb.MetaStreamInfo{}
err := proto.Unmarshal(lastSegmentMeta.Data, &msi)
if err != nil {
return Meta{}, err
}
return Meta{
Modified: lastSegmentMeta.Modified,
Expiration: lastSegmentMeta.Expiration,
Size: ((msi.NumberOfSegments - 1) * msi.SegmentsSize) + msi.LastSegmentSize,
Data: msi.Metadata,
}, nil
}
// Store interface methods for streams to satisfy to be a store
type Store interface {
Meta(ctx context.Context, path paths.Path) (Meta, error)
Get(ctx context.Context, path paths.Path) (ranger.Ranger, Meta, error)
Put(ctx context.Context, path paths.Path, data io.Reader, metadata []byte, expiration time.Time) (Meta, error)
Delete(ctx context.Context, path paths.Path) error
List(ctx context.Context, prefix, startAfter, endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error)
}
// streamStore is a store for streams
type streamStore struct {
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
segments segments.Store
segmentSize int64
rootKey []byte
encBlockSize int
encType eestream.Cipher
}
// NewStreamStore stuff
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
func NewStreamStore(segments segments.Store, segmentSize int64, rootKey string, encBlockSize int, encType int) (Store, error) {
if segmentSize <= 0 {
return nil, errs.New("segment size must be larger than 0")
}
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
if rootKey == "" {
return nil, errs.New("encryption key must not be empty")
}
if encBlockSize <= 0 {
return nil, errs.New("encryption block size must be larger than 0")
}
return &streamStore{
segments: segments,
segmentSize: segmentSize,
rootKey: []byte(rootKey),
encBlockSize: encBlockSize,
encType: eestream.Cipher(encType),
}, nil
}
// Put breaks up data as it comes in into s.segmentSize length pieces, then
// store the first piece at s0/<path>, second piece at s1/<path>, and the
// *last* piece at l/<path>. Store the given metadata, along with the number
// of segments, in a new protobuf, in the metadata of l/<path>.
func (s *streamStore) Put(ctx context.Context, path paths.Path, data io.Reader, metadata []byte, expiration time.Time) (m Meta, err error) {
defer mon.Task()(&ctx)(&err)
// previously file uploaded?
err = s.Delete(ctx, path)
if err != nil && !storage.ErrKeyNotFound.Has(err) {
//something wrong happened checking for an existing
//file with the same name
return Meta{}, err
}
var currentSegment int64
var streamSize int64
var putMeta segments.Meta
defer func() {
select {
case <-ctx.Done():
s.cancelHandler(context.Background(), currentSegment, path)
default:
}
}()
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
derivedKey, err := path.DeriveContentKey(s.rootKey)
if err != nil {
return Meta{}, err
}
cipher := s.encType
eofReader := NewEOFReader(data)
for !eofReader.isEOF() && !eofReader.hasError() {
var encKey eestream.Key
_, err = rand.Read(encKey[:])
if err != nil {
return Meta{}, err
}
var nonce eestream.Nonce
_, err := nonce.Increment(currentSegment)
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
if err != nil {
return Meta{}, err
}
encrypter, err := cipher.NewEncrypter(&encKey, &nonce, s.encBlockSize)
if err != nil {
return Meta{}, err
}
encryptedEncKey, err := cipher.Encrypt(encKey[:], (*eestream.Key)(derivedKey), &nonce)
if err != nil {
return Meta{}, err
}
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
sizeReader := NewSizeReader(eofReader)
segmentReader := io.LimitReader(sizeReader, s.segmentSize)
peekReader := segments.NewPeekThresholdReader(segmentReader)
largeData, err := peekReader.IsLargerThan(encrypter.InBlockSize())
if err != nil {
return Meta{}, err
}
var transformedReader io.Reader
if largeData {
paddedReader := eestream.PadReader(ioutil.NopCloser(peekReader), encrypter.InBlockSize())
transformedReader = eestream.TransformReader(paddedReader, encrypter, 0)
} else {
data, err := ioutil.ReadAll(peekReader)
if err != nil {
return Meta{}, err
}
cipherData, err := cipher.Encrypt(data, &encKey, &nonce)
if err != nil {
return Meta{}, err
}
transformedReader = bytes.NewReader(cipherData)
}
putMeta, err = s.segments.Put(ctx, transformedReader, expiration, func() (paths.Path, []byte, error) {
if !eofReader.isEOF() {
segmentPath := getSegmentPath(path, currentSegment)
return segmentPath, encryptedEncKey, nil
}
lastSegmentPath := path.Prepend("l")
msi := pb.MetaStreamInfo{
NumberOfSegments: currentSegment + 1,
SegmentsSize: s.segmentSize,
LastSegmentSize: sizeReader.Size(),
Metadata: metadata,
EncryptionType: int32(s.encType),
EncryptionBlockSize: int32(s.encBlockSize),
LastSegmentEncryptionKey: encryptedEncKey,
}
lastSegmentMeta, err := proto.Marshal(&msi)
if err != nil {
return nil, nil, err
}
return lastSegmentPath, lastSegmentMeta, nil
})
if err != nil {
return Meta{}, err
}
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
currentSegment++
streamSize += sizeReader.Size()
}
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
if eofReader.hasError() {
return Meta{}, eofReader.err
}
resultMeta := Meta{
Modified: putMeta.Modified,
Expiration: expiration,
Size: streamSize,
Data: metadata,
}
return resultMeta, nil
}
// getSegmentPath returns the unique path for a particular segment
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
func getSegmentPath(p paths.Path, segNum int64) paths.Path {
return p.Prepend(fmt.Sprintf("s%d", segNum))
}
// Get returns a ranger that knows what the overall size is (from l/<path>)
// and then returns the appropriate data from segments s0/<path>, s1/<path>,
// ..., l/<path>.
func (s *streamStore) Get(ctx context.Context, path paths.Path) (rr ranger.Ranger, meta Meta, err error) {
defer mon.Task()(&ctx)(&err)
lastSegmentRanger, lastSegmentMeta, err := s.segments.Get(ctx, path.Prepend("l"))
if err != nil {
return nil, Meta{}, err
}
msi := pb.MetaStreamInfo{}
err = proto.Unmarshal(lastSegmentMeta.Data, &msi)
if err != nil {
return nil, Meta{}, err
}
streamMeta, err := convertMeta(lastSegmentMeta)
if err != nil {
return nil, Meta{}, err
}
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
derivedKey, err := path.DeriveContentKey(s.rootKey)
if err != nil {
return nil, Meta{}, err
}
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
var rangers []ranger.Ranger
for i := int64(0); i < msi.NumberOfSegments-1; i++ {
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
currentPath := getSegmentPath(path, i)
2018-09-12 14:06:44 +01:00
size := msi.SegmentsSize
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
var nonce eestream.Nonce
_, err := nonce.Increment(i)
if err != nil {
return nil, Meta{}, err
}
2018-09-12 14:06:44 +01:00
rr := &lazySegmentRanger{
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
segments: s.segments,
path: currentPath,
size: size,
derivedKey: (*eestream.Key)(derivedKey),
startingNonce: &nonce,
encBlockSize: int(msi.EncryptionBlockSize),
encType: eestream.Cipher(msi.EncryptionType),
2018-09-12 14:06:44 +01:00
}
rangers = append(rangers, rr)
}
var nonce eestream.Nonce
_, err = nonce.Increment(msi.NumberOfSegments - 1)
if err != nil {
return nil, Meta{}, err
}
decryptedLastSegmentRanger, err := decryptRanger(
ctx,
lastSegmentRanger,
msi.LastSegmentSize,
eestream.Cipher(msi.EncryptionType),
msi.LastSegmentEncryptionKey,
(*eestream.Key)(derivedKey),
&nonce,
int(msi.EncryptionBlockSize),
)
if err != nil {
return nil, Meta{}, err
}
rangers = append(rangers, decryptedLastSegmentRanger)
catRangers := ranger.Concat(rangers...)
return catRangers, streamMeta, nil
}
// Meta implements Store.Meta
func (s *streamStore) Meta(ctx context.Context, path paths.Path) (Meta, error) {
lastSegmentMeta, err := s.segments.Meta(ctx, path.Prepend("l"))
if err != nil {
return Meta{}, err
}
streamMeta, err := convertMeta(lastSegmentMeta)
if err != nil {
return Meta{}, err
}
return streamMeta, nil
}
// Delete all the segments, with the last one last
func (s *streamStore) Delete(ctx context.Context, path paths.Path) (err error) {
defer mon.Task()(&ctx)(&err)
lastSegmentMeta, err := s.segments.Meta(ctx, path.Prepend("l"))
if err != nil {
return err
}
msi := pb.MetaStreamInfo{}
err = proto.Unmarshal(lastSegmentMeta.Data, &msi)
if err != nil {
return err
}
for i := 0; i < int(msi.NumberOfSegments-1); i++ {
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
currentPath := getSegmentPath(path, int64(i))
err := s.segments.Delete(ctx, currentPath)
if err != nil {
return err
}
}
return s.segments.Delete(ctx, path.Prepend("l"))
}
// ListItem is a single item in a listing
type ListItem struct {
Path paths.Path
Meta Meta
IsPrefix bool
}
// List all the paths inside l/, stripping off the l/ prefix
func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore paths.Path, recursive bool, limit int, metaFlags uint32) (items []ListItem, more bool, err error) {
defer mon.Task()(&ctx)(&err)
if metaFlags&meta.Size != 0 {
// Calculating the stream's size require also the user-defined metadata,
// where stream store keeps info about the number of segments and their size.
metaFlags |= meta.UserDefined
}
segments, more, err := s.segments.List(ctx, prefix.Prepend("l"), startAfter, endBefore, recursive, limit, metaFlags)
if err != nil {
return nil, false, err
}
items = make([]ListItem, len(segments))
for i, item := range segments {
newMeta, err := convertMeta(item.Meta)
if err != nil {
return nil, false, err
}
2018-09-09 07:34:23 +01:00
items[i] = ListItem{Path: item.Path, Meta: newMeta, IsPrefix: item.IsPrefix}
}
return items, more, nil
}
2018-09-12 14:06:44 +01:00
type lazySegmentRanger struct {
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
ranger ranger.Ranger
segments segments.Store
path paths.Path
size int64
derivedKey *eestream.Key
startingNonce *eestream.Nonce
encBlockSize int
encType eestream.Cipher
2018-09-12 14:06:44 +01:00
}
// Size implements Ranger.Size
func (lr *lazySegmentRanger) Size() int64 {
return lr.size
}
// Range implements Ranger.Range to be lazily connected
func (lr *lazySegmentRanger) Range(ctx context.Context, offset, length int64) (io.ReadCloser, error) {
if lr.ranger == nil {
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
rr, m, err := lr.segments.Get(ctx, lr.path)
if err != nil {
return nil, err
}
lr.ranger, err = decryptRanger(ctx, rr, lr.size, lr.encType, m.Data, lr.derivedKey, lr.startingNonce, lr.encBlockSize)
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
if err != nil {
return nil, err
}
}
return lr.ranger.Range(ctx, offset, length)
}
// decryptRanger returns a decrypted ranger of the given rr ranger
func decryptRanger(ctx context.Context, rr ranger.Ranger, decryptedSize int64, cipher eestream.Cipher, encryptedEncKey []byte, derivedKey *eestream.Key, startingNonce *eestream.Nonce, encBlockSize int) (ranger.Ranger, error) {
e, err := cipher.Decrypt(encryptedEncKey, derivedKey, startingNonce)
if err != nil {
return nil, err
}
var encKey eestream.Key
copy(encKey[:], e)
decrypter, err := cipher.NewDecrypter(&encKey, startingNonce, encBlockSize)
if err != nil {
return nil, err
}
var rd ranger.Ranger
if rr.Size()%int64(decrypter.InBlockSize()) != 0 {
reader, err := rr.Range(ctx, 0, rr.Size())
2018-09-12 14:06:44 +01:00
if err != nil {
return nil, err
}
cipherData, err := ioutil.ReadAll(reader)
if err != nil {
return nil, err
Stream encryption (#302) * begin adding encryption for remote pieces * begin adding decryption * add encryption key as arg to Put and Get * move encryption/decryption to object store * Add encryption key to object store constructor * Add the erasure scheme to object store constructor * Ensure decrypter is initialized with the stripe size used by encrypter * Revert "Ensure decrypter is initialized with the stripe size used by encrypter" This reverts commit 07272333f461606edfb43ad106cc152f37a3bd46. * Revert "Add the erasure scheme to object store constructor" This reverts commit ea5e793b536159d993b96e3db69a37c1656a193c. * move encryption to stream store * move decryption stuff to stream store * revert changes in object store * add encryptedBlockSize and close rangers on error during Get * calculate padding sizes correctly * encryptedBlockSize -> encryptionBlockSize * pass encryption key and block size into stream store * remove encryption key and block size from object store constructor * move encrypter/decrypter initialization * remove unnecessary cast * Fix padding issue * Fix linter * add todos * use random encryption key for data encryption. Store an encrypted copy of this key in segment metadata * use different encryption key for each segment * encrypt data in one step if it is small enough * refactor and move encryption stuff * fix errors related to nil slices passed to copy * fix encrypter vs. decrypter bug * put encryption stuff in eestream * get captplanet test to pass * fix linting errors * add types for encryption keys/nonces and clean up * fix tests * more review changes * add Cipher type for encryption stuff * fix rs_test * Simplify type casting of key and nonce * Init starting nonce to the segment index * don't copy derived key * remove default encryption key; force user to explicitly set it * move getSegmentPath to streams package * dont require user to specify encryption key for captplanet * rename GenericKey and GenericNonce to Key and Nonce * review changes * fix linting error * Download uses the encryption type from metadata * Store enc block size in metadata and use it for download
2018-09-26 14:32:23 +01:00
}
data, err := cipher.Decrypt(cipherData, &encKey, startingNonce)
if err != nil {
return nil, err
}
return ranger.ByteRanger(data), nil
2018-09-12 14:06:44 +01:00
}
rd, err = eestream.Transform(rr, decrypter)
if err != nil {
return nil, err
}
return eestream.Unpad(rd, int(rd.Size()-decryptedSize))
2018-09-12 14:06:44 +01:00
}
// CancelHandler handles clean up of segments on receiving CTRL+C
func (s *streamStore) cancelHandler(ctx context.Context, totalSegments int64, path paths.Path) {
for i := int64(0); i < totalSegments; i++ {
currentPath := getSegmentPath(path, i)
err := s.segments.Delete(ctx, currentPath)
if err != nil {
zap.S().Warnf("Failed deleting a segment %v %v", currentPath, err)
}
}
}