JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

plumb EncryptionScheme, RedundancyScheme through to buckets (#1638)

Browse Source 
We want to use those fields in the bucket-level Pointer objects as
bucket defaults, but we need to be able to get at them first.

I don't see any strong reason not to make these available, except
that it was kind of a pain.
This commit is contained in:
paul cannon 2019-04-02 15:15:31 -06:00 committed by GitHub
parent 5134766d57
commit e4a70e3fac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 159 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/metainfo/kvmetainfo/buckets.go
View File

@ -106,8 +106,11 @@ func getPathCipher(info *storj.Bucket) storj.Cipher {
func bucketFromMeta(bucket string, meta buckets.Meta) storj.Bucket {
return storj.Bucket{
Name: bucket,
Created: meta.Created,
PathCipher: meta.PathEncryptionType,
Name: bucket,
Created: meta.Created,
PathCipher: meta.PathEncryptionType,
SegmentsSize: meta.SegmentsSize,
RedundancyScheme: meta.RedundancyScheme,
EncryptionScheme: meta.EncryptionScheme,
}
}

18
pkg/metainfo/kvmetainfo/objects.go
View File

@ -265,7 +265,7 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st
Data: pointer.GetMetadata(),
}
streamInfoData, err := streams.DecryptStreamInfo(ctx, lastSegmentMeta, fullpath, db.rootKey)
streamInfoData, streamMeta, err := streams.DecryptStreamInfo(ctx, lastSegmentMeta.Data, fullpath, db.rootKey)
if err != nil {
return object{}, storj.Object{}, err
}
@ -276,16 +276,11 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st
return object{}, storj.Object{}, err
}
streamMeta := pb.StreamMeta{}
err = proto.Unmarshal(lastSegmentMeta.Data, &streamMeta)
if err != nil {
return object{}, storj.Object{}, err
}
info, err = objectStreamFromMeta(bucketInfo, path, lastSegmentMeta, streamInfo, streamMeta, redundancyScheme)
if err != nil {
return object{}, storj.Object{}, err
}
lastSegmentMeta.RedundancyScheme = info.Stream.RedundancyScheme
return object{
fullpath: fullpath,
@ -347,14 +342,7 @@ func objectStreamFromMeta(bucket storj.Bucket, path storj.Path, lastSegment segm
SegmentCount: stream.NumberOfSegments,
FixedSegmentSize: stream.SegmentsSize,
RedundancyScheme: storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
ShareSize: redundancyScheme.GetErasureShareSize(),
RequiredShares: int16(redundancyScheme.GetMinReq()),
RepairShares: int16(redundancyScheme.GetRepairThreshold()),
OptimalShares: int16(redundancyScheme.GetSuccessThreshold()),
TotalShares: int16(redundancyScheme.GetTotal()),
},
RedundancyScheme: segments.RedundancySchemeFromProto(redundancyScheme),
EncryptionScheme: storj.EncryptionScheme{
Cipher: storj.Cipher(streamMeta.EncryptionType),
BlockSize: streamMeta.EncryptionBlockSize,

6
pkg/storage/buckets/store.go
View File

@ -47,6 +47,9 @@ type BucketStore struct {
type Meta struct {
Created time.Time
PathEncryptionType storj.Cipher
SegmentsSize int64
RedundancyScheme storj.RedundancyScheme
EncryptionScheme storj.EncryptionScheme
}
// NewStore instantiates BucketStore
@ -182,5 +185,8 @@ func convertMeta(m objects.Meta) (Meta, error) {
return Meta{
Created: m.Modified,
PathEncryptionType: cipher,
RedundancyScheme: m.RedundancyScheme,
EncryptionScheme: m.EncryptionScheme,
SegmentsSize: m.SegmentsSize,
}, nil
}

14
pkg/storage/objects/store.go
View File

@ -24,10 +24,13 @@ var mon = monkit.Package()
// Meta is the full object metadata
type Meta struct {
pb.SerializableMeta
Modified time.Time
Expiration time.Time
Size int64
Checksum string
Modified time.Time
Expiration time.Time
Size int64
Checksum string
SegmentsSize int64
RedundancyScheme storj.RedundancyScheme
EncryptionScheme storj.EncryptionScheme
}
// ListItem is a single item in a listing
@ -156,5 +159,8 @@ func convertMeta(m streams.Meta) Meta {
Expiration: m.Expiration,
Size: m.Size,
SerializableMeta: ser,
SegmentsSize: m.SegmentsSize,
RedundancyScheme: m.RedundancyScheme,
EncryptionScheme: m.EncryptionScheme,
}
}

30
pkg/storage/segments/store.go
View File

@ -30,10 +30,11 @@ var (
// Meta info about a segment
type Meta struct {
Modified time.Time
Expiration time.Time
Size int64
Data []byte
Modified time.Time
Expiration time.Time
Size int64
Data []byte
RedundancyScheme storj.RedundancyScheme
}
// ListItem is a single item in a listing
@ -354,13 +355,26 @@ func CalcNeededNodes(rs *pb.RedundancyScheme) int32 {
return needed
}
// RedundancySchemeFromProto translates a pb.RedundancyScheme to a storj.RedundancyScheme.
func RedundancySchemeFromProto(redundancyScheme *pb.RedundancyScheme) storj.RedundancyScheme {
return storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
ShareSize: redundancyScheme.GetErasureShareSize(),
RequiredShares: int16(redundancyScheme.GetMinReq()),
RepairShares: int16(redundancyScheme.GetRepairThreshold()),
OptimalShares: int16(redundancyScheme.GetSuccessThreshold()),
TotalShares: int16(redundancyScheme.GetTotal()),
}
}
// convertMeta converts pointer to segment metadata
func convertMeta(pr *pb.Pointer) Meta {
return Meta{
Modified: convertTime(pr.GetCreationDate()),
Expiration: convertTime(pr.GetExpirationDate()),
Size: pr.GetSegmentSize(),
Data: pr.GetMetadata(),
Modified: convertTime(pr.GetCreationDate()),
Expiration: convertTime(pr.GetExpirationDate()),
Size: pr.GetSegmentSize(),
Data: pr.GetMetadata(),
RedundancyScheme: RedundancySchemeFromProto(pr.GetRemote().GetRedundancy()),
}
}

110
pkg/storage/streams/store.go
View File

@ -30,28 +30,31 @@ import (
var mon = monkit.Package()
// Meta info about a segment
// Meta info about a stream
type Meta struct {
Modified time.Time
Expiration time.Time
Size int64
Data []byte
Modified time.Time
Expiration time.Time
Size int64
Data []byte
SegmentsSize int64
EncryptionScheme storj.EncryptionScheme
RedundancyScheme storj.RedundancyScheme
}
// convertMeta converts segment metadata to stream metadata
func convertMeta(lastSegmentMeta segments.Meta) (Meta, error) {
stream := pb.StreamInfo{}
err := proto.Unmarshal(lastSegmentMeta.Data, &stream)
if err != nil {
return Meta{}, err
}
func convertMeta(lastSegmentMeta segments.Meta, stream pb.StreamInfo, streamMeta pb.StreamMeta) Meta {
return Meta{
Modified: lastSegmentMeta.Modified,
Expiration: lastSegmentMeta.Expiration,
Size: ((stream.NumberOfSegments - 1) * stream.SegmentsSize) + stream.LastSegmentSize,
Data: stream.Metadata,
}, nil
Modified: lastSegmentMeta.Modified,
Expiration: lastSegmentMeta.Expiration,
Size: ((stream.NumberOfSegments - 1) * stream.SegmentsSize) + stream.LastSegmentSize,
Data: stream.Metadata,
SegmentsSize: stream.SegmentsSize,
RedundancyScheme: lastSegmentMeta.RedundancyScheme,
EncryptionScheme: storj.EncryptionScheme{
Cipher: storj.Cipher(streamMeta.EncryptionType),
BlockSize: streamMeta.EncryptionBlockSize,
},
}
}
// Store interface methods for streams to satisfy to be a store
@ -269,10 +272,15 @@ func (s *streamStore) upload(ctx context.Context, path storj.Path, pathCipher st
}
resultMeta := Meta{
Modified: putMeta.Modified,
Expiration: expiration,
Size: streamSize,
Data: metadata,
Modified: putMeta.Modified,
Expiration: expiration,
Size: streamSize,
Data: metadata,
SegmentsSize: s.segmentSize,
EncryptionScheme: storj.EncryptionScheme{
Cipher: s.cipher,
BlockSize: int32(s.encBlockSize),
},
}
return resultMeta, currentSegment, nil
@ -299,7 +307,7 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj
return nil, Meta{}, err
}
streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey)
streamInfo, streamMeta, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey)
if err != nil {
return nil, Meta{}, err
}
@ -310,12 +318,6 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj
return nil, Meta{}, err
}
streamMeta := pb.StreamMeta{}
err = proto.Unmarshal(lastSegmentMeta.Data, &streamMeta)
if err != nil {
return nil, Meta{}, err
}
derivedKey, err := encryption.DeriveContentKey(path, s.rootKey)
if err != nil {
return nil, Meta{}, err
@ -362,16 +364,10 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj
if err != nil {
return nil, Meta{}, err
}
rangers = append(rangers, decryptedLastSegmentRanger)
catRangers := ranger.Concat(rangers...)
lastSegmentMeta.Data = streamInfo
meta, err = convertMeta(lastSegmentMeta)
if err != nil {
return nil, Meta{}, err
}
meta = convertMeta(lastSegmentMeta, stream, streamMeta)
return catRangers, meta, nil
}
@ -389,18 +385,16 @@ func (s *streamStore) Meta(ctx context.Context, path storj.Path, pathCipher stor
return Meta{}, err
}
streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey)
streamInfo, streamMeta, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey)
if err != nil {
return Meta{}, err
}
lastSegmentMeta.Data = streamInfo
newStreamMeta, err := convertMeta(lastSegmentMeta)
if err != nil {
var stream pb.StreamInfo
if err := proto.Unmarshal(streamInfo, &stream); err != nil {
return Meta{}, err
}
return newStreamMeta, nil
return convertMeta(lastSegmentMeta, stream, streamMeta), nil
}
// Delete all the segments, with the last one last
@ -416,14 +410,12 @@ func (s *streamStore) Delete(ctx context.Context, path storj.Path, pathCipher st
return err
}
streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey)
streamInfo, _, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey)
if err != nil {
return err
}
stream := pb.StreamInfo{}
err = proto.Unmarshal(streamInfo, &stream)
if err != nil {
var stream pb.StreamInfo
if err := proto.Unmarshal(streamInfo, &stream); err != nil {
return err
}
@ -493,17 +485,16 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore st
return nil, false, err
}
streamInfo, err := DecryptStreamInfo(ctx, item.Meta, storj.JoinPaths(prefix, path), s.rootKey)
streamInfo, streamMeta, err := DecryptStreamInfo(ctx, item.Meta.Data, storj.JoinPaths(prefix, path), s.rootKey)
if err != nil {
return nil, false, err
}
item.Meta.Data = streamInfo
newMeta, err := convertMeta(item.Meta)
if err != nil {
var stream pb.StreamInfo
if err := proto.Unmarshal(streamInfo, &stream); err != nil {
return nil, false, err
}
newMeta := convertMeta(item.Meta, stream, streamMeta)
items[i] = ListItem{Path: path, Meta: newMeta, IsPrefix: item.IsPrefix}
}
@ -666,25 +657,26 @@ func getEncryptedKeyAndNonce(m *pb.SegmentMeta) (storj.EncryptedPrivateKey, *sto
}
// DecryptStreamInfo decrypts stream info
func DecryptStreamInfo(ctx context.Context, item segments.Meta, path storj.Path, rootKey *storj.Key) (streamInfo []byte, err error) {
streamMeta := pb.StreamMeta{}
err = proto.Unmarshal(item.Data, &streamMeta)
func DecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path storj.Path, rootKey *storj.Key) (
streamInfo []byte, streamMeta pb.StreamMeta, err error) {
err = proto.Unmarshal(streamMetaBytes, &streamMeta)
if err != nil {
return nil, err
return nil, pb.StreamMeta{}, err
}
derivedKey, err := encryption.DeriveContentKey(path, rootKey)
if err != nil {
return nil, err
return nil, pb.StreamMeta{}, err
}
cipher := storj.Cipher(streamMeta.EncryptionType)
encryptedKey, keyNonce := getEncryptedKeyAndNonce(streamMeta.LastSegmentMeta)
contentKey, err := encryption.DecryptKey(encryptedKey, cipher, derivedKey, keyNonce)
if err != nil {
return nil, err
return nil, pb.StreamMeta{}, err
}
// decrypt metadata with the content encryption key and zero nonce
return encryption.Decrypt(streamMeta.EncryptedStreamInfo, cipher, contentKey, &storj.Nonce{})
streamInfo, err = encryption.Decrypt(streamMeta.EncryptedStreamInfo, cipher, contentKey, &storj.Nonce{})
return streamInfo, streamMeta, err
}

81
pkg/storage/streams/store_test.go
View File

@ -31,12 +31,13 @@ func TestStreamStoreMeta(t *testing.T) {
mockSegmentStore := segments.NewMockStore(ctrl)
stream, err := proto.Marshal(&pb.StreamInfo{
streamInfo := pb.StreamInfo{
NumberOfSegments: 2,
SegmentsSize: 10,
LastSegmentSize: 0,
Metadata: []byte{},
})
Metadata: nil,
}
stream, err := proto.Marshal(&streamInfo)
if err != nil {
t.Fatal(err)
}
@ -53,6 +54,9 @@ func TestStreamStoreMeta(t *testing.T) {
Expiration: staticTime,
Size: 50,
Data: lastSegmentMetadata,
RedundancyScheme: storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
},
}
streamMetaUnmarshaled := pb.StreamMeta{}
@ -62,16 +66,14 @@ func TestStreamStoreMeta(t *testing.T) {
}
segmentMetaStreamInfo := segments.Meta{
Modified: staticTime,
Expiration: staticTime,
Size: 50,
Data: streamMetaUnmarshaled.EncryptedStreamInfo,
Modified: staticTime,
Expiration: staticTime,
Size: 50,
Data: streamMetaUnmarshaled.EncryptedStreamInfo,
RedundancyScheme: segmentMeta.RedundancyScheme,
}
streamMeta, err := convertMeta(segmentMetaStreamInfo)
if err != nil {
t.Fatal(err)
}
streamMeta := convertMeta(segmentMetaStreamInfo, streamInfo, streamMetaUnmarshaled)
for i, test := range []struct {
// input for test function
@ -102,6 +104,7 @@ func TestStreamStoreMeta(t *testing.T) {
}
assert.Equal(t, test.streamMeta, meta, errTag)
assert.Equal(t, storj.ReedSolomon, test.streamMeta.RedundancyScheme.Algorithm)
}
}
@ -111,19 +114,31 @@ func TestStreamStorePut(t *testing.T) {
mockSegmentStore := segments.NewMockStore(ctrl)
const (
encBlockSize = 10
segSize = 10
pathCipher = storj.AESGCM
dataCipher = storj.Unencrypted
)
staticTime := time.Now()
segmentMeta := segments.Meta{
Modified: staticTime,
Expiration: staticTime,
Size: 10,
Size: segSize,
Data: []byte{},
}
streamMeta := Meta{
Modified: segmentMeta.Modified,
Expiration: segmentMeta.Expiration,
Size: 4,
Data: []byte("metadata"),
Modified: segmentMeta.Modified,
Expiration: segmentMeta.Expiration,
Size: 4,
Data: []byte("metadata"),
SegmentsSize: segSize,
EncryptionScheme: storj.EncryptionScheme{
Cipher: dataCipher,
BlockSize: encBlockSize,
},
}
for i, test := range []struct {
@ -163,12 +178,12 @@ func TestStreamStorePut(t *testing.T) {
Delete(gomock.Any(), gomock.Any()).
Return(test.segmentError)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0)
streamStore, err := NewStreamStore(mockSegmentStore, segSize, new(storj.Key), encBlockSize, dataCipher)
if err != nil {
t.Fatal(err)
}
meta, err := streamStore.Put(ctx, test.path, storj.AESGCM, test.data, test.metadata, test.expiration)
meta, err := streamStore.Put(ctx, test.path, pathCipher, test.data, test.metadata, test.expiration)
if err != nil {
t.Fatal(err)
}
@ -198,6 +213,13 @@ func TestStreamStoreGet(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
const (
segSize = 10
encBlockSize = 10
dataCipher = storj.Unencrypted
pathCipher = storj.AESGCM
)
mockSegmentStore := segments.NewMockStore(ctrl)
staticTime := time.Now()
@ -209,7 +231,7 @@ func TestStreamStoreGet(t *testing.T) {
stream, err := proto.Marshal(&pb.StreamInfo{
NumberOfSegments: 1,
SegmentsSize: 10,
SegmentsSize: segSize,
LastSegmentSize: 0,
})
if err != nil {
@ -218,6 +240,8 @@ func TestStreamStoreGet(t *testing.T) {
lastSegmentMeta, err := proto.Marshal(&pb.StreamMeta{
EncryptedStreamInfo: stream,
EncryptionType: int32(dataCipher),
EncryptionBlockSize: encBlockSize,
})
if err != nil {
t.Fatal(err)
@ -226,17 +250,22 @@ func TestStreamStoreGet(t *testing.T) {
segmentMeta := segments.Meta{
Modified: staticTime,
Expiration: staticTime,
Size: 10,
Size: segSize,
Data: lastSegmentMeta,
}
streamRanger := ranger.ByteRanger(nil)
streamMeta := Meta{
Modified: staticTime,
Expiration: staticTime,
Size: 0,
Data: nil,
Modified: staticTime,
Expiration: staticTime,
Size: 0,
Data: nil,
SegmentsSize: segSize,
EncryptionScheme: storj.EncryptionScheme{
Cipher: dataCipher,
BlockSize: encBlockSize,
},
}
for i, test := range []struct {
@ -263,12 +292,12 @@ func TestStreamStoreGet(t *testing.T) {
gomock.InOrder(calls...)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0)
streamStore, err := NewStreamStore(mockSegmentStore, segSize, new(storj.Key), encBlockSize, dataCipher)
if err != nil {
t.Fatal(err)
}
ranger, meta, err := streamStore.Get(ctx, test.path, storj.AESGCM)
ranger, meta, err := streamStore.Get(ctx, test.path, pathCipher)
if err != nil {
t.Fatal(err)
}

9
pkg/storj/object.go
View File

@ -25,9 +25,12 @@ var (
// Bucket contains information about a specific bucket
type Bucket struct {
Name string
Created time.Time
PathCipher Cipher
Name string
Created time.Time
PathCipher Cipher
SegmentsSize int64
RedundancyScheme RedundancyScheme
EncryptionScheme EncryptionScheme
}
// Object contains information about a specific object