JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

plumb EncryptionScheme, RedundancyScheme through to buckets (#1638)

Browse Source 
We want to use those fields in the bucket-level Pointer objects as
bucket defaults, but we need to be able to get at them first.

I don't see any strong reason not to make these available, except
that it was kind of a pain.
This commit is contained in:
paul cannon 2019-04-02 15:15:31 -06:00 committed by GitHub
parent 5134766d57
commit e4a70e3fac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 159 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
pkg/metainfo/kvmetainfo/buckets.go
View File

@ -106,8 +106,11 @@ func getPathCipher(info *storj.Bucket) storj.Cipher {
func bucketFromMeta(bucket string, meta buckets.Meta) storj.Bucket { func bucketFromMeta(bucket string, meta buckets.Meta) storj.Bucket {
return storj.Bucket{ return storj.Bucket{
Name: bucket, Name: bucket,
Created: meta.Created, Created: meta.Created,
PathCipher: meta.PathEncryptionType, PathCipher: meta.PathEncryptionType,
SegmentsSize: meta.SegmentsSize,
RedundancyScheme: meta.RedundancyScheme,
EncryptionScheme: meta.EncryptionScheme,
} }
} }

18
pkg/metainfo/kvmetainfo/objects.go
View File

@ -265,7 +265,7 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st
Data: pointer.GetMetadata(), Data: pointer.GetMetadata(),
} }
streamInfoData, err := streams.DecryptStreamInfo(ctx, lastSegmentMeta, fullpath, db.rootKey) streamInfoData, streamMeta, err := streams.DecryptStreamInfo(ctx, lastSegmentMeta.Data, fullpath, db.rootKey)
if err != nil { if err != nil {
return object{}, storj.Object{}, err return object{}, storj.Object{}, err
} }
@ -276,16 +276,11 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st
return object{}, storj.Object{}, err return object{}, storj.Object{}, err
} }
streamMeta := pb.StreamMeta{}
err = proto.Unmarshal(lastSegmentMeta.Data, &streamMeta)
if err != nil {
return object{}, storj.Object{}, err
}
info, err = objectStreamFromMeta(bucketInfo, path, lastSegmentMeta, streamInfo, streamMeta, redundancyScheme) info, err = objectStreamFromMeta(bucketInfo, path, lastSegmentMeta, streamInfo, streamMeta, redundancyScheme)
if err != nil { if err != nil {
return object{}, storj.Object{}, err return object{}, storj.Object{}, err
} }
lastSegmentMeta.RedundancyScheme = info.Stream.RedundancyScheme
return object{ return object{
fullpath: fullpath, fullpath: fullpath,
@ -347,14 +342,7 @@ func objectStreamFromMeta(bucket storj.Bucket, path storj.Path, lastSegment segm
SegmentCount: stream.NumberOfSegments, SegmentCount: stream.NumberOfSegments,
FixedSegmentSize: stream.SegmentsSize, FixedSegmentSize: stream.SegmentsSize,
RedundancyScheme: storj.RedundancyScheme{ RedundancyScheme: segments.RedundancySchemeFromProto(redundancyScheme),
Algorithm: storj.ReedSolomon,
ShareSize: redundancyScheme.GetErasureShareSize(),
RequiredShares: int16(redundancyScheme.GetMinReq()),
RepairShares: int16(redundancyScheme.GetRepairThreshold()),
OptimalShares: int16(redundancyScheme.GetSuccessThreshold()),
TotalShares: int16(redundancyScheme.GetTotal()),
},
EncryptionScheme: storj.EncryptionScheme{ EncryptionScheme: storj.EncryptionScheme{
Cipher: storj.Cipher(streamMeta.EncryptionType), Cipher: storj.Cipher(streamMeta.EncryptionType),
BlockSize: streamMeta.EncryptionBlockSize, BlockSize: streamMeta.EncryptionBlockSize,

6
pkg/storage/buckets/store.go
View File

@ -47,6 +47,9 @@ type BucketStore struct {
type Meta struct { type Meta struct {
Created time.Time Created time.Time
PathEncryptionType storj.Cipher PathEncryptionType storj.Cipher
SegmentsSize int64
RedundancyScheme storj.RedundancyScheme
EncryptionScheme storj.EncryptionScheme
} }
// NewStore instantiates BucketStore // NewStore instantiates BucketStore
@ -182,5 +185,8 @@ func convertMeta(m objects.Meta) (Meta, error) {
return Meta{ return Meta{
Created: m.Modified, Created: m.Modified,
PathEncryptionType: cipher, PathEncryptionType: cipher,
RedundancyScheme: m.RedundancyScheme,
EncryptionScheme: m.EncryptionScheme,
SegmentsSize: m.SegmentsSize,
}, nil }, nil
} }

14
pkg/storage/objects/store.go
View File

@ -24,10 +24,13 @@ var mon = monkit.Package()
// Meta is the full object metadata // Meta is the full object metadata
type Meta struct { type Meta struct {
pb.SerializableMeta pb.SerializableMeta
Modified time.Time Modified time.Time
Expiration time.Time Expiration time.Time
Size int64 Size int64
Checksum string Checksum string
SegmentsSize int64
RedundancyScheme storj.RedundancyScheme
EncryptionScheme storj.EncryptionScheme
} }
// ListItem is a single item in a listing // ListItem is a single item in a listing
@ -156,5 +159,8 @@ func convertMeta(m streams.Meta) Meta {
Expiration: m.Expiration, Expiration: m.Expiration,
Size: m.Size, Size: m.Size,
SerializableMeta: ser, SerializableMeta: ser,
SegmentsSize: m.SegmentsSize,
RedundancyScheme: m.RedundancyScheme,
EncryptionScheme: m.EncryptionScheme,
} }
} }

30
pkg/storage/segments/store.go
View File

@ -30,10 +30,11 @@ var (
// Meta info about a segment // Meta info about a segment
type Meta struct { type Meta struct {
Modified time.Time Modified time.Time
Expiration time.Time Expiration time.Time
Size int64 Size int64
Data []byte Data []byte
RedundancyScheme storj.RedundancyScheme
} }
// ListItem is a single item in a listing // ListItem is a single item in a listing
@ -354,13 +355,26 @@ func CalcNeededNodes(rs *pb.RedundancyScheme) int32 {
return needed return needed
} }
// RedundancySchemeFromProto translates a pb.RedundancyScheme to a storj.RedundancyScheme.
func RedundancySchemeFromProto(redundancyScheme *pb.RedundancyScheme) storj.RedundancyScheme {
return storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
ShareSize: redundancyScheme.GetErasureShareSize(),
RequiredShares: int16(redundancyScheme.GetMinReq()),
RepairShares: int16(redundancyScheme.GetRepairThreshold()),
OptimalShares: int16(redundancyScheme.GetSuccessThreshold()),
TotalShares: int16(redundancyScheme.GetTotal()),
}
}
// convertMeta converts pointer to segment metadata // convertMeta converts pointer to segment metadata
func convertMeta(pr *pb.Pointer) Meta { func convertMeta(pr *pb.Pointer) Meta {
return Meta{ return Meta{
Modified: convertTime(pr.GetCreationDate()), Modified: convertTime(pr.GetCreationDate()),
Expiration: convertTime(pr.GetExpirationDate()), Expiration: convertTime(pr.GetExpirationDate()),
Size: pr.GetSegmentSize(), Size: pr.GetSegmentSize(),
Data: pr.GetMetadata(), Data: pr.GetMetadata(),
RedundancyScheme: RedundancySchemeFromProto(pr.GetRemote().GetRedundancy()),
} }
} }

110
pkg/storage/streams/store.go
View File

@ -30,28 +30,31 @@ import (
var mon = monkit.Package() var mon = monkit.Package()
// Meta info about a segment // Meta info about a stream
type Meta struct { type Meta struct {
Modified time.Time Modified time.Time
Expiration time.Time Expiration time.Time
Size int64 Size int64
Data []byte Data []byte
SegmentsSize int64
EncryptionScheme storj.EncryptionScheme
RedundancyScheme storj.RedundancyScheme
} }
// convertMeta converts segment metadata to stream metadata // convertMeta converts segment metadata to stream metadata
func convertMeta(lastSegmentMeta segments.Meta) (Meta, error) { func convertMeta(lastSegmentMeta segments.Meta, stream pb.StreamInfo, streamMeta pb.StreamMeta) Meta {
stream := pb.StreamInfo{}
err := proto.Unmarshal(lastSegmentMeta.Data, &stream)
if err != nil {
return Meta{}, err
}
return Meta{ return Meta{
Modified: lastSegmentMeta.Modified, Modified: lastSegmentMeta.Modified,
Expiration: lastSegmentMeta.Expiration, Expiration: lastSegmentMeta.Expiration,
Size: ((stream.NumberOfSegments - 1) * stream.SegmentsSize) + stream.LastSegmentSize, Size: ((stream.NumberOfSegments - 1) * stream.SegmentsSize) + stream.LastSegmentSize,
Data: stream.Metadata, Data: stream.Metadata,
}, nil SegmentsSize: stream.SegmentsSize,
RedundancyScheme: lastSegmentMeta.RedundancyScheme,
EncryptionScheme: storj.EncryptionScheme{
Cipher: storj.Cipher(streamMeta.EncryptionType),
BlockSize: streamMeta.EncryptionBlockSize,
},
}
} }
// Store interface methods for streams to satisfy to be a store // Store interface methods for streams to satisfy to be a store
@ -269,10 +272,15 @@ func (s *streamStore) upload(ctx context.Context, path storj.Path, pathCipher st
} }
resultMeta := Meta{ resultMeta := Meta{
Modified: putMeta.Modified, Modified: putMeta.Modified,
Expiration: expiration, Expiration: expiration,
Size: streamSize, Size: streamSize,
Data: metadata, Data: metadata,
SegmentsSize: s.segmentSize,
EncryptionScheme: storj.EncryptionScheme{
Cipher: s.cipher,
BlockSize: int32(s.encBlockSize),
},
} }
return resultMeta, currentSegment, nil return resultMeta, currentSegment, nil
@ -299,7 +307,7 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj
return nil, Meta{}, err return nil, Meta{}, err
} }
streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey) streamInfo, streamMeta, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey)
if err != nil { if err != nil {
return nil, Meta{}, err return nil, Meta{}, err
} }
@ -310,12 +318,6 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj
return nil, Meta{}, err return nil, Meta{}, err
} }
streamMeta := pb.StreamMeta{}
err = proto.Unmarshal(lastSegmentMeta.Data, &streamMeta)
if err != nil {
return nil, Meta{}, err
}
derivedKey, err := encryption.DeriveContentKey(path, s.rootKey) derivedKey, err := encryption.DeriveContentKey(path, s.rootKey)
if err != nil { if err != nil {
return nil, Meta{}, err return nil, Meta{}, err
@ -362,16 +364,10 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj
if err != nil { if err != nil {
return nil, Meta{}, err return nil, Meta{}, err
} }
rangers = append(rangers, decryptedLastSegmentRanger) rangers = append(rangers, decryptedLastSegmentRanger)
catRangers := ranger.Concat(rangers...) catRangers := ranger.Concat(rangers...)
meta = convertMeta(lastSegmentMeta, stream, streamMeta)
lastSegmentMeta.Data = streamInfo
meta, err = convertMeta(lastSegmentMeta)
if err != nil {
return nil, Meta{}, err
}
return catRangers, meta, nil return catRangers, meta, nil
} }
@ -389,18 +385,16 @@ func (s *streamStore) Meta(ctx context.Context, path storj.Path, pathCipher stor
return Meta{}, err return Meta{}, err
} }
streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey) streamInfo, streamMeta, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey)
if err != nil { if err != nil {
return Meta{}, err return Meta{}, err
} }
var stream pb.StreamInfo
lastSegmentMeta.Data = streamInfo if err := proto.Unmarshal(streamInfo, &stream); err != nil {
newStreamMeta, err := convertMeta(lastSegmentMeta)
if err != nil {
return Meta{}, err return Meta{}, err
} }
return newStreamMeta, nil return convertMeta(lastSegmentMeta, stream, streamMeta), nil
} }
// Delete all the segments, with the last one last // Delete all the segments, with the last one last
@ -416,14 +410,12 @@ func (s *streamStore) Delete(ctx context.Context, path storj.Path, pathCipher st
return err return err
} }
streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey) streamInfo, _, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey)
if err != nil { if err != nil {
return err return err
} }
var stream pb.StreamInfo
stream := pb.StreamInfo{} if err := proto.Unmarshal(streamInfo, &stream); err != nil {
err = proto.Unmarshal(streamInfo, &stream)
if err != nil {
return err return err
} }
@ -493,17 +485,16 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore st
return nil, false, err return nil, false, err
} }
streamInfo, err := DecryptStreamInfo(ctx, item.Meta, storj.JoinPaths(prefix, path), s.rootKey) streamInfo, streamMeta, err := DecryptStreamInfo(ctx, item.Meta.Data, storj.JoinPaths(prefix, path), s.rootKey)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
var stream pb.StreamInfo
item.Meta.Data = streamInfo if err := proto.Unmarshal(streamInfo, &stream); err != nil {
newMeta, err := convertMeta(item.Meta)
if err != nil {
return nil, false, err return nil, false, err
} }
newMeta := convertMeta(item.Meta, stream, streamMeta)
items[i] = ListItem{Path: path, Meta: newMeta, IsPrefix: item.IsPrefix} items[i] = ListItem{Path: path, Meta: newMeta, IsPrefix: item.IsPrefix}
} }
@ -666,25 +657,26 @@ func getEncryptedKeyAndNonce(m *pb.SegmentMeta) (storj.EncryptedPrivateKey, *sto
} }
// DecryptStreamInfo decrypts stream info // DecryptStreamInfo decrypts stream info
func DecryptStreamInfo(ctx context.Context, item segments.Meta, path storj.Path, rootKey *storj.Key) (streamInfo []byte, err error) { func DecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path storj.Path, rootKey *storj.Key) (
streamMeta := pb.StreamMeta{} streamInfo []byte, streamMeta pb.StreamMeta, err error) {
err = proto.Unmarshal(item.Data, &streamMeta) err = proto.Unmarshal(streamMetaBytes, &streamMeta)
if err != nil { if err != nil {
return nil, err return nil, pb.StreamMeta{}, err
} }
derivedKey, err := encryption.DeriveContentKey(path, rootKey) derivedKey, err := encryption.DeriveContentKey(path, rootKey)
if err != nil { if err != nil {
return nil, err return nil, pb.StreamMeta{}, err
} }
cipher := storj.Cipher(streamMeta.EncryptionType) cipher := storj.Cipher(streamMeta.EncryptionType)
encryptedKey, keyNonce := getEncryptedKeyAndNonce(streamMeta.LastSegmentMeta) encryptedKey, keyNonce := getEncryptedKeyAndNonce(streamMeta.LastSegmentMeta)
contentKey, err := encryption.DecryptKey(encryptedKey, cipher, derivedKey, keyNonce) contentKey, err := encryption.DecryptKey(encryptedKey, cipher, derivedKey, keyNonce)
if err != nil { if err != nil {
return nil, err return nil, pb.StreamMeta{}, err
} }
// decrypt metadata with the content encryption key and zero nonce // decrypt metadata with the content encryption key and zero nonce
return encryption.Decrypt(streamMeta.EncryptedStreamInfo, cipher, contentKey, &storj.Nonce{}) streamInfo, err = encryption.Decrypt(streamMeta.EncryptedStreamInfo, cipher, contentKey, &storj.Nonce{})
return streamInfo, streamMeta, err
} }

81
pkg/storage/streams/store_test.go
View File

@ -31,12 +31,13 @@ func TestStreamStoreMeta(t *testing.T) {
mockSegmentStore := segments.NewMockStore(ctrl) mockSegmentStore := segments.NewMockStore(ctrl)
stream, err := proto.Marshal(&pb.StreamInfo{ streamInfo := pb.StreamInfo{
NumberOfSegments: 2, NumberOfSegments: 2,
SegmentsSize: 10, SegmentsSize: 10,
LastSegmentSize: 0, LastSegmentSize: 0,
Metadata: []byte{}, Metadata: nil,
}) }
stream, err := proto.Marshal(&streamInfo)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -53,6 +54,9 @@ func TestStreamStoreMeta(t *testing.T) {
Expiration: staticTime, Expiration: staticTime,
Size: 50, Size: 50,
Data: lastSegmentMetadata, Data: lastSegmentMetadata,
RedundancyScheme: storj.RedundancyScheme{
Algorithm: storj.ReedSolomon,
},
} }
streamMetaUnmarshaled := pb.StreamMeta{} streamMetaUnmarshaled := pb.StreamMeta{}
@ -62,16 +66,14 @@ func TestStreamStoreMeta(t *testing.T) {
} }
segmentMetaStreamInfo := segments.Meta{ segmentMetaStreamInfo := segments.Meta{
Modified: staticTime, Modified: staticTime,
Expiration: staticTime, Expiration: staticTime,
Size: 50, Size: 50,
Data: streamMetaUnmarshaled.EncryptedStreamInfo, Data: streamMetaUnmarshaled.EncryptedStreamInfo,
RedundancyScheme: segmentMeta.RedundancyScheme,
} }
streamMeta, err := convertMeta(segmentMetaStreamInfo) streamMeta := convertMeta(segmentMetaStreamInfo, streamInfo, streamMetaUnmarshaled)
if err != nil {
t.Fatal(err)
}
for i, test := range []struct { for i, test := range []struct {
// input for test function // input for test function
@ -102,6 +104,7 @@ func TestStreamStoreMeta(t *testing.T) {
} }
assert.Equal(t, test.streamMeta, meta, errTag) assert.Equal(t, test.streamMeta, meta, errTag)
assert.Equal(t, storj.ReedSolomon, test.streamMeta.RedundancyScheme.Algorithm)
} }
} }
@ -111,19 +114,31 @@ func TestStreamStorePut(t *testing.T) {
mockSegmentStore := segments.NewMockStore(ctrl) mockSegmentStore := segments.NewMockStore(ctrl)
const (
encBlockSize = 10
segSize = 10
pathCipher = storj.AESGCM
dataCipher = storj.Unencrypted
)
staticTime := time.Now() staticTime := time.Now()
segmentMeta := segments.Meta{ segmentMeta := segments.Meta{
Modified: staticTime, Modified: staticTime,
Expiration: staticTime, Expiration: staticTime,
Size: 10, Size: segSize,
Data: []byte{}, Data: []byte{},
} }
streamMeta := Meta{ streamMeta := Meta{
Modified: segmentMeta.Modified, Modified: segmentMeta.Modified,
Expiration: segmentMeta.Expiration, Expiration: segmentMeta.Expiration,
Size: 4, Size: 4,
Data: []byte("metadata"), Data: []byte("metadata"),
SegmentsSize: segSize,
EncryptionScheme: storj.EncryptionScheme{
Cipher: dataCipher,
BlockSize: encBlockSize,
},
} }
for i, test := range []struct { for i, test := range []struct {
@ -163,12 +178,12 @@ func TestStreamStorePut(t *testing.T) {
Delete(gomock.Any(), gomock.Any()). Delete(gomock.Any(), gomock.Any()).
Return(test.segmentError) Return(test.segmentError)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) streamStore, err := NewStreamStore(mockSegmentStore, segSize, new(storj.Key), encBlockSize, dataCipher)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
meta, err := streamStore.Put(ctx, test.path, storj.AESGCM, test.data, test.metadata, test.expiration) meta, err := streamStore.Put(ctx, test.path, pathCipher, test.data, test.metadata, test.expiration)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -198,6 +213,13 @@ func TestStreamStoreGet(t *testing.T) {
ctrl := gomock.NewController(t) ctrl := gomock.NewController(t)
defer ctrl.Finish() defer ctrl.Finish()
const (
segSize = 10
encBlockSize = 10
dataCipher = storj.Unencrypted
pathCipher = storj.AESGCM
)
mockSegmentStore := segments.NewMockStore(ctrl) mockSegmentStore := segments.NewMockStore(ctrl)
staticTime := time.Now() staticTime := time.Now()
@ -209,7 +231,7 @@ func TestStreamStoreGet(t *testing.T) {
stream, err := proto.Marshal(&pb.StreamInfo{ stream, err := proto.Marshal(&pb.StreamInfo{
NumberOfSegments: 1, NumberOfSegments: 1,
SegmentsSize: 10, SegmentsSize: segSize,
LastSegmentSize: 0, LastSegmentSize: 0,
}) })
if err != nil { if err != nil {
@ -218,6 +240,8 @@ func TestStreamStoreGet(t *testing.T) {
lastSegmentMeta, err := proto.Marshal(&pb.StreamMeta{ lastSegmentMeta, err := proto.Marshal(&pb.StreamMeta{
EncryptedStreamInfo: stream, EncryptedStreamInfo: stream,
EncryptionType: int32(dataCipher),
EncryptionBlockSize: encBlockSize,
}) })
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -226,17 +250,22 @@ func TestStreamStoreGet(t *testing.T) {
segmentMeta := segments.Meta{ segmentMeta := segments.Meta{
Modified: staticTime, Modified: staticTime,
Expiration: staticTime, Expiration: staticTime,
Size: 10, Size: segSize,
Data: lastSegmentMeta, Data: lastSegmentMeta,
} }
streamRanger := ranger.ByteRanger(nil) streamRanger := ranger.ByteRanger(nil)
streamMeta := Meta{ streamMeta := Meta{
Modified: staticTime, Modified: staticTime,
Expiration: staticTime, Expiration: staticTime,
Size: 0, Size: 0,
Data: nil, Data: nil,
SegmentsSize: segSize,
EncryptionScheme: storj.EncryptionScheme{
Cipher: dataCipher,
BlockSize: encBlockSize,
},
} }
for i, test := range []struct { for i, test := range []struct {
@ -263,12 +292,12 @@ func TestStreamStoreGet(t *testing.T) {
gomock.InOrder(calls...) gomock.InOrder(calls...)
streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) streamStore, err := NewStreamStore(mockSegmentStore, segSize, new(storj.Key), encBlockSize, dataCipher)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
ranger, meta, err := streamStore.Get(ctx, test.path, storj.AESGCM) ranger, meta, err := streamStore.Get(ctx, test.path, pathCipher)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

9
pkg/storj/object.go
View File

@ -25,9 +25,12 @@ var (
// Bucket contains information about a specific bucket // Bucket contains information about a specific bucket
type Bucket struct { type Bucket struct {
Name string Name string
Created time.Time Created time.Time
PathCipher Cipher PathCipher Cipher
SegmentsSize int64
RedundancyScheme RedundancyScheme
EncryptionScheme EncryptionScheme
} }
// Object contains information about a specific object // Object contains information about a specific object