diff --git a/pkg/metainfo/kvmetainfo/buckets.go b/pkg/metainfo/kvmetainfo/buckets.go index b9a870a15..11409b365 100644 --- a/pkg/metainfo/kvmetainfo/buckets.go +++ b/pkg/metainfo/kvmetainfo/buckets.go @@ -106,8 +106,11 @@ func getPathCipher(info *storj.Bucket) storj.Cipher { func bucketFromMeta(bucket string, meta buckets.Meta) storj.Bucket { return storj.Bucket{ - Name: bucket, - Created: meta.Created, - PathCipher: meta.PathEncryptionType, + Name: bucket, + Created: meta.Created, + PathCipher: meta.PathEncryptionType, + SegmentsSize: meta.SegmentsSize, + RedundancyScheme: meta.RedundancyScheme, + EncryptionScheme: meta.EncryptionScheme, } } diff --git a/pkg/metainfo/kvmetainfo/objects.go b/pkg/metainfo/kvmetainfo/objects.go index bc3ea335f..bda545ad4 100644 --- a/pkg/metainfo/kvmetainfo/objects.go +++ b/pkg/metainfo/kvmetainfo/objects.go @@ -265,7 +265,7 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st Data: pointer.GetMetadata(), } - streamInfoData, err := streams.DecryptStreamInfo(ctx, lastSegmentMeta, fullpath, db.rootKey) + streamInfoData, streamMeta, err := streams.DecryptStreamInfo(ctx, lastSegmentMeta.Data, fullpath, db.rootKey) if err != nil { return object{}, storj.Object{}, err } @@ -276,16 +276,11 @@ func (db *DB) getInfo(ctx context.Context, prefix string, bucket string, path st return object{}, storj.Object{}, err } - streamMeta := pb.StreamMeta{} - err = proto.Unmarshal(lastSegmentMeta.Data, &streamMeta) - if err != nil { - return object{}, storj.Object{}, err - } - info, err = objectStreamFromMeta(bucketInfo, path, lastSegmentMeta, streamInfo, streamMeta, redundancyScheme) if err != nil { return object{}, storj.Object{}, err } + lastSegmentMeta.RedundancyScheme = info.Stream.RedundancyScheme return object{ fullpath: fullpath, @@ -347,14 +342,7 @@ func objectStreamFromMeta(bucket storj.Bucket, path storj.Path, lastSegment segm SegmentCount: stream.NumberOfSegments, FixedSegmentSize: stream.SegmentsSize, - RedundancyScheme: storj.RedundancyScheme{ - Algorithm: storj.ReedSolomon, - ShareSize: redundancyScheme.GetErasureShareSize(), - RequiredShares: int16(redundancyScheme.GetMinReq()), - RepairShares: int16(redundancyScheme.GetRepairThreshold()), - OptimalShares: int16(redundancyScheme.GetSuccessThreshold()), - TotalShares: int16(redundancyScheme.GetTotal()), - }, + RedundancyScheme: segments.RedundancySchemeFromProto(redundancyScheme), EncryptionScheme: storj.EncryptionScheme{ Cipher: storj.Cipher(streamMeta.EncryptionType), BlockSize: streamMeta.EncryptionBlockSize, diff --git a/pkg/storage/buckets/store.go b/pkg/storage/buckets/store.go index fa4d81618..5780458c9 100644 --- a/pkg/storage/buckets/store.go +++ b/pkg/storage/buckets/store.go @@ -47,6 +47,9 @@ type BucketStore struct { type Meta struct { Created time.Time PathEncryptionType storj.Cipher + SegmentsSize int64 + RedundancyScheme storj.RedundancyScheme + EncryptionScheme storj.EncryptionScheme } // NewStore instantiates BucketStore @@ -182,5 +185,8 @@ func convertMeta(m objects.Meta) (Meta, error) { return Meta{ Created: m.Modified, PathEncryptionType: cipher, + RedundancyScheme: m.RedundancyScheme, + EncryptionScheme: m.EncryptionScheme, + SegmentsSize: m.SegmentsSize, }, nil } diff --git a/pkg/storage/objects/store.go b/pkg/storage/objects/store.go index a9fa17798..a820d8f7b 100644 --- a/pkg/storage/objects/store.go +++ b/pkg/storage/objects/store.go @@ -24,10 +24,13 @@ var mon = monkit.Package() // Meta is the full object metadata type Meta struct { pb.SerializableMeta - Modified time.Time - Expiration time.Time - Size int64 - Checksum string + Modified time.Time + Expiration time.Time + Size int64 + Checksum string + SegmentsSize int64 + RedundancyScheme storj.RedundancyScheme + EncryptionScheme storj.EncryptionScheme } // ListItem is a single item in a listing @@ -156,5 +159,8 @@ func convertMeta(m streams.Meta) Meta { Expiration: m.Expiration, Size: m.Size, SerializableMeta: ser, + SegmentsSize: m.SegmentsSize, + RedundancyScheme: m.RedundancyScheme, + EncryptionScheme: m.EncryptionScheme, } } diff --git a/pkg/storage/segments/store.go b/pkg/storage/segments/store.go index 289c0f9a9..e1fd7d919 100644 --- a/pkg/storage/segments/store.go +++ b/pkg/storage/segments/store.go @@ -30,10 +30,11 @@ var ( // Meta info about a segment type Meta struct { - Modified time.Time - Expiration time.Time - Size int64 - Data []byte + Modified time.Time + Expiration time.Time + Size int64 + Data []byte + RedundancyScheme storj.RedundancyScheme } // ListItem is a single item in a listing @@ -354,13 +355,26 @@ func CalcNeededNodes(rs *pb.RedundancyScheme) int32 { return needed } +// RedundancySchemeFromProto translates a pb.RedundancyScheme to a storj.RedundancyScheme. +func RedundancySchemeFromProto(redundancyScheme *pb.RedundancyScheme) storj.RedundancyScheme { + return storj.RedundancyScheme{ + Algorithm: storj.ReedSolomon, + ShareSize: redundancyScheme.GetErasureShareSize(), + RequiredShares: int16(redundancyScheme.GetMinReq()), + RepairShares: int16(redundancyScheme.GetRepairThreshold()), + OptimalShares: int16(redundancyScheme.GetSuccessThreshold()), + TotalShares: int16(redundancyScheme.GetTotal()), + } +} + // convertMeta converts pointer to segment metadata func convertMeta(pr *pb.Pointer) Meta { return Meta{ - Modified: convertTime(pr.GetCreationDate()), - Expiration: convertTime(pr.GetExpirationDate()), - Size: pr.GetSegmentSize(), - Data: pr.GetMetadata(), + Modified: convertTime(pr.GetCreationDate()), + Expiration: convertTime(pr.GetExpirationDate()), + Size: pr.GetSegmentSize(), + Data: pr.GetMetadata(), + RedundancyScheme: RedundancySchemeFromProto(pr.GetRemote().GetRedundancy()), } } diff --git a/pkg/storage/streams/store.go b/pkg/storage/streams/store.go index 6f56bc4b3..6a59e5fe9 100644 --- a/pkg/storage/streams/store.go +++ b/pkg/storage/streams/store.go @@ -30,28 +30,31 @@ import ( var mon = monkit.Package() -// Meta info about a segment +// Meta info about a stream type Meta struct { - Modified time.Time - Expiration time.Time - Size int64 - Data []byte + Modified time.Time + Expiration time.Time + Size int64 + Data []byte + SegmentsSize int64 + EncryptionScheme storj.EncryptionScheme + RedundancyScheme storj.RedundancyScheme } // convertMeta converts segment metadata to stream metadata -func convertMeta(lastSegmentMeta segments.Meta) (Meta, error) { - stream := pb.StreamInfo{} - err := proto.Unmarshal(lastSegmentMeta.Data, &stream) - if err != nil { - return Meta{}, err - } - +func convertMeta(lastSegmentMeta segments.Meta, stream pb.StreamInfo, streamMeta pb.StreamMeta) Meta { return Meta{ - Modified: lastSegmentMeta.Modified, - Expiration: lastSegmentMeta.Expiration, - Size: ((stream.NumberOfSegments - 1) * stream.SegmentsSize) + stream.LastSegmentSize, - Data: stream.Metadata, - }, nil + Modified: lastSegmentMeta.Modified, + Expiration: lastSegmentMeta.Expiration, + Size: ((stream.NumberOfSegments - 1) * stream.SegmentsSize) + stream.LastSegmentSize, + Data: stream.Metadata, + SegmentsSize: stream.SegmentsSize, + RedundancyScheme: lastSegmentMeta.RedundancyScheme, + EncryptionScheme: storj.EncryptionScheme{ + Cipher: storj.Cipher(streamMeta.EncryptionType), + BlockSize: streamMeta.EncryptionBlockSize, + }, + } } // Store interface methods for streams to satisfy to be a store @@ -269,10 +272,15 @@ func (s *streamStore) upload(ctx context.Context, path storj.Path, pathCipher st } resultMeta := Meta{ - Modified: putMeta.Modified, - Expiration: expiration, - Size: streamSize, - Data: metadata, + Modified: putMeta.Modified, + Expiration: expiration, + Size: streamSize, + Data: metadata, + SegmentsSize: s.segmentSize, + EncryptionScheme: storj.EncryptionScheme{ + Cipher: s.cipher, + BlockSize: int32(s.encBlockSize), + }, } return resultMeta, currentSegment, nil @@ -299,7 +307,7 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj return nil, Meta{}, err } - streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey) + streamInfo, streamMeta, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey) if err != nil { return nil, Meta{}, err } @@ -310,12 +318,6 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj return nil, Meta{}, err } - streamMeta := pb.StreamMeta{} - err = proto.Unmarshal(lastSegmentMeta.Data, &streamMeta) - if err != nil { - return nil, Meta{}, err - } - derivedKey, err := encryption.DeriveContentKey(path, s.rootKey) if err != nil { return nil, Meta{}, err @@ -362,16 +364,10 @@ func (s *streamStore) Get(ctx context.Context, path storj.Path, pathCipher storj if err != nil { return nil, Meta{}, err } + rangers = append(rangers, decryptedLastSegmentRanger) - catRangers := ranger.Concat(rangers...) - - lastSegmentMeta.Data = streamInfo - meta, err = convertMeta(lastSegmentMeta) - if err != nil { - return nil, Meta{}, err - } - + meta = convertMeta(lastSegmentMeta, stream, streamMeta) return catRangers, meta, nil } @@ -389,18 +385,16 @@ func (s *streamStore) Meta(ctx context.Context, path storj.Path, pathCipher stor return Meta{}, err } - streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey) + streamInfo, streamMeta, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey) if err != nil { return Meta{}, err } - - lastSegmentMeta.Data = streamInfo - newStreamMeta, err := convertMeta(lastSegmentMeta) - if err != nil { + var stream pb.StreamInfo + if err := proto.Unmarshal(streamInfo, &stream); err != nil { return Meta{}, err } - return newStreamMeta, nil + return convertMeta(lastSegmentMeta, stream, streamMeta), nil } // Delete all the segments, with the last one last @@ -416,14 +410,12 @@ func (s *streamStore) Delete(ctx context.Context, path storj.Path, pathCipher st return err } - streamInfo, err := DecryptStreamInfo(ctx, lastSegmentMeta, path, s.rootKey) + streamInfo, _, err := DecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.rootKey) if err != nil { return err } - - stream := pb.StreamInfo{} - err = proto.Unmarshal(streamInfo, &stream) - if err != nil { + var stream pb.StreamInfo + if err := proto.Unmarshal(streamInfo, &stream); err != nil { return err } @@ -493,17 +485,16 @@ func (s *streamStore) List(ctx context.Context, prefix, startAfter, endBefore st return nil, false, err } - streamInfo, err := DecryptStreamInfo(ctx, item.Meta, storj.JoinPaths(prefix, path), s.rootKey) + streamInfo, streamMeta, err := DecryptStreamInfo(ctx, item.Meta.Data, storj.JoinPaths(prefix, path), s.rootKey) if err != nil { return nil, false, err } - - item.Meta.Data = streamInfo - newMeta, err := convertMeta(item.Meta) - if err != nil { + var stream pb.StreamInfo + if err := proto.Unmarshal(streamInfo, &stream); err != nil { return nil, false, err } + newMeta := convertMeta(item.Meta, stream, streamMeta) items[i] = ListItem{Path: path, Meta: newMeta, IsPrefix: item.IsPrefix} } @@ -666,25 +657,26 @@ func getEncryptedKeyAndNonce(m *pb.SegmentMeta) (storj.EncryptedPrivateKey, *sto } // DecryptStreamInfo decrypts stream info -func DecryptStreamInfo(ctx context.Context, item segments.Meta, path storj.Path, rootKey *storj.Key) (streamInfo []byte, err error) { - streamMeta := pb.StreamMeta{} - err = proto.Unmarshal(item.Data, &streamMeta) +func DecryptStreamInfo(ctx context.Context, streamMetaBytes []byte, path storj.Path, rootKey *storj.Key) ( + streamInfo []byte, streamMeta pb.StreamMeta, err error) { + err = proto.Unmarshal(streamMetaBytes, &streamMeta) if err != nil { - return nil, err + return nil, pb.StreamMeta{}, err } derivedKey, err := encryption.DeriveContentKey(path, rootKey) if err != nil { - return nil, err + return nil, pb.StreamMeta{}, err } cipher := storj.Cipher(streamMeta.EncryptionType) encryptedKey, keyNonce := getEncryptedKeyAndNonce(streamMeta.LastSegmentMeta) contentKey, err := encryption.DecryptKey(encryptedKey, cipher, derivedKey, keyNonce) if err != nil { - return nil, err + return nil, pb.StreamMeta{}, err } // decrypt metadata with the content encryption key and zero nonce - return encryption.Decrypt(streamMeta.EncryptedStreamInfo, cipher, contentKey, &storj.Nonce{}) + streamInfo, err = encryption.Decrypt(streamMeta.EncryptedStreamInfo, cipher, contentKey, &storj.Nonce{}) + return streamInfo, streamMeta, err } diff --git a/pkg/storage/streams/store_test.go b/pkg/storage/streams/store_test.go index 3e58a3bbe..130db3882 100644 --- a/pkg/storage/streams/store_test.go +++ b/pkg/storage/streams/store_test.go @@ -31,12 +31,13 @@ func TestStreamStoreMeta(t *testing.T) { mockSegmentStore := segments.NewMockStore(ctrl) - stream, err := proto.Marshal(&pb.StreamInfo{ + streamInfo := pb.StreamInfo{ NumberOfSegments: 2, SegmentsSize: 10, LastSegmentSize: 0, - Metadata: []byte{}, - }) + Metadata: nil, + } + stream, err := proto.Marshal(&streamInfo) if err != nil { t.Fatal(err) } @@ -53,6 +54,9 @@ func TestStreamStoreMeta(t *testing.T) { Expiration: staticTime, Size: 50, Data: lastSegmentMetadata, + RedundancyScheme: storj.RedundancyScheme{ + Algorithm: storj.ReedSolomon, + }, } streamMetaUnmarshaled := pb.StreamMeta{} @@ -62,16 +66,14 @@ func TestStreamStoreMeta(t *testing.T) { } segmentMetaStreamInfo := segments.Meta{ - Modified: staticTime, - Expiration: staticTime, - Size: 50, - Data: streamMetaUnmarshaled.EncryptedStreamInfo, + Modified: staticTime, + Expiration: staticTime, + Size: 50, + Data: streamMetaUnmarshaled.EncryptedStreamInfo, + RedundancyScheme: segmentMeta.RedundancyScheme, } - streamMeta, err := convertMeta(segmentMetaStreamInfo) - if err != nil { - t.Fatal(err) - } + streamMeta := convertMeta(segmentMetaStreamInfo, streamInfo, streamMetaUnmarshaled) for i, test := range []struct { // input for test function @@ -102,6 +104,7 @@ func TestStreamStoreMeta(t *testing.T) { } assert.Equal(t, test.streamMeta, meta, errTag) + assert.Equal(t, storj.ReedSolomon, test.streamMeta.RedundancyScheme.Algorithm) } } @@ -111,19 +114,31 @@ func TestStreamStorePut(t *testing.T) { mockSegmentStore := segments.NewMockStore(ctrl) + const ( + encBlockSize = 10 + segSize = 10 + pathCipher = storj.AESGCM + dataCipher = storj.Unencrypted + ) + staticTime := time.Now() segmentMeta := segments.Meta{ Modified: staticTime, Expiration: staticTime, - Size: 10, + Size: segSize, Data: []byte{}, } streamMeta := Meta{ - Modified: segmentMeta.Modified, - Expiration: segmentMeta.Expiration, - Size: 4, - Data: []byte("metadata"), + Modified: segmentMeta.Modified, + Expiration: segmentMeta.Expiration, + Size: 4, + Data: []byte("metadata"), + SegmentsSize: segSize, + EncryptionScheme: storj.EncryptionScheme{ + Cipher: dataCipher, + BlockSize: encBlockSize, + }, } for i, test := range []struct { @@ -163,12 +178,12 @@ func TestStreamStorePut(t *testing.T) { Delete(gomock.Any(), gomock.Any()). Return(test.segmentError) - streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) + streamStore, err := NewStreamStore(mockSegmentStore, segSize, new(storj.Key), encBlockSize, dataCipher) if err != nil { t.Fatal(err) } - meta, err := streamStore.Put(ctx, test.path, storj.AESGCM, test.data, test.metadata, test.expiration) + meta, err := streamStore.Put(ctx, test.path, pathCipher, test.data, test.metadata, test.expiration) if err != nil { t.Fatal(err) } @@ -198,6 +213,13 @@ func TestStreamStoreGet(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() + const ( + segSize = 10 + encBlockSize = 10 + dataCipher = storj.Unencrypted + pathCipher = storj.AESGCM + ) + mockSegmentStore := segments.NewMockStore(ctrl) staticTime := time.Now() @@ -209,7 +231,7 @@ func TestStreamStoreGet(t *testing.T) { stream, err := proto.Marshal(&pb.StreamInfo{ NumberOfSegments: 1, - SegmentsSize: 10, + SegmentsSize: segSize, LastSegmentSize: 0, }) if err != nil { @@ -218,6 +240,8 @@ func TestStreamStoreGet(t *testing.T) { lastSegmentMeta, err := proto.Marshal(&pb.StreamMeta{ EncryptedStreamInfo: stream, + EncryptionType: int32(dataCipher), + EncryptionBlockSize: encBlockSize, }) if err != nil { t.Fatal(err) @@ -226,17 +250,22 @@ func TestStreamStoreGet(t *testing.T) { segmentMeta := segments.Meta{ Modified: staticTime, Expiration: staticTime, - Size: 10, + Size: segSize, Data: lastSegmentMeta, } streamRanger := ranger.ByteRanger(nil) streamMeta := Meta{ - Modified: staticTime, - Expiration: staticTime, - Size: 0, - Data: nil, + Modified: staticTime, + Expiration: staticTime, + Size: 0, + Data: nil, + SegmentsSize: segSize, + EncryptionScheme: storj.EncryptionScheme{ + Cipher: dataCipher, + BlockSize: encBlockSize, + }, } for i, test := range []struct { @@ -263,12 +292,12 @@ func TestStreamStoreGet(t *testing.T) { gomock.InOrder(calls...) - streamStore, err := NewStreamStore(mockSegmentStore, 10, new(storj.Key), 10, 0) + streamStore, err := NewStreamStore(mockSegmentStore, segSize, new(storj.Key), encBlockSize, dataCipher) if err != nil { t.Fatal(err) } - ranger, meta, err := streamStore.Get(ctx, test.path, storj.AESGCM) + ranger, meta, err := streamStore.Get(ctx, test.path, pathCipher) if err != nil { t.Fatal(err) } diff --git a/pkg/storj/object.go b/pkg/storj/object.go index f2e252800..3e6ceef34 100644 --- a/pkg/storj/object.go +++ b/pkg/storj/object.go @@ -25,9 +25,12 @@ var ( // Bucket contains information about a specific bucket type Bucket struct { - Name string - Created time.Time - PathCipher Cipher + Name string + Created time.Time + PathCipher Cipher + SegmentsSize int64 + RedundancyScheme RedundancyScheme + EncryptionScheme EncryptionScheme } // Object contains information about a specific object