From 12d50ebb995f782ff654104aafe78629fe1ce26d Mon Sep 17 00:00:00 2001 From: JT Olio Date: Thu, 22 Aug 2019 15:15:58 -0600 Subject: [PATCH] streams: don't encrypt segment count (#2859) What: this change makes sure the count of segments is not encrypted. Why: having the segment count encrypted just makes things hard for no reason - a satellite operator can figure out how many segments an object has by looking at the other segments in the database. but if a user has access but has lost their encryption key, they now can't clean up or delete old segments because they can't know how many there are without just guessing until they get errors. :( Backwards compatibility: clients will still understand old pointers and will still write old pointers. at some point in the future perhaps we can do a migration for remaining old pointers so we can delete the old code. Please describe the tests: covered by existing tests Please describe the performance impact: none --- pkg/pb/streams.pb.go | 68 ++++++++++++++---------- pkg/pb/streams.proto | 3 +- proto.lock | 7 ++- satellite/accounting/tally/tally_test.go | 2 +- uplink/metainfo/kvmetainfo/objects.go | 9 +++- uplink/storage/streams/store.go | 26 +++++---- 6 files changed, 72 insertions(+), 43 deletions(-) diff --git a/pkg/pb/streams.pb.go b/pkg/pb/streams.pb.go index 51d09c490..700137c27 100644 --- a/pkg/pb/streams.pb.go +++ b/pkg/pb/streams.pb.go @@ -67,13 +67,13 @@ func (m *SegmentMeta) GetKeyNonce() []byte { } type StreamInfo struct { - NumberOfSegments int64 `protobuf:"varint,1,opt,name=number_of_segments,json=numberOfSegments,proto3" json:"number_of_segments,omitempty"` - SegmentsSize int64 `protobuf:"varint,2,opt,name=segments_size,json=segmentsSize,proto3" json:"segments_size,omitempty"` - LastSegmentSize int64 `protobuf:"varint,3,opt,name=last_segment_size,json=lastSegmentSize,proto3" json:"last_segment_size,omitempty"` - Metadata []byte `protobuf:"bytes,4,opt,name=metadata,proto3" json:"metadata,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + DeprecatedNumberOfSegments int64 `protobuf:"varint,1,opt,name=deprecated_number_of_segments,json=deprecatedNumberOfSegments,proto3" json:"deprecated_number_of_segments,omitempty"` + SegmentsSize int64 `protobuf:"varint,2,opt,name=segments_size,json=segmentsSize,proto3" json:"segments_size,omitempty"` + LastSegmentSize int64 `protobuf:"varint,3,opt,name=last_segment_size,json=lastSegmentSize,proto3" json:"last_segment_size,omitempty"` + Metadata []byte `protobuf:"bytes,4,opt,name=metadata,proto3" json:"metadata,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *StreamInfo) Reset() { *m = StreamInfo{} } @@ -100,9 +100,9 @@ func (m *StreamInfo) XXX_DiscardUnknown() { var xxx_messageInfo_StreamInfo proto.InternalMessageInfo -func (m *StreamInfo) GetNumberOfSegments() int64 { +func (m *StreamInfo) GetDeprecatedNumberOfSegments() int64 { if m != nil { - return m.NumberOfSegments + return m.DeprecatedNumberOfSegments } return 0 } @@ -133,6 +133,7 @@ type StreamMeta struct { EncryptionType int32 `protobuf:"varint,2,opt,name=encryption_type,json=encryptionType,proto3" json:"encryption_type,omitempty"` EncryptionBlockSize int32 `protobuf:"varint,3,opt,name=encryption_block_size,json=encryptionBlockSize,proto3" json:"encryption_block_size,omitempty"` LastSegmentMeta *SegmentMeta `protobuf:"bytes,4,opt,name=last_segment_meta,json=lastSegmentMeta,proto3" json:"last_segment_meta,omitempty"` + NumberOfSegments int64 `protobuf:"varint,5,opt,name=number_of_segments,json=numberOfSegments,proto3" json:"number_of_segments,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -190,6 +191,13 @@ func (m *StreamMeta) GetLastSegmentMeta() *SegmentMeta { return nil } +func (m *StreamMeta) GetNumberOfSegments() int64 { + if m != nil { + return m.NumberOfSegments + } + return 0 +} + func init() { proto.RegisterType((*SegmentMeta)(nil), "streams.SegmentMeta") proto.RegisterType((*StreamInfo)(nil), "streams.StreamInfo") @@ -199,24 +207,26 @@ func init() { func init() { proto.RegisterFile("streams.proto", fileDescriptor_c6bbf8af0ec331d6) } var fileDescriptor_c6bbf8af0ec331d6 = []byte{ - // 304 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x51, 0xcb, 0x4e, 0xc3, 0x30, - 0x10, 0x54, 0x5f, 0x50, 0xb6, 0x29, 0x05, 0x03, 0x52, 0x04, 0x17, 0x14, 0x0e, 0x20, 0x84, 0x7a, - 0x28, 0x3f, 0x80, 0x7a, 0x43, 0x08, 0x2a, 0x25, 0x9c, 0xb8, 0x58, 0x4e, 0xba, 0x41, 0x51, 0x1a, - 0x3b, 0x8a, 0xcd, 0xc1, 0xfd, 0x21, 0x3e, 0x8b, 0x5f, 0x41, 0xb6, 0xf3, 0x82, 0xe3, 0xce, 0x8c, - 0x66, 0x67, 0x76, 0x61, 0x2e, 0x55, 0x85, 0xac, 0x90, 0xcb, 0xb2, 0x12, 0x4a, 0x90, 0xc3, 0x7a, - 0x0c, 0x36, 0x30, 0x8b, 0xf0, 0xb3, 0x40, 0xae, 0x5e, 0x51, 0x31, 0x72, 0x03, 0x73, 0xe4, 0x49, - 0xa5, 0x4b, 0x85, 0x5b, 0x9a, 0xa3, 0xf6, 0x07, 0xd7, 0x83, 0x3b, 0x2f, 0xf4, 0x5a, 0xf0, 0x05, - 0x35, 0xb9, 0x82, 0xa3, 0x1c, 0x35, 0xe5, 0x82, 0x27, 0xe8, 0x0f, 0xad, 0x60, 0x9a, 0xa3, 0x7e, - 0x33, 0x73, 0xf0, 0x3d, 0x00, 0x88, 0xac, 0xf9, 0x33, 0x4f, 0x05, 0x79, 0x00, 0xc2, 0xbf, 0x8a, - 0x18, 0x2b, 0x2a, 0x52, 0x2a, 0xdd, 0x26, 0x69, 0x5d, 0x47, 0xe1, 0x89, 0x63, 0x36, 0x69, 0x9d, - 0x40, 0x9a, 0xf5, 0x8d, 0x86, 0xca, 0x6c, 0xef, 0xdc, 0x47, 0xa1, 0xd7, 0x80, 0x51, 0xb6, 0x47, - 0x72, 0x0f, 0xa7, 0x3b, 0x26, 0x55, 0xe3, 0xe6, 0x84, 0x23, 0x2b, 0x5c, 0x18, 0xa2, 0x76, 0xb3, - 0xda, 0x4b, 0x98, 0x16, 0xa8, 0xd8, 0x96, 0x29, 0xe6, 0x8f, 0x5d, 0xd2, 0x66, 0x0e, 0x7e, 0xda, - 0xa4, 0xb6, 0xfa, 0x0a, 0x2e, 0xba, 0xea, 0xee, 0x3c, 0x34, 0xe3, 0xa9, 0xa8, 0x4f, 0x70, 0xd6, - 0x92, 0xbd, 0x76, 0xb7, 0xb0, 0xa8, 0xe1, 0x4c, 0x70, 0xaa, 0x74, 0xe9, 0x12, 0x4f, 0xc2, 0xe3, - 0x0e, 0x7e, 0xd7, 0x25, 0xf6, 0xcc, 0x8d, 0x30, 0xde, 0x89, 0x24, 0xef, 0x72, 0x4f, 0x5a, 0xf3, - 0x4c, 0xf0, 0xb5, 0xe1, 0x6c, 0xf6, 0xa7, 0x7f, 0x3d, 0x4d, 0x70, 0x5b, 0x62, 0xb6, 0x3a, 0x5f, - 0x36, 0xef, 0xec, 0x3d, 0xef, 0x4f, 0x7b, 0x03, 0xac, 0xc7, 0x1f, 0xc3, 0x32, 0x8e, 0x0f, 0xec, - 0xcb, 0x1f, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xdf, 0x5e, 0x32, 0x31, 0x03, 0x02, 0x00, 0x00, + // 330 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x52, 0x4d, 0x4b, 0xc3, 0x40, + 0x10, 0xa5, 0x5f, 0x5a, 0xa7, 0xad, 0xd5, 0xa8, 0x50, 0x2a, 0x82, 0xd4, 0x83, 0x22, 0xd2, 0x43, + 0xfd, 0x03, 0xda, 0x9b, 0x88, 0x2d, 0xa4, 0x9e, 0xbc, 0x2c, 0x9b, 0x74, 0x22, 0x21, 0xcd, 0x6e, + 0xc8, 0xae, 0x87, 0xed, 0x7f, 0xf0, 0x1f, 0xf9, 0xe3, 0x64, 0x27, 0xd9, 0x26, 0x16, 0x8f, 0xfb, + 0xe6, 0xf1, 0xe6, 0xbd, 0x37, 0x0b, 0x03, 0xa5, 0x73, 0xe4, 0xa9, 0x9a, 0x66, 0xb9, 0xd4, 0xd2, + 0x3b, 0x2c, 0x9f, 0x93, 0x25, 0xf4, 0x56, 0xf8, 0x99, 0xa2, 0xd0, 0x6f, 0xa8, 0xb9, 0x77, 0x03, + 0x03, 0x14, 0x61, 0x6e, 0x32, 0x8d, 0x6b, 0x96, 0xa0, 0x19, 0x35, 0xae, 0x1b, 0x77, 0x7d, 0xbf, + 0xbf, 0x03, 0x5f, 0xd1, 0x78, 0x97, 0x70, 0x94, 0xa0, 0x61, 0x42, 0x8a, 0x10, 0x47, 0x4d, 0x22, + 0x74, 0x13, 0x34, 0x0b, 0xfb, 0x9e, 0xfc, 0x34, 0x00, 0x56, 0x24, 0xfe, 0x22, 0x22, 0xe9, 0x3d, + 0xc3, 0xd5, 0x1a, 0xb3, 0x1c, 0x43, 0x6e, 0x15, 0xc5, 0x57, 0x1a, 0x60, 0xce, 0x64, 0xc4, 0x54, + 0xb1, 0x54, 0xd1, 0x82, 0x96, 0x3f, 0xae, 0x48, 0x0b, 0xe2, 0x2c, 0xa3, 0xd2, 0x96, 0xb2, 0x9e, + 0x1c, 0x9b, 0xa9, 0x78, 0x5b, 0xac, 0x6c, 0xf9, 0x7d, 0x07, 0xae, 0xe2, 0x2d, 0x7a, 0xf7, 0x70, + 0xba, 0xe1, 0x4a, 0x3b, 0xdd, 0x82, 0xd8, 0x22, 0xe2, 0xd0, 0x0e, 0x4a, 0x35, 0xe2, 0x8e, 0xa1, + 0x9b, 0xa2, 0xe6, 0x6b, 0xae, 0xf9, 0xa8, 0x5d, 0xd8, 0x77, 0xef, 0xc9, 0x77, 0xd3, 0xd9, 0xa7, + 0x3e, 0x66, 0x70, 0x51, 0xf5, 0x51, 0x74, 0xc6, 0x62, 0x11, 0xc9, 0xb2, 0x97, 0xb3, 0xdd, 0xb0, + 0x16, 0xf9, 0x16, 0x86, 0x25, 0x1c, 0x4b, 0xc1, 0xb4, 0xc9, 0x0a, 0xc7, 0x1d, 0xff, 0xb8, 0x82, + 0xdf, 0x4d, 0x86, 0x35, 0x71, 0x4b, 0x0c, 0x36, 0x32, 0x4c, 0x2a, 0xdf, 0x9d, 0x9d, 0x78, 0x2c, + 0xc5, 0xdc, 0xce, 0xc8, 0xfb, 0xd3, 0x5e, 0x4e, 0x6b, 0x9c, 0x42, 0xf4, 0x66, 0xe7, 0x53, 0x77, + 0xe3, 0xda, 0x45, 0xff, 0xa4, 0xa7, 0x48, 0x0f, 0xe0, 0xfd, 0x73, 0x86, 0x0e, 0x55, 0x75, 0x22, + 0xf6, 0xca, 0x9f, 0xb7, 0x3f, 0x9a, 0x59, 0x10, 0x1c, 0xd0, 0xaf, 0x79, 0xfc, 0x0d, 0x00, 0x00, + 0xff, 0xff, 0xc4, 0xcb, 0xe6, 0x4a, 0x46, 0x02, 0x00, 0x00, } diff --git a/pkg/pb/streams.proto b/pkg/pb/streams.proto index ef99548f0..4da965377 100644 --- a/pkg/pb/streams.proto +++ b/pkg/pb/streams.proto @@ -12,7 +12,7 @@ message SegmentMeta { } message StreamInfo { - int64 number_of_segments = 1; + int64 deprecated_number_of_segments = 1; int64 segments_size = 2; int64 last_segment_size = 3; bytes metadata = 4; @@ -23,4 +23,5 @@ message StreamMeta { int32 encryption_type = 2; int32 encryption_block_size = 3; SegmentMeta last_segment_meta = 4; + int64 number_of_segments = 5; } diff --git a/proto.lock b/proto.lock index 8fc133402..d3c379dde 100644 --- a/proto.lock +++ b/proto.lock @@ -5535,7 +5535,7 @@ "fields": [ { "id": 1, - "name": "number_of_segments", + "name": "deprecated_number_of_segments", "type": "int64" }, { @@ -5577,6 +5577,11 @@ "id": 4, "name": "last_segment_meta", "type": "SegmentMeta" + }, + { + "id": 5, + "name": "number_of_segments", + "type": "int64" } ] } diff --git a/satellite/accounting/tally/tally_test.go b/satellite/accounting/tally/tally_test.go index 6ade98e65..50119e5d0 100644 --- a/satellite/accounting/tally/tally_test.go +++ b/satellite/accounting/tally/tally_test.go @@ -95,7 +95,7 @@ func TestOnlyInline(t *testing.T) { InlineFiles: 1, Bytes: int64(expectedTotalBytes), InlineBytes: int64(expectedTotalBytes), - MetadataSize: 111, // brittle, this is hardcoded since its too difficult to get this value progamatically + MetadataSize: 113, // brittle, this is hardcoded since its too difficult to get this value progamatically } // The projectID should be the 16 bytes uuid representation, not 36 byte string representation assert.Equal(t, 16, len(projectID[:])) diff --git a/uplink/metainfo/kvmetainfo/objects.go b/uplink/metainfo/kvmetainfo/objects.go index 18c325b58..bb501148f 100644 --- a/uplink/metainfo/kvmetainfo/objects.go +++ b/uplink/metainfo/kvmetainfo/objects.go @@ -334,6 +334,11 @@ func objectStreamFromMeta(bucket storj.Bucket, path storj.Path, lastSegment segm return storj.Object{}, err } + numberOfSegments := streamMeta.NumberOfSegments + if streamMeta.NumberOfSegments == 0 { + numberOfSegments = stream.DeprecatedNumberOfSegments + } + return storj.Object{ Version: 0, // TODO: Bucket: bucket, @@ -348,10 +353,10 @@ func objectStreamFromMeta(bucket storj.Bucket, path storj.Path, lastSegment segm Expires: lastSegment.Expiration, // TODO: use correct field Stream: storj.Stream{ - Size: stream.SegmentsSize*(stream.NumberOfSegments-1) + stream.LastSegmentSize, + Size: stream.SegmentsSize*(numberOfSegments-1) + stream.LastSegmentSize, // Checksum: []byte(object.Checksum), - SegmentCount: stream.NumberOfSegments, + SegmentCount: numberOfSegments, FixedSegmentSize: stream.SegmentsSize, RedundancyScheme: storj.RedundancyScheme{ diff --git a/uplink/storage/streams/store.go b/uplink/storage/streams/store.go index d67687c6b..01d20a87d 100644 --- a/uplink/storage/streams/store.go +++ b/uplink/storage/streams/store.go @@ -39,12 +39,19 @@ type Meta struct { Data []byte } +func numberOfSegments(stream *pb.StreamInfo, streamMeta *pb.StreamMeta) int64 { + if streamMeta.NumberOfSegments > 0 { + return streamMeta.NumberOfSegments + } + return stream.DeprecatedNumberOfSegments +} + // convertMeta converts segment metadata to stream metadata func convertMeta(lastSegmentMeta segments.Meta, stream pb.StreamInfo, streamMeta pb.StreamMeta) Meta { return Meta{ Modified: lastSegmentMeta.Modified, Expiration: lastSegmentMeta.Expiration, - Size: ((stream.NumberOfSegments - 1) * stream.SegmentsSize) + stream.LastSegmentSize, + Size: ((numberOfSegments(&stream, &streamMeta) - 1) * stream.SegmentsSize) + stream.LastSegmentSize, Data: stream.Metadata, } } @@ -223,10 +230,10 @@ func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Ci } streamInfo, err := proto.Marshal(&pb.StreamInfo{ - NumberOfSegments: currentSegment + 1, - SegmentsSize: s.segmentSize, - LastSegmentSize: sizeReader.Size(), - Metadata: metadata, + DeprecatedNumberOfSegments: currentSegment + 1, + SegmentsSize: s.segmentSize, + LastSegmentSize: sizeReader.Size(), + Metadata: metadata, }) if err != nil { return "", nil, err @@ -239,6 +246,7 @@ func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Ci } streamMeta := pb.StreamMeta{ + NumberOfSegments: currentSegment + 1, EncryptedStreamInfo: encryptedStreamInfo, EncryptionType: int32(s.cipher), EncryptionBlockSize: int32(s.encBlockSize), @@ -318,7 +326,7 @@ func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.Ciphe } var rangers []ranger.Ranger - for i := int64(0); i < stream.NumberOfSegments-1; i++ { + for i := int64(0); i < numberOfSegments(&stream, &streamMeta)-1; i++ { currentPath, err := createSegmentPath(ctx, i, path.Bucket(), encPath) if err != nil { return nil, Meta{}, err @@ -342,7 +350,7 @@ func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.Ciphe } var contentNonce storj.Nonce - _, err = encryption.Increment(&contentNonce, stream.NumberOfSegments) + _, err = encryption.Increment(&contentNonce, numberOfSegments(&stream, &streamMeta)) if err != nil { return nil, Meta{}, err } @@ -420,7 +428,7 @@ func (s *streamStore) Delete(ctx context.Context, path Path, pathCipher storj.Ci return err } - streamInfo, _, err := TypedDecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.encStore) + streamInfo, streamMeta, err := TypedDecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.encStore) if err != nil { return err } @@ -429,7 +437,7 @@ func (s *streamStore) Delete(ctx context.Context, path Path, pathCipher storj.Ci return err } - for i := 0; i < int(stream.NumberOfSegments-1); i++ { + for i := 0; i < int(numberOfSegments(&stream, &streamMeta)-1); i++ { currentPath, err := createSegmentPath(ctx, int64(i), path.Bucket(), encPath) if err != nil { return err