streams: don't encrypt segment count (#2859)
What: this change makes sure the count of segments is not encrypted. Why: having the segment count encrypted just makes things hard for no reason - a satellite operator can figure out how many segments an object has by looking at the other segments in the database. but if a user has access but has lost their encryption key, they now can't clean up or delete old segments because they can't know how many there are without just guessing until they get errors. :( Backwards compatibility: clients will still understand old pointers and will still write old pointers. at some point in the future perhaps we can do a migration for remaining old pointers so we can delete the old code. Please describe the tests: covered by existing tests Please describe the performance impact: none
This commit is contained in:
JT Olio
GitHub
parent
357a4cdf5e
commit
12d50ebb99
@ -67,7 +67,7 @@ func (m *SegmentMeta) GetKeyNonce() []byte {
|
||||
}
|
||||
|
||||
type StreamInfo struct {
|
||||
NumberOfSegments int64 `protobuf:"varint,1,opt,name=number_of_segments,json=numberOfSegments,proto3" json:"number_of_segments,omitempty"`
|
||||
DeprecatedNumberOfSegments int64 `protobuf:"varint,1,opt,name=deprecated_number_of_segments,json=deprecatedNumberOfSegments,proto3" json:"deprecated_number_of_segments,omitempty"`
|
||||
SegmentsSize int64 `protobuf:"varint,2,opt,name=segments_size,json=segmentsSize,proto3" json:"segments_size,omitempty"`
|
||||
LastSegmentSize int64 `protobuf:"varint,3,opt,name=last_segment_size,json=lastSegmentSize,proto3" json:"last_segment_size,omitempty"`
|
||||
Metadata []byte `protobuf:"bytes,4,opt,name=metadata,proto3" json:"metadata,omitempty"`
|
||||
@ -100,9 +100,9 @@ func (m *StreamInfo) XXX_DiscardUnknown() {
|
||||
|
||||
var xxx_messageInfo_StreamInfo proto.InternalMessageInfo
|
||||
|
||||
func (m *StreamInfo) GetNumberOfSegments() int64 {
|
||||
func (m *StreamInfo) GetDeprecatedNumberOfSegments() int64 {
|
||||
if m != nil {
|
||||
return m.NumberOfSegments
|
||||
return m.DeprecatedNumberOfSegments
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@ -133,6 +133,7 @@ type StreamMeta struct {
|
||||
EncryptionType int32 `protobuf:"varint,2,opt,name=encryption_type,json=encryptionType,proto3" json:"encryption_type,omitempty"`
|
||||
EncryptionBlockSize int32 `protobuf:"varint,3,opt,name=encryption_block_size,json=encryptionBlockSize,proto3" json:"encryption_block_size,omitempty"`
|
||||
LastSegmentMeta *SegmentMeta `protobuf:"bytes,4,opt,name=last_segment_meta,json=lastSegmentMeta,proto3" json:"last_segment_meta,omitempty"`
|
||||
NumberOfSegments int64 `protobuf:"varint,5,opt,name=number_of_segments,json=numberOfSegments,proto3" json:"number_of_segments,omitempty"`
|
||||
XXX_NoUnkeyedLiteral struct{} `json:"-"`
|
||||
XXX_unrecognized []byte `json:"-"`
|
||||
XXX_sizecache int32 `json:"-"`
|
||||
@ -190,6 +191,13 @@ func (m *StreamMeta) GetLastSegmentMeta() *SegmentMeta {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (m *StreamMeta) GetNumberOfSegments() int64 {
|
||||
if m != nil {
|
||||
return m.NumberOfSegments
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func init() {
|
||||
proto.RegisterType((*SegmentMeta)(nil), "streams.SegmentMeta")
|
||||
proto.RegisterType((*StreamInfo)(nil), "streams.StreamInfo")
|
||||
@ -199,24 +207,26 @@ func init() {
|
||||
func init() { proto.RegisterFile("streams.proto", fileDescriptor_c6bbf8af0ec331d6) }
|
||||
|
||||
var fileDescriptor_c6bbf8af0ec331d6 = []byte{
|
||||
// 304 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x51, 0xcb, 0x4e, 0xc3, 0x30,
|
||||
0x10, 0x54, 0x5f, 0x50, 0xb6, 0x29, 0x05, 0x03, 0x52, 0x04, 0x17, 0x14, 0x0e, 0x20, 0x84, 0x7a,
|
||||
0x28, 0x3f, 0x80, 0x7a, 0x43, 0x08, 0x2a, 0x25, 0x9c, 0xb8, 0x58, 0x4e, 0xba, 0x41, 0x51, 0x1a,
|
||||
0x3b, 0x8a, 0xcd, 0xc1, 0xfd, 0x21, 0x3e, 0x8b, 0x5f, 0x41, 0xb6, 0xf3, 0x82, 0xe3, 0xce, 0x8c,
|
||||
0x66, 0x67, 0x76, 0x61, 0x2e, 0x55, 0x85, 0xac, 0x90, 0xcb, 0xb2, 0x12, 0x4a, 0x90, 0xc3, 0x7a,
|
||||
0x0c, 0x36, 0x30, 0x8b, 0xf0, 0xb3, 0x40, 0xae, 0x5e, 0x51, 0x31, 0x72, 0x03, 0x73, 0xe4, 0x49,
|
||||
0xa5, 0x4b, 0x85, 0x5b, 0x9a, 0xa3, 0xf6, 0x07, 0xd7, 0x83, 0x3b, 0x2f, 0xf4, 0x5a, 0xf0, 0x05,
|
||||
0x35, 0xb9, 0x82, 0xa3, 0x1c, 0x35, 0xe5, 0x82, 0x27, 0xe8, 0x0f, 0xad, 0x60, 0x9a, 0xa3, 0x7e,
|
||||
0x33, 0x73, 0xf0, 0x3d, 0x00, 0x88, 0xac, 0xf9, 0x33, 0x4f, 0x05, 0x79, 0x00, 0xc2, 0xbf, 0x8a,
|
||||
0x18, 0x2b, 0x2a, 0x52, 0x2a, 0xdd, 0x26, 0x69, 0x5d, 0x47, 0xe1, 0x89, 0x63, 0x36, 0x69, 0x9d,
|
||||
0x40, 0x9a, 0xf5, 0x8d, 0x86, 0xca, 0x6c, 0xef, 0xdc, 0x47, 0xa1, 0xd7, 0x80, 0x51, 0xb6, 0x47,
|
||||
0x72, 0x0f, 0xa7, 0x3b, 0x26, 0x55, 0xe3, 0xe6, 0x84, 0x23, 0x2b, 0x5c, 0x18, 0xa2, 0x76, 0xb3,
|
||||
0xda, 0x4b, 0x98, 0x16, 0xa8, 0xd8, 0x96, 0x29, 0xe6, 0x8f, 0x5d, 0xd2, 0x66, 0x0e, 0x7e, 0xda,
|
||||
0xa4, 0xb6, 0xfa, 0x0a, 0x2e, 0xba, 0xea, 0xee, 0x3c, 0x34, 0xe3, 0xa9, 0xa8, 0x4f, 0x70, 0xd6,
|
||||
0x92, 0xbd, 0x76, 0xb7, 0xb0, 0xa8, 0xe1, 0x4c, 0x70, 0xaa, 0x74, 0xe9, 0x12, 0x4f, 0xc2, 0xe3,
|
||||
0x0e, 0x7e, 0xd7, 0x25, 0xf6, 0xcc, 0x8d, 0x30, 0xde, 0x89, 0x24, 0xef, 0x72, 0x4f, 0x5a, 0xf3,
|
||||
0x4c, 0xf0, 0xb5, 0xe1, 0x6c, 0xf6, 0xa7, 0x7f, 0x3d, 0x4d, 0x70, 0x5b, 0x62, 0xb6, 0x3a, 0x5f,
|
||||
0x36, 0xef, 0xec, 0x3d, 0xef, 0x4f, 0x7b, 0x03, 0xac, 0xc7, 0x1f, 0xc3, 0x32, 0x8e, 0x0f, 0xec,
|
||||
0xcb, 0x1f, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xdf, 0x5e, 0x32, 0x31, 0x03, 0x02, 0x00, 0x00,
|
||||
// 330 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x52, 0x4d, 0x4b, 0xc3, 0x40,
|
||||
0x10, 0xa5, 0x5f, 0x5a, 0xa7, 0xad, 0xd5, 0xa8, 0x50, 0x2a, 0x82, 0xd4, 0x83, 0x22, 0xd2, 0x43,
|
||||
0xfd, 0x03, 0xda, 0x9b, 0x88, 0x2d, 0xa4, 0x9e, 0xbc, 0x2c, 0x9b, 0x74, 0x22, 0x21, 0xcd, 0x6e,
|
||||
0xc8, 0xae, 0x87, 0xed, 0x7f, 0xf0, 0x1f, 0xf9, 0xe3, 0x64, 0x27, 0xd9, 0x26, 0x16, 0x8f, 0xfb,
|
||||
0xe6, 0xf1, 0xe6, 0xbd, 0x37, 0x0b, 0x03, 0xa5, 0x73, 0xe4, 0xa9, 0x9a, 0x66, 0xb9, 0xd4, 0xd2,
|
||||
0x3b, 0x2c, 0x9f, 0x93, 0x25, 0xf4, 0x56, 0xf8, 0x99, 0xa2, 0xd0, 0x6f, 0xa8, 0xb9, 0x77, 0x03,
|
||||
0x03, 0x14, 0x61, 0x6e, 0x32, 0x8d, 0x6b, 0x96, 0xa0, 0x19, 0x35, 0xae, 0x1b, 0x77, 0x7d, 0xbf,
|
||||
0xbf, 0x03, 0x5f, 0xd1, 0x78, 0x97, 0x70, 0x94, 0xa0, 0x61, 0x42, 0x8a, 0x10, 0x47, 0x4d, 0x22,
|
||||
0x74, 0x13, 0x34, 0x0b, 0xfb, 0x9e, 0xfc, 0x34, 0x00, 0x56, 0x24, 0xfe, 0x22, 0x22, 0xe9, 0x3d,
|
||||
0xc3, 0xd5, 0x1a, 0xb3, 0x1c, 0x43, 0x6e, 0x15, 0xc5, 0x57, 0x1a, 0x60, 0xce, 0x64, 0xc4, 0x54,
|
||||
0xb1, 0x54, 0xd1, 0x82, 0x96, 0x3f, 0xae, 0x48, 0x0b, 0xe2, 0x2c, 0xa3, 0xd2, 0x96, 0xb2, 0x9e,
|
||||
0x1c, 0x9b, 0xa9, 0x78, 0x5b, 0xac, 0x6c, 0xf9, 0x7d, 0x07, 0xae, 0xe2, 0x2d, 0x7a, 0xf7, 0x70,
|
||||
0xba, 0xe1, 0x4a, 0x3b, 0xdd, 0x82, 0xd8, 0x22, 0xe2, 0xd0, 0x0e, 0x4a, 0x35, 0xe2, 0x8e, 0xa1,
|
||||
0x9b, 0xa2, 0xe6, 0x6b, 0xae, 0xf9, 0xa8, 0x5d, 0xd8, 0x77, 0xef, 0xc9, 0x77, 0xd3, 0xd9, 0xa7,
|
||||
0x3e, 0x66, 0x70, 0x51, 0xf5, 0x51, 0x74, 0xc6, 0x62, 0x11, 0xc9, 0xb2, 0x97, 0xb3, 0xdd, 0xb0,
|
||||
0x16, 0xf9, 0x16, 0x86, 0x25, 0x1c, 0x4b, 0xc1, 0xb4, 0xc9, 0x0a, 0xc7, 0x1d, 0xff, 0xb8, 0x82,
|
||||
0xdf, 0x4d, 0x86, 0x35, 0x71, 0x4b, 0x0c, 0x36, 0x32, 0x4c, 0x2a, 0xdf, 0x9d, 0x9d, 0x78, 0x2c,
|
||||
0xc5, 0xdc, 0xce, 0xc8, 0xfb, 0xd3, 0x5e, 0x4e, 0x6b, 0x9c, 0x42, 0xf4, 0x66, 0xe7, 0x53, 0x77,
|
||||
0xe3, 0xda, 0x45, 0xff, 0xa4, 0xa7, 0x48, 0x0f, 0xe0, 0xfd, 0x73, 0x86, 0x0e, 0x55, 0x75, 0x22,
|
||||
0xf6, 0xca, 0x9f, 0xb7, 0x3f, 0x9a, 0x59, 0x10, 0x1c, 0xd0, 0xaf, 0x79, 0xfc, 0x0d, 0x00, 0x00,
|
||||
0xff, 0xff, 0xc4, 0xcb, 0xe6, 0x4a, 0x46, 0x02, 0x00, 0x00,
|
||||
}
|
||||
|
||||
@ -12,7 +12,7 @@ message SegmentMeta {
|
||||
}
|
||||
|
||||
message StreamInfo {
|
||||
int64 number_of_segments = 1;
|
||||
int64 deprecated_number_of_segments = 1;
|
||||
int64 segments_size = 2;
|
||||
int64 last_segment_size = 3;
|
||||
bytes metadata = 4;
|
||||
@ -23,4 +23,5 @@ message StreamMeta {
|
||||
int32 encryption_type = 2;
|
||||
int32 encryption_block_size = 3;
|
||||
SegmentMeta last_segment_meta = 4;
|
||||
int64 number_of_segments = 5;
|
||||
}
|
||||
|
||||
@ -5535,7 +5535,7 @@
|
||||
"fields": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "number_of_segments",
|
||||
"name": "deprecated_number_of_segments",
|
||||
"type": "int64"
|
||||
},
|
||||
{
|
||||
@ -5577,6 +5577,11 @@
|
||||
"id": 4,
|
||||
"name": "last_segment_meta",
|
||||
"type": "SegmentMeta"
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"name": "number_of_segments",
|
||||
"type": "int64"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -95,7 +95,7 @@ func TestOnlyInline(t *testing.T) {
|
||||
InlineFiles: 1,
|
||||
Bytes: int64(expectedTotalBytes),
|
||||
InlineBytes: int64(expectedTotalBytes),
|
||||
MetadataSize: 111, // brittle, this is hardcoded since its too difficult to get this value progamatically
|
||||
MetadataSize: 113, // brittle, this is hardcoded since its too difficult to get this value progamatically
|
||||
}
|
||||
// The projectID should be the 16 bytes uuid representation, not 36 byte string representation
|
||||
assert.Equal(t, 16, len(projectID[:]))
|
||||
|
||||
@ -334,6 +334,11 @@ func objectStreamFromMeta(bucket storj.Bucket, path storj.Path, lastSegment segm
|
||||
return storj.Object{}, err
|
||||
}
|
||||
|
||||
numberOfSegments := streamMeta.NumberOfSegments
|
||||
if streamMeta.NumberOfSegments == 0 {
|
||||
numberOfSegments = stream.DeprecatedNumberOfSegments
|
||||
}
|
||||
|
||||
return storj.Object{
|
||||
Version: 0, // TODO:
|
||||
Bucket: bucket,
|
||||
@ -348,10 +353,10 @@ func objectStreamFromMeta(bucket storj.Bucket, path storj.Path, lastSegment segm
|
||||
Expires: lastSegment.Expiration, // TODO: use correct field
|
||||
|
||||
Stream: storj.Stream{
|
||||
Size: stream.SegmentsSize*(stream.NumberOfSegments-1) + stream.LastSegmentSize,
|
||||
Size: stream.SegmentsSize*(numberOfSegments-1) + stream.LastSegmentSize,
|
||||
// Checksum: []byte(object.Checksum),
|
||||
|
||||
SegmentCount: stream.NumberOfSegments,
|
||||
SegmentCount: numberOfSegments,
|
||||
FixedSegmentSize: stream.SegmentsSize,
|
||||
|
||||
RedundancyScheme: storj.RedundancyScheme{
|
||||
|
||||
@ -39,12 +39,19 @@ type Meta struct {
|
||||
Data []byte
|
||||
}
|
||||
|
||||
func numberOfSegments(stream *pb.StreamInfo, streamMeta *pb.StreamMeta) int64 {
|
||||
if streamMeta.NumberOfSegments > 0 {
|
||||
return streamMeta.NumberOfSegments
|
||||
}
|
||||
return stream.DeprecatedNumberOfSegments
|
||||
}
|
||||
|
||||
// convertMeta converts segment metadata to stream metadata
|
||||
func convertMeta(lastSegmentMeta segments.Meta, stream pb.StreamInfo, streamMeta pb.StreamMeta) Meta {
|
||||
return Meta{
|
||||
Modified: lastSegmentMeta.Modified,
|
||||
Expiration: lastSegmentMeta.Expiration,
|
||||
Size: ((stream.NumberOfSegments - 1) * stream.SegmentsSize) + stream.LastSegmentSize,
|
||||
Size: ((numberOfSegments(&stream, &streamMeta) - 1) * stream.SegmentsSize) + stream.LastSegmentSize,
|
||||
Data: stream.Metadata,
|
||||
}
|
||||
}
|
||||
@ -223,7 +230,7 @@ func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Ci
|
||||
}
|
||||
|
||||
streamInfo, err := proto.Marshal(&pb.StreamInfo{
|
||||
NumberOfSegments: currentSegment + 1,
|
||||
DeprecatedNumberOfSegments: currentSegment + 1,
|
||||
SegmentsSize: s.segmentSize,
|
||||
LastSegmentSize: sizeReader.Size(),
|
||||
Metadata: metadata,
|
||||
@ -239,6 +246,7 @@ func (s *streamStore) upload(ctx context.Context, path Path, pathCipher storj.Ci
|
||||
}
|
||||
|
||||
streamMeta := pb.StreamMeta{
|
||||
NumberOfSegments: currentSegment + 1,
|
||||
EncryptedStreamInfo: encryptedStreamInfo,
|
||||
EncryptionType: int32(s.cipher),
|
||||
EncryptionBlockSize: int32(s.encBlockSize),
|
||||
@ -318,7 +326,7 @@ func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.Ciphe
|
||||
}
|
||||
|
||||
var rangers []ranger.Ranger
|
||||
for i := int64(0); i < stream.NumberOfSegments-1; i++ {
|
||||
for i := int64(0); i < numberOfSegments(&stream, &streamMeta)-1; i++ {
|
||||
currentPath, err := createSegmentPath(ctx, i, path.Bucket(), encPath)
|
||||
if err != nil {
|
||||
return nil, Meta{}, err
|
||||
@ -342,7 +350,7 @@ func (s *streamStore) Get(ctx context.Context, path Path, pathCipher storj.Ciphe
|
||||
}
|
||||
|
||||
var contentNonce storj.Nonce
|
||||
_, err = encryption.Increment(&contentNonce, stream.NumberOfSegments)
|
||||
_, err = encryption.Increment(&contentNonce, numberOfSegments(&stream, &streamMeta))
|
||||
if err != nil {
|
||||
return nil, Meta{}, err
|
||||
}
|
||||
@ -420,7 +428,7 @@ func (s *streamStore) Delete(ctx context.Context, path Path, pathCipher storj.Ci
|
||||
return err
|
||||
}
|
||||
|
||||
streamInfo, _, err := TypedDecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.encStore)
|
||||
streamInfo, streamMeta, err := TypedDecryptStreamInfo(ctx, lastSegmentMeta.Data, path, s.encStore)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -429,7 +437,7 @@ func (s *streamStore) Delete(ctx context.Context, path Path, pathCipher storj.Ci
|
||||
return err
|
||||
}
|
||||
|
||||
for i := 0; i < int(stream.NumberOfSegments-1); i++ {
|
||||
for i := 0; i < int(numberOfSegments(&stream, &streamMeta)-1); i++ {
|
||||
currentPath, err := createSegmentPath(ctx, int64(i), path.Bucket(), encPath)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
Loading…
Reference in New Issue
Block a user