JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo: wire versioning with commit object operation

Browse Source 
Change-Id: Ib19b5bade69dfb3f02256c638cc2f6281befa415
This commit is contained in:
Michal Niewrzal 2023-11-03 16:37:57 +01:00 committed by Michał Niewrzał
parent 539b32d01e
commit c9430c22b5
6 changed files with 111 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
satellite/internalpb/delegated_repair_drpc.pb.go
View File

@ -1,5 +1,5 @@
// Code generated by protoc-gen-go-drpc. DO NOT EDIT.
// protoc-gen-go-drpc version: v0.0.28
// protoc-gen-go-drpc version: v0.0.33
// source: delegated_repair.proto
package internalpb

84
satellite/internalpb/metainfo_sat.pb.go
View File

@ -39,6 +39,7 @@ type StreamID struct {
// temporary field to determine if we should go with new pending_objects table or
// fallback to pending object in objects table.
UsePendingObjectsTable bool `protobuf:"varint,14,opt,name=use_pending_objects_table,json=usePendingObjectsTable,proto3" json:"use_pending_objects_table,omitempty"`
Versioned bool `protobuf:"varint,15,opt,name=versioned,proto3" json:"versioned,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -145,6 +146,13 @@ func (m *StreamID) GetUsePendingObjectsTable() bool {
return false
}
func (m *StreamID) GetVersioned() bool {
if m != nil {
return m.Versioned
}
return false
}
type SegmentID struct {
StreamId *StreamID `protobuf:"bytes,1,opt,name=stream_id,json=streamId,proto3" json:"stream_id,omitempty"`
PartNumber int32 `protobuf:"varint,2,opt,name=part_number,json=partNumber,proto3" json:"part_number,omitempty"`
@ -232,42 +240,42 @@ func init() {
func init() { proto.RegisterFile("metainfo_sat.proto", fileDescriptor_47c60bd892d94aaf) }
var fileDescriptor_47c60bd892d94aaf = []byte{
// 580 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x52, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0xc5, 0x84, 0xa4, 0xc9, 0x24, 0x7d, 0x68, 0xfa, 0xd0, 0x10, 0x8a, 0x62, 0x15, 0x21, 0x99,
0x8d, 0x8d, 0xda, 0x55, 0x97, 0x44, 0x61, 0x11, 0xf1, 0x68, 0x70, 0xca, 0x86, 0x8d, 0x35, 0xb6,
0x6f, 0xad, 0x69, 0xed, 0x19, 0x6b, 0x66, 0x8c, 0x9a, 0x25, 0x7f, 0xc0, 0xaf, 0xf0, 0x17, 0x7c,
0x03, 0x8b, 0xf2, 0x2b, 0xc8, 0xe3, 0x47, 0x22, 0xd1, 0x2e, 0x60, 0x37, 0xf7, 0xdc, 0x33, 0x67,
0xce, 0x9c, 0x7b, 0x11, 0xce, 0x40, 0x53, 0xc6, 0xaf, 0x44, 0xa0, 0xa8, 0x76, 0x73, 0x29, 0xb4,
0xc0, 0x58, 0x51, 0x0d, 0x69, 0xca, 0x34, 0xb8, 0x4d, 0x77, 0xbc, 0x07, 0x3c, 0x92, 0xab, 0x5c,
0x33, 0xc1, 0x2b, 0xd6, 0x18, 0x25, 0x22, 0x11, 0xf5, 0x79, 0x92, 0x08, 0x91, 0xa4, 0xe0, 0x99,
0x2a, 0x2c, 0xae, 0x3c, 0xcd, 0x32, 0x50, 0x9a, 0x66, 0x79, 0x4d, 0xd8, 0x69, 0x84, 0xaa, 0xfa,
0xe4, 0xc7, 0x13, 0xd4, 0x5f, 0x6a, 0x09, 0x34, 0x9b, 0xcf, 0xf0, 0x11, 0xea, 0x85, 0x45, 0x74,
0x03, 0x9a, 0x58, 0xb6, 0xe5, 0x8c, 0xfc, 0xba, 0xc2, 0xaf, 0xd1, 0x41, 0xfd, 0x2a, 0xc4, 0x81,
0x08, 0xaf, 0x21, 0xd2, 0xc1, 0x0d, 0xac, 0xc8, 0x63, 0xc3, 0xc2, 0x6d, 0xef, 0xc2, 0xb4, 0xde,
0xc1, 0x0a, 0x13, 0xb4, 0xf5, 0x15, 0xa4, 0x62, 0x82, 0x93, 0x8e, 0x6d, 0x39, 0x1d, 0xbf, 0x29,
0xf1, 0x67, 0x74, 0xb8, 0xfe, 0x41, 0x90, 0x53, 0x49, 0x33, 0xd0, 0x20, 0x15, 0x19, 0xd9, 0x96,
0x33, 0x3c, 0xb5, 0xdd, 0x8d, 0xff, 0xbd, 0x6d, 0x8f, 0x8b, 0x96, 0xe7, 0x1f, 0xc0, 0x3d, 0x28,
0x9e, 0xa3, 0xed, 0x48, 0x02, 0x35, 0xa2, 0x31, 0xd5, 0x40, 0xba, 0x46, 0x6e, 0xec, 0x56, 0x81,
0xb8, 0x4d, 0x20, 0xee, 0x65, 0x13, 0xc8, 0xb4, 0xff, 0xf3, 0x6e, 0xf2, 0xe8, 0xfb, 0xef, 0x89,
0xe5, 0x8f, 0x9a, 0xab, 0x33, 0xaa, 0x01, 0x7f, 0x40, 0xbb, 0x70, 0x9b, 0x33, 0xb9, 0x21, 0xd6,
0xfb, 0x07, 0xb1, 0x9d, 0xf5, 0x65, 0x23, 0xf7, 0x0a, 0xed, 0x65, 0x45, 0xaa, 0x59, 0x4e, 0xa5,
0xae, 0xc3, 0x23, 0x43, 0xdb, 0x72, 0xfa, 0xfe, 0x6e, 0x8b, 0x57, 0xc1, 0x61, 0x0f, 0xed, 0xb7,
0x13, 0x0f, 0x14, 0x4b, 0x38, 0xd5, 0x85, 0x04, 0x32, 0xa8, 0x62, 0x6e, 0x5b, 0xcb, 0xa6, 0x83,
0x9f, 0xa1, 0x81, 0x32, 0xc3, 0x0b, 0x58, 0x4c, 0x90, 0xa1, 0xf5, 0x2b, 0x60, 0x1e, 0xe3, 0x63,
0x34, 0xc8, 0x53, 0x1a, 0x41, 0x06, 0x5c, 0x93, 0x6d, 0xdb, 0x72, 0xba, 0xfe, 0x1a, 0xc0, 0xe7,
0xe8, 0x69, 0xa1, 0x20, 0xc8, 0x81, 0xc7, 0x8c, 0x27, 0xb5, 0x31, 0x15, 0x68, 0x1a, 0xa6, 0x40,
0x76, 0x8c, 0xbf, 0xa3, 0x42, 0xc1, 0xa2, 0xea, 0x57, 0x06, 0xd5, 0x65, 0xd9, 0x3d, 0xf9, 0xd6,
0x41, 0x83, 0x25, 0x24, 0xa5, 0xcc, 0x7c, 0x86, 0xcf, 0x37, 0x3d, 0x58, 0x26, 0xa8, 0x63, 0xf7,
0xef, 0xc5, 0x75, 0x9b, 0x2d, 0xdb, 0x70, 0x38, 0x41, 0x43, 0x93, 0x0a, 0x2f, 0xb2, 0x10, 0xa4,
0x59, 0xa7, 0xae, 0x8f, 0x4a, 0xe8, 0xa3, 0x41, 0xf0, 0x01, 0xea, 0x32, 0x1e, 0xc3, 0xad, 0x59,
0xa2, 0xae, 0x5f, 0x15, 0xf8, 0x0c, 0x6d, 0x4b, 0x21, 0x74, 0x90, 0x33, 0x88, 0xa0, 0x7c, 0xb5,
0x9c, 0xf5, 0x68, 0xba, 0x5b, 0x8e, 0xe0, 0xd7, 0xdd, 0x64, 0x6b, 0x51, 0xe2, 0xf3, 0x99, 0x3f,
0x2c, 0x59, 0x55, 0x11, 0xe3, 0x4f, 0xe8, 0x50, 0x48, 0x96, 0x30, 0x4e, 0xd3, 0x40, 0xc8, 0x18,
0x64, 0x90, 0xb2, 0x8c, 0x69, 0x45, 0x7a, 0x76, 0xc7, 0x19, 0x9e, 0x3e, 0x5f, 0x1b, 0x7d, 0x13,
0xc7, 0x12, 0x94, 0x82, 0xf8, 0xa2, 0xa4, 0xbd, 0x2f, 0x59, 0xfe, 0x7e, 0x73, 0x77, 0x8d, 0xdd,
0xb3, 0x73, 0x5b, 0xff, 0xbd, 0x73, 0x0f, 0x4c, 0xbe, 0xff, 0xd0, 0xe4, 0xa7, 0x2f, 0xbf, 0xbc,
0x50, 0x5a, 0xc8, 0x6b, 0x97, 0x09, 0xcf, 0x1c, 0xbc, 0x96, 0xe4, 0x31, 0xae, 0x41, 0x72, 0x9a,
0xe6, 0x61, 0xd8, 0x33, 0x1e, 0xce, 0xfe, 0x04, 0x00, 0x00, 0xff, 0xff, 0xec, 0x7c, 0xdf, 0x4d,
0x5e, 0x04, 0x00, 0x00,
// 589 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x52, 0xcb, 0x6e, 0xd4, 0x30,
0x14, 0x25, 0x0c, 0x33, 0x9d, 0xf1, 0x4c, 0x3b, 0x95, 0xfb, 0x90, 0x29, 0x45, 0x13, 0x15, 0x21,
0x0d, 0x9b, 0x04, 0xb5, 0xab, 0x2e, 0xa9, 0x86, 0xc5, 0x88, 0x47, 0x4b, 0x5a, 0x36, 0x6c, 0x22,
0x27, 0xbe, 0x8d, 0xdc, 0x26, 0x76, 0x64, 0x3b, 0xa8, 0x5d, 0xf2, 0x07, 0x7c, 0x16, 0x3b, 0xf6,
0x2c, 0xca, 0xaf, 0xa0, 0x38, 0xaf, 0x91, 0x68, 0x17, 0xb0, 0xf3, 0x3d, 0xf7, 0xf8, 0xfa, 0xf8,
0xdc, 0x83, 0x70, 0x06, 0x86, 0x72, 0x71, 0x29, 0x43, 0x4d, 0x8d, 0x97, 0x2b, 0x69, 0x24, 0xc6,
0x9a, 0x1a, 0x48, 0x53, 0x6e, 0xc0, 0x6b, 0xba, 0x7b, 0x9b, 0x20, 0x62, 0x75, 0x9b, 0x1b, 0x2e,
0x45, 0xc5, 0xda, 0x43, 0x89, 0x4c, 0x64, 0x7d, 0x9e, 0x25, 0x52, 0x26, 0x29, 0xf8, 0xb6, 0x8a,
0x8a, 0x4b, 0xdf, 0xf0, 0x0c, 0xb4, 0xa1, 0x59, 0x5e, 0x13, 0x36, 0x9a, 0x41, 0x55, 0x7d, 0xf0,
0xf3, 0x09, 0x1a, 0x9e, 0x1b, 0x05, 0x34, 0x5b, 0x2e, 0xf0, 0x2e, 0x1a, 0x44, 0x45, 0x7c, 0x0d,
0x86, 0x38, 0xae, 0x33, 0x9f, 0x04, 0x75, 0x85, 0x5f, 0xa3, 0xed, 0xfa, 0x55, 0x60, 0xa1, 0x8c,
0xae, 0x20, 0x36, 0xe1, 0x35, 0xdc, 0x92, 0xc7, 0x96, 0x85, 0xdb, 0xde, 0xa9, 0x6d, 0xbd, 0x83,
0x5b, 0x4c, 0xd0, 0xda, 0x57, 0x50, 0x9a, 0x4b, 0x41, 0x7a, 0xae, 0x33, 0xef, 0x05, 0x4d, 0x89,
0x3f, 0xa3, 0x9d, 0xee, 0x07, 0x61, 0x4e, 0x15, 0xcd, 0xc0, 0x80, 0xd2, 0x64, 0xe2, 0x3a, 0xf3,
0xf1, 0xa1, 0xeb, 0xad, 0xfc, 0xef, 0x6d, 0x7b, 0x3c, 0x6b, 0x79, 0xc1, 0x36, 0xdc, 0x83, 0xe2,
0x25, 0x5a, 0x8f, 0x15, 0x50, 0x3b, 0x94, 0x51, 0x03, 0xa4, 0x6f, 0xc7, 0xed, 0x79, 0x95, 0x21,
0x5e, 0x63, 0x88, 0x77, 0xd1, 0x18, 0x72, 0x32, 0xfc, 0x71, 0x37, 0x7b, 0xf4, 0xfd, 0xf7, 0xcc,
0x09, 0x26, 0xcd, 0xd5, 0x05, 0x35, 0x80, 0x3f, 0xa0, 0x29, 0xdc, 0xe4, 0x5c, 0xad, 0x0c, 0x1b,
0xfc, 0xc3, 0xb0, 0x8d, 0xee, 0xb2, 0x1d, 0xf7, 0x0a, 0x6d, 0x66, 0x45, 0x6a, 0x78, 0x4e, 0x95,
0xa9, 0xcd, 0x23, 0x63, 0xd7, 0x99, 0x0f, 0x83, 0x69, 0x8b, 0x57, 0xc6, 0x61, 0x1f, 0x6d, 0xb5,
0x1b, 0x0f, 0x35, 0x4f, 0x04, 0x35, 0x85, 0x02, 0x32, 0xaa, 0x6c, 0x6e, 0x5b, 0xe7, 0x4d, 0x07,
0x3f, 0x43, 0x23, 0x6d, 0x97, 0x17, 0x72, 0x46, 0x90, 0xa5, 0x0d, 0x2b, 0x60, 0xc9, 0xf0, 0x3e,
0x1a, 0xe5, 0x29, 0x8d, 0x21, 0x03, 0x61, 0xc8, 0xba, 0xeb, 0xcc, 0xfb, 0x41, 0x07, 0xe0, 0x63,
0xf4, 0xb4, 0xd0, 0x10, 0xe6, 0x20, 0x18, 0x17, 0x49, 0x2d, 0x4c, 0x87, 0x86, 0x46, 0x29, 0x90,
0x0d, 0xab, 0x6f, 0xb7, 0xd0, 0x70, 0x56, 0xf5, 0x2b, 0x81, 0xfa, 0xa2, 0xec, 0x96, 0x83, 0xeb,
0x6d, 0x02, 0x23, 0x53, 0x4b, 0xed, 0x80, 0x83, 0x6f, 0x3d, 0x34, 0x3a, 0x87, 0xa4, 0x7c, 0x64,
0xb9, 0xc0, 0xc7, 0xab, 0x0a, 0x1d, 0x6b, 0xe3, 0xbe, 0xf7, 0x77, 0xac, 0xbd, 0x26, 0x83, 0x2b,
0xfa, 0x67, 0x68, 0x6c, 0x3d, 0x13, 0x45, 0x16, 0x81, 0xb2, 0x61, 0xeb, 0x07, 0xa8, 0x84, 0x3e,
0x5a, 0x04, 0x6f, 0xa3, 0x3e, 0x17, 0x0c, 0x6e, 0x6c, 0xc4, 0xfa, 0x41, 0x55, 0xe0, 0x23, 0xb4,
0xae, 0xa4, 0x34, 0x61, 0xce, 0x21, 0x86, 0xf2, 0xd5, 0x32, 0x09, 0x93, 0x93, 0x69, 0xb9, 0xa0,
0x5f, 0x77, 0xb3, 0xb5, 0xb3, 0x12, 0x5f, 0x2e, 0x82, 0x71, 0xc9, 0xaa, 0x0a, 0x86, 0x3f, 0xa1,
0x1d, 0xa9, 0x78, 0xc2, 0x05, 0x4d, 0x43, 0xa9, 0x18, 0xa8, 0x30, 0xe5, 0x19, 0x37, 0x9a, 0x0c,
0xdc, 0xde, 0x7c, 0x7c, 0xf8, 0xbc, 0x13, 0xfa, 0x86, 0x31, 0x05, 0x5a, 0x03, 0x3b, 0x2d, 0x69,
0xef, 0x4b, 0x56, 0xb0, 0xd5, 0xdc, 0xed, 0xb0, 0x7b, 0x12, 0xb9, 0xf6, 0xdf, 0x89, 0x7c, 0x20,
0x17, 0xc3, 0x87, 0x72, 0x71, 0xf2, 0xf2, 0xcb, 0x0b, 0x6d, 0xa4, 0xba, 0xf2, 0xb8, 0xf4, 0xed,
0xc1, 0x6f, 0x49, 0x3e, 0x17, 0x06, 0x94, 0xa0, 0x69, 0x1e, 0x45, 0x03, 0xab, 0xe1, 0xe8, 0x4f,
0x00, 0x00, 0x00, 0xff, 0xff, 0x81, 0xe3, 0xf1, 0xc8, 0x7c, 0x04, 0x00, 0x00,
}

2
satellite/internalpb/metainfo_sat.proto
View File

@ -32,6 +32,8 @@ message StreamID {
// temporary field to determine if we should go with new pending_objects table or
// fallback to pending object in objects table.
bool use_pending_objects_table = 14;
bool versioned = 15;
}
message SegmentID {

7
satellite/metainfo/endpoint_object.go
View File

@ -107,7 +107,7 @@ func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRe
}
// TODO this needs to be optimized to avoid DB call on each request
placement, err := endpoint.buckets.GetBucketPlacement(ctx, req.Bucket, keyInfo.ProjectID)
bucket, err := endpoint.buckets.GetBucket(ctx, req.Bucket, keyInfo.ProjectID)
if err != nil {
if buckets.ErrBucketNotFound.Has(err) {
return nil, rpcstatus.Errorf(rpcstatus.NotFound, "bucket not found: %s", req.Bucket)
@ -174,7 +174,8 @@ func (endpoint *Endpoint) BeginObject(ctx context.Context, req *pb.ObjectBeginRe
StreamId: object.StreamID[:],
MultipartObject: object.FixedSegmentSize <= 0,
EncryptionParameters: req.EncryptionParameters,
Placement: int32(placement),
Placement: int32(bucket.Placement),
Versioned: bucket.Versioning == buckets.VersioningEnabled,
UsePendingObjectsTable: usePendingObjectsTable,
})
@ -295,6 +296,8 @@ func (endpoint *Endpoint) CommitObject(ctx context.Context, req *pb.ObjectCommit
DisallowDelete: !allowDelete,
UsePendingObjectsTable: streamID.UsePendingObjectsTable,
Versioned: streamID.Versioned,
}
// uplink can send empty metadata with not empty key/nonce
// we need to fix it on uplink side but that part will be

56
satellite/metainfo/endpoint_object_test.go
View File

@ -3111,3 +3111,59 @@ func TestNodeTagPlacement(t *testing.T) {
},
)
}
func TestEndpoint_Object_No_StorageNodes_Versioning(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, UplinkCount: 1,
Reconfigure: testplanet.Reconfigure{
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
config.Metainfo.UseBucketLevelObjectVersioning = true
},
},
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
satelliteSys := planet.Satellites[0]
apiKey := planet.Uplinks[0].APIKey[planet.Satellites[0].ID()].SerializeRaw()
projectID := planet.Uplinks[0].Projects[0].ID
bucketName := "versioned-bucket"
objectKey := "versioned-object"
createBucket := func(name string) error {
_, err := satelliteSys.API.Metainfo.Endpoint.CreateBucket(ctx, &pb.CreateBucketRequest{
Header: &pb.RequestHeader{ApiKey: apiKey},
Name: []byte(name),
})
if err != nil {
return err
}
return satelliteSys.API.DB.Buckets().EnableBucketVersioning(ctx, []byte(bucketName), projectID)
}
deleteBucket := func(name string) func() error {
return func() error {
_, err := satelliteSys.API.Metainfo.Endpoint.DeleteBucket(ctx, &pb.DeleteBucketRequest{
Header: &pb.RequestHeader{ApiKey: apiKey},
Name: []byte(name),
DeleteAll: true,
})
return err
}
}
t.Run("object with 2 versions", func(t *testing.T) {
defer ctx.Check(deleteBucket(bucketName))
require.NoError(t, createBucket(bucketName))
err := planet.Uplinks[0].Upload(ctx, satelliteSys, bucketName, objectKey, testrand.Bytes(100))
require.NoError(t, err)
err = planet.Uplinks[0].Upload(ctx, satelliteSys, bucketName, objectKey, testrand.Bytes(100))
require.NoError(t, err)
objects, err := satelliteSys.Metabase.DB.TestingAllObjects(ctx)
require.NoError(t, err)
require.Len(t, objects, 2)
})
})
}

1
satellite/satellitedb/bucketsdb.go
View File

@ -355,6 +355,7 @@ func convertDBXtoBucket(dbxBucket *dbx.BucketMetainfo) (bucket buckets.Bucket, e
CipherSuite: storj.CipherSuite(dbxBucket.DefaultEncryptionCipherSuite),
BlockSize: int32(dbxBucket.DefaultEncryptionBlockSize),
},
Versioning: buckets.Versioning(dbxBucket.Versioning),
}
if dbxBucket.Placement != nil {