From 7be844351d79fd5d2a4c1ba842d83c5465bbace3 Mon Sep 17 00:00:00 2001 From: Michal Niewrzal Date: Fri, 4 Aug 2023 11:27:08 +0200 Subject: [PATCH] satellite/metainfo: remove ServerSideCopyDuplicateMetadata https://github.com/storj/storj/issues/5891 Change-Id: Ib5440169107acca6e832c2280e1ad12dfd380f28 --- satellite/accounting/projectusage_test.go | 2 - satellite/audit/audit_test.go | 7 - satellite/gc/gc_test.go | 8 +- satellite/gracefulexit/gracefulexit_test.go | 8 +- satellite/metabase/copy_object.go | 125 +-- satellite/metabase/copy_object_test.go | 1116 +------------------ satellite/metabase/delete_bucket_test.go | 13 +- satellite/metabase/delete_test.go | 12 +- satellite/metabase/get.go | 56 - satellite/metabase/get_test.go | 102 +- satellite/metabase/list_segments.go | 4 - satellite/metabase/list_segments_test.go | 5 +- satellite/metabase/metabasetest/common.go | 8 - satellite/metabase/metabasetest/create.go | 12 +- satellite/metabase/raw.go | 43 - satellite/metabase/streamstat.go | 2 +- satellite/metabase/streamstat_test.go | 2 +- satellite/metainfo/config.go | 5 +- satellite/metainfo/endpoint_object.go | 4 +- satellite/metainfo/endpoint_object_test.go | 6 - satellite/repair/checker/observer_test.go | 15 +- scripts/testdata/satellite-config.yaml.lock | 3 - 22 files changed, 79 insertions(+), 1479 deletions(-) diff --git a/satellite/accounting/projectusage_test.go b/satellite/accounting/projectusage_test.go index 57aaf22e2..572326699 100644 --- a/satellite/accounting/projectusage_test.go +++ b/satellite/accounting/projectusage_test.go @@ -1161,8 +1161,6 @@ func TestProjectBandwidthUsageWithCopies(t *testing.T) { // this effectively disable live accounting cache config.LiveAccounting.BandwidthCacheTTL = -1 config.LiveAccounting.AsOfSystemInterval = 0 - - config.Metainfo.ServerSideCopyDuplicateMetadata = true }, }, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { diff --git a/satellite/audit/audit_test.go b/satellite/audit/audit_test.go index 4129adbcb..ecc255707 100644 --- a/satellite/audit/audit_test.go +++ b/satellite/audit/audit_test.go @@ -10,7 +10,6 @@ import ( "time" "github.com/stretchr/testify/require" - "go.uber.org/zap" "storj.io/common/memory" "storj.io/common/pb" @@ -18,7 +17,6 @@ import ( "storj.io/common/testcontext" "storj.io/common/testrand" "storj.io/storj/private/testplanet" - "storj.io/storj/satellite" "storj.io/storj/satellite/accounting" "storj.io/storj/satellite/audit" ) @@ -84,11 +82,6 @@ func TestAuditOrderLimit(t *testing.T) { func TestAuditSkipsRemoteCopies(t *testing.T) { testWithRangedLoop(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, - Reconfigure: testplanet.Reconfigure{ - Satellite: func(log *zap.Logger, index int, config *satellite.Config) { - config.Metainfo.ServerSideCopyDuplicateMetadata = true - }, - }, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet, pauseQueueing pauseQueueingFunc, runQueueingOnce runQueueingOnceFunc) { satellite := planet.Satellites[0] audits := satellite.Audit diff --git a/satellite/gc/gc_test.go b/satellite/gc/gc_test.go index 321f7b718..1d38d1673 100644 --- a/satellite/gc/gc_test.go +++ b/satellite/gc/gc_test.go @@ -22,7 +22,6 @@ import ( "storj.io/common/testcontext" "storj.io/common/testrand" "storj.io/storj/private/testplanet" - "storj.io/storj/satellite" "storj.io/storj/satellite/gc/bloomfilter" "storj.io/storj/satellite/metabase" "storj.io/storj/satellite/metabase/rangedloop" @@ -306,12 +305,7 @@ func TestGarbageCollectionWithCopiesWithDuplicateMetadata(t *testing.T) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, Reconfigure: testplanet.Reconfigure{ - Satellite: testplanet.Combine( - testplanet.ReconfigureRS(2, 3, 4, 4), - func(log *zap.Logger, index int, config *satellite.Config) { - config.Metainfo.ServerSideCopyDuplicateMetadata = true - }, - ), + Satellite: testplanet.ReconfigureRS(2, 3, 4, 4), }, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { satellite := planet.Satellites[0] diff --git a/satellite/gracefulexit/gracefulexit_test.go b/satellite/gracefulexit/gracefulexit_test.go index 0da84b367..db6b218fb 100644 --- a/satellite/gracefulexit/gracefulexit_test.go +++ b/satellite/gracefulexit/gracefulexit_test.go @@ -11,7 +11,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/zap" "golang.org/x/exp/slices" "storj.io/common/memory" @@ -235,12 +234,7 @@ func TestGracefulExit_CopiedObjects(t *testing.T) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, Reconfigure: testplanet.Reconfigure{ - Satellite: testplanet.Combine( - testplanet.ReconfigureRS(2, 3, 4, 4), - func(log *zap.Logger, index int, config *satellite.Config) { - config.Metainfo.ServerSideCopyDuplicateMetadata = true - }, - ), + Satellite: testplanet.ReconfigureRS(2, 3, 4, 4), }, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { project, err := planet.Uplinks[0].OpenProject(ctx, planet.Satellites[0]) diff --git a/satellite/metabase/copy_object.go b/satellite/metabase/copy_object.go index fa5b300bc..e76cb6cc3 100644 --- a/satellite/metabase/copy_object.go +++ b/satellite/metabase/copy_object.go @@ -52,10 +52,6 @@ type FinishCopyObject struct { NewSegmentKeys []EncryptedKeyAndNonce - // If set, copy the object by duplicating the metadata and - // remote_alias_pieces list, rather than using segment_copies. - DuplicateMetadata bool - // VerifyLimits holds a callback by which the caller can interrupt the copy // if it turns out completing the copy would exceed a limit. // It will be called only once. @@ -156,8 +152,7 @@ func (db *DB) FinishCopyObject(ctx context.Context, opts FinishCopyObject) (obje redundancySchemes := make([]int64, sourceObject.SegmentCount) - if opts.DuplicateMetadata { - err = withRows(db.db.QueryContext(ctx, ` + err = withRows(db.db.QueryContext(ctx, ` SELECT position, expires_at, @@ -172,75 +167,35 @@ func (db *DB) FinishCopyObject(ctx context.Context, opts FinishCopyObject) (obje ORDER BY position ASC LIMIT $2 `, sourceObject.StreamID, sourceObject.SegmentCount))(func(rows tagsql.Rows) error { - index := 0 - for rows.Next() { - err := rows.Scan( - &positions[index], - &expiresAts[index], - &rootPieceIDs[index], - &encryptedSizes[index], &plainOffsets[index], &plainSizes[index], - &redundancySchemes[index], - &remoteAliasPiecesLists[index], - &placementConstraints[index], - &inlineDatas[index], - ) - if err != nil { - return err - } - index++ - } - - if err := rows.Err(); err != nil { + index := 0 + for rows.Next() { + err := rows.Scan( + &positions[index], + &expiresAts[index], + &rootPieceIDs[index], + &encryptedSizes[index], &plainOffsets[index], &plainSizes[index], + &redundancySchemes[index], + &remoteAliasPiecesLists[index], + &placementConstraints[index], + &inlineDatas[index], + ) + if err != nil { return err } + index++ + } - if index != int(sourceObject.SegmentCount) { - return Error.New("could not load all of the segment information") - } + if err := rows.Err(); err != nil { + return err + } - return nil - }) - } else { - err = withRows(db.db.QueryContext(ctx, ` - SELECT - position, - expires_at, - root_piece_id, - encrypted_size, plain_offset, plain_size, - redundancy, - inline_data - FROM segments - WHERE stream_id = $1 - ORDER BY position ASC - LIMIT $2 - `, sourceObject.StreamID, sourceObject.SegmentCount))(func(rows tagsql.Rows) error { - index := 0 - for rows.Next() { - err := rows.Scan( - &positions[index], - &expiresAts[index], - &rootPieceIDs[index], - &encryptedSizes[index], &plainOffsets[index], &plainSizes[index], - &redundancySchemes[index], - &inlineDatas[index], - ) - if err != nil { - return err - } - index++ - } + if index != int(sourceObject.SegmentCount) { + return Error.New("could not load all of the segment information") + } - if err := rows.Err(); err != nil { - return err - } + return nil + }) - if index != int(sourceObject.SegmentCount) { - return Error.New("could not load all of the segment information") - } - - return nil - }) - } if err != nil { return Error.New("unable to copy object: %w", err) } @@ -351,19 +306,6 @@ func (db *DB) FinishCopyObject(ctx context.Context, opts FinishCopyObject) (obje return nil } - if !opts.DuplicateMetadata { - _, err = tx.ExecContext(ctx, ` - INSERT INTO segment_copies ( - stream_id, ancestor_stream_id - ) VALUES ( - $1, $2 - ) - `, opts.NewStreamID, ancestorStreamID) - if err != nil { - return Error.New("unable to copy object: %w", err) - } - } - return nil }) @@ -387,7 +329,6 @@ func (db *DB) FinishCopyObject(ctx context.Context, opts FinishCopyObject) (obje // Fetch the following in a single query: // - object at copy source location (error if it's not there) -// - source ancestor stream id (if any) // - next version available // - object at copy destination location (if any). func getObjectAtCopySourceAndDestination( @@ -395,7 +336,6 @@ func getObjectAtCopySourceAndDestination( ) (sourceObject Object, ancestorStreamID uuid.UUID, destinationObject *Object, nextAvailableVersion Version, err error) { defer mon.Task()(&ctx)(&err) - var ancestorStreamIDBytes []byte var highestVersion Version sourceObject.ProjectID = opts.ProjectID @@ -422,11 +362,9 @@ func getObjectAtCopySourceAndDestination( encrypted_metadata, total_plain_size, total_encrypted_size, fixed_segment_size, encryption, - segment_copies.ancestor_stream_id, 0, coalesce((SELECT max(version) FROM destination_current_versions),0) AS highest_version FROM objects - LEFT JOIN segment_copies ON objects.stream_id = segment_copies.stream_id WHERE project_id = $1 AND bucket_name = $3 AND @@ -441,7 +379,6 @@ func getObjectAtCopySourceAndDestination( NULL, total_plain_size, total_encrypted_size, fixed_segment_size, encryption, - NULL, version, (SELECT max(version) FROM destination_current_versions) AS highest_version FROM objects @@ -472,7 +409,6 @@ func getObjectAtCopySourceAndDestination( &sourceObject.EncryptedMetadata, &sourceObject.TotalPlainSize, &sourceObject.TotalEncryptedSize, &sourceObject.FixedSegmentSize, encryptionParameters{&sourceObject.Encryption}, - &ancestorStreamIDBytes, &highestVersion, &highestVersion, ) @@ -483,19 +419,7 @@ func getObjectAtCopySourceAndDestination( return Object{}, uuid.UUID{}, nil, 0, ErrObjectNotFound.New("object was changed during copy") } - if len(ancestorStreamIDBytes) != 0 { - // Source object already was a copy, the new copy becomes yet another copy of the existing ancestor - ancestorStreamID, err = uuid.FromBytes(ancestorStreamIDBytes) - if err != nil { - return Object{}, uuid.UUID{}, nil, 0, err - } - } else { - // Source object was not a copy, it will now become an ancestor (unless it has only inline segments) - ancestorStreamID = sourceObject.StreamID - } - if rows.Next() { - var _bogusBytes []byte destinationObject = &Object{} destinationObject.ProjectID = opts.ProjectID destinationObject.BucketName = opts.NewBucket @@ -509,7 +433,6 @@ func getObjectAtCopySourceAndDestination( &destinationObject.EncryptedMetadata, &destinationObject.TotalPlainSize, &destinationObject.TotalEncryptedSize, &destinationObject.FixedSegmentSize, encryptionParameters{&destinationObject.Encryption}, - &_bogusBytes, &destinationObject.Version, &highestVersion, ) diff --git a/satellite/metabase/copy_object_test.go b/satellite/metabase/copy_object_test.go index b03eacf4d..67d40a91d 100644 --- a/satellite/metabase/copy_object_test.go +++ b/satellite/metabase/copy_object_test.go @@ -86,8 +86,6 @@ func TestBeginCopyObject(t *testing.T) { } func TestFinishCopyObject(t *testing.T) { - t.Skip("test will be removed in subsequent change") - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { obj := metabasetest.RandObjectStream() newBucketName := "New bucket name" @@ -483,7 +481,6 @@ func TestFinishCopyObject(t *testing.T) { metabasetest.Verify{ Objects: expectedRawObjects, - Copies: nil, // no copies because we have only inline segments }.Check(ctx, t, db) }) @@ -505,7 +502,7 @@ func TestFinishCopyObject(t *testing.T) { copyObj, expectedOriginalSegments, expectedCopySegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, CopyObjectStream: ©Stream, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) var expectedRawSegments []metabase.RawSegment expectedRawSegments = append(expectedRawSegments, expectedOriginalSegments...) @@ -517,10 +514,6 @@ func TestFinishCopyObject(t *testing.T) { metabase.RawObject(copyObj), }, Segments: expectedRawSegments, - Copies: []metabase.RawCopy{{ - StreamID: copyObj.StreamID, - AncestorStreamID: originalObj.StreamID, - }}, }.Check(ctx, t, db) // TODO find better names @@ -528,7 +521,7 @@ func TestFinishCopyObject(t *testing.T) { copyOfCopyObj, _, expectedCopyOfCopySegments := metabasetest.CreateObjectCopy{ OriginalObject: copyObj, CopyObjectStream: ©OfCopyStream, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) expectedRawSegments = append(expectedRawSegments, expectedCopyOfCopySegments...) @@ -539,13 +532,6 @@ func TestFinishCopyObject(t *testing.T) { metabase.RawObject(copyOfCopyObj), }, Segments: expectedRawSegments, - Copies: []metabase.RawCopy{{ - StreamID: copyStream.StreamID, - AncestorStreamID: originalObj.StreamID, - }, { - StreamID: copyOfCopyObj.StreamID, - AncestorStreamID: originalObj.StreamID, - }}, }.Check(ctx, t, db) }) @@ -592,7 +578,7 @@ func TestFinishCopyObject(t *testing.T) { NewEncryptedMetadataKeyNonce: newMetadataKeyNonce, NewEncryptedMetadataKey: newMetadataKey, }, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) require.Equal(t, originalMetadata, copyObjNoOverride.EncryptedMetadata) require.Equal(t, newMetadataKey, copyObjNoOverride.EncryptedMetadataEncryptedKey) @@ -616,7 +602,7 @@ func TestFinishCopyObject(t *testing.T) { NewEncryptedMetadataKeyNonce: newMetadataKeyNonce, NewEncryptedMetadataKey: newMetadataKey, }, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) require.Equal(t, newMetadata, copyObj.EncryptedMetadata) require.Equal(t, newMetadataKey, copyObj.EncryptedMetadataEncryptedKey) @@ -628,7 +614,6 @@ func TestFinishCopyObject(t *testing.T) { metabase.RawObject(copyObj), metabase.RawObject(copyObjNoOverride), }, - Copies: nil, }.Check(ctx, t, db) }) @@ -687,7 +672,7 @@ func TestFinishCopyObject(t *testing.T) { objBprime, expectedSegmentsOfB, expectedSegmentsOfBprime := metabasetest.CreateObjectCopy{ OriginalObject: objB, CopyObjectStream: &objStreamBprime, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) // check that we indeed overwrote object A require.Equal(t, objA.BucketName, objBprime.BucketName) @@ -708,10 +693,6 @@ func TestFinishCopyObject(t *testing.T) { metabase.RawObject(objC), }, Segments: expectedRawSegments, - Copies: []metabase.RawCopy{{ - StreamID: objBprime.StreamID, - AncestorStreamID: objB.StreamID, - }}, }.Check(ctx, t, db) // C' is a copy of C to B @@ -720,7 +701,7 @@ func TestFinishCopyObject(t *testing.T) { objCprime, _, expectedSegmentsOfCprime := metabasetest.CreateObjectCopy{ OriginalObject: objC, CopyObjectStream: &objStreamCprime, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) require.Equal(t, objStreamB.BucketName, objCprime.BucketName) require.Equal(t, objStreamB.ProjectID, objCprime.ProjectID) @@ -745,10 +726,6 @@ func TestFinishCopyObject(t *testing.T) { metabase.RawObject(objC), }, Segments: expectedSegments, - Copies: []metabase.RawCopy{{ - StreamID: objCprime.StreamID, - AncestorStreamID: objC.StreamID, - }}, }.Check(ctx, t, db) }) @@ -775,7 +752,7 @@ func TestFinishCopyObject(t *testing.T) { copyObj, _, copySegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, CopyObjectStream: ©ObjStream, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) // Copy the copy back to the source location opts := metabase.FinishCopyObject{ @@ -797,7 +774,7 @@ func TestFinishCopyObject(t *testing.T) { OriginalObject: copyObj, CopyObjectStream: ©BackObjStream, FinishObject: &opts, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) // expected object at the location which was previously the original object copyBackObj := originalObj @@ -810,7 +787,6 @@ func TestFinishCopyObject(t *testing.T) { copySegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy originalSegments[i].StreamID = opts.NewStreamID - originalSegments[i].Pieces = nil originalSegments[i].InlineData = nil originalSegments[i].EncryptedKey = opts.NewSegmentKeys[i].EncryptedKey originalSegments[i].EncryptedKeyNonce = opts.NewSegmentKeys[i].EncryptedKeyNonce @@ -823,10 +799,6 @@ func TestFinishCopyObject(t *testing.T) { metabase.RawObject(copyBackObj), }, Segments: append(metabasetest.SegmentsToRaw(originalSegments), copySegments...), - Copies: []metabase.RawCopy{{ - StreamID: opts.NewStreamID, - AncestorStreamID: copyObjStream.StreamID, - }}, }.Check(ctx, t, db) }) @@ -853,7 +825,7 @@ func TestFinishCopyObject(t *testing.T) { copyObj, originalSegments, copySegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, CopyObjectStream: ©ObjStream, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) opts := metabase.FinishCopyObject{ // source @@ -875,7 +847,7 @@ func TestFinishCopyObject(t *testing.T) { OriginalObject: originalObj, CopyObjectStream: ©BackObjStream, FinishObject: &opts, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) copyBackObj := originalObj copyBackObj.Version++ // copy is placed into first available version @@ -887,7 +859,6 @@ func TestFinishCopyObject(t *testing.T) { copySegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy originalSegments[i].StreamID = opts.NewStreamID - originalSegments[i].Pieces = nil originalSegments[i].InlineData = nil originalSegments[i].EncryptedKey = opts.NewSegmentKeys[i].EncryptedKey originalSegments[i].EncryptedKeyNonce = opts.NewSegmentKeys[i].EncryptedKeyNonce @@ -900,10 +871,6 @@ func TestFinishCopyObject(t *testing.T) { metabase.RawObject(copyBackObj), }, Segments: append(originalSegments, copySegments...), - Copies: []metabase.RawCopy{{ - StreamID: copyBackObjStream.StreamID, - AncestorStreamID: copyObjStream.StreamID, - }}, }.Check(ctx, t, db) }) @@ -981,10 +948,6 @@ func TestFinishCopyObject(t *testing.T) { metabase.RawObject(copyObj), }, Segments: metabasetest.SegmentsToRaw(listSegments), - Copies: []metabase.RawCopy{{ - StreamID: copyObj.StreamID, - AncestorStreamID: originalObj.StreamID, - }}, }.Check(ctx, t, db) }) @@ -1006,14 +969,13 @@ func TestFinishCopyObject(t *testing.T) { _, expectedOriginalSegments, _ := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, CopyObjectStream: &obj, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) metabasetest.Verify{ Objects: []metabase.RawObject{ metabase.RawObject(originalObj), }, Segments: expectedOriginalSegments, - Copies: []metabase.RawCopy{}, }.Check(ctx, t, db) }) @@ -1141,7 +1103,7 @@ func TestFinishCopyObject(t *testing.T) { copyObj, expectedOriginalSegments, _ := metabasetest.CreateObjectCopy{ OriginalObject: sourceObj, CopyObjectStream: &destinationObjStream, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) require.Equal(t, tc.expectedCopyVersion, copyObj.Version) @@ -1150,1060 +1112,6 @@ func TestFinishCopyObject(t *testing.T) { metabasetest.Verify{ Objects: rawObjects, Segments: expectedOriginalSegments, - Copies: []metabase.RawCopy{}, - }.Check(ctx, t, db) - } - }) - }) -} - -func TestFinishCopyObjectWithDuplicateMetadata(t *testing.T) { - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - obj := metabasetest.RandObjectStream() - newBucketName := "New bucket name" - - newStreamID := testrand.UUID() - for _, test := range metabasetest.InvalidObjectStreams(obj) { - test := test - t.Run(test.Name, func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - ObjectStream: test.ObjectStream, - NewStreamID: newStreamID, - DuplicateMetadata: true, - }, - ErrClass: test.ErrClass, - ErrText: test.ErrText, - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - } - - t.Run("invalid NewBucket", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - ObjectStream: obj, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewEncryptedMetadataKey: []byte{1, 2, 3}, - NewEncryptedMetadataKeyNonce: testrand.Nonce(), - NewStreamID: newStreamID, - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "NewBucket is missing", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("invalid NewStreamID", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - ObjectStream: obj, - NewBucket: newBucketName, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewEncryptedMetadataKey: []byte{1, 2, 3}, - NewEncryptedMetadataKeyNonce: testrand.Nonce(), - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "NewStreamID is missing", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - t.Run("copy to the same StreamID", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - ObjectStream: obj, - NewBucket: newBucketName, - NewStreamID: obj.StreamID, - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "StreamIDs are identical", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("invalid NewEncryptedObjectKey", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - ObjectStream: obj, - NewStreamID: newStreamID, - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "NewEncryptedObjectKey is missing", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("invalid EncryptedMetadataKeyNonce", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - ObjectStream: obj, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewStreamID: newStreamID, - NewEncryptedMetadataKey: []byte{0}, - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "EncryptedMetadataKeyNonce is missing", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("invalid EncryptedMetadataKey", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - ObjectStream: obj, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewEncryptedMetadataKeyNonce: testrand.Nonce(), - NewStreamID: newStreamID, - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "EncryptedMetadataKey is missing", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("empty EncryptedMetadataKey and EncryptedMetadataKeyNonce", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - ObjectStream: obj, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewStreamID: newStreamID, - DuplicateMetadata: true, - }, - // validation pass without EncryptedMetadataKey and EncryptedMetadataKeyNonce - ErrClass: &metabase.ErrObjectNotFound, - ErrText: "source object not found", - }.Check(ctx, t, db) - }) - - t.Run("empty EncryptedMetadata with OverrideMetadata=true", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - ObjectStream: obj, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - - OverrideMetadata: true, - NewEncryptedMetadataKey: []byte{1}, - NewEncryptedMetadataKeyNonce: testrand.Nonce(), - NewStreamID: newStreamID, - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be not set if EncryptedMetadata is not set", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("empty NewEncryptedMetadataKey and NewEncryptedMetadataKeyNonce with OverrideMetadata=true", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - ObjectStream: obj, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewStreamID: newStreamID, - - OverrideMetadata: true, - NewEncryptedMetadata: testrand.BytesInt(256), - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be set if EncryptedMetadata is set", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - t.Run("object does not exist", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - newObj := metabasetest.RandObjectStream() - - newEncryptedMetadataKeyNonce := testrand.Nonce() - newEncryptedMetadataKey := testrand.Bytes(32) - newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, 10) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - NewStreamID: newStreamID, - ObjectStream: newObj, - NewSegmentKeys: newEncryptedKeysNonces, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce, - NewEncryptedMetadataKey: newEncryptedMetadataKey, - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrObjectNotFound, - ErrText: "source object not found", - }.Check(ctx, t, db) - - metabasetest.Verify{}.Check(ctx, t, db) - }) - - // Assert that an error occurs when a new object has been put at the source key - // between BeginCopyObject and FinishCopyObject. (stream_id of source key changed) - t.Run("source object changed", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - newObj, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: obj, - OverrideEncryptedMetadata: true, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, obj, 2) - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewStreamID: testrand.UUID(), - NewBucket: newBucketName, - ObjectStream: metabase.ObjectStream{ - ProjectID: newObj.ProjectID, - BucketName: newObj.BucketName, - ObjectKey: newObj.ObjectKey, - Version: newObj.Version, - StreamID: testrand.UUID(), - }, - NewSegmentKeys: []metabase.EncryptedKeyAndNonce{ - metabasetest.RandEncryptedKeyAndNonce(0), - metabasetest.RandEncryptedKeyAndNonce(1), - }, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewEncryptedMetadataKeyNonce: testrand.Nonce(), - NewEncryptedMetadataKey: testrand.Bytes(32), - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrObjectNotFound, - ErrText: "object was changed during copy", - }.Check(ctx, t, db) - }) - - t.Run("not enough segments", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - numberOfSegments := 10 - - newObj, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: obj, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, obj, byte(numberOfSegments)) - - newEncryptedMetadataKeyNonce := testrand.Nonce() - newEncryptedMetadataKey := testrand.Bytes(32) - newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount-1) - expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount) - - for i := 0; i < int(newObj.SegmentCount-1); i++ { - newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{ - Position: metabase.SegmentPosition{Index: uint32(i)}, - EncryptedKeyNonce: testrand.Nonce().Bytes(), - EncryptedKey: testrand.Bytes(32), - } - - expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)}) - expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce - expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey - expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize) - expectedSegments[i].EncryptedSize = int32(0) - } - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewBucket: newBucketName, - ObjectStream: obj, - NewStreamID: newStreamID, - NewSegmentKeys: newEncryptedKeysNonces, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce, - NewEncryptedMetadataKey: newEncryptedMetadataKey, - DuplicateMetadata: true, - }, - ErrClass: &metabase.ErrInvalidRequest, - ErrText: "wrong number of segments keys received (received 9, need 10)", - }.Check(ctx, t, db) - }) - - t.Run("wrong segment indexes", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - numberOfSegments := 10 - - newObj, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: obj, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, obj, byte(numberOfSegments)) - - newEncryptedMetadataKeyNonce := testrand.Nonce() - newEncryptedMetadataKey := testrand.Bytes(32) - newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount) - expectedEncryptedSize := 1060 - expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount) - - for i := 0; i < int(newObj.SegmentCount); i++ { - newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{ - Position: metabase.SegmentPosition{Index: uint32(i + 5)}, - EncryptedKeyNonce: testrand.Nonce().Bytes(), - EncryptedKey: testrand.Bytes(32), - } - - expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)}) - expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce - expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey - expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize) - expectedSegments[i].EncryptedSize = int32(expectedEncryptedSize) - } - - metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - NewStreamID: newStreamID, - NewBucket: newBucketName, - ObjectStream: obj, - NewSegmentKeys: newEncryptedKeysNonces, - NewEncryptedObjectKey: metabasetest.RandObjectKey(), - NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce, - NewEncryptedMetadataKey: newEncryptedMetadataKey, - DuplicateMetadata: true, - }, - ErrClass: &metabase.Error, - ErrText: "missing new segment keys for segment 0", - }.Check(ctx, t, db) - }) - - t.Run("returned object", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - expectedRawObjects := []metabase.RawObject{} - - for _, expectedVersion := range []metabase.Version{1} { - objStream := metabasetest.RandObjectStream() - objStream.Version = expectedVersion - - copyStream := metabasetest.RandObjectStream() - copyStream.ProjectID = objStream.ProjectID - copyStream.BucketName = objStream.BucketName - - originalObj, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: objStream, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, objStream, 0) - - metadataNonce := testrand.Nonce() - expectedCopyObject := originalObj - expectedCopyObject.ObjectKey = copyStream.ObjectKey - expectedCopyObject.StreamID = copyStream.StreamID - expectedCopyObject.Version = metabase.DefaultVersion // it will always copy into first available version - expectedCopyObject.EncryptedMetadataEncryptedKey = testrand.Bytes(32) - expectedCopyObject.EncryptedMetadataNonce = metadataNonce.Bytes() - - objectCopy := metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - ObjectStream: objStream, - NewBucket: copyStream.BucketName, - NewStreamID: copyStream.StreamID, - NewEncryptedObjectKey: copyStream.ObjectKey, - NewEncryptedMetadataKey: expectedCopyObject.EncryptedMetadataEncryptedKey, - NewEncryptedMetadataKeyNonce: metadataNonce, - DuplicateMetadata: true, - }, - Result: expectedCopyObject, - }.Check(ctx, t, db) - - require.NotEqual(t, originalObj.CreatedAt, objectCopy.CreatedAt) - - expectedRawObjects = append(expectedRawObjects, metabase.RawObject(originalObj)) - expectedRawObjects = append(expectedRawObjects, metabase.RawObject(expectedCopyObject)) - } - - metabasetest.Verify{ - Objects: expectedRawObjects, - Copies: nil, // no copies because we have only inline segments - }.Check(ctx, t, db) - }) - - t.Run("finish copy object with existing metadata", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - numberOfSegments := 10 - copyStream := metabasetest.RandObjectStream() - - originalObj, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: obj, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, obj, byte(numberOfSegments)) - - copyObj, expectedOriginalSegments, expectedCopySegments := metabasetest.CreateObjectCopy{ - OriginalObject: originalObj, - CopyObjectStream: ©Stream, - }.Run(ctx, t, db, true) - - var expectedRawSegments []metabase.RawSegment - expectedRawSegments = append(expectedRawSegments, expectedOriginalSegments...) - expectedRawSegments = append(expectedRawSegments, expectedCopySegments...) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(originalObj), - metabase.RawObject(copyObj), - }, - Segments: expectedRawSegments, - }.Check(ctx, t, db) - - // TODO find better names - copyOfCopyStream := metabasetest.RandObjectStream() - copyOfCopyObj, _, expectedCopyOfCopySegments := metabasetest.CreateObjectCopy{ - OriginalObject: copyObj, - CopyObjectStream: ©OfCopyStream, - }.Run(ctx, t, db, true) - - expectedRawSegments = append(expectedRawSegments, expectedCopyOfCopySegments...) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(originalObj), - metabase.RawObject(copyObj), - metabase.RawObject(copyOfCopyObj), - }, - Segments: expectedRawSegments, - }.Check(ctx, t, db) - }) - - t.Run("finish copy object with new metadata", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - copyStream := metabasetest.RandObjectStream() - copyStreamNoOverride := metabasetest.RandObjectStream() - - originalMetadata := testrand.Bytes(64) - originalMetadataNonce := testrand.Nonce().Bytes() - originalMetadataEncryptedKey := testrand.Bytes(265) - - originalObj, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: obj, - OverrideEncryptedMetadata: true, - EncryptedMetadata: originalMetadata, - EncryptedMetadataNonce: originalMetadataNonce, - EncryptedMetadataEncryptedKey: originalMetadataEncryptedKey, - }, - }.Run(ctx, t, db, obj, 0) - - newMetadata := testrand.Bytes(256) - newMetadataKey := testrand.Bytes(32) - newMetadataKeyNonce := testrand.Nonce() - - // do a copy without OverrideMetadata field set to true, - // metadata shouldn't be updated even if NewEncryptedMetadata - // field is set - copyObjNoOverride, _, _ := metabasetest.CreateObjectCopy{ - OriginalObject: originalObj, - CopyObjectStream: ©StreamNoOverride, - FinishObject: &metabase.FinishCopyObject{ - ObjectStream: originalObj.ObjectStream, - - NewBucket: copyStreamNoOverride.BucketName, - NewStreamID: copyStreamNoOverride.StreamID, - - NewEncryptedObjectKey: copyStreamNoOverride.ObjectKey, - - OverrideMetadata: false, - NewEncryptedMetadata: newMetadata, - NewEncryptedMetadataKeyNonce: newMetadataKeyNonce, - NewEncryptedMetadataKey: newMetadataKey, - }, - }.Run(ctx, t, db, true) - - require.Equal(t, originalMetadata, copyObjNoOverride.EncryptedMetadata) - require.Equal(t, newMetadataKey, copyObjNoOverride.EncryptedMetadataEncryptedKey) - require.Equal(t, newMetadataKeyNonce.Bytes(), copyObjNoOverride.EncryptedMetadataNonce) - - // do a copy WITH OverrideMetadata field set to true, - // metadata should be updated to NewEncryptedMetadata - copyObj, _, _ := metabasetest.CreateObjectCopy{ - OriginalObject: originalObj, - CopyObjectStream: ©Stream, - FinishObject: &metabase.FinishCopyObject{ - ObjectStream: originalObj.ObjectStream, - - NewBucket: copyStream.BucketName, - NewStreamID: copyStream.StreamID, - - NewEncryptedObjectKey: copyStream.ObjectKey, - - OverrideMetadata: true, - NewEncryptedMetadata: newMetadata, - NewEncryptedMetadataKeyNonce: newMetadataKeyNonce, - NewEncryptedMetadataKey: newMetadataKey, - }, - }.Run(ctx, t, db, true) - - require.Equal(t, newMetadata, copyObj.EncryptedMetadata) - require.Equal(t, newMetadataKey, copyObj.EncryptedMetadataEncryptedKey) - require.Equal(t, newMetadataKeyNonce.Bytes(), copyObj.EncryptedMetadataNonce) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(originalObj), - metabase.RawObject(copyObj), - metabase.RawObject(copyObjNoOverride), - }, - Copies: nil, - }.Check(ctx, t, db) - }) - - t.Run("finish copy object to already existing destination", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - // Test: - // - 3 objects: objA, objB, objC - // - copy objB to objA - creating objBprime - // - check that segments of original objA have been deleted - // - check that we now have three objects: objBprime, objB, objC - // - copy objC to objB creating objCprime - // - check that we now have three objects: objBprime, objCprime, objC - // - check that objBprime has become an original object, now that its ancestor - // objB has been overwritten - - // object that already exists - objStreamA := metabasetest.RandObjectStream() - objStreamB := metabasetest.RandObjectStream() - objStreamC := metabasetest.RandObjectStream() - - // set same projectID for all - objStreamB.ProjectID = objStreamA.ProjectID - objStreamC.ProjectID = objStreamA.ProjectID - - objA, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: objStreamA, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, objStreamA, 4) - - objB, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: objStreamB, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, objStreamB, 3) - - objC, segmentsOfC := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: objStreamC, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, objStreamC, 1) - - // B' is a copy of B to A - objStreamBprime := objStreamA - objStreamBprime.StreamID = testrand.UUID() - objBprime, expectedSegmentsOfB, expectedSegmentsOfBprime := metabasetest.CreateObjectCopy{ - OriginalObject: objB, - CopyObjectStream: &objStreamBprime, - }.Run(ctx, t, db, true) - - // check that we indeed overwrote object A - require.Equal(t, objA.BucketName, objBprime.BucketName) - require.Equal(t, objA.ProjectID, objBprime.ProjectID) - require.Equal(t, objA.ObjectKey, objBprime.ObjectKey) - - require.NotEqual(t, objA.StreamID, objBprime.StreamID) - - var expectedRawSegments []metabase.RawSegment - expectedRawSegments = append(expectedRawSegments, expectedSegmentsOfBprime...) - expectedRawSegments = append(expectedRawSegments, expectedSegmentsOfB...) - expectedRawSegments = append(expectedRawSegments, metabasetest.SegmentsToRaw(segmentsOfC)...) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(objBprime), - metabase.RawObject(objB), - metabase.RawObject(objC), - }, - Segments: expectedRawSegments, - }.Check(ctx, t, db) - - // C' is a copy of C to B - objStreamCprime := objStreamB - objStreamCprime.StreamID = testrand.UUID() - objCprime, _, expectedSegmentsOfCprime := metabasetest.CreateObjectCopy{ - OriginalObject: objC, - CopyObjectStream: &objStreamCprime, - }.Run(ctx, t, db, true) - - require.Equal(t, objStreamB.BucketName, objCprime.BucketName) - require.Equal(t, objStreamB.ProjectID, objCprime.ProjectID) - require.Equal(t, objStreamB.ObjectKey, objCprime.ObjectKey) - require.NotEqual(t, objB.StreamID, objCprime) - - // B' should become the original of B and now hold pieces. - for i := range expectedSegmentsOfBprime { - expectedSegmentsOfBprime[i].EncryptedETag = nil - expectedSegmentsOfBprime[i].Pieces = expectedSegmentsOfB[i].Pieces - } - - var expectedSegments []metabase.RawSegment - expectedSegments = append(expectedSegments, expectedSegmentsOfBprime...) - expectedSegments = append(expectedSegments, expectedSegmentsOfCprime...) - expectedSegments = append(expectedSegments, metabasetest.SegmentsToRaw(segmentsOfC)...) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(objBprime), - metabase.RawObject(objCprime), - metabase.RawObject(objC), - }, - Segments: expectedSegments, - }.Check(ctx, t, db) - }) - - // checks that a copy can be copied to it's ancestor location - t.Run("Copy child to ancestor", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - originalObjStream := metabasetest.RandObjectStream() - copyObjStream := metabasetest.RandObjectStream() - // Copy back to original object key. - // StreamID is independent of key. - copyBackObjStream := originalObjStream - copyBackObjStream.StreamID = testrand.UUID() - - originalObj, originalSegments := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: originalObjStream, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, originalObjStream, 4) - - copyObj, _, copySegments := metabasetest.CreateObjectCopy{ - OriginalObject: originalObj, - CopyObjectStream: ©ObjStream, - }.Run(ctx, t, db, true) - - // Copy the copy back to the source location - opts := metabase.FinishCopyObject{ - // source - ObjectStream: copyObj.ObjectStream, - // destination - NewBucket: originalObj.BucketName, - NewEncryptedObjectKey: originalObj.ObjectKey, - NewStreamID: copyBackObjStream.StreamID, - OverrideMetadata: false, - NewSegmentKeys: []metabase.EncryptedKeyAndNonce{ - metabasetest.RandEncryptedKeyAndNonce(0), - metabasetest.RandEncryptedKeyAndNonce(1), - metabasetest.RandEncryptedKeyAndNonce(2), - metabasetest.RandEncryptedKeyAndNonce(3), - }, - } - metabasetest.CreateObjectCopy{ - OriginalObject: copyObj, - CopyObjectStream: ©BackObjStream, - FinishObject: &opts, - }.Run(ctx, t, db, true) - - // expected object at the location which was previously the original object - copyBackObj := originalObj - copyBackObj.Version++ // copy is placed into first available version - copyBackObj.StreamID = opts.NewStreamID - - for i := 0; i < 4; i++ { - copySegments[i].Pieces = originalSegments[i].Pieces - copySegments[i].InlineData = originalSegments[i].InlineData - copySegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy - - originalSegments[i].StreamID = opts.NewStreamID - originalSegments[i].InlineData = nil - originalSegments[i].EncryptedKey = opts.NewSegmentKeys[i].EncryptedKey - originalSegments[i].EncryptedKeyNonce = opts.NewSegmentKeys[i].EncryptedKeyNonce - originalSegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy - } - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(copyObj), - metabase.RawObject(copyBackObj), - }, - Segments: append(metabasetest.SegmentsToRaw(originalSegments), copySegments...), - }.Check(ctx, t, db) - }) - - // checks that a copy ancestor can be copied to itself - t.Run("Copy ancestor to itself", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - originalObjStream := metabasetest.RandObjectStream() - copyObjStream := metabasetest.RandObjectStream() - // Copy back to same object key. - // StreamID is independent of key. - copyBackObjStream := originalObjStream - copyBackObjStream.StreamID = testrand.UUID() - - originalObj, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: originalObjStream, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, originalObjStream, 4) - - copyObj, originalSegments, copySegments := metabasetest.CreateObjectCopy{ - OriginalObject: originalObj, - CopyObjectStream: ©ObjStream, - }.Run(ctx, t, db, true) - - opts := metabase.FinishCopyObject{ - // source - ObjectStream: copyObj.ObjectStream, - // destination - NewBucket: originalObj.BucketName, - NewEncryptedObjectKey: originalObj.ObjectKey, - NewStreamID: copyBackObjStream.StreamID, - OverrideMetadata: false, - NewSegmentKeys: []metabase.EncryptedKeyAndNonce{ - metabasetest.RandEncryptedKeyAndNonce(0), - metabasetest.RandEncryptedKeyAndNonce(1), - metabasetest.RandEncryptedKeyAndNonce(2), - metabasetest.RandEncryptedKeyAndNonce(3), - }, - } - // Copy the copy back to the source location - metabasetest.CreateObjectCopy{ - OriginalObject: originalObj, - CopyObjectStream: ©BackObjStream, - FinishObject: &opts, - }.Run(ctx, t, db, true) - - copyBackObj := originalObj - copyBackObj.Version++ // copy is placed into first available version - copyBackObj.StreamID = copyBackObjStream.StreamID - - for i := 0; i < 4; i++ { - copySegments[i].Pieces = originalSegments[i].Pieces - copySegments[i].InlineData = originalSegments[i].InlineData - copySegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy - - originalSegments[i].StreamID = opts.NewStreamID - originalSegments[i].InlineData = nil - originalSegments[i].EncryptedKey = opts.NewSegmentKeys[i].EncryptedKey - originalSegments[i].EncryptedKeyNonce = opts.NewSegmentKeys[i].EncryptedKeyNonce - originalSegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy - } - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(copyObj), - metabase.RawObject(copyBackObj), - }, - Segments: append(originalSegments, copySegments...), - }.Check(ctx, t, db) - }) - - t.Run("copied segments has same expires_at as original", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - expiresAt := time.Now().Add(2 * time.Hour) - - objStream := metabasetest.RandObjectStream() - copyStream := metabasetest.RandObjectStream() - copyStream.ProjectID = objStream.ProjectID - copyStream.BucketName = objStream.BucketName - - originalObj := metabasetest.CreateExpiredObject(ctx, t, db, objStream, 10, expiresAt) - - metadataNonce := testrand.Nonce() - expectedCopyObject := originalObj - expectedCopyObject.ObjectKey = copyStream.ObjectKey - expectedCopyObject.StreamID = copyStream.StreamID - expectedCopyObject.EncryptedMetadataEncryptedKey = testrand.Bytes(32) - expectedCopyObject.EncryptedMetadataNonce = metadataNonce.Bytes() - - newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, originalObj.SegmentCount) - expectedSegments := make([]metabase.RawSegment, originalObj.SegmentCount) - - for i := 0; i < int(originalObj.SegmentCount); i++ { - newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{ - Position: metabase.SegmentPosition{Index: uint32(i)}, - EncryptedKeyNonce: testrand.Nonce().Bytes(), - EncryptedKey: testrand.Bytes(32), - } - - expectedSegments[i] = metabasetest.DefaultRawSegment(originalObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)}) - expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce - expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey - expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize) - expectedSegments[i].EncryptedSize = int32(0) - } - - copyObj := metabasetest.FinishCopyObject{ - Opts: metabase.FinishCopyObject{ - ObjectStream: objStream, - NewBucket: copyStream.BucketName, - NewStreamID: copyStream.StreamID, - NewEncryptedObjectKey: copyStream.ObjectKey, - NewEncryptedMetadataKey: expectedCopyObject.EncryptedMetadataEncryptedKey, - NewEncryptedMetadataKeyNonce: metadataNonce, - NewSegmentKeys: newEncryptedKeysNonces, - DuplicateMetadata: true, - }, - Result: expectedCopyObject, - }.Check(ctx, t, db) - - var listSegments []metabase.Segment - - copiedSegments, err := db.ListSegments(ctx, metabase.ListSegments{ - StreamID: copyObj.StreamID, - }) - require.NoError(t, err) - - originalSegments, err := db.ListSegments(ctx, metabase.ListSegments{ - StreamID: originalObj.StreamID, - }) - require.NoError(t, err) - - listSegments = append(listSegments, originalSegments.Segments...) - listSegments = append(listSegments, copiedSegments.Segments...) - - for _, v := range listSegments { - require.Equal(t, expiresAt.Unix(), v.ExpiresAt.Unix()) - } - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(originalObj), - metabase.RawObject(copyObj), - }, - Segments: metabasetest.SegmentsToRaw(listSegments), - }.Check(ctx, t, db) - }) - - t.Run("finish copy object to same destination", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - obj := metabasetest.RandObjectStream() - numberOfSegments := 10 - originalObj, _ := metabasetest.CreateTestObject{ - CommitObject: &metabase.CommitObject{ - ObjectStream: obj, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, obj, byte(numberOfSegments)) - - obj.StreamID = testrand.UUID() - _, expectedOriginalSegments, _ := metabasetest.CreateObjectCopy{ - OriginalObject: originalObj, - CopyObjectStream: &obj, - }.Run(ctx, t, db, true) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - metabase.RawObject(originalObj), - }, - Segments: expectedOriginalSegments, - Copies: []metabase.RawCopy{}, - }.Check(ctx, t, db) - }) - - t.Run("finish copy object to existing pending destination", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - sourceObjStream := metabasetest.RandObjectStream() - destinationObjStream := metabasetest.RandObjectStream() - destinationObjStream.ProjectID = sourceObjStream.ProjectID - // testcases: - // - versions of pending objects - // - version of committed object - // - expected copy version - - testCases := []struct { - Bucket string - Key metabase.ObjectKey - NewBucket string - NewKey metabase.ObjectKey - sourcePendingVersions []metabase.Version - sourceCommittedVersion metabase.Version - destinationPendingVersions []metabase.Version - destinationCommittedVersion metabase.Version - expectedCopyVersion metabase.Version - }{ - // the same bucket - {"testbucket", "object", "testbucket", "new-object", - []metabase.Version{}, 2, - []metabase.Version{}, 1, - 2}, - {"testbucket", "object", "testbucket", "new-object", - []metabase.Version{}, 1, - []metabase.Version{1}, 2, - 3}, - {"testbucket", "object", "testbucket", "new-object", - []metabase.Version{}, 1, - []metabase.Version{1, 3}, 2, - 4}, - {"testbucket", "object", "testbucket", "new-object", - []metabase.Version{1, 5}, 2, - []metabase.Version{1, 3}, 2, - 4}, - {"testbucket", "object", "newbucket", "object", - []metabase.Version{2, 3}, 1, - []metabase.Version{1, 5}, 2, - 6}, - } - - for _, tc := range testCases { - metabasetest.DeleteAll{}.Check(ctx, t, db) - sourceObjStream.BucketName = tc.Bucket - sourceObjStream.ObjectKey = tc.Key - destinationObjStream.BucketName = tc.NewBucket - destinationObjStream.ObjectKey = tc.NewKey - - var rawObjects []metabase.RawObject - for _, version := range tc.sourcePendingVersions { - sourceObjStream.Version = version - sourceObjStream.StreamID = testrand.UUID() - metabasetest.CreatePendingObject(ctx, t, db, sourceObjStream, 0) - - rawObjects = append(rawObjects, metabase.RawObject{ - ObjectStream: sourceObjStream, - CreatedAt: now, - Status: metabase.Pending, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }) - } - sourceObjStream.Version = tc.sourceCommittedVersion - sourceObjStream.StreamID = testrand.UUID() - sourceObj, _ := metabasetest.CreateTestObject{ - BeginObjectExactVersion: &metabase.BeginObjectExactVersion{ - ObjectStream: sourceObjStream, - Encryption: metabasetest.DefaultEncryption, - }, - CommitObject: &metabase.CommitObject{ - ObjectStream: sourceObjStream, - OverrideEncryptedMetadata: true, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, sourceObjStream, 0) - - rawObjects = append(rawObjects, metabase.RawObject(sourceObj)) - - for _, version := range tc.destinationPendingVersions { - destinationObjStream.Version = version - destinationObjStream.StreamID = testrand.UUID() - metabasetest.CreatePendingObject(ctx, t, db, destinationObjStream, 0) - - rawObjects = append(rawObjects, metabase.RawObject{ - ObjectStream: destinationObjStream, - CreatedAt: now, - Status: metabase.Pending, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }) - } - - if tc.destinationCommittedVersion != 0 { - destinationObjStream.StreamID = testrand.UUID() - destinationObjStream.Version = tc.destinationCommittedVersion - _, _ = metabasetest.CreateTestObject{ - BeginObjectExactVersion: &metabase.BeginObjectExactVersion{ - ObjectStream: destinationObjStream, - Encryption: metabasetest.DefaultEncryption, - }, - CommitObject: &metabase.CommitObject{ - ObjectStream: destinationObjStream, - OverrideEncryptedMetadata: true, - EncryptedMetadata: testrand.Bytes(64), - EncryptedMetadataNonce: testrand.Nonce().Bytes(), - EncryptedMetadataEncryptedKey: testrand.Bytes(265), - }, - }.Run(ctx, t, db, destinationObjStream, 0) - } - - copyObj, expectedOriginalSegments, _ := metabasetest.CreateObjectCopy{ - OriginalObject: sourceObj, - CopyObjectStream: &destinationObjStream, - }.Run(ctx, t, db, true) - - require.Equal(t, tc.expectedCopyVersion, copyObj.Version) - - rawObjects = append(rawObjects, metabase.RawObject(copyObj)) - - metabasetest.Verify{ - Objects: rawObjects, - Segments: expectedOriginalSegments, - Copies: []metabase.RawCopy{}, }.Check(ctx, t, db) } }) diff --git a/satellite/metabase/delete_bucket_test.go b/satellite/metabase/delete_bucket_test.go index 8359fb470..b1f97bc68 100644 --- a/satellite/metabase/delete_bucket_test.go +++ b/satellite/metabase/delete_bucket_test.go @@ -297,7 +297,6 @@ func TestDeleteBucketObjectsCancel(t *testing.T) { } func TestDeleteBucketWithCopies(t *testing.T) { - duplicateMetadata := true metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { for _, numberOfSegments := range []int{0, 1, 3} { t.Run(fmt.Sprintf("%d segments", numberOfSegments), func(t *testing.T) { @@ -322,7 +321,7 @@ func TestDeleteBucketWithCopies(t *testing.T) { metabasetest.CreateObjectCopy{ OriginalObject: originalObj, CopyObjectStream: ©ObjectStream, - }.Run(ctx, t, db, duplicateMetadata) + }.Run(ctx, t, db) _, err := db.DeleteBucketObjects(ctx, metabase.DeleteBucketObjects{ Bucket: metabase.BucketLocation{ @@ -363,7 +362,7 @@ func TestDeleteBucketWithCopies(t *testing.T) { copyObj, _, copySegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, CopyObjectStream: ©ObjectStream, - }.Run(ctx, t, db, duplicateMetadata) + }.Run(ctx, t, db) _, err := db.DeleteBucketObjects(ctx, metabase.DeleteBucketObjects{ Bucket: metabase.BucketLocation{ @@ -421,12 +420,12 @@ func TestDeleteBucketWithCopies(t *testing.T) { metabasetest.CreateObjectCopy{ OriginalObject: originalObj1, CopyObjectStream: ©ObjectStream1, - }.Run(ctx, t, db, duplicateMetadata) + }.Run(ctx, t, db) copyObj2, _, copySegments2 := metabasetest.CreateObjectCopy{ OriginalObject: originalObj2, CopyObjectStream: ©ObjectStream2, - }.Run(ctx, t, db, duplicateMetadata) + }.Run(ctx, t, db) // done preparing, delete bucket 1 _, err := db.DeleteBucketObjects(ctx, metabase.DeleteBucketObjects{ @@ -487,12 +486,12 @@ func TestDeleteBucketWithCopies(t *testing.T) { metabasetest.CreateObjectCopy{ OriginalObject: originalObj1, CopyObjectStream: ©ObjectStream1, - }.Run(ctx, t, db, duplicateMetadata) + }.Run(ctx, t, db) copyObj2, _, copySegments2 := metabasetest.CreateObjectCopy{ OriginalObject: originalObj2, CopyObjectStream: ©ObjectStream2, - }.Run(ctx, t, db, duplicateMetadata) + }.Run(ctx, t, db) // done preparing, delete bucket 1 _, err := db.DeleteBucketObjects(ctx, metabase.DeleteBucketObjects{ diff --git a/satellite/metabase/delete_test.go b/satellite/metabase/delete_test.go index 643028614..389fbb096 100644 --- a/satellite/metabase/delete_test.go +++ b/satellite/metabase/delete_test.go @@ -753,7 +753,7 @@ func TestDeleteCopyWithDuplicateMetadata(t *testing.T) { copyObj, _, copySegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) // check that copy went OK metabasetest.Verify{ @@ -798,10 +798,10 @@ func TestDeleteCopyWithDuplicateMetadata(t *testing.T) { copyObject1, _, _ := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) copyObject2, _, copySegments2 := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) metabasetest.DeleteObjectExactVersion{ Opts: metabase.DeleteObjectExactVersion{ @@ -838,7 +838,7 @@ func TestDeleteCopyWithDuplicateMetadata(t *testing.T) { copyObject, _, copySegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) metabasetest.DeleteObjectExactVersion{ Opts: metabase.DeleteObjectExactVersion{ @@ -878,10 +878,10 @@ func TestDeleteCopyWithDuplicateMetadata(t *testing.T) { copyObject1, _, copySegments1 := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) copyObject2, _, copySegments2 := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) _, err := db.DeleteObjectExactVersion(ctx, metabase.DeleteObjectExactVersion{ Version: originalObj.Version, diff --git a/satellite/metabase/get.go b/satellite/metabase/get.go index 273a118cb..a4d617bf1 100644 --- a/satellite/metabase/get.go +++ b/satellite/metabase/get.go @@ -41,12 +41,6 @@ func (s Segment) Inline() bool { return s.Redundancy.IsZero() && len(s.Pieces) == 0 } -// PiecesInAncestorSegment returns true if remote alias pieces are to be found in an ancestor segment. -// TODO we will remove this method and related to code when all metadata will be migrated to segment copies. -func (s Segment) PiecesInAncestorSegment() bool { - return s.EncryptedSize != 0 && len(s.InlineData) == 0 && len(s.Pieces) == 0 -} - // Expired checks if segment is expired relative to now. func (s Segment) Expired(now time.Time) bool { return s.ExpiresAt != nil && s.ExpiresAt.Before(now) @@ -271,13 +265,6 @@ func (db *DB) GetSegmentByPosition(ctx context.Context, opts GetSegmentByPositio segment.StreamID = opts.StreamID segment.Position = opts.Position - if db.config.ServerSideCopy { - err = db.updateWithAncestorSegment(ctx, &segment) - if err != nil { - return Segment{}, err - } - } - return segment, nil } @@ -342,52 +329,9 @@ func (db *DB) GetLatestObjectLastSegment(ctx context.Context, opts GetLatestObje } } - if db.config.ServerSideCopy { - err = db.updateWithAncestorSegment(ctx, &segment) - if err != nil { - return Segment{}, err - } - } - return segment, nil } -func (db *DB) updateWithAncestorSegment(ctx context.Context, segment *Segment) (err error) { - if !segment.PiecesInAncestorSegment() { - return nil - } - - var aliasPieces AliasPieces - - err = db.db.QueryRowContext(ctx, ` - SELECT - root_piece_id, - repaired_at, - remote_alias_pieces - FROM segments - WHERE - stream_id IN (SELECT ancestor_stream_id FROM segment_copies WHERE stream_id = $1) - AND position = $2 - `, segment.StreamID, segment.Position.Encode()).Scan( - &segment.RootPieceID, - &segment.RepairedAt, - &aliasPieces, - ) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return ErrSegmentNotFound.New("segment missing") - } - return Error.New("unable to query segment: %w", err) - } - - segment.Pieces, err = db.aliasCache.ConvertAliasesToPieces(ctx, aliasPieces) - if err != nil { - return Error.New("unable to convert aliases to pieces: %w", err) - } - - return nil -} - // BucketEmpty contains arguments necessary for checking if bucket is empty. type BucketEmpty struct { ProjectID uuid.UUID diff --git a/satellite/metabase/get_test.go b/satellite/metabase/get_test.go index 56c3f4dae..7d20af777 100644 --- a/satellite/metabase/get_test.go +++ b/satellite/metabase/get_test.go @@ -337,52 +337,6 @@ func TestGetObjectLastCommitted(t *testing.T) { }}.Check(ctx, t, db) }) - t.Run("Get latest copied object version", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - copyObjStream := metabasetest.RandObjectStream() - originalObject := metabasetest.CreateObject(ctx, t, db, obj, 0) - - copiedObj, _, _ := metabasetest.CreateObjectCopy{ - OriginalObject: originalObject, - CopyObjectStream: ©ObjStream, - }.Run(ctx, t, db, false) - - metabasetest.DeleteObjectExactVersion{ - Opts: metabase.DeleteObjectExactVersion{ - Version: 1, - ObjectLocation: obj.Location(), - }, - Result: metabase.DeleteObjectResult{ - Objects: []metabase.Object{originalObject}, - }, - }.Check(ctx, t, db) - - metabasetest.GetObjectLastCommitted{ - Opts: metabase.GetObjectLastCommitted{ - ObjectLocation: copiedObj.Location(), - }, - Result: copiedObj, - }.Check(ctx, t, db) - - metabasetest.Verify{Objects: []metabase.RawObject{ - { - ObjectStream: metabase.ObjectStream{ - ProjectID: copiedObj.ProjectID, - BucketName: copiedObj.BucketName, - ObjectKey: copiedObj.ObjectKey, - Version: copiedObj.Version, - StreamID: copiedObj.StreamID, - }, - CreatedAt: now, - Status: metabase.Committed, - Encryption: metabasetest.DefaultEncryption, - EncryptedMetadata: copiedObj.EncryptedMetadata, - EncryptedMetadataNonce: copiedObj.EncryptedMetadataNonce, - EncryptedMetadataEncryptedKey: copiedObj.EncryptedMetadataEncryptedKey, - }, - }}.Check(ctx, t, db) - }) - t.Run("Get latest copied object version with duplicate metadata", func(t *testing.T) { defer metabasetest.DeleteAll{}.Check(ctx, t, db) copyObjStream := metabasetest.RandObjectStream() @@ -391,7 +345,7 @@ func TestGetObjectLastCommitted(t *testing.T) { copiedObj, _, _ := metabasetest.CreateObjectCopy{ OriginalObject: originalObject, CopyObjectStream: ©ObjStream, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) metabasetest.DeleteObjectExactVersion{ Opts: metabase.DeleteObjectExactVersion{ @@ -621,33 +575,12 @@ func TestGetSegmentByPosition(t *testing.T) { Redundancy: metabasetest.DefaultRedundancy, } - expectedCopiedSegmentRaw := metabase.Segment{ - StreamID: copyObjStream.StreamID, - Position: metabase.SegmentPosition{ - Index: 0, - }, - CreatedAt: obj.CreatedAt, - ExpiresAt: obj.ExpiresAt, - RootPieceID: storj.PieceID{1}, - - Pieces: metabase.Pieces{}, - - EncryptedKey: newEncryptedKeyNonces[0].EncryptedKey, - EncryptedKeyNonce: newEncryptedKeyNonces[0].EncryptedKeyNonce, - EncryptedSize: 1024, - PlainSize: 512, - - Redundancy: metabasetest.DefaultRedundancy, - InlineData: []byte{}, - } - - expectedCopiedSegmentGet := expectedSegment - - expectedCopiedSegmentGet.EncryptedETag = nil - expectedCopiedSegmentGet.StreamID = copyObjStream.StreamID - expectedCopiedSegmentGet.EncryptedKey = newEncryptedKeyNonces[0].EncryptedKey - expectedCopiedSegmentGet.EncryptedKeyNonce = newEncryptedKeyNonces[0].EncryptedKeyNonce - expectedCopiedSegmentGet.InlineData = []byte{} + expectedCopiedSegment := expectedSegment + expectedCopiedSegment.StreamID = copyObjStream.StreamID + expectedCopiedSegment.EncryptedETag = nil + expectedCopiedSegment.EncryptedKey = newEncryptedKeyNonces[0].EncryptedKey + expectedCopiedSegment.EncryptedKeyNonce = newEncryptedKeyNonces[0].EncryptedKeyNonce + expectedCopiedSegment.InlineData = []byte{} metabasetest.GetSegmentByPosition{ Opts: metabase.GetSegmentByPosition{ @@ -666,7 +599,7 @@ func TestGetSegmentByPosition(t *testing.T) { Index: 0, }, }, - Result: expectedCopiedSegmentGet, + Result: expectedCopiedSegment, }.Check(ctx, t, db) metabasetest.Verify{ @@ -701,13 +634,8 @@ func TestGetSegmentByPosition(t *testing.T) { }, Segments: []metabase.RawSegment{ metabase.RawSegment(expectedSegment), - metabase.RawSegment(expectedCopiedSegmentRaw), + metabase.RawSegment(expectedCopiedSegment), }, - Copies: []metabase.RawCopy{ - { - StreamID: copyObjStream.StreamID, - AncestorStreamID: objStream.StreamID, - }}, }.Check(ctx, t, db) }) @@ -871,7 +799,6 @@ func TestGetSegmentByPosition(t *testing.T) { metabase.RawSegment(expectedSegment), metabase.RawSegment(expectedCopiedSegmentRaw), }, - Copies: nil, }.Check(ctx, t, db) }) @@ -1047,7 +974,6 @@ func TestGetSegmentByPosition(t *testing.T) { metabase.RawSegment(expectedSegment), metabase.RawSegment(expectedCopiedSegmentRaw), }, - Copies: nil, }.Check(ctx, t, db) }) }) @@ -1160,7 +1086,7 @@ func TestGetLatestObjectLastSegment(t *testing.T) { copyObj, _, newSegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) metabasetest.GetLatestObjectLastSegment{ Opts: metabase.GetLatestObjectLastSegment{ @@ -1189,10 +1115,6 @@ func TestGetLatestObjectLastSegment(t *testing.T) { metabase.RawObject(copyObj), }, Segments: append(metabasetest.SegmentsToRaw(originalSegments), newSegments...), - Copies: []metabase.RawCopy{{ - StreamID: copyObj.StreamID, - AncestorStreamID: originalObj.StreamID, - }}, }.Check(ctx, t, db) }) @@ -1212,7 +1134,7 @@ func TestGetLatestObjectLastSegment(t *testing.T) { copyObj, _, newSegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObj, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) metabasetest.GetLatestObjectLastSegment{ Opts: metabase.GetLatestObjectLastSegment{ @@ -1401,7 +1323,6 @@ func TestGetLatestObjectLastSegment(t *testing.T) { metabase.RawSegment(expectedSegment), metabase.RawSegment(expectedCopiedSegmentRaw), }, - Copies: nil, }.Check(ctx, t, db) }) @@ -1573,7 +1494,6 @@ func TestGetLatestObjectLastSegment(t *testing.T) { metabase.RawSegment(expectedSegment), metabase.RawSegment(expectedCopiedSegmentRaw), }, - Copies: nil, }.Check(ctx, t, db) }) }) diff --git a/satellite/metabase/list_segments.go b/satellite/metabase/list_segments.go index c79431b41..5e419084a 100644 --- a/satellite/metabase/list_segments.go +++ b/satellite/metabase/list_segments.go @@ -20,10 +20,6 @@ type ListSegments struct { Limit int Range *StreamRange - - // This causes ListSegments to update the first Segment in the response - // with the ancestor info if it exists and server side copy is enabled. - UpdateFirstWithAncestor bool } // ListSegmentsResult result of listing segments. diff --git a/satellite/metabase/list_segments_test.go b/satellite/metabase/list_segments_test.go index 79d06e31e..45d1f4dc0 100644 --- a/satellite/metabase/list_segments_test.go +++ b/satellite/metabase/list_segments_test.go @@ -282,7 +282,7 @@ func TestListSegments(t *testing.T) { _, _, copySegments := metabasetest.CreateObjectCopy{ OriginalObject: originalObject, CopyObjectStream: ©Stream, - }.Run(ctx, t, db, true) + }.Run(ctx, t, db) expectedSegments := []metabase.Segment{} for _, segment := range copySegments { @@ -304,8 +304,7 @@ func TestListSegments(t *testing.T) { metabasetest.ListSegments{ Opts: metabase.ListSegments{ - StreamID: copyStream.StreamID, - UpdateFirstWithAncestor: true, + StreamID: copyStream.StreamID, }, Result: metabase.ListSegmentsResult{ Segments: expectedSegments, diff --git a/satellite/metabase/metabasetest/common.go b/satellite/metabase/metabasetest/common.go index 72c24a614..41b97c624 100644 --- a/satellite/metabase/metabasetest/common.go +++ b/satellite/metabase/metabasetest/common.go @@ -40,8 +40,6 @@ func (step Verify) Check(ctx *testcontext.Context, t testing.TB, db *metabase.DB sortRawPendingObjects(step.PendingObjects) sortRawSegments(state.Segments) sortRawSegments(step.Segments) - sortRawCopies(state.Copies) - sortRawCopies(step.Copies) diff := cmp.Diff(metabase.RawState(step), *state, DefaultTimeDiff(), @@ -85,12 +83,6 @@ func sortRawSegments(segments []metabase.RawSegment) { }) } -func sortRawCopies(copies []metabase.RawCopy) { - sort.Slice(copies, func(i, j int) bool { - return copies[i].StreamID.Less(copies[j].StreamID) - }) -} - func checkError(t require.TestingT, err error, errClass *errs.Class, errText string) { if errClass != nil { require.True(t, errClass.Has(err), "expected an error %v got %v", *errClass, err) diff --git a/satellite/metabase/metabasetest/create.go b/satellite/metabase/metabasetest/create.go index 3a2dabedb..f6e7158fb 100644 --- a/satellite/metabase/metabasetest/create.go +++ b/satellite/metabase/metabasetest/create.go @@ -321,10 +321,7 @@ type CreateObjectCopy struct { } // Run creates the copy. -// -// The duplicateMetadata argument is a hack and it will be great to get rid of it once -// duplicateMetadata is no longer an option. -func (cc CreateObjectCopy) Run(ctx *testcontext.Context, t testing.TB, db *metabase.DB, duplicateMetadata bool) (copyObj metabase.Object, expectedOriginalSegments []metabase.RawSegment, expectedCopySegments []metabase.RawSegment) { +func (cc CreateObjectCopy) Run(ctx *testcontext.Context, t testing.TB, db *metabase.DB) (copyObj metabase.Object, expectedOriginalSegments []metabase.RawSegment, expectedCopySegments []metabase.RawSegment) { var copyStream metabase.ObjectStream if cc.CopyObjectStream != nil { copyStream = *cc.CopyObjectStream @@ -364,10 +361,8 @@ func (cc CreateObjectCopy) Run(ctx *testcontext.Context, t testing.TB, db *metab expectedCopySegments[i].InlineData = []byte{} } - if duplicateMetadata { - expectedCopySegments[i].Pieces = make(metabase.Pieces, len(expectedOriginalSegments[i].Pieces)) - copy(expectedCopySegments[i].Pieces, expectedOriginalSegments[i].Pieces) - } + expectedCopySegments[i].Pieces = make(metabase.Pieces, len(expectedOriginalSegments[i].Pieces)) + copy(expectedCopySegments[i].Pieces, expectedOriginalSegments[i].Pieces) } opts := cc.FinishObject @@ -382,7 +377,6 @@ func (cc CreateObjectCopy) Run(ctx *testcontext.Context, t testing.TB, db *metab NewEncryptedMetadataKey: testrand.Bytes(32), } } - opts.DuplicateMetadata = duplicateMetadata copyObj, err := db.FinishCopyObject(ctx, *opts) require.NoError(t, err) diff --git a/satellite/metabase/raw.go b/satellite/metabase/raw.go index b2a45855d..d2c19b684 100644 --- a/satellite/metabase/raw.go +++ b/satellite/metabase/raw.go @@ -99,7 +99,6 @@ type RawState struct { Objects []RawObject PendingObjects []RawPendingObject Segments []RawSegment - Copies []RawCopy } // TestingGetState returns the state of the database. @@ -121,11 +120,6 @@ func (db *DB) TestingGetState(ctx context.Context) (_ *RawState, err error) { return nil, Error.New("GetState: %w", err) } - state.Copies, err = db.testingGetAllCopies(ctx) - if err != nil { - return nil, Error.New("GetState: %w", err) - } - return state, nil } @@ -135,7 +129,6 @@ func (db *DB) TestingDeleteAll(ctx context.Context) (err error) { WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM objects; WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM pending_objects; WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM segments; - WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM segment_copies; WITH ignore_full_scan_for_test AS (SELECT 1) DELETE FROM node_aliases; WITH ignore_full_scan_for_test AS (SELECT 1) SELECT setval('node_alias_seq', 1, false); `) @@ -325,39 +318,3 @@ func (db *DB) testingGetAllSegments(ctx context.Context) (_ []RawSegment, err er } return segs, nil } - -// testingGetAllCopies returns the state of the database. -func (db *DB) testingGetAllCopies(ctx context.Context) (_ []RawCopy, err error) { - copies := []RawCopy{} - - rows, err := db.db.QueryContext(ctx, ` - WITH ignore_full_scan_for_test AS (SELECT 1) - SELECT - stream_id, ancestor_stream_id - FROM segment_copies - ORDER BY stream_id ASC, ancestor_stream_id ASC - `) - if err != nil { - return nil, Error.New("testingGetAllCopies query: %w", err) - } - defer func() { err = errs.Combine(err, rows.Close()) }() - for rows.Next() { - var copy RawCopy - err := rows.Scan( - ©.StreamID, - ©.AncestorStreamID, - ) - if err != nil { - return nil, Error.New("testingGetAllCopies scan failed: %w", err) - } - copies = append(copies, copy) - } - if err := rows.Err(); err != nil { - return nil, Error.New("testingGetAllCopies scan failed: %w", err) - } - - if len(copies) == 0 { - return nil, nil - } - return copies, nil -} diff --git a/satellite/metabase/streamstat.go b/satellite/metabase/streamstat.go index c78f443e1..e4531d31f 100644 --- a/satellite/metabase/streamstat.go +++ b/satellite/metabase/streamstat.go @@ -31,7 +31,7 @@ func (db *DB) GetStreamPieceCountByNodeID(ctx context.Context, opts GetStreamPie err = withRows(db.db.QueryContext(ctx, ` SELECT remote_alias_pieces FROM segments - WHERE stream_id IN (SELECT $1 UNION SELECT ancestor_stream_id FROM segment_copies WHERE stream_id = $1) AND remote_alias_pieces IS NOT null + WHERE stream_id = $1 AND remote_alias_pieces IS NOT null `, opts.StreamID))(func(rows tagsql.Rows) error { for rows.Next() { var aliasPieces AliasPieces diff --git a/satellite/metabase/streamstat_test.go b/satellite/metabase/streamstat_test.go index cf78b80c6..317d18c89 100644 --- a/satellite/metabase/streamstat_test.go +++ b/satellite/metabase/streamstat_test.go @@ -193,7 +193,7 @@ func TestGetStreamPieceCountByNodeID(t *testing.T) { _, _, _ = metabasetest.CreateObjectCopy{ OriginalObject: originalObj, CopyObjectStream: ©Stream, - }.Run(ctx, t, db, false) + }.Run(ctx, t, db) metabasetest.GetStreamPieceCountByNodeID{ Opts: metabase.GetStreamPieceCountByNodeID{ diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 4e5aef5f3..ceb04be32 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -143,9 +143,8 @@ type Config struct { ProjectLimits ProjectLimitConfig `help:"project limit configuration"` // TODO remove this flag when server-side copy implementation will be finished - ServerSideCopy bool `help:"enable code for server-side copy, deprecated. please leave this to true." default:"true"` - ServerSideCopyDisabled bool `help:"disable already enabled server-side copy. this is because once server side copy is enabled, delete code should stay changed, even if you want to disable server side copy" default:"false"` - ServerSideCopyDuplicateMetadata bool `help:"perform server-side copy by duplicating metadata, instead of using segment_copies" default:"false"` + ServerSideCopy bool `help:"enable code for server-side copy, deprecated. please leave this to true." default:"true"` + ServerSideCopyDisabled bool `help:"disable already enabled server-side copy. this is because once server side copy is enabled, delete code should stay changed, even if you want to disable server side copy" default:"false"` UsePendingObjectsTable bool `help:"enable new flow for upload which is using pending_objects table" default:"false"` diff --git a/satellite/metainfo/endpoint_object.go b/satellite/metainfo/endpoint_object.go index c96e35617..78a8e9c03 100644 --- a/satellite/metainfo/endpoint_object.go +++ b/satellite/metainfo/endpoint_object.go @@ -477,8 +477,6 @@ func (endpoint *Endpoint) DownloadObject(ctx context.Context, req *pb.ObjectDown StreamID: object.StreamID, Range: streamRange, Limit: int(req.Limit), - - UpdateFirstWithAncestor: true, }) if err != nil { return nil, endpoint.convertMetabaseErr(err) @@ -1958,7 +1956,7 @@ func (endpoint *Endpoint) FinishCopyObject(ctx context.Context, req *pb.ObjectFi NewEncryptedMetadata: req.NewEncryptedMetadata, NewEncryptedMetadataKeyNonce: req.NewEncryptedMetadataKeyNonce, NewEncryptedMetadataKey: req.NewEncryptedMetadataKey, - DuplicateMetadata: endpoint.config.ServerSideCopyDuplicateMetadata, + VerifyLimits: func(encryptedObjectSize int64, nSegments int64) error { return endpoint.addStorageUsageUpToLimit(ctx, keyInfo.ProjectID, encryptedObjectSize, nSegments) }, diff --git a/satellite/metainfo/endpoint_object_test.go b/satellite/metainfo/endpoint_object_test.go index 379bf67af..d6ffb0e39 100644 --- a/satellite/metainfo/endpoint_object_test.go +++ b/satellite/metainfo/endpoint_object_test.go @@ -841,7 +841,6 @@ func TestEndpoint_Object_With_StorageNodes(t *testing.T) { Reconfigure: testplanet.Reconfigure{ Satellite: func(logger *zap.Logger, index int, config *satellite.Config) { config.Overlay.GeoIP.MockCountries = []string{"DE"} - config.Metainfo.ServerSideCopyDuplicateMetadata = true }, }, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { @@ -2197,11 +2196,6 @@ func TestEndpoint_UpdateObjectMetadata(t *testing.T) { func TestEndpoint_Object_CopyObject(t *testing.T) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, - Reconfigure: testplanet.Reconfigure{ - Satellite: func(log *zap.Logger, index int, config *satellite.Config) { - config.Metainfo.ServerSideCopyDuplicateMetadata = true - }, - }, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { checkDownload := func(objectKey string, expectedData []byte) { data, err := planet.Uplinks[0].Download(ctx, planet.Satellites[0], "multipleversions", objectKey) diff --git a/satellite/repair/checker/observer_test.go b/satellite/repair/checker/observer_test.go index 9c9b6e20d..07895270b 100644 --- a/satellite/repair/checker/observer_test.go +++ b/satellite/repair/checker/observer_test.go @@ -191,7 +191,7 @@ func TestIdentifyIrreparableSegmentsObserver(t *testing.T) { }) } -func TestIgnoringCopiedSegmentsObserver(t *testing.T) { +func TestObserver_CheckSegmentCopy(t *testing.T) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1, Reconfigure: testplanet.Reconfigure{ @@ -241,15 +241,16 @@ func TestIgnoringCopiedSegmentsObserver(t *testing.T) { _, err = rangedLoopService.RunOnce(ctx) require.NoError(t, err) - // check that injured segment in repair queue streamID is same that in original segment. - injuredSegment, err := repairQueue.Select(ctx) - require.NoError(t, err) - require.Equal(t, segments[0].StreamID, injuredSegment.StreamID) + // check that repair queue has original segment and copied one as it has exactly the same metadata + for _, segment := range segmentsAfterCopy { + injuredSegment, err := repairQueue.Select(ctx) + require.NoError(t, err) + require.Equal(t, segment.StreamID, injuredSegment.StreamID) + } - // check that repair queue has only original segment, and not copied one. injuredSegments, err := repairQueue.Count(ctx) require.NoError(t, err) - require.Equal(t, 1, injuredSegments) + require.Equal(t, 2, injuredSegments) }) } diff --git a/scripts/testdata/satellite-config.yaml.lock b/scripts/testdata/satellite-config.yaml.lock index 426f6d333..34b03d374 100755 --- a/scripts/testdata/satellite-config.yaml.lock +++ b/scripts/testdata/satellite-config.yaml.lock @@ -691,9 +691,6 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key # disable already enabled server-side copy. this is because once server side copy is enabled, delete code should stay changed, even if you want to disable server side copy # metainfo.server-side-copy-disabled: false -# perform server-side copy by duplicating metadata, instead of using segment_copies -# metainfo.server-side-copy-duplicate-metadata: false - # test the new query for non-recursive listing # metainfo.test-listing-query: false