From f0ce8996c3de25d1dc3fe76b232b830ed7e920c7 Mon Sep 17 00:00:00 2001 From: Michal Niewrzal Date: Mon, 14 Nov 2022 15:41:55 +0100 Subject: [PATCH] satellite/metainfo: enable metainfo.multiple-versions flag by default We tested new upload flow (with multiple versions) to fix inconsistency while uploading object on QA/EUN1/SLC. Now we would like to enable it for all satellites by default. Tests required small adjustments. Fixes https://github.com/storj/storj/issues/5283 Change-Id: I0d53c041abebc0d182ba5a88bb1dac906c29caf0 --- satellite/metabase/commit_test.go | 280 ++++---------------- satellite/metabase/copy_object_test.go | 9 + satellite/metainfo/config.go | 2 +- scripts/testdata/satellite-config.yaml.lock | 2 +- 4 files changed, 65 insertions(+), 228 deletions(-) diff --git a/satellite/metabase/commit_test.go b/satellite/metabase/commit_test.go index 1df195261..9f9823751 100644 --- a/satellite/metabase/commit_test.go +++ b/satellite/metabase/commit_test.go @@ -155,231 +155,6 @@ func TestBeginObjectNextVersion(t *testing.T) { // TODO: expires at date // TODO: zombie deletion deadline - t.Run("older committed version exists", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now1 := time.Now() - objectStream.Version = metabase.NextVersion - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - }, - Version: 1, - }.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 1, - StreamID: obj.StreamID, - }, - }, - }.Check(ctx, t, db) - - now2 := time.Now() - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - }, - Version: 2, - }.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 2, - StreamID: obj.StreamID, - }, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 1, - StreamID: obj.StreamID, - }, - CreatedAt: now1, - Status: metabase.Committed, - - Encryption: metabasetest.DefaultEncryption, - }, - { - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 2, - StreamID: obj.StreamID, - }, - CreatedAt: now2, - Status: metabase.Committed, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("newer committed version exists", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now1 := time.Now() - - objectStream.Version = metabase.NextVersion - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - }, - Version: 1, - }.Check(ctx, t, db) - - now2 := time.Now() - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - }, - Version: 2, - }.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 2, - StreamID: obj.StreamID, - }, - }, - }.Check(ctx, t, db) - - metabasetest.CommitObject{ - Opts: metabase.CommitObject{ - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 1, - StreamID: obj.StreamID, - }, - }, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 1, - StreamID: obj.StreamID, - }, - CreatedAt: now1, - Status: metabase.Committed, - - Encryption: metabasetest.DefaultEncryption, - }, - { - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: 2, - StreamID: obj.StreamID, - }, - CreatedAt: now2, - Status: metabase.Committed, - - Encryption: metabasetest.DefaultEncryption, - }, - }, - }.Check(ctx, t, db) - }) - - t.Run("begin object next version with metadata", func(t *testing.T) { - defer metabasetest.DeleteAll{}.Check(ctx, t, db) - - now := time.Now() - zombieDeadline := now.Add(24 * time.Hour) - - objectStream.Version = metabase.NextVersion - - encryptedMetadata := testrand.BytesInt(64) - encryptedMetadataNonce := testrand.Nonce() - encryptedMetadataEncryptedKey := testrand.BytesInt(32) - - metabasetest.BeginObjectNextVersion{ - Opts: metabase.BeginObjectNextVersion{ - ObjectStream: objectStream, - Encryption: metabasetest.DefaultEncryption, - - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadataEncryptedKey: encryptedMetadataEncryptedKey, - }, - Version: 1, - }.Check(ctx, t, db) - - metabasetest.Verify{ - Objects: []metabase.RawObject{ - { - ObjectStream: metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - Version: metabase.DefaultVersion, - StreamID: obj.StreamID, - }, - CreatedAt: now, - Status: metabase.Pending, - - EncryptedMetadata: encryptedMetadata, - EncryptedMetadataNonce: encryptedMetadataNonce[:], - EncryptedMetadataEncryptedKey: encryptedMetadataEncryptedKey, - - Encryption: metabasetest.DefaultEncryption, - ZombieDeletionDeadline: &zombieDeadline, - }, - }, - }.Check(ctx, t, db) - }) - }) -} - -func TestBeginObjectNextVersionMultipleVersions(t *testing.T) { - // TODO partially duplicated TestBeginObjectNextVersion tests to cover MultipleVersions enabled - // to be removed when flag will be removed - metabasetest.RunWithConfig(t, metabase.Config{ - ApplicationName: "satellite-test", - MultipleVersions: true, - }, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { - obj := metabasetest.RandObjectStream() - - objectStream := metabase.ObjectStream{ - ProjectID: obj.ProjectID, - BucketName: obj.BucketName, - ObjectKey: obj.ObjectKey, - StreamID: obj.StreamID, - } - t.Run("older committed version exists", func(t *testing.T) { defer metabasetest.DeleteAll{}.Check(ctx, t, db) @@ -512,11 +287,64 @@ func TestBeginObjectNextVersionMultipleVersions(t *testing.T) { }, }.Check(ctx, t, db) }) + + t.Run("begin object next version with metadata", func(t *testing.T) { + defer metabasetest.DeleteAll{}.Check(ctx, t, db) + + now := time.Now() + zombieDeadline := now.Add(24 * time.Hour) + + objectStream.Version = metabase.NextVersion + + encryptedMetadata := testrand.BytesInt(64) + encryptedMetadataNonce := testrand.Nonce() + encryptedMetadataEncryptedKey := testrand.BytesInt(32) + + metabasetest.BeginObjectNextVersion{ + Opts: metabase.BeginObjectNextVersion{ + ObjectStream: objectStream, + Encryption: metabasetest.DefaultEncryption, + + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadataEncryptedKey: encryptedMetadataEncryptedKey, + }, + Version: 1, + }.Check(ctx, t, db) + + metabasetest.Verify{ + Objects: []metabase.RawObject{ + { + ObjectStream: metabase.ObjectStream{ + ProjectID: obj.ProjectID, + BucketName: obj.BucketName, + ObjectKey: obj.ObjectKey, + Version: metabase.DefaultVersion, + StreamID: obj.StreamID, + }, + CreatedAt: now, + Status: metabase.Pending, + + EncryptedMetadata: encryptedMetadata, + EncryptedMetadataNonce: encryptedMetadataNonce[:], + EncryptedMetadataEncryptedKey: encryptedMetadataEncryptedKey, + + Encryption: metabasetest.DefaultEncryption, + ZombieDeletionDeadline: &zombieDeadline, + }, + }, + }.Check(ctx, t, db) + }) }) } func TestBeginObjectExactVersion(t *testing.T) { - metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { + // BeginObjectExactVersion will be removed when MultipleVersions feature will be enabled in prod + // but for now lets keep this test but just disable MultipleVersions + metabasetest.RunWithConfig(t, metabase.Config{ + ApplicationName: "satellite-test", + MultipleVersions: false, + }, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { obj := metabasetest.RandObjectStream() for _, test := range metabasetest.InvalidObjectStreams(obj) { diff --git a/satellite/metabase/copy_object_test.go b/satellite/metabase/copy_object_test.go index 254cfc9e5..464003d2b 100644 --- a/satellite/metabase/copy_object_test.go +++ b/satellite/metabase/copy_object_test.go @@ -458,6 +458,9 @@ func TestFinishCopyObject(t *testing.T) { expectedCopyObject := originalObj expectedCopyObject.ObjectKey = copyStream.ObjectKey expectedCopyObject.StreamID = copyStream.StreamID + // copy operation will always try to find first free version for new/copied object + // so if there are no objects under this location it will be always 1 + expectedCopyObject.Version = metabase.DefaultVersion expectedCopyObject.EncryptedMetadataEncryptedKey = testrand.Bytes(32) expectedCopyObject.EncryptedMetadataNonce = metadataNonce.Bytes() @@ -814,6 +817,9 @@ func TestFinishCopyObject(t *testing.T) { originalSegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy } + // copy operation is finding first available version for new/copied object + // so we need to bump copyBackObj version + copyBackObj.Version++ metabasetest.Verify{ Objects: []metabase.RawObject{ metabase.RawObject(copyObj), @@ -890,6 +896,9 @@ func TestFinishCopyObject(t *testing.T) { originalSegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy } + // copy operation is finding first available version for new/copied object + // so we need to bump copyBackObj version + copyBackObj.Version++ metabasetest.Verify{ Objects: []metabase.RawObject{ metabase.RawObject(copyObj), diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 134f813bb..643e2faa3 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -139,7 +139,7 @@ type Config struct { // TODO remove this flag when server-side copy implementation will be finished ServerSideCopy bool `help:"enable code for server-side copy, deprecated. please leave this to true." default:"true"` ServerSideCopyDisabled bool `help:"disable already enabled server-side copy. this is because once server side copy is enabled, delete code should stay changed, even if you want to disable server side copy" default:"false"` - MultipleVersions bool `help:"feature flag to enable using multple objects versions in the system internally" default:"false"` + MultipleVersions bool `help:"feature flag to enable using multple objects versions in the system internally" default:"true"` // TODO remove when we benchmarking are done and decision is made. TestListingQuery bool `default:"false" help:"test the new query for non-recursive listing"` } diff --git a/scripts/testdata/satellite-config.yaml.lock b/scripts/testdata/satellite-config.yaml.lock index c92d52b29..c85f137fe 100755 --- a/scripts/testdata/satellite-config.yaml.lock +++ b/scripts/testdata/satellite-config.yaml.lock @@ -584,7 +584,7 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key # metainfo.min-remote-segment-size: 1.2 KiB # feature flag to enable using multple objects versions in the system internally -# metainfo.multiple-versions: false +# metainfo.multiple-versions: true # toggle flag if overlay is enabled # metainfo.overlay: true