satellite/metabase: use commit order for objects

This changes metabase behavior such that the latest object
will be the committed object which may result in a new version
assigned to the objects.

Change-Id: I7a731d7db4696caba75fff65107a248569b6111f

satellite/metabase/commit.go

@@ -692,11 +692,11 @@ func (db *DB) CommitObject(ctx context.Context, opts CommitObject) (object Objec
 		var highestVersion Version
 		if opts.Versioned {
 			// TODO(ver): fold this into the queries below using a subquery.
-			v, err := db.queryHighestVersion(ctx, opts.Location(), tx)
+			maxVersion, err := db.queryHighestVersion(ctx, opts.Location(), tx)
 			if err != nil {
 				return Error.Wrap(err)
 			}
-			highestVersion = v
+			highestVersion = maxVersion
 		} else {
 			// TODO(ver): fold this into the query below using a subquery using `ON CONFLICT` on the unique index.
 			// Note, we are prematurely deleting the object without permissions
@@ -708,7 +708,6 @@ func (db *DB) CommitObject(ctx context.Context, opts CommitObject) (object Objec
 			if deleteResult.DeletedObjectCount > 0 && opts.DisallowDelete {
 				return ErrPermissionDenied.New("no permissions to delete existing object")
 			}
-
 			highestVersion = deleteResult.MaxVersion
 		}
 
@@ -784,6 +783,13 @@ func (db *DB) CommitObject(ctx context.Context, opts CommitObject) (object Objec
 				return Error.New("failed to update object: %w", err)
 			}
 		} else {
+			nextVersion := opts.Version
+			if nextVersion < highestVersion {
+				nextVersion = highestVersion + 1
+			}
+			args = append(args, nextVersion)
+			opts.Version = nextVersion
+
 			metadataColumns := ""
 			if opts.OverrideEncryptedMetadata {
 				args = append(args,
@@ -792,13 +798,14 @@ func (db *DB) CommitObject(ctx context.Context, opts CommitObject) (object Objec
 					opts.EncryptedMetadataEncryptedKey,
 				)
 				metadataColumns = `,
-					encrypted_metadata_nonce         = $12,
+					encrypted_metadata_nonce         = $13,
-					encrypted_metadata               = $13,
+					encrypted_metadata               = $14,
-					encrypted_metadata_encrypted_key = $14
+					encrypted_metadata_encrypted_key = $15
 				`
 			}
 			err = tx.QueryRowContext(ctx, `
 				UPDATE objects SET
+					version = $12,
 					status = $6,
 					segment_count = $7,
 

satellite/metabase/commit_object.go

@@ -51,23 +51,31 @@ func (db *DB) CommitObjectWithSegments(ctx context.Context, opts CommitObjectWit
 		return Object{}, nil, err
 	}
 
-	var previousObject deleteObjectUnversionedCommittedResult
+	var deleted deleteObjectUnversionedCommittedResult
 
 	err = txutil.WithTx(ctx, db.db, nil, func(ctx context.Context, tx tagsql.Tx) error {
 		// TODO: should we prevent this from executing when the object has been committed
 		// currently this requires quite a lot of database communication, so invalid handling can be expensive.
 
+		var highestVersion Version
 		if !opts.Versioned {
 			var err error
 			// Note, we are prematurely deleting the object without permissions
 			// and then rolling the action back, if we were not allowed to.
-			previousObject, err = db.deleteObjectUnversionedCommitted(ctx, opts.Location(), tx)
+			deleted, err = db.deleteObjectUnversionedCommitted(ctx, opts.Location(), tx)
 			if err != nil {
 				return err
 			}
-			if previousObject.DeletedObjectCount > 0 && opts.DisallowDelete {
+			if deleted.DeletedObjectCount > 0 && opts.DisallowDelete {
 				return ErrPermissionDenied.New("no permissions to delete existing object")
 			}
+			highestVersion = deleted.MaxVersion
+		} else {
+			v, err := db.queryHighestVersion(ctx, opts.Location(), tx)
+			if err != nil {
+				return err
+			}
+			highestVersion = v
 		}
 
 		segmentsInDatabase, err := fetchSegmentsForCommit(ctx, tx, opts.StreamID)
@@ -113,9 +121,14 @@ func (db *DB) CommitObjectWithSegments(ctx context.Context, opts CommitObjectWit
 		}
 
 		nextStatus := committedWhereVersioned(opts.Versioned)
+		nextVersion := opts.Version
+		if nextVersion < highestVersion {
+			nextVersion = highestVersion + 1
+		}
 
 		err = tx.QueryRowContext(ctx, `
 			UPDATE objects SET
+				version = $14,
 				status = $6,
 				segment_count = $7,
 
@@ -138,6 +151,7 @@ func (db *DB) CommitObjectWithSegments(ctx context.Context, opts CommitObjectWit
 			totalPlainSize,
 			totalEncryptedSize,
 			fixedSegmentSize,
+			nextVersion,
 		).
 			Scan(
 				&object.CreatedAt, &object.ExpiresAt,
@@ -154,7 +168,7 @@ func (db *DB) CommitObjectWithSegments(ctx context.Context, opts CommitObjectWit
 		object.ProjectID = opts.ProjectID
 		object.BucketName = opts.BucketName
 		object.ObjectKey = opts.ObjectKey
-		object.Version = opts.Version
+		object.Version = nextVersion
 		object.Status = nextStatus
 		object.SegmentCount = int32(len(finalSegments))
 		object.EncryptedMetadataNonce = opts.EncryptedMetadataNonce

satellite/metabase/commit_object_test.go

@@ -332,6 +332,7 @@ func TestCommitObjectWithSegments(t *testing.T) {
 					EncryptedMetadata:             encryptedMetadata,
 					EncryptedMetadataEncryptedKey: encryptedMetadataKey,
 				},
+				ExpectVersion: obj.Version + 1,
 			}.Check(ctx, t, db)
 
 			metabasetest.Verify{
@@ -341,7 +342,7 @@ func TestCommitObjectWithSegments(t *testing.T) {
 							ProjectID:  obj.ProjectID,
 							BucketName: obj.BucketName,
 							ObjectKey:  obj.ObjectKey,
-							Version:    5,
+							Version:    obj.Version + 1,
 							StreamID:   obj.StreamID,
 						},
 						CreatedAt: now,

satellite/metabase/commit_test.go

@@ -264,6 +264,7 @@ func TestBeginObjectNextVersion(t *testing.T) {
 						StreamID:   obj.StreamID,
 					},
 				},
+				ExpectVersion: 3,
 			}.Check(ctx, t, db)
 
 			// currently CommitObject always deletes previous versions so only version 1 left
@@ -274,7 +275,7 @@ func TestBeginObjectNextVersion(t *testing.T) {
 							ProjectID:  obj.ProjectID,
 							BucketName: obj.BucketName,
 							ObjectKey:  obj.ObjectKey,
-							Version:    1,
+							Version:    3,
 							StreamID:   obj.StreamID,
 						},
 						CreatedAt: now1,
@@ -799,6 +800,7 @@ func TestBeginObjectExactVersion(t *testing.T) {
 				Opts: metabase.CommitObject{
 					ObjectStream: objectStream,
 				},
+				ExpectVersion: 301,
 			}.Check(ctx, t, db)
 
 			// currently CommitObject always deletes previous versions so only version 1 left
@@ -809,7 +811,7 @@ func TestBeginObjectExactVersion(t *testing.T) {
 							ProjectID:  obj.ProjectID,
 							BucketName: obj.BucketName,
 							ObjectKey:  obj.ObjectKey,
-							Version:    100,
+							Version:    301,
 							StreamID:   obj.StreamID,
 						},
 						CreatedAt: now1,
@@ -4432,21 +4434,27 @@ func TestCommitObjectVersioned(t *testing.T) {
 					ObjectStream: v1,
 					Versioned:    true,
 				},
+				ExpectVersion: v3.Version + 1,
 			}.Check(ctx, t, db)
+			v1.Version = v3.Version + 1
 
 			metabasetest.CommitObject{
 				Opts: metabase.CommitObject{
 					ObjectStream: v2,
 					Versioned:    true,
 				},
+				ExpectVersion: v3.Version + 2,
 			}.Check(ctx, t, db)
+			v2.Version = v1.Version + 1
 
 			metabasetest.CommitObject{
 				Opts: metabase.CommitObject{
 					ObjectStream: v3,
 					Versioned:    true,
 				},
+				ExpectVersion: v3.Version + 3,
 			}.Check(ctx, t, db)
+			v3.Version = v2.Version + 1
 
 			metabasetest.Verify{
 				Objects: []metabase.RawObject{
@@ -4558,13 +4566,17 @@ func TestCommitObjectVersioned(t *testing.T) {
 				Opts: metabase.CommitObject{
 					ObjectStream: v3,
 				},
+				ExpectVersion: 5,
 			}.Check(ctx, t, db)
+			v3.Version = 5
 
 			metabasetest.CommitObject{
 				Opts: metabase.CommitObject{
 					ObjectStream: v1,
 				},
+				ExpectVersion: 6,
 			}.Check(ctx, t, db)
+			v1.Version = 6
 
 			// The latter commit should overwrite the v3.
 			// When pending objects table is enabled, then objects
@@ -4601,14 +4613,18 @@ func TestCommitObjectVersioned(t *testing.T) {
 					ObjectStream: v2,
 					Versioned:    true,
 				},
+				ExpectVersion: 7,
 			}.Check(ctx, t, db)
+			v2.Version = 7
 
 			metabasetest.CommitObject{
 				Opts: metabase.CommitObject{
 					ObjectStream: v4,
 					Versioned:    true,
 				},
+				ExpectVersion: 8,
 			}.Check(ctx, t, db)
+			v4.Version = 8
 
 			metabasetest.Verify{
 				Objects: []metabase.RawObject{
@@ -4719,14 +4735,18 @@ func TestCommitObjectVersioned(t *testing.T) {
 					ObjectStream: v1,
 					Versioned:    true,
 				},
+				ExpectVersion: 5,
 			}.Check(ctx, t, db)
+			v1.Version = 5
 
 			metabasetest.CommitObject{
 				Opts: metabase.CommitObject{
 					ObjectStream: v3,
 					Versioned:    true,
 				},
+				ExpectVersion: 6,
 			}.Check(ctx, t, db)
+			v3.Version = 6
 
 			metabasetest.Verify{
 				Objects: []metabase.RawObject{
@@ -4763,13 +4783,17 @@ func TestCommitObjectVersioned(t *testing.T) {
 				Opts: metabase.CommitObject{
 					ObjectStream: v2,
 				},
+				ExpectVersion: 7,
 			}.Check(ctx, t, db)
+			v2.Version = 7
 
 			metabasetest.CommitObject{
 				Opts: metabase.CommitObject{
 					ObjectStream: v4,
 				},
+				ExpectVersion: 8,
 			}.Check(ctx, t, db)
+			v4.Version = 8
 
 			// committing v4 should overwrite the previous unversioned commit (v2),
 			// so v2 is not in the result check
@@ -4882,7 +4906,9 @@ func TestCommitObjectVersioned(t *testing.T) {
 					ObjectStream: v1,
 					Versioned:    true,
 				},
+				ExpectVersion: 5,
 			}.Check(ctx, t, db)
+			v1.Version = 5
 
 			metabasetest.Verify{
 				Objects: []metabase.RawObject{
@@ -4920,7 +4946,9 @@ func TestCommitObjectVersioned(t *testing.T) {
 				Opts: metabase.CommitObject{
 					ObjectStream: v2,
 				},
+				ExpectVersion: 6,
 			}.Check(ctx, t, db)
+			v2.Version = 6
 
 			metabasetest.Verify{
 				Objects: []metabase.RawObject{
@@ -4958,7 +4986,9 @@ func TestCommitObjectVersioned(t *testing.T) {
 					ObjectStream: v3,
 					Versioned:    true,
 				},
+				ExpectVersion: 7,
 			}.Check(ctx, t, db)
+			v3.Version = 7
 
 			metabasetest.Verify{
 				Objects: []metabase.RawObject{
@@ -4994,7 +5024,9 @@ func TestCommitObjectVersioned(t *testing.T) {
 				Opts: metabase.CommitObject{
 					ObjectStream: v4,
 				},
+				ExpectVersion: 8,
 			}.Check(ctx, t, db)
+			v4.Version = 8
 
 			// committing v4 should overwrite the previous unversioned commit (v2),
 			// so v2 is not in the result check
@@ -5043,6 +5075,7 @@ func TestCommitObjectVersioned(t *testing.T) {
 			}
 
 			rawObjects := make([]metabase.RawObject, 0, len(objs))
+			expectVersion := metabase.Version(numCommits + 1)
 			for i := 0; i < len(objs); i++ {
 				versioned := i%2 == 0
 
@@ -5051,7 +5084,10 @@ func TestCommitObjectVersioned(t *testing.T) {
 						ObjectStream: *objs[i],
 						Versioned:    versioned,
 					},
+					ExpectVersion: expectVersion,
 				}.Check(ctx, t, db)
+				objs[i].Version = expectVersion
+				expectVersion++
 
 				if versioned {
 					rawObjects = append(rawObjects, metabase.RawObject{

satellite/metabase/copy_object_test.go

@@ -4,6 +4,7 @@
 package metabase_test
 
 import (
+	"strconv"
 	"testing"
 	"time"
 
@@ -1033,36 +1034,39 @@ func TestFinishCopyObject(t *testing.T) {
 				NewBucket                    string
 				NewKey                       metabase.ObjectKey
 				sourcePendingVersions        []metabase.Version
+				sourceCommitVersion          metabase.Version
 				sourceCommittedVersion       metabase.Version
 				destinationPendingVersions   []metabase.Version
-				destinationCommittedVersion metabase.Version
+				destinationCommitVersion     metabase.Version
+				destionationCommittedVersion metabase.Version
 				expectedCopyVersion          metabase.Version
 			}{
 				// the same bucket
-				{"testbucket", "object", "testbucket", "new-object",
+				0: {"testbucket", "object", "testbucket", "new-object",
-					[]metabase.Version{}, 2,
+					[]metabase.Version{}, 2, 2,
-					[]metabase.Version{}, 1,
+					[]metabase.Version{}, 1, 1,
 					2},
-				{"testbucket", "object", "testbucket", "new-object",
+				1: {"testbucket", "object", "testbucket", "new-object",
-					[]metabase.Version{}, 1,
+					[]metabase.Version{}, 1, 1,
-					[]metabase.Version{1}, 2,
+					[]metabase.Version{1}, 2, 2,
 					3},
-				{"testbucket", "object", "testbucket", "new-object",
+				2: {"testbucket", "object", "testbucket", "new-object",
-					[]metabase.Version{}, 1,
+					[]metabase.Version{}, 1, 1,
-					[]metabase.Version{1, 3}, 2,
+					[]metabase.Version{1, 3}, 2, 4,
-					4},
+					5},
-				{"testbucket", "object", "testbucket", "new-object",
+				3: {"testbucket", "object", "testbucket", "new-object",
-					[]metabase.Version{1, 5}, 2,
+					[]metabase.Version{1, 5}, 2, 6,
-					[]metabase.Version{1, 3}, 2,
+					[]metabase.Version{1, 3}, 2, 4,
-					4},
+					5},
-				{"testbucket", "object", "newbucket", "object",
+				4: {"testbucket", "object", "newbucket", "object",
-					[]metabase.Version{2, 3}, 1,
+					[]metabase.Version{2, 3}, 1, 4,
-					[]metabase.Version{1, 5}, 2,
+					[]metabase.Version{1, 5}, 2, 6,
-					6},
+					7},
 			}
 
-			for _, tc := range testCases {
+			for i, tc := range testCases {
-				metabasetest.DeleteAll{}.Check(ctx, t, db)
+				t.Run(strconv.Itoa(i), func(t *testing.T) {
+					defer metabasetest.DeleteAll{}.Check(ctx, t, db)
 					sourceObjStream.BucketName = tc.Bucket
 					sourceObjStream.ObjectKey = tc.Key
 					destinationObjStream.BucketName = tc.NewBucket
@@ -1083,7 +1087,7 @@ func TestFinishCopyObject(t *testing.T) {
 							ZombieDeletionDeadline: &zombieDeadline,
 						})
 					}
-				sourceObjStream.Version = tc.sourceCommittedVersion
+					sourceObjStream.Version = tc.sourceCommitVersion
 					sourceObjStream.StreamID = testrand.UUID()
 					sourceObj, _ := metabasetest.CreateTestObject{
 						BeginObjectExactVersion: &metabase.BeginObjectExactVersion{
@@ -1097,6 +1101,7 @@ func TestFinishCopyObject(t *testing.T) {
 							EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
 							EncryptedMetadataEncryptedKey: testrand.Bytes(265),
 						},
+						ExpectVersion: tc.sourceCommittedVersion,
 					}.Run(ctx, t, db, sourceObjStream, 0)
 
 					rawObjects = append(rawObjects, metabase.RawObject(sourceObj))
@@ -1116,9 +1121,9 @@ func TestFinishCopyObject(t *testing.T) {
 						})
 					}
 
-				if tc.destinationCommittedVersion != 0 {
+					if tc.destinationCommitVersion != 0 {
 						destinationObjStream.StreamID = testrand.UUID()
-					destinationObjStream.Version = tc.destinationCommittedVersion
+						destinationObjStream.Version = tc.destinationCommitVersion
 						_, _ = metabasetest.CreateTestObject{
 							BeginObjectExactVersion: &metabase.BeginObjectExactVersion{
 								ObjectStream: destinationObjStream,
@@ -1131,6 +1136,7 @@ func TestFinishCopyObject(t *testing.T) {
 								EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
 								EncryptedMetadataEncryptedKey: testrand.Bytes(265),
 							},
+							ExpectVersion: tc.destionationCommittedVersion,
 						}.Run(ctx, t, db, destinationObjStream, 0)
 					}
 
@@ -1147,6 +1153,7 @@ func TestFinishCopyObject(t *testing.T) {
 						Objects:  rawObjects,
 						Segments: expectedOriginalSegments,
 					}.Check(ctx, t, db)
+				})
 			}
 		})
 

satellite/metabase/list_objects_test.go

@@ -902,13 +902,13 @@ func TestListObjectsVersioned(t *testing.T) {
 
 			a1 := a0
 			a1.Version = 1001
-			b1 := a0
+			b1 := b0
 			b1.Version = 500
 
 			objA0 := metabasetest.CreateObjectVersioned(ctx, t, db, a0, 0)
 			objA1 := metabasetest.CreateObjectVersioned(ctx, t, db, a1, 0)
 			objB0 := metabasetest.CreateObjectVersioned(ctx, t, db, b0, 0)
-			objB1 := metabasetest.CreateObjectVersioned(ctx, t, db, b1, 0)
+			objB1 := metabasetest.CreateObjectVersionedOutOfOrder(ctx, t, db, b1, 0, 1001)
 
 			metabasetest.ListObjects{
 				Opts: metabase.ListObjects{
@@ -922,7 +922,7 @@ func TestListObjectsVersioned(t *testing.T) {
 				Result: metabase.ListObjectsResult{
 					Objects: []metabase.ObjectEntry{
 						objectEntryFromRaw(metabase.RawObject(objA1)),
-						objectEntryFromRaw(metabase.RawObject(objB0)),
+						objectEntryFromRaw(metabase.RawObject(objB1)),
 					},
 				}}.Check(ctx, t, db)
 
@@ -952,13 +952,13 @@ func TestListObjectsVersioned(t *testing.T) {
 
 			a1 := a0
 			a1.Version = 1001
-			b1 := a0
+			b1 := b0
 			b1.Version = 500
 
 			objA0 := metabasetest.CreateObjectVersioned(ctx, t, db, a0, 0)
 			objA1 := metabasetest.CreateObjectVersioned(ctx, t, db, a1, 0)
 			objB0 := metabasetest.CreateObjectVersioned(ctx, t, db, b0, 0)
-			objB1 := metabasetest.CreateObjectVersioned(ctx, t, db, b1, 0)
+			objB1 := metabasetest.CreateObjectVersionedOutOfOrder(ctx, t, db, b1, 0, 1001)
 
 			deletionResult := metabasetest.DeleteObjectLastCommitted{
 				Opts: metabase.DeleteObjectLastCommitted{
@@ -992,7 +992,7 @@ func TestListObjectsVersioned(t *testing.T) {
 				},
 				Result: metabase.ListObjectsResult{
 					Objects: []metabase.ObjectEntry{
-						objectEntryFromRaw(metabase.RawObject(objB0)),
+						objectEntryFromRaw(metabase.RawObject(objB1)),
 					},
 				}}.Check(ctx, t, db)
 

satellite/metabase/metabasetest/create.go

@@ -124,6 +124,19 @@ func CreateObjectVersioned(ctx *testcontext.Context, t testing.TB, db *metabase.
 	}.Check(ctx, t, db)
 }
 
+// CreateObjectVersionedOutOfOrder creates a new committed object with the specified number of segments.
+func CreateObjectVersionedOutOfOrder(ctx *testcontext.Context, t testing.TB, db *metabase.DB, obj metabase.ObjectStream, numberOfSegments byte, expectVersion metabase.Version) metabase.Object {
+	CreatePendingObject(ctx, t, db, obj, numberOfSegments)
+
+	return CommitObject{
+		Opts: metabase.CommitObject{
+			ObjectStream: obj,
+			Versioned:    true,
+		},
+		ExpectVersion: expectVersion,
+	}.Check(ctx, t, db)
+}
+
 // CreateExpiredObject creates a new committed expired object with the specified number of segments.
 func CreateExpiredObject(ctx *testcontext.Context, t testing.TB, db *metabase.DB, obj metabase.ObjectStream, numberOfSegments byte, expiresAt time.Time) metabase.Object {
 	BeginObjectExactVersion{
@@ -206,6 +219,7 @@ func CreateSegments(ctx *testcontext.Context, t testing.TB, db *metabase.DB, obj
 type CreateTestObject struct {
 	BeginObjectExactVersion *metabase.BeginObjectExactVersion
 	CommitObject            *metabase.CommitObject
+	ExpectVersion           metabase.Version
 	CreateSegment           func(object metabase.Object, index int) metabase.Segment
 }
 
@@ -303,6 +317,7 @@ func (co CreateTestObject) Run(ctx *testcontext.Context, t testing.TB, db *metab
 
 	createdObject := CommitObject{
 		Opts:          coOpts,
+		ExpectVersion: co.ExpectVersion,
 	}.Check(ctx, t, db)
 
 	return createdObject, createdSegments

satellite/metabase/metabasetest/test.go

@@ -88,6 +88,7 @@ func (step BeginObjectExactVersion) Check(ctx *testcontext.Context, t require.Te
 // CommitObject is for testing metabase.CommitObject.
 type CommitObject struct {
 	Opts          metabase.CommitObject
+	ExpectVersion metabase.Version
 	ErrClass      *errs.Class
 	ErrText       string
 }
@@ -97,6 +98,9 @@ func (step CommitObject) Check(ctx *testcontext.Context, t require.TestingT, db
 	object, err := db.CommitObject(ctx, step.Opts)
 	checkError(t, err, step.ErrClass, step.ErrText)
 	if err == nil {
+		if step.ExpectVersion != 0 {
+			step.Opts.ObjectStream.Version = step.ExpectVersion
+		}
 		require.Equal(t, step.Opts.ObjectStream, object.ObjectStream)
 	}
 	return object
@@ -106,6 +110,8 @@ func (step CommitObject) Check(ctx *testcontext.Context, t require.TestingT, db
 type CommitObjectWithSegments struct {
 	Opts          metabase.CommitObjectWithSegments
 	Deleted       []metabase.DeletedSegmentInfo
+	ExpectVersion metabase.Version
+
 	ErrClass *errs.Class
 	ErrText  string
 }
@@ -115,6 +121,9 @@ func (step CommitObjectWithSegments) Check(ctx *testcontext.Context, t testing.T
 	object, deleted, err := db.CommitObjectWithSegments(ctx, step.Opts)
 	checkError(t, err, step.ErrClass, step.ErrText)
 	if err == nil {
+		if step.ExpectVersion != 0 {
+			step.Opts.ObjectStream.Version = step.ExpectVersion
+		}
 		require.Equal(t, step.Opts.ObjectStream, object.ObjectStream)
 	}
 	require.Equal(t, step.Deleted, deleted)