storj/satellite/metabase/copy_object_test.go

// Copyright (C) 2022 Storj Labs, Inc.
// See LICENSE for copying information.

package metabase_test

import (
	"testing"
	"time"

	"github.com/stretchr/testify/require"

	"storj.io/common/storj"
	"storj.io/common/testcontext"
	"storj.io/common/testrand"
	"storj.io/storj/satellite/metabase"
	"storj.io/storj/satellite/metabase/metabasetest"
)

func TestBeginCopyObject(t *testing.T) {
	metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
		obj := metabasetest.RandObjectStream()

		for _, test := range metabasetest.InvalidObjectLocations(obj.Location()) {
			test := test
			t.Run(test.Name, func(t *testing.T) {
				defer metabasetest.DeleteAll{}.Check(ctx, t, db)
				metabasetest.BeginCopyObject{
					Opts: metabase.BeginCopyObject{
						ObjectLocation: test.ObjectLocation,
					},
					ErrClass: test.ErrClass,
					ErrText:  test.ErrText,
				}.Check(ctx, t, db)

				metabasetest.Verify{}.Check(ctx, t, db)
			})
		}

		t.Run("begin copy object", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			expectedRawObjects := []metabase.RawObject{}
			expectedRawSegments := []metabase.RawSegment{}

			for _, expectedVersion := range []metabase.Version{1, 2, 3, 11} {
				obj.ObjectKey = metabasetest.RandObjectKey()
				obj.StreamID = testrand.UUID()
				obj.Version = expectedVersion
				expectedObject, expectedSegments := metabasetest.CreateTestObject{
					CommitObject: &metabase.CommitObject{
						ObjectStream: obj,
					},
				}.Run(ctx, t, db, obj, 10)

				expectedRawObjects = append(expectedRawObjects, metabase.RawObject(expectedObject))

				var encKeyAndNonces []metabase.EncryptedKeyAndNonce
				for _, expectedSegment := range expectedSegments {
					encKeyAndNonces = append(encKeyAndNonces, metabase.EncryptedKeyAndNonce{
						EncryptedKeyNonce: expectedSegment.EncryptedKeyNonce,
						EncryptedKey:      expectedSegment.EncryptedKey,
						Position:          expectedSegment.Position,
					})

					expectedRawSegments = append(expectedRawSegments, metabase.RawSegment(expectedSegment))
				}

				metabasetest.BeginCopyObject{
					Opts: metabase.BeginCopyObject{
						ObjectLocation: obj.Location(),
					},
					Result: metabase.BeginCopyObjectResult{
						StreamID:             expectedObject.StreamID,
						Version:              expectedVersion,
						EncryptedKeysNonces:  encKeyAndNonces,
						EncryptionParameters: expectedObject.Encryption,
					},
				}.Check(ctx, t, db)
			}

			metabasetest.Verify{
				Objects:  expectedRawObjects,
				Segments: expectedRawSegments,
			}.Check(ctx, t, db)
		})
	})
}

func TestFinishCopyObject(t *testing.T) {
	metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
		obj := metabasetest.RandObjectStream()
		newBucketName := "New bucket name"

		newStreamID := testrand.UUID()
		for _, test := range metabasetest.InvalidObjectStreams(obj) {
			test := test
			t.Run(test.Name, func(t *testing.T) {
				defer metabasetest.DeleteAll{}.Check(ctx, t, db)
				metabasetest.FinishCopyObject{
					Opts: metabase.FinishCopyObject{
						NewBucket:    newBucketName,
						ObjectStream: test.ObjectStream,
						NewStreamID:  newStreamID,
					},
					ErrClass: test.ErrClass,
					ErrText:  test.ErrText,
				}.Check(ctx, t, db)

				metabasetest.Verify{}.Check(ctx, t, db)
			})
		}

		t.Run("invalid NewBucket", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					ObjectStream:                 obj,
					NewEncryptedObjectKey:        metabasetest.RandObjectKey(),
					NewEncryptedMetadataKey:      []byte{1, 2, 3},
					NewEncryptedMetadataKeyNonce: testrand.Nonce(),
					NewStreamID:                  newStreamID,
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "NewBucket is missing",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("invalid NewStreamID", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					ObjectStream:                 obj,
					NewBucket:                    newBucketName,
					NewEncryptedObjectKey:        metabasetest.RandObjectKey(),
					NewEncryptedMetadataKey:      []byte{1, 2, 3},
					NewEncryptedMetadataKeyNonce: testrand.Nonce(),
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "NewStreamID is missing",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})
		t.Run("copy to the same StreamID", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					ObjectStream: obj,
					NewBucket:    newBucketName,
					NewStreamID:  obj.StreamID,
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "StreamIDs are identical",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("invalid NewEncryptedObjectKey", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewBucket:    newBucketName,
					ObjectStream: obj,
					NewStreamID:  newStreamID,
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "NewEncryptedObjectKey is missing",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("invalid EncryptedMetadataKeyNonce", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewBucket:               newBucketName,
					ObjectStream:            obj,
					NewEncryptedObjectKey:   metabasetest.RandObjectKey(),
					NewStreamID:             newStreamID,
					NewEncryptedMetadataKey: []byte{0},
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "EncryptedMetadataKeyNonce is missing",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("invalid EncryptedMetadataKey", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewBucket:                    newBucketName,
					ObjectStream:                 obj,
					NewEncryptedObjectKey:        metabasetest.RandObjectKey(),
					NewEncryptedMetadataKeyNonce: testrand.Nonce(),
					NewStreamID:                  newStreamID,
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "EncryptedMetadataKey is missing",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("empty EncryptedMetadataKey and EncryptedMetadataKeyNonce", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewBucket:             newBucketName,
					ObjectStream:          obj,
					NewEncryptedObjectKey: metabasetest.RandObjectKey(),
					NewStreamID:           newStreamID,
				},
				// validation pass without EncryptedMetadataKey and EncryptedMetadataKeyNonce
				ErrClass: &storj.ErrObjectNotFound,
				ErrText:  "source object not found",
			}.Check(ctx, t, db)
		})

		t.Run("empty EncryptedMetadata with OverrideMetadata=true", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewBucket:             newBucketName,
					ObjectStream:          obj,
					NewEncryptedObjectKey: metabasetest.RandObjectKey(),

					OverrideMetadata:             true,
					NewEncryptedMetadataKey:      []byte{1},
					NewEncryptedMetadataKeyNonce: testrand.Nonce(),
					NewStreamID:                  newStreamID,
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be not set if EncryptedMetadata is not set",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("empty NewEncryptedMetadataKey and NewEncryptedMetadataKeyNonce with OverrideMetadata=true", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewBucket:             newBucketName,
					ObjectStream:          obj,
					NewEncryptedObjectKey: metabasetest.RandObjectKey(),
					NewStreamID:           newStreamID,

					OverrideMetadata:     true,
					NewEncryptedMetadata: testrand.BytesInt(256),
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be set if EncryptedMetadata is set",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		t.Run("object does not exist", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			newObj := metabasetest.RandObjectStream()

			newEncryptedMetadataKeyNonce := testrand.Nonce()
			newEncryptedMetadataKey := testrand.Bytes(32)
			newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, 10)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewBucket:                    newBucketName,
					NewStreamID:                  newStreamID,
					ObjectStream:                 newObj,
					NewSegmentKeys:               newEncryptedKeysNonces,
					NewEncryptedObjectKey:        metabasetest.RandObjectKey(),
					NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
					NewEncryptedMetadataKey:      newEncryptedMetadataKey,
				},
				ErrClass: &storj.ErrObjectNotFound,
				ErrText:  "source object not found",
			}.Check(ctx, t, db)

			metabasetest.Verify{}.Check(ctx, t, db)
		})

		// Assert that an error occurs when a new object has been put at the source key
		// between BeginCopyObject and FinishCopyObject. (stream_id of source key changed)
		t.Run("source object changed", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			newObj, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  obj,
					OverrideEncryptedMetadata:     true,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, obj, 2)

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewStreamID: testrand.UUID(),
					NewBucket:   newBucketName,
					ObjectStream: metabase.ObjectStream{
						ProjectID:  newObj.ProjectID,
						BucketName: newObj.BucketName,
						ObjectKey:  newObj.ObjectKey,
						Version:    newObj.Version,
						StreamID:   testrand.UUID(),
					},
					NewSegmentKeys: []metabase.EncryptedKeyAndNonce{
						metabasetest.RandEncryptedKeyAndNonce(0),
						metabasetest.RandEncryptedKeyAndNonce(1),
					},
					NewEncryptedObjectKey:        metabasetest.RandObjectKey(),
					NewEncryptedMetadataKeyNonce: testrand.Nonce(),
					NewEncryptedMetadataKey:      testrand.Bytes(32),
				},
				ErrClass: &storj.ErrObjectNotFound,
				ErrText:  "object was changed during copy",
			}.Check(ctx, t, db)
		})

		t.Run("not enough segments", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			numberOfSegments := 10

			newObj, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  obj,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, obj, byte(numberOfSegments))

			newEncryptedMetadataKeyNonce := testrand.Nonce()
			newEncryptedMetadataKey := testrand.Bytes(32)
			newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount-1)
			expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount)

			for i := 0; i < int(newObj.SegmentCount-1); i++ {
				newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
					Position:          metabase.SegmentPosition{Index: uint32(i)},
					EncryptedKeyNonce: testrand.Nonce().Bytes(),
					EncryptedKey:      testrand.Bytes(32),
				}

				expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
				expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
				expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
				expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
				expectedSegments[i].EncryptedSize = int32(0)
			}

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewBucket:                    newBucketName,
					ObjectStream:                 obj,
					NewStreamID:                  newStreamID,
					NewSegmentKeys:               newEncryptedKeysNonces,
					NewEncryptedObjectKey:        metabasetest.RandObjectKey(),
					NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
					NewEncryptedMetadataKey:      newEncryptedMetadataKey,
				},
				ErrClass: &metabase.ErrInvalidRequest,
				ErrText:  "wrong number of segments keys received (received 9, need 10)",
			}.Check(ctx, t, db)
		})

		t.Run("wrong segment indexes", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			numberOfSegments := 10

			newObj, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  obj,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, obj, byte(numberOfSegments))

			newEncryptedMetadataKeyNonce := testrand.Nonce()
			newEncryptedMetadataKey := testrand.Bytes(32)
			newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount)
			expectedEncryptedSize := 1060
			expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount)

			for i := 0; i < int(newObj.SegmentCount); i++ {
				newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
					Position:          metabase.SegmentPosition{Index: uint32(i + 5)},
					EncryptedKeyNonce: testrand.Nonce().Bytes(),
					EncryptedKey:      testrand.Bytes(32),
				}

				expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
				expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
				expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
				expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
				expectedSegments[i].EncryptedSize = int32(expectedEncryptedSize)
			}

			metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					NewStreamID:                  newStreamID,
					NewBucket:                    newBucketName,
					ObjectStream:                 obj,
					NewSegmentKeys:               newEncryptedKeysNonces,
					NewEncryptedObjectKey:        metabasetest.RandObjectKey(),
					NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
					NewEncryptedMetadataKey:      newEncryptedMetadataKey,
				},
				ErrClass: &metabase.Error,
				ErrText:  "missing new segment keys for segment 0",
			}.Check(ctx, t, db)
		})

		t.Run("returned object", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			expectedRawObjects := []metabase.RawObject{}

			for _, expectedVersion := range []metabase.Version{1} {
				objStream := metabasetest.RandObjectStream()
				objStream.Version = expectedVersion

				copyStream := metabasetest.RandObjectStream()
				copyStream.ProjectID = objStream.ProjectID
				copyStream.BucketName = objStream.BucketName

				originalObj, _ := metabasetest.CreateTestObject{
					CommitObject: &metabase.CommitObject{
						ObjectStream:                  objStream,
						EncryptedMetadata:             testrand.Bytes(64),
						EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
						EncryptedMetadataEncryptedKey: testrand.Bytes(265),
					},
				}.Run(ctx, t, db, objStream, 0)

				metadataNonce := testrand.Nonce()
				expectedCopyObject := originalObj
				expectedCopyObject.ObjectKey = copyStream.ObjectKey
				expectedCopyObject.StreamID = copyStream.StreamID
				expectedCopyObject.Version = metabase.DefaultVersion // it will always copy into first available version
				expectedCopyObject.EncryptedMetadataEncryptedKey = testrand.Bytes(32)
				expectedCopyObject.EncryptedMetadataNonce = metadataNonce.Bytes()

				objectCopy := metabasetest.FinishCopyObject{
					Opts: metabase.FinishCopyObject{
						ObjectStream:                 objStream,
						NewBucket:                    copyStream.BucketName,
						NewStreamID:                  copyStream.StreamID,
						NewEncryptedObjectKey:        copyStream.ObjectKey,
						NewEncryptedMetadataKey:      expectedCopyObject.EncryptedMetadataEncryptedKey,
						NewEncryptedMetadataKeyNonce: metadataNonce,
					},
					Result: expectedCopyObject,
				}.Check(ctx, t, db)

				require.NotEqual(t, originalObj.CreatedAt, objectCopy.CreatedAt)

				expectedRawObjects = append(expectedRawObjects, metabase.RawObject(originalObj))
				expectedRawObjects = append(expectedRawObjects, metabase.RawObject(expectedCopyObject))
			}

			metabasetest.Verify{
				Objects: expectedRawObjects,
				Copies:  nil, // no copies because we have only inline segments
			}.Check(ctx, t, db)
		})

		t.Run("finish copy object with existing metadata", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			numberOfSegments := 10
			copyStream := metabasetest.RandObjectStream()

			originalObj, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  obj,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, obj, byte(numberOfSegments))

			copyObj, expectedOriginalSegments, expectedCopySegments := metabasetest.CreateObjectCopy{
				OriginalObject:   originalObj,
				CopyObjectStream: &copyStream,
			}.Run(ctx, t, db)

			var expectedRawSegments []metabase.RawSegment
			expectedRawSegments = append(expectedRawSegments, expectedOriginalSegments...)
			expectedRawSegments = append(expectedRawSegments, expectedCopySegments...)

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(originalObj),
					metabase.RawObject(copyObj),
				},
				Segments: expectedRawSegments,
				Copies: []metabase.RawCopy{{
					StreamID:         copyObj.StreamID,
					AncestorStreamID: originalObj.StreamID,
				}},
			}.Check(ctx, t, db)

			// TODO find better names
			copyOfCopyStream := metabasetest.RandObjectStream()
			copyOfCopyObj, _, expectedCopyOfCopySegments := metabasetest.CreateObjectCopy{
				OriginalObject:   copyObj,
				CopyObjectStream: &copyOfCopyStream,
			}.Run(ctx, t, db)

			expectedRawSegments = append(expectedRawSegments, expectedCopyOfCopySegments...)

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(originalObj),
					metabase.RawObject(copyObj),
					metabase.RawObject(copyOfCopyObj),
				},
				Segments: expectedRawSegments,
				Copies: []metabase.RawCopy{{
					StreamID:         copyStream.StreamID,
					AncestorStreamID: originalObj.StreamID,
				}, {
					StreamID:         copyOfCopyObj.StreamID,
					AncestorStreamID: originalObj.StreamID,
				}},
			}.Check(ctx, t, db)
		})

		t.Run("finish copy object with new metadata", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			copyStream := metabasetest.RandObjectStream()
			copyStreamNoOverride := metabasetest.RandObjectStream()

			originalMetadata := testrand.Bytes(64)
			originalMetadataNonce := testrand.Nonce().Bytes()
			originalMetadataEncryptedKey := testrand.Bytes(265)

			originalObj, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  obj,
					OverrideEncryptedMetadata:     true,
					EncryptedMetadata:             originalMetadata,
					EncryptedMetadataNonce:        originalMetadataNonce,
					EncryptedMetadataEncryptedKey: originalMetadataEncryptedKey,
				},
			}.Run(ctx, t, db, obj, 0)

			newMetadata := testrand.Bytes(256)
			newMetadataKey := testrand.Bytes(32)
			newMetadataKeyNonce := testrand.Nonce()

			// do a copy without OverrideMetadata field set to true,
			// metadata shouldn't be updated even if NewEncryptedMetadata
			// field is set
			copyObjNoOverride, _, _ := metabasetest.CreateObjectCopy{
				OriginalObject:   originalObj,
				CopyObjectStream: &copyStreamNoOverride,
				FinishObject: &metabase.FinishCopyObject{
					ObjectStream: originalObj.ObjectStream,

					NewBucket:   copyStreamNoOverride.BucketName,
					NewStreamID: copyStreamNoOverride.StreamID,

					NewEncryptedObjectKey: copyStreamNoOverride.ObjectKey,

					OverrideMetadata:             false,
					NewEncryptedMetadata:         newMetadata,
					NewEncryptedMetadataKeyNonce: newMetadataKeyNonce,
					NewEncryptedMetadataKey:      newMetadataKey,
				},
			}.Run(ctx, t, db)

			require.Equal(t, originalMetadata, copyObjNoOverride.EncryptedMetadata)
			require.Equal(t, newMetadataKey, copyObjNoOverride.EncryptedMetadataEncryptedKey)
			require.Equal(t, newMetadataKeyNonce.Bytes(), copyObjNoOverride.EncryptedMetadataNonce)

			// do a copy WITH OverrideMetadata field set to true,
			// metadata should be updated to NewEncryptedMetadata
			copyObj, _, _ := metabasetest.CreateObjectCopy{
				OriginalObject:   originalObj,
				CopyObjectStream: &copyStream,
				FinishObject: &metabase.FinishCopyObject{
					ObjectStream: originalObj.ObjectStream,

					NewBucket:   copyStream.BucketName,
					NewStreamID: copyStream.StreamID,

					NewEncryptedObjectKey: copyStream.ObjectKey,

					OverrideMetadata:             true,
					NewEncryptedMetadata:         newMetadata,
					NewEncryptedMetadataKeyNonce: newMetadataKeyNonce,
					NewEncryptedMetadataKey:      newMetadataKey,
				},
			}.Run(ctx, t, db)

			require.Equal(t, newMetadata, copyObj.EncryptedMetadata)
			require.Equal(t, newMetadataKey, copyObj.EncryptedMetadataEncryptedKey)
			require.Equal(t, newMetadataKeyNonce.Bytes(), copyObj.EncryptedMetadataNonce)

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(originalObj),
					metabase.RawObject(copyObj),
					metabase.RawObject(copyObjNoOverride),
				},
				Copies: nil,
			}.Check(ctx, t, db)
		})

		t.Run("finish copy object to already existing destination", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			// Test:
			// - 3 objects: objA, objB, objC
			// - copy objB to objA - creating objBprime
			// - check that segments of original objA have been deleted
			// - check that we now have three objects: objBprime, objB, objC
			// - copy objC to objB creating objCprime
			// - check that we now have three objects: objBprime, objCprime, objC
			// - check that objBprime has become an original object, now that its ancestor
			// objB has been overwritten

			// object that already exists
			objStreamA := metabasetest.RandObjectStream()
			objStreamB := metabasetest.RandObjectStream()
			objStreamC := metabasetest.RandObjectStream()

			// set same projectID for all
			objStreamB.ProjectID = objStreamA.ProjectID
			objStreamC.ProjectID = objStreamA.ProjectID

			objA, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  objStreamA,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, objStreamA, 4)

			objB, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  objStreamB,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, objStreamB, 3)

			objC, segmentsOfC := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  objStreamC,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, objStreamC, 1)

			// B' is a copy of B to A
			objStreamBprime := objStreamA
			objStreamBprime.StreamID = testrand.UUID()
			objBprime, expectedSegmentsOfB, expectedSegmentsOfBprime := metabasetest.CreateObjectCopy{
				OriginalObject:   objB,
				CopyObjectStream: &objStreamBprime,
			}.Run(ctx, t, db)

			// check that we indeed overwrote object A
			require.Equal(t, objA.BucketName, objBprime.BucketName)
			require.Equal(t, objA.ProjectID, objBprime.ProjectID)
			require.Equal(t, objA.ObjectKey, objBprime.ObjectKey)

			require.NotEqual(t, objA.StreamID, objBprime.StreamID)

			var expectedRawSegments []metabase.RawSegment
			expectedRawSegments = append(expectedRawSegments, expectedSegmentsOfBprime...)
			expectedRawSegments = append(expectedRawSegments, expectedSegmentsOfB...)
			expectedRawSegments = append(expectedRawSegments, metabasetest.SegmentsToRaw(segmentsOfC)...)

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(objBprime),
					metabase.RawObject(objB),
					metabase.RawObject(objC),
				},
				Segments: expectedRawSegments,
				Copies: []metabase.RawCopy{{
					StreamID:         objBprime.StreamID,
					AncestorStreamID: objB.StreamID,
				}},
			}.Check(ctx, t, db)

			// C' is a copy of C to B
			objStreamCprime := objStreamB
			objStreamCprime.StreamID = testrand.UUID()
			objCprime, _, expectedSegmentsOfCprime := metabasetest.CreateObjectCopy{
				OriginalObject:   objC,
				CopyObjectStream: &objStreamCprime,
			}.Run(ctx, t, db)

			require.Equal(t, objStreamB.BucketName, objCprime.BucketName)
			require.Equal(t, objStreamB.ProjectID, objCprime.ProjectID)
			require.Equal(t, objStreamB.ObjectKey, objCprime.ObjectKey)
			require.NotEqual(t, objB.StreamID, objCprime)

			// B' should become the original of B and now hold pieces.
			for i := range expectedSegmentsOfBprime {
				expectedSegmentsOfBprime[i].EncryptedETag = nil
				expectedSegmentsOfBprime[i].Pieces = expectedSegmentsOfB[i].Pieces
			}

			var expectedSegments []metabase.RawSegment
			expectedSegments = append(expectedSegments, expectedSegmentsOfBprime...)
			expectedSegments = append(expectedSegments, expectedSegmentsOfCprime...)
			expectedSegments = append(expectedSegments, metabasetest.SegmentsToRaw(segmentsOfC)...)

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(objBprime),
					metabase.RawObject(objCprime),
					metabase.RawObject(objC),
				},
				Segments: expectedSegments,
				Copies: []metabase.RawCopy{{
					StreamID:         objCprime.StreamID,
					AncestorStreamID: objC.StreamID,
				}},
			}.Check(ctx, t, db)
		})

		// checks that a copy can be copied to it's ancestor location
		t.Run("Copy child to ancestor", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			originalObjStream := metabasetest.RandObjectStream()
			copyObjStream := metabasetest.RandObjectStream()
			// Copy back to original object key.
			// StreamID is independent of key.
			copyBackObjStream := originalObjStream
			copyBackObjStream.StreamID = testrand.UUID()

			originalObj, originalSegments := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  originalObjStream,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, originalObjStream, 4)

			copyObj, _, copySegments := metabasetest.CreateObjectCopy{
				OriginalObject:   originalObj,
				CopyObjectStream: &copyObjStream,
			}.Run(ctx, t, db)

			// Copy the copy back to the source location
			opts := metabase.FinishCopyObject{
				// source
				ObjectStream: copyObj.ObjectStream,
				// destination
				NewBucket:             originalObj.BucketName,
				NewEncryptedObjectKey: originalObj.ObjectKey,
				NewStreamID:           copyBackObjStream.StreamID,
				OverrideMetadata:      false,
				NewSegmentKeys: []metabase.EncryptedKeyAndNonce{
					metabasetest.RandEncryptedKeyAndNonce(0),
					metabasetest.RandEncryptedKeyAndNonce(1),
					metabasetest.RandEncryptedKeyAndNonce(2),
					metabasetest.RandEncryptedKeyAndNonce(3),
				},
			}
			metabasetest.CreateObjectCopy{
				OriginalObject:   copyObj,
				CopyObjectStream: &copyBackObjStream,
				FinishObject:     &opts,
			}.Run(ctx, t, db)

			// expected object at the location which was previously the original object
			copyBackObj := originalObj
			copyBackObj.Version++ // copy is placed into first available version
			copyBackObj.StreamID = opts.NewStreamID

			for i := 0; i < 4; i++ {
				copySegments[i].Pieces = originalSegments[i].Pieces
				copySegments[i].InlineData = originalSegments[i].InlineData
				copySegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy

				originalSegments[i].StreamID = opts.NewStreamID
				originalSegments[i].Pieces = nil
				originalSegments[i].InlineData = nil
				originalSegments[i].EncryptedKey = opts.NewSegmentKeys[i].EncryptedKey
				originalSegments[i].EncryptedKeyNonce = opts.NewSegmentKeys[i].EncryptedKeyNonce
				originalSegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy
			}

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(copyObj),
					metabase.RawObject(copyBackObj),
				},
				Segments: append(metabasetest.SegmentsToRaw(originalSegments), copySegments...),
				Copies: []metabase.RawCopy{{
					StreamID:         opts.NewStreamID,
					AncestorStreamID: copyObjStream.StreamID,
				}},
			}.Check(ctx, t, db)
		})

		// checks that a copy ancestor can be copied to itself
		t.Run("Copy ancestor to itself", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			originalObjStream := metabasetest.RandObjectStream()
			copyObjStream := metabasetest.RandObjectStream()
			// Copy back to same object key.
			// StreamID is independent of key.
			copyBackObjStream := originalObjStream
			copyBackObjStream.StreamID = testrand.UUID()

			originalObj, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  originalObjStream,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, originalObjStream, 4)

			copyObj, originalSegments, copySegments := metabasetest.CreateObjectCopy{
				OriginalObject:   originalObj,
				CopyObjectStream: &copyObjStream,
			}.Run(ctx, t, db)

			opts := metabase.FinishCopyObject{
				// source
				ObjectStream: copyObj.ObjectStream,
				// destination
				NewBucket:             originalObj.BucketName,
				NewEncryptedObjectKey: originalObj.ObjectKey,
				NewStreamID:           copyBackObjStream.StreamID,
				OverrideMetadata:      false,
				NewSegmentKeys: []metabase.EncryptedKeyAndNonce{
					metabasetest.RandEncryptedKeyAndNonce(0),
					metabasetest.RandEncryptedKeyAndNonce(1),
					metabasetest.RandEncryptedKeyAndNonce(2),
					metabasetest.RandEncryptedKeyAndNonce(3),
				},
			}
			// Copy the copy back to the source location
			metabasetest.CreateObjectCopy{
				OriginalObject:   originalObj,
				CopyObjectStream: &copyBackObjStream,
				FinishObject:     &opts,
			}.Run(ctx, t, db)

			copyBackObj := originalObj
			copyBackObj.Version++ // copy is placed into first available version
			copyBackObj.StreamID = copyBackObjStream.StreamID

			for i := 0; i < 4; i++ {
				copySegments[i].Pieces = originalSegments[i].Pieces
				copySegments[i].InlineData = originalSegments[i].InlineData
				copySegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy

				originalSegments[i].StreamID = opts.NewStreamID
				originalSegments[i].Pieces = nil
				originalSegments[i].InlineData = nil
				originalSegments[i].EncryptedKey = opts.NewSegmentKeys[i].EncryptedKey
				originalSegments[i].EncryptedKeyNonce = opts.NewSegmentKeys[i].EncryptedKeyNonce
				originalSegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy
			}

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(copyObj),
					metabase.RawObject(copyBackObj),
				},
				Segments: append(originalSegments, copySegments...),
				Copies: []metabase.RawCopy{{
					StreamID:         copyBackObjStream.StreamID,
					AncestorStreamID: copyObjStream.StreamID,
				}},
			}.Check(ctx, t, db)
		})

		t.Run("copied segments has same expires_at as original", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			expiresAt := time.Now().Add(2 * time.Hour)

			objStream := metabasetest.RandObjectStream()
			copyStream := metabasetest.RandObjectStream()
			copyStream.ProjectID = objStream.ProjectID
			copyStream.BucketName = objStream.BucketName

			originalObj := metabasetest.CreateExpiredObject(ctx, t, db, objStream, 10, expiresAt)

			metadataNonce := testrand.Nonce()
			expectedCopyObject := originalObj
			expectedCopyObject.ObjectKey = copyStream.ObjectKey
			expectedCopyObject.StreamID = copyStream.StreamID
			expectedCopyObject.EncryptedMetadataEncryptedKey = testrand.Bytes(32)
			expectedCopyObject.EncryptedMetadataNonce = metadataNonce.Bytes()

			newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, originalObj.SegmentCount)
			expectedSegments := make([]metabase.RawSegment, originalObj.SegmentCount)

			for i := 0; i < int(originalObj.SegmentCount); i++ {
				newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
					Position:          metabase.SegmentPosition{Index: uint32(i)},
					EncryptedKeyNonce: testrand.Nonce().Bytes(),
					EncryptedKey:      testrand.Bytes(32),
				}

				expectedSegments[i] = metabasetest.DefaultRawSegment(originalObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
				expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
				expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
				expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
				expectedSegments[i].EncryptedSize = int32(0)
			}

			copyObj := metabasetest.FinishCopyObject{
				Opts: metabase.FinishCopyObject{
					ObjectStream:                 objStream,
					NewBucket:                    copyStream.BucketName,
					NewStreamID:                  copyStream.StreamID,
					NewEncryptedObjectKey:        copyStream.ObjectKey,
					NewEncryptedMetadataKey:      expectedCopyObject.EncryptedMetadataEncryptedKey,
					NewEncryptedMetadataKeyNonce: metadataNonce,
					NewSegmentKeys:               newEncryptedKeysNonces,
				},
				Result: expectedCopyObject,
			}.Check(ctx, t, db)

			var listSegments []metabase.Segment

			copiedSegments, err := db.ListSegments(ctx, metabase.ListSegments{
				StreamID: copyObj.StreamID,
			})
			require.NoError(t, err)

			originalSegments, err := db.ListSegments(ctx, metabase.ListSegments{
				StreamID: originalObj.StreamID,
			})
			require.NoError(t, err)

			listSegments = append(listSegments, originalSegments.Segments...)
			listSegments = append(listSegments, copiedSegments.Segments...)

			for _, v := range listSegments {
				require.Equal(t, expiresAt.Unix(), v.ExpiresAt.Unix())
			}

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(originalObj),
					metabase.RawObject(copyObj),
				},
				Segments: metabasetest.SegmentsToRaw(listSegments),
				Copies: []metabase.RawCopy{{
					StreamID:         copyObj.StreamID,
					AncestorStreamID: originalObj.StreamID,
				}},
			}.Check(ctx, t, db)
		})

		t.Run("finish copy object to same destination", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			obj := metabasetest.RandObjectStream()
			numberOfSegments := 10
			originalObj, _ := metabasetest.CreateTestObject{
				CommitObject: &metabase.CommitObject{
					ObjectStream:                  obj,
					EncryptedMetadata:             testrand.Bytes(64),
					EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
					EncryptedMetadataEncryptedKey: testrand.Bytes(265),
				},
			}.Run(ctx, t, db, obj, byte(numberOfSegments))

			obj.StreamID = testrand.UUID()
			_, expectedOriginalSegments, _ := metabasetest.CreateObjectCopy{
				OriginalObject:   originalObj,
				CopyObjectStream: &obj,
			}.Run(ctx, t, db)

			metabasetest.Verify{
				Objects: []metabase.RawObject{
					metabase.RawObject(originalObj),
				},
				Segments: expectedOriginalSegments,
				Copies:   []metabase.RawCopy{},
			}.Check(ctx, t, db)
		})

		t.Run("finish copy object to existing pending destination", func(t *testing.T) {
			defer metabasetest.DeleteAll{}.Check(ctx, t, db)

			now := time.Now()
			zombieDeadline := now.Add(24 * time.Hour)

			sourceObjStream := metabasetest.RandObjectStream()
			destinationObjStream := metabasetest.RandObjectStream()
			destinationObjStream.ProjectID = sourceObjStream.ProjectID
			// testcases:
			// - versions of pending objects
			// - version of committed object
			// - expected copy version

			testCases := []struct {
				Bucket                      string
				Key                         metabase.ObjectKey
				NewBucket                   string
				NewKey                      metabase.ObjectKey
				sourcePendingVersions       []metabase.Version
				sourceCommittedVersion      metabase.Version
				destinationPendingVersions  []metabase.Version
				destinationCommittedVersion metabase.Version
				expectedCopyVersion         metabase.Version
			}{
				// the same bucket
				{"testbucket", "object", "testbucket", "new-object",
					[]metabase.Version{}, 2,
					[]metabase.Version{}, 1,
					2},
				{"testbucket", "object", "testbucket", "new-object",
					[]metabase.Version{}, 1,
					[]metabase.Version{1}, 2,
					3},
				{"testbucket", "object", "testbucket", "new-object",
					[]metabase.Version{}, 1,
					[]metabase.Version{1, 3}, 2,
					4},
				{"testbucket", "object", "testbucket", "new-object",
					[]metabase.Version{1, 5}, 2,
					[]metabase.Version{1, 3}, 2,
					4},
				{"testbucket", "object", "newbucket", "object",
					[]metabase.Version{2, 3}, 1,
					[]metabase.Version{1, 5}, 2,
					6},
			}

			for _, tc := range testCases {
				metabasetest.DeleteAll{}.Check(ctx, t, db)
				db.TestingEnableMultipleVersions(false)
				sourceObjStream.BucketName = tc.Bucket
				sourceObjStream.ObjectKey = tc.Key
				destinationObjStream.BucketName = tc.NewBucket
				destinationObjStream.ObjectKey = tc.NewKey

				var rawObjects []metabase.RawObject
				for _, version := range tc.sourcePendingVersions {
					sourceObjStream.Version = version
					sourceObjStream.StreamID = testrand.UUID()
					metabasetest.CreatePendingObject(ctx, t, db, sourceObjStream, 0)

					rawObjects = append(rawObjects, metabase.RawObject{
						ObjectStream: sourceObjStream,
						CreatedAt:    now,
						Status:       metabase.Pending,

						Encryption:             metabasetest.DefaultEncryption,
						ZombieDeletionDeadline: &zombieDeadline,
					})
				}
				sourceObjStream.Version = tc.sourceCommittedVersion
				sourceObjStream.StreamID = testrand.UUID()
				sourceObj, _ := metabasetest.CreateTestObject{
					BeginObjectExactVersion: &metabase.BeginObjectExactVersion{
						ObjectStream: sourceObjStream,
						Encryption:   metabasetest.DefaultEncryption,
					},
					CommitObject: &metabase.CommitObject{
						ObjectStream:                  sourceObjStream,
						OverrideEncryptedMetadata:     true,
						EncryptedMetadata:             testrand.Bytes(64),
						EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
						EncryptedMetadataEncryptedKey: testrand.Bytes(265),
					},
				}.Run(ctx, t, db, sourceObjStream, 0)

				rawObjects = append(rawObjects, metabase.RawObject(sourceObj))

				for _, version := range tc.destinationPendingVersions {
					destinationObjStream.Version = version
					destinationObjStream.StreamID = testrand.UUID()
					metabasetest.CreatePendingObject(ctx, t, db, destinationObjStream, 0)

					rawObjects = append(rawObjects, metabase.RawObject{
						ObjectStream: destinationObjStream,
						CreatedAt:    now,
						Status:       metabase.Pending,

						Encryption:             metabasetest.DefaultEncryption,
						ZombieDeletionDeadline: &zombieDeadline,
					})
				}

				if tc.destinationCommittedVersion != 0 {
					destinationObjStream.StreamID = testrand.UUID()
					destinationObjStream.Version = tc.destinationCommittedVersion
					_, _ = metabasetest.CreateTestObject{
						BeginObjectExactVersion: &metabase.BeginObjectExactVersion{
							ObjectStream: destinationObjStream,
							Encryption:   metabasetest.DefaultEncryption,
						},
						CommitObject: &metabase.CommitObject{
							ObjectStream:                  destinationObjStream,
							OverrideEncryptedMetadata:     true,
							EncryptedMetadata:             testrand.Bytes(64),
							EncryptedMetadataNonce:        testrand.Nonce().Bytes(),
							EncryptedMetadataEncryptedKey: testrand.Bytes(265),
						},
					}.Run(ctx, t, db, destinationObjStream, 0)
				}

				db.TestingEnableMultipleVersions(true)
				copyObj, expectedOriginalSegments, _ := metabasetest.CreateObjectCopy{
					OriginalObject:   sourceObj,
					CopyObjectStream: &destinationObjStream,
				}.Run(ctx, t, db)

				require.Equal(t, tc.expectedCopyVersion, copyObj.Version)

				rawObjects = append(rawObjects, metabase.RawObject(copyObj))

				metabasetest.Verify{
					Objects:  rawObjects,
					Segments: expectedOriginalSegments,
					Copies:   []metabase.RawCopy{},
				}.Check(ctx, t, db)
			}
		})
	})
}