fa76dfe577
We are preparing to use object versions internally and to do that we need to prepare different parts of the system to handle object versions different than '1'. This change adjust code responsible for server-side move and copy. What was done: * begin methods for move and copy are now using GetObjectLastCommitted to find object * results from begin move and copy operation contains now version to be able to map object correctly with finish operation * begin methods are putting version into satellite stream id and finish methods are using this version as parameter instead hardcoded value Fixes https://github.com/storj/storj/issues/4867 Change-Id: I1380911279c21e10a3fff0342793efd2e73eafad
987 lines
35 KiB
Go
987 lines
35 KiB
Go
// Copyright (C) 2022 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package metabase_test
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"storj.io/common/storj"
|
|
"storj.io/common/testcontext"
|
|
"storj.io/common/testrand"
|
|
"storj.io/storj/satellite/metabase"
|
|
"storj.io/storj/satellite/metabase/metabasetest"
|
|
)
|
|
|
|
func TestBeginCopyObject(t *testing.T) {
|
|
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
|
|
obj := metabasetest.RandObjectStream()
|
|
|
|
for _, test := range metabasetest.InvalidObjectLocations(obj.Location()) {
|
|
test := test
|
|
t.Run(test.Name, func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
metabasetest.BeginCopyObject{
|
|
Opts: metabase.BeginCopyObject{
|
|
ObjectLocation: test.ObjectLocation,
|
|
},
|
|
ErrClass: test.ErrClass,
|
|
ErrText: test.ErrText,
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
}
|
|
|
|
t.Run("begin copy object", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
expectedRawObjects := []metabase.RawObject{}
|
|
expectedRawSegments := []metabase.RawSegment{}
|
|
|
|
for _, expectedVersion := range []metabase.Version{1, 2, 3, 11} {
|
|
obj.StreamID = testrand.UUID()
|
|
obj.Version = expectedVersion
|
|
expectedObject, expectedSegments := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
},
|
|
}.Run(ctx, t, db, obj, 10)
|
|
|
|
expectedRawObjects = append(expectedRawObjects, metabase.RawObject(expectedObject))
|
|
|
|
var encKeyAndNonces []metabase.EncryptedKeyAndNonce
|
|
for _, expectedSegment := range expectedSegments {
|
|
encKeyAndNonces = append(encKeyAndNonces, metabase.EncryptedKeyAndNonce{
|
|
EncryptedKeyNonce: expectedSegment.EncryptedKeyNonce,
|
|
EncryptedKey: expectedSegment.EncryptedKey,
|
|
Position: expectedSegment.Position,
|
|
})
|
|
|
|
expectedRawSegments = append(expectedRawSegments, metabase.RawSegment(expectedSegment))
|
|
}
|
|
|
|
metabasetest.BeginCopyObject{
|
|
Opts: metabase.BeginCopyObject{
|
|
ObjectLocation: obj.Location(),
|
|
},
|
|
Result: metabase.BeginCopyObjectResult{
|
|
StreamID: expectedObject.StreamID,
|
|
Version: expectedVersion,
|
|
EncryptedKeysNonces: encKeyAndNonces,
|
|
EncryptionParameters: expectedObject.Encryption,
|
|
},
|
|
}.Check(ctx, t, db)
|
|
}
|
|
|
|
metabasetest.Verify{
|
|
Objects: expectedRawObjects,
|
|
Segments: expectedRawSegments,
|
|
}.Check(ctx, t, db)
|
|
})
|
|
})
|
|
}
|
|
|
|
func TestFinishCopyObject(t *testing.T) {
|
|
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
|
|
obj := metabasetest.RandObjectStream()
|
|
newBucketName := "New bucket name"
|
|
|
|
newStreamID := testrand.UUID()
|
|
for _, test := range metabasetest.InvalidObjectStreams(obj) {
|
|
test := test
|
|
t.Run(test.Name, func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
ObjectStream: test.ObjectStream,
|
|
NewStreamID: newStreamID,
|
|
},
|
|
ErrClass: test.ErrClass,
|
|
ErrText: test.ErrText,
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
}
|
|
|
|
t.Run("invalid NewBucket", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
ObjectStream: obj,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewEncryptedMetadataKey: []byte{1, 2, 3},
|
|
NewEncryptedMetadataKeyNonce: testrand.Nonce(),
|
|
NewStreamID: newStreamID,
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "NewBucket is missing",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("invalid NewStreamID", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
ObjectStream: obj,
|
|
NewBucket: newBucketName,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewEncryptedMetadataKey: []byte{1, 2, 3},
|
|
NewEncryptedMetadataKeyNonce: testrand.Nonce(),
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "NewStreamID is missing",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
t.Run("copy to the same StreamID", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
ObjectStream: obj,
|
|
NewBucket: newBucketName,
|
|
NewStreamID: obj.StreamID,
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "StreamIDs are identical",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("invalid NewEncryptedObjectKey", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
ObjectStream: obj,
|
|
NewStreamID: newStreamID,
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "NewEncryptedObjectKey is missing",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("invalid EncryptedMetadataKeyNonce", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
ObjectStream: obj,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewStreamID: newStreamID,
|
|
NewEncryptedMetadataKey: []byte{0},
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "EncryptedMetadataKeyNonce is missing",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("invalid EncryptedMetadataKey", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
ObjectStream: obj,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewEncryptedMetadataKeyNonce: testrand.Nonce(),
|
|
NewStreamID: newStreamID,
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "EncryptedMetadataKey is missing",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("empty EncryptedMetadataKey and EncryptedMetadataKeyNonce", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
ObjectStream: obj,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewStreamID: newStreamID,
|
|
},
|
|
// validation pass without EncryptedMetadataKey and EncryptedMetadataKeyNonce
|
|
ErrClass: &storj.ErrObjectNotFound,
|
|
ErrText: "source object not found",
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("empty EncryptedMetadata with OverrideMetadata=true", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
ObjectStream: obj,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
|
|
OverrideMetadata: true,
|
|
NewEncryptedMetadataKey: []byte{1},
|
|
NewEncryptedMetadataKeyNonce: testrand.Nonce(),
|
|
NewStreamID: newStreamID,
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be not set if EncryptedMetadata is not set",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("empty NewEncryptedMetadataKey and NewEncryptedMetadataKeyNonce with OverrideMetadata=true", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
ObjectStream: obj,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewStreamID: newStreamID,
|
|
|
|
OverrideMetadata: true,
|
|
NewEncryptedMetadata: testrand.BytesInt(256),
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "EncryptedMetadataNonce and EncryptedMetadataEncryptedKey must be set if EncryptedMetadata is set",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("object does not exist", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
newObj := metabasetest.RandObjectStream()
|
|
|
|
newEncryptedMetadataKeyNonce := testrand.Nonce()
|
|
newEncryptedMetadataKey := testrand.Bytes(32)
|
|
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, 10)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
NewStreamID: newStreamID,
|
|
ObjectStream: newObj,
|
|
NewSegmentKeys: newEncryptedKeysNonces,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
|
|
NewEncryptedMetadataKey: newEncryptedMetadataKey,
|
|
},
|
|
ErrClass: &storj.ErrObjectNotFound,
|
|
ErrText: "source object not found",
|
|
}.Check(ctx, t, db)
|
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
|
})
|
|
|
|
// Assert that an error occurs when a new object has been put at the source key
|
|
// between BeginCopyObject and FinishCopyObject. (stream_id of source key changed)
|
|
t.Run("source object changed", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
newObj, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
OverrideEncryptedMetadata: true,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, obj, 2)
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewStreamID: testrand.UUID(),
|
|
NewBucket: newBucketName,
|
|
ObjectStream: metabase.ObjectStream{
|
|
ProjectID: newObj.ProjectID,
|
|
BucketName: newObj.BucketName,
|
|
ObjectKey: newObj.ObjectKey,
|
|
Version: newObj.Version,
|
|
StreamID: testrand.UUID(),
|
|
},
|
|
NewSegmentKeys: []metabase.EncryptedKeyAndNonce{
|
|
metabasetest.RandEncryptedKeyAndNonce(0),
|
|
metabasetest.RandEncryptedKeyAndNonce(1),
|
|
},
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewEncryptedMetadataKeyNonce: testrand.Nonce(),
|
|
NewEncryptedMetadataKey: testrand.Bytes(32),
|
|
},
|
|
ErrClass: &storj.ErrObjectNotFound,
|
|
ErrText: "object was changed during copy",
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("not enough segments", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
numberOfSegments := 10
|
|
|
|
newObj, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, obj, byte(numberOfSegments))
|
|
|
|
newEncryptedMetadataKeyNonce := testrand.Nonce()
|
|
newEncryptedMetadataKey := testrand.Bytes(32)
|
|
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount-1)
|
|
expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount)
|
|
|
|
for i := 0; i < int(newObj.SegmentCount-1); i++ {
|
|
newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
|
|
Position: metabase.SegmentPosition{Index: uint32(i)},
|
|
EncryptedKeyNonce: testrand.Nonce().Bytes(),
|
|
EncryptedKey: testrand.Bytes(32),
|
|
}
|
|
|
|
expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
|
|
expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
|
|
expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
|
|
expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
|
|
expectedSegments[i].EncryptedSize = int32(0)
|
|
}
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewBucket: newBucketName,
|
|
ObjectStream: obj,
|
|
NewStreamID: newStreamID,
|
|
NewSegmentKeys: newEncryptedKeysNonces,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
|
|
NewEncryptedMetadataKey: newEncryptedMetadataKey,
|
|
},
|
|
ErrClass: &metabase.ErrInvalidRequest,
|
|
ErrText: "wrong number of segments keys received (received 9, need 10)",
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("wrong segment indexes", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
numberOfSegments := 10
|
|
|
|
newObj, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, obj, byte(numberOfSegments))
|
|
|
|
newEncryptedMetadataKeyNonce := testrand.Nonce()
|
|
newEncryptedMetadataKey := testrand.Bytes(32)
|
|
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount)
|
|
expectedEncryptedSize := 1060
|
|
expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount)
|
|
|
|
for i := 0; i < int(newObj.SegmentCount); i++ {
|
|
newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
|
|
Position: metabase.SegmentPosition{Index: uint32(i + 5)},
|
|
EncryptedKeyNonce: testrand.Nonce().Bytes(),
|
|
EncryptedKey: testrand.Bytes(32),
|
|
}
|
|
|
|
expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
|
|
expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
|
|
expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
|
|
expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
|
|
expectedSegments[i].EncryptedSize = int32(expectedEncryptedSize)
|
|
}
|
|
|
|
metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
NewStreamID: newStreamID,
|
|
NewBucket: newBucketName,
|
|
ObjectStream: obj,
|
|
NewSegmentKeys: newEncryptedKeysNonces,
|
|
NewEncryptedObjectKey: metabasetest.RandObjectKey(),
|
|
NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
|
|
NewEncryptedMetadataKey: newEncryptedMetadataKey,
|
|
},
|
|
ErrClass: &metabase.Error,
|
|
ErrText: "missing new segment keys for segment 0",
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("returned object", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
expectedRawObjects := []metabase.RawObject{}
|
|
|
|
for _, expectedVersion := range []metabase.Version{1, 2, 3, 11} {
|
|
objStream := metabasetest.RandObjectStream()
|
|
objStream.Version = expectedVersion
|
|
|
|
copyStream := metabasetest.RandObjectStream()
|
|
copyStream.ProjectID = objStream.ProjectID
|
|
copyStream.BucketName = objStream.BucketName
|
|
|
|
originalObj, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: objStream,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, objStream, 0)
|
|
|
|
metadataNonce := testrand.Nonce()
|
|
expectedCopyObject := originalObj
|
|
expectedCopyObject.ObjectKey = copyStream.ObjectKey
|
|
expectedCopyObject.StreamID = copyStream.StreamID
|
|
expectedCopyObject.EncryptedMetadataEncryptedKey = testrand.Bytes(32)
|
|
expectedCopyObject.EncryptedMetadataNonce = metadataNonce.Bytes()
|
|
|
|
objectCopy := metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
ObjectStream: objStream,
|
|
NewBucket: copyStream.BucketName,
|
|
NewStreamID: copyStream.StreamID,
|
|
NewEncryptedObjectKey: copyStream.ObjectKey,
|
|
NewEncryptedMetadataKey: expectedCopyObject.EncryptedMetadataEncryptedKey,
|
|
NewEncryptedMetadataKeyNonce: metadataNonce,
|
|
},
|
|
Result: expectedCopyObject,
|
|
}.Check(ctx, t, db)
|
|
|
|
require.NotEqual(t, originalObj.CreatedAt, objectCopy.CreatedAt)
|
|
|
|
expectedRawObjects = append(expectedRawObjects, metabase.RawObject(originalObj))
|
|
expectedRawObjects = append(expectedRawObjects, metabase.RawObject(expectedCopyObject))
|
|
}
|
|
|
|
metabasetest.Verify{
|
|
Objects: expectedRawObjects,
|
|
Copies: nil, // no copies because we have only inline segments
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("finish copy object with existing metadata", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
numberOfSegments := 10
|
|
copyStream := metabasetest.RandObjectStream()
|
|
|
|
originalObj, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, obj, byte(numberOfSegments))
|
|
|
|
copyObj, expectedOriginalSegments, expectedCopySegments := metabasetest.CreateObjectCopy{
|
|
OriginalObject: originalObj,
|
|
CopyObjectStream: ©Stream,
|
|
}.Run(ctx, t, db)
|
|
|
|
var expectedRawSegments []metabase.RawSegment
|
|
expectedRawSegments = append(expectedRawSegments, expectedOriginalSegments...)
|
|
expectedRawSegments = append(expectedRawSegments, expectedCopySegments...)
|
|
|
|
metabasetest.Verify{
|
|
Objects: []metabase.RawObject{
|
|
metabase.RawObject(originalObj),
|
|
metabase.RawObject(copyObj),
|
|
},
|
|
Segments: expectedRawSegments,
|
|
Copies: []metabase.RawCopy{{
|
|
StreamID: copyObj.StreamID,
|
|
AncestorStreamID: originalObj.StreamID,
|
|
}},
|
|
}.Check(ctx, t, db)
|
|
|
|
// TODO find better names
|
|
copyOfCopyStream := metabasetest.RandObjectStream()
|
|
copyOfCopyObj, _, expectedCopyOfCopySegments := metabasetest.CreateObjectCopy{
|
|
OriginalObject: copyObj,
|
|
CopyObjectStream: ©OfCopyStream,
|
|
}.Run(ctx, t, db)
|
|
|
|
expectedRawSegments = append(expectedRawSegments, expectedCopyOfCopySegments...)
|
|
|
|
metabasetest.Verify{
|
|
Objects: []metabase.RawObject{
|
|
metabase.RawObject(originalObj),
|
|
metabase.RawObject(copyObj),
|
|
metabase.RawObject(copyOfCopyObj),
|
|
},
|
|
Segments: expectedRawSegments,
|
|
Copies: []metabase.RawCopy{{
|
|
StreamID: copyStream.StreamID,
|
|
AncestorStreamID: originalObj.StreamID,
|
|
}, {
|
|
StreamID: copyOfCopyObj.StreamID,
|
|
AncestorStreamID: originalObj.StreamID,
|
|
}},
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("finish copy object with new metadata", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
copyStream := metabasetest.RandObjectStream()
|
|
copyStreamNoOverride := metabasetest.RandObjectStream()
|
|
|
|
originalMetadata := testrand.Bytes(64)
|
|
originalMetadataNonce := testrand.Nonce().Bytes()
|
|
originalMetadataEncryptedKey := testrand.Bytes(265)
|
|
|
|
originalObj, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
OverrideEncryptedMetadata: true,
|
|
EncryptedMetadata: originalMetadata,
|
|
EncryptedMetadataNonce: originalMetadataNonce,
|
|
EncryptedMetadataEncryptedKey: originalMetadataEncryptedKey,
|
|
},
|
|
}.Run(ctx, t, db, obj, 0)
|
|
|
|
newMetadata := testrand.Bytes(256)
|
|
newMetadataKey := testrand.Bytes(32)
|
|
newMetadataKeyNonce := testrand.Nonce()
|
|
|
|
// do a copy without OverrideMetadata field set to true,
|
|
// metadata shouldn't be updated even if NewEncryptedMetadata
|
|
// field is set
|
|
copyObjNoOverride, _, _ := metabasetest.CreateObjectCopy{
|
|
OriginalObject: originalObj,
|
|
CopyObjectStream: ©StreamNoOverride,
|
|
FinishObject: &metabase.FinishCopyObject{
|
|
ObjectStream: originalObj.ObjectStream,
|
|
|
|
NewBucket: copyStreamNoOverride.BucketName,
|
|
NewStreamID: copyStreamNoOverride.StreamID,
|
|
|
|
NewEncryptedObjectKey: copyStreamNoOverride.ObjectKey,
|
|
|
|
OverrideMetadata: false,
|
|
NewEncryptedMetadata: newMetadata,
|
|
NewEncryptedMetadataKeyNonce: newMetadataKeyNonce,
|
|
NewEncryptedMetadataKey: newMetadataKey,
|
|
},
|
|
}.Run(ctx, t, db)
|
|
|
|
require.Equal(t, originalMetadata, copyObjNoOverride.EncryptedMetadata)
|
|
require.Equal(t, newMetadataKey, copyObjNoOverride.EncryptedMetadataEncryptedKey)
|
|
require.Equal(t, newMetadataKeyNonce.Bytes(), copyObjNoOverride.EncryptedMetadataNonce)
|
|
|
|
// do a copy WITH OverrideMetadata field set to true,
|
|
// metadata should be updated to NewEncryptedMetadata
|
|
copyObj, _, _ := metabasetest.CreateObjectCopy{
|
|
OriginalObject: originalObj,
|
|
CopyObjectStream: ©Stream,
|
|
FinishObject: &metabase.FinishCopyObject{
|
|
ObjectStream: originalObj.ObjectStream,
|
|
|
|
NewBucket: copyStream.BucketName,
|
|
NewStreamID: copyStream.StreamID,
|
|
|
|
NewEncryptedObjectKey: copyStream.ObjectKey,
|
|
|
|
OverrideMetadata: true,
|
|
NewEncryptedMetadata: newMetadata,
|
|
NewEncryptedMetadataKeyNonce: newMetadataKeyNonce,
|
|
NewEncryptedMetadataKey: newMetadataKey,
|
|
},
|
|
}.Run(ctx, t, db)
|
|
|
|
require.Equal(t, newMetadata, copyObj.EncryptedMetadata)
|
|
require.Equal(t, newMetadataKey, copyObj.EncryptedMetadataEncryptedKey)
|
|
require.Equal(t, newMetadataKeyNonce.Bytes(), copyObj.EncryptedMetadataNonce)
|
|
|
|
metabasetest.Verify{
|
|
Objects: []metabase.RawObject{
|
|
metabase.RawObject(originalObj),
|
|
metabase.RawObject(copyObj),
|
|
metabase.RawObject(copyObjNoOverride),
|
|
},
|
|
Copies: nil,
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("finish copy object to already existing destination", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
// Test:
|
|
// - 3 objects: objA, objB, objC
|
|
// - copy objB to objA - creating objBprime
|
|
// - check that segments of original objA have been deleted
|
|
// - check that we now have three objects: objBprime, objB, objC
|
|
// - copy objC to objB creating objCprime
|
|
// - check that we now have three objects: objBprime, objCprime, objC
|
|
// - check that objBprime has become an original object, now that its ancestor
|
|
// objB has been overwritten
|
|
|
|
// object that already exists
|
|
objStreamA := metabasetest.RandObjectStream()
|
|
objStreamB := metabasetest.RandObjectStream()
|
|
objStreamC := metabasetest.RandObjectStream()
|
|
|
|
// set same projectID for all
|
|
objStreamB.ProjectID = objStreamA.ProjectID
|
|
objStreamC.ProjectID = objStreamA.ProjectID
|
|
|
|
objA, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: objStreamA,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, objStreamA, 4)
|
|
|
|
objB, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: objStreamB,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, objStreamB, 3)
|
|
|
|
objC, segmentsOfC := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: objStreamC,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, objStreamC, 1)
|
|
|
|
// B' is a copy of B to A
|
|
objStreamBprime := objStreamA
|
|
objStreamBprime.StreamID = testrand.UUID()
|
|
objBprime, expectedSegmentsOfB, expectedSegmentsOfBprime := metabasetest.CreateObjectCopy{
|
|
OriginalObject: objB,
|
|
CopyObjectStream: &objStreamBprime,
|
|
}.Run(ctx, t, db)
|
|
|
|
// check that we indeed overwrote object A
|
|
require.Equal(t, objA.BucketName, objBprime.BucketName)
|
|
require.Equal(t, objA.ProjectID, objBprime.ProjectID)
|
|
require.Equal(t, objA.ObjectKey, objBprime.ObjectKey)
|
|
|
|
require.NotEqual(t, objA.StreamID, objBprime.StreamID)
|
|
|
|
var expectedRawSegments []metabase.RawSegment
|
|
expectedRawSegments = append(expectedRawSegments, expectedSegmentsOfBprime...)
|
|
expectedRawSegments = append(expectedRawSegments, expectedSegmentsOfB...)
|
|
expectedRawSegments = append(expectedRawSegments, metabasetest.SegmentsToRaw(segmentsOfC)...)
|
|
|
|
metabasetest.Verify{
|
|
Objects: []metabase.RawObject{
|
|
metabase.RawObject(objBprime),
|
|
metabase.RawObject(objB),
|
|
metabase.RawObject(objC),
|
|
},
|
|
Segments: expectedRawSegments,
|
|
Copies: []metabase.RawCopy{{
|
|
StreamID: objBprime.StreamID,
|
|
AncestorStreamID: objB.StreamID,
|
|
}},
|
|
}.Check(ctx, t, db)
|
|
|
|
// C' is a copy of C to B
|
|
objStreamCprime := objStreamB
|
|
objStreamCprime.StreamID = testrand.UUID()
|
|
objCprime, _, expectedSegmentsOfCprime := metabasetest.CreateObjectCopy{
|
|
OriginalObject: objC,
|
|
CopyObjectStream: &objStreamCprime,
|
|
}.Run(ctx, t, db)
|
|
|
|
require.Equal(t, objStreamB.BucketName, objCprime.BucketName)
|
|
require.Equal(t, objStreamB.ProjectID, objCprime.ProjectID)
|
|
require.Equal(t, objStreamB.ObjectKey, objCprime.ObjectKey)
|
|
require.NotEqual(t, objB.StreamID, objCprime)
|
|
|
|
// B' should become the original of B and now hold pieces.
|
|
for i := range expectedSegmentsOfBprime {
|
|
expectedSegmentsOfBprime[i].EncryptedETag = nil
|
|
expectedSegmentsOfBprime[i].Pieces = expectedSegmentsOfB[i].Pieces
|
|
}
|
|
|
|
var expectedSegments []metabase.RawSegment
|
|
expectedSegments = append(expectedSegments, expectedSegmentsOfBprime...)
|
|
expectedSegments = append(expectedSegments, expectedSegmentsOfCprime...)
|
|
expectedSegments = append(expectedSegments, metabasetest.SegmentsToRaw(segmentsOfC)...)
|
|
|
|
metabasetest.Verify{
|
|
Objects: []metabase.RawObject{
|
|
metabase.RawObject(objBprime),
|
|
metabase.RawObject(objCprime),
|
|
metabase.RawObject(objC),
|
|
},
|
|
Segments: expectedSegments,
|
|
Copies: []metabase.RawCopy{{
|
|
StreamID: objCprime.StreamID,
|
|
AncestorStreamID: objC.StreamID,
|
|
}},
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
// checks that a copy can be copied to it's ancestor location
|
|
t.Run("Copy child to ancestor", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
originalObjStream := metabasetest.RandObjectStream()
|
|
copyObjStream := metabasetest.RandObjectStream()
|
|
// Copy back to original object key.
|
|
// StreamID is independent of key.
|
|
copyBackObjStream := originalObjStream
|
|
copyBackObjStream.StreamID = testrand.UUID()
|
|
|
|
originalObj, originalSegments := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: originalObjStream,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, originalObjStream, 4)
|
|
|
|
copyObj, _, copySegments := metabasetest.CreateObjectCopy{
|
|
OriginalObject: originalObj,
|
|
CopyObjectStream: ©ObjStream,
|
|
}.Run(ctx, t, db)
|
|
|
|
// Copy the copy back to the source location
|
|
opts := metabase.FinishCopyObject{
|
|
// source
|
|
ObjectStream: copyObj.ObjectStream,
|
|
// destination
|
|
NewBucket: originalObj.BucketName,
|
|
NewEncryptedObjectKey: originalObj.ObjectKey,
|
|
NewStreamID: copyBackObjStream.StreamID,
|
|
OverrideMetadata: false,
|
|
NewSegmentKeys: []metabase.EncryptedKeyAndNonce{
|
|
metabasetest.RandEncryptedKeyAndNonce(0),
|
|
metabasetest.RandEncryptedKeyAndNonce(1),
|
|
metabasetest.RandEncryptedKeyAndNonce(2),
|
|
metabasetest.RandEncryptedKeyAndNonce(3),
|
|
},
|
|
}
|
|
metabasetest.CreateObjectCopy{
|
|
OriginalObject: copyObj,
|
|
CopyObjectStream: ©BackObjStream,
|
|
FinishObject: &opts,
|
|
}.Run(ctx, t, db)
|
|
|
|
// expected object at the location which was previously the original object
|
|
copyBackObj := originalObj
|
|
copyBackObj.StreamID = opts.NewStreamID
|
|
|
|
for i := 0; i < 4; i++ {
|
|
copySegments[i].Pieces = originalSegments[i].Pieces
|
|
copySegments[i].InlineData = originalSegments[i].InlineData
|
|
copySegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy
|
|
|
|
originalSegments[i].StreamID = opts.NewStreamID
|
|
originalSegments[i].Pieces = nil
|
|
originalSegments[i].InlineData = nil
|
|
originalSegments[i].EncryptedKey = opts.NewSegmentKeys[i].EncryptedKey
|
|
originalSegments[i].EncryptedKeyNonce = opts.NewSegmentKeys[i].EncryptedKeyNonce
|
|
originalSegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy
|
|
}
|
|
|
|
metabasetest.Verify{
|
|
Objects: []metabase.RawObject{
|
|
metabase.RawObject(copyObj),
|
|
metabase.RawObject(copyBackObj),
|
|
},
|
|
Segments: append(metabasetest.SegmentsToRaw(originalSegments), copySegments...),
|
|
Copies: []metabase.RawCopy{{
|
|
StreamID: opts.NewStreamID,
|
|
AncestorStreamID: copyObjStream.StreamID,
|
|
}},
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
// checks that a copy ancestor can be copied to itself
|
|
t.Run("Copy ancestor to itself", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
originalObjStream := metabasetest.RandObjectStream()
|
|
copyObjStream := metabasetest.RandObjectStream()
|
|
// Copy back to same object key.
|
|
// StreamID is independent of key.
|
|
copyBackObjStream := originalObjStream
|
|
copyBackObjStream.StreamID = testrand.UUID()
|
|
|
|
originalObj, _ := metabasetest.CreateTestObject{
|
|
CommitObject: &metabase.CommitObject{
|
|
ObjectStream: originalObjStream,
|
|
EncryptedMetadata: testrand.Bytes(64),
|
|
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
|
|
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
|
|
},
|
|
}.Run(ctx, t, db, originalObjStream, 4)
|
|
|
|
copyObj, originalSegments, copySegments := metabasetest.CreateObjectCopy{
|
|
OriginalObject: originalObj,
|
|
CopyObjectStream: ©ObjStream,
|
|
}.Run(ctx, t, db)
|
|
|
|
opts := metabase.FinishCopyObject{
|
|
// source
|
|
ObjectStream: copyObj.ObjectStream,
|
|
// destination
|
|
NewBucket: originalObj.BucketName,
|
|
NewEncryptedObjectKey: originalObj.ObjectKey,
|
|
NewStreamID: copyBackObjStream.StreamID,
|
|
OverrideMetadata: false,
|
|
NewSegmentKeys: []metabase.EncryptedKeyAndNonce{
|
|
metabasetest.RandEncryptedKeyAndNonce(0),
|
|
metabasetest.RandEncryptedKeyAndNonce(1),
|
|
metabasetest.RandEncryptedKeyAndNonce(2),
|
|
metabasetest.RandEncryptedKeyAndNonce(3),
|
|
},
|
|
}
|
|
// Copy the copy back to the source location
|
|
metabasetest.CreateObjectCopy{
|
|
OriginalObject: originalObj,
|
|
CopyObjectStream: ©BackObjStream,
|
|
FinishObject: &opts,
|
|
}.Run(ctx, t, db)
|
|
|
|
copyBackObj := originalObj
|
|
copyBackObj.StreamID = copyBackObjStream.StreamID
|
|
|
|
for i := 0; i < 4; i++ {
|
|
copySegments[i].Pieces = originalSegments[i].Pieces
|
|
copySegments[i].InlineData = originalSegments[i].InlineData
|
|
copySegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy
|
|
|
|
originalSegments[i].StreamID = opts.NewStreamID
|
|
originalSegments[i].Pieces = nil
|
|
originalSegments[i].InlineData = nil
|
|
originalSegments[i].EncryptedKey = opts.NewSegmentKeys[i].EncryptedKey
|
|
originalSegments[i].EncryptedKeyNonce = opts.NewSegmentKeys[i].EncryptedKeyNonce
|
|
originalSegments[i].EncryptedETag = nil // TODO: ETag seems lost after copy
|
|
}
|
|
|
|
metabasetest.Verify{
|
|
Objects: []metabase.RawObject{
|
|
metabase.RawObject(copyObj),
|
|
metabase.RawObject(copyBackObj),
|
|
},
|
|
Segments: append(originalSegments, copySegments...),
|
|
Copies: []metabase.RawCopy{{
|
|
StreamID: copyBackObjStream.StreamID,
|
|
AncestorStreamID: copyObjStream.StreamID,
|
|
}},
|
|
}.Check(ctx, t, db)
|
|
})
|
|
|
|
t.Run("copied segments has same expires_at as original", func(t *testing.T) {
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
expiresAt := time.Now().Add(2 * time.Hour)
|
|
|
|
objStream := metabasetest.RandObjectStream()
|
|
copyStream := metabasetest.RandObjectStream()
|
|
copyStream.ProjectID = objStream.ProjectID
|
|
copyStream.BucketName = objStream.BucketName
|
|
|
|
originalObj := metabasetest.CreateExpiredObject(ctx, t, db, objStream, 10, expiresAt)
|
|
|
|
metadataNonce := testrand.Nonce()
|
|
expectedCopyObject := originalObj
|
|
expectedCopyObject.ObjectKey = copyStream.ObjectKey
|
|
expectedCopyObject.StreamID = copyStream.StreamID
|
|
expectedCopyObject.EncryptedMetadataEncryptedKey = testrand.Bytes(32)
|
|
expectedCopyObject.EncryptedMetadataNonce = metadataNonce.Bytes()
|
|
|
|
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, originalObj.SegmentCount)
|
|
expectedSegments := make([]metabase.RawSegment, originalObj.SegmentCount)
|
|
|
|
for i := 0; i < int(originalObj.SegmentCount); i++ {
|
|
newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
|
|
Position: metabase.SegmentPosition{Index: uint32(i)},
|
|
EncryptedKeyNonce: testrand.Nonce().Bytes(),
|
|
EncryptedKey: testrand.Bytes(32),
|
|
}
|
|
|
|
expectedSegments[i] = metabasetest.DefaultRawSegment(originalObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
|
|
expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
|
|
expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
|
|
expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
|
|
expectedSegments[i].EncryptedSize = int32(0)
|
|
}
|
|
|
|
copyObj := metabasetest.FinishCopyObject{
|
|
Opts: metabase.FinishCopyObject{
|
|
ObjectStream: objStream,
|
|
NewBucket: copyStream.BucketName,
|
|
NewStreamID: copyStream.StreamID,
|
|
NewEncryptedObjectKey: copyStream.ObjectKey,
|
|
NewEncryptedMetadataKey: expectedCopyObject.EncryptedMetadataEncryptedKey,
|
|
NewEncryptedMetadataKeyNonce: metadataNonce,
|
|
NewSegmentKeys: newEncryptedKeysNonces,
|
|
},
|
|
Result: expectedCopyObject,
|
|
}.Check(ctx, t, db)
|
|
|
|
var listSegments []metabase.Segment
|
|
|
|
copiedSegments, err := db.ListSegments(ctx, metabase.ListSegments{
|
|
StreamID: copyObj.StreamID,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
originalSegments, err := db.ListSegments(ctx, metabase.ListSegments{
|
|
StreamID: originalObj.StreamID,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
listSegments = append(listSegments, originalSegments.Segments...)
|
|
listSegments = append(listSegments, copiedSegments.Segments...)
|
|
|
|
for _, v := range listSegments {
|
|
require.Equal(t, expiresAt.Unix(), v.ExpiresAt.Unix())
|
|
}
|
|
|
|
metabasetest.Verify{
|
|
Objects: []metabase.RawObject{
|
|
metabase.RawObject(originalObj),
|
|
metabase.RawObject(copyObj),
|
|
},
|
|
Segments: metabasetest.SegmentsToRaw(listSegments),
|
|
Copies: []metabase.RawCopy{{
|
|
StreamID: copyObj.StreamID,
|
|
AncestorStreamID: originalObj.StreamID,
|
|
}},
|
|
}.Check(ctx, t, db)
|
|
})
|
|
})
|
|
}
|