storj/satellite/metabase/move_object_test.go
Michal Niewrzal fa76dfe577 satellite/metabase: use GetObjectLastCommitted with move/copy
We are preparing to use object versions internally and to do
that we need to prepare different parts of the system to handle
object versions different than '1'. This change adjust code
responsible for server-side move and copy.

What was done:
* begin methods for move and copy are now using GetObjectLastCommitted
to find object
* results from begin move and copy operation contains now version to
be able to map object correctly with finish operation
* begin methods are putting version into satellite stream id and
finish methods are using this version as parameter instead hardcoded
value

Fixes https://github.com/storj/storj/issues/4867

Change-Id: I1380911279c21e10a3fff0342793efd2e73eafad
2022-08-30 06:36:02 +00:00

530 lines
18 KiB
Go

// Copyright (C) 2021 Storj Labs, Inc.
// See LICENSE for copying information.
package metabase_test
import (
"testing"
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metabase/metabasetest"
)
func TestBeginMoveObject(t *testing.T) {
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
obj := metabasetest.RandObjectStream()
for _, test := range metabasetest.InvalidObjectLocations(obj.Location()) {
test := test
t.Run(test.Name, func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
metabasetest.BeginMoveObject{
Opts: metabase.BeginMoveObject{
ObjectLocation: test.ObjectLocation,
},
ErrClass: test.ErrClass,
ErrText: test.ErrText,
}.Check(ctx, t, db)
metabasetest.Verify{}.Check(ctx, t, db)
})
}
t.Run("begin move object", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
expectedRawObjects := []metabase.RawObject{}
expectedRawSegments := []metabase.RawSegment{}
for _, expectedVersion := range []metabase.Version{1, 2, 3, 11} {
obj.StreamID = testrand.UUID()
obj.Version = expectedVersion
expectedObject, expectedSegments := metabasetest.CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
},
}.Run(ctx, t, db, obj, 10)
expectedRawObjects = append(expectedRawObjects, metabase.RawObject(expectedObject))
var encKeyAndNonces []metabase.EncryptedKeyAndNonce
for _, expectedSegment := range expectedSegments {
encKeyAndNonces = append(encKeyAndNonces, metabase.EncryptedKeyAndNonce{
EncryptedKeyNonce: expectedSegment.EncryptedKeyNonce,
EncryptedKey: expectedSegment.EncryptedKey,
Position: expectedSegment.Position,
})
expectedRawSegments = append(expectedRawSegments, metabase.RawSegment(expectedSegment))
}
metabasetest.BeginMoveObject{
Opts: metabase.BeginMoveObject{
ObjectLocation: obj.Location(),
},
Result: metabase.BeginMoveObjectResult{
StreamID: expectedObject.StreamID,
Version: expectedVersion,
EncryptedKeysNonces: encKeyAndNonces,
EncryptionParameters: expectedObject.Encryption,
},
}.Check(ctx, t, db)
}
metabasetest.Verify{
Objects: expectedRawObjects,
Segments: expectedRawSegments,
}.Check(ctx, t, db)
})
})
}
func TestFinishMoveObject(t *testing.T) {
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
obj := metabasetest.RandObjectStream()
newBucketName := "New bucket name"
for _, test := range metabasetest.InvalidObjectStreams(obj) {
test := test
t.Run(test.Name, func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: test.ObjectStream,
},
ErrClass: test.ErrClass,
ErrText: test.ErrText,
}.Check(ctx, t, db)
metabasetest.Verify{}.Check(ctx, t, db)
})
}
t.Run("invalid NewBucket", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
ObjectStream: obj,
NewEncryptedObjectKey: []byte{1, 2, 3},
NewEncryptedMetadataKey: []byte{1, 2, 3},
NewEncryptedMetadataKeyNonce: testrand.Nonce(),
},
ErrClass: &metabase.ErrInvalidRequest,
ErrText: "NewBucket is missing",
}.Check(ctx, t, db)
metabasetest.Verify{}.Check(ctx, t, db)
})
t.Run("invalid NewEncryptedObjectKey", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: obj,
},
ErrClass: &metabase.ErrInvalidRequest,
ErrText: "NewEncryptedObjectKey is missing",
}.Check(ctx, t, db)
metabasetest.Verify{}.Check(ctx, t, db)
})
t.Run("invalid EncryptedMetadataKeyNonce", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
numberOfSegments := 10
newObjectKey := testrand.Bytes(32)
newEncryptedMetadataKey := testrand.Bytes(32)
newObj, segments := metabasetest.CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
OverrideEncryptedMetadata: true,
EncryptedMetadata: testrand.Bytes(64),
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
},
}.Run(ctx, t, db, obj, byte(numberOfSegments))
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: obj,
NewSegmentKeys: newEncryptedKeysNonces,
NewEncryptedObjectKey: newObjectKey,
NewEncryptedMetadataKeyNonce: storj.Nonce{},
NewEncryptedMetadataKey: newEncryptedMetadataKey,
},
ErrClass: &metabase.ErrInvalidRequest,
ErrText: "EncryptedMetadataKeyNonce is missing",
}.Check(ctx, t, db)
metabasetest.Verify{
Objects: []metabase.RawObject{
metabase.RawObject(newObj),
},
Segments: metabasetest.SegmentsToRaw(segments),
}.Check(ctx, t, db)
})
t.Run("invalid EncryptedMetadataKey", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
numberOfSegments := 10
newObjectKey := testrand.Bytes(32)
newObj, segments := metabasetest.CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
OverrideEncryptedMetadata: true,
EncryptedMetadata: testrand.Bytes(64),
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
},
}.Run(ctx, t, db, obj, byte(numberOfSegments))
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount)
newEncryptedMetadataKeyNonce := testrand.Nonce()
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: obj,
NewSegmentKeys: newEncryptedKeysNonces,
NewEncryptedObjectKey: newObjectKey,
NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
NewEncryptedMetadataKey: nil,
},
ErrClass: &metabase.ErrInvalidRequest,
ErrText: "EncryptedMetadataKey is missing",
}.Check(ctx, t, db)
metabasetest.Verify{
Objects: []metabase.RawObject{
metabase.RawObject(newObj),
},
Segments: metabasetest.SegmentsToRaw(segments),
}.Check(ctx, t, db)
})
t.Run("empty EncryptedMetadataKey and EncryptedMetadataKeyNonce", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: obj,
NewEncryptedObjectKey: []byte{0},
},
// validation pass without EncryptedMetadataKey and EncryptedMetadataKeyNonce
ErrClass: &storj.ErrObjectNotFound,
ErrText: "object not found",
}.Check(ctx, t, db)
metabasetest.Verify{}.Check(ctx, t, db)
})
t.Run("object already exists", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
moveObjStream := metabasetest.RandObjectStream()
metabasetest.CreateObject(ctx, t, db, moveObjStream, 0)
conflictObjStream := metabasetest.RandObjectStream()
conflictObjStream.ProjectID = moveObjStream.ProjectID
metabasetest.CreateObject(ctx, t, db, conflictObjStream, 0)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: conflictObjStream.BucketName,
ObjectStream: moveObjStream,
NewEncryptedObjectKey: []byte(conflictObjStream.ObjectKey),
NewEncryptedMetadataKeyNonce: testrand.Nonce(),
NewEncryptedMetadataKey: testrand.Bytes(265),
},
ErrClass: &metabase.ErrObjectAlreadyExists,
}.Check(ctx, t, db)
})
t.Run("object does not exist", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
newObj := metabasetest.RandObjectStream()
newEncryptedMetadataKeyNonce := testrand.Nonce()
newEncryptedMetadataKey := testrand.Bytes(32)
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, 10)
newObjectKey := testrand.Bytes(32)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: newObj,
NewSegmentKeys: newEncryptedKeysNonces,
NewEncryptedObjectKey: newObjectKey,
NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
NewEncryptedMetadataKey: newEncryptedMetadataKey,
},
ErrClass: &storj.ErrObjectNotFound,
ErrText: "object not found",
}.Check(ctx, t, db)
metabasetest.Verify{}.Check(ctx, t, db)
})
t.Run("not enough segments", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
numberOfSegments := 10
newObjectKey := testrand.Bytes(32)
newObj, _ := metabasetest.CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
EncryptedMetadata: testrand.Bytes(64),
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
},
}.Run(ctx, t, db, obj, byte(numberOfSegments))
newEncryptedMetadataKeyNonce := testrand.Nonce()
newEncryptedMetadataKey := testrand.Bytes(32)
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount-1)
expectedEncryptedSize := 1060
expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount)
for i := 0; i < int(newObj.SegmentCount-1); i++ {
newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
Position: metabase.SegmentPosition{Index: uint32(i)},
EncryptedKeyNonce: testrand.Nonce().Bytes(),
EncryptedKey: testrand.Bytes(32),
}
expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
expectedSegments[i].EncryptedSize = int32(expectedEncryptedSize)
}
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: obj,
NewSegmentKeys: newEncryptedKeysNonces,
NewEncryptedObjectKey: newObjectKey,
NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
NewEncryptedMetadataKey: newEncryptedMetadataKey,
},
ErrClass: &metabase.ErrInvalidRequest,
ErrText: "wrong number of segments keys received",
}.Check(ctx, t, db)
})
t.Run("wrong segment indexes", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
numberOfSegments := 10
newObjectKey := testrand.Bytes(32)
newObj, _ := metabasetest.CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
EncryptedMetadata: testrand.Bytes(64),
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
},
}.Run(ctx, t, db, obj, byte(numberOfSegments))
newEncryptedMetadataKeyNonce := testrand.Nonce()
newEncryptedMetadataKey := testrand.Bytes(32)
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount)
expectedEncryptedSize := 1060
expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount)
for i := 0; i < int(newObj.SegmentCount); i++ {
newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
Position: metabase.SegmentPosition{Index: uint32(i + 5)},
EncryptedKeyNonce: testrand.Nonce().Bytes(),
EncryptedKey: testrand.Bytes(32),
}
expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
expectedSegments[i].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
expectedSegments[i].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
expectedSegments[i].EncryptedSize = int32(expectedEncryptedSize)
}
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: obj,
NewSegmentKeys: newEncryptedKeysNonces,
NewEncryptedObjectKey: newObjectKey,
NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
NewEncryptedMetadataKey: newEncryptedMetadataKey,
},
ErrClass: &metabase.Error,
ErrText: "segment is missing",
}.Check(ctx, t, db)
})
// Assert that an error occurs when a new object is put at the key (different stream_id)
// between BeginMoveObject and FinishMoveObject,
t.Run("source object changed", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
newObj, _ := metabasetest.CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
OverrideEncryptedMetadata: true,
EncryptedMetadata: testrand.Bytes(64),
EncryptedMetadataNonce: testrand.Nonce().Bytes(),
EncryptedMetadataEncryptedKey: testrand.Bytes(265),
},
}.Run(ctx, t, db, obj, 2)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: metabase.ObjectStream{
ProjectID: newObj.ProjectID,
BucketName: newObj.BucketName,
ObjectKey: newObj.ObjectKey,
Version: newObj.Version,
StreamID: testrand.UUID(),
},
NewSegmentKeys: []metabase.EncryptedKeyAndNonce{
metabasetest.RandEncryptedKeyAndNonce(0),
metabasetest.RandEncryptedKeyAndNonce(1),
},
NewEncryptedObjectKey: testrand.Bytes(32),
NewEncryptedMetadataKeyNonce: testrand.Nonce(),
NewEncryptedMetadataKey: testrand.Bytes(32),
},
ErrClass: &storj.ErrObjectNotFound,
ErrText: "object was changed during move",
}.Check(ctx, t, db)
})
t.Run("finish move object", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
expectedRawObjects := []metabase.RawObject{}
expectedRawSegments := []metabase.RawSegment{}
for _, expectedVersion := range []metabase.Version{1, 2, 3, 11} {
obj := metabasetest.RandObjectStream()
obj.Version = expectedVersion
object, segments := metabasetest.CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
},
}.Run(ctx, t, db, obj, 10)
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, object.SegmentCount)
for i, segment := range segments {
newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
Position: segment.Position,
EncryptedKeyNonce: testrand.Nonce().Bytes(),
EncryptedKey: testrand.Bytes(32),
}
segment.EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
segment.EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
expectedRawSegments = append(expectedRawSegments, metabase.RawSegment(segment))
}
newObjectKey := testrand.Bytes(32)
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: obj,
NewSegmentKeys: newEncryptedKeysNonces,
NewEncryptedObjectKey: newObjectKey,
},
ErrText: "",
}.Check(ctx, t, db)
object.ObjectKey = metabase.ObjectKey(newObjectKey)
object.BucketName = newBucketName
expectedRawObjects = append(expectedRawObjects, metabase.RawObject(object))
}
metabasetest.Verify{
Objects: expectedRawObjects,
Segments: expectedRawSegments,
}.Check(ctx, t, db)
})
t.Run("finish move object with empty metadata, key, nonce object", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
numberOfSegments := 10
newObjectKey := testrand.Bytes(32)
newObj, _ := metabasetest.CreateTestObject{
CommitObject: &metabase.CommitObject{
ObjectStream: obj,
OverrideEncryptedMetadata: true,
},
}.Run(ctx, t, db, obj, byte(numberOfSegments))
newEncryptedMetadataKeyNonce := testrand.Nonce()
newEncryptedMetadataKey := testrand.Bytes(32)
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, newObj.SegmentCount)
expectedEncryptedSize := 1060
expectedSegments := make([]metabase.RawSegment, newObj.SegmentCount)
for i := 0; i < int(newObj.SegmentCount); i++ {
newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
Position: metabase.SegmentPosition{Index: uint32(i)},
EncryptedKeyNonce: nil,
EncryptedKey: nil,
}
expectedSegments[i] = metabasetest.DefaultRawSegment(newObj.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
expectedSegments[i].EncryptedKeyNonce = nil
expectedSegments[i].EncryptedKey = nil
expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
expectedSegments[i].EncryptedSize = int32(expectedEncryptedSize)
}
metabasetest.FinishMoveObject{
Opts: metabase.FinishMoveObject{
NewBucket: newBucketName,
ObjectStream: obj,
NewSegmentKeys: newEncryptedKeysNonces,
NewEncryptedObjectKey: newObjectKey,
NewEncryptedMetadataKeyNonce: newEncryptedMetadataKeyNonce,
NewEncryptedMetadataKey: newEncryptedMetadataKey,
},
ErrText: "",
}.Check(ctx, t, db)
newObj.ObjectKey = metabase.ObjectKey(newObjectKey)
newObj.BucketName = newBucketName
metabasetest.Verify{
Objects: []metabase.RawObject{
metabase.RawObject(newObj),
},
Segments: expectedSegments,
}.Check(ctx, t, db)
})
})
}