0358a08d6b
Part of server-side copy implementation. Creates the methods BeginCopyObject and FinishCopyObject in satellite/metabase/copy_object.go. This commit makes it possible to copy objects and their segments in metabase. https://github.com/storj/team-metainfo/issues/79 Change-Id: Icc64afce16e5e0e15b83c43cff36797bb9bd39bc
335 lines
10 KiB
Go
335 lines
10 KiB
Go
// Copyright (C) 2021 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package metabasetest
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"storj.io/common/storj"
|
|
"storj.io/common/testcontext"
|
|
"storj.io/common/testrand"
|
|
"storj.io/common/uuid"
|
|
"storj.io/storj/satellite/metabase"
|
|
)
|
|
|
|
// RandObjectStream returns a random object stream.
|
|
func RandObjectStream() metabase.ObjectStream {
|
|
return metabase.ObjectStream{
|
|
ProjectID: testrand.UUID(),
|
|
BucketName: testrand.BucketName(),
|
|
ObjectKey: metabase.ObjectKey(testrand.Bytes(16)),
|
|
Version: 1,
|
|
StreamID: testrand.UUID(),
|
|
}
|
|
}
|
|
|
|
// CreatePendingObject creates a new pending object with the specified number of segments.
|
|
func CreatePendingObject(ctx *testcontext.Context, t *testing.T, db *metabase.DB, obj metabase.ObjectStream, numberOfSegments byte) {
|
|
BeginObjectExactVersion{
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
ObjectStream: obj,
|
|
Encryption: DefaultEncryption,
|
|
},
|
|
Version: obj.Version,
|
|
}.Check(ctx, t, db)
|
|
|
|
for i := byte(0); i < numberOfSegments; i++ {
|
|
BeginSegment{
|
|
Opts: metabase.BeginSegment{
|
|
ObjectStream: obj,
|
|
Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)},
|
|
RootPieceID: storj.PieceID{i + 1},
|
|
Pieces: []metabase.Piece{{
|
|
Number: 1,
|
|
StorageNode: testrand.NodeID(),
|
|
}},
|
|
},
|
|
}.Check(ctx, t, db)
|
|
|
|
CommitSegment{
|
|
Opts: metabase.CommitSegment{
|
|
ObjectStream: obj,
|
|
Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)},
|
|
RootPieceID: storj.PieceID{1},
|
|
Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}},
|
|
|
|
EncryptedKey: []byte{3},
|
|
EncryptedKeyNonce: []byte{4},
|
|
EncryptedETag: []byte{5},
|
|
|
|
EncryptedSize: 1024,
|
|
PlainSize: 512,
|
|
PlainOffset: 0,
|
|
Redundancy: DefaultRedundancy,
|
|
},
|
|
}.Check(ctx, t, db)
|
|
}
|
|
}
|
|
|
|
// CreateObject creates a new committed object with the specified number of segments.
|
|
func CreateObject(ctx *testcontext.Context, t *testing.T, db *metabase.DB, obj metabase.ObjectStream, numberOfSegments byte) metabase.Object {
|
|
BeginObjectExactVersion{
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
ObjectStream: obj,
|
|
Encryption: DefaultEncryption,
|
|
},
|
|
Version: obj.Version,
|
|
}.Check(ctx, t, db)
|
|
|
|
for i := byte(0); i < numberOfSegments; i++ {
|
|
BeginSegment{
|
|
Opts: metabase.BeginSegment{
|
|
ObjectStream: obj,
|
|
Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)},
|
|
RootPieceID: storj.PieceID{i + 1},
|
|
Pieces: []metabase.Piece{{
|
|
Number: 1,
|
|
StorageNode: testrand.NodeID(),
|
|
}},
|
|
},
|
|
}.Check(ctx, t, db)
|
|
|
|
CommitSegment{
|
|
Opts: metabase.CommitSegment{
|
|
ObjectStream: obj,
|
|
Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)},
|
|
RootPieceID: storj.PieceID{1},
|
|
Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}},
|
|
|
|
EncryptedKey: []byte{3},
|
|
EncryptedKeyNonce: []byte{4},
|
|
EncryptedETag: []byte{5},
|
|
|
|
EncryptedSize: 1024,
|
|
PlainSize: 512,
|
|
PlainOffset: 0,
|
|
Redundancy: DefaultRedundancy,
|
|
},
|
|
}.Check(ctx, t, db)
|
|
}
|
|
|
|
return CommitObject{
|
|
Opts: metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
},
|
|
}.Check(ctx, t, db)
|
|
}
|
|
|
|
// CreateExpiredObject creates a new committed expired object with the specified number of segments.
|
|
func CreateExpiredObject(ctx *testcontext.Context, t *testing.T, db *metabase.DB, obj metabase.ObjectStream, numberOfSegments byte, expiresAt time.Time) metabase.Object {
|
|
BeginObjectExactVersion{
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
ObjectStream: obj,
|
|
Encryption: DefaultEncryption,
|
|
ExpiresAt: &expiresAt,
|
|
},
|
|
Version: obj.Version,
|
|
}.Check(ctx, t, db)
|
|
|
|
for i := byte(0); i < numberOfSegments; i++ {
|
|
BeginSegment{
|
|
Opts: metabase.BeginSegment{
|
|
ObjectStream: obj,
|
|
Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)},
|
|
RootPieceID: storj.PieceID{i + 1},
|
|
Pieces: []metabase.Piece{{
|
|
Number: 1,
|
|
StorageNode: testrand.NodeID(),
|
|
}},
|
|
},
|
|
}.Check(ctx, t, db)
|
|
|
|
CommitSegment{
|
|
Opts: metabase.CommitSegment{
|
|
ObjectStream: obj,
|
|
Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)},
|
|
ExpiresAt: &expiresAt,
|
|
RootPieceID: storj.PieceID{1},
|
|
Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}},
|
|
|
|
EncryptedKey: []byte{3},
|
|
EncryptedKeyNonce: []byte{4},
|
|
EncryptedETag: []byte{5},
|
|
|
|
EncryptedSize: 1024,
|
|
PlainSize: 512,
|
|
PlainOffset: 0,
|
|
Redundancy: DefaultRedundancy,
|
|
},
|
|
}.Check(ctx, t, db)
|
|
}
|
|
|
|
return CommitObject{
|
|
Opts: metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
},
|
|
}.Check(ctx, t, db)
|
|
}
|
|
|
|
// CreateFullObjectsWithKeys creates multiple objects with the specified keys.
|
|
func CreateFullObjectsWithKeys(ctx *testcontext.Context, t *testing.T, db *metabase.DB, projectID uuid.UUID, bucketName string, keys []metabase.ObjectKey) map[metabase.ObjectKey]metabase.LoopObjectEntry {
|
|
objects := make(map[metabase.ObjectKey]metabase.LoopObjectEntry, len(keys))
|
|
for _, key := range keys {
|
|
obj := RandObjectStream()
|
|
obj.ProjectID = projectID
|
|
obj.BucketName = bucketName
|
|
obj.ObjectKey = key
|
|
|
|
CreateObject(ctx, t, db, obj, 0)
|
|
|
|
objects[key] = metabase.LoopObjectEntry{
|
|
ObjectStream: obj,
|
|
Status: metabase.Committed,
|
|
CreatedAt: time.Now(),
|
|
}
|
|
}
|
|
|
|
return objects
|
|
}
|
|
|
|
// CreateTestObject is for testing metabase.CreateTestObject.
|
|
type CreateTestObject struct {
|
|
BeginObjectExactVersion *metabase.BeginObjectExactVersion
|
|
CommitObject *metabase.CommitObject
|
|
// TODO add BeginSegment, CommitSegment
|
|
}
|
|
|
|
// Run runs the test.
|
|
func (co CreateTestObject) Run(ctx *testcontext.Context, t testing.TB, db *metabase.DB, obj metabase.ObjectStream, numberOfSegments byte) metabase.Object {
|
|
boeOpts := metabase.BeginObjectExactVersion{
|
|
ObjectStream: obj,
|
|
Encryption: DefaultEncryption,
|
|
}
|
|
if co.BeginObjectExactVersion != nil {
|
|
boeOpts = *co.BeginObjectExactVersion
|
|
}
|
|
|
|
BeginObjectExactVersion{
|
|
Opts: boeOpts,
|
|
Version: obj.Version,
|
|
}.Check(ctx, t, db)
|
|
|
|
for i := byte(0); i < numberOfSegments; i++ {
|
|
BeginSegment{
|
|
Opts: metabase.BeginSegment{
|
|
ObjectStream: obj,
|
|
Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)},
|
|
RootPieceID: storj.PieceID{i + 1},
|
|
Pieces: []metabase.Piece{{
|
|
Number: 1,
|
|
StorageNode: testrand.NodeID(),
|
|
}},
|
|
},
|
|
}.Check(ctx, t, db)
|
|
|
|
CommitSegment{
|
|
Opts: metabase.CommitSegment{
|
|
ObjectStream: obj,
|
|
ExpiresAt: boeOpts.ExpiresAt,
|
|
Position: metabase.SegmentPosition{Part: 0, Index: uint32(i)},
|
|
RootPieceID: storj.PieceID{1},
|
|
Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}},
|
|
|
|
EncryptedKey: []byte{3},
|
|
EncryptedKeyNonce: []byte{4},
|
|
EncryptedETag: []byte{5},
|
|
|
|
EncryptedSize: 1060,
|
|
PlainSize: 512,
|
|
PlainOffset: int64(i) * 512,
|
|
Redundancy: DefaultRedundancy,
|
|
},
|
|
}.Check(ctx, t, db)
|
|
}
|
|
|
|
coOpts := metabase.CommitObject{
|
|
ObjectStream: obj,
|
|
}
|
|
if co.CommitObject != nil {
|
|
coOpts = *co.CommitObject
|
|
}
|
|
|
|
return CommitObject{
|
|
Opts: coOpts,
|
|
}.Check(ctx, t, db)
|
|
}
|
|
|
|
// CreateObjectCopy is for testing object copy.
|
|
type CreateObjectCopy struct {
|
|
OriginalObject metabase.Object
|
|
FinishObject *metabase.FinishCopyObject
|
|
CopyObjectStream *metabase.ObjectStream
|
|
}
|
|
|
|
// Run creates the copy.
|
|
func (cc CreateObjectCopy) Run(ctx *testcontext.Context, t testing.TB, db *metabase.DB) (metabase.Object, []metabase.RawSegment) {
|
|
|
|
var copyStream metabase.ObjectStream
|
|
if cc.CopyObjectStream != nil {
|
|
copyStream = *cc.CopyObjectStream
|
|
} else {
|
|
copyStream = RandObjectStream()
|
|
}
|
|
|
|
newEncryptedKeysNonces := make([]metabase.EncryptedKeyAndNonce, cc.OriginalObject.SegmentCount)
|
|
expectedSegments := make([]metabase.RawSegment, cc.OriginalObject.SegmentCount*2)
|
|
expectedEncryptedSize := 1060
|
|
|
|
for i := 0; i < int(cc.OriginalObject.SegmentCount); i++ {
|
|
newEncryptedKeysNonces[i] = metabase.EncryptedKeyAndNonce{
|
|
Position: metabase.SegmentPosition{Index: uint32(i)},
|
|
EncryptedKeyNonce: testrand.Nonce().Bytes(),
|
|
EncryptedKey: testrand.Bytes(32),
|
|
}
|
|
|
|
copyIndex := i + int(cc.OriginalObject.SegmentCount)
|
|
expectedSegments[i] = DefaultRawSegment(cc.OriginalObject.ObjectStream, metabase.SegmentPosition{Index: uint32(i)})
|
|
|
|
// TODO: place this calculation in metabasetest.
|
|
expectedSegments[i].PlainOffset = int64(int32(i) * expectedSegments[i].PlainSize)
|
|
// TODO: we should use the same value for encrypted size in both test methods.
|
|
expectedSegments[i].EncryptedSize = int32(expectedEncryptedSize)
|
|
|
|
expectedSegments[copyIndex] = metabase.RawSegment{}
|
|
expectedSegments[copyIndex].StreamID = copyStream.StreamID
|
|
expectedSegments[copyIndex].EncryptedKeyNonce = newEncryptedKeysNonces[i].EncryptedKeyNonce
|
|
expectedSegments[copyIndex].EncryptedKey = newEncryptedKeysNonces[i].EncryptedKey
|
|
expectedSegments[copyIndex].EncryptedSize = expectedSegments[i].EncryptedSize
|
|
expectedSegments[copyIndex].Position = expectedSegments[i].Position
|
|
expectedSegments[copyIndex].RootPieceID = expectedSegments[i].RootPieceID
|
|
expectedSegments[copyIndex].Redundancy = expectedSegments[i].Redundancy
|
|
expectedSegments[copyIndex].PlainSize = expectedSegments[i].PlainSize
|
|
expectedSegments[copyIndex].PlainOffset = expectedSegments[i].PlainOffset
|
|
expectedSegments[copyIndex].CreatedAt = time.Now().UTC()
|
|
}
|
|
|
|
opts := cc.FinishObject
|
|
if opts == nil {
|
|
opts = &metabase.FinishCopyObject{
|
|
NewStreamID: copyStream.StreamID,
|
|
NewBucket: copyStream.BucketName,
|
|
ObjectStream: cc.OriginalObject.ObjectStream,
|
|
NewSegmentKeys: newEncryptedKeysNonces,
|
|
NewEncryptedObjectKey: []byte(copyStream.ObjectKey),
|
|
NewEncryptedMetadataKeyNonce: testrand.Nonce().Bytes(),
|
|
NewEncryptedMetadataKey: testrand.Bytes(32),
|
|
}
|
|
}
|
|
FinishCopyObject{
|
|
Opts: *opts,
|
|
ErrText: "",
|
|
}.Check(ctx, t, db)
|
|
|
|
copyObj := cc.OriginalObject
|
|
copyObj.StreamID = copyStream.StreamID
|
|
copyObj.ObjectKey = copyStream.ObjectKey
|
|
copyObj.EncryptedMetadataEncryptedKey = opts.NewEncryptedMetadataKey
|
|
copyObj.EncryptedMetadataNonce = opts.NewEncryptedMetadataKeyNonce
|
|
copyObj.BucketName = copyStream.BucketName
|
|
copyObj.ZombieDeletionDeadline = nil
|
|
|
|
return copyObj, expectedSegments
|
|
}
|