2020-11-12 11:29:19 +00:00
|
|
|
// Copyright (C) 2020 Storj Labs, Inc.
|
|
|
|
// See LICENSE for copying information.
|
|
|
|
|
|
|
|
package metabase_test
|
|
|
|
|
|
|
|
import (
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"storj.io/common/testcontext"
|
|
|
|
"storj.io/common/testrand"
|
2021-04-21 13:42:57 +01:00
|
|
|
"storj.io/storj/satellite/metabase"
|
2021-04-26 10:35:44 +01:00
|
|
|
"storj.io/storj/satellite/metabase/metabasetest"
|
2020-11-12 11:29:19 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func TestCommitObjectWithSegments(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
|
|
|
|
obj := metabasetest.RandObjectStream()
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
for _, test := range metabasetest.InvalidObjectStreams(obj) {
|
2020-11-12 11:29:19 +00:00
|
|
|
test := test
|
|
|
|
t.Run(test.Name, func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: test.ObjectStream,
|
|
|
|
},
|
|
|
|
ErrClass: test.ErrClass,
|
|
|
|
ErrText: test.ErrText,
|
|
|
|
}.Check(ctx, t, db)
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
|
|
|
t.Run("invalid order", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
pos00 := metabase.SegmentPosition{Part: 0, Index: 0}
|
|
|
|
pos01 := metabase.SegmentPosition{Part: 0, Index: 1}
|
|
|
|
pos10 := metabase.SegmentPosition{Part: 1, Index: 0}
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos01,
|
|
|
|
pos00,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
ErrClass: &metabase.Error,
|
|
|
|
ErrText: "segments not in ascending order, got {0 1} before {0 0}",
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos10,
|
|
|
|
pos00,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
ErrClass: &metabase.Error,
|
|
|
|
ErrText: "segments not in ascending order, got {1 0} before {0 0}",
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos00,
|
|
|
|
pos00,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
ErrClass: &metabase.Error,
|
|
|
|
ErrText: "segments not in ascending order, got {0 0} before {0 0}",
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("version without pending", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
},
|
2023-04-19 10:54:16 +01:00
|
|
|
ErrClass: &metabase.ErrObjectNotFound,
|
2020-11-12 11:29:19 +00:00
|
|
|
ErrText: "metabase: object with specified version and pending status is missing", // TODO: this error message could be better
|
|
|
|
}.Check(ctx, t, db)
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("version", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.BeginObjectExactVersion{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
encryptedMetadata := testrand.Bytes(1024)
|
|
|
|
encryptedMetadataNonce := testrand.Nonce()
|
|
|
|
encryptedMetadataKey := testrand.Bytes(265)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
2021-10-12 14:37:12 +01:00
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
2020-11-12 11:29:19 +00:00
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
// disallow for double commit
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
},
|
2023-04-19 10:54:16 +01:00
|
|
|
ErrClass: &metabase.ErrObjectNotFound,
|
2020-11-12 11:29:19 +00:00
|
|
|
ErrText: "metabase: object with specified version and pending status is missing", // TODO: this error message could be better
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{
|
2020-11-12 11:29:19 +00:00
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
CreatedAt: now,
|
2023-10-13 17:13:18 +01:00
|
|
|
Status: metabase.CommittedUnversioned,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-10-12 14:37:12 +01:00
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
2020-11-12 11:29:19 +00:00
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
2023-10-20 01:01:27 +01:00
|
|
|
t.Run("disallow delete but nothing to delete", func(t *testing.T) {
|
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
metabasetest.BeginObjectExactVersion{
|
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
encryptedMetadata := testrand.Bytes(1024)
|
|
|
|
encryptedMetadataNonce := testrand.Nonce()
|
|
|
|
encryptedMetadataKey := testrand.Bytes(265)
|
|
|
|
|
|
|
|
metabasetest.CommitObjectWithSegments{
|
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
DisallowDelete: true,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
metabasetest.Verify{
|
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
CreatedAt: now,
|
|
|
|
Status: metabase.CommittedUnversioned,
|
|
|
|
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("disallow delete when committing unversioned", func(t *testing.T) {
|
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
unversionedStream := obj
|
|
|
|
unversionedStream.Version = 3
|
|
|
|
unversionedObject := metabasetest.CreateObject(ctx, t, db, unversionedStream, 0)
|
|
|
|
|
|
|
|
metabasetest.BeginObjectExactVersion{
|
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
zombieDeadline := now.Add(24 * time.Hour)
|
|
|
|
|
|
|
|
encryptedMetadata := testrand.Bytes(1024)
|
|
|
|
encryptedMetadataNonce := testrand.Nonce()
|
|
|
|
encryptedMetadataKey := testrand.Bytes(265)
|
|
|
|
|
|
|
|
metabasetest.CommitObjectWithSegments{
|
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
DisallowDelete: true,
|
|
|
|
},
|
|
|
|
ErrClass: &metabase.ErrPermissionDenied,
|
|
|
|
ErrText: "no permissions to delete existing object",
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
metabasetest.Verify{
|
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
metabase.RawObject(unversionedObject),
|
|
|
|
{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
CreatedAt: now,
|
|
|
|
Status: metabase.Pending,
|
|
|
|
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
ZombieDeletionDeadline: &zombieDeadline,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
2023-10-16 13:04:07 +01:00
|
|
|
|
2023-10-20 01:01:27 +01:00
|
|
|
t.Run("overwrite previous object when committing unversioned", func(t *testing.T) {
|
2023-10-16 13:04:07 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
metabasetest.CreateObject(ctx, t, db, obj, 1)
|
|
|
|
|
|
|
|
metabasetest.BeginObjectExactVersion{
|
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
encryptedMetadata := testrand.Bytes(1024)
|
|
|
|
encryptedMetadataNonce := testrand.Nonce()
|
|
|
|
encryptedMetadataKey := testrand.Bytes(265)
|
|
|
|
|
|
|
|
metabasetest.CommitObjectWithSegments{
|
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
metabasetest.Verify{
|
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
CreatedAt: now,
|
|
|
|
Status: metabase.CommittedUnversioned,
|
|
|
|
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("keep previous objects when committing versioned", func(t *testing.T) {
|
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
unversionedStream := obj
|
|
|
|
unversionedStream.Version = 3
|
|
|
|
unversionedObject := metabasetest.CreateObject(ctx, t, db, unversionedStream, 0)
|
|
|
|
versionedStream := obj
|
|
|
|
versionedStream.Version = 4
|
|
|
|
versionedObject := metabasetest.CreateObjectVersioned(ctx, t, db, versionedStream, 0)
|
|
|
|
|
|
|
|
metabasetest.BeginObjectExactVersion{
|
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
encryptedMetadata := testrand.Bytes(1024)
|
|
|
|
encryptedMetadataNonce := testrand.Nonce()
|
|
|
|
encryptedMetadataKey := testrand.Bytes(265)
|
|
|
|
|
|
|
|
metabasetest.CommitObjectWithSegments{
|
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
|
|
|
|
Versioned: true,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
metabasetest.Verify{
|
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
metabase.RawObject(unversionedObject),
|
|
|
|
metabase.RawObject(versionedObject),
|
|
|
|
{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
CreatedAt: now,
|
|
|
|
Status: metabase.CommittedVersioned,
|
|
|
|
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
2023-10-20 01:01:27 +01:00
|
|
|
t.Run("keep versioned objects when committing unversioned", func(t *testing.T) {
|
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
versionedStream1 := obj
|
|
|
|
versionedStream1.Version = 3
|
|
|
|
versionedObject1 := metabasetest.CreateObjectVersioned(ctx, t, db, versionedStream1, 0)
|
|
|
|
versionedStream2 := obj
|
|
|
|
versionedStream2.Version = 4
|
|
|
|
versionedObject2 := metabasetest.CreateObjectVersioned(ctx, t, db, versionedStream2, 0)
|
|
|
|
|
|
|
|
metabasetest.BeginObjectExactVersion{
|
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
encryptedMetadata := testrand.Bytes(1024)
|
|
|
|
encryptedMetadataNonce := testrand.Nonce()
|
|
|
|
encryptedMetadataKey := testrand.Bytes(265)
|
|
|
|
|
|
|
|
metabasetest.CommitObjectWithSegments{
|
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
|
|
|
metabasetest.Verify{
|
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
metabase.RawObject(versionedObject1),
|
|
|
|
metabase.RawObject(versionedObject2),
|
|
|
|
{
|
|
|
|
ObjectStream: metabase.ObjectStream{
|
|
|
|
ProjectID: obj.ProjectID,
|
|
|
|
BucketName: obj.BucketName,
|
|
|
|
ObjectKey: obj.ObjectKey,
|
|
|
|
Version: 5,
|
|
|
|
StreamID: obj.StreamID,
|
|
|
|
},
|
|
|
|
CreatedAt: now,
|
|
|
|
Status: metabase.CommittedUnversioned,
|
|
|
|
|
|
|
|
EncryptedMetadataNonce: encryptedMetadataNonce[:],
|
|
|
|
EncryptedMetadata: encryptedMetadata,
|
|
|
|
EncryptedMetadataEncryptedKey: encryptedMetadataKey,
|
|
|
|
|
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
2020-11-12 11:29:19 +00:00
|
|
|
t.Run("segments missing in database", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.BeginObjectExactVersion{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: obj,
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
2021-05-06 12:42:20 +01:00
|
|
|
zombieDeadline := now.Add(24 * time.Hour)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
pos00 := metabase.SegmentPosition{Part: 0, Index: 0}
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos00,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
ErrClass: &metabase.Error,
|
|
|
|
ErrText: "segments and database does not match: {0 0}: segment not committed",
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{
|
2020-11-12 11:29:19 +00:00
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: obj,
|
|
|
|
CreatedAt: now,
|
|
|
|
Status: metabase.Pending,
|
|
|
|
|
2021-05-06 12:42:20 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
|
|
|
ZombieDeletionDeadline: &zombieDeadline,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("delete segments that are not in proofs", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.BeginObjectExactVersion{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: obj,
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
pos00 := metabase.SegmentPosition{Part: 0, Index: 0}
|
|
|
|
rootPieceID00 := testrand.PieceID()
|
|
|
|
pieces00 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}}
|
|
|
|
encryptedKey00 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce00 := testrand.Bytes(32)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
pos01 := metabase.SegmentPosition{Part: 0, Index: 1}
|
|
|
|
rootPieceID01 := testrand.PieceID()
|
|
|
|
pieces01 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}}
|
|
|
|
encryptedKey01 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce01 := testrand.Bytes(32)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitSegment{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
|
|
|
|
Position: pos00,
|
|
|
|
RootPieceID: rootPieceID00,
|
|
|
|
Pieces: pieces00,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey00,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce00,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitSegment{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
|
|
|
|
Position: pos01,
|
|
|
|
RootPieceID: rootPieceID01,
|
|
|
|
Pieces: pieces01,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey01,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce01,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos01,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Deleted: []metabase.DeletedSegmentInfo{
|
|
|
|
{RootPieceID: rootPieceID00, Pieces: pieces00},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{
|
2020-11-12 11:29:19 +00:00
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: obj,
|
|
|
|
CreatedAt: now,
|
2023-10-13 17:13:18 +01:00
|
|
|
Status: metabase.CommittedUnversioned,
|
2020-11-12 11:29:19 +00:00
|
|
|
SegmentCount: 1,
|
|
|
|
|
|
|
|
TotalPlainSize: 512,
|
|
|
|
TotalEncryptedSize: 1024,
|
2021-01-25 11:30:56 +00:00
|
|
|
FixedSegmentSize: -1,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
Segments: []metabase.RawSegment{
|
|
|
|
{
|
2021-03-12 17:43:30 +00:00
|
|
|
StreamID: obj.StreamID,
|
|
|
|
Position: pos01,
|
2021-08-05 00:56:50 +01:00
|
|
|
CreatedAt: now,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
RootPieceID: rootPieceID01,
|
|
|
|
EncryptedKey: encryptedKey01,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce01,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainOffset: 0,
|
|
|
|
PlainSize: 512,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
Pieces: pieces01,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("delete inline segments that are not in proofs", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.BeginObjectExactVersion{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: obj,
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
pos00 := metabase.SegmentPosition{Part: 0, Index: 0}
|
|
|
|
data00 := testrand.Bytes(32)
|
|
|
|
encryptedKey00 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce00 := testrand.Bytes(32)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
pos01 := metabase.SegmentPosition{Part: 0, Index: 1}
|
|
|
|
data01 := testrand.Bytes(1024)
|
|
|
|
encryptedKey01 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce01 := testrand.Bytes(32)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitInlineSegment{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitInlineSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Position: pos00,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey00,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce00,
|
|
|
|
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
|
|
|
|
|
|
|
InlineData: data00,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitInlineSegment{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitInlineSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Position: pos01,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey01,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce01,
|
|
|
|
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
|
|
|
|
|
|
|
InlineData: data01,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos01,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Deleted: nil,
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{
|
2020-11-12 11:29:19 +00:00
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: obj,
|
|
|
|
CreatedAt: now,
|
2023-10-13 17:13:18 +01:00
|
|
|
Status: metabase.CommittedUnversioned,
|
2020-11-12 11:29:19 +00:00
|
|
|
SegmentCount: 1,
|
|
|
|
|
|
|
|
TotalPlainSize: 512,
|
|
|
|
TotalEncryptedSize: 1024,
|
2021-01-25 11:30:56 +00:00
|
|
|
FixedSegmentSize: -1,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
Segments: []metabase.RawSegment{
|
|
|
|
{
|
2021-03-12 17:43:30 +00:00
|
|
|
StreamID: obj.StreamID,
|
|
|
|
Position: pos01,
|
2021-08-05 00:56:50 +01:00
|
|
|
CreatedAt: now,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
EncryptedKey: encryptedKey01,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce01,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainOffset: 0,
|
|
|
|
PlainSize: 512,
|
|
|
|
|
|
|
|
InlineData: data01,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("updated plain offset", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.BeginObjectExactVersion{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: obj,
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
pos00 := metabase.SegmentPosition{Part: 0, Index: 0}
|
|
|
|
rootPieceID00 := testrand.PieceID()
|
|
|
|
pieces00 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}}
|
|
|
|
encryptedKey00 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce00 := testrand.Bytes(32)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
pos01 := metabase.SegmentPosition{Part: 0, Index: 1}
|
|
|
|
rootPieceID01 := testrand.PieceID()
|
|
|
|
pieces01 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}}
|
|
|
|
encryptedKey01 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce01 := testrand.Bytes(32)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitSegment{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
|
|
|
|
Position: pos00,
|
|
|
|
RootPieceID: rootPieceID00,
|
|
|
|
Pieces: pieces00,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey00,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce00,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitSegment{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
|
|
|
|
Position: pos01,
|
|
|
|
RootPieceID: rootPieceID01,
|
|
|
|
Pieces: pieces01,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey01,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce01,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos00,
|
|
|
|
pos01,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{
|
2020-11-12 11:29:19 +00:00
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: obj,
|
|
|
|
CreatedAt: now,
|
2023-10-13 17:13:18 +01:00
|
|
|
Status: metabase.CommittedUnversioned,
|
2020-11-12 11:29:19 +00:00
|
|
|
SegmentCount: 2,
|
|
|
|
|
|
|
|
TotalPlainSize: 1024,
|
|
|
|
TotalEncryptedSize: 2048,
|
2021-01-14 11:47:29 +00:00
|
|
|
FixedSegmentSize: 512,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
Segments: []metabase.RawSegment{
|
|
|
|
{
|
2021-03-12 17:43:30 +00:00
|
|
|
StreamID: obj.StreamID,
|
|
|
|
Position: pos00,
|
2021-08-05 00:56:50 +01:00
|
|
|
CreatedAt: now,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
RootPieceID: rootPieceID00,
|
|
|
|
EncryptedKey: encryptedKey00,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce00,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainOffset: 0,
|
|
|
|
PlainSize: 512,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
Pieces: pieces00,
|
|
|
|
},
|
|
|
|
{
|
2021-03-12 17:43:30 +00:00
|
|
|
StreamID: obj.StreamID,
|
|
|
|
Position: pos01,
|
2021-08-05 00:56:50 +01:00
|
|
|
CreatedAt: now,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
RootPieceID: rootPieceID01,
|
|
|
|
EncryptedKey: encryptedKey01,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce01,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainOffset: 512,
|
|
|
|
PlainSize: 512,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
|
|
|
Pieces: pieces01,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
2021-01-11 12:34:36 +00:00
|
|
|
t.Run("fixed segment size", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2021-01-11 12:34:36 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.BeginObjectExactVersion{
|
2021-01-11 12:34:36 +00:00
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: obj,
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2021-01-11 12:34:36 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
pos00 := metabase.SegmentPosition{Part: 0, Index: 0}
|
|
|
|
rootPieceID00 := testrand.PieceID()
|
|
|
|
pieces00 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}}
|
|
|
|
encryptedKey00 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce00 := testrand.Bytes(32)
|
2021-01-11 12:34:36 +00:00
|
|
|
|
|
|
|
pos10 := metabase.SegmentPosition{Part: 1, Index: 0}
|
|
|
|
rootPieceID10 := testrand.PieceID()
|
|
|
|
pieces10 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}}
|
|
|
|
encryptedKey10 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce10 := testrand.Bytes(32)
|
2021-01-11 12:34:36 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitSegment{
|
2021-01-11 12:34:36 +00:00
|
|
|
Opts: metabase.CommitSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
|
|
|
|
Position: pos00,
|
|
|
|
RootPieceID: rootPieceID00,
|
|
|
|
Pieces: pieces00,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey00,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce00,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2021-01-11 12:34:36 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitSegment{
|
2021-01-11 12:34:36 +00:00
|
|
|
Opts: metabase.CommitSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
|
|
|
|
Position: pos10,
|
|
|
|
RootPieceID: rootPieceID10,
|
|
|
|
Pieces: pieces10,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey10,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce10,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2021-01-11 12:34:36 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2021-01-11 12:34:36 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos00,
|
|
|
|
pos10,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{
|
2021-01-11 12:34:36 +00:00
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: obj,
|
|
|
|
CreatedAt: now,
|
2023-10-13 17:13:18 +01:00
|
|
|
Status: metabase.CommittedUnversioned,
|
2021-01-11 12:34:36 +00:00
|
|
|
SegmentCount: 2,
|
|
|
|
|
|
|
|
TotalPlainSize: 1024,
|
|
|
|
TotalEncryptedSize: 2048,
|
|
|
|
FixedSegmentSize: -1,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2021-01-11 12:34:36 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
Segments: []metabase.RawSegment{
|
|
|
|
{
|
2021-03-12 17:43:30 +00:00
|
|
|
StreamID: obj.StreamID,
|
|
|
|
Position: pos00,
|
2021-08-05 00:56:50 +01:00
|
|
|
CreatedAt: now,
|
2021-01-11 12:34:36 +00:00
|
|
|
|
|
|
|
RootPieceID: rootPieceID00,
|
|
|
|
EncryptedKey: encryptedKey00,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce00,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainOffset: 0,
|
|
|
|
PlainSize: 512,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2021-01-11 12:34:36 +00:00
|
|
|
|
|
|
|
Pieces: pieces00,
|
|
|
|
},
|
|
|
|
{
|
2021-03-12 17:43:30 +00:00
|
|
|
StreamID: obj.StreamID,
|
|
|
|
Position: pos10,
|
2021-08-05 00:56:50 +01:00
|
|
|
CreatedAt: now,
|
2021-01-11 12:34:36 +00:00
|
|
|
|
|
|
|
RootPieceID: rootPieceID10,
|
|
|
|
EncryptedKey: encryptedKey10,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce10,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainOffset: 512,
|
|
|
|
PlainSize: 512,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2021-01-11 12:34:36 +00:00
|
|
|
|
|
|
|
Pieces: pieces10,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
2021-01-25 11:30:56 +00:00
|
|
|
t.Run("skipped fixed segment size", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2021-01-25 11:30:56 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.BeginObjectExactVersion{
|
2021-01-25 11:30:56 +00:00
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: obj,
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2021-01-25 11:30:56 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
|
|
|
pos00 := metabase.SegmentPosition{Part: 0, Index: 0}
|
|
|
|
rootPieceID00 := testrand.PieceID()
|
|
|
|
pieces00 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}}
|
|
|
|
encryptedKey00 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce00 := testrand.Bytes(32)
|
2021-01-25 11:30:56 +00:00
|
|
|
|
|
|
|
pos02 := metabase.SegmentPosition{Part: 0, Index: 2}
|
|
|
|
rootPieceID02 := testrand.PieceID()
|
|
|
|
pieces02 := metabase.Pieces{{Number: 0, StorageNode: testrand.NodeID()}}
|
|
|
|
encryptedKey02 := testrand.Bytes(32)
|
2021-10-12 14:37:12 +01:00
|
|
|
encryptedKeyNonce02 := testrand.Bytes(32)
|
2021-01-25 11:30:56 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitSegment{
|
2021-01-25 11:30:56 +00:00
|
|
|
Opts: metabase.CommitSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
|
|
|
|
Position: pos00,
|
|
|
|
RootPieceID: rootPieceID00,
|
|
|
|
Pieces: pieces00,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey00,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce00,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2021-01-25 11:30:56 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitSegment{
|
2021-01-25 11:30:56 +00:00
|
|
|
Opts: metabase.CommitSegment{
|
|
|
|
ObjectStream: obj,
|
|
|
|
|
|
|
|
Position: pos02,
|
|
|
|
RootPieceID: rootPieceID02,
|
|
|
|
Pieces: pieces02,
|
|
|
|
|
|
|
|
EncryptedKey: encryptedKey02,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce02,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainSize: 512,
|
|
|
|
PlainOffset: 0,
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2021-01-25 11:30:56 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2021-01-25 11:30:56 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
Segments: []metabase.SegmentPosition{
|
|
|
|
pos00,
|
|
|
|
pos02,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{
|
2021-01-25 11:30:56 +00:00
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: obj,
|
|
|
|
CreatedAt: now,
|
2023-10-13 17:13:18 +01:00
|
|
|
Status: metabase.CommittedUnversioned,
|
2021-01-25 11:30:56 +00:00
|
|
|
SegmentCount: 2,
|
|
|
|
|
|
|
|
TotalPlainSize: 1024,
|
|
|
|
TotalEncryptedSize: 2048,
|
|
|
|
FixedSegmentSize: -1,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2021-01-25 11:30:56 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
Segments: []metabase.RawSegment{
|
|
|
|
{
|
2021-03-12 17:43:30 +00:00
|
|
|
StreamID: obj.StreamID,
|
|
|
|
Position: pos00,
|
2021-08-05 00:56:50 +01:00
|
|
|
CreatedAt: now,
|
2021-01-25 11:30:56 +00:00
|
|
|
|
|
|
|
RootPieceID: rootPieceID00,
|
|
|
|
EncryptedKey: encryptedKey00,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce00,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainOffset: 0,
|
|
|
|
PlainSize: 512,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2021-01-25 11:30:56 +00:00
|
|
|
|
|
|
|
Pieces: pieces00,
|
|
|
|
},
|
|
|
|
{
|
2021-03-12 17:43:30 +00:00
|
|
|
StreamID: obj.StreamID,
|
|
|
|
Position: pos02,
|
2021-08-05 00:56:50 +01:00
|
|
|
CreatedAt: now,
|
2021-01-25 11:30:56 +00:00
|
|
|
|
|
|
|
RootPieceID: rootPieceID02,
|
|
|
|
EncryptedKey: encryptedKey02,
|
|
|
|
EncryptedKeyNonce: encryptedKeyNonce02,
|
|
|
|
|
|
|
|
EncryptedSize: 1024,
|
|
|
|
PlainOffset: 512,
|
|
|
|
PlainSize: 512,
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Redundancy: metabasetest.DefaultRedundancy,
|
2021-01-25 11:30:56 +00:00
|
|
|
|
|
|
|
Pieces: pieces02,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
|
2020-11-12 11:29:19 +00:00
|
|
|
t.Run("no segments", func(t *testing.T) {
|
2021-04-26 10:35:44 +01:00
|
|
|
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.BeginObjectExactVersion{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.BeginObjectExactVersion{
|
|
|
|
ObjectStream: obj,
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
now := time.Now()
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.CommitObjectWithSegments{
|
2020-11-12 11:29:19 +00:00
|
|
|
Opts: metabase.CommitObjectWithSegments{
|
|
|
|
ObjectStream: obj,
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
metabasetest.Verify{
|
2020-11-12 11:29:19 +00:00
|
|
|
Objects: []metabase.RawObject{
|
|
|
|
{
|
|
|
|
ObjectStream: obj,
|
|
|
|
CreatedAt: now,
|
2023-10-13 17:13:18 +01:00
|
|
|
Status: metabase.CommittedUnversioned,
|
2020-11-12 11:29:19 +00:00
|
|
|
|
2021-04-26 10:35:44 +01:00
|
|
|
Encryption: metabasetest.DefaultEncryption,
|
2020-11-12 11:29:19 +00:00
|
|
|
},
|
|
|
|
},
|
|
|
|
}.Check(ctx, t, db)
|
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|