JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo: fix setting object ZombieDeletionDeadline

Browse Source 
So far we were setting ZombieDeletionDeadline alwasy as nil and because of that DB default was never set. This change adds separate query for inserting object if deadline is not set.

Change-Id: I3d6a16570e7c74b5304e13edad8c7adcd021340c
This commit is contained in:
Michał Niewrzał 2021-05-06 13:42:20 +02:00 committed by Michal Niewrzal
parent 7fb86617fc
commit 033006403f
9 changed files with 116 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
satellite/metabase/commit.go
View File

@ -21,6 +21,8 @@ import (
// we need to disable PlainSize validation for old uplinks.
const validatePlainSize = false
const defaultZombieDeletionPeriod = 24 * time.Hour
var (
// ErrInvalidRequest is used to indicate invalid requests.
ErrInvalidRequest = errs.Class("metabase: invalid request")
@ -50,6 +52,11 @@ func (db *DB) BeginObjectNextVersion(ctx context.Context, opts BeginObjectNextVe
return -1, ErrInvalidRequest.New("Version should be metabase.NextVersion")
}
if opts.ZombieDeletionDeadline == nil {
deadline := time.Now().Add(defaultZombieDeletionPeriod)
opts.ZombieDeletionDeadline = &deadline
}
row := db.db.QueryRow(ctx, `
INSERT INTO objects (
project_id, bucket_name, object_key, version, stream_id,
@ -101,6 +108,11 @@ func (db *DB) BeginObjectExactVersion(ctx context.Context, opts BeginObjectExact
return Object{}, ErrInvalidRequest.New("Version should not be metabase.NextVersion")
}
if opts.ZombieDeletionDeadline == nil {
deadline := time.Now().Add(defaultZombieDeletionPeriod)
opts.ZombieDeletionDeadline = &deadline
}
object := Object{
ObjectStream: ObjectStream{
ProjectID: opts.ProjectID,
@ -119,7 +131,7 @@ func (db *DB) BeginObjectExactVersion(ctx context.Context, opts BeginObjectExact
project_id, bucket_name, object_key, version, stream_id,
expires_at, encryption,
zombie_deletion_deadline
) values (
) VALUES (
$1, $2, $3, $4, $5,
$6, $7,
$8

4
satellite/metabase/commit_object_test.go
View File

@ -184,6 +184,7 @@ func TestCommitObjectWithSegments(t *testing.T) {
Version: 1,
}.Check(ctx, t, db)
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
pos00 := metabase.SegmentPosition{Part: 0, Index: 0}
metabasetest.CommitObjectWithSegments{
@ -204,7 +205,8 @@ func TestCommitObjectWithSegments(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)

65
satellite/metabase/commit_test.go
View File

@ -63,6 +63,8 @@ func TestBeginObjectNextVersion(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
now1 := time.Now()
zombieDeadline := now1.Add(24 * time.Hour)
futureTime := now1.Add(10 * 24 * time.Hour)
objectStream.Version = metabase.NextVersion
@ -78,8 +80,9 @@ func TestBeginObjectNextVersion(t *testing.T) {
metabasetest.BeginObjectNextVersion{
Opts: metabase.BeginObjectNextVersion{
ObjectStream: objectStream,
Encryption: metabasetest.DefaultEncryption,
ObjectStream: objectStream,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &futureTime,
},
Version: 2,
}.Check(ctx, t, db)
@ -97,7 +100,8 @@ func TestBeginObjectNextVersion(t *testing.T) {
CreatedAt: now1,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
{
ObjectStream: metabase.ObjectStream{
@ -110,7 +114,8 @@ func TestBeginObjectNextVersion(t *testing.T) {
CreatedAt: now2,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &futureTime,
},
},
}.Check(ctx, t, db)
@ -328,6 +333,7 @@ func TestBeginObjectExactVersion(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
now1 := time.Now()
zombieDeadline := now1.Add(24 * time.Hour)
objectStream.Version = 5
@ -352,7 +358,8 @@ func TestBeginObjectExactVersion(t *testing.T) {
CreatedAt: now1,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -362,6 +369,7 @@ func TestBeginObjectExactVersion(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
now1 := time.Now()
zombieDeadline := now1.Add(24 * time.Hour)
objectStream.Version = 5
@ -396,7 +404,8 @@ func TestBeginObjectExactVersion(t *testing.T) {
CreatedAt: now1,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -772,6 +781,7 @@ func TestBeginSegment(t *testing.T) {
t.Run("begin segment successfully", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
metabasetest.BeginObjectExactVersion{
Opts: metabase.BeginObjectExactVersion{
@ -799,7 +809,8 @@ func TestBeginSegment(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -808,6 +819,7 @@ func TestBeginSegment(t *testing.T) {
t.Run("multiple begin segment successfully", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
metabasetest.BeginObjectExactVersion{
Opts: metabase.BeginObjectExactVersion{
@ -837,7 +849,8 @@ func TestBeginSegment(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -869,6 +882,7 @@ func TestCommitSegment(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
metabasetest.BeginObjectExactVersion{
Opts: metabase.BeginObjectExactVersion{
ObjectStream: obj,
@ -1090,7 +1104,8 @@ func TestCommitSegment(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -1100,6 +1115,7 @@ func TestCommitSegment(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
now1 := time.Now()
zombieDeadline := now1.Add(24 * time.Hour)
metabasetest.BeginObjectExactVersion{
Opts: metabase.BeginObjectExactVersion{
ObjectStream: obj,
@ -1165,7 +1181,8 @@ func TestCommitSegment(t *testing.T) {
CreatedAt: now1,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
Segments: []metabase.RawSegment{
@ -1283,6 +1300,7 @@ func TestCommitSegment(t *testing.T) {
encryptedETag := testrand.Bytes(32)
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
metabasetest.BeginObjectExactVersion{
Opts: metabase.BeginObjectExactVersion{
ObjectStream: obj,
@ -1315,7 +1333,8 @@ func TestCommitSegment(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
Segments: []metabase.RawSegment{
@ -1345,7 +1364,7 @@ func TestCommitInlineSegment(t *testing.T) {
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
obj := metabasetest.RandObjectStream()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
for _, test := range metabasetest.InvalidObjectStreams(obj) {
test := test
t.Run(test.Name, func(t *testing.T) {
@ -1439,6 +1458,7 @@ func TestCommitInlineSegment(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
now1 := time.Now()
metabasetest.BeginObjectExactVersion{
Opts: metabase.BeginObjectExactVersion{
ObjectStream: obj,
@ -1487,7 +1507,8 @@ func TestCommitInlineSegment(t *testing.T) {
CreatedAt: now1,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
Segments: []metabase.RawSegment{
@ -1601,10 +1622,11 @@ func TestCommitInlineSegment(t *testing.T) {
metabasetest.Verify{
Objects: []metabase.RawObject{
{
ObjectStream: obj,
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
ObjectStream: obj,
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
Segments: []metabase.RawSegment{
@ -1658,10 +1680,11 @@ func TestCommitInlineSegment(t *testing.T) {
metabasetest.Verify{
Objects: []metabase.RawObject{
{
ObjectStream: obj,
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
ObjectStream: obj,
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
Segments: []metabase.RawSegment{

21
satellite/metabase/delete_objects_test.go
View File

@ -21,6 +21,7 @@ func TestDeleteExpiredObjects(t *testing.T) {
obj3 := metabasetest.RandObjectStream()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
pastTime := now.Add(-1 * time.Hour)
futureTime := now.Add(1 * time.Hour)
@ -80,7 +81,8 @@ func TestDeleteExpiredObjects(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
{
ObjectStream: obj3,
@ -88,7 +90,8 @@ func TestDeleteExpiredObjects(t *testing.T) {
ExpiresAt: &futureTime,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -171,6 +174,7 @@ func TestDeleteZombieObjects(t *testing.T) {
obj3 := metabasetest.RandObjectStream()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
pastTime := now.Add(-1 * time.Hour)
futureTime := now.Add(1 * time.Hour)
@ -230,15 +234,16 @@ func TestDeleteZombieObjects(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
{
ObjectStream: obj3,
CreatedAt: now,
ZombieDeletionDeadline: &futureTime,
Status: metabase.Pending,
ObjectStream: obj3,
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &futureTime,
},
},
}.Check(ctx, t, db)

19
satellite/metabase/delete_test.go
View File

@ -19,6 +19,7 @@ func TestDeletePendingObject(t *testing.T) {
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
obj := metabasetest.RandObjectStream()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
for _, test := range metabasetest.InvalidObjectStreams(obj) {
test := test
@ -79,7 +80,8 @@ func TestDeletePendingObject(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -144,7 +146,8 @@ func TestDeletePendingObject(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -263,6 +266,7 @@ func TestDeleteObjectExactVersion(t *testing.T) {
location := obj.Location()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
for _, test := range metabasetest.InvalidObjectLocations(location) {
test := test
@ -348,7 +352,8 @@ func TestDeleteObjectExactVersion(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -461,6 +466,7 @@ func TestDeleteObjectLatestVersion(t *testing.T) {
location := obj.Location()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
for _, test := range metabasetest.InvalidObjectLocations(location) {
test := test
@ -523,7 +529,8 @@ func TestDeleteObjectLatestVersion(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -892,6 +899,7 @@ func TestDeleteObjectsAllVersions(t *testing.T) {
location := obj.Location()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
for _, test := range metabasetest.InvalidObjectLocations(location) {
test := test
@ -983,7 +991,8 @@ func TestDeleteObjectsAllVersions(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)

18
satellite/metabase/get_test.go
View File

@ -21,6 +21,7 @@ func TestGetObjectExactVersion(t *testing.T) {
location := obj.Location()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
for _, test := range metabasetest.InvalidObjectLocations(location) {
test := test
@ -123,7 +124,8 @@ func TestGetObjectExactVersion(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -168,6 +170,7 @@ func TestGetObjectLatestVersion(t *testing.T) {
location := obj.Location()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
for _, test := range metabasetest.InvalidObjectLocations(location) {
test := test
@ -226,7 +229,8 @@ func TestGetObjectLatestVersion(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)
@ -762,6 +766,7 @@ func TestBucketEmpty(t *testing.T) {
metabasetest.Run(t, func(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
obj := metabasetest.RandObjectStream()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
t.Run("ProjectID missing", func(t *testing.T) {
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
@ -826,10 +831,11 @@ func TestBucketEmpty(t *testing.T) {
metabasetest.Verify{
Objects: []metabase.RawObject{
{
ObjectStream: obj,
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
ObjectStream: obj,
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
},
},
}.Check(ctx, t, db)

14
satellite/metabase/iterator_test.go
View File

@ -1059,6 +1059,7 @@ func TestIteratePendingObjectsWithObjectKey(t *testing.T) {
location := obj.Location()
now := time.Now()
zombieDeadline := now.Add(24 * time.Hour)
for _, test := range metabasetest.InvalidObjectLocations(location) {
test := test
@ -1099,7 +1100,8 @@ func TestIteratePendingObjectsWithObjectKey(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
}
metabasetest.IteratePendingObjectsByKey{
@ -1139,7 +1141,8 @@ func TestIteratePendingObjectsWithObjectKey(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
}
expected[i] = objectEntryFromRaw(objects[i])
}
@ -1179,7 +1182,8 @@ func TestIteratePendingObjectsWithObjectKey(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
}
metabasetest.IteratePendingObjectsByKey{
@ -1213,7 +1217,8 @@ func TestIteratePendingObjectsWithObjectKey(t *testing.T) {
CreatedAt: now,
Status: metabase.Pending,
Encryption: metabasetest.DefaultEncryption,
Encryption: metabasetest.DefaultEncryption,
ZombieDeletionDeadline: &zombieDeadline,
}
expected[i] = objectEntryFromRaw(objects[i])
}
@ -1318,6 +1323,5 @@ func objectEntryFromRaw(m metabase.RawObject) metabase.ObjectEntry {
TotalEncryptedSize: m.TotalEncryptedSize,
FixedSegmentSize: m.FixedSegmentSize,
Encryption: m.Encryption,
ZombieDeletionDeadline: m.ZombieDeletionDeadline,
}
}

5
satellite/metabase/list.go
View File

@ -34,11 +34,6 @@ type ObjectEntry struct {
FixedSegmentSize int32
Encryption storj.EncryptionParameters
// ZombieDeletionDeadline defines when the pending raw object should be deleted from the database.
// This is as a safeguard against objects that failed to upload and the client has not indicated
// whether they want to continue uploading or delete the already uploaded data.
ZombieDeletionDeadline *time.Time
}
// ObjectsIterator iterates over a sequence of ObjectEntry items.

9
satellite/metabase/metabasetest/test.go
View File

@ -52,7 +52,14 @@ func (step BeginObjectExactVersion) Check(ctx *testcontext.Context, t testing.TB
require.WithinDuration(t, time.Now(), got.CreatedAt, 5*time.Second)
require.Equal(t, step.Opts.ObjectStream, got.ObjectStream)
require.Equal(t, step.Opts.ExpiresAt, got.ExpiresAt)
require.Equal(t, step.Opts.ZombieDeletionDeadline, got.ZombieDeletionDeadline)
gotDeadline := got.ZombieDeletionDeadline
optsDeadline := step.Opts.ZombieDeletionDeadline
if optsDeadline == nil {
require.WithinDuration(t, time.Now().Add(24*time.Hour), *gotDeadline, 5*time.Second)
} else {
require.WithinDuration(t, *optsDeadline, *gotDeadline, 5*time.Second)
}
require.Equal(t, step.Opts.Encryption, got.Encryption)
}
}