satellite/metabase: delete zombie object that has no new segments for specific period of time
Added options flag to define after which object won't be marked as inactive. All segments CreatedAt time needs to be bellow this flag to treat object as inactive. Change-Id: Ib5cffc776c6ee1b62b51eb8595438f968b42528c
This commit is contained in:
Michał Niewrzał
Michal Niewrzal
parent
1e02504be6
commit
0344790c20
@ -88,9 +88,10 @@ func (db *DB) DeleteExpiredObjects(ctx context.Context, opts DeleteExpiredObject
|
||||
|
||||
// DeleteZombieObjects contains all the information necessary to delete zombie objects and segments.
|
||||
type DeleteZombieObjects struct {
|
||||
DeadlineBefore time.Time
|
||||
AsOfSystemTime time.Time
|
||||
BatchSize int
|
||||
DeadlineBefore time.Time
|
||||
InactiveDeadline time.Time
|
||||
AsOfSystemTime time.Time
|
||||
BatchSize int
|
||||
}
|
||||
|
||||
// DeleteZombieObjects deletes all objects that zombie deletion deadline passed.
|
||||
@ -123,7 +124,7 @@ func (db *DB) DeleteZombieObjects(ctx context.Context, opts DeleteZombieObjects)
|
||||
return Error.New("unable to delete zombie objects: %w", err)
|
||||
}
|
||||
|
||||
db.log.Info("Deleting zombie object",
|
||||
db.log.Debug("Deleting zombie object",
|
||||
zap.Stringer("Project", last.ProjectID),
|
||||
zap.String("Bucket", last.BucketName),
|
||||
zap.String("Object Key", string(last.ObjectKey)),
|
||||
@ -139,7 +140,7 @@ func (db *DB) DeleteZombieObjects(ctx context.Context, opts DeleteZombieObjects)
|
||||
return ObjectStream{}, Error.New("unable to delete zombie objects: %w", err)
|
||||
}
|
||||
|
||||
err = db.deleteObjectsAndSegments(ctx, objects)
|
||||
err = db.deleteInactiveObjectsAndSegments(ctx, objects, opts.InactiveDeadline)
|
||||
if err != nil {
|
||||
return ObjectStream{}, err
|
||||
}
|
||||
@ -225,3 +226,52 @@ func (db *DB) deleteObjectsAndSegments(ctx context.Context, objects []ObjectStre
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (db *DB) deleteInactiveObjectsAndSegments(ctx context.Context, objects []ObjectStream, inactiveDeadline time.Time) (err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
if len(objects) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
err = pgxutil.Conn(ctx, db.db, func(conn *pgx.Conn) error {
|
||||
var batch pgx.Batch
|
||||
for _, obj := range objects {
|
||||
batch.Queue(`
|
||||
WITH deleted_objects AS (
|
||||
DELETE FROM objects
|
||||
WHERE
|
||||
(project_id, bucket_name, object_key, version) = ($1::BYTEA, $2::BYTEA, $3::BYTEA, $4) AND
|
||||
stream_id = $5::BYTEA AND (
|
||||
-- TODO figure out something more optimal
|
||||
NOT EXISTS (SELECT stream_id FROM segments WHERE stream_id = $5::BYTEA)
|
||||
OR
|
||||
-- check that all segments where created before inactive time
|
||||
NOT EXISTS (SELECT stream_id FROM segments WHERE stream_id = $5::BYTEA AND created_at > $6)
|
||||
)
|
||||
RETURNING version -- return anything
|
||||
)
|
||||
DELETE FROM segments
|
||||
WHERE
|
||||
segments.stream_id = $5::BYTEA AND
|
||||
NOT EXISTS (SELECT stream_id FROM segments WHERE stream_id = $5::BYTEA AND created_at > $6)
|
||||
`, obj.ProjectID, []byte(obj.BucketName), []byte(obj.ObjectKey), obj.Version, obj.StreamID, inactiveDeadline)
|
||||
}
|
||||
|
||||
results := conn.SendBatch(ctx, &batch)
|
||||
defer func() { err = errs.Combine(err, results.Close()) }()
|
||||
|
||||
var errlist errs.Group
|
||||
for i := 0; i < batch.Len(); i++ {
|
||||
_, err := results.Exec()
|
||||
errlist.Add(err)
|
||||
}
|
||||
|
||||
return errlist.Err()
|
||||
})
|
||||
if err != nil {
|
||||
return Error.New("unable to delete zombie objects: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -184,7 +184,7 @@ func TestDeleteZombieObjects(t *testing.T) {
|
||||
|
||||
metabasetest.DeleteZombieObjects{
|
||||
Opts: metabase.DeleteZombieObjects{
|
||||
DeadlineBefore: time.Now(),
|
||||
DeadlineBefore: now,
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
metabasetest.Verify{}.Check(ctx, t, db)
|
||||
@ -224,7 +224,8 @@ func TestDeleteZombieObjects(t *testing.T) {
|
||||
|
||||
metabasetest.DeleteZombieObjects{
|
||||
Opts: metabase.DeleteZombieObjects{
|
||||
DeadlineBefore: time.Now(),
|
||||
DeadlineBefore: now,
|
||||
InactiveDeadline: now,
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
@ -250,16 +251,102 @@ func TestDeleteZombieObjects(t *testing.T) {
|
||||
}.Check(ctx, t, db)
|
||||
})
|
||||
|
||||
t.Run("partial object with segment", func(t *testing.T) {
|
||||
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
||||
|
||||
metabasetest.BeginObjectExactVersion{
|
||||
Opts: metabase.BeginObjectExactVersion{
|
||||
ObjectStream: obj1,
|
||||
Encryption: metabasetest.DefaultEncryption,
|
||||
ZombieDeletionDeadline: &now,
|
||||
},
|
||||
Version: 1,
|
||||
}.Check(ctx, t, db)
|
||||
metabasetest.BeginSegment{
|
||||
Opts: metabase.BeginSegment{
|
||||
ObjectStream: obj1,
|
||||
RootPieceID: storj.PieceID{1},
|
||||
Pieces: []metabase.Piece{{
|
||||
Number: 1,
|
||||
StorageNode: testrand.NodeID(),
|
||||
}},
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
metabasetest.CommitSegment{
|
||||
Opts: metabase.CommitSegment{
|
||||
ObjectStream: obj1,
|
||||
RootPieceID: storj.PieceID{1},
|
||||
Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}},
|
||||
|
||||
EncryptedKey: []byte{3},
|
||||
EncryptedKeyNonce: []byte{4},
|
||||
EncryptedETag: []byte{5},
|
||||
|
||||
EncryptedSize: 1024,
|
||||
PlainSize: 512,
|
||||
PlainOffset: 0,
|
||||
Redundancy: metabasetest.DefaultRedundancy,
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
// object will be checked if is inactive but inactive time is in future
|
||||
metabasetest.DeleteZombieObjects{
|
||||
Opts: metabase.DeleteZombieObjects{
|
||||
DeadlineBefore: now.Add(1 * time.Hour),
|
||||
InactiveDeadline: now.Add(-1 * time.Hour),
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
metabasetest.Verify{
|
||||
Objects: []metabase.RawObject{
|
||||
{
|
||||
ObjectStream: obj1,
|
||||
CreatedAt: now,
|
||||
Status: metabase.Pending,
|
||||
|
||||
Encryption: metabasetest.DefaultEncryption,
|
||||
ZombieDeletionDeadline: &now,
|
||||
},
|
||||
},
|
||||
Segments: []metabase.RawSegment{
|
||||
{
|
||||
StreamID: obj1.StreamID,
|
||||
RootPieceID: storj.PieceID{1},
|
||||
Pieces: metabase.Pieces{{Number: 0, StorageNode: storj.NodeID{2}}},
|
||||
CreatedAt: now,
|
||||
|
||||
EncryptedKey: []byte{3},
|
||||
EncryptedKeyNonce: []byte{4},
|
||||
EncryptedETag: []byte{5},
|
||||
|
||||
EncryptedSize: 1024,
|
||||
PlainSize: 512,
|
||||
PlainOffset: 0,
|
||||
Redundancy: metabasetest.DefaultRedundancy,
|
||||
},
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
// object will be checked if is inactive and will be deleted with segment
|
||||
metabasetest.DeleteZombieObjects{
|
||||
Opts: metabase.DeleteZombieObjects{
|
||||
DeadlineBefore: now.Add(1 * time.Hour),
|
||||
InactiveDeadline: now.Add(2 * time.Hour),
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
metabasetest.Verify{}.Check(ctx, t, db)
|
||||
})
|
||||
|
||||
t.Run("batch size", func(t *testing.T) {
|
||||
deadlineAt := time.Now().Add(-30 * 24 * time.Hour)
|
||||
for i := 0; i < 33; i++ {
|
||||
obj := metabasetest.RandObjectStream()
|
||||
|
||||
metabasetest.BeginObjectExactVersion{
|
||||
Opts: metabase.BeginObjectExactVersion{
|
||||
ObjectStream: obj,
|
||||
Encryption: metabasetest.DefaultEncryption,
|
||||
ZombieDeletionDeadline: &deadlineAt,
|
||||
ObjectStream: obj,
|
||||
Encryption: metabasetest.DefaultEncryption,
|
||||
// use default 24h zombie deletion deadline
|
||||
},
|
||||
Version: obj.Version,
|
||||
}.Check(ctx, t, db)
|
||||
@ -299,8 +386,9 @@ func TestDeleteZombieObjects(t *testing.T) {
|
||||
|
||||
metabasetest.DeleteZombieObjects{
|
||||
Opts: metabase.DeleteZombieObjects{
|
||||
DeadlineBefore: time.Now().Add(time.Hour),
|
||||
BatchSize: 4,
|
||||
DeadlineBefore: now.Add(25 * time.Hour),
|
||||
InactiveDeadline: now.Add(48 * time.Hour),
|
||||
BatchSize: 4,
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
@ -350,7 +438,8 @@ func TestDeleteZombieObjects(t *testing.T) {
|
||||
|
||||
metabasetest.DeleteZombieObjects{
|
||||
Opts: metabase.DeleteZombieObjects{
|
||||
DeadlineBefore: time.Now(),
|
||||
DeadlineBefore: now,
|
||||
InactiveDeadline: now.Add(1 * time.Hour),
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user