diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 92c062e63..755c976ac 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -197,8 +197,8 @@ type MetabaseDB interface { GetLatestObjectLastSegment(ctx context.Context, opts metabase.GetLatestObjectLastSegment) (segment metabase.Segment, err error) // ListSegments lists specified stream segments. ListSegments(ctx context.Context, opts metabase.ListSegments) (result metabase.ListSegmentsResult, err error) - // IterateObjectsAllVersions iterates through all versions of all committed objects. - IterateObjectsAllVersions(ctx context.Context, opts metabase.IterateObjects, fn func(context.Context, metabase.ObjectsIterator) error) (err error) + // IterateObjectsAllVersionsWithStatus iterates through all versions of all objects with specified status. + IterateObjectsAllVersionsWithStatus(ctx context.Context, opts metabase.IterateObjectsWithStatus, fn func(context.Context, metabase.ObjectsIterator) error) (err error) // BucketEmpty returns true if bucket does not contain objects (pending or committed). // This method doesn't check bucket existence. BucketEmpty(ctx context.Context, opts metabase.BucketEmpty) (empty bool, err error) diff --git a/satellite/metainfo/expireddeletion/expireddeletion_test.go b/satellite/metainfo/expireddeletion/expireddeletion_test.go index eaa40c27b..530b8f776 100644 --- a/satellite/metainfo/expireddeletion/expireddeletion_test.go +++ b/satellite/metainfo/expireddeletion/expireddeletion_test.go @@ -4,7 +4,6 @@ package expireddeletion_test import ( - "context" "testing" "time" @@ -16,7 +15,6 @@ import ( "storj.io/common/testrand" "storj.io/storj/private/testplanet" "storj.io/storj/satellite" - "storj.io/storj/satellite/metainfo/metabase" ) func TestExpiredDeletion(t *testing.T) { @@ -49,21 +47,9 @@ func TestExpiredDeletion(t *testing.T) { require.NoError(t, err) // Verify that all four objects are in the metabase - count := 0 - err = satellite.Metainfo.Metabase.IterateObjectsAllVersions(ctx, - metabase.IterateObjects{ - ProjectID: upl.Projects[0].ID, - BucketName: "testbucket", - Status: metabase.Committed, - }, func(ctx context.Context, it metabase.ObjectsIterator) error { - for it.Next(ctx, &metabase.ObjectEntry{}) { - count++ - } - return nil - }, - ) + objects, err := satellite.Metainfo.Metabase.TestingAllObjects(ctx) require.NoError(t, err) - require.EqualValues(t, 4, count) + require.Len(t, objects, 4) // Trigger the next iteration of expired cleanup and wait to finish expiredChore.SetNow(func() time.Time { @@ -73,20 +59,8 @@ func TestExpiredDeletion(t *testing.T) { expiredChore.Loop.TriggerWait() // Verify that only two objects remain in the metabase - count = 0 - err = satellite.Metainfo.Metabase.IterateObjectsAllVersions(ctx, - metabase.IterateObjects{ - ProjectID: upl.Projects[0].ID, - BucketName: "testbucket", - Status: metabase.Committed, - }, func(ctx context.Context, it metabase.ObjectsIterator) error { - for it.Next(ctx, &metabase.ObjectEntry{}) { - count++ - } - return nil - }, - ) + objects, err = satellite.Metainfo.Metabase.TestingAllObjects(ctx) require.NoError(t, err) - require.EqualValues(t, 2, count) + require.Len(t, objects, 2) }) } diff --git a/satellite/metainfo/loop.go b/satellite/metainfo/loop.go index b33447e62..28a19fe86 100644 --- a/satellite/metainfo/loop.go +++ b/satellite/metainfo/loop.go @@ -329,7 +329,7 @@ func iterateObjects(ctx context.Context, projectID uuid.UUID, bucket string, met // TODO we should improve performance here, this is just most straightforward solution - err = metabaseDB.IterateObjectsAllVersions(ctx, metabase.IterateObjects{ + err = metabaseDB.IterateObjectsAllVersionsWithStatus(ctx, metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucket, BatchSize: limit, diff --git a/satellite/metainfo/metabase/get.go b/satellite/metainfo/metabase/get.go index e4df29b75..e8724a0ce 100644 --- a/satellite/metainfo/metabase/get.go +++ b/satellite/metainfo/metabase/get.go @@ -371,8 +371,8 @@ func (db *DB) TestingAllPendingObjects(ctx context.Context, projectID uuid.UUID, func (db *DB) testingAllObjectsByStatus(ctx context.Context, projectID uuid.UUID, bucketName string, status ObjectStatus) (objects []ObjectEntry, err error) { defer mon.Task()(&ctx)(&err) - err = db.IterateObjectsAllVersions(ctx, - IterateObjects{ + err = db.IterateObjectsAllVersionsWithStatus(ctx, + IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, diff --git a/satellite/metainfo/metabase/iterator.go b/satellite/metainfo/metabase/iterator.go index 5b8b85ed1..5b8ba4576 100644 --- a/satellite/metainfo/metabase/iterator.go +++ b/satellite/metainfo/metabase/iterator.go @@ -32,7 +32,7 @@ type objectsIterator struct { skipPrefix ObjectKey } -func iterateAllVersions(ctx context.Context, db *DB, opts IterateObjects, fn func(context.Context, ObjectsIterator) error) (err error) { +func iterateAllVersionsWithStatus(ctx context.Context, db *DB, opts IterateObjectsWithStatus, fn func(context.Context, ObjectsIterator) error) (err error) { defer mon.Task()(&ctx)(&err) it := &objectsIterator{ diff --git a/satellite/metainfo/metabase/iterator_test.go b/satellite/metainfo/metabase/iterator_test.go index e731cd6ea..7fabc8ab2 100644 --- a/satellite/metainfo/metabase/iterator_test.go +++ b/satellite/metainfo/metabase/iterator_test.go @@ -20,8 +20,8 @@ func TestIterateObjects(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) t.Run("ProjectID missing", func(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{}, BucketName: "sj://mybucket", Recursive: true, @@ -32,8 +32,8 @@ func TestIterateObjects(t *testing.T) { }.Check(ctx, t, db) Verify{}.Check(ctx, t, db) }) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{1}, BucketName: "", Recursive: true, @@ -46,8 +46,8 @@ func TestIterateObjects(t *testing.T) { }) t.Run("Limit is negative", func(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{1}, BucketName: "mybucket", BatchSize: -1, @@ -61,8 +61,8 @@ func TestIterateObjects(t *testing.T) { }) t.Run("Status is invalid", func(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{1}, BucketName: "test", Recursive: true, @@ -77,8 +77,8 @@ func TestIterateObjects(t *testing.T) { t.Run("empty bucket", func(t *testing.T) { defer DeleteAll{}.Check(ctx, t, db) objects := createObjects(ctx, t, db, 2, uuid.UUID{1}, "mybucket") - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{1}, BucketName: "myemptybucket", BatchSize: 10, @@ -131,8 +131,8 @@ func TestIterateObjects(t *testing.T) { }, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -151,8 +151,8 @@ func TestIterateObjects(t *testing.T) { }}, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -178,8 +178,8 @@ func TestIterateObjects(t *testing.T) { for i, obj := range objects { expected[i] = objectEntryFromRaw(obj) } - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{1}, BucketName: "mybucket", Recursive: true, @@ -200,8 +200,8 @@ func TestIterateObjects(t *testing.T) { for i, obj := range objects { expected[i] = objectEntryFromRaw(obj) } - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{1}, BucketName: "mybucket", Recursive: true, @@ -222,8 +222,8 @@ func TestIterateObjects(t *testing.T) { for i, obj := range objectsBucketA { expected[i] = objectEntryFromRaw(obj) } - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{1}, BucketName: "bucket-a", Recursive: true, @@ -243,8 +243,8 @@ func TestIterateObjects(t *testing.T) { for i, obj := range objectsProject1 { expected[i] = objectEntryFromRaw(obj) } - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: uuid.UUID{1}, BucketName: "mybucket", Recursive: true, @@ -271,8 +271,8 @@ func TestIterateObjects(t *testing.T) { "g", }) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -291,8 +291,8 @@ func TestIterateObjects(t *testing.T) { }, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -312,8 +312,8 @@ func TestIterateObjects(t *testing.T) { }, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -333,8 +333,8 @@ func TestIterateObjects(t *testing.T) { }, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -349,8 +349,8 @@ func TestIterateObjects(t *testing.T) { ), }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -366,8 +366,8 @@ func TestIterateObjects(t *testing.T) { ), }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -382,8 +382,8 @@ func TestIterateObjects(t *testing.T) { ), }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Recursive: true, @@ -412,8 +412,8 @@ func TestIterateObjects(t *testing.T) { "g", }) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, @@ -427,8 +427,8 @@ func TestIterateObjects(t *testing.T) { }, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, @@ -443,8 +443,8 @@ func TestIterateObjects(t *testing.T) { }, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, @@ -459,8 +459,8 @@ func TestIterateObjects(t *testing.T) { }, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, @@ -474,8 +474,8 @@ func TestIterateObjects(t *testing.T) { ), }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, @@ -490,8 +490,8 @@ func TestIterateObjects(t *testing.T) { ), }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, @@ -505,8 +505,8 @@ func TestIterateObjects(t *testing.T) { ), }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, @@ -517,8 +517,8 @@ func TestIterateObjects(t *testing.T) { Result: nil, }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, @@ -533,8 +533,8 @@ func TestIterateObjects(t *testing.T) { ), }.Check(ctx, t, db) - IterateObjects{ - Opts: metabase.IterateObjects{ + IterateObjectsWithStatus{ + Opts: metabase.IterateObjectsWithStatus{ ProjectID: projectID, BucketName: bucketName, Status: metabase.Committed, diff --git a/satellite/metainfo/metabase/list.go b/satellite/metainfo/metabase/list.go index d06f578f4..a64120ed2 100644 --- a/satellite/metainfo/metabase/list.go +++ b/satellite/metainfo/metabase/list.go @@ -52,8 +52,8 @@ type IterateCursor struct { Version Version } -// IterateObjects contains arguments necessary for listing objects in a bucket. -type IterateObjects struct { +// IterateObjectsWithStatus contains arguments necessary for listing objects in a bucket. +type IterateObjectsWithStatus struct { ProjectID uuid.UUID BucketName string Recursive bool @@ -63,17 +63,17 @@ type IterateObjects struct { Status ObjectStatus } -// IterateObjectsAllVersions iterates through all versions of all committed objects. -func (db *DB) IterateObjectsAllVersions(ctx context.Context, opts IterateObjects, fn func(context.Context, ObjectsIterator) error) (err error) { +// IterateObjectsAllVersionsWithStatus iterates through all versions of all objects with specified status. +func (db *DB) IterateObjectsAllVersionsWithStatus(ctx context.Context, opts IterateObjectsWithStatus, fn func(context.Context, ObjectsIterator) error) (err error) { defer mon.Task()(&ctx)(&err) if err = opts.Verify(); err != nil { return err } - return iterateAllVersions(ctx, db, opts, fn) + return iterateAllVersionsWithStatus(ctx, db, opts, fn) } // Verify verifies get object request fields. -func (opts *IterateObjects) Verify() error { +func (opts *IterateObjectsWithStatus) Verify() error { switch { case opts.ProjectID.IsZero(): return ErrInvalidRequest.New("ProjectID missing") diff --git a/satellite/metainfo/metabase/test_test.go b/satellite/metainfo/metabase/test_test.go index 353974072..f8a874495 100644 --- a/satellite/metainfo/metabase/test_test.go +++ b/satellite/metainfo/metabase/test_test.go @@ -350,18 +350,18 @@ func (coll *IterateCollector) Add(ctx context.Context, it metabase.ObjectsIterat return nil } -type IterateObjects struct { - Opts metabase.IterateObjects +type IterateObjectsWithStatus struct { + Opts metabase.IterateObjectsWithStatus Result []metabase.ObjectEntry ErrClass *errs.Class ErrText string } -func (step IterateObjects) Check(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { +func (step IterateObjectsWithStatus) Check(ctx *testcontext.Context, t *testing.T, db *metabase.DB) { var result IterateCollector - err := db.IterateObjectsAllVersions(ctx, step.Opts, result.Add) + err := db.IterateObjectsAllVersionsWithStatus(ctx, step.Opts, result.Add) checkError(t, err, step.ErrClass, step.ErrText) diff := cmp.Diff(step.Result, []metabase.ObjectEntry(result), cmpopts.EquateApproxTime(5*time.Second)) diff --git a/satellite/metainfo/metainfo.go b/satellite/metainfo/metainfo.go index e22b04971..46a93b2aa 100644 --- a/satellite/metainfo/metainfo.go +++ b/satellite/metainfo/metainfo.go @@ -885,8 +885,8 @@ func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListReq resp = &pb.ObjectListResponse{} // TODO: Replace with IterateObjectsLatestVersion when ready - err = endpoint.metainfo.metabaseDB.IterateObjectsAllVersions(ctx, - metabase.IterateObjects{ + err = endpoint.metainfo.metabaseDB.IterateObjectsAllVersionsWithStatus(ctx, + metabase.IterateObjectsWithStatus{ ProjectID: keyInfo.ProjectID, BucketName: string(req.Bucket), Prefix: prefix,