JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo/metabase: rename IterateObjectsAllVersions to

Browse Source 
IterateObjectsAllVersionsWithStatus

We need different implementation for IterateObjectsAllVersions because
we want to iterate over all object without specifying object status.
Existing method will have new name but implementation details are not
changed.

Change-Id: I01b987996772fa7f8fd73da9910d52db2d1aa0d7
This commit is contained in:
Michal Niewrzal 2020-12-21 16:07:00 +01:00
parent bafc6af992
commit dad8360b39
9 changed files with 76 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
satellite/metainfo/config.go
View File

@ -197,8 +197,8 @@ type MetabaseDB interface {
GetLatestObjectLastSegment(ctx context.Context, opts metabase.GetLatestObjectLastSegment) (segment metabase.Segment, err error)
// ListSegments lists specified stream segments.
ListSegments(ctx context.Context, opts metabase.ListSegments) (result metabase.ListSegmentsResult, err error)
// IterateObjectsAllVersions iterates through all versions of all committed objects.
IterateObjectsAllVersions(ctx context.Context, opts metabase.IterateObjects, fn func(context.Context, metabase.ObjectsIterator) error) (err error)
// IterateObjectsAllVersionsWithStatus iterates through all versions of all objects with specified status.
IterateObjectsAllVersionsWithStatus(ctx context.Context, opts metabase.IterateObjectsWithStatus, fn func(context.Context, metabase.ObjectsIterator) error) (err error)
// BucketEmpty returns true if bucket does not contain objects (pending or committed).
// This method doesn't check bucket existence.
BucketEmpty(ctx context.Context, opts metabase.BucketEmpty) (empty bool, err error)

34
satellite/metainfo/expireddeletion/expireddeletion_test.go
View File

@ -4,7 +4,6 @@
package expireddeletion_test
import (
"context"
"testing"
"time"
@ -16,7 +15,6 @@ import (
"storj.io/common/testrand"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite"
"storj.io/storj/satellite/metainfo/metabase"
)
func TestExpiredDeletion(t *testing.T) {
@ -49,21 +47,9 @@ func TestExpiredDeletion(t *testing.T) {
require.NoError(t, err)
// Verify that all four objects are in the metabase
count := 0
err = satellite.Metainfo.Metabase.IterateObjectsAllVersions(ctx,
metabase.IterateObjects{
ProjectID: upl.Projects[0].ID,
BucketName: "testbucket",
Status: metabase.Committed,
}, func(ctx context.Context, it metabase.ObjectsIterator) error {
for it.Next(ctx, &metabase.ObjectEntry{}) {
count++
}
return nil
},
)
objects, err := satellite.Metainfo.Metabase.TestingAllObjects(ctx)
require.NoError(t, err)
require.EqualValues(t, 4, count)
require.Len(t, objects, 4)
// Trigger the next iteration of expired cleanup and wait to finish
expiredChore.SetNow(func() time.Time {
@ -73,20 +59,8 @@ func TestExpiredDeletion(t *testing.T) {
expiredChore.Loop.TriggerWait()
// Verify that only two objects remain in the metabase
count = 0
err = satellite.Metainfo.Metabase.IterateObjectsAllVersions(ctx,
metabase.IterateObjects{
ProjectID: upl.Projects[0].ID,
BucketName: "testbucket",
Status: metabase.Committed,
}, func(ctx context.Context, it metabase.ObjectsIterator) error {
for it.Next(ctx, &metabase.ObjectEntry{}) {
count++
}
return nil
},
)
objects, err = satellite.Metainfo.Metabase.TestingAllObjects(ctx)
require.NoError(t, err)
require.EqualValues(t, 2, count)
require.Len(t, objects, 2)
})
}

2
satellite/metainfo/loop.go
View File

@ -329,7 +329,7 @@ func iterateObjects(ctx context.Context, projectID uuid.UUID, bucket string, met
// TODO we should improve performance here, this is just most straightforward solution
err = metabaseDB.IterateObjectsAllVersions(ctx, metabase.IterateObjects{
err = metabaseDB.IterateObjectsAllVersionsWithStatus(ctx, metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucket,
BatchSize: limit,

4
satellite/metainfo/metabase/get.go
View File

@ -371,8 +371,8 @@ func (db *DB) TestingAllPendingObjects(ctx context.Context, projectID uuid.UUID,
func (db *DB) testingAllObjectsByStatus(ctx context.Context, projectID uuid.UUID, bucketName string, status ObjectStatus) (objects []ObjectEntry, err error) {
defer mon.Task()(&ctx)(&err)
err = db.IterateObjectsAllVersions(ctx,
IterateObjects{
err = db.IterateObjectsAllVersionsWithStatus(ctx,
IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,

2
satellite/metainfo/metabase/iterator.go
View File

@ -32,7 +32,7 @@ type objectsIterator struct {
skipPrefix ObjectKey
}
func iterateAllVersions(ctx context.Context, db *DB, opts IterateObjects, fn func(context.Context, ObjectsIterator) error) (err error) {
func iterateAllVersionsWithStatus(ctx context.Context, db *DB, opts IterateObjectsWithStatus, fn func(context.Context, ObjectsIterator) error) (err error) {
defer mon.Task()(&ctx)(&err)
it := &objectsIterator{

108
satellite/metainfo/metabase/iterator_test.go
View File

@ -20,8 +20,8 @@ func TestIterateObjects(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
t.Run("ProjectID missing", func(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{},
BucketName: "sj://mybucket",
Recursive: true,
@ -32,8 +32,8 @@ func TestIterateObjects(t *testing.T) {
}.Check(ctx, t, db)
Verify{}.Check(ctx, t, db)
})
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{1},
BucketName: "",
Recursive: true,
@ -46,8 +46,8 @@ func TestIterateObjects(t *testing.T) {
})
t.Run("Limit is negative", func(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{1},
BucketName: "mybucket",
BatchSize: -1,
@ -61,8 +61,8 @@ func TestIterateObjects(t *testing.T) {
})
t.Run("Status is invalid", func(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{1},
BucketName: "test",
Recursive: true,
@ -77,8 +77,8 @@ func TestIterateObjects(t *testing.T) {
t.Run("empty bucket", func(t *testing.T) {
defer DeleteAll{}.Check(ctx, t, db)
objects := createObjects(ctx, t, db, 2, uuid.UUID{1}, "mybucket")
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{1},
BucketName: "myemptybucket",
BatchSize: 10,
@ -131,8 +131,8 @@ func TestIterateObjects(t *testing.T) {
},
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -151,8 +151,8 @@ func TestIterateObjects(t *testing.T) {
}},
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -178,8 +178,8 @@ func TestIterateObjects(t *testing.T) {
for i, obj := range objects {
expected[i] = objectEntryFromRaw(obj)
}
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{1},
BucketName: "mybucket",
Recursive: true,
@ -200,8 +200,8 @@ func TestIterateObjects(t *testing.T) {
for i, obj := range objects {
expected[i] = objectEntryFromRaw(obj)
}
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{1},
BucketName: "mybucket",
Recursive: true,
@ -222,8 +222,8 @@ func TestIterateObjects(t *testing.T) {
for i, obj := range objectsBucketA {
expected[i] = objectEntryFromRaw(obj)
}
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{1},
BucketName: "bucket-a",
Recursive: true,
@ -243,8 +243,8 @@ func TestIterateObjects(t *testing.T) {
for i, obj := range objectsProject1 {
expected[i] = objectEntryFromRaw(obj)
}
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: uuid.UUID{1},
BucketName: "mybucket",
Recursive: true,
@ -271,8 +271,8 @@ func TestIterateObjects(t *testing.T) {
"g",
})
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -291,8 +291,8 @@ func TestIterateObjects(t *testing.T) {
},
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -312,8 +312,8 @@ func TestIterateObjects(t *testing.T) {
},
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -333,8 +333,8 @@ func TestIterateObjects(t *testing.T) {
},
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -349,8 +349,8 @@ func TestIterateObjects(t *testing.T) {
),
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -366,8 +366,8 @@ func TestIterateObjects(t *testing.T) {
),
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -382,8 +382,8 @@ func TestIterateObjects(t *testing.T) {
),
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Recursive: true,
@ -412,8 +412,8 @@ func TestIterateObjects(t *testing.T) {
"g",
})
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,
@ -427,8 +427,8 @@ func TestIterateObjects(t *testing.T) {
},
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,
@ -443,8 +443,8 @@ func TestIterateObjects(t *testing.T) {
},
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,
@ -459,8 +459,8 @@ func TestIterateObjects(t *testing.T) {
},
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,
@ -474,8 +474,8 @@ func TestIterateObjects(t *testing.T) {
),
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,
@ -490,8 +490,8 @@ func TestIterateObjects(t *testing.T) {
),
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,
@ -505,8 +505,8 @@ func TestIterateObjects(t *testing.T) {
),
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,
@ -517,8 +517,8 @@ func TestIterateObjects(t *testing.T) {
Result: nil,
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,
@ -533,8 +533,8 @@ func TestIterateObjects(t *testing.T) {
),
}.Check(ctx, t, db)
IterateObjects{
Opts: metabase.IterateObjects{
IterateObjectsWithStatus{
Opts: metabase.IterateObjectsWithStatus{
ProjectID: projectID,
BucketName: bucketName,
Status: metabase.Committed,

12
satellite/metainfo/metabase/list.go
View File

@ -52,8 +52,8 @@ type IterateCursor struct {
Version Version
}
// IterateObjects contains arguments necessary for listing objects in a bucket.
type IterateObjects struct {
// IterateObjectsWithStatus contains arguments necessary for listing objects in a bucket.
type IterateObjectsWithStatus struct {
ProjectID uuid.UUID
BucketName string
Recursive bool
@ -63,17 +63,17 @@ type IterateObjects struct {
Status ObjectStatus
}
// IterateObjectsAllVersions iterates through all versions of all committed objects.
func (db *DB) IterateObjectsAllVersions(ctx context.Context, opts IterateObjects, fn func(context.Context, ObjectsIterator) error) (err error) {
// IterateObjectsAllVersionsWithStatus iterates through all versions of all objects with specified status.
func (db *DB) IterateObjectsAllVersionsWithStatus(ctx context.Context, opts IterateObjectsWithStatus, fn func(context.Context, ObjectsIterator) error) (err error) {
defer mon.Task()(&ctx)(&err)
if err = opts.Verify(); err != nil {
return err
}
return iterateAllVersions(ctx, db, opts, fn)
return iterateAllVersionsWithStatus(ctx, db, opts, fn)
}
// Verify verifies get object request fields.
func (opts *IterateObjects) Verify() error {
func (opts *IterateObjectsWithStatus) Verify() error {
switch {
case opts.ProjectID.IsZero():
return ErrInvalidRequest.New("ProjectID missing")

8
satellite/metainfo/metabase/test_test.go
View File

@ -350,18 +350,18 @@ func (coll *IterateCollector) Add(ctx context.Context, it metabase.ObjectsIterat
return nil
}
type IterateObjects struct {
Opts metabase.IterateObjects
type IterateObjectsWithStatus struct {
Opts metabase.IterateObjectsWithStatus
Result []metabase.ObjectEntry
ErrClass *errs.Class
ErrText string
}
func (step IterateObjects) Check(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
func (step IterateObjectsWithStatus) Check(ctx *testcontext.Context, t *testing.T, db *metabase.DB) {
var result IterateCollector
err := db.IterateObjectsAllVersions(ctx, step.Opts, result.Add)
err := db.IterateObjectsAllVersionsWithStatus(ctx, step.Opts, result.Add)
checkError(t, err, step.ErrClass, step.ErrText)
diff := cmp.Diff(step.Result, []metabase.ObjectEntry(result), cmpopts.EquateApproxTime(5*time.Second))

4
satellite/metainfo/metainfo.go
View File

@ -885,8 +885,8 @@ func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListReq
resp = &pb.ObjectListResponse{}
// TODO: Replace with IterateObjectsLatestVersion when ready
err = endpoint.metainfo.metabaseDB.IterateObjectsAllVersions(ctx,
metabase.IterateObjects{
err = endpoint.metainfo.metabaseDB.IterateObjectsAllVersionsWithStatus(ctx,
metabase.IterateObjectsWithStatus{
ProjectID: keyInfo.ProjectID,
BucketName: string(req.Bucket),
Prefix: prefix,