JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo: fix duplicates while listing pending objects

Browse Source 
While working on fixing listing for committed objects we didn't fix
the same case for pending objects. For case were we have many
pending objects under different locations we need to set cursor
version to highest value to avoid duplicates.

For case where we have many pending objects under the same location
we will need to make a separate fix.

https://github.com/storj/storj/issues/5570

Change-Id: Id5c8eb728868e8e1177fdbcf65a493142be4eaf0
This commit is contained in:
Michal Niewrzal 2023-02-15 11:12:39 +01:00
parent 9138e84fb1
commit 0a525292e4
2 changed files with 45 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
satellite/metainfo/endpoint_object.go
View File

@ -805,14 +805,14 @@ func (endpoint *Endpoint) ListObjects(ctx context.Context, req *pb.ObjectListReq
if len(cursor.Key) != 0 {
cursor.Key = prefix + cursor.Key
if status == metabase.Committed {
// TODO this is a workaround to avoid duplicates while listing objects by libuplink.
// because version is not part of cursor yet and we can have object with version higher
// than 1 we cannot use hardcoded version 1 as default.
// This workaround should be in place for a longer time even if metainfo protocol will be
// fix as we still want to avoid this problem for older libuplink versions.
cursor.Version = metabase.MaxVersion
}
// TODO this is a workaround to avoid duplicates while listing objects by libuplink.
// because version is not part of cursor yet and we can have object with version higher
// than 1 we cannot use hardcoded version 1 as default.
// This workaround should be in place for a longer time even if metainfo protocol will be
// fix as we still want to avoid this problem for older libuplink versions.
//
// it should be set in case of pending and committed objects
cursor.Version = metabase.MaxVersion
}
includeCustomMetadata := true

37
satellite/metainfo/endpoint_object_test.go
View File

@ -2406,3 +2406,40 @@ func TestListObjectDuplicates(t *testing.T) {
}
})
}
func TestListUploads(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1,
StorageNodeCount: 0,
UplinkCount: 1,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
// basic ListUploads tests, more tests are on storj/uplink side
u := planet.Uplinks[0]
s := planet.Satellites[0]
project, err := u.OpenProject(ctx, s)
require.NoError(t, err)
defer ctx.Check(project.Close)
require.NoError(t, u.CreateBucket(ctx, s, "testbucket"))
// TODO number of objects created can be limited when uplink will
// have an option to control listing limit value for ListUploads
for i := 0; i < 1001; i++ {
_, err := project.BeginUpload(ctx, "testbucket", "object"+strconv.Itoa(i), nil)
require.NoError(t, err)
}
list := project.ListUploads(ctx, "testbucket", nil)
items := 0
for list.Next() {
items++
}
require.NoError(t, list.Err())
// TODO result should be 1001 but we have bug in libuplink
// were it's not possible to get second page of results for
// pending objets.
// test will fail when we will fix uplink and we will need to adjust this test
require.Equal(t, 1000, items)
})
}