JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite: adjust code to handle context cancelation for SQL queries

Browse Source 
Our DB support in storj/private was updated to enable basic context
support for executing SQL queries. This change requires some small
adjustments as not all parts were working correctly.

storj/private commit with change:
4bc77107b7acfcc2f7ad65796d5dd3d7c64801e4

Change-Id: I64d7ed92788ea0920d12cecd1aa0e414720e9b9c
This commit is contained in:
Michal Niewrzal 2023-01-27 10:05:49 +01:00
parent 6f11c8b32c
commit bd8867cd09
10 changed files with 29 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -55,7 +55,7 @@ require (
storj.io/common v0.0.0-20230125172029-ac90a56db6dd
storj.io/drpc v0.0.32
storj.io/monkit-jaeger v0.0.0-20220915074555-d100d7589f41
storj.io/private v0.0.0-20221108123115-3a27297f0b78
storj.io/private v0.0.0-20230123202745-d3e63b336444
storj.io/uplink v1.10.1-0.20230125195524-beab4fa36cf1
)

4
go.sum
View File

@ -945,7 +945,7 @@ storj.io/drpc v0.0.32 h1:5p5ZwsK/VOgapaCu+oxaPVwO6UwIs+iwdMiD50+R4PI=
storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
storj.io/monkit-jaeger v0.0.0-20220915074555-d100d7589f41 h1:SVuEocEhZfFc13J1AmlVLitdGXTVrvmbzN4Z9C9Ms40=
storj.io/monkit-jaeger v0.0.0-20220915074555-d100d7589f41/go.mod h1:iK+dmHZZXQlW7ahKdNSOo+raMk5BDL2wbD62FIeXLWs=
storj.io/private v0.0.0-20221108123115-3a27297f0b78 h1:kHJEx5AAASp+zGaBPwBJyqZ5zFkde6nLyELGlxdZmiM=
storj.io/private v0.0.0-20221108123115-3a27297f0b78/go.mod h1:hc63X333SVJ3UrOxXpGN4VnwqnmbWgAHln3wYJ8AroI=
storj.io/private v0.0.0-20230123202745-d3e63b336444 h1:qwjIC8HoYGH8H5jYWhxPVUqKK2y3lWSnKZiPtWb08GA=
storj.io/private v0.0.0-20230123202745-d3e63b336444/go.mod h1:hc63X333SVJ3UrOxXpGN4VnwqnmbWgAHln3wYJ8AroI=
storj.io/uplink v1.10.1-0.20230125195524-beab4fa36cf1 h1:UrnuA+dMgn3e9c8k15a1hyN2W+IqqBrt+cXJGV6+7qk=
storj.io/uplink v1.10.1-0.20230125195524-beab4fa36cf1/go.mod h1:fYTIRgr06karY5t3ZHaVi5TYABLnQJThpVMZkObU+J4=

8
private/testplanet/planet.go
View File

@ -357,7 +357,13 @@ func (planet *Planet) Shutdown() error {
errlist.Add(planet.VersionControl.Close())
errlist.Add(os.RemoveAll(planet.directory))
return errlist.Err()
// workaround for not being able to catch context.Canceled error from net package
err := errlist.Err()
if err != nil && strings.Contains(err.Error(), "operation was canceled") {
return nil
}
return err
}
// Identities returns the identity provider for this planet.

5
satellite/gracefulexit/endpoint.go
View File

@ -13,6 +13,7 @@ import (
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"storj.io/common/context2"
"storj.io/common/errs2"
"storj.io/common/identity"
"storj.io/common/pb"
@ -238,6 +239,10 @@ func (endpoint *Endpoint) Process(stream pb.DRPCSatelliteGracefulExit_ProcessStr
if !wasSuccessful {
return rpcstatus.Error(rpcstatus.Canceled, "graceful exit processing interrupted (node should reconnect and continue)")
}
// ignore cancelled context which was triggered to finish loop but we still need to do some DB operations
ctx = context2.WithoutCancellation(ctx)
isDisqualified, err := endpoint.handleDisqualifiedNode(ctx, nodeID)
if err != nil {
return rpcstatus.Error(rpcstatus.Internal, err.Error())

7
satellite/metabase/delete_bucket_test.go
View File

@ -11,6 +11,7 @@ import (
"time"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
"storj.io/common/storj"
"storj.io/common/testcontext"
@ -303,8 +304,9 @@ func TestDeleteBucketObjectsParallel(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 5, len(objects))
var errgroup errgroup.Group
for i := 0; i < 3; i++ {
ctx.Go(func() error {
errgroup.Go(func() error {
_, err := db.DeleteBucketObjects(ctx, metabase.DeleteBucketObjects{
Bucket: root.Location().Bucket(),
BatchSize: 2,
@ -315,8 +317,7 @@ func TestDeleteBucketObjectsParallel(t *testing.T) {
return err
})
}
ctx.Wait()
require.NoError(t, errgroup.Wait())
metabasetest.Verify{}.Check(ctx, t, db)
})

6
satellite/metainfo/attribution_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sync/errgroup"
"storj.io/common/memory"
"storj.io/common/testcontext"
@ -365,9 +366,10 @@ func TestBucketAttributionConcurrentUpload(t *testing.T) {
project, err := config.OpenProject(ctx, planet.Uplinks[0].Access[satellite.ID()])
require.NoError(t, err)
var errgroup errgroup.Group
for i := 0; i < 3; i++ {
i := i
ctx.Go(func() error {
errgroup.Go(func() error {
upload, err := project.UploadObject(ctx, "attr-bucket", "key"+strconv.Itoa(i), nil)
require.NoError(t, err)
@ -380,7 +382,7 @@ func TestBucketAttributionConcurrentUpload(t *testing.T) {
})
}
ctx.Wait()
require.NoError(t, errgroup.Wait())
attributionInfo, err := planet.Satellites[0].DB.Attribution().Get(ctx, planet.Uplinks[0].Projects[0].ID, []byte("attr-bucket"))
require.NoError(t, err)

2
testsuite/storjscan/go.mod
View File

@ -11,7 +11,7 @@ require (
go.uber.org/zap v1.21.0
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
storj.io/common v0.0.0-20230125172029-ac90a56db6dd
storj.io/private v0.0.0-20221108123115-3a27297f0b78
storj.io/private v0.0.0-20230123202745-d3e63b336444
storj.io/storj v1.63.1
storj.io/storjscan v0.0.0-20220926140643-1623c3b391b0
storj.io/uplink v1.10.1-0.20230125195524-beab4fa36cf1

4
testsuite/storjscan/go.sum
View File

@ -1266,8 +1266,8 @@ storj.io/monkit-jaeger v0.0.0-20220726162929-c3a9898b5bca/go.mod h1:iK+dmHZZXQlW
storj.io/monkit-jaeger v0.0.0-20220915074555-d100d7589f41 h1:SVuEocEhZfFc13J1AmlVLitdGXTVrvmbzN4Z9C9Ms40=
storj.io/monkit-jaeger v0.0.0-20220915074555-d100d7589f41/go.mod h1:iK+dmHZZXQlW7ahKdNSOo+raMk5BDL2wbD62FIeXLWs=
storj.io/private v0.0.0-20220823161836-79135887c9be/go.mod h1:GbCItLcLLzsrMClBAa2aaj8xlawtHXWrqhr3D1I3NPM=
storj.io/private v0.0.0-20221108123115-3a27297f0b78 h1:kHJEx5AAASp+zGaBPwBJyqZ5zFkde6nLyELGlxdZmiM=
storj.io/private v0.0.0-20221108123115-3a27297f0b78/go.mod h1:hc63X333SVJ3UrOxXpGN4VnwqnmbWgAHln3wYJ8AroI=
storj.io/private v0.0.0-20230123202745-d3e63b336444 h1:qwjIC8HoYGH8H5jYWhxPVUqKK2y3lWSnKZiPtWb08GA=
storj.io/private v0.0.0-20230123202745-d3e63b336444/go.mod h1:hc63X333SVJ3UrOxXpGN4VnwqnmbWgAHln3wYJ8AroI=
storj.io/storjscan v0.0.0-20220926140643-1623c3b391b0 h1:pSfGf9E9OlUd17W7LSpL4tTONIyFji6dz8I2iTDd8BY=
storj.io/storjscan v0.0.0-20220926140643-1623c3b391b0/go.mod h1:5nLgAOl1KTDVyqORAhvrp+167PtShEuS1L3pJgXPjwo=
storj.io/uplink v1.10.1-0.20230125195524-beab4fa36cf1 h1:UrnuA+dMgn3e9c8k15a1hyN2W+IqqBrt+cXJGV6+7qk=

2
testsuite/ui/go.mod
View File

@ -12,7 +12,7 @@ require (
go.uber.org/zap v1.17.0
storj.io/common v0.0.0-20230125172029-ac90a56db6dd
storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31
storj.io/private v0.0.0-20221108123115-3a27297f0b78
storj.io/private v0.0.0-20230123202745-d3e63b336444
storj.io/storj v0.12.1-0.20221125175451-ef4b564b82f7
)

4
testsuite/ui/go.sum
View File

@ -1514,8 +1514,8 @@ storj.io/monkit-jaeger v0.0.0-20210225162224-66fb37637bf6/go.mod h1:gj4vuCeyCRjR
storj.io/monkit-jaeger v0.0.0-20220915074555-d100d7589f41 h1:SVuEocEhZfFc13J1AmlVLitdGXTVrvmbzN4Z9C9Ms40=
storj.io/monkit-jaeger v0.0.0-20220915074555-d100d7589f41/go.mod h1:iK+dmHZZXQlW7ahKdNSOo+raMk5BDL2wbD62FIeXLWs=
storj.io/private v0.0.0-20210810102517-434aeab3f17d/go.mod h1:wvudoWSyOyB2daZNaMykjjqsSUad/ttFlUwgelg9+qg=
storj.io/private v0.0.0-20221108123115-3a27297f0b78 h1:kHJEx5AAASp+zGaBPwBJyqZ5zFkde6nLyELGlxdZmiM=
storj.io/private v0.0.0-20221108123115-3a27297f0b78/go.mod h1:hc63X333SVJ3UrOxXpGN4VnwqnmbWgAHln3wYJ8AroI=
storj.io/private v0.0.0-20230123202745-d3e63b336444 h1:qwjIC8HoYGH8H5jYWhxPVUqKK2y3lWSnKZiPtWb08GA=
storj.io/private v0.0.0-20230123202745-d3e63b336444/go.mod h1:hc63X333SVJ3UrOxXpGN4VnwqnmbWgAHln3wYJ8AroI=
storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo=
storj.io/uplink v1.7.1-0.20211103104100-a785482780d8/go.mod h1:pKqsMpNMIAz//2TXzUGOR6tpu3iyabvXV4VWINj4jaY=
storj.io/uplink v1.10.1-0.20230125195524-beab4fa36cf1 h1:UrnuA+dMgn3e9c8k15a1hyN2W+IqqBrt+cXJGV6+7qk=