JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo: support desired node number for download object/segment

Browse Source 
This modification introduce support of the new "desired node" field of download segment/object.

This can be used to request more nodes than the suggested minimum. It can be used to achieve better performance in exchange of using more bandwidth. (more parallel downloads).

Change-Id: Ia167d6979e6d70a597c85070a4ccd1c3a573e406
This commit is contained in:
Márton Elek 2023-01-24 11:22:27 +01:00 committed by Storj Robot
parent ca6e3a9e88
commit 8f8e97de23
6 changed files with 23 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
satellite/audit/disqualification_test.go
View File

@ -140,7 +140,7 @@ func TestDisqualifiedNodesGetNoDownload(t *testing.T) {
err = satellitePeer.Reputation.Service.TestDisqualifyNode(ctx, disqualifiedNode, overlay.DisqualificationReasonUnknown)
require.NoError(t, err)
limits, _, err := satellitePeer.Orders.Service.CreateGetOrderLimits(ctx, bucket, segment, 0)
limits, _, err := satellitePeer.Orders.Service.CreateGetOrderLimits(ctx, bucket, segment, 0, 0)
require.NoError(t, err)
notNilLimits := []*pb.AddressedOrderLimit{}

2
satellite/metainfo/endpoint_object.go
View File

@ -543,7 +543,7 @@ func (endpoint *Endpoint) DownloadObject(ctx context.Context, req *pb.ObjectDown
}}, nil
}
limits, privateKey, err := endpoint.orders.CreateGetOrderLimits(ctx, object.Location().Bucket(), segment, downloadSizes.orderLimit)
limits, privateKey, err := endpoint.orders.CreateGetOrderLimits(ctx, object.Location().Bucket(), segment, req.GetDesiredNodes(), downloadSizes.orderLimit)
if err != nil {
if orders.ErrDownloadFailedNotEnoughPieces.Has(err) {
endpoint.log.Error("Unable to create order limits.",

2
satellite/metainfo/endpoint_segment.go
View File

@ -703,7 +703,7 @@ func (endpoint *Endpoint) DownloadSegment(ctx context.Context, req *pb.SegmentDo
}
// Remote segment
limits, privateKey, err := endpoint.orders.CreateGetOrderLimits(ctx, bucket, segment, 0)
limits, privateKey, err := endpoint.orders.CreateGetOrderLimits(ctx, bucket, segment, req.GetDesiredNodes(), 0)
if err != nil {
if orders.ErrDownloadFailedNotEnoughPieces.Has(err) {
endpoint.log.Error("Unable to create order limits.",

5
satellite/orders/service.go
View File

@ -126,7 +126,7 @@ func (service *Service) updateBandwidth(ctx context.Context, bucket metabase.Buc
}
// CreateGetOrderLimits creates the order limits for downloading the pieces of a segment.
func (service *Service) CreateGetOrderLimits(ctx context.Context, bucket metabase.BucketLocation, segment metabase.Segment, overrideLimit int64) (_ []*pb.AddressedOrderLimit, privateKey storj.PiecePrivateKey, err error) {
func (service *Service) CreateGetOrderLimits(ctx context.Context, bucket metabase.BucketLocation, segment metabase.Segment, desiredNodes int32, overrideLimit int64) (_ []*pb.AddressedOrderLimit, privateKey storj.PiecePrivateKey, err error) {
defer mon.Task()(&ctx)(&err)
redundancy, err := eestream.NewRedundancyStrategyFromStorj(segment.Redundancy)
@ -155,7 +155,10 @@ func (service *Service) CreateGetOrderLimits(ctx context.Context, bucket metabas
}
neededLimits := segment.Redundancy.DownloadNodes()
if desiredNodes > neededLimits {
neededLimits = desiredNodes
}
pieces := segment.Pieces
for _, pieceIndex := range service.perm(len(pieces)) {
piece := pieces[pieceIndex]

2
satellite/orders/service_integration_test.go
View File

@ -38,7 +38,7 @@ func TestOrderLimitsEncryptedMetadata(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 1, len(segments))
limits, _, err := satellitePeer.Orders.Service.CreateGetOrderLimits(ctx, bucket, segments[0], 0)
limits, _, err := satellitePeer.Orders.Service.CreateGetOrderLimits(ctx, bucket, segments[0], 0, 0)
require.NoError(t, err)
require.Equal(t, 3, len(limits))

18
satellite/orders/service_test.go
View File

@ -81,8 +81,8 @@ func TestGetOrderLimits(t *testing.T) {
RootPieceID: testrand.PieceID(),
}
checkExpectedLimits := func(received int) {
limits, _, err := service.CreateGetOrderLimits(ctx, bucket, segment, 0)
checkExpectedLimits := func(requested int32, received int) {
limits, _, err := service.CreateGetOrderLimits(ctx, bucket, segment, requested, 0)
require.NoError(t, err)
realLimits := 0
for _, limit := range limits {
@ -94,7 +94,19 @@ func TestGetOrderLimits(t *testing.T) {
}
t.Run("Do not request any specific number", func(t *testing.T) {
checkExpectedLimits(6)
checkExpectedLimits(0, 6)
})
t.Run("Request less than the optimal", func(t *testing.T) {
checkExpectedLimits(2, 6)
})
t.Run("Request more than the optimal", func(t *testing.T) {
checkExpectedLimits(8, 8)
})
t.Run("Request more than the replication", func(t *testing.T) {
checkExpectedLimits(1000, 8)
})
}