satellite/overlay: Add ability to exclude country codes on upload

Create global config to specify a list of country codes that should be
excluded from node selection during uploads.

This exclusion is not implemented when the upload selection cache is
disabled.

Change-Id: Ic41e8b4f18857a11045668eac23107da99668a72
This commit is contained in:
Moby von Briesen 2022-02-25 11:53:24 -05:00 committed by Maximillian von Briesen
parent b4e42ceb23
commit b2d342aa9b
7 changed files with 58 additions and 13 deletions

View File

@ -5,6 +5,7 @@ package uploadselection
import (
"storj.io/common/storj"
"storj.io/common/storj/location"
)
// Criteria to filter nodes.
@ -12,6 +13,7 @@ type Criteria struct {
ExcludeNodeIDs []storj.NodeID
AutoExcludeSubnets map[string]struct{} // initialize it with empty map to keep only one node per subnet.
Placement storj.PlacementConstraint
ExcludedCountryCodes []location.CountryCode
}
// MatchInclude returns with true if node is selected.
@ -30,6 +32,13 @@ func (c *Criteria) MatchInclude(node *Node) bool {
}
c.AutoExcludeSubnets[node.LastNet] = struct{}{}
}
for _, code := range c.ExcludedCountryCodes {
if node.CountryCode == code {
return false
}
}
return true
}

View File

@ -10,6 +10,7 @@ import (
"github.com/zeebo/errs"
"storj.io/common/storj"
"storj.io/common/storj/location"
)
// ErrNotEnoughNodes is when selecting nodes failed with the given parameters.
@ -88,6 +89,7 @@ type Request struct {
Distinct bool
ExcludedIDs []storj.NodeID
Placement storj.PlacementConstraint
ExcludedCountryCodes []string
}
// Select selects requestedCount nodes where there will be newFraction nodes.
@ -111,6 +113,10 @@ func (state *State) Select(ctx context.Context, request Request) (_ []*Node, err
criteria.ExcludeNodeIDs = request.ExcludedIDs
}
for _, code := range request.ExcludedCountryCodes {
criteria.ExcludedCountryCodes = append(criteria.ExcludedCountryCodes, location.ToCountryCode(code))
}
criteria.Placement = request.Placement
if request.Distinct {

View File

@ -44,6 +44,8 @@ type NodeSelectionConfig struct {
MinimumDiskSpace memory.Size `help:"how much disk space a node at minimum must have to be selected for upload" default:"500.00MB" testDefault:"100.00MB"`
AsOfSystemTime AsOfSystemTimeConfig
UploadExcludedCountryCodes []string `help:"list of country codes to exclude from node selection for uploads" default:"" testDefault:"FR,BE"`
}
// GeoIPConfig is a configuration struct that helps configure the GeoIP lookup features on the satellite.

View File

@ -398,6 +398,7 @@ func (service *Service) FindStorageNodesForUpload(ctx context.Context, req FindS
req.AsOfSystemInterval = service.config.Node.AsOfSystemTime.DefaultInterval
}
// TODO excluding country codes on upload if cache is disabled is not implemented
if service.config.NodeSelectionCache.Disabled {
return service.FindStorageNodesWithPreferences(ctx, req, &service.config.Node)
}

View File

@ -111,6 +111,7 @@ func (cache *UploadSelectionCache) GetNodes(ctx context.Context, req FindStorage
Distinct: cache.selectionConfig.DistinctIP,
ExcludedIDs: req.ExcludedIDs,
Placement: req.Placement,
ExcludedCountryCodes: cache.selectionConfig.UploadExcludedCountryCodes,
})
if uploadselection.ErrNotEnoughNodes.Has(err) {
err = ErrNotEnoughNodes.Wrap(err)

View File

@ -20,6 +20,7 @@ import (
"storj.io/common/sync2"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite"
"storj.io/storj/satellite/overlay"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
@ -226,6 +227,28 @@ func TestGetNodes(t *testing.T) {
})
}
func TestGetNodesExcludeCountryCodes(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 2, UplinkCount: 0,
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
err := planet.Satellites[0].Overlay.Service.TestNodeCountryCode(ctx, planet.StorageNodes[0].ID(), "FR")
require.NoError(t, err)
cache := planet.Satellites[0].Overlay.Service.UploadSelectionCache
// confirm cache.GetNodes returns the correct nodes
selectedNodes, err := cache.GetNodes(ctx, overlay.FindStorageNodesRequest{RequestedCount: 2})
// we only expect one node to be returned, even though we requested two, so there will be an error
require.Error(t, err)
_, new := cache.Size()
require.Equal(t, 2, new)
require.Equal(t, 1, len(selectedNodes))
// the node that was returned should be the one that does not have the "FR" country code
require.Equal(t, planet.StorageNodes[1].ID(), selectedNodes[0].ID)
})
}
func TestGetNodesConcurrent(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()

View File

@ -616,6 +616,9 @@ identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# the amount of time without seeing a node before its considered offline
# overlay.node.online-window: 4h0m0s
# list of country codes to exclude from node selection for uploads
# overlay.node.upload-excluded-country-codes: '[]'
# list of country codes to exclude nodes from target repair selection
# overlay.repair-excluded-country-codes: '[]'