From 9370bc4580008c59f4f6f952cbd9eb09fc0b0388 Mon Sep 17 00:00:00 2001 From: Egon Elbre Date: Mon, 10 Jul 2023 18:01:48 +0300 Subject: [PATCH] satellite/{nodeselection,overlay}: bump common and fix some potential issues * Handle failed country code conversion. * Avoid potential issues with a data-race due to shared slice. Updates #6028 Change-Id: If7beef2619abd084e1f4109de2d323f834a6090a --- go.mod | 2 +- go.sum | 4 +-- satellite/nodeselection/filter.go | 13 ++++---- satellite/nodeselection/filter_test.go | 2 +- satellite/nodeselection/region.go | 44 ++++++++++++++++++++++++++ satellite/overlay/placement.go | 22 +++++++------ satellite/overlay/placement_test.go | 6 ++++ testsuite/storjscan/go.mod | 2 +- testsuite/storjscan/go.sum | 4 +-- testsuite/ui/go.mod | 2 +- testsuite/ui/go.sum | 4 +-- 11 files changed, 78 insertions(+), 27 deletions(-) create mode 100644 satellite/nodeselection/region.go diff --git a/go.mod b/go.mod index 205193e04..7122085b8 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/segmentio/analytics-go.v3 v3.1.0 gopkg.in/yaml.v3 v3.0.1 - storj.io/common v0.0.0-20230703090058-81cb588c23b6 + storj.io/common v0.0.0-20230707075619-cbf38d719fcb storj.io/drpc v0.0.33 storj.io/monkit-jaeger v0.0.0-20220915074555-d100d7589f41 storj.io/private v0.0.0-20230703113355-ccd4db5ae659 diff --git a/go.sum b/go.sum index 9e7d5ed10..8ad056d80 100644 --- a/go.sum +++ b/go.sum @@ -1015,8 +1015,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= storj.io/common v0.0.0-20220719163320-cd2ef8e1b9b0/go.mod h1:mCYV6Ud5+cdbuaxdPD5Zht/HYaIn0sffnnws9ErkrMQ= -storj.io/common v0.0.0-20230703090058-81cb588c23b6 h1:UXMLomBVcTeIbyrTJWCvaxOixMwE9OIUuZR//17kYO4= -storj.io/common v0.0.0-20230703090058-81cb588c23b6/go.mod h1:zu2L8WdpvfIBrCbBTgPsz4qhHSArYSiDgRcV1RLlIF8= +storj.io/common v0.0.0-20230707075619-cbf38d719fcb h1:8Ma3a9a6mYhRUrwUN71J2M+EPiZOcCkL3clFCNv2oTo= +storj.io/common v0.0.0-20230707075619-cbf38d719fcb/go.mod h1:zu2L8WdpvfIBrCbBTgPsz4qhHSArYSiDgRcV1RLlIF8= storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI= storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4= diff --git a/satellite/nodeselection/filter.go b/satellite/nodeselection/filter.go index 9fe5af480..9173c9e68 100644 --- a/satellite/nodeselection/filter.go +++ b/satellite/nodeselection/filter.go @@ -21,18 +21,17 @@ type NodeFilters []NodeFilter // NodeFilterFunc is helper to use func as NodeFilter. type NodeFilterFunc func(node *SelectedNode) bool -// ExcludeAll will never select any node. -var ExcludeAll = NodeFilters{ - NodeFilterFunc(func(node *SelectedNode) bool { - return false - }), -} - // MatchInclude implements NodeFilter interface. func (n NodeFilterFunc) MatchInclude(node *SelectedNode) bool { return n(node) } +// ExcludeAllFilter will never select any node. +type ExcludeAllFilter struct{} + +// MatchInclude implements NodeFilter interface. +func (ExcludeAllFilter) MatchInclude(node *SelectedNode) bool { return false } + // MatchInclude implements NodeFilter interface. func (n NodeFilters) MatchInclude(node *SelectedNode) bool { for _, filter := range n { diff --git a/satellite/nodeselection/filter_test.go b/satellite/nodeselection/filter_test.go index 4f9e86cb6..a0098e120 100644 --- a/satellite/nodeselection/filter_test.go +++ b/satellite/nodeselection/filter_test.go @@ -74,7 +74,7 @@ func TestCriteria_NodeIDAndSubnet(t *testing.T) { func TestCriteria_Geofencing(t *testing.T) { eu := NodeFilters{}.WithCountryFilter(func(code location.CountryCode) bool { - for _, c := range location.EuCountries { + for _, c := range EuCountries { if c == code { return true } diff --git a/satellite/nodeselection/region.go b/satellite/nodeselection/region.go new file mode 100644 index 000000000..375d89ee6 --- /dev/null +++ b/satellite/nodeselection/region.go @@ -0,0 +1,44 @@ +// Copyright (C) 2023 Storj Labs, Inc. +// See LICENSE for copying information. + +package nodeselection + +import "storj.io/common/storj/location" + +// EuCountries defines the member countries of European Union. +var EuCountries = []location.CountryCode{ + location.Austria, + location.Belgium, + location.Bulgaria, + location.Croatia, + location.Cyprus, + location.Czechia, + location.Denmark, + location.Estonia, + location.Finland, + location.France, + location.Germany, + location.Greece, + location.Hungary, + location.Ireland, + location.Italy, + location.Lithuania, + location.Latvia, + location.Luxembourg, + location.Malta, + location.Netherlands, + location.Poland, + location.Portugal, + location.Romania, + location.Slovenia, + location.Slovakia, + location.Spain, + location.Sweden, +} + +// EeaCountries defined the EEA countries. +var EeaCountries = append(EuCountries, + location.Iceland, + location.Liechtenstein, + location.Norway, +) diff --git a/satellite/overlay/placement.go b/satellite/overlay/placement.go index cebc26e40..b4367c6bc 100644 --- a/satellite/overlay/placement.go +++ b/satellite/overlay/placement.go @@ -11,6 +11,7 @@ import ( "github.com/jtolio/mito" "github.com/spf13/pflag" "github.com/zeebo/errs" + "golang.org/x/exp/slices" "storj.io/common/storj" "storj.io/common/storj/location" @@ -64,12 +65,7 @@ func NewPlacementRules() *ConfigurablePlacementRule { // AddLegacyStaticRules initializes all the placement rules defined earlier in static golang code. func (d *ConfigurablePlacementRule) AddLegacyStaticRules() { d.placements[storj.EEA] = nodeselection.NodeFilters{}.WithCountryFilter(func(isoCountryCode location.CountryCode) bool { - for _, c := range location.EeaNonEuCountries { - if c == isoCountryCode { - return true - } - } - for _, c := range location.EuCountries { + for _, c := range nodeselection.EeaCountries { if c == isoCountryCode { return true } @@ -77,7 +73,7 @@ func (d *ConfigurablePlacementRule) AddLegacyStaticRules() { return false }) d.placements[storj.EU] = nodeselection.NodeFilters{}.WithCountryFilter(func(isoCountryCode location.CountryCode) bool { - for _, c := range location.EuCountries { + for _, c := range nodeselection.EuCountries { if c == isoCountryCode { return true } @@ -106,7 +102,11 @@ func (d *ConfigurablePlacementRule) AddPlacementFromString(definitions string) e "country": func(countries ...string) (nodeselection.NodeFilters, error) { countryCodes := make([]location.CountryCode, len(countries)) for i, country := range countries { - countryCodes[i] = location.ToCountryCode(country) + code := location.ToCountryCode(country) + if code == location.None { + return nil, errs.New("invalid country code %q", code) + } + countryCodes[i] = code } return nodeselection.NodeFilters{}.WithCountryFilter(func(code location.CountryCode) bool { for _, expectedCode := range countryCodes { @@ -170,7 +170,9 @@ func (d *ConfigurablePlacementRule) CreateFilters(constraint storj.PlacementCons return nodeselection.NodeFilters{} } if filters, found := d.placements[constraint]; found { - return filters + return slices.Clone(filters) + } + return nodeselection.NodeFilters{ + nodeselection.ExcludeAllFilter{}, } - return nodeselection.ExcludeAll } diff --git a/satellite/overlay/placement_test.go b/satellite/overlay/placement_test.go index c6bd32e39..edf5b5d26 100644 --- a/satellite/overlay/placement_test.go +++ b/satellite/overlay/placement_test.go @@ -17,6 +17,12 @@ func TestPlacementFromString(t *testing.T) { signer, err := storj.NodeIDFromString("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4") require.NoError(t, err) + t.Run("invalid country-code", func(t *testing.T) { + p := NewPlacementRules() + err := p.AddPlacementFromString(`1:country("ZZZZ")`) + require.Error(t, err) + }) + t.Run("single country", func(t *testing.T) { p := NewPlacementRules() err := p.AddPlacementFromString(`11:country("GB")`) diff --git a/testsuite/storjscan/go.mod b/testsuite/storjscan/go.mod index c32f27a65..2324d96b9 100644 --- a/testsuite/storjscan/go.mod +++ b/testsuite/storjscan/go.mod @@ -9,7 +9,7 @@ require ( github.com/zeebo/errs v1.3.0 go.uber.org/zap v1.21.0 golang.org/x/sync v0.1.0 - storj.io/common v0.0.0-20230703090058-81cb588c23b6 + storj.io/common v0.0.0-20230707075619-cbf38d719fcb storj.io/private v0.0.0-20230703113355-ccd4db5ae659 storj.io/storj v1.63.1 storj.io/storjscan v0.0.0-20220926140643-1623c3b391b0 diff --git a/testsuite/storjscan/go.sum b/testsuite/storjscan/go.sum index c239187da..d4928a27c 100644 --- a/testsuite/storjscan/go.sum +++ b/testsuite/storjscan/go.sum @@ -1249,8 +1249,8 @@ rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= storj.io/common v0.0.0-20220719163320-cd2ef8e1b9b0/go.mod h1:mCYV6Ud5+cdbuaxdPD5Zht/HYaIn0sffnnws9ErkrMQ= -storj.io/common v0.0.0-20230703090058-81cb588c23b6 h1:UXMLomBVcTeIbyrTJWCvaxOixMwE9OIUuZR//17kYO4= -storj.io/common v0.0.0-20230703090058-81cb588c23b6/go.mod h1:zu2L8WdpvfIBrCbBTgPsz4qhHSArYSiDgRcV1RLlIF8= +storj.io/common v0.0.0-20230707075619-cbf38d719fcb h1:8Ma3a9a6mYhRUrwUN71J2M+EPiZOcCkL3clFCNv2oTo= +storj.io/common v0.0.0-20230707075619-cbf38d719fcb/go.mod h1:zu2L8WdpvfIBrCbBTgPsz4qhHSArYSiDgRcV1RLlIF8= storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI= storj.io/drpc v0.0.33/go.mod h1:vR804UNzhBa49NOJ6HeLjd2H3MakC1j5Gv8bsOQT6N4= diff --git a/testsuite/ui/go.mod b/testsuite/ui/go.mod index 337abff69..7d397365b 100644 --- a/testsuite/ui/go.mod +++ b/testsuite/ui/go.mod @@ -10,7 +10,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.2 go.uber.org/zap v1.23.0 - storj.io/common v0.0.0-20230703090058-81cb588c23b6 + storj.io/common v0.0.0-20230707075619-cbf38d719fcb storj.io/gateway-mt v1.51.1-0.20230417204402-7d9bb25bc297 storj.io/private v0.0.0-20230703113355-ccd4db5ae659 storj.io/storj v0.12.1-0.20221125175451-ef4b564b82f7 diff --git a/testsuite/ui/go.sum b/testsuite/ui/go.sum index 194802a63..3dd94b8bc 100644 --- a/testsuite/ui/go.sum +++ b/testsuite/ui/go.sum @@ -1960,8 +1960,8 @@ sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1 sourcegraph.com/sourcegraph/go-diff v0.5.0/go.mod h1:kuch7UrkMzY0X+p9CRK03kfuPQ2zzQcaEFbx8wA8rck= sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0= storj.io/common v0.0.0-20220719163320-cd2ef8e1b9b0/go.mod h1:mCYV6Ud5+cdbuaxdPD5Zht/HYaIn0sffnnws9ErkrMQ= -storj.io/common v0.0.0-20230703090058-81cb588c23b6 h1:UXMLomBVcTeIbyrTJWCvaxOixMwE9OIUuZR//17kYO4= -storj.io/common v0.0.0-20230703090058-81cb588c23b6/go.mod h1:zu2L8WdpvfIBrCbBTgPsz4qhHSArYSiDgRcV1RLlIF8= +storj.io/common v0.0.0-20230707075619-cbf38d719fcb h1:8Ma3a9a6mYhRUrwUN71J2M+EPiZOcCkL3clFCNv2oTo= +storj.io/common v0.0.0-20230707075619-cbf38d719fcb/go.mod h1:zu2L8WdpvfIBrCbBTgPsz4qhHSArYSiDgRcV1RLlIF8= storj.io/dotworld v0.0.0-20210324183515-0d11aeccd840 h1:oqMwoF6vaOrCe92SKRyr8cc2WSjLYAd8fjpAHA7rNqY= storj.io/drpc v0.0.32/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= storj.io/drpc v0.0.33 h1:yCGZ26r66ZdMP0IcTYsj7WDAUIIjzXk6DJhbhvt9FHI=