diff --git a/satellite/nodeselection/filter_test.go b/satellite/nodeselection/filter_test.go index ef1cf96e5..4cfc453e2 100644 --- a/satellite/nodeselection/filter_test.go +++ b/satellite/nodeselection/filter_test.go @@ -73,7 +73,7 @@ func TestCriteria_NodeIDAndSubnet(t *testing.T) { } func TestCriteria_Geofencing(t *testing.T) { - eu := NodeFilters{}.WithCountryFilter(EuCountries) + eu := NodeFilters{}.WithCountryFilter(location.NewSet(EuCountries...)) us := NodeFilters{}.WithCountryFilter(location.NewSet(location.UnitedStates)) cases := []struct { diff --git a/satellite/nodeselection/region.go b/satellite/nodeselection/region.go index 81dcf857e..49c9a4d9c 100644 --- a/satellite/nodeselection/region.go +++ b/satellite/nodeselection/region.go @@ -6,7 +6,7 @@ package nodeselection import "storj.io/common/storj/location" // EuCountries defines the member countries of European Union. -var EuCountries = location.NewSet( +var EuCountries = []location.CountryCode{ location.Austria, location.Belgium, location.Bulgaria, @@ -34,11 +34,11 @@ var EuCountries = location.NewSet( location.Slovakia, location.Spain, location.Sweden, -) +} -// EeaCountries defined the EEA countries. -var EeaCountries = EuCountries.With( +// EeaCountriesWithoutEu defined the EEA countries. +var EeaCountriesWithoutEu = []location.CountryCode{ location.Iceland, location.Liechtenstein, location.Norway, -) +} diff --git a/satellite/overlay/placement.go b/satellite/overlay/placement.go index db114db37..3bbbaba9d 100644 --- a/satellite/overlay/placement.go +++ b/satellite/overlay/placement.go @@ -66,8 +66,8 @@ func NewPlacementRules() *ConfigurablePlacementRule { // AddLegacyStaticRules initializes all the placement rules defined earlier in static golang code. func (d *ConfigurablePlacementRule) AddLegacyStaticRules() { - d.placements[storj.EEA] = nodeselection.NodeFilters{nodeselection.NewCountryFilter(nodeselection.EeaCountries)} - d.placements[storj.EU] = nodeselection.NodeFilters{nodeselection.NewCountryFilter(nodeselection.EuCountries)} + d.placements[storj.EEA] = nodeselection.NodeFilters{nodeselection.NewCountryFilter(location.NewSet(nodeselection.EeaCountriesWithoutEu...).With(nodeselection.EuCountries...))} + d.placements[storj.EU] = nodeselection.NodeFilters{nodeselection.NewCountryFilter(location.NewSet(nodeselection.EuCountries...))} d.placements[storj.US] = nodeselection.NodeFilters{nodeselection.NewCountryFilter(location.NewSet(location.UnitedStates))} d.placements[storj.DE] = nodeselection.NodeFilters{nodeselection.NewCountryFilter(location.NewSet(location.Germany))} d.placements[storj.NR] = nodeselection.NodeFilters{nodeselection.NewCountryFilter(location.NewFullSet().Without(location.Russia, location.Belarus, location.None))} @@ -84,14 +84,38 @@ func (d *ConfigurablePlacementRule) AddPlacementFromString(definitions string) e "country": func(countries ...string) (nodeselection.NodeFilters, error) { var set location.Set for _, country := range countries { - code := location.ToCountryCode(country) - if code == location.None { - return nil, errs.New("invalid country code %q", code) + apply := func(modified location.Set, code ...location.CountryCode) location.Set { + return modified.With(code...) + } + if country[0] == '!' { + apply = func(modified location.Set, code ...location.CountryCode) location.Set { + return modified.Without(code...) + } + country = country[1:] + } + switch strings.ToLower(country) { + case "all", "*", "any": + set = location.NewFullSet() + case "none": + set = apply(set, location.None) + case "eu": + set = apply(set, nodeselection.EuCountries...) + case "eea": + set = apply(set, nodeselection.EuCountries...) + set = apply(set, nodeselection.EeaCountriesWithoutEu...) + default: + code := location.ToCountryCode(country) + if code == location.None { + return nil, errs.New("invalid country code %q", code) + } + set = apply(set, code) } - set.Include(code) } return nodeselection.NodeFilters{nodeselection.NewCountryFilter(set)}, nil }, + "placement": func(ix int64) nodeselection.NodeFilter { + return d.placements[storj.PlacementConstraint(ix)] + }, "all": func(filters ...nodeselection.NodeFilters) (nodeselection.NodeFilters, error) { res := nodeselection.NodeFilters{} for _, filter := range filters { @@ -99,6 +123,15 @@ func (d *ConfigurablePlacementRule) AddPlacementFromString(definitions string) e } return res, nil }, + mito.OpAnd: func(env map[any]any, a, b any) (any, error) { + filter1, ok1 := a.(nodeselection.NodeFilter) + filter2, ok2 := b.(nodeselection.NodeFilter) + if !ok1 || !ok2 { + return nil, errs.New("&& is supported only between NodeFilter instances") + } + res := nodeselection.NodeFilters{filter1, filter2} + return res, nil + }, "tag": func(nodeIDstr string, key string, value any) (nodeselection.NodeFilters, error) { nodeID, err := storj.NodeIDFromString(nodeIDstr) if err != nil { diff --git a/satellite/overlay/placement_test.go b/satellite/overlay/placement_test.go index 684e51834..65c3e41cd 100644 --- a/satellite/overlay/placement_test.go +++ b/satellite/overlay/placement_test.go @@ -6,6 +6,7 @@ package overlay import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "storj.io/common/storj" @@ -23,18 +24,30 @@ func TestPlacementFromString(t *testing.T) { require.Error(t, err) }) - t.Run("single country", func(t *testing.T) { - p := NewPlacementRules() - err := p.AddPlacementFromString(`11:country("GB")`) - require.NoError(t, err) - filters := p.placements[storj.PlacementConstraint(11)] - require.NotNil(t, filters) - require.True(t, filters.MatchInclude(&nodeselection.SelectedNode{ - CountryCode: location.UnitedKingdom, - })) - require.False(t, filters.MatchInclude(&nodeselection.SelectedNode{ - CountryCode: location.Germany, - })) + t.Run("country tests", func(t *testing.T) { + countryTest := func(placementDef string, shouldBeIncluded []location.CountryCode, shouldBeExcluded []location.CountryCode) { + p := NewPlacementRules() + err := p.AddPlacementFromString("11:" + placementDef) + require.NoError(t, err) + filters := p.placements[storj.PlacementConstraint(11)] + require.NotNil(t, filters) + for _, code := range shouldBeExcluded { + require.False(t, filters.MatchInclude(&nodeselection.SelectedNode{ + CountryCode: code, + }), "%s shouldn't be included in placement %s", code, placementDef) + } + for _, code := range shouldBeIncluded { + require.True(t, filters.MatchInclude(&nodeselection.SelectedNode{ + CountryCode: code, + }), "%s is not included in placement %s", code, placementDef) + } + } + countryTest(`country("GB")`, []location.CountryCode{location.UnitedKingdom}, []location.CountryCode{location.Germany, location.UnitedStates}) + countryTest(`country("EU")`, []location.CountryCode{location.Germany, location.Hungary}, []location.CountryCode{location.UnitedStates, location.Norway, location.Iceland}) + countryTest(`country("EEA")`, []location.CountryCode{location.Germany, location.Hungary, location.Norway, location.Iceland}, []location.CountryCode{location.UnitedStates}) + countryTest(`country("ALL","!EU")`, []location.CountryCode{location.Norway, location.India}, []location.CountryCode{location.Germany, location.Hungary}) + countryTest(`country("ALL", "!RU", "!BY")`, []location.CountryCode{location.Norway, location.India, location.UnitedStates}, []location.CountryCode{location.Russia, location.Belarus}) + }) t.Run("tag rule", func(t *testing.T) { @@ -57,14 +70,12 @@ func TestPlacementFromString(t *testing.T) { })) }) - t.Run("all rules", func(t *testing.T) { + t.Run("placement reuse", func(t *testing.T) { p := NewPlacementRules() - err := p.AddPlacementFromString(`11:all(country("GB"),tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","foo","bar"))`) + err := p.AddPlacementFromString(`1:tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","foo","bar");2:exclude(placement(1))`) require.NoError(t, err) - filters := p.placements[storj.PlacementConstraint(11)] - require.NotNil(t, filters) - require.True(t, filters.MatchInclude(&nodeselection.SelectedNode{ - CountryCode: location.UnitedKingdom, + + require.True(t, p.placements[storj.PlacementConstraint(1)].MatchInclude(&nodeselection.SelectedNode{ Tags: nodeselection.NodeTags{ { Signer: signer, @@ -73,19 +84,62 @@ func TestPlacementFromString(t *testing.T) { }, }, })) - require.False(t, filters.MatchInclude(&nodeselection.SelectedNode{ - CountryCode: location.UnitedKingdom, + + placement2 := p.placements[storj.PlacementConstraint(2)] + require.False(t, placement2.MatchInclude(&nodeselection.SelectedNode{ + Tags: nodeselection.NodeTags{ + { + Signer: signer, + Name: "foo", + Value: []byte("bar"), + }, + }, })) - require.False(t, filters.MatchInclude(&nodeselection.SelectedNode{ + require.True(t, placement2.MatchInclude(&nodeselection.SelectedNode{ CountryCode: location.Germany, - Tags: nodeselection.NodeTags{ - { - Signer: signer, - Name: "foo", - Value: []byte("bar"), - }, - }, })) + + }) + + t.Run("all rules", func(t *testing.T) { + for _, syntax := range []string{ + `11:all(country("GB"),tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","foo","bar"))`, + `11:country("GB") && tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","foo","bar")`, + } { + p := NewPlacementRules() + err := p.AddPlacementFromString(syntax) + require.NoError(t, err) + filters := p.placements[storj.PlacementConstraint(11)] + require.NotNil(t, filters) + require.True(t, filters.MatchInclude(&nodeselection.SelectedNode{ + CountryCode: location.UnitedKingdom, + Tags: nodeselection.NodeTags{ + { + Signer: signer, + Name: "foo", + Value: []byte("bar"), + }, + }, + })) + require.False(t, filters.MatchInclude(&nodeselection.SelectedNode{ + CountryCode: location.UnitedKingdom, + })) + require.False(t, filters.MatchInclude(&nodeselection.SelectedNode{ + CountryCode: location.Germany, + Tags: nodeselection.NodeTags{ + { + Signer: signer, + Name: "foo", + Value: []byte("bar"), + }, + }, + })) + } + t.Run("invalid", func(t *testing.T) { + p := NewPlacementRules() + err := p.AddPlacementFromString("10:1 && 2") + require.Error(t, err) + }) }) t.Run("multi rule", func(t *testing.T) { @@ -168,4 +222,74 @@ func TestPlacementFromString(t *testing.T) { }) + t.Run("full example", func(t *testing.T) { + // this is a realistic configuration, compatible with legacy rules + using one node tag for specific placement + + rules1 := NewPlacementRules() + err := rules1.AddPlacementFromString(` + 10:tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","selected","true"); + 0:exclude(placement(10)); + 1:country("EU") && exclude(placement(10)); + 2:country("EEA") && exclude(placement(10)); + 3:country("US") && exclude(placement(10)); + 4:country("DE") && exclude(placement(10)); + 6:country("*","!BY", "!RU", "!NONE") && exclude(placement(10))`) + require.NoError(t, err) + + // for countries, it should be the same as above + rules2 := NewPlacementRules() + rules2.AddLegacyStaticRules() + + testCountries := []location.CountryCode{ + location.Russia, + location.India, + location.Belarus, + location.UnitedStates, + location.Canada, + location.Brazil, + location.Ghana, + } + testCountries = append(testCountries, nodeselection.EeaCountriesWithoutEu...) + testCountries = append(testCountries, nodeselection.EuCountries...) + + // check if old geofencing rules are working as before (and string based config is the same as the code base) + for _, placement := range []storj.PlacementConstraint{storj.EU, storj.EEA, storj.DE, storj.US, storj.NR} { + filter1 := rules1.CreateFilters(placement) + filter2 := rules2.CreateFilters(placement) + for _, country := range testCountries { + old := placement.AllowedCountry(country) + result1 := filter1.MatchInclude(&nodeselection.SelectedNode{ + CountryCode: country, + }) + result2 := filter2.MatchInclude(&nodeselection.SelectedNode{ + CountryCode: country, + }) + assert.Equal(t, old, result1, "old placement doesn't match string based configuration for placement %d and country %s", placement, country) + assert.Equal(t, old, result2, "old placement doesn't match code based configuration for placement %d and country %s", placement, country) + } + } + + // make sure that new rules exclude location.None from NR + assert.False(t, rules1.CreateFilters(storj.NR).MatchInclude(&nodeselection.SelectedNode{})) + assert.False(t, rules2.CreateFilters(storj.NR).MatchInclude(&nodeselection.SelectedNode{})) + + // make sure tagged nodes (even from EU) matches only the special placement + node := &nodeselection.SelectedNode{ + CountryCode: location.Germany, + Tags: nodeselection.NodeTags{ + { + Signer: signer, + Name: "selected", + Value: []byte("true"), + }, + }, + } + + for _, placement := range []storj.PlacementConstraint{storj.EveryCountry, storj.EU, storj.EEA, storj.DE, storj.US, storj.NR} { + assert.False(t, rules1.CreateFilters(placement).MatchInclude(node)) + } + assert.False(t, rules1.CreateFilters(6).MatchInclude(node)) + + }) + }