JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/{placement,nodeselection}: introduce empty() and notEmpty() for tag value selection

Browse Source 
It helps to implement rules like `tag("nodeid","select",notEmpty())

Change-Id: If7a4532eacc0e4e670ffe81d504aab9d5b34302f
This commit is contained in:
Márton Elek 2023-09-14 18:42:59 +02:00 committed by Storj Robot
parent 92a69c7de4
commit f4fe983b1e
3 changed files with 134 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
satellite/nodeselection/filter.go
View File

@ -4,8 +4,6 @@
package nodeselection
import (
"bytes"
"storj.io/common/storj"
"storj.io/common/storj/location"
)
@ -205,26 +203,31 @@ func (e ExcludedIDs) Match(node *SelectedNode) bool {
var _ NodeFilter = ExcludedIDs{}
// ValueMatch defines how to compare tag value with the defined one.
type ValueMatch func(a []byte, b []byte) bool
// TagFilter matches nodes with specific tags.
type TagFilter struct {
signer storj.NodeID
name string
value []byte
match ValueMatch
}
// NewTagFilter creates a new tag filter.
func NewTagFilter(id storj.NodeID, name string, value []byte) TagFilter {
func NewTagFilter(id storj.NodeID, name string, value []byte, match ValueMatch) TagFilter {
return TagFilter{
signer: id,
name: name,
value: value,
match: match,
}
}
// Match implements NodeFilter interface.
func (t TagFilter) Match(node *SelectedNode) bool {
for _, tag := range node.Tags {
if tag.Name == t.name && bytes.Equal(tag.Value, t.value) && tag.Signer == t.signer {
if tag.Name == t.name && t.match(tag.Value, t.value) && tag.Signer == t.signer {
return true
}
}

18
satellite/overlay/placement.go
View File

@ -4,6 +4,7 @@
package overlay
import (
"bytes"
"fmt"
"strconv"
"strings"
@ -78,6 +79,8 @@ func (d *ConfigurablePlacementRule) AddPlacementRule(id storj.PlacementConstrain
d.placements[id] = filter
}
type stringNotMatch string
// AddPlacementFromString parses placement definition form string representations from id:definition;id:definition;...
func (d *ConfigurablePlacementRule) AddPlacementFromString(definitions string) error {
env := map[any]any{
@ -137,17 +140,24 @@ func (d *ConfigurablePlacementRule) AddPlacementFromString(definitions string) e
if err != nil {
return nil, err
}
var rawValue []byte
match := bytes.Equal
switch v := value.(type) {
case string:
rawValue = []byte(v)
case []byte:
rawValue = v
case stringNotMatch:
match = func(a, b []byte) bool {
return !bytes.Equal(a, b)
}
rawValue = []byte(v)
default:
return nil, errs.New("3rd argument of tag() should be string or []byte")
}
res := nodeselection.NodeFilters{
nodeselection.NewTagFilter(nodeID, key, rawValue),
nodeselection.NewTagFilter(nodeID, key, rawValue, match),
}
return res, nil
},
@ -166,6 +176,12 @@ func (d *ConfigurablePlacementRule) AddPlacementFromString(definitions string) e
"exclude": func(filter nodeselection.NodeFilter) (nodeselection.NodeFilter, error) {
return nodeselection.NewExcludeFilter(filter), nil
},
"empty": func() string {
return ""
},
"notEmpty": func() any {
return stringNotMatch("")
},
}
for _, definition := range strings.Split(definitions, ";") {
definition = strings.TrimSpace(definition)

132
satellite/overlay/placement_test.go
View File

@ -52,23 +52,89 @@ func TestPlacementFromString(t *testing.T) {
})
t.Run("tag rule", func(t *testing.T) {
p := NewPlacementRules()
err := p.AddPlacementFromString(`11:tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","foo","bar")`)
require.NoError(t, err)
filters := p.placements[storj.PlacementConstraint(11)]
require.NotNil(t, filters)
require.True(t, filters.Match(&nodeselection.SelectedNode{
Tags: nodeselection.NodeTags{
tagged := func(key string, value string) nodeselection.NodeTags {
return nodeselection.NodeTags{
{
Signer: signer,
Name: "foo",
Value: []byte("bar"),
Name: key,
Value: []byte(value),
},
}
}
testCases := []struct {
name string
placement string
includedNodes []*nodeselection.SelectedNode
excludedNodes []*nodeselection.SelectedNode
}{
{
name: "simple tag",
placement: `11:tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","foo","bar")`,
includedNodes: []*nodeselection.SelectedNode{
{
Tags: tagged("foo", "bar"),
},
},
excludedNodes: []*nodeselection.SelectedNode{
{
CountryCode: location.Germany,
},
},
},
}))
require.False(t, filters.Match(&nodeselection.SelectedNode{
CountryCode: location.Germany,
}))
{
name: "tag not empty",
placement: `11:tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","foo",notEmpty())`,
includedNodes: []*nodeselection.SelectedNode{
{
Tags: tagged("foo", "barx"),
},
{
Tags: tagged("foo", "bar"),
},
},
excludedNodes: []*nodeselection.SelectedNode{
{
Tags: tagged("foo", ""),
},
{
CountryCode: location.Germany,
},
},
},
{
name: "tag empty",
placement: `11:tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","foo",empty())`,
includedNodes: []*nodeselection.SelectedNode{
{
Tags: tagged("foo", ""),
},
},
excludedNodes: []*nodeselection.SelectedNode{
{
Tags: tagged("foo", "bar"),
},
{
CountryCode: location.Germany,
},
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
p := NewPlacementRules()
err := p.AddPlacementFromString(tc.placement)
require.NoError(t, err)
filters := p.placements[storj.PlacementConstraint(11)]
require.NotNil(t, filters)
for _, i := range tc.includedNodes {
require.True(t, filters.Match(i), "%v should be included", i)
}
for _, e := range tc.excludedNodes {
require.False(t, filters.Match(e), "%v should be excluded", e)
}
})
}
})
t.Run("placement reuse", func(t *testing.T) {
@ -272,15 +338,15 @@ func TestPlacementFromString(t *testing.T) {
rules1 := NewPlacementRules()
err := rules1.AddPlacementFromString(`
10:tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","selected","true");
11:annotated(placement(10),annotation("autoExcludeSubnet","off"));
12:placement(11) && country("US");
0:exclude(placement(10));
1:country("EU") && exclude(placement(10));
2:country("EEA") && exclude(placement(10));
3:country("US") && exclude(placement(10));
4:country("DE") && exclude(placement(10));
6:country("*","!BY", "!RU", "!NONE") && exclude(placement(10))`)
10:tag("12whfK1EDvHJtajBiAUeajQLYcWqxcQmdYQU5zX5cCf6bAxfgu4","selected",notEmpty());
11:placement(10) && annotation("autoExcludeSubnet","off") && annotation("location","do-not-use");
12:placement(10) && annotation("autoExcludeSubnet","off") && country("US") && annotation("location","us-select-1");
0:exclude(placement(10)) && annotation("location","global");
1:country("EU") && exclude(placement(10)) && annotation("location","eu-1");
2:country("EEA") && exclude(placement(10)) && annotation("location","eea-1");
3:country("US") && exclude(placement(10)) && annotation("location","us-1");
4:country("DE") && exclude(placement(10)) && annotation("location","de-1");
6:country("*","!BY", "!RU", "!NONE") && exclude(placement(10)) && annotation("location","custom-1");`)
require.NoError(t, err)
// for countries, it should be the same as above
@ -335,6 +401,28 @@ func TestPlacementFromString(t *testing.T) {
}
assert.False(t, rules1.CreateFilters(6).Match(node))
// any value is accepted
assert.True(t, rules1.CreateFilters(11).Match(&nodeselection.SelectedNode{
Tags: nodeselection.NodeTags{
{
Signer: signer,
Name: "selected",
Value: []byte("true,something"),
},
},
}))
// but not empty
assert.False(t, rules1.CreateFilters(11).Match(&nodeselection.SelectedNode{
Tags: nodeselection.NodeTags{
{
Signer: signer,
Name: "selected",
Value: []byte(""),
},
},
}))
// check if annotation present on 11,12, but not on other
for i := 0; i < 20; i++ {
subnetDisabled := nodeselection.GetAnnotation(rules1.CreateFilters(storj.PlacementConstraint(i)), nodeselection.AutoExcludeSubnet) == nodeselection.AutoExcludeSubnetOFF