JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

wired up online config (#1827)

Browse Source 
* wired up online config
This commit is contained in:
Bill Thorp 2019-04-26 08:15:06 -04:00 committed by GitHub
parent ba152790b1
commit a11dc76169
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 35 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/overlay/cache.go
View File

@ -6,6 +6,7 @@ package main
import (
"context"
"fmt"
"time"
"github.com/zeebo/errs"
"go.uber.org/zap"
@ -31,5 +32,5 @@ func (c cacheConfig) open(ctx context.Context) (cache *overlay.Cache, dbClose fu
}
}
return overlay.NewCache(zap.L(), database.OverlayCache(), overlay.NodeSelectionConfig{}), dbClose, nil
return overlay.NewCache(zap.L(), database.OverlayCache(), overlay.NodeSelectionConfig{OnlineWindow: time.Hour}), dbClose, nil
}

1
internal/testplanet/planet.go
View File

@ -450,6 +450,7 @@ func (planet *Planet) newSatellites(count int) ([]*satellite.Peer, error) {
AuditSuccessRatio: 0,
AuditCount: 0,
NewNodePercentage: 0,
OnlineWindow: time.Hour,
},
},
Discovery: discovery.Config{

32
pkg/overlay/cache.go
View File

@ -16,11 +16,6 @@ import (
"storj.io/storj/storage"
)
const (
// OnlineWindow is the maximum amount of time that can pass without seeing a node before that node is considered offline
OnlineWindow = 1 * time.Hour
)
// ErrEmptyNode is returned when the nodeID is empty
var ErrEmptyNode = errs.New("empty node ID")
@ -68,28 +63,23 @@ type DB interface {
type FindStorageNodesRequest struct {
MinimumRequiredNodes int
RequestedCount int
FreeBandwidth int64
FreeDisk int64
ExcludedNodes []storj.NodeID
MinimumVersion string // semver or empty
FreeBandwidth int64
FreeDisk int64
ExcludedNodes []storj.NodeID
MinimumVersion string // semver or empty
}
// NodeCriteria are the requirements for selecting nodes
type NodeCriteria struct {
FreeBandwidth int64
FreeDisk int64
FreeBandwidth int64
FreeDisk int64
AuditCount int64
AuditSuccessRatio float64
UptimeCount int64
UptimeSuccessRatio float64
Excluded []storj.NodeID
MinimumVersion string // semver or empty
Excluded []storj.NodeID
MinimumVersion string // semver or empty
OnlineWindow time.Duration
}
// UpdateRequest is used to update a node status.
@ -176,7 +166,7 @@ func (cache *Cache) IsNew(node *NodeDossier) bool {
// IsOnline checks if a node is 'online' based on the collected statistics.
func (cache *Cache) IsOnline(node *NodeDossier) bool {
return time.Now().Sub(node.Reputation.LastContactSuccess) < OnlineWindow &&
return time.Now().Sub(node.Reputation.LastContactSuccess) < cache.preferences.OnlineWindow &&
node.Reputation.LastContactSuccess.After(node.Reputation.LastContactFailure)
}
@ -219,6 +209,7 @@ func (cache *Cache) FindStorageNodesWithPreferences(ctx context.Context, req Fin
AuditSuccessRatio: preferences.AuditSuccessRatio,
Excluded: excluded,
MinimumVersion: preferences.MinimumVersion,
OnlineWindow: preferences.OnlineWindow,
})
if err != nil {
return nil, err
@ -241,6 +232,7 @@ func (cache *Cache) FindStorageNodesWithPreferences(ctx context.Context, req Fin
UptimeSuccessRatio: preferences.UptimeRatio,
Excluded: excluded,
MinimumVersion: preferences.MinimumVersion,
OnlineWindow: preferences.OnlineWindow,
})
if err != nil {
return nil, err

8
pkg/overlay/cache_test.go
View File

@ -7,6 +7,7 @@ import (
"context"
"math/rand"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -40,7 +41,7 @@ func testCache(ctx context.Context, t *testing.T, store overlay.DB) {
_, _ = rand.Read(valid2ID[:])
_, _ = rand.Read(missingID[:])
cache := overlay.NewCache(zaptest.NewLogger(t), store, overlay.NodeSelectionConfig{})
cache := overlay.NewCache(zaptest.NewLogger(t), store, overlay.NodeSelectionConfig{OnlineWindow: time.Hour})
{ // Put
err := cache.Put(ctx, valid1ID, pb.Node{Id: valid1ID})
@ -161,11 +162,12 @@ func TestRandomizedSelection(t *testing.T) {
var err error
if i%2 == 0 {
nodes, err = cache.SelectStorageNodes(ctx, numNodesToSelect, &overlay.NodeCriteria{})
nodes, err = cache.SelectStorageNodes(ctx, numNodesToSelect, &overlay.NodeCriteria{OnlineWindow: time.Hour})
require.NoError(t, err)
} else {
nodes, err = cache.SelectNewStorageNodes(ctx, numNodesToSelect, &overlay.NodeCriteria{
AuditCount: 1,
OnlineWindow: time.Hour,
AuditCount: 1,
})
require.NoError(t, err)
}

2
pkg/overlay/config.go
View File

@ -40,7 +40,7 @@ type NodeSelectionConfig struct {
AuditCount int64 `help:"the number of times a node has been audited to not be considered a New Node" releaseDefault:"500" devDefault:"0"`
NewNodePercentage float64 `help:"the percentage of new nodes allowed per request" default:"0.05"` // TODO: fix, this is not percentage, it's ratio
MinimumVersion string `help:"the minimum node software version for node selection queries" default:""`
OnlineWindow time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"1h"` //todo: wire up instead of constants
OnlineWindow time.Duration `help:"the amount of time without seeing a node before its considered offline" default:"1h"`
}
// ParseIDs converts the base58check encoded node ID strings from the config into node IDs

12
pkg/overlay/selection_test.go
View File

@ -5,6 +5,7 @@ package overlay_test
import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/zeebo/errs"
@ -54,6 +55,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 0,
NewNodePercentage: 0,
OnlineWindow: time.Hour,
},
RequestCount: 5,
ExpectedCount: 5,
@ -62,6 +64,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 0,
NewNodePercentage: 1,
OnlineWindow: time.Hour,
},
RequestCount: 5,
ExpectedCount: 5,
@ -70,6 +73,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 1,
NewNodePercentage: 1,
OnlineWindow: time.Hour,
},
RequestCount: 5,
ExpectedCount: 6,
@ -78,6 +82,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 5,
NewNodePercentage: 1,
OnlineWindow: time.Hour,
},
RequestCount: 2,
ExpectedCount: 4,
@ -86,6 +91,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 5,
NewNodePercentage: 0.5,
OnlineWindow: time.Hour,
},
RequestCount: 4,
ExpectedCount: 6,
@ -94,6 +100,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 8,
NewNodePercentage: 1,
OnlineWindow: time.Hour,
},
RequestCount: 1,
ExpectedCount: 2,
@ -102,6 +109,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 9,
NewNodePercentage: 1,
OnlineWindow: time.Hour,
},
RequestCount: 2,
ExpectedCount: 3,
@ -111,6 +119,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 50,
NewNodePercentage: 1,
OnlineWindow: time.Hour,
},
RequestCount: 2,
ExpectedCount: 2,
@ -120,6 +129,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 9,
NewNodePercentage: 0,
OnlineWindow: time.Hour,
},
RequestCount: 1,
ExpectedCount: 1,
@ -128,6 +138,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 0,
NewNodePercentage: 1,
OnlineWindow: time.Hour,
},
RequestCount: 1,
ExpectedCount: 1,
@ -136,6 +147,7 @@ func TestNodeSelection(t *testing.T) {
Preferences: overlay.NodeSelectionConfig{
AuditCount: 5,
NewNodePercentage: 0,
OnlineWindow: time.Hour,
},
ExcludeCount: 7,
RequestCount: 5,

4
satellite/satellitedb/overlaycache.go
View File

@ -47,7 +47,7 @@ func (cache *overlaycache) SelectStorageNodes(ctx context.Context, count int, cr
args := append(make([]interface{}, 0, 13),
nodeType, criteria.FreeBandwidth, criteria.FreeDisk,
criteria.AuditCount, criteria.AuditSuccessRatio, criteria.UptimeCount, criteria.UptimeSuccessRatio,
time.Now().Add(-overlay.OnlineWindow))
time.Now().Add(-criteria.OnlineWindow))
if criteria.MinimumVersion != "" {
v, err := version.NewSemVer(criteria.MinimumVersion)
@ -73,7 +73,7 @@ func (cache *overlaycache) SelectNewStorageNodes(ctx context.Context, count int,
AND last_contact_success > ?
AND last_contact_success > last_contact_failure`
args := append(make([]interface{}, 0, 10),
nodeType, criteria.FreeBandwidth, criteria.FreeDisk, criteria.AuditCount, criteria.AuditSuccessRatio, time.Now().Add(-overlay.OnlineWindow))
nodeType, criteria.FreeBandwidth, criteria.FreeDisk, criteria.AuditCount, criteria.AuditSuccessRatio, time.Now().Add(-criteria.OnlineWindow))
if criteria.MinimumVersion != "" {
v, err := version.NewSemVer(criteria.MinimumVersion)