2020-04-14 21:50:02 +01:00
|
|
|
// Copyright (C) 2019 Storj Labs, Incache.
|
|
|
|
// See LICENSE for copying information.
|
|
|
|
|
|
|
|
package overlay
|
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"go.uber.org/zap"
|
2020-05-18 16:10:17 +01:00
|
|
|
|
2022-06-28 12:53:39 +01:00
|
|
|
"storj.io/common/sync2"
|
2023-07-07 09:31:58 +01:00
|
|
|
"storj.io/storj/satellite/nodeselection"
|
2020-04-14 21:50:02 +01:00
|
|
|
)
|
|
|
|
|
2021-01-28 11:46:18 +00:00
|
|
|
// UploadSelectionDB implements the database for upload selection cache.
|
2020-04-14 21:50:02 +01:00
|
|
|
//
|
|
|
|
// architecture: Database
|
2021-01-28 11:46:18 +00:00
|
|
|
type UploadSelectionDB interface {
|
2020-04-14 21:50:02 +01:00
|
|
|
// SelectAllStorageNodesUpload returns all nodes that qualify to store data, organized as reputable nodes and new nodes
|
2023-07-07 09:31:58 +01:00
|
|
|
SelectAllStorageNodesUpload(ctx context.Context, selectionCfg NodeSelectionConfig) (reputable, new []*nodeselection.SelectedNode, err error)
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
|
|
|
|
2021-01-28 11:46:18 +00:00
|
|
|
// UploadSelectionCacheConfig is a configuration for upload selection cache.
|
|
|
|
type UploadSelectionCacheConfig struct {
|
2020-05-19 18:25:53 +01:00
|
|
|
Disabled bool `help:"disable node cache" default:"false"`
|
testplanet/satellite: reduce the number of places default values need to be configured
Satellites set their configuration values to default values using
cfgstruct, however, it turns out our tests don't test these values
at all! Instead, they have a completely separate definition system
that is easy to forget about.
As is to be expected, these values have drifted, and it appears
in a few cases test planet is testing unreasonable values that we
won't see in production, or perhaps worse, features enabled in
production were missed and weren't enabled in testplanet.
This change makes it so all values are configured the same,
systematic way, so it's easy to see when test values are different
than dev values or release values, and it's less hard to forget
to enable features in testplanet.
In terms of reviewing, this change should be actually fairly
easy to review, considering private/testplanet/satellite.go keeps
the current config system and the new one and confirms that they
result in identical configurations, so you can be certain that
nothing was missed and the config is all correct.
You can also check the config lock to see what actual config
values changed.
Change-Id: I6715d0794887f577e21742afcf56fd2b9d12170e
2021-05-31 22:15:00 +01:00
|
|
|
Staleness time.Duration `help:"how stale the node selection cache can be" releaseDefault:"3m" devDefault:"5m" testDefault:"3m"`
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
|
|
|
|
2021-01-28 11:46:18 +00:00
|
|
|
// UploadSelectionCache keeps a list of all the storage nodes that are qualified to store data
|
2020-04-14 21:50:02 +01:00
|
|
|
// We organize the nodes by if they are reputable or a new node on the network.
|
|
|
|
// The cache will sync with the nodes table in the database and get refreshed once the staleness time has past.
|
2021-01-28 11:46:18 +00:00
|
|
|
type UploadSelectionCache struct {
|
2020-04-14 21:50:02 +01:00
|
|
|
log *zap.Logger
|
2021-01-28 11:46:18 +00:00
|
|
|
db UploadSelectionDB
|
2020-04-14 21:50:02 +01:00
|
|
|
selectionConfig NodeSelectionConfig
|
|
|
|
|
2023-07-07 09:31:58 +01:00
|
|
|
cache sync2.ReadCacheOf[*nodeselection.State]
|
2023-06-30 11:13:18 +01:00
|
|
|
|
2023-07-07 09:31:58 +01:00
|
|
|
defaultFilters nodeselection.NodeFilters
|
2023-06-30 11:13:18 +01:00
|
|
|
placementRules PlacementRules
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
|
|
|
|
2021-01-28 11:46:18 +00:00
|
|
|
// NewUploadSelectionCache creates a new cache that keeps a list of all the storage nodes that are qualified to store data.
|
2023-07-07 09:31:58 +01:00
|
|
|
func NewUploadSelectionCache(log *zap.Logger, db UploadSelectionDB, staleness time.Duration, config NodeSelectionConfig, defaultFilter nodeselection.NodeFilters, placementRules PlacementRules) (*UploadSelectionCache, error) {
|
2022-06-28 12:53:39 +01:00
|
|
|
cache := &UploadSelectionCache{
|
2020-04-14 21:50:02 +01:00
|
|
|
log: log,
|
|
|
|
db: db,
|
|
|
|
selectionConfig: config,
|
2023-06-30 11:13:18 +01:00
|
|
|
defaultFilters: defaultFilter,
|
|
|
|
placementRules: placementRules,
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
2022-06-28 12:53:39 +01:00
|
|
|
return cache, cache.cache.Init(staleness/2, staleness, cache.read)
|
|
|
|
}
|
|
|
|
|
|
|
|
// Run runs the background task for cache.
|
|
|
|
func (cache *UploadSelectionCache) Run(ctx context.Context) (err error) {
|
|
|
|
return cache.cache.Run(ctx)
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// Refresh populates the cache with all of the reputableNodes and newNode nodes
|
2020-07-16 15:18:02 +01:00
|
|
|
// This method is useful for tests.
|
2021-01-28 11:46:18 +00:00
|
|
|
func (cache *UploadSelectionCache) Refresh(ctx context.Context) (err error) {
|
2020-04-14 21:50:02 +01:00
|
|
|
defer mon.Task()(&ctx)(&err)
|
2022-06-28 12:53:39 +01:00
|
|
|
_, err = cache.cache.RefreshAndGet(ctx, time.Now())
|
2020-04-14 21:50:02 +01:00
|
|
|
return err
|
|
|
|
}
|
|
|
|
|
|
|
|
// refresh calls out to the database and refreshes the cache with the most up-to-date
|
|
|
|
// data from the nodes table, then sets time that the last refresh occurred so we know when
|
2020-07-16 15:18:02 +01:00
|
|
|
// to refresh again in the future.
|
2023-07-07 09:31:58 +01:00
|
|
|
func (cache *UploadSelectionCache) read(ctx context.Context) (_ *nodeselection.State, err error) {
|
2020-04-14 21:50:02 +01:00
|
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
|
|
|
|
reputableNodes, newNodes, err := cache.db.SelectAllStorageNodesUpload(ctx, cache.selectionConfig)
|
|
|
|
if err != nil {
|
2022-06-28 12:53:39 +01:00
|
|
|
return nil, Error.Wrap(err)
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
|
|
|
|
2023-07-07 09:31:58 +01:00
|
|
|
state := nodeselection.NewState(reputableNodes, newNodes)
|
2020-05-18 16:10:17 +01:00
|
|
|
|
2020-04-14 21:50:02 +01:00
|
|
|
mon.IntVal("refresh_cache_size_reputable").Observe(int64(len(reputableNodes)))
|
|
|
|
mon.IntVal("refresh_cache_size_new").Observe(int64(len(newNodes)))
|
2022-06-28 12:53:39 +01:00
|
|
|
|
|
|
|
return state, nil
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// GetNodes selects nodes from the cache that will be used to upload a file.
|
|
|
|
// Every node selected will be from a distinct network.
|
|
|
|
// If the cache hasn't been refreshed recently it will do so first.
|
2023-07-07 09:31:58 +01:00
|
|
|
func (cache *UploadSelectionCache) GetNodes(ctx context.Context, req FindStorageNodesRequest) (_ []*nodeselection.SelectedNode, err error) {
|
2020-04-14 21:50:02 +01:00
|
|
|
defer mon.Task()(&ctx)(&err)
|
|
|
|
|
2023-04-20 14:08:26 +01:00
|
|
|
state, err := cache.cache.Get(ctx, time.Now())
|
2022-06-28 12:53:39 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, Error.Wrap(err)
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
|
|
|
|
2023-08-01 12:50:22 +01:00
|
|
|
placementRules := cache.placementRules(req.Placement)
|
2023-08-17 10:53:05 +01:00
|
|
|
useSubnetExclusion := nodeselection.GetAnnotation(placementRules, nodeselection.AutoExcludeSubnet) != nodeselection.AutoExcludeSubnetOFF
|
2023-08-01 12:50:22 +01:00
|
|
|
|
|
|
|
filters := nodeselection.NodeFilters{placementRules}
|
2023-06-30 11:13:18 +01:00
|
|
|
if len(req.ExcludedIDs) > 0 {
|
2023-08-01 12:50:22 +01:00
|
|
|
if useSubnetExclusion {
|
|
|
|
filters = append(filters, state.ExcludeNetworksBasedOnNodes(req.ExcludedIDs))
|
|
|
|
} else {
|
|
|
|
filters = append(filters, nodeselection.ExcludedIDs(req.ExcludedIDs))
|
|
|
|
}
|
2023-06-30 11:13:18 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
filters = append(filters, cache.defaultFilters)
|
|
|
|
|
2023-08-01 12:50:22 +01:00
|
|
|
selectionReq := nodeselection.Request{
|
2023-06-30 11:13:18 +01:00
|
|
|
Count: req.RequestedCount,
|
|
|
|
NewFraction: cache.selectionConfig.NewNodeFraction,
|
|
|
|
NodeFilters: filters,
|
2023-08-01 12:50:22 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
if !useSubnetExclusion {
|
|
|
|
selectionReq.SelectionType = nodeselection.SelectionTypeByID
|
|
|
|
}
|
|
|
|
|
|
|
|
selected, err := state.Select(ctx, selectionReq)
|
2023-07-07 09:31:58 +01:00
|
|
|
if nodeselection.ErrNotEnoughNodes.Has(err) {
|
2020-05-18 16:10:17 +01:00
|
|
|
err = ErrNotEnoughNodes.Wrap(err)
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|
2023-06-30 11:35:07 +01:00
|
|
|
return selected, err
|
2020-04-14 21:50:02 +01:00
|
|
|
}
|