JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/satellitedb: simplify select nodes query construction

Browse Source 
Change-Id: I07009b28762d4485929a2a999e8f4be8179bee51
This commit is contained in:
Egon Elbre 2021-10-20 13:54:52 +03:00
parent 3989107031
commit f2d8e97d97
2 changed files with 68 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

72
satellite/overlay/benchmark_test.go
View File

@ -206,6 +206,10 @@ func BenchmarkNodeSelection(b *testing.B) {
OnlineWindow: time.Hour,
DistinctIP: true,
MinimumDiskSpace: 0,
AsOfSystemTime: overlay.AsOfSystemTimeConfig{
Enabled: true,
DefaultInterval: -time.Microsecond,
},
}
var excludedIDs []storj.NodeID
@ -279,20 +283,22 @@ func BenchmarkNodeSelection(b *testing.B) {
}
criteria := &overlay.NodeCriteria{
FreeDisk: 0,
ExcludedIDs: nil,
ExcludedNetworks: nil,
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: false,
FreeDisk: 0,
ExcludedIDs: nil,
ExcludedNetworks: nil,
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: false,
AsOfSystemInterval: -time.Microsecond,
}
excludedCriteria := &overlay.NodeCriteria{
FreeDisk: 0,
ExcludedIDs: excludedIDs,
ExcludedNetworks: excludedNets,
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: false,
FreeDisk: 0,
ExcludedIDs: excludedIDs,
ExcludedNetworks: excludedNets,
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: false,
AsOfSystemInterval: -time.Microsecond,
}
b.Run("SelectStorageNodes", func(b *testing.B) {
@ -362,9 +368,10 @@ func BenchmarkNodeSelection(b *testing.B) {
b.Run("FindStorageNodesWithPreference", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.FindStorageNodesWithPreferences(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
}, &nodeSelectionConfig)
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -374,9 +381,10 @@ func BenchmarkNodeSelection(b *testing.B) {
b.Run("FindStorageNodesWithPreferenceExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.FindStorageNodesWithPreferences(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
}, &nodeSelectionConfig)
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -386,9 +394,10 @@ func BenchmarkNodeSelection(b *testing.B) {
b.Run("FindStorageNodes", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.FindStorageNodesForUpload(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
})
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -398,9 +407,10 @@ func BenchmarkNodeSelection(b *testing.B) {
b.Run("FindStorageNodesExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.FindStorageNodesForUpload(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
})
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -410,9 +420,10 @@ func BenchmarkNodeSelection(b *testing.B) {
b.Run("UploadSelectionCacheGetNodes", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.UploadSelectionCache.GetNodes(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
})
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -422,9 +433,10 @@ func BenchmarkNodeSelection(b *testing.B) {
b.Run("UploadSelectionCacheGetNodesExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.UploadSelectionCache.GetNodes(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
})
require.NoError(b, err)
require.NotEmpty(b, selected)

66
satellite/satellitedb/nodeselection.go
View File

@ -104,43 +104,38 @@ func (cache *overlaycache) selectStorageNodesOnce(ctx context.Context, reputable
var reputableNodeQuery, newNodeQuery partialQuery
asOf := cache.db.impl.AsOfSystemInterval(criteria.AsOfSystemInterval)
// Note: the true/false at the end of each selection string indicates if the selection is for new nodes or not.
// Later, the flag allows us to distinguish if a node is new when scanning the db rows.
if !criteria.DistinctIP {
reputableNodeQuery = partialQuery{
selection: `SELECT last_net, id, address, last_ip_port, false FROM nodes ` + asOf,
condition: reputableNodesCondition,
limit: reputableNodeCount,
aostClause: asOf,
selection: `SELECT last_net, id, address, last_ip_port, false FROM nodes`,
condition: reputableNodesCondition,
limit: reputableNodeCount,
}
newNodeQuery = partialQuery{
selection: `SELECT last_net, id, address, last_ip_port, true FROM nodes ` + asOf,
condition: newNodesCondition,
limit: newNodeCount,
aostClause: asOf,
selection: `SELECT last_net, id, address, last_ip_port, true FROM nodes`,
condition: newNodesCondition,
limit: newNodeCount,
}
} else {
reputableNodeQuery = partialQuery{
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, false FROM nodes ` + asOf,
condition: reputableNodesCondition,
distinct: true,
limit: reputableNodeCount,
orderBy: "last_net",
aostClause: asOf,
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, false FROM nodes`,
condition: reputableNodesCondition,
distinct: true,
limit: reputableNodeCount,
orderBy: "last_net",
}
newNodeQuery = partialQuery{
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, true FROM nodes ` + asOf,
condition: newNodesCondition,
distinct: true,
limit: newNodeCount,
orderBy: "last_net",
aostClause: asOf,
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, true FROM nodes`,
condition: newNodesCondition,
distinct: true,
limit: newNodeCount,
orderBy: "last_net",
}
}
query := unionAll(asOf, newNodeQuery, reputableNodeQuery)
query := unionAll(newNodeQuery, reputableNodeQuery)
query.query = cache.db.impl.WrapAsOfSystemInterval(query.query, criteria.AsOfSystemInterval)
rows, err := cache.db.Query(ctx, cache.db.Rebind(query.query), query.args...)
if err != nil {
@ -238,12 +233,11 @@ func nodeSelectionCondition(ctx context.Context, criteria *overlay.NodeCriteria,
// SELECT * FROM ($selection WHERE $condition ORDER BY $orderBy, RANDOM()) filtered ORDER BY RANDOM() LIMIT $limit
//
type partialQuery struct {
selection string
condition condition
distinct bool
orderBy string
limit int
aostClause string
selection string
condition condition
distinct bool
orderBy string
limit int
}
// isEmpty returns whether the result for the query is definitely empty.
@ -274,7 +268,7 @@ func (partial partialQuery) asQuery() query {
fmt.Fprint(&q, " LIMIT ? ")
args = append(args, partial.limit)
} else {
fmt.Fprint(&q, ") filtered "+partial.aostClause+" ORDER BY RANDOM() LIMIT ?")
fmt.Fprint(&q, ") filtered ORDER BY RANDOM() LIMIT ?")
args = append(args, partial.limit)
}
@ -282,7 +276,7 @@ func (partial partialQuery) asQuery() query {
}
// unionAll combines multiple partial queries into a single query.
func unionAll(asOf string, partials ...partialQuery) query {
func unionAll(partials ...partialQuery) query {
var queries []string
var args []interface{}
for _, partial := range partials {
@ -302,16 +296,8 @@ func unionAll(asOf string, partials ...partialQuery) query {
return query{query: queries[0], args: args}
}
union := "(" + strings.Join(queries, ") UNION ALL (") + ")"
if asOf == "" {
return query{
query: union,
args: args,
}
}
return query{
query: "SELECT * FROM (" + union + ") " + asOf,
query: "(" + strings.Join(queries, ") UNION ALL (") + ")",
args: args,
}
}