JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/satellitedb: simplify select nodes query construction

Browse Source 
Change-Id: I07009b28762d4485929a2a999e8f4be8179bee51
This commit is contained in:
Egon Elbre 2021-10-20 13:54:52 +03:00
parent 3989107031
commit f2d8e97d97
2 changed files with 68 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
satellite/overlay/benchmark_test.go
View File

@ -206,6 +206,10 @@ func BenchmarkNodeSelection(b *testing.B) {
OnlineWindow: time.Hour,
DistinctIP: true,
MinimumDiskSpace: 0,
AsOfSystemTime: overlay.AsOfSystemTimeConfig{
Enabled: true,
DefaultInterval: -time.Microsecond,
},
}
var excludedIDs []storj.NodeID
@ -285,6 +289,7 @@ func BenchmarkNodeSelection(b *testing.B) {
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: false,
AsOfSystemInterval: -time.Microsecond,
}
excludedCriteria := &overlay.NodeCriteria{
FreeDisk: 0,
@ -293,6 +298,7 @@ func BenchmarkNodeSelection(b *testing.B) {
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: false,
AsOfSystemInterval: -time.Microsecond,
}
b.Run("SelectStorageNodes", func(b *testing.B) {
@ -365,6 +371,7 @@ func BenchmarkNodeSelection(b *testing.B) {
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
}, &nodeSelectionConfig)
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -377,6 +384,7 @@ func BenchmarkNodeSelection(b *testing.B) {
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
}, &nodeSelectionConfig)
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -389,6 +397,7 @@ func BenchmarkNodeSelection(b *testing.B) {
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
})
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -401,6 +410,7 @@ func BenchmarkNodeSelection(b *testing.B) {
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
})
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -413,6 +423,7 @@ func BenchmarkNodeSelection(b *testing.B) {
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
})
require.NoError(b, err)
require.NotEmpty(b, selected)
@ -425,6 +436,7 @@ func BenchmarkNodeSelection(b *testing.B) {
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
AsOfSystemInterval: -time.Microsecond,
})
require.NoError(b, err)
require.NotEmpty(b, selected)

32
satellite/satellitedb/nodeselection.go
View File

@ -104,43 +104,38 @@ func (cache *overlaycache) selectStorageNodesOnce(ctx context.Context, reputable
var reputableNodeQuery, newNodeQuery partialQuery
asOf := cache.db.impl.AsOfSystemInterval(criteria.AsOfSystemInterval)
// Note: the true/false at the end of each selection string indicates if the selection is for new nodes or not.
// Later, the flag allows us to distinguish if a node is new when scanning the db rows.
if !criteria.DistinctIP {
reputableNodeQuery = partialQuery{
selection: `SELECT last_net, id, address, last_ip_port, false FROM nodes ` + asOf,
selection: `SELECT last_net, id, address, last_ip_port, false FROM nodes`,
condition: reputableNodesCondition,
limit: reputableNodeCount,
aostClause: asOf,
}
newNodeQuery = partialQuery{
selection: `SELECT last_net, id, address, last_ip_port, true FROM nodes ` + asOf,
selection: `SELECT last_net, id, address, last_ip_port, true FROM nodes`,
condition: newNodesCondition,
limit: newNodeCount,
aostClause: asOf,
}
} else {
reputableNodeQuery = partialQuery{
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, false FROM nodes ` + asOf,
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, false FROM nodes`,
condition: reputableNodesCondition,
distinct: true,
limit: reputableNodeCount,
orderBy: "last_net",
aostClause: asOf,
}
newNodeQuery = partialQuery{
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, true FROM nodes ` + asOf,
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, true FROM nodes`,
condition: newNodesCondition,
distinct: true,
limit: newNodeCount,
orderBy: "last_net",
aostClause: asOf,
}
}
query := unionAll(asOf, newNodeQuery, reputableNodeQuery)
query := unionAll(newNodeQuery, reputableNodeQuery)
query.query = cache.db.impl.WrapAsOfSystemInterval(query.query, criteria.AsOfSystemInterval)
rows, err := cache.db.Query(ctx, cache.db.Rebind(query.query), query.args...)
if err != nil {
@ -243,7 +238,6 @@ type partialQuery struct {
distinct bool
orderBy string
limit int
aostClause string
}
// isEmpty returns whether the result for the query is definitely empty.
@ -274,7 +268,7 @@ func (partial partialQuery) asQuery() query {
fmt.Fprint(&q, " LIMIT ? ")
args = append(args, partial.limit)
} else {
fmt.Fprint(&q, ") filtered "+partial.aostClause+" ORDER BY RANDOM() LIMIT ?")
fmt.Fprint(&q, ") filtered ORDER BY RANDOM() LIMIT ?")
args = append(args, partial.limit)
}
@ -282,7 +276,7 @@ func (partial partialQuery) asQuery() query {
}
// unionAll combines multiple partial queries into a single query.
func unionAll(asOf string, partials ...partialQuery) query {
func unionAll(partials ...partialQuery) query {
var queries []string
var args []interface{}
for _, partial := range partials {
@ -302,16 +296,8 @@ func unionAll(asOf string, partials ...partialQuery) query {
return query{query: queries[0], args: args}
}
union := "(" + strings.Join(queries, ") UNION ALL (") + ")"
if asOf == "" {
return query{
query: union,
args: args,
}
}
return query{
query: "SELECT * FROM (" + union + ") " + asOf,
query: "(" + strings.Join(queries, ") UNION ALL (") + ")",
args: args,
}
}