satellite/satellitedb: simplify select nodes query construction
Change-Id: I07009b28762d4485929a2a999e8f4be8179bee51
This commit is contained in:
parent
3989107031
commit
f2d8e97d97
@ -206,6 +206,10 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
OnlineWindow: time.Hour,
|
OnlineWindow: time.Hour,
|
||||||
DistinctIP: true,
|
DistinctIP: true,
|
||||||
MinimumDiskSpace: 0,
|
MinimumDiskSpace: 0,
|
||||||
|
AsOfSystemTime: overlay.AsOfSystemTimeConfig{
|
||||||
|
Enabled: true,
|
||||||
|
DefaultInterval: -time.Microsecond,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
var excludedIDs []storj.NodeID
|
var excludedIDs []storj.NodeID
|
||||||
@ -285,6 +289,7 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
MinimumVersion: "v1.0.0",
|
MinimumVersion: "v1.0.0",
|
||||||
OnlineWindow: time.Hour,
|
OnlineWindow: time.Hour,
|
||||||
DistinctIP: false,
|
DistinctIP: false,
|
||||||
|
AsOfSystemInterval: -time.Microsecond,
|
||||||
}
|
}
|
||||||
excludedCriteria := &overlay.NodeCriteria{
|
excludedCriteria := &overlay.NodeCriteria{
|
||||||
FreeDisk: 0,
|
FreeDisk: 0,
|
||||||
@ -293,6 +298,7 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
MinimumVersion: "v1.0.0",
|
MinimumVersion: "v1.0.0",
|
||||||
OnlineWindow: time.Hour,
|
OnlineWindow: time.Hour,
|
||||||
DistinctIP: false,
|
DistinctIP: false,
|
||||||
|
AsOfSystemInterval: -time.Microsecond,
|
||||||
}
|
}
|
||||||
|
|
||||||
b.Run("SelectStorageNodes", func(b *testing.B) {
|
b.Run("SelectStorageNodes", func(b *testing.B) {
|
||||||
@ -365,6 +371,7 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
RequestedCount: SelectCount,
|
RequestedCount: SelectCount,
|
||||||
ExcludedIDs: nil,
|
ExcludedIDs: nil,
|
||||||
MinimumVersion: "v1.0.0",
|
MinimumVersion: "v1.0.0",
|
||||||
|
AsOfSystemInterval: -time.Microsecond,
|
||||||
}, &nodeSelectionConfig)
|
}, &nodeSelectionConfig)
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
require.NotEmpty(b, selected)
|
require.NotEmpty(b, selected)
|
||||||
@ -377,6 +384,7 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
RequestedCount: SelectCount,
|
RequestedCount: SelectCount,
|
||||||
ExcludedIDs: excludedIDs,
|
ExcludedIDs: excludedIDs,
|
||||||
MinimumVersion: "v1.0.0",
|
MinimumVersion: "v1.0.0",
|
||||||
|
AsOfSystemInterval: -time.Microsecond,
|
||||||
}, &nodeSelectionConfig)
|
}, &nodeSelectionConfig)
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
require.NotEmpty(b, selected)
|
require.NotEmpty(b, selected)
|
||||||
@ -389,6 +397,7 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
RequestedCount: SelectCount,
|
RequestedCount: SelectCount,
|
||||||
ExcludedIDs: nil,
|
ExcludedIDs: nil,
|
||||||
MinimumVersion: "v1.0.0",
|
MinimumVersion: "v1.0.0",
|
||||||
|
AsOfSystemInterval: -time.Microsecond,
|
||||||
})
|
})
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
require.NotEmpty(b, selected)
|
require.NotEmpty(b, selected)
|
||||||
@ -401,6 +410,7 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
RequestedCount: SelectCount,
|
RequestedCount: SelectCount,
|
||||||
ExcludedIDs: excludedIDs,
|
ExcludedIDs: excludedIDs,
|
||||||
MinimumVersion: "v1.0.0",
|
MinimumVersion: "v1.0.0",
|
||||||
|
AsOfSystemInterval: -time.Microsecond,
|
||||||
})
|
})
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
require.NotEmpty(b, selected)
|
require.NotEmpty(b, selected)
|
||||||
@ -413,6 +423,7 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
RequestedCount: SelectCount,
|
RequestedCount: SelectCount,
|
||||||
ExcludedIDs: nil,
|
ExcludedIDs: nil,
|
||||||
MinimumVersion: "v1.0.0",
|
MinimumVersion: "v1.0.0",
|
||||||
|
AsOfSystemInterval: -time.Microsecond,
|
||||||
})
|
})
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
require.NotEmpty(b, selected)
|
require.NotEmpty(b, selected)
|
||||||
@ -425,6 +436,7 @@ func BenchmarkNodeSelection(b *testing.B) {
|
|||||||
RequestedCount: SelectCount,
|
RequestedCount: SelectCount,
|
||||||
ExcludedIDs: excludedIDs,
|
ExcludedIDs: excludedIDs,
|
||||||
MinimumVersion: "v1.0.0",
|
MinimumVersion: "v1.0.0",
|
||||||
|
AsOfSystemInterval: -time.Microsecond,
|
||||||
})
|
})
|
||||||
require.NoError(b, err)
|
require.NoError(b, err)
|
||||||
require.NotEmpty(b, selected)
|
require.NotEmpty(b, selected)
|
||||||
|
@ -104,43 +104,38 @@ func (cache *overlaycache) selectStorageNodesOnce(ctx context.Context, reputable
|
|||||||
|
|
||||||
var reputableNodeQuery, newNodeQuery partialQuery
|
var reputableNodeQuery, newNodeQuery partialQuery
|
||||||
|
|
||||||
asOf := cache.db.impl.AsOfSystemInterval(criteria.AsOfSystemInterval)
|
|
||||||
|
|
||||||
// Note: the true/false at the end of each selection string indicates if the selection is for new nodes or not.
|
// Note: the true/false at the end of each selection string indicates if the selection is for new nodes or not.
|
||||||
// Later, the flag allows us to distinguish if a node is new when scanning the db rows.
|
// Later, the flag allows us to distinguish if a node is new when scanning the db rows.
|
||||||
if !criteria.DistinctIP {
|
if !criteria.DistinctIP {
|
||||||
reputableNodeQuery = partialQuery{
|
reputableNodeQuery = partialQuery{
|
||||||
selection: `SELECT last_net, id, address, last_ip_port, false FROM nodes ` + asOf,
|
selection: `SELECT last_net, id, address, last_ip_port, false FROM nodes`,
|
||||||
condition: reputableNodesCondition,
|
condition: reputableNodesCondition,
|
||||||
limit: reputableNodeCount,
|
limit: reputableNodeCount,
|
||||||
aostClause: asOf,
|
|
||||||
}
|
}
|
||||||
newNodeQuery = partialQuery{
|
newNodeQuery = partialQuery{
|
||||||
selection: `SELECT last_net, id, address, last_ip_port, true FROM nodes ` + asOf,
|
selection: `SELECT last_net, id, address, last_ip_port, true FROM nodes`,
|
||||||
condition: newNodesCondition,
|
condition: newNodesCondition,
|
||||||
limit: newNodeCount,
|
limit: newNodeCount,
|
||||||
aostClause: asOf,
|
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
reputableNodeQuery = partialQuery{
|
reputableNodeQuery = partialQuery{
|
||||||
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, false FROM nodes ` + asOf,
|
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, false FROM nodes`,
|
||||||
condition: reputableNodesCondition,
|
condition: reputableNodesCondition,
|
||||||
distinct: true,
|
distinct: true,
|
||||||
limit: reputableNodeCount,
|
limit: reputableNodeCount,
|
||||||
orderBy: "last_net",
|
orderBy: "last_net",
|
||||||
aostClause: asOf,
|
|
||||||
}
|
}
|
||||||
newNodeQuery = partialQuery{
|
newNodeQuery = partialQuery{
|
||||||
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, true FROM nodes ` + asOf,
|
selection: `SELECT DISTINCT ON (last_net) last_net, id, address, last_ip_port, true FROM nodes`,
|
||||||
condition: newNodesCondition,
|
condition: newNodesCondition,
|
||||||
distinct: true,
|
distinct: true,
|
||||||
limit: newNodeCount,
|
limit: newNodeCount,
|
||||||
orderBy: "last_net",
|
orderBy: "last_net",
|
||||||
aostClause: asOf,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
query := unionAll(asOf, newNodeQuery, reputableNodeQuery)
|
query := unionAll(newNodeQuery, reputableNodeQuery)
|
||||||
|
query.query = cache.db.impl.WrapAsOfSystemInterval(query.query, criteria.AsOfSystemInterval)
|
||||||
|
|
||||||
rows, err := cache.db.Query(ctx, cache.db.Rebind(query.query), query.args...)
|
rows, err := cache.db.Query(ctx, cache.db.Rebind(query.query), query.args...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -243,7 +238,6 @@ type partialQuery struct {
|
|||||||
distinct bool
|
distinct bool
|
||||||
orderBy string
|
orderBy string
|
||||||
limit int
|
limit int
|
||||||
aostClause string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// isEmpty returns whether the result for the query is definitely empty.
|
// isEmpty returns whether the result for the query is definitely empty.
|
||||||
@ -274,7 +268,7 @@ func (partial partialQuery) asQuery() query {
|
|||||||
fmt.Fprint(&q, " LIMIT ? ")
|
fmt.Fprint(&q, " LIMIT ? ")
|
||||||
args = append(args, partial.limit)
|
args = append(args, partial.limit)
|
||||||
} else {
|
} else {
|
||||||
fmt.Fprint(&q, ") filtered "+partial.aostClause+" ORDER BY RANDOM() LIMIT ?")
|
fmt.Fprint(&q, ") filtered ORDER BY RANDOM() LIMIT ?")
|
||||||
args = append(args, partial.limit)
|
args = append(args, partial.limit)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -282,7 +276,7 @@ func (partial partialQuery) asQuery() query {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// unionAll combines multiple partial queries into a single query.
|
// unionAll combines multiple partial queries into a single query.
|
||||||
func unionAll(asOf string, partials ...partialQuery) query {
|
func unionAll(partials ...partialQuery) query {
|
||||||
var queries []string
|
var queries []string
|
||||||
var args []interface{}
|
var args []interface{}
|
||||||
for _, partial := range partials {
|
for _, partial := range partials {
|
||||||
@ -302,16 +296,8 @@ func unionAll(asOf string, partials ...partialQuery) query {
|
|||||||
return query{query: queries[0], args: args}
|
return query{query: queries[0], args: args}
|
||||||
}
|
}
|
||||||
|
|
||||||
union := "(" + strings.Join(queries, ") UNION ALL (") + ")"
|
|
||||||
if asOf == "" {
|
|
||||||
return query{
|
return query{
|
||||||
query: union,
|
query: "(" + strings.Join(queries, ") UNION ALL (") + ")",
|
||||||
args: args,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return query{
|
|
||||||
query: "SELECT * FROM (" + union + ") " + asOf,
|
|
||||||
args: args,
|
args: args,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user