storj/satellite/overlay/benchmark_test.go
Egon Elbre 8f15f975a2 satellite/overlay: improve contended update checkin
Improve UpdateCheckIn on a contended row:

  name                             old time/op  new time/op delta
  UpdateCheckInContended-100x-32   2.29s ±55%   0.17s ±61%  -92.45%  (p=0.008 n=5+5)

Change-Id: I053ab9f1cff136c306e5fb57f5e355cdc0269a8c
2021-05-16 20:41:12 +03:00

550 lines
15 KiB
Go

// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package overlay_test
import (
"context"
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/common/errs2"
"storj.io/common/pb"
"storj.io/common/storj"
"storj.io/common/testrand"
"storj.io/storj/satellite"
"storj.io/storj/satellite/overlay"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
)
func BenchmarkOverlay(b *testing.B) {
satellitedbtest.Bench(b, func(b *testing.B, db satellite.DB) {
const (
TotalNodeCount = 211
OnlineCount = 90
OfflineCount = 10
)
overlaydb := db.OverlayCache()
ctx := context.Background()
var all []storj.NodeID
var check []storj.NodeID
for i := 0; i < TotalNodeCount; i++ {
id := testrand.NodeID()
all = append(all, id)
if i < OnlineCount {
check = append(check, id)
}
}
for i, id := range all {
addr := fmt.Sprintf("127.0.%d.0:8080", i)
lastNet := fmt.Sprintf("127.0.%d", i)
d := overlay.NodeCheckInInfo{
NodeID: id,
Address: &pb.NodeAddress{Address: addr},
LastIPPort: addr,
LastNet: lastNet,
Version: &pb.NodeVersion{Version: "v1.0.0"},
IsUp: true,
}
err := overlaydb.UpdateCheckIn(ctx, d, time.Now().UTC(), overlay.NodeSelectionConfig{})
require.NoError(b, err)
}
// create random offline node ids to check
for i := 0; i < OfflineCount; i++ {
check = append(check, testrand.NodeID())
}
b.Run("KnownUnreliableOrOffline", func(b *testing.B) {
criteria := &overlay.NodeCriteria{
OnlineWindow: 1000 * time.Hour,
}
for i := 0; i < b.N; i++ {
badNodes, err := overlaydb.KnownUnreliableOrOffline(ctx, criteria, check)
require.NoError(b, err)
require.Len(b, badNodes, OfflineCount)
}
})
b.Run("UpdateCheckIn", func(b *testing.B) {
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
addr := fmt.Sprintf("127.0.%d.0:8080", i)
lastNet := fmt.Sprintf("127.0.%d", i)
d := overlay.NodeCheckInInfo{
NodeID: id,
Address: &pb.NodeAddress{Address: addr},
LastIPPort: addr,
LastNet: lastNet,
Version: &pb.NodeVersion{Version: "v1.0.0"},
}
err := overlaydb.UpdateCheckIn(ctx, d, time.Now().UTC(), overlay.NodeSelectionConfig{})
require.NoError(b, err)
}
})
b.Run("UpdateCheckInContended-100x", func(b *testing.B) {
for k := 0; k < b.N; k++ {
var g errs2.Group
for i := 0; i < 100; i++ {
g.Go(func() error {
d := overlay.NodeCheckInInfo{
NodeID: all[0],
Address: &pb.NodeAddress{Address: "127.0.0.0:8080"},
LastIPPort: "127.0.0.0:8080",
LastNet: "127.0.0",
Operator: &pb.NodeOperator{
Email: "hello@example.com",
Wallet: "123123123123",
},
Version: &pb.NodeVersion{Version: "v1.0.0"},
IsUp: true,
}
return overlaydb.UpdateCheckIn(ctx, d, time.Now().UTC(), overlay.NodeSelectionConfig{})
})
}
require.NoError(b, errs.Combine(g.Wait()...))
}
})
b.Run("UpdateStatsSuccess", func(b *testing.B) {
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
_, err := overlaydb.UpdateStats(ctx, &overlay.UpdateRequest{
NodeID: id,
AuditOutcome: overlay.AuditSuccess,
AuditHistory: testAuditHistoryConfig(),
}, time.Now())
require.NoError(b, err)
}
})
b.Run("UpdateStatsFailure", func(b *testing.B) {
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
_, err := overlaydb.UpdateStats(ctx, &overlay.UpdateRequest{
NodeID: id,
AuditOutcome: overlay.AuditFailure,
AuditHistory: testAuditHistoryConfig(),
}, time.Now())
require.NoError(b, err)
}
})
b.Run("UpdateStatsUnknown", func(b *testing.B) {
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
_, err := overlaydb.UpdateStats(ctx, &overlay.UpdateRequest{
NodeID: id,
AuditOutcome: overlay.AuditUnknown,
AuditHistory: testAuditHistoryConfig(),
}, time.Now())
require.NoError(b, err)
}
})
b.Run("UpdateStatsOffline", func(b *testing.B) {
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
_, err := overlaydb.UpdateStats(ctx, &overlay.UpdateRequest{
NodeID: id,
AuditOutcome: overlay.AuditOffline,
AuditHistory: testAuditHistoryConfig(),
}, time.Now())
require.NoError(b, err)
}
})
b.Run("BatchUpdateStatsSuccess", func(b *testing.B) {
var updateRequests []*overlay.UpdateRequest
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
updateRequests = append(updateRequests, &overlay.UpdateRequest{
NodeID: id,
AuditOutcome: overlay.AuditSuccess,
AuditHistory: testAuditHistoryConfig(),
})
}
_, err := overlaydb.BatchUpdateStats(ctx, updateRequests, 100, time.Now())
require.NoError(b, err)
})
b.Run("BatchUpdateStatsFailure", func(b *testing.B) {
var updateRequests []*overlay.UpdateRequest
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
updateRequests = append(updateRequests, &overlay.UpdateRequest{
NodeID: id,
AuditOutcome: overlay.AuditFailure,
AuditHistory: testAuditHistoryConfig(),
})
}
_, err := overlaydb.BatchUpdateStats(ctx, updateRequests, 100, time.Now())
require.NoError(b, err)
})
b.Run("BatchUpdateStatsUnknown", func(b *testing.B) {
var updateRequests []*overlay.UpdateRequest
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
updateRequests = append(updateRequests, &overlay.UpdateRequest{
NodeID: id,
AuditOutcome: overlay.AuditUnknown,
AuditHistory: testAuditHistoryConfig(),
})
}
_, err := overlaydb.BatchUpdateStats(ctx, updateRequests, 100, time.Now())
require.NoError(b, err)
})
b.Run("BatchUpdateStatsOffline", func(b *testing.B) {
var updateRequests []*overlay.UpdateRequest
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
updateRequests = append(updateRequests, &overlay.UpdateRequest{
NodeID: id,
AuditOutcome: overlay.AuditOffline,
AuditHistory: testAuditHistoryConfig(),
})
}
_, err := overlaydb.BatchUpdateStats(ctx, updateRequests, 100, time.Now())
require.NoError(b, err)
})
b.Run("UpdateNodeInfo", func(b *testing.B) {
now := time.Now()
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
_, err := overlaydb.UpdateNodeInfo(ctx, id, &overlay.InfoResponse{
Type: pb.NodeType_STORAGE,
Operator: &pb.NodeOperator{
Wallet: "0x0123456789012345678901234567890123456789",
Email: "a@mail.test",
},
Capacity: &pb.NodeCapacity{
FreeDisk: 1000,
},
Version: &pb.NodeVersion{
Version: "1.0.0",
CommitHash: "0",
Timestamp: now,
Release: false,
},
})
require.NoError(b, err)
}
})
b.Run("UpdateCheckIn", func(b *testing.B) {
now := time.Now()
for i := 0; i < b.N; i++ {
id := all[i%len(all)]
err := overlaydb.UpdateCheckIn(ctx, overlay.NodeCheckInInfo{
NodeID: id,
Address: &pb.NodeAddress{
Address: "1.2.4.4",
},
IsUp: true,
Capacity: &pb.NodeCapacity{
FreeDisk: int64(i),
},
Operator: &pb.NodeOperator{
Email: "a@mail.test",
Wallet: "0x0123456789012345678901234567890123456789",
},
Version: &pb.NodeVersion{
Version: "1.0.0",
CommitHash: "0",
Timestamp: now,
Release: false,
},
},
now,
overlay.NodeSelectionConfig{},
)
require.NoError(b, err)
}
})
})
}
func BenchmarkNodeSelection(b *testing.B) {
satellitedbtest.Bench(b, func(b *testing.B, db satellite.DB) {
var (
Total = 10000
Offline = 1000
NodesPerNet = 2
SelectCount = 100
ExcludedCount = 90
newNodeFraction = 0.05
)
if testing.Short() {
Total /= 10
Offline /= 10
SelectCount /= 10
ExcludedCount /= 10
}
SelectNewCount := int(100 * newNodeFraction)
now := time.Now()
twoHoursAgo := now.Add(-2 * time.Hour)
overlaydb := db.OverlayCache()
ctx := context.Background()
nodeSelectionConfig := overlay.NodeSelectionConfig{
AuditCount: 1,
NewNodeFraction: newNodeFraction,
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: true,
MinimumDiskSpace: 0,
}
var excludedIDs []storj.NodeID
var excludedNets []string
for i := 0; i < Total/NodesPerNet; i++ {
for k := 0; k < NodesPerNet; k++ {
nodeID := testrand.NodeID()
address := fmt.Sprintf("127.%d.%d.%d", byte(i>>8), byte(i), byte(k))
lastNet := fmt.Sprintf("127.%d.%d.0", byte(i>>8), byte(i))
if i < ExcludedCount && k == 0 {
excludedIDs = append(excludedIDs, nodeID)
excludedNets = append(excludedNets, lastNet)
}
addr := address + ":12121"
d := overlay.NodeCheckInInfo{
NodeID: nodeID,
Address: &pb.NodeAddress{Address: addr},
LastIPPort: addr,
LastNet: lastNet,
Version: &pb.NodeVersion{Version: "v1.0.0"},
Capacity: &pb.NodeCapacity{
FreeDisk: 1_000_000_000,
},
}
err := overlaydb.UpdateCheckIn(ctx, d, time.Now().UTC(), overlay.NodeSelectionConfig{})
require.NoError(b, err)
_, err = overlaydb.UpdateNodeInfo(ctx, nodeID, &overlay.InfoResponse{
Type: pb.NodeType_STORAGE,
Capacity: &pb.NodeCapacity{
FreeDisk: 1_000_000_000,
},
Version: &pb.NodeVersion{
Version: "v1.0.0",
Timestamp: now,
Release: true,
},
})
require.NoError(b, err)
if i%2 == 0 { // make half of nodes "new" and half "vetted"
_, err = overlaydb.UpdateStats(ctx, &overlay.UpdateRequest{
NodeID: nodeID,
AuditOutcome: overlay.AuditSuccess,
AuditLambda: 1,
AuditWeight: 1,
AuditDQ: 0.5,
AuditHistory: testAuditHistoryConfig(),
}, time.Now())
require.NoError(b, err)
}
if i > Total-Offline {
switch i % 3 {
case 0:
err := overlaydb.SuspendNodeUnknownAudit(ctx, nodeID, now)
require.NoError(b, err)
case 1:
err := overlaydb.DisqualifyNode(ctx, nodeID)
require.NoError(b, err)
case 2:
err := overlaydb.UpdateCheckIn(ctx, overlay.NodeCheckInInfo{
NodeID: nodeID,
Address: &pb.NodeAddress{
Address: address,
},
Operator: nil,
Version: nil,
}, twoHoursAgo, nodeSelectionConfig)
require.NoError(b, err)
}
}
}
}
criteria := &overlay.NodeCriteria{
FreeDisk: 0,
ExcludedIDs: nil,
ExcludedNetworks: nil,
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: false,
}
excludedCriteria := &overlay.NodeCriteria{
FreeDisk: 0,
ExcludedIDs: excludedIDs,
ExcludedNetworks: excludedNets,
MinimumVersion: "v1.0.0",
OnlineWindow: time.Hour,
DistinctIP: false,
}
b.Run("SelectStorageNodes", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := overlaydb.SelectStorageNodes(ctx, SelectCount, 0, criteria)
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("SelectNewStorageNodes", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := overlaydb.SelectStorageNodes(ctx, SelectCount, SelectCount, criteria)
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("SelectStorageNodesExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := overlaydb.SelectStorageNodes(ctx, SelectCount, 0, excludedCriteria)
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("SelectNewStorageNodesExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := overlaydb.SelectStorageNodes(ctx, SelectCount, SelectCount, excludedCriteria)
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("SelectStorageNodesBoth", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := overlaydb.SelectStorageNodes(ctx, SelectCount, SelectNewCount, criteria)
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("SelectStorageNodesBothExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := overlaydb.SelectStorageNodes(ctx, SelectCount, SelectNewCount, excludedCriteria)
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("GetNodesNetwork", func(b *testing.B) {
for i := 0; i < b.N; i++ {
excludedNetworks, err := overlaydb.GetNodesNetwork(ctx, excludedIDs)
require.NoError(b, err)
require.NotEmpty(b, excludedNetworks)
}
})
service, err := overlay.NewService(zap.NewNop(), overlaydb, overlay.Config{
Node: nodeSelectionConfig,
NodeSelectionCache: overlay.UploadSelectionCacheConfig{
Staleness: time.Hour,
},
})
require.NoError(b, err)
b.Run("FindStorageNodesWithPreference", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.FindStorageNodesWithPreferences(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
}, &nodeSelectionConfig)
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("FindStorageNodesWithPreferenceExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.FindStorageNodesWithPreferences(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
}, &nodeSelectionConfig)
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("FindStorageNodes", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.FindStorageNodesForUpload(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
})
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("FindStorageNodesExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.FindStorageNodesForUpload(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
})
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("UploadSelectionCacheGetNodes", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.UploadSelectionCache.GetNodes(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: nil,
MinimumVersion: "v1.0.0",
})
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
b.Run("UploadSelectionCacheGetNodesExclusion", func(b *testing.B) {
for i := 0; i < b.N; i++ {
selected, err := service.UploadSelectionCache.GetNodes(ctx, overlay.FindStorageNodesRequest{
RequestedCount: SelectCount,
ExcludedIDs: excludedIDs,
MinimumVersion: "v1.0.0",
})
require.NoError(b, err)
require.NotEmpty(b, selected)
}
})
})
}