satellite/durability: make benchmark even quicker

To make sure that Benchmark tests are good, we run them with -short flag, eg:

```
go test -short -run=BenchmarkDurabilityProcess
```

Durability benchmark already supports this, but we can make it slightly more faster with
using less sgements and pieces during the `-short` run.

Change-Id: I9547ca1e3cd0178eb395a7a388f2e7936a9862d7

satellite/durability/observer.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/jtolio/eventkit"
 	"github.com/zeebo/errs"
+	"golang.org/x/exp/slices"
 
 	"storj.io/common/storj"
 	"storj.io/storj/satellite/metabase"
@@ -73,6 +74,7 @@ type ReportConfig struct {
 //	in this case this reporter will return 38 for the class "country:DE" (assuming all the other segments are more lucky).
 type Report struct {
 	healthStat         map[string]*HealthStat
+	busFactor          HealthStat
 	classifiers        []NodeClassifier
 	aliasMap           *metabase.NodeAliasMap
 	nodes              map[storj.NodeID]*nodeselection.SelectedNode
@@ -80,6 +82,7 @@ type Report struct {
 	metabaseDB         *metabase.DB
 	reporter           func(n time.Time, name string, stat *HealthStat)
 	reportThreshold    int
+	busFactorThreshold int
 	asOfSystemInterval time.Duration
 
 	// map between classes (like "country:hu" and integer IDs)
@@ -88,19 +91,23 @@ type Report struct {
 
 	// contains the available classes for each node alias.
 	classified [][]classID
+
+	maxPieceCount int
 }
 
 // NewDurability creates the new instance.
-func NewDurability(db overlay.DB, metabaseDB *metabase.DB, classifiers []NodeClassifier, reportThreshold int, asOfSystemInterval time.Duration) *Report {
+func NewDurability(db overlay.DB, metabaseDB *metabase.DB, classifiers []NodeClassifier, maxPieceCount int, reportThreshold int, busFactorThreshold int, asOfSystemInterval time.Duration) *Report {
 	return &Report{
 		db:                 db,
 		metabaseDB:         metabaseDB,
 		classifiers:        classifiers,
 		reportThreshold:    reportThreshold,
+		busFactorThreshold: busFactorThreshold,
 		asOfSystemInterval: asOfSystemInterval,
 		nodes:              make(map[storj.NodeID]*nodeselection.SelectedNode),
 		healthStat:         make(map[string]*HealthStat),
 		reporter:           reportToEventkit,
+		maxPieceCount:      maxPieceCount,
 	}
 }
 
@@ -159,6 +166,7 @@ func (c *Report) Fork(ctx context.Context) (rangedloop.Partial, error) {
 		reportThreshold:        c.reportThreshold,
 		healthStat:             make([]HealthStat, len(c.classID)),
 		controlledByClassCache: make([]int32, len(c.classID)),
+		busFactorCache:         make([]int32, 0, c.maxPieceCount),
 		classified:             c.classified,
 	}
 	return d, nil
@@ -184,6 +192,7 @@ func (c *Report) Join(ctx context.Context, partial rangedloop.Partial) (err erro
 		}
 
 	}
+	c.busFactor.Update(fork.busFactor.Min(), fork.busFactor.Exemplar)
 	return nil
 }
 
@@ -209,12 +218,15 @@ type ObserverFork struct {
 	controlledByClassCache []int32
 
 	healthStat      []HealthStat
+	busFactor       HealthStat
 	classifiers     []NodeClassifier
 	aliasMap        *metabase.NodeAliasMap
 	nodes           map[storj.NodeID]*nodeselection.SelectedNode
 	classifierCache [][]string
+	busFactorCache  []int32
 
-	reportThreshold int
+	reportThreshold    int
+	busFactorThreshold int
 
 	classified [][]classID
 }
@@ -248,6 +260,8 @@ func (c *ObserverFork) Process(ctx context.Context, segments []rangedloop.Segmen
 			}
 		}
 
+		busFactorGroups := c.busFactorCache
+
 		streamLocation := fmt.Sprintf("%s/%d", s.StreamID, s.Position.Encode())
 		for classID, count := range controlledByClass {
 			if count == 0 {
@@ -259,12 +273,30 @@ func (c *ObserverFork) Process(ctx context.Context, segments []rangedloop.Segmen
 
 			diff := healthyPieceCount - int(count)
 
+			busFactorGroups = append(busFactorGroups, count)
+
 			if c.reportThreshold > 0 && diff > c.reportThreshold {
 				continue
 			}
-			c.healthStat[classID].Update(diff, streamLocation)
 
+			c.healthStat[classID].Update(diff, streamLocation)
 		}
+
+		slices.SortFunc[int32](busFactorGroups, func(a int32, b int32) bool {
+			return a > b
+		})
+		rollingSum := 0
+		busFactor := 0
+		for _, count := range busFactorGroups {
+			if rollingSum < c.busFactorThreshold {
+				busFactor++
+				rollingSum += int(count)
+			} else {
+				break
+			}
+		}
+
+		c.busFactor.Update(busFactor, streamLocation)
 	}
 	return nil
 }

satellite/durability/observer_test.go

@@ -67,7 +67,7 @@ func TestDurability(t *testing.T) {
 	c := NewDurability(nil, nil, []NodeClassifier{
 		func(node *nodeselection.SelectedNode) string {
 			return "net:" + node.LastNet
-		}}, 0, 0)
+		}}, 110, 0, 0, 0)
 
 	c.aliasMap = metabase.NewNodeAliasMap(aliases)
 	for _, node := range storageNodes {
@@ -123,7 +123,7 @@ func TestDurabilityUnknownNode(t *testing.T) {
 	c := NewDurability(nil, nil, []NodeClassifier{
 		func(node *nodeselection.SelectedNode) string {
 			return "net:" + node.LastNet
-		}}, 0, 0)
+		}}, 110, 0, 0, 0)
 
 	c.aliasMap = metabase.NewNodeAliasMap(aliases)
 	for _, node := range storageNodes {
@@ -165,14 +165,61 @@ func TestDurabilityUnknownNode(t *testing.T) {
 	require.Equal(t, 0, c.healthStat["net:127.0.0.1"].Min())
 }
 
+func TestBusFactor(t *testing.T) {
+	ctx := testcontext.New(t)
+	f := ObserverFork{}
+
+	for i := 0; i < 100; i++ {
+		f.classified = append(f.classified, []classID{classID((i))})
+	}
+	f.controlledByClassCache = make([]int32, 100)
+	f.busFactorCache = make([]int32, 300)
+	f.healthStat = make([]HealthStat, 100)
+	f.busFactorThreshold = 26
+
+	createSegments := func(groups ...int) []rangedloop.Segment {
+		var pieces []metabase.AliasPiece
+		ix := uint16(0)
+		groupIndex := 0
+		for _, group := range groups {
+			for i := 0; i < group; i++ {
+				pieces = append(pieces, metabase.AliasPiece{
+					Number: ix,
+					Alias:  metabase.NodeAlias(groupIndex),
+				})
+				ix++
+			}
+			groupIndex++
+		}
+		return []rangedloop.Segment{
+			{
+				StreamID:    testrand.UUID(),
+				AliasPieces: pieces,
+				Redundancy: storj.RedundancyScheme{
+					ShareSize: 123,
+				},
+			},
+		}
+	}
+
+	err := f.Process(ctx, createSegments(10, 10, 10, 10, 5, 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1))
+	require.NoError(t, err)
+	require.Equal(t, 3, f.busFactor.Min())
+}
+
 func BenchmarkDurabilityProcess(b *testing.B) {
 	ctx := context.TODO()
 
 	rng := rand.New(rand.NewSource(0))
 
 	nodeNo := 20000
+	// create 2500 segments (usual observer loop batch size) with 80 pieces
+	segmentNo := 2500
+	pieceNo := 80
 	if testing.Short() {
 		nodeNo = 10
+		segmentNo = 10
+		pieceNo = 10
 	}
 
 	nodeMap := make(map[storj.NodeID]*nodeselection.SelectedNode)
@@ -201,14 +248,14 @@ func BenchmarkDurabilityProcess(b *testing.B) {
 
 	var segments []rangedloop.Segment
 	{
-		// create 2500 segments (usual observer loop batch size) with 80 pieces
+
-		for i := 0; i < 2500; i++ {
+		for i := 0; i < segmentNo; i++ {
 			var id uuid.UUID
 			rng.Read(id[:])
 
 			var pieces metabase.Pieces
 			var aliasPieces metabase.AliasPieces
-			for j := 0; j < 80; j++ {
+			for j := 0; j < pieceNo; j++ {
 				nodeIx := rand.Intn(len(aliasToNode) - 1)
 				pieces = append(pieces, metabase.Piece{
 					Number:      uint16(j),

satellite/rangedloop.go

@@ -160,7 +160,7 @@ func NewRangedLoop(log *zap.Logger, db DB, metabaseDB *metabase.DB, config *Conf
 			func(node *nodeselection.SelectedNode) string {
 				return "c:" + node.CountryCode.String()
 			},
-		}, config.Metainfo.RS.Repair, config.RangedLoop.AsOfSystemInterval)
+		}, config.Metainfo.RS.Total, config.Metainfo.RS.Repair, config.Metainfo.RS.Repair-config.Metainfo.RS.Min, config.RangedLoop.AsOfSystemInterval)
 	}
 
 	{ // setup overlay