ff6d640fca
Minimal implementation of the ranged (=threaded) segment loop service, to improve performance over the existing loop. Has tests with a an inmemory segment database and example observer. Does not have yet: database link, observer duration tracking, suspicious processed ratio guard, rate limiting, minimum execution interval per observer, etc. Part of https://github.com/storj/storj/issues/5223 Change-Id: I08ffb392c3539e380f4e7b4f1afd56c4c394668d
64 lines
1.5 KiB
Go
64 lines
1.5 KiB
Go
// Copyright (C) 2022 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package rangedloop_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/zap/zaptest"
|
|
|
|
"storj.io/storj/satellite/metabase/rangedloop"
|
|
"storj.io/storj/satellite/metabase/rangedloop/rangedlooptest"
|
|
"storj.io/storj/satellite/metabase/segmentloop"
|
|
)
|
|
|
|
func TestLoop(t *testing.T) {
|
|
for _, parallelism := range []int{1, 2, 3} {
|
|
for _, nSegments := range []int{0, 1, 2, 11} {
|
|
for _, nObservers := range []int{0, 1, 2} {
|
|
t.Run(
|
|
fmt.Sprintf("par%d_seg%d_obs%d", parallelism, nSegments, nObservers),
|
|
func(t *testing.T) {
|
|
RunTest(t, parallelism, nSegments, nObservers)
|
|
},
|
|
)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func RunTest(t *testing.T, parallelism int, nSegments int, nObservers int) {
|
|
batchSize := 2
|
|
ctx := context.Background()
|
|
|
|
observers := []rangedloop.Observer{}
|
|
for i := 0; i < nObservers; i++ {
|
|
observers = append(observers, &rangedlooptest.CountObserver{})
|
|
}
|
|
|
|
loopService := rangedloop.NewService(
|
|
zaptest.NewLogger(t),
|
|
rangedloop.Config{
|
|
BatchSize: batchSize,
|
|
Parallelism: parallelism,
|
|
AsOfSystemInterval: 0,
|
|
},
|
|
&rangedlooptest.RangeSplitterMock{
|
|
Segments: make([]segmentloop.Segment, nSegments),
|
|
},
|
|
observers,
|
|
)
|
|
|
|
err := loopService.RunOnce(ctx)
|
|
require.NoError(t, err)
|
|
|
|
for _, observer := range observers {
|
|
countObserver := observer.(*rangedlooptest.CountObserver)
|
|
require.Equal(t, nSegments, countObserver.NumSegments)
|
|
}
|
|
}
|