93fad70e4b
This change fixes the access of unset segments and keys on the reservoir when the reservoir size is less than the max OR the number of sampled segments is smaller than the reservoir size. It does so by tucking away the segments and keys behind methods that return properly sized slices into the segments/keys arrays. It also fixes a bug in the housekeeping for the internal index variable that holds onto how many items in the array have been populated. As part of this fix, it changes the type of index to int8, which reduces the size of the reservoir struct by 8 bytes. The tests have been updated to provide better coverage for this case. Change-Id: I3ceb17b692fe456fc4c1ca5d67d35c96aeb0a169
118 lines
3.6 KiB
Go
118 lines
3.6 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package audit_test
|
|
|
|
import (
|
|
"math/rand"
|
|
"strconv"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"storj.io/common/memory"
|
|
"storj.io/common/testcontext"
|
|
"storj.io/common/testrand"
|
|
"storj.io/storj/private/testplanet"
|
|
"storj.io/storj/satellite/audit"
|
|
"storj.io/storj/satellite/metabase/segmentloop"
|
|
)
|
|
|
|
// TestAuditCollector does the following:
|
|
// - start testplanet with 5 nodes and a reservoir size of 3
|
|
// - upload 5 files
|
|
// - iterate over all the segments in satellite.Metainfo and store them in allPieces map
|
|
// - create a audit observer and call metaloop.Join(auditObs)
|
|
//
|
|
// Then for every node in testplanet:
|
|
// - expect that there is a reservoir for that node on the audit observer
|
|
// - that the reservoir size is <= 2 (the maxReservoirSize)
|
|
// - that every item in the reservoir is unique
|
|
func TestAuditCollector(t *testing.T) {
|
|
testplanet.Run(t, testplanet.Config{
|
|
SatelliteCount: 1, StorageNodeCount: 5, UplinkCount: 1,
|
|
Reconfigure: testplanet.Reconfigure{
|
|
Satellite: testplanet.ReconfigureRS(3, 4, 5, 5),
|
|
},
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
satellite := planet.Satellites[0]
|
|
satellite.Audit.Worker.Loop.Pause()
|
|
|
|
ul := planet.Uplinks[0]
|
|
|
|
// upload 5 remote files with 1 segment
|
|
for i := 0; i < 5; i++ {
|
|
testData := testrand.Bytes(8 * memory.KiB)
|
|
path := "/some/remote/path/" + strconv.Itoa(i)
|
|
err := ul.Upload(ctx, satellite, "testbucket", path, testData)
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
r := rand.New(rand.NewSource(time.Now().Unix()))
|
|
observer := audit.NewCollector(4, r)
|
|
err := satellite.Metabase.SegmentLoop.Join(ctx, observer)
|
|
require.NoError(t, err)
|
|
|
|
for _, node := range planet.StorageNodes {
|
|
// expect a reservoir for every node
|
|
require.NotNil(t, observer.Reservoirs[node.ID()])
|
|
require.True(t, len(observer.Reservoirs[node.ID()].Segments()) > 1)
|
|
|
|
// Require that len segments are <= 3 even though the Collector was instantiated with 4
|
|
// because the maxReservoirSize is currently 3.
|
|
require.True(t, len(observer.Reservoirs[node.ID()].Segments()) <= 3)
|
|
|
|
repeats := make(map[audit.Segment]bool)
|
|
for _, loopSegment := range observer.Reservoirs[node.ID()].Segments() {
|
|
segment := audit.NewSegment(loopSegment)
|
|
assert.False(t, repeats[segment], "expected every item in reservoir to be unique")
|
|
repeats[segment] = true
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func BenchmarkRemoteSegment(b *testing.B) {
|
|
testplanet.Bench(b, testplanet.Config{
|
|
SatelliteCount: 1, StorageNodeCount: 4, UplinkCount: 1,
|
|
}, func(b *testing.B, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
for i := 0; i < 10; i++ {
|
|
err := planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", "object"+strconv.Itoa(i), testrand.Bytes(10*memory.KiB))
|
|
require.NoError(b, err)
|
|
}
|
|
|
|
observer := audit.NewCollector(3, rand.New(rand.NewSource(time.Now().Unix())))
|
|
|
|
segments, err := planet.Satellites[0].Metabase.DB.TestingAllSegments(ctx)
|
|
require.NoError(b, err)
|
|
|
|
loopSegments := []*segmentloop.Segment{}
|
|
|
|
for _, segment := range segments {
|
|
loopSegments = append(loopSegments, &segmentloop.Segment{
|
|
StreamID: segment.StreamID,
|
|
Position: segment.Position,
|
|
CreatedAt: segment.CreatedAt,
|
|
ExpiresAt: segment.ExpiresAt,
|
|
Redundancy: segment.Redundancy,
|
|
Pieces: segment.Pieces,
|
|
})
|
|
}
|
|
|
|
b.Run("multiple segments", func(b *testing.B) {
|
|
for i := 0; i < b.N; i++ {
|
|
for _, loopSegment := range loopSegments {
|
|
err := observer.RemoteSegment(ctx, loopSegment)
|
|
if err != nil {
|
|
b.FailNow()
|
|
}
|
|
}
|
|
}
|
|
})
|
|
})
|
|
|
|
}
|