06f4aede9b
This is first attempt to use AliasPieces inastead Pieces with segments/range loop. So far we were always using Pieces which are always converted from AliasPieces for easy use. Side effect is that using NodeID with loop observers is heavy e.g. we are using maps which behaves slower with NodeIDs. We are starting with audit observer because it's easy to change it as in feact it doesn't need access to real NodeID at all. We just need to reference node in some way and this way is NodeAlias. Results of BenchmarkRemoteSegment: name old time/op new time/op delta RemoteSegment/Cockroach/multiple_segments-8 1.79µs ± 6% 0.03µs ± 4% -98.29% (p=0.008 n=5+5) name old alloc/op new alloc/op delta RemoteSegment/Cockroach/multiple_segments-8 0.00B 0.00B ~ (all equal) name old allocs/op new allocs/op delta RemoteSegment/Cockroach/multiple_segments-8 0.00 0.00 ~ (all equal) Change-Id: Ib7fc87e568a4d3a9af27b5e3b644ea68ab6db7aa
72 lines
2.1 KiB
Go
72 lines
2.1 KiB
Go
// Copyright (C) 2019 Storj Labs, Inc.
|
|
// See LICENSE for copying information.
|
|
|
|
package audit
|
|
|
|
import (
|
|
"context"
|
|
"math/rand"
|
|
|
|
"storj.io/storj/satellite/metabase"
|
|
"storj.io/storj/satellite/metabase/segmentloop"
|
|
)
|
|
|
|
var _ segmentloop.Observer = (*Collector)(nil)
|
|
|
|
// Collector uses the segment loop to add segments to node reservoirs.
|
|
type Collector struct {
|
|
Reservoirs map[metabase.NodeAlias]*Reservoir
|
|
slotCount int
|
|
rand *rand.Rand
|
|
}
|
|
|
|
// NewCollector instantiates a segment collector.
|
|
func NewCollector(reservoirSlots int, r *rand.Rand) *Collector {
|
|
return &Collector{
|
|
Reservoirs: make(map[metabase.NodeAlias]*Reservoir),
|
|
slotCount: reservoirSlots,
|
|
rand: r,
|
|
}
|
|
}
|
|
|
|
// LoopStarted is called at each start of a loop.
|
|
func (collector *Collector) LoopStarted(context.Context, segmentloop.LoopInfo) (err error) {
|
|
return nil
|
|
}
|
|
|
|
// RemoteSegment takes a remote segment found in metainfo and creates a reservoir for it if it doesn't exist already.
|
|
func (collector *Collector) RemoteSegment(ctx context.Context, segment *segmentloop.Segment) error {
|
|
// we are expliticy not adding monitoring here as we are tracking loop observers separately
|
|
|
|
for _, piece := range segment.AliasPieces {
|
|
res, ok := collector.Reservoirs[piece.Alias]
|
|
if !ok {
|
|
res = NewReservoir(collector.slotCount)
|
|
collector.Reservoirs[piece.Alias] = res
|
|
}
|
|
res.Sample(collector.rand, segment)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// InlineSegment returns nil because we're only auditing for storage nodes for now.
|
|
func (collector *Collector) InlineSegment(ctx context.Context, segment *segmentloop.Segment) (err error) {
|
|
return nil
|
|
}
|
|
|
|
// Process performs per-node reservoir sampling on remote segments for addition into the audit queue.
|
|
func (collector *Collector) Process(ctx context.Context, segments []segmentloop.Segment) (err error) {
|
|
for _, segment := range segments {
|
|
// The reservoir ends up deferencing and copying the segment internally
|
|
// but that's not obvious, so alias the loop variable.
|
|
segment := segment
|
|
if segment.Inline() {
|
|
continue
|
|
}
|
|
if err := collector.RemoteSegment(ctx, &segment); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|