satellite/gc: Optionally run the GC bloomfilter process once, instead of in a loop

The current deployment strategy requires that the GC bloomfilter generation process executes only once and exits.

Change-Id: I952991f126596aa165d1f2e9fce6f8548c21bdba
This commit is contained in:
Ethan 2022-10-27 16:47:26 -05:00 committed by Ethan Adams
parent 4edef9e05c
commit 4efde65c9e
4 changed files with 29 additions and 9 deletions

View File

@ -84,6 +84,11 @@ func cmdGCBloomFilterRun(cmd *cobra.Command, args []string) (err error) {
}
runError := peer.Run(ctx)
if err := process.Report(ctx); err != nil {
log.Warn("could not send telemetry", zap.Error(err))
}
closeError := peer.Close()
return errs.Combine(runError, closeError)
}

View File

@ -57,6 +57,7 @@ type GarbageCollectionBF struct {
}
GarbageCollection struct {
Config bloomfilter.Config
Service *bloomfilter.Service
}
}
@ -127,12 +128,15 @@ func NewGarbageCollectionBF(log *zap.Logger, full *identity.FullIdentity, db DB,
}
{ // setup garbage collection bloom filters
peer.GarbageCollection.Config = config.GarbageCollectionBF
peer.GarbageCollection.Service = bloomfilter.NewService(
peer.Log.Named("garbage-collection-bf"),
config.GarbageCollectionBF,
peer.Overlay.DB,
peer.Metainfo.SegmentLoop,
)
if !config.GarbageCollectionBF.RunOnce {
peer.Services.Add(lifecycle.Item{
Name: "garbage-collection-bf",
Run: peer.GarbageCollection.Service.Run,
@ -140,6 +144,7 @@ func NewGarbageCollectionBF(log *zap.Logger, full *identity.FullIdentity, db DB,
peer.Debug.Server.Panel.Add(
debug.Cycle("Garbage Collection Bloom Filters", peer.GarbageCollection.Service.Loop))
}
}
return peer, nil
}
@ -154,10 +159,15 @@ func (peer *GarbageCollectionBF) Run(ctx context.Context) (err error) {
peer.Servers.Run(ctx, group)
peer.Services.Run(ctx, group)
if peer.GarbageCollection.Config.RunOnce {
err = peer.GarbageCollection.Service.RunOnce(ctx)
} else {
pprof.Do(ctx, pprof.Labels("name", "subsystem-wait"), func(ctx context.Context) {
err = group.Wait()
})
}
})
return err
}

View File

@ -30,6 +30,8 @@ type Config struct {
// TODO service is not enabled by default for testing until will be finished
Enabled bool `help:"set if garbage collection bloom filters is enabled or not" default:"true" testDefault:"false"`
RunOnce bool `help:"set if garbage collection bloom filter process should only run once then exit" default:"false"`
// value for InitialPieces currently based on average pieces per node
InitialPieces int64 `help:"the initial number of pieces expected for a storage node to have, used for creating a filter" releaseDefault:"400000" devDefault:"10"`
FalsePositiveRate float64 `help:"the false positive rate used for creating a garbage collection bloom filter" releaseDefault:"0.1" devDefault:"0.1"`

View File

@ -412,6 +412,9 @@ contact.external-address: ""
# the time between each garbage collection executions
# garbage-collection-bf.interval: 120h0m0s
# set if garbage collection bloom filter process should only run once then exit
# garbage-collection-bf.run-once: false
# how many bloom filters will be packed in a single zip
# garbage-collection-bf.zip-batch-size: 500