JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

metabase-orphaned-segments: avoid processing recently created elements

Browse Source 
To be sure we are comparing the same set of objects
and segments lets ignore objects and segments created
after processing was started.
Segmets without objects cannot be created in normal
way so we will have them only if we broke something in
the past.

Change-Id: I96c07caf9e5091775d4dc8dfc0fef2b08b87957c
This commit is contained in:
Michał Niewrzał 2021-07-26 17:17:04 +02:00 committed by Ivan Fraixedes
parent 42b113d92b
commit 420d2f6275
1 changed files with 10 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/metabase-orphaned-segments/main.go
View File

@ -174,6 +174,11 @@ func findOrphanedSegments(ctx context.Context, log *zap.Logger, config Config) (
}, func(ctx context.Context, it metabase.LoopSegmentsIterator) error {
var entry metabase.LoopSegmentEntry
for it.Next(ctx, &entry) {
// avoid segments created after starting processing
if entry.CreatedAt != nil && entry.CreatedAt.After(startingTime) {
continue
}
streamIDs[entry.StreamID] = struct{}{}
if numberOfElements%100000 == 0 {
@ -195,6 +200,11 @@ func findOrphanedSegments(ctx context.Context, log *zap.Logger, config Config) (
}, func(ctx context.Context, it metabase.LoopObjectsIterator) error {
var entry metabase.LoopObjectEntry
for it.Next(ctx, &entry) {
// avoid objects created after starting processing
if entry.CreatedAt.After(startingTime) {
continue
}
delete(streamIDs, entry.StreamID)
if numberOfElements%100000 == 0 {