cmd/tools/segment-verify: add CreatedBefore and CreatedAfter flags
We would like to verify only segments from specific periods of time. https://github.com/storj/storj/issues/6431 Change-Id: I42610962022bdf6ee36815fe1c157d67792147b8
This commit is contained in:
parent
bca46736bb
commit
4cbdc0342a
@ -15,6 +15,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/spacemonkeygo/monkit/v3"
|
||||
"github.com/spf13/pflag"
|
||||
"github.com/zeebo/errs"
|
||||
"go.uber.org/zap"
|
||||
|
||||
@ -72,6 +73,9 @@ type ServiceConfig struct {
|
||||
|
||||
OfflineStatusCacheTime time.Duration `help:"how long to cache a \"node offline\" status" default:"30m"`
|
||||
|
||||
CreatedBefore DateFlag `help:"verify only segments created before specific date (date format 'YYYY-MM-DD')" default:""`
|
||||
CreatedAfter DateFlag `help:"verify only segments created after specific date (date format 'YYYY-MM-DD')" default:"1970-01-01"`
|
||||
|
||||
AsOfSystemInterval time.Duration `help:"as of system interval" releaseDefault:"-5m" devDefault:"-1us" testDefault:"-1us"`
|
||||
}
|
||||
|
||||
@ -300,6 +304,9 @@ func (service *Service) ProcessRange(ctx context.Context, low, high uuid.UUID) (
|
||||
CursorPosition: cursorPosition,
|
||||
Limit: service.config.BatchSize,
|
||||
|
||||
CreatedAfter: service.config.CreatedAfter.time(),
|
||||
CreatedBefore: service.config.CreatedBefore.time(),
|
||||
|
||||
AsOfSystemInterval: service.config.AsOfSystemInterval,
|
||||
})
|
||||
if err != nil {
|
||||
@ -627,3 +634,42 @@ func uuidBefore(v uuid.UUID) uuid.UUID {
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// DateFlag flag implementation for date, format YYYY-MM-DD.
|
||||
type DateFlag struct {
|
||||
time.Time
|
||||
}
|
||||
|
||||
// String implements pflag.Value.
|
||||
func (t *DateFlag) String() string {
|
||||
return t.Format(time.DateOnly)
|
||||
}
|
||||
|
||||
// Set implements pflag.Value.
|
||||
func (t *DateFlag) Set(s string) error {
|
||||
if s == "" {
|
||||
t.Time = time.Now()
|
||||
return nil
|
||||
}
|
||||
|
||||
parsedTime, err := time.Parse(time.DateOnly, s)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.Time = parsedTime
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t *DateFlag) time() *time.Time {
|
||||
if t.IsZero() {
|
||||
return nil
|
||||
}
|
||||
return &t.Time
|
||||
}
|
||||
|
||||
// Type implements pflag.Value.
|
||||
func (t *DateFlag) Type() string {
|
||||
return "time-flag"
|
||||
}
|
||||
|
||||
var _ pflag.Value = &DateFlag{}
|
||||
|
@ -24,6 +24,9 @@ type ListVerifySegments struct {
|
||||
StreamIDs []uuid.UUID
|
||||
Limit int
|
||||
|
||||
CreatedAfter *time.Time
|
||||
CreatedBefore *time.Time
|
||||
|
||||
AsOfSystemTime time.Time
|
||||
AsOfSystemInterval time.Duration
|
||||
}
|
||||
@ -62,12 +65,16 @@ func (opts *ListVerifySegments) getQueryAndParameters(asof string) (string, []in
|
||||
(stream_id, position) > ($1, $2) AND
|
||||
inline_data IS NULL AND
|
||||
remote_alias_pieces IS NOT NULL AND
|
||||
(segments.expires_at IS NULL OR segments.expires_at > now())
|
||||
(segments.expires_at IS NULL OR segments.expires_at > now()) AND
|
||||
($3::TIMESTAMPTZ IS NULL OR segments.created_at > $3) AND -- created after
|
||||
($4::TIMESTAMPTZ IS NULL OR segments.created_at < $4) -- created before
|
||||
ORDER BY stream_id ASC, position ASC
|
||||
LIMIT $3
|
||||
LIMIT $5
|
||||
`, []interface{}{
|
||||
opts.CursorStreamID,
|
||||
opts.CursorPosition,
|
||||
opts.CreatedAfter,
|
||||
opts.CreatedBefore,
|
||||
opts.Limit,
|
||||
}
|
||||
}
|
||||
@ -84,13 +91,17 @@ func (opts *ListVerifySegments) getQueryAndParameters(asof string) (string, []in
|
||||
(segments.stream_id, segments.position) > ($2, $3) AND
|
||||
segments.inline_data IS NULL AND
|
||||
segments.remote_alias_pieces IS NOT NULL AND
|
||||
(segments.expires_at IS NULL OR segments.expires_at > now())
|
||||
(segments.expires_at IS NULL OR segments.expires_at > now()) AND
|
||||
($4::TIMESTAMPTZ IS NULL OR segments.created_at > $4) AND -- created after
|
||||
($5::TIMESTAMPTZ IS NULL OR segments.created_at < $5) -- created before
|
||||
ORDER BY segments.stream_id ASC, segments.position ASC
|
||||
LIMIT $4
|
||||
LIMIT $6
|
||||
`, []interface{}{
|
||||
pgutil.UUIDArray(opts.StreamIDs),
|
||||
opts.CursorStreamID,
|
||||
opts.CursorPosition,
|
||||
opts.CreatedAfter,
|
||||
opts.CreatedBefore,
|
||||
opts.Limit,
|
||||
}
|
||||
}
|
||||
|
@ -292,6 +292,76 @@ func TestListVerifySegments(t *testing.T) {
|
||||
},
|
||||
}.Check(ctx, t, db)
|
||||
})
|
||||
|
||||
t.Run("creation time filtering", func(t *testing.T) {
|
||||
defer metabasetest.DeleteAll{}.Check(ctx, t, db)
|
||||
|
||||
now := time.Now()
|
||||
|
||||
expectedVerifySegments := []metabase.VerifySegment{}
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
obj = metabasetest.RandObjectStream()
|
||||
obj.StreamID[0] = byte(i) // make StreamIDs ordered
|
||||
_ = metabasetest.CreateObject(ctx, t, db, obj, 1)
|
||||
|
||||
expectedVerifySegments = append(expectedVerifySegments, defaultVerifySegment(obj.StreamID, 0))
|
||||
}
|
||||
|
||||
date := func(t time.Time) *time.Time {
|
||||
return &t
|
||||
}
|
||||
|
||||
metabasetest.ListVerifySegments{
|
||||
Opts: metabase.ListVerifySegments{
|
||||
CreatedAfter: date(now.Add(-time.Hour)),
|
||||
Limit: 10,
|
||||
},
|
||||
Result: metabase.ListVerifySegmentsResult{Segments: expectedVerifySegments},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
metabasetest.ListVerifySegments{
|
||||
Opts: metabase.ListVerifySegments{
|
||||
CreatedAfter: date(now.Add(time.Hour)),
|
||||
Limit: 10,
|
||||
},
|
||||
Result: metabase.ListVerifySegmentsResult{},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
metabasetest.ListVerifySegments{
|
||||
Opts: metabase.ListVerifySegments{
|
||||
CreatedBefore: date(now.Add(time.Hour)),
|
||||
Limit: 10,
|
||||
},
|
||||
Result: metabase.ListVerifySegmentsResult{Segments: expectedVerifySegments},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
metabasetest.ListVerifySegments{
|
||||
Opts: metabase.ListVerifySegments{
|
||||
CreatedBefore: date(now.Add(-time.Hour)),
|
||||
Limit: 10,
|
||||
},
|
||||
Result: metabase.ListVerifySegmentsResult{},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
metabasetest.ListVerifySegments{
|
||||
Opts: metabase.ListVerifySegments{
|
||||
CreatedAfter: date(now.Add(-time.Hour)),
|
||||
CreatedBefore: date(now.Add(time.Hour)),
|
||||
Limit: 10,
|
||||
},
|
||||
Result: metabase.ListVerifySegmentsResult{Segments: expectedVerifySegments},
|
||||
}.Check(ctx, t, db)
|
||||
|
||||
metabasetest.ListVerifySegments{
|
||||
Opts: metabase.ListVerifySegments{
|
||||
CreatedAfter: date(now.Add(time.Hour)),
|
||||
CreatedBefore: date(now.Add(2 * time.Hour)),
|
||||
Limit: 10,
|
||||
},
|
||||
Result: metabase.ListVerifySegmentsResult{},
|
||||
}.Check(ctx, t, db)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user