JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/accounting: use custom query for bucket tally by default

Browse Source 
We added alternative way to calculate bucket tallies for accounting and
now it's tested and we will enable it by default.

CollectBucketTallies was extended to support overriding current time
to be able to test handling expired objects.

Change-Id: I738b99a33fd2e086245f92d874c1cbb806e834c0
This commit is contained in:
Michal Niewrzal 2022-11-17 14:06:17 +01:00 committed by Storj Robot
parent 954d703533
commit d5eea2db61
3 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
satellite/accounting/tally/tally.go
View File

@ -29,7 +29,7 @@ type Config struct {
Interval time.Duration `help:"how frequently the tally service should run" releaseDefault:"1h" devDefault:"30s" testDefault:"$TESTINTERVAL"`
SaveRollupBatchSize int `help:"how large of batches SaveRollup should process at a time" default:"1000"`
ReadRollupBatchSize int `help:"how large of batches GetBandwidthSince should process at a time" default:"10000"`
UseObjectsLoop bool `help:"flag to switch between calculating bucket tallies using objects loop or custom query" default:"true"`
UseObjectsLoop bool `help:"flag to switch between calculating bucket tallies using objects loop or custom query" default:"false"`
ListLimit int `help:"how many objects to query in a batch" default:"2500"`
AsOfSystemInterval time.Duration `help:"as of system interval" releaseDefault:"-5m" devDefault:"-1us" testDefault:"-1us"`
@ -313,6 +313,7 @@ func (observer *BucketTallyCollector) fillBucketTallies(ctx context.Context, sta
To: bucketLocations[len(bucketLocations)-1],
AsOfSystemTime: startingTime,
AsOfSystemInterval: observer.config.AsOfSystemInterval,
Now: observer.Now,
})
if err != nil {
return err

9
satellite/metabase/loop.go
View File

@ -433,6 +433,7 @@ type CollectBucketTallies struct {
To BucketLocation
AsOfSystemTime time.Time
AsOfSystemInterval time.Duration
Now time.Time
}
// Verify verifies CollectBucketTallies request fields.
@ -454,15 +455,19 @@ func (db *DB) CollectBucketTallies(ctx context.Context, opts CollectBucketTallie
return []BucketTally{}, err
}
if opts.Now.IsZero() {
opts.Now = time.Now()
}
err = withRows(db.db.QueryContext(ctx, `
SELECT project_id, bucket_name, SUM(total_encrypted_size), SUM(segment_count), COALESCE(SUM(length(encrypted_metadata)), 0), count(*)
FROM objects
`+db.asOfTime(opts.AsOfSystemTime, opts.AsOfSystemInterval)+`
WHERE (project_id, bucket_name) BETWEEN ($1, $2) AND ($3, $4) AND
(expires_at IS NULL OR expires_at > now())
(expires_at IS NULL OR expires_at > $5)
GROUP BY (project_id, bucket_name)
ORDER BY (project_id, bucket_name) ASC
`, opts.From.ProjectID, opts.From.BucketName, opts.To.ProjectID, opts.To.BucketName))(func(rows tagsql.Rows) error {
`, opts.From.ProjectID, opts.From.BucketName, opts.To.ProjectID, opts.To.BucketName, opts.Now))(func(rows tagsql.Rows) error {
for rows.Next() {
var bucketTally BucketTally

2
scripts/testdata/satellite-config.yaml.lock vendored
View File

@ -1004,7 +1004,7 @@ server.private-address: 127.0.0.1:7778
# tally.save-rollup-batch-size: 1000
# flag to switch between calculating bucket tallies using objects loop or custom query
# tally.use-objects-loop: true
# tally.use-objects-loop: false
# address for jaeger agent
# tracing.agent-addr: agent.tracing.datasci.storj.io:5775