JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/satellitedb: fix bucket sum for daily usage

Browse Source 
The code to get daily storage usage is a little confusing. There is a
portion of the query which looks like it is trying to sum the storage
for buckets by each day, but it doesn't work. There is also a section
which sums the buckets by reading the returned rows, but it assumes that
the rows are ordered by the date, which they are not. This results in
the potential for multiple entries for the same day being created: one
for each bucket. The UI doesn't know about this, so the entries for the
same day overwrite each other and result in the appearance of less
stored data on the graph.

To fix it, don't group by bucket and allow the query to sum the buckets
together.

Change-Id: Id3ec25238e6e6346b93b10f834bdc9c70023f1e1
This commit is contained in:
Cameron 2023-11-22 16:25:40 -05:00
parent fb894b3720
commit 479fbb628c
2 changed files with 32 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
satellite/satellitedb/projectaccounting.go
View File

@ -303,7 +303,7 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
FROM project_usage
ORDER BY project_id, bucket_name, interval_day, interval_start DESC) pu
` + db.db.impl.AsOfSystemInterval(crdbInterval) + `
GROUP BY project_id, bucket_name, interval_day
GROUP BY project_id, interval_day
`)
batch.Queue(storageQuery, projectID, fromBeginningOfDay, toEndOfDay)
@ -331,9 +331,6 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
return err
}
var current time.Time
var index int
for storageRows.Next() {
var day time.Time
var amount int64
@ -344,23 +341,6 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
return err
}
if len(storage) == 0 {
current = day
storage = append(storage, accounting.ProjectUsageByDay{
Date: day.UTC(),
Value: amount,
})
continue
}
if current == day {
storage[index].Value += amount
continue
}
current = day
index++
storage = append(storage, accounting.ProjectUsageByDay{
Date: day.UTC(),
Value: amount,

42
satellite/satellitedb/projectaccounting_test.go
View File

@ -32,8 +32,12 @@ func Test_DailyUsage(t *testing.T) {
secondBucketName = "testbucket1"
)
now := time.Now()
inFiveMinutes := time.Now().Add(5 * time.Minute)
now := time.Now().UTC()
// set time to middle of day to make sure we don't cross the day boundary during tally creation
twelveToday := time.Date(now.Year(), now.Month(), now.Day(), 12, 0, 0, 0, time.UTC)
fivePastTwelve := twelveToday.Add(5 * time.Minute)
yesterday := twelveToday.Add(-24 * time.Hour)
twoDaysAgo := yesterday.Add(-24 * time.Hour)
var (
satelliteSys = planet.Satellites[0]
@ -53,7 +57,7 @@ func Test_DailyUsage(t *testing.T) {
_, err = satelliteSys.DB.Console().ProjectMembers().Insert(ctx, user.ID, projectID)
require.NoError(t, err)
usage0, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, now, inFiveMinutes, 0)
usage0, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, twoDaysAgo, fivePastTwelve, 0)
require.NoError(t, err)
require.Zero(t, len(usage0.AllocatedBandwidthUsage))
require.Zero(t, len(usage0.SettledBandwidthUsage))
@ -80,20 +84,36 @@ func Test_DailyUsage(t *testing.T) {
},
}
err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, now, tallies)
// test multiple rows existing each day
createTallies := func(interval time.Time, tallies map[metabase.BucketLocation]*accounting.BucketTally) {
for i := 0; i < 3; i++ {
if i != 0 {
interval = interval.Add(1 * time.Hour)
}
err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, interval, tallies)
require.NoError(t, err)
}
}
createTallies(twoDaysAgo, tallies)
createTallies(yesterday, tallies)
createTallies(twelveToday, tallies)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, 0, fivePastTwelve)
require.NoError(t, err)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, 0, inFiveMinutes)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, 0, fivePastTwelve)
require.NoError(t, err)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, 0, inFiveMinutes)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, fivePastTwelve)
require.NoError(t, err)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, inFiveMinutes)
require.NoError(t, err)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, inFiveMinutes)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, fivePastTwelve)
require.NoError(t, err)
usage1, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, now, inFiveMinutes, 0)
usage1, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, twoDaysAgo, fivePastTwelve, 0)
require.NoError(t, err)
require.Equal(t, 2*segment, usage1.StorageUsage[0].Value)
require.Len(t, usage1.StorageUsage, 3)
for _, u := range usage1.StorageUsage {
require.Equal(t, 2*segment, u.Value)
}
require.Equal(t, 2*segment, usage1.AllocatedBandwidthUsage[0].Value)
require.Equal(t, 2*segment, usage1.SettledBandwidthUsage[0].Value)
},