JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/projectaccounting: sum up bucket usages for daily usage query

Browse Source 
Fixed daily usage query returning single bucket usage.
We sum up bucket usages now.
Also fixed https://github.com/storj/storj/issues/4559.

Change-Id: I2eb6299f1ef500d68150879195011b6fbb5f37ed
This commit is contained in:
Vitalii 2022-05-16 18:47:47 +03:00 committed by Storj Robot
parent 84d02f7fbf
commit 07e65cd338
3 changed files with 61 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
satellite/console/consoleweb/consoleapi/usagelimits.go
View File

@ -133,8 +133,8 @@ func (ul *UsageLimits) DailyUsage(w http.ResponseWriter, r *http.Request) {
return
}
since := time.Unix(sinceStamp, 0).UTC()
before := time.Unix(beforeStamp, 0).UTC()
since := time.Unix(sinceStamp, 0)
before := time.Unix(beforeStamp, 0)
dailyUsage, err := ul.service.GetDailyProjectUsage(ctx, projectID, since, before)
if err != nil {

28
satellite/satellitedb/projectaccounting.go
View File

@ -205,8 +205,8 @@ func (db *ProjectAccounting) GetProjectDailyBandwidth(ctx context.Context, proje
func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context, projectID uuid.UUID, from, to time.Time, crdbInterval time.Duration) (_ *accounting.ProjectDailyUsage, err error) {
defer mon.Task()(&ctx)(&err)
// use end of the day for 'to' caveat.
endOfDay := time.Date(to.Year(), to.Month(), to.Day(), 23, 59, 59, 0, time.UTC)
fromBeginningOfDay := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC)
toEndOfDay := time.Date(to.Year(), to.Month(), to.Day(), 23, 59, 59, 0, time.UTC)
allocatedBandwidth := make([]accounting.ProjectUsageByDay, 0)
settledBandwidth := make([]accounting.ProjectUsageByDay, 0)
@ -219,7 +219,7 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
SELECT interval_day, egress_allocated, egress_settled
FROM project_bandwidth_daily_rollups
WHERE project_id = $1 AND (interval_day BETWEEN $2 AND $3)
`), projectID, from, endOfDay)
`), projectID, fromBeginningOfDay, toEndOfDay)
storageQuery := db.db.Rebind(`
WITH project_usage AS (
@ -251,7 +251,7 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
` + db.db.impl.AsOfSystemInterval(crdbInterval) + `
GROUP BY project_id, bucket_name, interval_day
`)
batch.Queue(storageQuery, projectID, from, endOfDay)
batch.Queue(storageQuery, projectID, fromBeginningOfDay, toEndOfDay)
results := conn.SendBatch(ctx, &batch)
defer func() { err = errs.Combine(err, results.Close()) }()
@ -293,6 +293,9 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
return err
}
var current time.Time
var index int
for storageRows.Next() {
var day time.Time
var amount int64
@ -302,6 +305,23 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
return err
}
if len(storage) == 0 {
current = day
storage = append(storage, accounting.ProjectUsageByDay{
Date: day.UTC(),
Value: amount,
})
continue
}
if current == day {
storage[index].Value += amount
continue
}
current = day
index++
storage = append(storage, accounting.ProjectUsageByDay{
Date: day.UTC(),
Value: amount,

57
satellite/satellitedb/projectaccounting_test.go
View File

@ -10,19 +10,21 @@ import (
"github.com/stretchr/testify/require"
"storj.io/common/memory"
"storj.io/common/pb"
"storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/private/testplanet"
"storj.io/storj/satellite/accounting"
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/metabase"
)
func Test_DailyUsage(t *testing.T) {
testplanet.Run(t, testplanet.Config{SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 1},
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
const (
bucketName = "testbucket"
firstPath = "path"
secondPath = "another_path"
firstBucketName = "testbucket0"
secondBucketName = "testbucket1"
)
now := time.Now()
@ -46,38 +48,49 @@ func Test_DailyUsage(t *testing.T) {
_, err = satelliteSys.DB.Console().ProjectMembers().Insert(ctx, user.ID, projectID)
require.NoError(t, err)
planet.Satellites[0].Orders.Chore.Loop.Pause()
satelliteSys.Accounting.Tally.Loop.Pause()
usage0, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, now, inFiveMinutes, 0)
require.NoError(t, err)
require.Zero(t, len(usage0.AllocatedBandwidthUsage))
require.Zero(t, len(usage0.SettledBandwidthUsage))
require.Zero(t, len(usage0.StorageUsage))
firstSegment := testrand.Bytes(5 * memory.KiB)
secondSegment := testrand.Bytes(10 * memory.KiB)
segment := int64(15000)
err = uplink.Upload(ctx, satelliteSys, bucketName, firstPath, firstSegment)
firstBucketLocation := metabase.BucketLocation{
ProjectID: projectID,
BucketName: firstBucketName,
}
secondBucketLocation := metabase.BucketLocation{
ProjectID: projectID,
BucketName: secondBucketName,
}
tallies := map[metabase.BucketLocation]*accounting.BucketTally{
firstBucketLocation: {
BucketLocation: firstBucketLocation,
TotalBytes: segment,
},
secondBucketLocation: {
BucketLocation: secondBucketLocation,
TotalBytes: segment,
},
}
err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, now, tallies)
require.NoError(t, err)
err = uplink.Upload(ctx, satelliteSys, bucketName, secondPath, secondSegment)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, inFiveMinutes)
require.NoError(t, err)
_, err = uplink.Download(ctx, satelliteSys, bucketName, firstPath)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, 0, inFiveMinutes)
require.NoError(t, err)
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, inFiveMinutes)
require.NoError(t, err)
err = planet.Satellites[0].DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, 0, inFiveMinutes)
require.NoError(t, err)
require.NoError(t, planet.WaitForStorageNodeEndpoints(ctx))
tomorrow := time.Now().Add(24 * time.Hour)
planet.StorageNodes[0].Storage2.Orders.SendOrders(ctx, tomorrow)
planet.Satellites[0].Orders.Chore.Loop.TriggerWait()
satelliteSys.Accounting.Tally.Loop.TriggerWait()
usage1, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, now, inFiveMinutes, 0)
require.NoError(t, err)
require.GreaterOrEqual(t, usage1.StorageUsage[0].Value, 15*memory.KiB)
require.GreaterOrEqual(t, usage1.AllocatedBandwidthUsage[0].Value, 5*memory.KiB)
require.GreaterOrEqual(t, usage1.SettledBandwidthUsage[0].Value, 5*memory.KiB)
require.Equal(t, 2*segment, usage1.StorageUsage[0].Value)
require.Equal(t, 2*segment, usage1.AllocatedBandwidthUsage[0].Value)
require.Equal(t, 2*segment, usage1.SettledBandwidthUsage[0].Value)
},
)
}