satellite/projectaccounting: sum up bucket usages for daily usage query
Fixed daily usage query returning single bucket usage. We sum up bucket usages now. Also fixed https://github.com/storj/storj/issues/4559. Change-Id: I2eb6299f1ef500d68150879195011b6fbb5f37ed
This commit is contained in:
parent
84d02f7fbf
commit
07e65cd338
@ -133,8 +133,8 @@ func (ul *UsageLimits) DailyUsage(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
since := time.Unix(sinceStamp, 0).UTC()
|
||||
before := time.Unix(beforeStamp, 0).UTC()
|
||||
since := time.Unix(sinceStamp, 0)
|
||||
before := time.Unix(beforeStamp, 0)
|
||||
|
||||
dailyUsage, err := ul.service.GetDailyProjectUsage(ctx, projectID, since, before)
|
||||
if err != nil {
|
||||
|
@ -205,8 +205,8 @@ func (db *ProjectAccounting) GetProjectDailyBandwidth(ctx context.Context, proje
|
||||
func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context, projectID uuid.UUID, from, to time.Time, crdbInterval time.Duration) (_ *accounting.ProjectDailyUsage, err error) {
|
||||
defer mon.Task()(&ctx)(&err)
|
||||
|
||||
// use end of the day for 'to' caveat.
|
||||
endOfDay := time.Date(to.Year(), to.Month(), to.Day(), 23, 59, 59, 0, time.UTC)
|
||||
fromBeginningOfDay := time.Date(from.Year(), from.Month(), from.Day(), 0, 0, 0, 0, time.UTC)
|
||||
toEndOfDay := time.Date(to.Year(), to.Month(), to.Day(), 23, 59, 59, 0, time.UTC)
|
||||
|
||||
allocatedBandwidth := make([]accounting.ProjectUsageByDay, 0)
|
||||
settledBandwidth := make([]accounting.ProjectUsageByDay, 0)
|
||||
@ -219,7 +219,7 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
|
||||
SELECT interval_day, egress_allocated, egress_settled
|
||||
FROM project_bandwidth_daily_rollups
|
||||
WHERE project_id = $1 AND (interval_day BETWEEN $2 AND $3)
|
||||
`), projectID, from, endOfDay)
|
||||
`), projectID, fromBeginningOfDay, toEndOfDay)
|
||||
|
||||
storageQuery := db.db.Rebind(`
|
||||
WITH project_usage AS (
|
||||
@ -251,7 +251,7 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
|
||||
` + db.db.impl.AsOfSystemInterval(crdbInterval) + `
|
||||
GROUP BY project_id, bucket_name, interval_day
|
||||
`)
|
||||
batch.Queue(storageQuery, projectID, from, endOfDay)
|
||||
batch.Queue(storageQuery, projectID, fromBeginningOfDay, toEndOfDay)
|
||||
|
||||
results := conn.SendBatch(ctx, &batch)
|
||||
defer func() { err = errs.Combine(err, results.Close()) }()
|
||||
@ -293,6 +293,9 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
|
||||
return err
|
||||
}
|
||||
|
||||
var current time.Time
|
||||
var index int
|
||||
|
||||
for storageRows.Next() {
|
||||
var day time.Time
|
||||
var amount int64
|
||||
@ -302,6 +305,23 @@ func (db *ProjectAccounting) GetProjectDailyUsageByDateRange(ctx context.Context
|
||||
return err
|
||||
}
|
||||
|
||||
if len(storage) == 0 {
|
||||
current = day
|
||||
storage = append(storage, accounting.ProjectUsageByDay{
|
||||
Date: day.UTC(),
|
||||
Value: amount,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
if current == day {
|
||||
storage[index].Value += amount
|
||||
continue
|
||||
}
|
||||
|
||||
current = day
|
||||
index++
|
||||
|
||||
storage = append(storage, accounting.ProjectUsageByDay{
|
||||
Date: day.UTC(),
|
||||
Value: amount,
|
||||
|
@ -10,19 +10,21 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"storj.io/common/memory"
|
||||
"storj.io/common/pb"
|
||||
"storj.io/common/testcontext"
|
||||
"storj.io/common/testrand"
|
||||
"storj.io/storj/private/testplanet"
|
||||
"storj.io/storj/satellite/accounting"
|
||||
"storj.io/storj/satellite/console"
|
||||
"storj.io/storj/satellite/metabase"
|
||||
)
|
||||
|
||||
func Test_DailyUsage(t *testing.T) {
|
||||
testplanet.Run(t, testplanet.Config{SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 1},
|
||||
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
||||
const (
|
||||
bucketName = "testbucket"
|
||||
firstPath = "path"
|
||||
secondPath = "another_path"
|
||||
firstBucketName = "testbucket0"
|
||||
secondBucketName = "testbucket1"
|
||||
)
|
||||
|
||||
now := time.Now()
|
||||
@ -46,38 +48,49 @@ func Test_DailyUsage(t *testing.T) {
|
||||
_, err = satelliteSys.DB.Console().ProjectMembers().Insert(ctx, user.ID, projectID)
|
||||
require.NoError(t, err)
|
||||
|
||||
planet.Satellites[0].Orders.Chore.Loop.Pause()
|
||||
satelliteSys.Accounting.Tally.Loop.Pause()
|
||||
|
||||
usage0, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, now, inFiveMinutes, 0)
|
||||
require.NoError(t, err)
|
||||
require.Zero(t, len(usage0.AllocatedBandwidthUsage))
|
||||
require.Zero(t, len(usage0.SettledBandwidthUsage))
|
||||
require.Zero(t, len(usage0.StorageUsage))
|
||||
|
||||
firstSegment := testrand.Bytes(5 * memory.KiB)
|
||||
secondSegment := testrand.Bytes(10 * memory.KiB)
|
||||
segment := int64(15000)
|
||||
|
||||
err = uplink.Upload(ctx, satelliteSys, bucketName, firstPath, firstSegment)
|
||||
firstBucketLocation := metabase.BucketLocation{
|
||||
ProjectID: projectID,
|
||||
BucketName: firstBucketName,
|
||||
}
|
||||
secondBucketLocation := metabase.BucketLocation{
|
||||
ProjectID: projectID,
|
||||
BucketName: secondBucketName,
|
||||
}
|
||||
tallies := map[metabase.BucketLocation]*accounting.BucketTally{
|
||||
firstBucketLocation: {
|
||||
BucketLocation: firstBucketLocation,
|
||||
TotalBytes: segment,
|
||||
},
|
||||
secondBucketLocation: {
|
||||
BucketLocation: secondBucketLocation,
|
||||
TotalBytes: segment,
|
||||
},
|
||||
}
|
||||
|
||||
err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, now, tallies)
|
||||
require.NoError(t, err)
|
||||
err = uplink.Upload(ctx, satelliteSys, bucketName, secondPath, secondSegment)
|
||||
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, inFiveMinutes)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = uplink.Download(ctx, satelliteSys, bucketName, firstPath)
|
||||
err = satelliteSys.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, 0, inFiveMinutes)
|
||||
require.NoError(t, err)
|
||||
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, inFiveMinutes)
|
||||
require.NoError(t, err)
|
||||
err = planet.Satellites[0].DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, 0, inFiveMinutes)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, planet.WaitForStorageNodeEndpoints(ctx))
|
||||
tomorrow := time.Now().Add(24 * time.Hour)
|
||||
planet.StorageNodes[0].Storage2.Orders.SendOrders(ctx, tomorrow)
|
||||
|
||||
planet.Satellites[0].Orders.Chore.Loop.TriggerWait()
|
||||
satelliteSys.Accounting.Tally.Loop.TriggerWait()
|
||||
|
||||
usage1, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, now, inFiveMinutes, 0)
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, usage1.StorageUsage[0].Value, 15*memory.KiB)
|
||||
require.GreaterOrEqual(t, usage1.AllocatedBandwidthUsage[0].Value, 5*memory.KiB)
|
||||
require.GreaterOrEqual(t, usage1.SettledBandwidthUsage[0].Value, 5*memory.KiB)
|
||||
require.Equal(t, 2*segment, usage1.StorageUsage[0].Value)
|
||||
require.Equal(t, 2*segment, usage1.AllocatedBandwidthUsage[0].Value)
|
||||
require.Equal(t, 2*segment, usage1.SettledBandwidthUsage[0].Value)
|
||||
},
|
||||
)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user