2021-12-09 15:05:21 +00:00
|
|
|
// Copyright (C) 2022 Storj Labs, Inc.
|
|
|
|
// See LICENSE for copying information.
|
|
|
|
|
|
|
|
package satellitedb_test
|
|
|
|
|
|
|
|
import (
|
|
|
|
"testing"
|
|
|
|
"time"
|
|
|
|
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
|
|
|
|
"storj.io/common/memory"
|
2022-05-16 16:47:47 +01:00
|
|
|
"storj.io/common/pb"
|
2021-12-09 15:05:21 +00:00
|
|
|
"storj.io/common/testcontext"
|
|
|
|
"storj.io/common/testrand"
|
2023-02-23 16:27:37 +00:00
|
|
|
"storj.io/common/uuid"
|
2021-12-09 15:05:21 +00:00
|
|
|
"storj.io/storj/private/testplanet"
|
2023-06-20 09:53:42 +01:00
|
|
|
"storj.io/storj/satellite"
|
2022-05-16 16:47:47 +01:00
|
|
|
"storj.io/storj/satellite/accounting"
|
2023-04-13 13:04:07 +01:00
|
|
|
"storj.io/storj/satellite/buckets"
|
2021-12-09 15:05:21 +00:00
|
|
|
"storj.io/storj/satellite/console"
|
2022-05-16 16:47:47 +01:00
|
|
|
"storj.io/storj/satellite/metabase"
|
2022-11-10 21:02:44 +00:00
|
|
|
"storj.io/storj/satellite/orders"
|
2023-06-20 09:53:42 +01:00
|
|
|
"storj.io/storj/satellite/satellitedb/satellitedbtest"
|
2021-12-09 15:05:21 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func Test_DailyUsage(t *testing.T) {
|
|
|
|
testplanet.Run(t, testplanet.Config{SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 1},
|
|
|
|
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
const (
|
2022-05-16 16:47:47 +01:00
|
|
|
firstBucketName = "testbucket0"
|
|
|
|
secondBucketName = "testbucket1"
|
2021-12-09 15:05:21 +00:00
|
|
|
)
|
|
|
|
|
2023-11-22 21:25:40 +00:00
|
|
|
now := time.Now().UTC()
|
|
|
|
// set time to middle of day to make sure we don't cross the day boundary during tally creation
|
|
|
|
twelveToday := time.Date(now.Year(), now.Month(), now.Day(), 12, 0, 0, 0, time.UTC)
|
|
|
|
fivePastTwelve := twelveToday.Add(5 * time.Minute)
|
|
|
|
yesterday := twelveToday.Add(-24 * time.Hour)
|
|
|
|
twoDaysAgo := yesterday.Add(-24 * time.Hour)
|
2021-12-09 15:05:21 +00:00
|
|
|
|
|
|
|
var (
|
|
|
|
satelliteSys = planet.Satellites[0]
|
|
|
|
uplink = planet.Uplinks[0]
|
|
|
|
projectID = uplink.Projects[0].ID
|
|
|
|
)
|
|
|
|
|
|
|
|
newUser := console.CreateUser{
|
|
|
|
FullName: "Project Daily Usage Test",
|
|
|
|
ShortName: "",
|
|
|
|
Email: "du@test.test",
|
|
|
|
}
|
|
|
|
|
|
|
|
user, err := satelliteSys.AddUser(ctx, newUser, 3)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
_, err = satelliteSys.DB.Console().ProjectMembers().Insert(ctx, user.ID, projectID)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
2023-11-22 21:25:40 +00:00
|
|
|
usage0, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, twoDaysAgo, fivePastTwelve, 0)
|
2021-12-09 15:05:21 +00:00
|
|
|
require.NoError(t, err)
|
2021-12-17 14:47:35 +00:00
|
|
|
require.Zero(t, len(usage0.AllocatedBandwidthUsage))
|
|
|
|
require.Zero(t, len(usage0.SettledBandwidthUsage))
|
2021-12-09 15:05:21 +00:00
|
|
|
require.Zero(t, len(usage0.StorageUsage))
|
|
|
|
|
2022-05-16 16:47:47 +01:00
|
|
|
segment := int64(15000)
|
2021-12-09 15:05:21 +00:00
|
|
|
|
2022-05-16 16:47:47 +01:00
|
|
|
firstBucketLocation := metabase.BucketLocation{
|
|
|
|
ProjectID: projectID,
|
|
|
|
BucketName: firstBucketName,
|
|
|
|
}
|
|
|
|
secondBucketLocation := metabase.BucketLocation{
|
|
|
|
ProjectID: projectID,
|
|
|
|
BucketName: secondBucketName,
|
|
|
|
}
|
|
|
|
tallies := map[metabase.BucketLocation]*accounting.BucketTally{
|
|
|
|
firstBucketLocation: {
|
|
|
|
BucketLocation: firstBucketLocation,
|
|
|
|
TotalBytes: segment,
|
|
|
|
},
|
|
|
|
secondBucketLocation: {
|
|
|
|
BucketLocation: secondBucketLocation,
|
|
|
|
TotalBytes: segment,
|
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2023-11-22 21:25:40 +00:00
|
|
|
// test multiple rows existing each day
|
|
|
|
createTallies := func(interval time.Time, tallies map[metabase.BucketLocation]*accounting.BucketTally) {
|
|
|
|
for i := 0; i < 3; i++ {
|
|
|
|
if i != 0 {
|
|
|
|
interval = interval.Add(1 * time.Hour)
|
|
|
|
}
|
|
|
|
err = satelliteSys.DB.ProjectAccounting().SaveTallies(ctx, interval, tallies)
|
|
|
|
require.NoError(t, err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
createTallies(twoDaysAgo, tallies)
|
|
|
|
createTallies(yesterday, tallies)
|
|
|
|
createTallies(twelveToday, tallies)
|
|
|
|
|
|
|
|
err = satelliteSys.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, 0, fivePastTwelve)
|
2022-05-16 16:47:47 +01:00
|
|
|
require.NoError(t, err)
|
2023-11-22 21:25:40 +00:00
|
|
|
err = satelliteSys.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, 0, fivePastTwelve)
|
2023-09-12 12:55:57 +01:00
|
|
|
require.NoError(t, err)
|
2023-11-22 21:25:40 +00:00
|
|
|
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(firstBucketName), pb.PieceAction_GET, segment, fivePastTwelve)
|
2023-09-12 12:55:57 +01:00
|
|
|
require.NoError(t, err)
|
2023-11-22 21:25:40 +00:00
|
|
|
err = satelliteSys.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, []byte(secondBucketName), pb.PieceAction_GET, segment, fivePastTwelve)
|
2021-12-09 15:05:21 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
2023-11-22 21:25:40 +00:00
|
|
|
usage1, err := satelliteSys.DB.ProjectAccounting().GetProjectDailyUsageByDateRange(ctx, projectID, twoDaysAgo, fivePastTwelve, 0)
|
2021-12-09 15:05:21 +00:00
|
|
|
require.NoError(t, err)
|
2023-11-22 21:25:40 +00:00
|
|
|
|
|
|
|
require.Len(t, usage1.StorageUsage, 3)
|
|
|
|
for _, u := range usage1.StorageUsage {
|
|
|
|
require.Equal(t, 2*segment, u.Value)
|
|
|
|
}
|
2023-09-12 12:55:57 +01:00
|
|
|
require.Equal(t, 2*segment, usage1.AllocatedBandwidthUsage[0].Value)
|
2022-05-16 16:47:47 +01:00
|
|
|
require.Equal(t, 2*segment, usage1.SettledBandwidthUsage[0].Value)
|
2021-12-09 15:05:21 +00:00
|
|
|
},
|
|
|
|
)
|
|
|
|
}
|
2022-02-17 13:48:39 +00:00
|
|
|
|
|
|
|
func Test_GetSingleBucketRollup(t *testing.T) {
|
|
|
|
testplanet.Run(t, testplanet.Config{SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 1},
|
|
|
|
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
const (
|
|
|
|
bucketName = "testbucket"
|
|
|
|
firstPath = "path"
|
|
|
|
secondPath = "another_path"
|
|
|
|
)
|
|
|
|
|
|
|
|
now := time.Now().UTC()
|
|
|
|
inFiveMinutes := time.Now().Add(5 * time.Minute).UTC()
|
|
|
|
|
|
|
|
var (
|
|
|
|
satelliteSys = planet.Satellites[0]
|
2022-04-14 12:20:18 +01:00
|
|
|
upl = planet.Uplinks[0]
|
|
|
|
projectID = upl.Projects[0].ID
|
2022-02-17 13:48:39 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
newUser := console.CreateUser{
|
|
|
|
FullName: "Project Single Bucket Rollup",
|
|
|
|
ShortName: "",
|
|
|
|
Email: "sbur@test.test",
|
|
|
|
}
|
|
|
|
|
|
|
|
user, err := satelliteSys.AddUser(ctx, newUser, 3)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
_, err = satelliteSys.DB.Console().ProjectMembers().Insert(ctx, user.ID, projectID)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
planet.Satellites[0].Orders.Chore.Loop.Pause()
|
|
|
|
satelliteSys.Accounting.Tally.Loop.Pause()
|
|
|
|
|
|
|
|
timeTruncateDown := func(t time.Time) time.Time {
|
|
|
|
return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, t.Location())
|
|
|
|
}
|
|
|
|
|
|
|
|
usage0, err := satelliteSys.DB.ProjectAccounting().GetSingleBucketUsageRollup(ctx, projectID, bucketName, now, inFiveMinutes)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, bucketName, usage0.BucketName)
|
|
|
|
require.Equal(t, projectID, usage0.ProjectID)
|
|
|
|
require.Equal(t, timeTruncateDown(now), usage0.Since)
|
|
|
|
require.Equal(t, inFiveMinutes, usage0.Before)
|
|
|
|
require.Zero(t, usage0.GetEgress)
|
|
|
|
require.Zero(t, usage0.ObjectCount)
|
|
|
|
require.Zero(t, usage0.AuditEgress)
|
|
|
|
require.Zero(t, usage0.RepairEgress)
|
|
|
|
require.Zero(t, usage0.MetadataSize)
|
|
|
|
require.Zero(t, usage0.TotalSegments)
|
|
|
|
require.Zero(t, usage0.TotalStoredData)
|
|
|
|
|
|
|
|
firstSegment := testrand.Bytes(100 * memory.KiB)
|
|
|
|
secondSegment := testrand.Bytes(200 * memory.KiB)
|
|
|
|
|
2022-04-14 12:20:18 +01:00
|
|
|
err = upl.Upload(ctx, satelliteSys, bucketName, firstPath, firstSegment)
|
2022-02-17 13:48:39 +00:00
|
|
|
require.NoError(t, err)
|
2022-04-14 12:20:18 +01:00
|
|
|
|
|
|
|
err = upl.Upload(ctx, satelliteSys, bucketName, secondPath, secondSegment)
|
2022-02-17 13:48:39 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
2022-04-14 12:20:18 +01:00
|
|
|
_, err = upl.Download(ctx, satelliteSys, bucketName, firstPath)
|
2022-02-17 13:48:39 +00:00
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
require.NoError(t, planet.WaitForStorageNodeEndpoints(ctx))
|
|
|
|
tomorrow := time.Now().Add(24 * time.Hour)
|
|
|
|
planet.StorageNodes[0].Storage2.Orders.SendOrders(ctx, tomorrow)
|
|
|
|
|
|
|
|
planet.Satellites[0].Orders.Chore.Loop.TriggerWait()
|
|
|
|
satelliteSys.Accounting.Tally.Loop.TriggerWait()
|
|
|
|
// We trigger tally one more time because the most recent tally is skipped in service method.
|
|
|
|
satelliteSys.Accounting.Tally.Loop.TriggerWait()
|
|
|
|
|
|
|
|
usage1, err := satelliteSys.DB.ProjectAccounting().GetSingleBucketUsageRollup(ctx, projectID, bucketName, now, inFiveMinutes)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Greater(t, usage1.GetEgress, 0.0)
|
|
|
|
require.Greater(t, usage1.ObjectCount, 0.0)
|
|
|
|
require.Greater(t, usage1.MetadataSize, 0.0)
|
|
|
|
require.Greater(t, usage1.TotalSegments, 0.0)
|
|
|
|
require.Greater(t, usage1.TotalStoredData, 0.0)
|
|
|
|
},
|
|
|
|
)
|
|
|
|
}
|
2022-11-10 21:02:44 +00:00
|
|
|
|
2023-02-23 05:08:29 +00:00
|
|
|
func Test_GetProjectTotal(t *testing.T) {
|
2022-11-10 21:02:44 +00:00
|
|
|
testplanet.Run(t, testplanet.Config{SatelliteCount: 1, StorageNodeCount: 1},
|
|
|
|
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
2023-02-23 05:08:29 +00:00
|
|
|
bucketName := testrand.BucketName()
|
|
|
|
projectID := testrand.UUID()
|
|
|
|
|
|
|
|
db := planet.Satellites[0].DB
|
|
|
|
|
|
|
|
// The 3rd tally is only present to prevent CreateStorageTally from skipping the 2nd.
|
|
|
|
var tallies []accounting.BucketStorageTally
|
|
|
|
for i := 0; i < 3; i++ {
|
2023-02-23 16:27:37 +00:00
|
|
|
tally := randTally(bucketName, projectID, time.Time{}.Add(time.Duration(i)*time.Hour))
|
2023-02-23 05:08:29 +00:00
|
|
|
tallies = append(tallies, tally)
|
|
|
|
require.NoError(t, db.ProjectAccounting().CreateStorageTally(ctx, tally))
|
|
|
|
}
|
2023-01-27 05:34:08 +00:00
|
|
|
|
2023-02-23 05:08:29 +00:00
|
|
|
var rollups []orders.BucketBandwidthRollup
|
|
|
|
var expectedEgress int64
|
|
|
|
for i := 0; i < 2; i++ {
|
2023-02-23 16:27:37 +00:00
|
|
|
rollup := randRollup(bucketName, projectID, tallies[i].IntervalStart)
|
2023-02-23 05:08:29 +00:00
|
|
|
rollups = append(rollups, rollup)
|
|
|
|
expectedEgress += rollup.Inline + rollup.Settled
|
2022-11-10 21:02:44 +00:00
|
|
|
}
|
2023-02-23 05:08:29 +00:00
|
|
|
require.NoError(t, db.Orders().UpdateBandwidthBatch(ctx, rollups))
|
2022-11-10 21:02:44 +00:00
|
|
|
|
2023-02-23 05:08:29 +00:00
|
|
|
usage, err := db.ProjectAccounting().GetProjectTotal(ctx, projectID, tallies[0].IntervalStart, tallies[2].IntervalStart.Add(time.Minute))
|
|
|
|
require.NoError(t, err)
|
2022-11-10 21:02:44 +00:00
|
|
|
|
2023-02-23 05:08:29 +00:00
|
|
|
const epsilon = 1e-8
|
2023-09-07 17:39:36 +01:00
|
|
|
require.InDelta(t, usage.Storage, float64(tallies[0].Bytes()+tallies[1].Bytes()), epsilon)
|
|
|
|
require.InDelta(t, usage.SegmentCount, float64(tallies[0].TotalSegmentCount+tallies[1].TotalSegmentCount), epsilon)
|
|
|
|
require.InDelta(t, usage.ObjectCount, float64(tallies[0].ObjectCount+tallies[1].ObjectCount), epsilon)
|
|
|
|
require.Equal(t, usage.Egress, expectedEgress)
|
|
|
|
require.Equal(t, usage.Since, tallies[0].IntervalStart)
|
|
|
|
require.Equal(t, usage.Before, tallies[2].IntervalStart.Add(time.Minute))
|
2022-11-10 21:02:44 +00:00
|
|
|
|
2023-02-23 05:08:29 +00:00
|
|
|
// Ensure that GetProjectTotal treats the 'before' arg as exclusive
|
|
|
|
usage, err = db.ProjectAccounting().GetProjectTotal(ctx, projectID, tallies[0].IntervalStart, tallies[2].IntervalStart)
|
|
|
|
require.NoError(t, err)
|
2023-09-07 17:39:36 +01:00
|
|
|
require.InDelta(t, usage.Storage, float64(tallies[0].Bytes()), epsilon)
|
|
|
|
require.InDelta(t, usage.SegmentCount, float64(tallies[0].TotalSegmentCount), epsilon)
|
|
|
|
require.InDelta(t, usage.ObjectCount, float64(tallies[0].ObjectCount), epsilon)
|
|
|
|
require.Equal(t, usage.Egress, expectedEgress)
|
|
|
|
require.Equal(t, usage.Since, tallies[0].IntervalStart)
|
|
|
|
require.Equal(t, usage.Before, tallies[2].IntervalStart)
|
2023-01-27 05:34:08 +00:00
|
|
|
|
2023-02-23 05:08:29 +00:00
|
|
|
usage, err = db.ProjectAccounting().GetProjectTotal(ctx, projectID, rollups[0].IntervalStart, rollups[1].IntervalStart)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Zero(t, usage.Storage)
|
|
|
|
require.Zero(t, usage.SegmentCount)
|
|
|
|
require.Zero(t, usage.ObjectCount)
|
2023-09-07 17:39:36 +01:00
|
|
|
require.Equal(t, usage.Egress, rollups[0].Inline+rollups[0].Settled)
|
|
|
|
require.Equal(t, usage.Since, rollups[0].IntervalStart)
|
|
|
|
require.Equal(t, usage.Before, rollups[1].IntervalStart)
|
2022-11-10 21:02:44 +00:00
|
|
|
},
|
|
|
|
)
|
|
|
|
}
|
2023-02-23 16:27:37 +00:00
|
|
|
|
|
|
|
func Test_GetProjectTotalByPartner(t *testing.T) {
|
|
|
|
const (
|
|
|
|
epsilon = 1e-8
|
|
|
|
usagePeriod = time.Hour
|
|
|
|
tallyRollupCount = 2
|
|
|
|
)
|
|
|
|
since := time.Time{}
|
|
|
|
before := since.Add(2 * usagePeriod)
|
|
|
|
|
|
|
|
testplanet.Run(t, testplanet.Config{SatelliteCount: 1, StorageNodeCount: 1},
|
|
|
|
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
sat := planet.Satellites[0]
|
|
|
|
|
|
|
|
user, err := sat.AddUser(ctx, console.CreateUser{
|
|
|
|
FullName: "Test User",
|
|
|
|
Email: "user@mail.test",
|
|
|
|
}, 1)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
project, err := sat.AddProject(ctx, user.ID, "testproject")
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
type expectedTotal struct {
|
|
|
|
storage float64
|
|
|
|
segments float64
|
|
|
|
objects float64
|
|
|
|
egress int64
|
|
|
|
}
|
|
|
|
expectedTotals := make(map[string]expectedTotal)
|
|
|
|
var beforeTotal expectedTotal
|
|
|
|
|
|
|
|
requireTotal := func(t *testing.T, expected expectedTotal, expectedSince, expectedBefore time.Time, actual accounting.ProjectUsage) {
|
|
|
|
require.InDelta(t, expected.storage, actual.Storage, epsilon)
|
|
|
|
require.InDelta(t, expected.segments, actual.SegmentCount, epsilon)
|
|
|
|
require.InDelta(t, expected.objects, actual.ObjectCount, epsilon)
|
|
|
|
require.Equal(t, expected.egress, actual.Egress)
|
|
|
|
require.Equal(t, expectedSince, actual.Since)
|
|
|
|
require.Equal(t, expectedBefore, actual.Before)
|
|
|
|
}
|
|
|
|
|
|
|
|
partnerNames := []string{"", "partner1", "partner2"}
|
|
|
|
for _, name := range partnerNames {
|
|
|
|
total := expectedTotal{}
|
|
|
|
|
2023-04-13 13:04:07 +01:00
|
|
|
bucket := buckets.Bucket{
|
2023-02-23 16:27:37 +00:00
|
|
|
ID: testrand.UUID(),
|
|
|
|
Name: testrand.BucketName(),
|
|
|
|
ProjectID: project.ID,
|
|
|
|
}
|
|
|
|
if name != "" {
|
|
|
|
bucket.UserAgent = []byte(name)
|
|
|
|
}
|
|
|
|
_, err := sat.DB.Buckets().CreateBucket(ctx, bucket)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
// We use multiple tallies and rollups to ensure that
|
|
|
|
// GetProjectTotalByPartner is capable of summing them.
|
|
|
|
for i := 0; i <= tallyRollupCount; i++ {
|
|
|
|
tally := randTally(bucket.Name, project.ID, since.Add(time.Duration(i)*usagePeriod/tallyRollupCount))
|
|
|
|
require.NoError(t, sat.DB.ProjectAccounting().CreateStorageTally(ctx, tally))
|
|
|
|
|
|
|
|
// The last tally's usage data is unused.
|
|
|
|
usageHours := (usagePeriod / tallyRollupCount).Hours()
|
|
|
|
if i < tallyRollupCount {
|
|
|
|
total.storage += float64(tally.Bytes()) * usageHours
|
|
|
|
total.objects += float64(tally.ObjectCount) * usageHours
|
|
|
|
total.segments += float64(tally.TotalSegmentCount) * usageHours
|
|
|
|
}
|
|
|
|
|
|
|
|
if i < tallyRollupCount-1 {
|
|
|
|
beforeTotal.storage += float64(tally.Bytes()) * usageHours
|
|
|
|
beforeTotal.objects += float64(tally.ObjectCount) * usageHours
|
|
|
|
beforeTotal.segments += float64(tally.TotalSegmentCount) * usageHours
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
var rollups []orders.BucketBandwidthRollup
|
|
|
|
for i := 0; i < tallyRollupCount; i++ {
|
|
|
|
rollup := randRollup(bucket.Name, project.ID, since.Add(time.Duration(i)*usagePeriod/tallyRollupCount))
|
|
|
|
rollups = append(rollups, rollup)
|
|
|
|
total.egress += rollup.Inline + rollup.Settled
|
|
|
|
|
|
|
|
if i < tallyRollupCount {
|
|
|
|
beforeTotal.egress += rollup.Inline + rollup.Settled
|
|
|
|
}
|
|
|
|
}
|
|
|
|
require.NoError(t, sat.DB.Orders().UpdateBandwidthBatch(ctx, rollups))
|
|
|
|
|
|
|
|
expectedTotals[name] = total
|
|
|
|
}
|
|
|
|
|
|
|
|
t.Run("sum all partner usages", func(t *testing.T) {
|
|
|
|
ctx := testcontext.New(t)
|
|
|
|
usages, err := sat.DB.ProjectAccounting().GetProjectTotalByPartner(ctx, project.ID, nil, since, before)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, usages, 1)
|
|
|
|
require.Contains(t, usages, "")
|
|
|
|
|
|
|
|
var summedTotal expectedTotal
|
|
|
|
for _, total := range expectedTotals {
|
|
|
|
summedTotal.storage += total.storage
|
|
|
|
summedTotal.segments += total.segments
|
|
|
|
summedTotal.objects += total.objects
|
|
|
|
summedTotal.egress += total.egress
|
|
|
|
}
|
|
|
|
|
|
|
|
requireTotal(t, summedTotal, since, before, usages[""])
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("individual partner usages", func(t *testing.T) {
|
|
|
|
ctx := testcontext.New(t)
|
|
|
|
usages, err := sat.DB.ProjectAccounting().GetProjectTotalByPartner(ctx, project.ID, partnerNames, since, before)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, usages, len(expectedTotals))
|
|
|
|
for _, name := range partnerNames {
|
|
|
|
require.Contains(t, usages, name)
|
|
|
|
}
|
|
|
|
|
|
|
|
for partner, usage := range usages {
|
|
|
|
requireTotal(t, expectedTotals[partner], since, before, usage)
|
|
|
|
}
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("select one partner usage and sum remaining usages", func(t *testing.T) {
|
|
|
|
ctx := testcontext.New(t)
|
|
|
|
partner := partnerNames[len(partnerNames)-1]
|
|
|
|
usages, err := sat.DB.ProjectAccounting().GetProjectTotalByPartner(ctx, project.ID, []string{partner}, since, before)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, usages, 2)
|
|
|
|
require.Contains(t, usages, "")
|
|
|
|
require.Contains(t, usages, partner)
|
|
|
|
|
|
|
|
var summedTotal expectedTotal
|
|
|
|
for _, partner := range partnerNames[:len(partnerNames)-1] {
|
|
|
|
summedTotal.storage += expectedTotals[partner].storage
|
|
|
|
summedTotal.segments += expectedTotals[partner].segments
|
|
|
|
summedTotal.objects += expectedTotals[partner].objects
|
|
|
|
summedTotal.egress += expectedTotals[partner].egress
|
|
|
|
}
|
|
|
|
|
|
|
|
requireTotal(t, expectedTotals[partner], since, before, usages[partner])
|
|
|
|
requireTotal(t, summedTotal, since, before, usages[""])
|
|
|
|
})
|
|
|
|
|
|
|
|
t.Run("ensure the 'before' arg is exclusive", func(t *testing.T) {
|
|
|
|
ctx := testcontext.New(t)
|
|
|
|
before := since.Add(usagePeriod)
|
|
|
|
usages, err := sat.DB.ProjectAccounting().GetProjectTotalByPartner(ctx, project.ID, nil, since, before)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, usages, 1)
|
|
|
|
require.Contains(t, usages, "")
|
|
|
|
requireTotal(t, beforeTotal, since, before, usages[""])
|
|
|
|
})
|
|
|
|
},
|
|
|
|
)
|
|
|
|
}
|
|
|
|
|
|
|
|
func randTally(bucketName string, projectID uuid.UUID, intervalStart time.Time) accounting.BucketStorageTally {
|
|
|
|
return accounting.BucketStorageTally{
|
|
|
|
BucketName: bucketName,
|
|
|
|
ProjectID: projectID,
|
|
|
|
IntervalStart: intervalStart,
|
|
|
|
TotalBytes: int64(testrand.Intn(1000)),
|
|
|
|
ObjectCount: int64(testrand.Intn(1000)),
|
|
|
|
TotalSegmentCount: int64(testrand.Intn(1000)),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
func randRollup(bucketName string, projectID uuid.UUID, intervalStart time.Time) orders.BucketBandwidthRollup {
|
|
|
|
return orders.BucketBandwidthRollup{
|
|
|
|
ProjectID: projectID,
|
|
|
|
BucketName: bucketName,
|
|
|
|
IntervalStart: intervalStart,
|
|
|
|
Action: pb.PieceAction_GET,
|
|
|
|
Inline: int64(testrand.Intn(1000)),
|
|
|
|
Settled: int64(testrand.Intn(1000)),
|
|
|
|
}
|
|
|
|
}
|
2023-03-30 11:58:25 +01:00
|
|
|
|
|
|
|
func Test_GetProjectObjectsSegments(t *testing.T) {
|
|
|
|
testplanet.Run(t, testplanet.Config{SatelliteCount: 1, UplinkCount: 1},
|
|
|
|
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
projectID := planet.Uplinks[0].Projects[0].ID
|
|
|
|
|
|
|
|
projectStats, err := planet.Satellites[0].DB.ProjectAccounting().GetProjectObjectsSegments(ctx, projectID)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Zero(t, projectStats.ObjectCount)
|
|
|
|
require.Zero(t, projectStats.SegmentCount)
|
|
|
|
|
|
|
|
err = planet.Uplinks[0].Upload(ctx, planet.Satellites[0], "testbucket", "object", []byte("testdata"))
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
2023-04-17 09:24:10 +01:00
|
|
|
// bucket exists but no entry in bucket tallies yet
|
|
|
|
projectStats, err = planet.Satellites[0].DB.ProjectAccounting().GetProjectObjectsSegments(ctx, projectID)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Zero(t, projectStats.ObjectCount)
|
|
|
|
require.Zero(t, projectStats.SegmentCount)
|
|
|
|
|
2023-03-30 11:58:25 +01:00
|
|
|
planet.Satellites[0].Accounting.Tally.Loop.TriggerWait()
|
|
|
|
|
|
|
|
projectStats, err = planet.Satellites[0].DB.ProjectAccounting().GetProjectObjectsSegments(ctx, projectID)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.EqualValues(t, 1, projectStats.ObjectCount)
|
|
|
|
require.EqualValues(t, 1, projectStats.SegmentCount)
|
|
|
|
|
|
|
|
// delete object and bucket to see if projects stats will show zero
|
|
|
|
err = planet.Uplinks[0].DeleteObject(ctx, planet.Satellites[0], "testbucket", "object")
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
err = planet.Uplinks[0].DeleteBucket(ctx, planet.Satellites[0], "testbucket")
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
projectStats, err = planet.Satellites[0].DB.ProjectAccounting().GetProjectObjectsSegments(ctx, projectID)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Zero(t, projectStats.ObjectCount)
|
|
|
|
require.Zero(t, projectStats.SegmentCount)
|
|
|
|
})
|
|
|
|
}
|
2023-05-19 23:51:54 +01:00
|
|
|
|
2023-06-29 10:54:26 +01:00
|
|
|
func Test_GetProjectSettledBandwidth(t *testing.T) {
|
|
|
|
testplanet.Run(t, testplanet.Config{SatelliteCount: 1, UplinkCount: 1},
|
|
|
|
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
projectID := planet.Uplinks[0].Projects[0].ID
|
|
|
|
sat := planet.Satellites[0]
|
|
|
|
|
|
|
|
now := time.Now().UTC()
|
|
|
|
|
|
|
|
egress, err := sat.DB.ProjectAccounting().GetProjectSettledBandwidth(ctx, projectID, now.Year(), now.Month(), 0)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Zero(t, egress)
|
|
|
|
|
|
|
|
bucket := "testbucket"
|
|
|
|
err = planet.Uplinks[0].CreateBucket(ctx, sat, bucket)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
bucket1 := "testbucket1"
|
|
|
|
err = planet.Uplinks[0].CreateBucket(ctx, sat, bucket1)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
amount := int64(1000)
|
|
|
|
bucketBytes := []byte(bucket)
|
|
|
|
bucket1Bytes := []byte(bucket1)
|
|
|
|
startOfMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
|
|
|
|
err = sat.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, bucketBytes, pb.PieceAction_GET, amount, startOfMonth)
|
|
|
|
require.NoError(t, err)
|
|
|
|
err = sat.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, bucket1Bytes, pb.PieceAction_GET, 2*amount, startOfMonth)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
egress, err = sat.DB.ProjectAccounting().GetProjectSettledBandwidth(ctx, projectID, now.Year(), now.Month(), 0)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Zero(t, egress)
|
|
|
|
|
|
|
|
err = sat.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, bucketBytes, pb.PieceAction_GET, amount, 0, startOfMonth)
|
|
|
|
require.NoError(t, err)
|
|
|
|
err = sat.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, bucket1Bytes, pb.PieceAction_GET, 2*amount, 0, startOfMonth)
|
|
|
|
require.NoError(t, err)
|
|
|
|
|
|
|
|
egress, err = sat.DB.ProjectAccounting().GetProjectSettledBandwidth(ctx, projectID, now.Year(), now.Month(), 0)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, 3*amount, egress)
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2023-05-19 23:51:54 +01:00
|
|
|
func TestProjectUsageGap(t *testing.T) {
|
|
|
|
testplanet.Run(t, testplanet.Config{
|
|
|
|
SatelliteCount: 1, UplinkCount: 1,
|
|
|
|
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
|
|
|
sat := planet.Satellites[0]
|
|
|
|
uplink := planet.Uplinks[0]
|
|
|
|
tally := sat.Accounting.Tally
|
|
|
|
|
|
|
|
tally.Loop.Pause()
|
|
|
|
|
|
|
|
now := time.Time{}
|
|
|
|
tally.SetNow(func() time.Time {
|
|
|
|
return now
|
|
|
|
})
|
|
|
|
|
|
|
|
const (
|
|
|
|
bucketName = "testbucket"
|
|
|
|
objectPath = "test/path"
|
|
|
|
)
|
|
|
|
|
|
|
|
data := testrand.Bytes(10)
|
|
|
|
require.NoError(t, uplink.Upload(ctx, sat, bucketName, objectPath, data))
|
|
|
|
tally.Loop.TriggerWait()
|
|
|
|
|
|
|
|
objs, err := sat.Metabase.DB.TestingAllObjects(ctx)
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.Len(t, objs, 1)
|
|
|
|
expectedStorage := objs[0].TotalEncryptedSize
|
|
|
|
|
|
|
|
now = now.Add(time.Hour)
|
|
|
|
require.NoError(t, uplink.DeleteObject(ctx, sat, bucketName, objectPath))
|
|
|
|
tally.Loop.TriggerWait()
|
|
|
|
|
|
|
|
// This object is only uploaded and tallied so that the usage calculator knows
|
|
|
|
// how long it's been since the previous tally.
|
|
|
|
now = now.Add(time.Hour)
|
|
|
|
require.NoError(t, uplink.Upload(ctx, sat, bucketName, objectPath, data))
|
|
|
|
tally.Loop.TriggerWait()
|
|
|
|
|
|
|
|
// The bucket was full for only 1 hour, so expect `expectedStorage` byte-hours of storage usage.
|
|
|
|
usage, err := sat.DB.ProjectAccounting().GetProjectTotal(ctx, uplink.Projects[0].ID, time.Time{}, now.Add(time.Second))
|
|
|
|
require.NoError(t, err)
|
|
|
|
require.EqualValues(t, expectedStorage, usage.Storage)
|
|
|
|
})
|
|
|
|
}
|
2023-06-20 09:53:42 +01:00
|
|
|
|
|
|
|
func TestProjectaccounting_GetNonEmptyTallyBucketsInRange(t *testing.T) {
|
|
|
|
// test if invalid bucket name will be handled correctly
|
|
|
|
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
|
|
|
|
_, err := db.ProjectAccounting().GetNonEmptyTallyBucketsInRange(ctx, metabase.BucketLocation{
|
|
|
|
ProjectID: testrand.UUID(),
|
|
|
|
BucketName: "a\\",
|
|
|
|
}, metabase.BucketLocation{
|
|
|
|
ProjectID: testrand.UUID(),
|
|
|
|
BucketName: "b\\",
|
|
|
|
})
|
|
|
|
require.NoError(t, err)
|
|
|
|
})
|
|
|
|
}
|