From 8d8f6734de4e50fd18b7b704e06f0138f17826e2 Mon Sep 17 00:00:00 2001 From: Vitalii Date: Thu, 29 Jun 2023 12:54:26 +0300 Subject: [PATCH] satellite/{db, accounting}: added functionality to query settled bandwidth for given project Added functionality to return only settled traffic from project_bandwidth_daily_rollups table for given month. Updated {projectID}/usage-limits endpoint to return only settled bandwidth used. This is a possible fix for this issue https://github.com/storj/storj-private/issues/293 Change-Id: I12516dc898f449c2122e7442b8fbb88309a48ebe --- satellite/accounting/db.go | 2 + satellite/accounting/projectusage.go | 11 +++++ satellite/console/service.go | 15 +++++-- satellite/console/service_test.go | 15 +++++++ satellite/satellitedb/projectaccounting.go | 19 ++++++++ .../satellitedb/projectaccounting_test.go | 44 +++++++++++++++++++ 6 files changed, 102 insertions(+), 4 deletions(-) diff --git a/satellite/accounting/db.go b/satellite/accounting/db.go index 290626dc6..74e666d61 100644 --- a/satellite/accounting/db.go +++ b/satellite/accounting/db.go @@ -219,6 +219,8 @@ type ProjectAccounting interface { GetProjectSettledBandwidthTotal(ctx context.Context, projectID uuid.UUID, from time.Time) (_ int64, err error) // GetProjectBandwidth returns project allocated bandwidth for the specified year, month and day. GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int, asOfSystemInterval time.Duration) (int64, error) + // GetProjectSettledBandwidth returns the used settled bandwidth for the specified year and month. + GetProjectSettledBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, asOfSystemInterval time.Duration) (int64, error) // GetProjectDailyBandwidth returns bandwidth (allocated and settled) for the specified day. GetProjectDailyBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (int64, int64, int64, error) // DeleteProjectBandwidthBefore deletes project bandwidth rollups before the given time diff --git a/satellite/accounting/projectusage.go b/satellite/accounting/projectusage.go index 8a986bd74..5bff015ef 100644 --- a/satellite/accounting/projectusage.go +++ b/satellite/accounting/projectusage.go @@ -218,6 +218,17 @@ func (usage *Service) GetProjectBandwidthTotals(ctx context.Context, projectID u return total, ErrProjectUsage.Wrap(err) } +// GetProjectSettledBandwidth returns total amount of settled bandwidth used for past 30 days. +func (usage *Service) GetProjectSettledBandwidth(ctx context.Context, projectID uuid.UUID) (_ int64, err error) { + defer mon.Task()(&ctx, projectID)(&err) + + // from the beginning of the current month + year, month, _ := usage.nowFn().Date() + + total, err := usage.projectAccountingDB.GetProjectSettledBandwidth(ctx, projectID, year, month, usage.asOfSystemInterval) + return total, ErrProjectUsage.Wrap(err) +} + // GetProjectSegmentTotals returns total amount of allocated segments used for past 30 days. func (usage *Service) GetProjectSegmentTotals(ctx context.Context, projectID uuid.UUID) (total int64, err error) { defer mon.Task()(&ctx, projectID)(&err) diff --git a/satellite/console/service.go b/satellite/console/service.go index 61ef3c32e..791895534 100644 --- a/satellite/console/service.go +++ b/satellite/console/service.go @@ -2725,7 +2725,7 @@ func (s *Service) GetProjectUsageLimits(ctx context.Context, projectID uuid.UUID return nil, Error.Wrap(err) } - prUsageLimits, err := s.getProjectUsageLimits(ctx, isMember.project.ID) + prUsageLimits, err := s.getProjectUsageLimits(ctx, isMember.project.ID, true) if err != nil { return nil, Error.Wrap(err) } @@ -2767,7 +2767,7 @@ func (s *Service) GetTotalUsageLimits(ctx context.Context) (_ *ProjectUsageLimit var totalBandwidthUsed int64 for _, pr := range projects { - prUsageLimits, err := s.getProjectUsageLimits(ctx, pr.ID) + prUsageLimits, err := s.getProjectUsageLimits(ctx, pr.ID, false) if err != nil { return nil, Error.Wrap(err) } @@ -2786,7 +2786,7 @@ func (s *Service) GetTotalUsageLimits(ctx context.Context) (_ *ProjectUsageLimit }, nil } -func (s *Service) getProjectUsageLimits(ctx context.Context, projectID uuid.UUID) (_ *ProjectUsageLimits, err error) { +func (s *Service) getProjectUsageLimits(ctx context.Context, projectID uuid.UUID, onlySettledBandwidth bool) (_ *ProjectUsageLimits, err error) { defer mon.Task()(&ctx)(&err) storageLimit, err := s.projectUsage.GetProjectStorageLimit(ctx, projectID) @@ -2806,10 +2806,17 @@ func (s *Service) getProjectUsageLimits(ctx context.Context, projectID uuid.UUID if err != nil { return nil, err } - bandwidthUsed, err := s.projectUsage.GetProjectBandwidthTotals(ctx, projectID) + + var bandwidthUsed int64 + if onlySettledBandwidth { + bandwidthUsed, err = s.projectUsage.GetProjectSettledBandwidth(ctx, projectID) + } else { + bandwidthUsed, err = s.projectUsage.GetProjectBandwidthTotals(ctx, projectID) + } if err != nil { return nil, err } + segmentUsed, err := s.projectUsage.GetProjectSegmentTotals(ctx, projectID) if err != nil { return nil, err diff --git a/satellite/console/service_test.go b/satellite/console/service_test.go index 29fb1f025..e53f0c371 100644 --- a/satellite/console/service_test.go +++ b/satellite/console/service_test.go @@ -23,6 +23,7 @@ import ( "storj.io/common/currency" "storj.io/common/macaroon" "storj.io/common/memory" + "storj.io/common/pb" "storj.io/common/storj" "storj.io/common/testcontext" "storj.io/common/testrand" @@ -434,6 +435,20 @@ func TestService(t *testing.T) { require.Equal(t, updatedBandwidthLimit.Int64(), limits1.BandwidthLimit) require.Equal(t, updatedStorageLimit.Int64(), limits2.StorageLimit) require.Equal(t, updatedBandwidthLimit.Int64(), limits2.BandwidthLimit) + + bucket := "testbucket1" + err = planet.Uplinks[1].CreateBucket(ctx, sat, bucket) + require.NoError(t, err) + + now := time.Now().UTC() + startOfMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + err = sat.DB.Orders().UpdateBucketBandwidthAllocation(ctx, up2Proj.ID, []byte(bucket), pb.PieceAction_GET, 1000, startOfMonth) + require.NoError(t, err) + + limits2, err = service.GetProjectUsageLimits(userCtx2, up2Proj.PublicID) + require.NoError(t, err) + require.NotNil(t, limits2) + require.Equal(t, int64(0), limits2.BandwidthUsed) }) t.Run("ChangeEmail", func(t *testing.T) { diff --git a/satellite/satellitedb/projectaccounting.go b/satellite/satellitedb/projectaccounting.go index 798d211bb..331c1e9cb 100644 --- a/satellite/satellitedb/projectaccounting.go +++ b/satellite/satellitedb/projectaccounting.go @@ -223,6 +223,25 @@ func (db *ProjectAccounting) GetProjectBandwidth(ctx context.Context, projectID return *egress, err } +// GetProjectSettledBandwidth returns the used settled bandwidth for the specified year and month. +func (db *ProjectAccounting) GetProjectSettledBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, asOfSystemInterval time.Duration) (_ int64, err error) { + defer mon.Task()(&ctx)(&err) + var egress *int64 + + startOfMonth := time.Date(year, month, 1, 0, 0, 0, 0, time.UTC) + periodEnd := time.Date(year, month+1, 1, 0, 0, 0, 0, time.UTC) + + query := `SELECT sum(egress_settled) FROM project_bandwidth_daily_rollups` + + db.db.impl.AsOfSystemInterval(asOfSystemInterval) + + ` WHERE project_id = ? AND interval_day >= ? AND interval_day < ?` + err = db.db.QueryRow(ctx, db.db.Rebind(query), projectID[:], startOfMonth, periodEnd).Scan(&egress) + if errors.Is(err, sql.ErrNoRows) || egress == nil { + return 0, nil + } + + return *egress, err +} + // GetProjectDailyBandwidth returns project bandwidth (allocated and settled) for the specified day. func (db *ProjectAccounting) GetProjectDailyBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (allocated int64, settled, dead int64, err error) { defer mon.Task()(&ctx)(&err) diff --git a/satellite/satellitedb/projectaccounting_test.go b/satellite/satellitedb/projectaccounting_test.go index a0d7cf23d..8fe7c3d98 100644 --- a/satellite/satellitedb/projectaccounting_test.go +++ b/satellite/satellitedb/projectaccounting_test.go @@ -455,6 +455,50 @@ func Test_GetProjectObjectsSegments(t *testing.T) { }) } +func Test_GetProjectSettledBandwidth(t *testing.T) { + testplanet.Run(t, testplanet.Config{SatelliteCount: 1, UplinkCount: 1}, + func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + projectID := planet.Uplinks[0].Projects[0].ID + sat := planet.Satellites[0] + + now := time.Now().UTC() + + egress, err := sat.DB.ProjectAccounting().GetProjectSettledBandwidth(ctx, projectID, now.Year(), now.Month(), 0) + require.NoError(t, err) + require.Zero(t, egress) + + bucket := "testbucket" + err = planet.Uplinks[0].CreateBucket(ctx, sat, bucket) + require.NoError(t, err) + + bucket1 := "testbucket1" + err = planet.Uplinks[0].CreateBucket(ctx, sat, bucket1) + require.NoError(t, err) + + amount := int64(1000) + bucketBytes := []byte(bucket) + bucket1Bytes := []byte(bucket1) + startOfMonth := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC) + err = sat.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, bucketBytes, pb.PieceAction_GET, amount, startOfMonth) + require.NoError(t, err) + err = sat.DB.Orders().UpdateBucketBandwidthAllocation(ctx, projectID, bucket1Bytes, pb.PieceAction_GET, 2*amount, startOfMonth) + require.NoError(t, err) + + egress, err = sat.DB.ProjectAccounting().GetProjectSettledBandwidth(ctx, projectID, now.Year(), now.Month(), 0) + require.NoError(t, err) + require.Zero(t, egress) + + err = sat.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, bucketBytes, pb.PieceAction_GET, amount, 0, startOfMonth) + require.NoError(t, err) + err = sat.DB.Orders().UpdateBucketBandwidthSettle(ctx, projectID, bucket1Bytes, pb.PieceAction_GET, 2*amount, 0, startOfMonth) + require.NoError(t, err) + + egress, err = sat.DB.ProjectAccounting().GetProjectSettledBandwidth(ctx, projectID, now.Year(), now.Month(), 0) + require.NoError(t, err) + require.Equal(t, 3*amount, egress) + }) +} + func TestProjectUsageGap(t *testing.T) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, UplinkCount: 1,