JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/satellitedb: use AS OF SYSTEM TIME for GetProjectBandwidth query

Browse Source 
Adding AS OF SYSTEM TIME to query that is calculating project bandiwdth.
As an addition method for setting interval is added as test doesn't
work well with default interval.

Change-Id: Id1e15be4f6afff13b9dc2b7f595e2edb6de28db9
This commit is contained in:
Michał Niewrzał 2021-05-25 19:43:47 +02:00 committed by Michal Niewrzal
parent 70e6cdfd06
commit 4267a958d4
10 changed files with 25 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
satellite/accounting/db.go
View File

@ -184,7 +184,7 @@ type ProjectAccounting interface {
// GetAllocatedBandwidthTotal returns the sum of GET bandwidth usage allocated for a projectID in the past time frame
GetAllocatedBandwidthTotal(ctx context.Context, projectID uuid.UUID, from time.Time) (int64, error)
// GetProjectBandwidth returns project allocated bandwidth for the specified year, month and day.
GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (int64, error)
GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int, asOfSystemInterval time.Duration) (int64, error)
// GetProjectDailyBandwidth returns bandwidth (allocated and settled) for the specified day.
GetProjectDailyBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (int64, int64, error)
// DeleteProjectBandwidthBefore deletes project bandwidth rollups before the given time

1
satellite/accounting/live/cache.go
View File

@ -25,6 +25,7 @@ var (
type Config struct {
StorageBackend string `help:"what to use for storing real-time accounting data"`
BandwidthCacheTTL time.Duration `default:"5m" help:"bandwidth cache key time to live"`
AsOfSystemInterval time.Duration `default:"-10s" help:"as of system interval"`
}
// OpenCache creates a new accounting.Cache instance using the type specified backend in

1
satellite/accounting/projectbwcleanup/chore_test.go
View File

@ -39,6 +39,7 @@ func testProjectAllocatedBandwidthRetain(t *testing.T, retain int) {
Reconfigure: testplanet.Reconfigure{
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
config.ProjectBWCleanup.RetainMonths = retain
config.LiveAccounting.AsOfSystemInterval = time.Millisecond
},
},
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {

6
satellite/accounting/projectusage.go
View File

@ -31,16 +31,18 @@ type Service struct {
projectLimitCache *ProjectLimitCache
bandwidthCacheTTL time.Duration
nowFn func() time.Time
asOfSystemInterval time.Duration
}
// NewService created new instance of project usage service.
func NewService(projectAccountingDB ProjectAccounting, liveAccounting Cache, limitCache *ProjectLimitCache, bandwidthCacheTTL time.Duration) *Service {
func NewService(projectAccountingDB ProjectAccounting, liveAccounting Cache, limitCache *ProjectLimitCache, bandwidthCacheTTL, asOfSystemInterval time.Duration) *Service {
return &Service{
projectAccountingDB: projectAccountingDB,
liveAccounting: liveAccounting,
projectLimitCache: limitCache,
bandwidthCacheTTL: bandwidthCacheTTL,
nowFn: time.Now,
asOfSystemInterval: asOfSystemInterval,
}
}
@ -166,7 +168,7 @@ func (usage *Service) GetProjectBandwidthTotals(ctx context.Context, projectID u
func (usage *Service) GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (_ int64, err error) {
defer mon.Task()(&ctx, projectID)(&err)
total, err := usage.projectAccountingDB.GetProjectBandwidth(ctx, projectID, year, month, day)
total, err := usage.projectAccountingDB.GetProjectBandwidth(ctx, projectID, year, month, day, usage.asOfSystemInterval)
return total, ErrProjectUsage.Wrap(err)
}

8
satellite/accounting/projectusage_test.go
View File

@ -115,6 +115,11 @@ func TestProjectUsageBandwidth(t *testing.T) {
t.Run(testCase.name, func(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 6, UplinkCount: 1,
Reconfigure: testplanet.Reconfigure{
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
config.LiveAccounting.AsOfSystemInterval = time.Millisecond
},
},
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
saDB := planet.Satellites[0].DB
orderDB := saDB.Orders()
@ -248,7 +253,7 @@ func TestProjectBandwidthRollups(t *testing.T) {
err = db.Orders().UpdateBucketBandwidthBatch(ctx, hour, rollups)
require.NoError(t, err)
alloc, err := db.ProjectAccounting().GetProjectBandwidth(ctx, p1, now.Year(), now.Month(), now.Day())
alloc, err := db.ProjectAccounting().GetProjectBandwidth(ctx, p1, now.Year(), now.Month(), now.Day(), 0)
require.NoError(t, err)
require.EqualValues(t, 4000, alloc)
})
@ -629,6 +634,7 @@ func TestProjectUsageBandwidthResetAfter3days(t *testing.T) {
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
config.Console.UsageLimits.DefaultStorageLimit = 1 * memory.MB
config.Console.UsageLimits.DefaultBandwidthLimit = 1 * memory.MB
config.LiveAccounting.AsOfSystemInterval = time.Millisecond
},
},
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {

1
satellite/api.go
View File

@ -304,6 +304,7 @@ func NewAPI(log *zap.Logger, full *identity.FullIdentity, db DB,
peer.LiveAccounting.Cache,
peer.ProjectLimits.Cache,
config.LiveAccounting.BandwidthCacheTTL,
config.LiveAccounting.AsOfSystemInterval,
)
}

2
satellite/console/consoleweb/consoleql/mutation_test.go
View File

@ -66,7 +66,7 @@ func TestGraphqlMutation(t *testing.T) {
projectLimitCache := accounting.NewProjectLimitCache(db.ProjectAccounting(), 0, 0, accounting.ProjectLimitConfig{CacheCapacity: 100})
projectUsage := accounting.NewService(db.ProjectAccounting(), cache, projectLimitCache, 5*time.Minute)
projectUsage := accounting.NewService(db.ProjectAccounting(), cache, projectLimitCache, 5*time.Minute, -10*time.Second)
// TODO maybe switch this test to testplanet to avoid defining config and Stripe service
pc := paymentsconfig.Config{

2
satellite/console/consoleweb/consoleql/query_test.go
View File

@ -50,7 +50,7 @@ func TestGraphqlQuery(t *testing.T) {
projectLimitCache := accounting.NewProjectLimitCache(db.ProjectAccounting(), 0, 0, accounting.ProjectLimitConfig{CacheCapacity: 100})
projectUsage := accounting.NewService(db.ProjectAccounting(), cache, projectLimitCache, 5*time.Minute)
projectUsage := accounting.NewService(db.ProjectAccounting(), cache, projectLimitCache, 5*time.Minute, -10*time.Second)
// TODO maybe switch this test to testplanet to avoid defining config and Stripe service
pc := paymentsconfig.Config{

4
satellite/satellitedb/projectaccounting.go
View File

@ -138,7 +138,7 @@ func (db *ProjectAccounting) GetAllocatedBandwidthTotal(ctx context.Context, pro
}
// GetProjectBandwidth returns the used bandwidth (settled or allocated) for the specified year, month and day.
func (db *ProjectAccounting) GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int) (_ int64, err error) {
func (db *ProjectAccounting) GetProjectBandwidth(ctx context.Context, projectID uuid.UUID, year int, month time.Month, day int, asOfSystemInterval time.Duration) (_ int64, err error) {
defer mon.Task()(&ctx)(&err)
var egress *int64
@ -160,7 +160,7 @@ func (db *ProjectAccounting) GetProjectBandwidth(ctx context.Context, projectID
END AS amount
FROM project_bandwidth_daily_rollups
WHERE project_id = ? AND interval_day >= ? AND interval_day < ?
) SELECT sum(amount) from egress;`
) SELECT sum(amount) FROM egress` + db.db.impl.AsOfSystemInterval(asOfSystemInterval)
err = db.db.QueryRow(ctx, db.db.Rebind(query), expiredSince, projectID[:], startOfMonth, periodEnd).Scan(&egress)
if errors.Is(err, sql.ErrNoRows) || egress == nil {
return 0, nil

3
scripts/testdata/satellite-config.yaml.lock vendored
View File

@ -289,6 +289,9 @@ identity.cert-path: /root/.local/share/storj/identity/satellite/identity.cert
# path to the private key for this identity
identity.key-path: /root/.local/share/storj/identity/satellite/identity.key
# as of system interval
# live-accounting.as-of-system-interval: -10s
# bandwidth cache key time to live
# live-accounting.bandwidth-cache-ttl: 5m0s