JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storagenode/bandwidth: fix tests to not fail for 10 hours near the end of the month

Browse Source 
Change-Id: I390569a8702164c42edddd3be020e93782227c2e
This commit is contained in:
Jeff Wendling 2020-01-31 16:25:52 -07:00
parent 03166d6be3
commit 71ff044edb
5 changed files with 61 additions and 60 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
storagenode/bandwidth/db_test.go
View File

@ -62,7 +62,7 @@ func TestBandwidthDB(t *testing.T) {
satellite0 := testidentity.MustPregeneratedSignedIdentity(0, storj.LatestIDVersion()).ID
satellite1 := testidentity.MustPregeneratedSignedIdentity(1, storj.LatestIDVersion()).ID
now := time.Now()
now := time.Date(2010, 4, 7, 12, 30, 00, 0, time.UTC)
expectedUsage := &bandwidth.Usage{}
expectedUsageTotal := &bandwidth.Usage{}
@ -85,7 +85,7 @@ func TestBandwidthDB(t *testing.T) {
}
// test summarizing
t.Run("test total summary", func(t *testing.T) {
{
usage, err := bandwidthdb.Summary(ctx, now.Add(-10*time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedUsageTotal, usage)
@ -94,9 +94,9 @@ func TestBandwidthDB(t *testing.T) {
usage, err = bandwidthdb.Summary(ctx, now.Add(time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedUsage, usage)
})
}
t.Run("test summary by satellite", func(t *testing.T) {
{
usageBySatellite, err := bandwidthdb.SummaryBySatellite(ctx, now.Add(-10*time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedUsageBySatellite, usageBySatellite)
@ -109,9 +109,9 @@ func TestBandwidthDB(t *testing.T) {
usageBySatellite, err = bandwidthdb.SummaryBySatellite(ctx, now.Add(time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedUsageBySatellite, usageBySatellite)
})
}
t.Run("test satellite summary", func(t *testing.T) {
{
usage, err := bandwidthdb.SatelliteSummary(ctx, satellite0, time.Time{}, now)
require.NoError(t, err)
require.Equal(t, expectedUsageBySatellite[satellite0], usage)
@ -119,13 +119,13 @@ func TestBandwidthDB(t *testing.T) {
usage, err = bandwidthdb.SatelliteSummary(ctx, satellite1, time.Time{}, now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedUsageBySatellite[satellite1], usage)
})
}
t.Run("test cached bandwidth usage", func(t *testing.T) {
cachedBandwidthUsage, err := bandwidthdb.MonthSummary(ctx)
{
cachedBandwidthUsage, err := bandwidthdb.MonthSummary(ctx, now)
require.NoError(t, err)
require.Equal(t, expectedUsageTotal.Total(), cachedBandwidthUsage)
})
}
})
}
@ -136,7 +136,7 @@ func TestEgressSummary(t *testing.T) {
satellite0 := testidentity.MustPregeneratedSignedIdentity(0, storj.LatestIDVersion()).ID
satellite1 := testidentity.MustPregeneratedSignedIdentity(1, storj.LatestIDVersion()).ID
now := time.Now()
now := time.Date(2010, 4, 7, 12, 30, 00, 0, time.UTC)
expectedEgressUsage := &bandwidth.Usage{}
expectedEgressUsageTotal := &bandwidth.Usage{}
@ -159,7 +159,7 @@ func TestEgressSummary(t *testing.T) {
}
// test egress summarizing.
t.Run("test egress summary", func(t *testing.T) {
{
usage, err := bandwidthdb.EgressSummary(ctx, now.Add(-10*time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedEgressUsageTotal, usage)
@ -168,9 +168,9 @@ func TestEgressSummary(t *testing.T) {
usage, err = bandwidthdb.EgressSummary(ctx, now.Add(time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedEgressUsage, usage)
})
}
t.Run("test egress summary by satellite", func(t *testing.T) {
{
usageBySatellite, err := bandwidthdb.SummaryBySatellite(ctx, now.Add(-10*time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedEgressUsageBySatellite, usageBySatellite)
@ -183,9 +183,9 @@ func TestEgressSummary(t *testing.T) {
usageBySatellite, err = bandwidthdb.SummaryBySatellite(ctx, now.Add(time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedEgressUsageBySatellite, usageBySatellite)
})
}
t.Run("test satellite egress summary", func(t *testing.T) {
{
usage, err := bandwidthdb.SatelliteEgressSummary(ctx, satellite0, time.Time{}, now)
require.NoError(t, err)
require.Equal(t, expectedEgressUsageBySatellite[satellite0], usage)
@ -193,7 +193,7 @@ func TestEgressSummary(t *testing.T) {
usage, err = bandwidthdb.SatelliteEgressSummary(ctx, satellite1, time.Time{}, now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedEgressUsageBySatellite[satellite1], usage)
})
}
})
}
@ -204,7 +204,7 @@ func TestIngressSummary(t *testing.T) {
satellite0 := testidentity.MustPregeneratedSignedIdentity(0, storj.LatestIDVersion()).ID
satellite1 := testidentity.MustPregeneratedSignedIdentity(1, storj.LatestIDVersion()).ID
now := time.Now()
now := time.Date(2010, 4, 7, 12, 30, 00, 0, time.UTC)
expectedIngressUsage := &bandwidth.Usage{}
expectedIngressUsageTotal := &bandwidth.Usage{}
@ -227,7 +227,7 @@ func TestIngressSummary(t *testing.T) {
}
// test ingress summarizing.
t.Run("test ingress summary", func(t *testing.T) {
{
usage, err := bandwidthdb.IngressSummary(ctx, now.Add(-10*time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedIngressUsageTotal, usage)
@ -236,9 +236,9 @@ func TestIngressSummary(t *testing.T) {
usage, err = bandwidthdb.IngressSummary(ctx, now.Add(time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedIngressUsage, usage)
})
}
t.Run("test ingress summary by satellite", func(t *testing.T) {
{
usageBySatellite, err := bandwidthdb.SummaryBySatellite(ctx, now.Add(-10*time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedIngressUsageBySatellite, usageBySatellite)
@ -251,9 +251,9 @@ func TestIngressSummary(t *testing.T) {
usageBySatellite, err = bandwidthdb.SummaryBySatellite(ctx, now.Add(time.Hour), now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedUsageBySatellite, usageBySatellite)
})
}
t.Run("test satellite ingress summary", func(t *testing.T) {
{
usage, err := bandwidthdb.SatelliteIngressSummary(ctx, satellite0, time.Time{}, now)
require.NoError(t, err)
require.Equal(t, expectedIngressUsageBySatellite[satellite0], usage)
@ -261,7 +261,7 @@ func TestIngressSummary(t *testing.T) {
usage, err = bandwidthdb.SatelliteIngressSummary(ctx, satellite1, time.Time{}, now.Add(10*time.Hour))
require.NoError(t, err)
require.Equal(t, expectedIngressUsageBySatellite[satellite1], usage)
})
}
})
}
@ -269,37 +269,37 @@ func TestEmptyBandwidthDB(t *testing.T) {
storagenodedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db storagenode.DB) {
bandwidthdb := db.Bandwidth()
now := time.Now()
now := time.Date(2010, 4, 7, 12, 30, 00, 0, time.UTC)
t.Run("test total summary", func(t *testing.T) {
{
usage, err := bandwidthdb.Summary(ctx, now, now)
require.NoError(t, err)
require.Equal(t, &bandwidth.Usage{}, usage)
})
}
t.Run("test summary by satellite", func(t *testing.T) {
{
usageBySatellite, err := bandwidthdb.SummaryBySatellite(ctx, now, now)
require.NoError(t, err)
require.Equal(t, map[storj.NodeID]*bandwidth.Usage{}, usageBySatellite)
})
}
t.Run("test satellite summary", func(t *testing.T) {
{
usage, err := bandwidthdb.SatelliteSummary(ctx, storj.NodeID{}, now, now)
require.NoError(t, err)
require.Equal(t, &bandwidth.Usage{}, usage)
})
}
t.Run("test get daily rollups", func(t *testing.T) {
{
rollups, err := bandwidthdb.GetDailyRollups(ctx, now, now)
require.NoError(t, err)
require.Equal(t, []bandwidth.UsageRollup(nil), rollups)
})
}
t.Run("test get satellite daily rollups", func(t *testing.T) {
{
rollups, err := bandwidthdb.GetDailySatelliteRollups(ctx, storj.NodeID{}, now, now)
require.NoError(t, err)
require.Equal(t, []bandwidth.UsageRollup(nil), rollups)
})
}
})
}
@ -311,7 +311,7 @@ func TestBandwidthDailyRollups(t *testing.T) {
hours = 12
)
now := time.Now().UTC()
now := time.Date(2010, 4, 7, 12, 30, 00, 0, time.UTC)
startDate := time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, now.Location())
bandwidthDB := db.Bandwidth()
@ -406,7 +406,7 @@ func TestBandwidthDailyRollups(t *testing.T) {
usageRollups[satellite][day] = usageRollup
}
t.Run("test satellite daily usage rollups", func(t *testing.T) {
{
rolls, err := bandwidthDB.GetDailySatelliteRollups(ctx, satelliteID, time.Time{}, now)
assert.NoError(t, err)
@ -417,9 +417,9 @@ func TestBandwidthDailyRollups(t *testing.T) {
expected := *usageRollups[satelliteID][rollup.IntervalStart]
assert.Equal(t, expected, rollup)
}
})
}
t.Run("test daily usage rollups", func(t *testing.T) {
{
rolls, err := bandwidthDB.GetDailyRollups(ctx, time.Time{}, now)
assert.NoError(t, err)
@ -429,7 +429,7 @@ func TestBandwidthDailyRollups(t *testing.T) {
for _, rollup := range rolls {
assert.Equal(t, *totalUsageRollups[rollup.IntervalStart], rollup)
}
})
}
})
}
@ -439,9 +439,10 @@ func TestCachedBandwidthMonthRollover(t *testing.T) {
satellite0 := testidentity.MustPregeneratedSignedIdentity(0, storj.LatestIDVersion()).ID
y, m, _ := time.Now().UTC().Date()
// Last second of the previous month
previousMonth := time.Date(y, m, 0, 23, 59, 59, 0, time.Now().UTC().Location())
// Compute times that will be used for the test
now := time.Date(2010, 4, 7, 12, 30, 00, 0, time.UTC)
previousMonth := time.Date(2010, 3, 31, 23, 59, 59, 0, time.UTC)
thisMonth := previousMonth.Add(time.Second)
// Add data for the previous month.
for _, action := range actions {
@ -449,20 +450,19 @@ func TestCachedBandwidthMonthRollover(t *testing.T) {
require.NoError(t, err)
}
cached, err := bandwidthdb.MonthSummary(ctx)
require.NoError(t, err)
// Cached bandwidth for this month should still be 0 since CachedBandwidthUsed only looks up by the current month
cached, err := bandwidthdb.MonthSummary(ctx, now)
require.NoError(t, err)
require.Equal(t, int64(0), cached)
thisMonth := previousMonth.Add(time.Second + 1)
var totalAmount int64
for _, action := range actions {
totalAmount += int64(action)
err := bandwidthdb.Add(ctx, satellite0, action, int64(action), thisMonth)
require.NoError(t, err)
}
cached, err = bandwidthdb.MonthSummary(ctx)
cached, err = bandwidthdb.MonthSummary(ctx, now)
require.NoError(t, err)
require.Equal(t, totalAmount, cached)
})
@ -478,7 +478,7 @@ func TestBandwidthRollup(t *testing.T) {
testID2 := storj.NodeID{2}
testID3 := storj.NodeID{3}
now := time.Now()
now := time.Date(2010, 4, 7, 12, 30, 00, 0, time.UTC)
// Create data for 48 hours ago
err = db.Bandwidth().Add(ctx, testID1, pb.PieceAction_PUT, 1, now.Add(time.Hour*-48))
@ -579,7 +579,7 @@ func TestDB_Trivial(t *testing.T) {
}
{ // Ensure MonthSummary works at all
_, err := db.Bandwidth().MonthSummary(ctx)
_, err := db.Bandwidth().MonthSummary(ctx, time.Now())
require.NoError(t, err)
}

2
storagenode/bandwidth/usage.go
View File

@ -17,7 +17,7 @@ import (
type DB interface {
Add(ctx context.Context, satelliteID storj.NodeID, action pb.PieceAction, amount int64, created time.Time) error
// MonthSummary returns summary of the current months bandwidth usages.
MonthSummary(ctx context.Context) (int64, error)
MonthSummary(ctx context.Context, now time.Time) (int64, error)
Rollup(ctx context.Context) (err error)
// Summary returns summary of bandwidth usages.
Summary(ctx context.Context, from, to time.Time) (*Usage, error)

2
storagenode/console/service.go
View File

@ -165,7 +165,7 @@ func (s *Service) GetDashboardData(ctx context.Context) (_ *Dashboard, err error
return nil, SNOServiceErr.Wrap(err)
}
bandwidthUsage, err := s.bandwidthDB.MonthSummary(ctx)
bandwidthUsage, err := s.bandwidthDB.MonthSummary(ctx, time.Now())
if err != nil {
return nil, SNOServiceErr.Wrap(err)
}

4
storagenode/monitor/monitor.go
View File

@ -171,7 +171,7 @@ func (service *Service) usedSpace(ctx context.Context) (_ int64, err error) {
func (service *Service) usedBandwidth(ctx context.Context) (_ int64, err error) {
defer mon.Task()(&ctx)(&err)
usage, err := service.usageDB.MonthSummary(ctx)
usage, err := service.usageDB.MonthSummary(ctx, time.Now())
if err != nil {
return 0, err
}
@ -192,7 +192,7 @@ func (service *Service) AvailableSpace(ctx context.Context) (_ int64, err error)
// AvailableBandwidth returns available bandwidth for upload/download
func (service *Service) AvailableBandwidth(ctx context.Context) (_ int64, err error) {
defer mon.Task()(&ctx)(&err)
usage, err := service.usageDB.MonthSummary(ctx)
usage, err := service.usageDB.MonthSummary(ctx, time.Now())
if err != nil {
return 0, Error.Wrap(err)
}

13
storagenode/storagenodedb/bandwidthdb.go
View File

@ -48,7 +48,7 @@ func (db *bandwidthDB) Add(ctx context.Context, satelliteID storj.NodeID, action
if beginningOfMonth.Equal(db.usedSince) {
db.usedSpace += amount
} else if beginningOfMonth.After(db.usedSince) {
usage, err := db.Summary(ctx, beginningOfMonth, time.Now().UTC())
usage, err := db.Summary(ctx, beginningOfMonth, time.Now())
if err != nil {
return err
}
@ -60,17 +60,18 @@ func (db *bandwidthDB) Add(ctx context.Context, satelliteID storj.NodeID, action
}
// MonthSummary returns summary of the current months bandwidth usages
func (db *bandwidthDB) MonthSummary(ctx context.Context) (_ int64, err error) {
func (db *bandwidthDB) MonthSummary(ctx context.Context, now time.Time) (_ int64, err error) {
defer mon.Task()(&ctx)(&err)
db.usedMu.RLock()
beginningOfMonth := getBeginningOfMonth(time.Now().UTC())
beginningOfMonth := getBeginningOfMonth(now)
if beginningOfMonth.Equal(db.usedSince) {
defer db.usedMu.RUnlock()
return db.usedSpace, nil
}
db.usedMu.RUnlock()
usage, err := db.Summary(ctx, beginningOfMonth, time.Now())
usage, err := db.Summary(ctx, beginningOfMonth, now)
if err != nil {
return 0, err
}
@ -435,6 +436,6 @@ func (db *bandwidthDB) getDailyUsageRollups(ctx context.Context, cond string, ar
}
func getBeginningOfMonth(now time.Time) time.Time {
y, m, _ := now.Date()
return time.Date(y, m, 1, 0, 0, 0, 0, time.Now().UTC().Location())
y, m, _ := now.UTC().Date()
return time.Date(y, m, 1, 0, 0, 0, 0, time.UTC)
}