diff --git a/storagenode/console/consoleapi/storagenode_test.go b/storagenode/console/consoleapi/storagenode_test.go index f3d16e85d..ff1cf4a39 100644 --- a/storagenode/console/consoleapi/storagenode_test.go +++ b/storagenode/console/consoleapi/storagenode_test.go @@ -22,7 +22,7 @@ import ( "storj.io/storj/storagenode/payouts/estimatedpayouts" "storj.io/storj/storagenode/pricing" "storj.io/storj/storagenode/reputation" - "storj.io/storj/storagenode/storageusage" + "storj.io/storj/storagenode/storagenodedb/storagenodedbtest" ) var ( @@ -80,7 +80,7 @@ func TestStorageNodeApi(t *testing.T) { var satellites []storj.NodeID satellites = append(satellites, satellite.ID()) - stamps, _ := makeStorageUsageStamps(satellites) + stamps := storagenodedbtest.MakeStorageUsageStamps(satellites, 30, time.Now().UTC()) err := storageusagedb.Store(ctx, stamps) require.NoError(t, err) @@ -139,30 +139,3 @@ func TestStorageNodeApi(t *testing.T) { }, ) } - -// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites. -// Creates one entry per day for 30 days with last date as beginning of provided endDate. -func makeStorageUsageStamps(satellites []storj.NodeID) ([]storageusage.Stamp, map[storj.NodeID]float64) { - var stamps []storageusage.Stamp - summary := make(map[storj.NodeID]float64) - - now := time.Now().UTC() - currentDay := now.Day() - - for _, satellite := range satellites { - for i := 0; i < currentDay; i++ { - intervalEndTime := now.Add(time.Hour * -24 * time.Duration(i)) - stamp := storageusage.Stamp{ - SatelliteID: satellite, - AtRestTotal: 30000000000000, - IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()), - IntervalEndTime: intervalEndTime, - } - - summary[satellite] += stamp.AtRestTotal - stamps = append(stamps, stamp) - } - } - - return stamps, summary -} diff --git a/storagenode/multinode/payout_test.go b/storagenode/multinode/payout_test.go index d62e509c9..319e50fac 100644 --- a/storagenode/multinode/payout_test.go +++ b/storagenode/multinode/payout_test.go @@ -24,7 +24,6 @@ import ( "storj.io/storj/storagenode/pricing" "storj.io/storj/storagenode/reputation" "storj.io/storj/storagenode/storagenodedb/storagenodedbtest" - "storj.io/storj/storagenode/storageusage" "storj.io/storj/storagenode/trust" ) @@ -173,7 +172,7 @@ func TestPayoutsEndpointEstimations(t *testing.T) { var satellites []storj.NodeID satellites = append(satellites, satelliteID) - stamps, _ := makeStorageUsageStamps(satellites) + stamps := storagenodedbtest.MakeStorageUsageStamps(satellites, 30, time.Now().UTC()) err = storageusagedb.Store(ctx, stamps) require.NoError(t, err) @@ -278,29 +277,3 @@ func TestPayoutsUndistributedEndpoint(t *testing.T) { require.EqualValues(t, 500, resp.Total) }) } - -// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites. -// Creates one entry per day for 30 days with last date as beginning of provided endDate. -func makeStorageUsageStamps(satellites []storj.NodeID) ([]storageusage.Stamp, map[storj.NodeID]float64) { - var stamps []storageusage.Stamp - summary := make(map[storj.NodeID]float64) - - now := time.Now().UTC().Day() - - for _, satellite := range satellites { - for i := 0; i < now; i++ { - intervalEndTime := time.Now().UTC().Add(time.Hour * -24 * time.Duration(i)) - stamp := storageusage.Stamp{ - SatelliteID: satellite, - AtRestTotal: 31234567891234, - IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()), - IntervalEndTime: intervalEndTime, - } - - summary[satellite] += stamp.AtRestTotal - stamps = append(stamps, stamp) - } - } - - return stamps, summary -} diff --git a/storagenode/storagenodedb/storagenodedbtest/common.go b/storagenode/storagenodedb/storagenodedbtest/common.go new file mode 100644 index 000000000..d4053ad8c --- /dev/null +++ b/storagenode/storagenodedb/storagenodedbtest/common.go @@ -0,0 +1,36 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package storagenodedbtest + +import ( + "math" + "time" + + "storj.io/common/storj" + "storj.io/common/testrand" + "storj.io/storj/storagenode/storageusage" +) + +// MakeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites. +// Creates one entry per day for 30 days with last date as beginning of provided endDate. +func MakeStorageUsageStamps(satellites []storj.NodeID, days int, endDate time.Time) []storageusage.Stamp { + var stamps []storageusage.Stamp + + startDate := time.Date(endDate.Year(), endDate.Month(), endDate.Day()-days, 0, 0, 0, 0, endDate.Location()) + for _, satellite := range satellites { + for i := 0; i < days; i++ { + h := testrand.Intn(24) + intervalEndTime := startDate.Add(time.Hour * 24 * time.Duration(i)).Add(time.Hour * time.Duration(h)) + stamp := storageusage.Stamp{ + SatelliteID: satellite, + AtRestTotal: math.Round(testrand.Float64n(1000)), + IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()), + IntervalEndTime: intervalEndTime, + } + stamps = append(stamps, stamp) + } + } + + return stamps +} diff --git a/storagenode/storagenodedb/storageusage.go b/storagenode/storagenodedb/storageusage.go index b5e7327b9..351313517 100644 --- a/storagenode/storagenodedb/storageusage.go +++ b/storagenode/storagenodedb/storageusage.go @@ -55,14 +55,24 @@ func (db *storageUsageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID // the at_rest_total is in bytes*hours, so to find the total number // of hours used to get the at_rest_total, we find the hour difference, // between the interval_end_time of a row and that of the previous row - // and divide the at_rest_total by the hour interval and multiply by 24 hours - // 24 hours to estimate the value for a 24hour time window. - // i.e. 24 * (at_rest_total/hour_difference), where the + // and divide the at_rest_total by the hour interval + // i.e. (at_rest_total/hour_difference), where the // hour_difference = current row interval_end_time - previous row interval_end_time // Rows with 0-hour difference are assumed to be 24 hours. query := `SELECT satellite_id, - COALESCE(24 * (at_rest_total / COALESCE((CAST(strftime('%s', interval_end_time) AS NUMERIC) - CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)) / 3600, 24)), at_rest_total), - timestamp + ( + at_rest_total + / + COALESCE( + ( + CAST(strftime('%s', interval_end_time) AS NUMERIC) + - + CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC) + ) / 3600, + 24 + ) + ) AS at_rest_total, + timestamp FROM storage_usage WHERE satellite_id = ? AND ? <= timestamp AND timestamp <= ? @@ -103,15 +113,25 @@ func (db *storageUsageDB) GetDailyTotal(ctx context.Context, from, to time.Time) // the at_rest_total is in bytes*hours, so to find the total number // of hours used to get the at_rest_total, we find the hour difference, // between the interval_end_time of a row and that of the previous row - // and divide the at_rest_total by the hour interval and multiply by 24 hours - // 24 hours to estimate the value for a 24hour time window. - // i.e. 24 * (at_rest_total/hour_difference), where the + // and divide the at_rest_total by the hour interval + // i.e. (at_rest_total/hour_difference), where the // hour_difference = current row interval_end_time - previous row interval_end_time // Rows with 0-hour difference are assumed to be 24 hours. query := `SELECT SUM(usages.at_rest_total), usages.timestamp FROM ( SELECT timestamp, - COALESCE(24 * (at_rest_total / COALESCE((CAST(strftime('%s', interval_end_time) AS NUMERIC) - CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)) / 3600, 24)), at_rest_total) AS at_rest_total + ( + at_rest_total + / + COALESCE( + ( + CAST(strftime('%s', interval_end_time) AS NUMERIC) + - + CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC) + ) / 3600, + 24 + ) + ) AS at_rest_total FROM storage_usage WHERE ? <= timestamp AND timestamp <= ? ) as usages @@ -150,9 +170,24 @@ func (db *storageUsageDB) Summary(ctx context.Context, from, to time.Time) (_ fl defer mon.Task()(&ctx, from, to)(&err) var summary sql.NullFloat64 - query := `SELECT SUM(at_rest_total) - FROM storage_usage - WHERE ? <= timestamp AND timestamp <= ?` + query := `SELECT SUM(usages.at_rest_total) + FROM ( + SELECT + ( + at_rest_total + / + COALESCE( + ( + CAST(strftime('%s', interval_end_time) AS NUMERIC) + - + CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC) + ) / 3600, + 24 + ) + ) AS at_rest_total + FROM storage_usage + WHERE ? <= timestamp AND timestamp <= ? + ) as usages` err = db.QueryRowContext(ctx, query, from.UTC(), to.UTC()).Scan(&summary) return summary.Float64, err @@ -163,10 +198,25 @@ func (db *storageUsageDB) SatelliteSummary(ctx context.Context, satelliteID stor defer mon.Task()(&ctx, satelliteID, from, to)(&err) var summary sql.NullFloat64 - query := `SELECT SUM(at_rest_total) - FROM storage_usage - WHERE satellite_id = ? - AND ? <= timestamp AND timestamp <= ?` + query := `SELECT SUM(usages.at_rest_total) + FROM ( + SELECT + ( + at_rest_total + / + COALESCE( + ( + CAST(strftime('%s', interval_end_time) AS NUMERIC) + - + CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC) + ) / 3600, + 24 + ) + ) AS at_rest_total + FROM storage_usage + WHERE satellite_id = ? + AND ? <= timestamp AND timestamp <= ? + ) as usages` err = db.QueryRowContext(ctx, query, satelliteID, from.UTC(), to.UTC()).Scan(&summary) return summary.Float64, err diff --git a/storagenode/storageusage/storageusage_test.go b/storagenode/storageusage/storageusage_test.go index 64d714f87..78372903d 100644 --- a/storagenode/storageusage/storageusage_test.go +++ b/storagenode/storageusage/storageusage_test.go @@ -24,7 +24,7 @@ func TestStorageUsage(t *testing.T) { days = 30 ) - now := time.Now() + now := time.Now().UTC() var satellites []storj.NodeID satelliteID := testrand.NodeID() @@ -34,15 +34,12 @@ func TestStorageUsage(t *testing.T) { satellites = append(satellites, testrand.NodeID()) } - stamps, summary := makeStorageUsageStamps(satellites, days, now) + stamps := storagenodedbtest.MakeStorageUsageStamps(satellites, days, now) var totalSummary float64 - for _, summ := range summary { - totalSummary += summ - } - expectedDailyStamps := make(map[storj.NodeID]map[time.Time]storageusage.Stamp) expectedDailyStampsTotals := make(map[time.Time]float64) + summary := make(map[storj.NodeID]float64) for _, stamp := range stamps { if expectedDailyStamps[stamp.SatelliteID] == nil { @@ -55,10 +52,10 @@ func TestStorageUsage(t *testing.T) { for _, stamp := range expectedDailyStamps[satellite] { intervalStart := stamp.IntervalStart.UTC() prevTimestamp := intervalStart.AddDate(0, 0, -1) - atRestTotal := stamp.AtRestTotal + atRestTotal := stamp.AtRestTotal / 24 if prevStamp, ok := expectedDailyStamps[satellite][prevTimestamp]; ok { diff := stamp.IntervalEndTime.UTC().Sub(prevStamp.IntervalEndTime.UTC()).Hours() - atRestTotal = (stamp.AtRestTotal / diff) * 24 + atRestTotal = stamp.AtRestTotal / diff } expectedDailyStamps[satellite][intervalStart] = storageusage.Stamp{ SatelliteID: satellite, @@ -67,6 +64,8 @@ func TestStorageUsage(t *testing.T) { IntervalEndTime: stamp.IntervalEndTime, } expectedDailyStampsTotals[intervalStart] += atRestTotal + summary[satellite] += atRestTotal + totalSummary += atRestTotal } } @@ -106,13 +105,13 @@ func TestStorageUsage(t *testing.T) { t.Run("summary satellite", func(t *testing.T) { summ, err := storageUsageDB.SatelliteSummary(ctx, satelliteID, time.Time{}, now) assert.NoError(t, err) - assert.Equal(t, summary[satelliteID], summ) + assert.Equal(t, roundFloat(summary[satelliteID]), roundFloat(summ)) }) t.Run("summary", func(t *testing.T) { summ, err := storageUsageDB.Summary(ctx, time.Time{}, now) assert.NoError(t, err) - assert.Equal(t, totalSummary, summ) + assert.Equal(t, roundFloat(totalSummary), roundFloat(summ)) }) }) } @@ -186,32 +185,6 @@ func TestZeroStorageUsage(t *testing.T) { }) } -// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites. -// Creates one entry per day for 30 days with last date as beginning of provided endDate. -func makeStorageUsageStamps(satellites []storj.NodeID, days int, endDate time.Time) ([]storageusage.Stamp, map[storj.NodeID]float64) { - var stamps []storageusage.Stamp - summary := make(map[storj.NodeID]float64) - - startDate := time.Date(endDate.Year(), endDate.Month(), endDate.Day()-days, 0, 0, 0, 0, endDate.Location()) - for _, satellite := range satellites { - for i := 0; i < days; i++ { - h := testrand.Intn(24) - intervalEndTime := startDate.Add(time.Hour * 24 * time.Duration(i)).Add(time.Hour * time.Duration(h)) - stamp := storageusage.Stamp{ - SatelliteID: satellite, - AtRestTotal: math.Round(testrand.Float64n(1000)), - IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()), - IntervalEndTime: intervalEndTime, - } - - summary[satellite] += stamp.AtRestTotal - stamps = append(stamps, stamp) - } - } - - return stamps, summary -} - // RoundFloat rounds float value to 5 decimal places. func roundFloat(value float64) float64 { return math.Round(value*100000) / 100000 diff --git a/web/multinode/src/app/components/storage/DiskSpaceChart.vue b/web/multinode/src/app/components/storage/DiskSpaceChart.vue index 9f522519d..7b0520345 100644 --- a/web/multinode/src/app/components/storage/DiskSpaceChart.vue +++ b/web/multinode/src/app/components/storage/DiskSpaceChart.vue @@ -3,7 +3,7 @@