JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storagenode: use bytes instead of bytes*hour unit for used space graph

Browse Source 
Closes https://github.com/storj/storj/issues/5146

Change-Id: I1b135da81a68193b5b50c761088d79471ca3a2fe
This commit is contained in:
Clement Sam 2022-10-10 11:35:58 +00:00 committed by Clement Sam
parent ec9d63d0ca
commit d625eb85fd
13 changed files with 127 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
storagenode/console/consoleapi/storagenode_test.go
View File

@ -22,7 +22,7 @@ import (
"storj.io/storj/storagenode/payouts/estimatedpayouts"
"storj.io/storj/storagenode/pricing"
"storj.io/storj/storagenode/reputation"
"storj.io/storj/storagenode/storageusage"
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
)
var (
@ -80,7 +80,7 @@ func TestStorageNodeApi(t *testing.T) {
var satellites []storj.NodeID
satellites = append(satellites, satellite.ID())
stamps, _ := makeStorageUsageStamps(satellites)
stamps := storagenodedbtest.MakeStorageUsageStamps(satellites, 30, time.Now().UTC())
err := storageusagedb.Store(ctx, stamps)
require.NoError(t, err)
@ -139,30 +139,3 @@ func TestStorageNodeApi(t *testing.T) {
},
)
}
// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites.
// Creates one entry per day for 30 days with last date as beginning of provided endDate.
func makeStorageUsageStamps(satellites []storj.NodeID) ([]storageusage.Stamp, map[storj.NodeID]float64) {
var stamps []storageusage.Stamp
summary := make(map[storj.NodeID]float64)
now := time.Now().UTC()
currentDay := now.Day()
for _, satellite := range satellites {
for i := 0; i < currentDay; i++ {
intervalEndTime := now.Add(time.Hour * -24 * time.Duration(i))
stamp := storageusage.Stamp{
SatelliteID: satellite,
AtRestTotal: 30000000000000,
IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()),
IntervalEndTime: intervalEndTime,
}
summary[satellite] += stamp.AtRestTotal
stamps = append(stamps, stamp)
}
}
return stamps, summary
}

29
storagenode/multinode/payout_test.go
View File

@ -24,7 +24,6 @@ import (
"storj.io/storj/storagenode/pricing"
"storj.io/storj/storagenode/reputation"
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
"storj.io/storj/storagenode/storageusage"
"storj.io/storj/storagenode/trust"
)
@ -173,7 +172,7 @@ func TestPayoutsEndpointEstimations(t *testing.T) {
var satellites []storj.NodeID
satellites = append(satellites, satelliteID)
stamps, _ := makeStorageUsageStamps(satellites)
stamps := storagenodedbtest.MakeStorageUsageStamps(satellites, 30, time.Now().UTC())
err = storageusagedb.Store(ctx, stamps)
require.NoError(t, err)
@ -278,29 +277,3 @@ func TestPayoutsUndistributedEndpoint(t *testing.T) {
require.EqualValues(t, 500, resp.Total)
})
}
// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites.
// Creates one entry per day for 30 days with last date as beginning of provided endDate.
func makeStorageUsageStamps(satellites []storj.NodeID) ([]storageusage.Stamp, map[storj.NodeID]float64) {
var stamps []storageusage.Stamp
summary := make(map[storj.NodeID]float64)
now := time.Now().UTC().Day()
for _, satellite := range satellites {
for i := 0; i < now; i++ {
intervalEndTime := time.Now().UTC().Add(time.Hour * -24 * time.Duration(i))
stamp := storageusage.Stamp{
SatelliteID: satellite,
AtRestTotal: 31234567891234,
IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()),
IntervalEndTime: intervalEndTime,
}
summary[satellite] += stamp.AtRestTotal
stamps = append(stamps, stamp)
}
}
return stamps, summary
}

36
storagenode/storagenodedb/storagenodedbtest/common.go Normal file
View File

@ -0,0 +1,36 @@
// Copyright (C) 2022 Storj Labs, Inc.
// See LICENSE for copying information.
package storagenodedbtest
import (
"math"
"time"
"storj.io/common/storj"
"storj.io/common/testrand"
"storj.io/storj/storagenode/storageusage"
)
// MakeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites.
// Creates one entry per day for 30 days with last date as beginning of provided endDate.
func MakeStorageUsageStamps(satellites []storj.NodeID, days int, endDate time.Time) []storageusage.Stamp {
var stamps []storageusage.Stamp
startDate := time.Date(endDate.Year(), endDate.Month(), endDate.Day()-days, 0, 0, 0, 0, endDate.Location())
for _, satellite := range satellites {
for i := 0; i < days; i++ {
h := testrand.Intn(24)
intervalEndTime := startDate.Add(time.Hour * 24 * time.Duration(i)).Add(time.Hour * time.Duration(h))
stamp := storageusage.Stamp{
SatelliteID: satellite,
AtRestTotal: math.Round(testrand.Float64n(1000)),
IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()),
IntervalEndTime: intervalEndTime,
}
stamps = append(stamps, stamp)
}
}
return stamps
}

82
storagenode/storagenodedb/storageusage.go
View File

@ -55,14 +55,24 @@ func (db *storageUsageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID
// the at_rest_total is in bytes*hours, so to find the total number
// of hours used to get the at_rest_total, we find the hour difference,
// between the interval_end_time of a row and that of the previous row
// and divide the at_rest_total by the hour interval and multiply by 24 hours
// 24 hours to estimate the value for a 24hour time window.
// i.e. 24 * (at_rest_total/hour_difference), where the
// and divide the at_rest_total by the hour interval
// i.e. (at_rest_total/hour_difference), where the
// hour_difference = current row interval_end_time - previous row interval_end_time
// Rows with 0-hour difference are assumed to be 24 hours.
query := `SELECT satellite_id,
COALESCE(24 * (at_rest_total / COALESCE((CAST(strftime('%s', interval_end_time) AS NUMERIC) - CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)) / 3600, 24)), at_rest_total),
timestamp
(
at_rest_total
/
COALESCE(
(
CAST(strftime('%s', interval_end_time) AS NUMERIC)
-
CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)
) / 3600,
24
)
) AS at_rest_total,
timestamp
FROM storage_usage
WHERE satellite_id = ?
AND ? <= timestamp AND timestamp <= ?
@ -103,15 +113,25 @@ func (db *storageUsageDB) GetDailyTotal(ctx context.Context, from, to time.Time)
// the at_rest_total is in bytes*hours, so to find the total number
// of hours used to get the at_rest_total, we find the hour difference,
// between the interval_end_time of a row and that of the previous row
// and divide the at_rest_total by the hour interval and multiply by 24 hours
// 24 hours to estimate the value for a 24hour time window.
// i.e. 24 * (at_rest_total/hour_difference), where the
// and divide the at_rest_total by the hour interval
// i.e. (at_rest_total/hour_difference), where the
// hour_difference = current row interval_end_time - previous row interval_end_time
// Rows with 0-hour difference are assumed to be 24 hours.
query := `SELECT SUM(usages.at_rest_total), usages.timestamp
FROM (
SELECT timestamp,
COALESCE(24 * (at_rest_total / COALESCE((CAST(strftime('%s', interval_end_time) AS NUMERIC) - CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)) / 3600, 24)), at_rest_total) AS at_rest_total
(
at_rest_total
/
COALESCE(
(
CAST(strftime('%s', interval_end_time) AS NUMERIC)
-
CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)
) / 3600,
24
)
) AS at_rest_total
FROM storage_usage
WHERE ? <= timestamp AND timestamp <= ?
) as usages
@ -150,9 +170,24 @@ func (db *storageUsageDB) Summary(ctx context.Context, from, to time.Time) (_ fl
defer mon.Task()(&ctx, from, to)(&err)
var summary sql.NullFloat64
query := `SELECT SUM(at_rest_total)
FROM storage_usage
WHERE ? <= timestamp AND timestamp <= ?`
query := `SELECT SUM(usages.at_rest_total)
FROM (
SELECT
(
at_rest_total
/
COALESCE(
(
CAST(strftime('%s', interval_end_time) AS NUMERIC)
-
CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)
) / 3600,
24
)
) AS at_rest_total
FROM storage_usage
WHERE ? <= timestamp AND timestamp <= ?
) as usages`
err = db.QueryRowContext(ctx, query, from.UTC(), to.UTC()).Scan(&summary)
return summary.Float64, err
@ -163,10 +198,25 @@ func (db *storageUsageDB) SatelliteSummary(ctx context.Context, satelliteID stor
defer mon.Task()(&ctx, satelliteID, from, to)(&err)
var summary sql.NullFloat64
query := `SELECT SUM(at_rest_total)
FROM storage_usage
WHERE satellite_id = ?
AND ? <= timestamp AND timestamp <= ?`
query := `SELECT SUM(usages.at_rest_total)
FROM (
SELECT
(
at_rest_total
/
COALESCE(
(
CAST(strftime('%s', interval_end_time) AS NUMERIC)
-
CAST(strftime('%s', LAG(interval_end_time) OVER (PARTITION BY satellite_id ORDER BY interval_end_time)) AS NUMERIC)
) / 3600,
24
)
) AS at_rest_total
FROM storage_usage
WHERE satellite_id = ?
AND ? <= timestamp AND timestamp <= ?
) as usages`
err = db.QueryRowContext(ctx, query, satelliteID, from.UTC(), to.UTC()).Scan(&summary)
return summary.Float64, err

45
storagenode/storageusage/storageusage_test.go
View File

@ -24,7 +24,7 @@ func TestStorageUsage(t *testing.T) {
days = 30
)
now := time.Now()
now := time.Now().UTC()
var satellites []storj.NodeID
satelliteID := testrand.NodeID()
@ -34,15 +34,12 @@ func TestStorageUsage(t *testing.T) {
satellites = append(satellites, testrand.NodeID())
}
stamps, summary := makeStorageUsageStamps(satellites, days, now)
stamps := storagenodedbtest.MakeStorageUsageStamps(satellites, days, now)
var totalSummary float64
for _, summ := range summary {
totalSummary += summ
}
expectedDailyStamps := make(map[storj.NodeID]map[time.Time]storageusage.Stamp)
expectedDailyStampsTotals := make(map[time.Time]float64)
summary := make(map[storj.NodeID]float64)
for _, stamp := range stamps {
if expectedDailyStamps[stamp.SatelliteID] == nil {
@ -55,10 +52,10 @@ func TestStorageUsage(t *testing.T) {
for _, stamp := range expectedDailyStamps[satellite] {
intervalStart := stamp.IntervalStart.UTC()
prevTimestamp := intervalStart.AddDate(0, 0, -1)
atRestTotal := stamp.AtRestTotal
atRestTotal := stamp.AtRestTotal / 24
if prevStamp, ok := expectedDailyStamps[satellite][prevTimestamp]; ok {
diff := stamp.IntervalEndTime.UTC().Sub(prevStamp.IntervalEndTime.UTC()).Hours()
atRestTotal = (stamp.AtRestTotal / diff) * 24
atRestTotal = stamp.AtRestTotal / diff
}
expectedDailyStamps[satellite][intervalStart] = storageusage.Stamp{
SatelliteID: satellite,
@ -67,6 +64,8 @@ func TestStorageUsage(t *testing.T) {
IntervalEndTime: stamp.IntervalEndTime,
}
expectedDailyStampsTotals[intervalStart] += atRestTotal
summary[satellite] += atRestTotal
totalSummary += atRestTotal
}
}
@ -106,13 +105,13 @@ func TestStorageUsage(t *testing.T) {
t.Run("summary satellite", func(t *testing.T) {
summ, err := storageUsageDB.SatelliteSummary(ctx, satelliteID, time.Time{}, now)
assert.NoError(t, err)
assert.Equal(t, summary[satelliteID], summ)
assert.Equal(t, roundFloat(summary[satelliteID]), roundFloat(summ))
})
t.Run("summary", func(t *testing.T) {
summ, err := storageUsageDB.Summary(ctx, time.Time{}, now)
assert.NoError(t, err)
assert.Equal(t, totalSummary, summ)
assert.Equal(t, roundFloat(totalSummary), roundFloat(summ))
})
})
}
@ -186,32 +185,6 @@ func TestZeroStorageUsage(t *testing.T) {
})
}
// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites.
// Creates one entry per day for 30 days with last date as beginning of provided endDate.
func makeStorageUsageStamps(satellites []storj.NodeID, days int, endDate time.Time) ([]storageusage.Stamp, map[storj.NodeID]float64) {
var stamps []storageusage.Stamp
summary := make(map[storj.NodeID]float64)
startDate := time.Date(endDate.Year(), endDate.Month(), endDate.Day()-days, 0, 0, 0, 0, endDate.Location())
for _, satellite := range satellites {
for i := 0; i < days; i++ {
h := testrand.Intn(24)
intervalEndTime := startDate.Add(time.Hour * 24 * time.Duration(i)).Add(time.Hour * time.Duration(h))
stamp := storageusage.Stamp{
SatelliteID: satellite,
AtRestTotal: math.Round(testrand.Float64n(1000)),
IntervalStart: time.Date(intervalEndTime.Year(), intervalEndTime.Month(), intervalEndTime.Day(), 0, 0, 0, 0, intervalEndTime.Location()),
IntervalEndTime: intervalEndTime,
}
summary[satellite] += stamp.AtRestTotal
stamps = append(stamps, stamp)
}
}
return stamps, summary
}
// RoundFloat rounds float value to 5 decimal places.
func roundFloat(value float64) float64 {
return math.Round(value*100000) / 100000

4
web/multinode/src/app/components/storage/DiskSpaceChart.vue
View File

@ -3,7 +3,7 @@
<template>
<div class="chart">
<p class="disk-space-chart__data-dimension">{{ chartDataDimension }}*h</p>
<p class="disk-space-chart__data-dimension">{{ chartDataDimension }}</p>
<VChart
id="disk-space-chart"
:key="chartKey"
@ -86,7 +86,7 @@ export default class DiskSpaceChart extends BaseChart {
const dataPoint = new StampTooltip(this.allStamps[dataIndex]);
return `<div class='tooltip-body'>
<p class='tooltip-body__data'><b>${dataPoint.atRestTotal}*h</b></p>
<p class='tooltip-body__data'><b>${dataPoint.atRestTotal}</b></p>
<p class='tooltip-body__footer'>${dataPoint.date}</p>
</div>`;
}

2
web/multinode/src/app/views/bandwidth/BandwidthPage.vue
View File

@ -55,7 +55,7 @@
<div class="chart-container__title-area disk-space-title">
<p class="chart-container__title-area__title">Disk Space Used This Month</p>
</div>
<p class="chart-container__amount disk-space-amount"><b>{{ diskSpaceUsageSummary | bytesToBase10String }}*h</b></p>
<p class="chart-container__amount disk-space-amount"><b>{{ diskSpaceUsageSummary | bytesToBase10String }}*d</b></p>
<div ref="diskSpaceChart" class="chart-container__chart" onresize="recalculateChartDimensions()">
<disk-space-chart :height="diskSpaceChartHeight" :width="diskSpaceChartWidth" />
</div>

2
web/multinode/src/private/memory/size.ts
View File

@ -40,7 +40,7 @@ export class Size {
case _size >= SizeBreakpoints.KB * 2 / 3:
return `${parseFloat((size / SizeBreakpoints.KB).toFixed(decimals))}KB`;
default:
return `${size}B`;
return `${size ? size.toFixed(decimals) : size}B`;
}
}
}

6
web/multinode/tests/unit/components/views/__snapshots__/BandwidthPage.spec.ts.snap
View File

@ -28,7 +28,7 @@ exports[`BandwidthPage renders correctly 1`] = `
<div class="chart-container__title-area disk-space-title">
<p class="chart-container__title-area__title">Disk Space Used This Month</p>
</div>
<p class="chart-container__amount disk-space-amount"><b>0B*h</b></p>
<p class="chart-container__amount disk-space-amount"><b>0B*d</b></p>
<div onresize="recalculateChartDimensions()" class="chart-container__chart">
<disk-space-chart-stub width="0" height="0"></disk-space-chart-stub>
</div>
@ -68,7 +68,7 @@ exports[`BandwidthPage renders correctly with egress chart 1`] = `
<div class="chart-container__title-area disk-space-title">
<p class="chart-container__title-area__title">Disk Space Used This Month</p>
</div>
<p class="chart-container__amount disk-space-amount"><b>0B*h</b></p>
<p class="chart-container__amount disk-space-amount"><b>0B*d</b></p>
<div onresize="recalculateChartDimensions()" class="chart-container__chart">
<disk-space-chart-stub width="0" height="0"></disk-space-chart-stub>
</div>
@ -108,7 +108,7 @@ exports[`BandwidthPage renders correctly with ingress chart 1`] = `
<div class="chart-container__title-area disk-space-title">
<p class="chart-container__title-area__title">Disk Space Used This Month</p>
</div>
<p class="chart-container__amount disk-space-amount"><b>0B*h</b></p>
<p class="chart-container__amount disk-space-amount"><b>0B*d</b></p>
<div onresize="recalculateChartDimensions()" class="chart-container__chart">
<disk-space-chart-stub width="0" height="0"></disk-space-chart-stub>
</div>

4
web/storagenode/src/app/components/DiskSpaceChart.vue
View File

@ -3,7 +3,7 @@
<template>
<div class="chart">
<p class="disk-space-chart__data-dimension">{{ chartDataDimension }}*h</p>
<p class="disk-space-chart__data-dimension">{{ chartDataDimension }}</p>
<VChart
id="disk-space-chart"
:key="chartKey"
@ -93,7 +93,7 @@ export default class DiskSpaceChart extends BaseChart {
const dataPoint = new StampTooltip(this.allStamps[dataIndex]);
return `<div class='tooltip-body'>
<p class='tooltip-body__data'><b>${dataPoint.atRestTotal}*h</b></p>
<p class='tooltip-body__data'><b>${dataPoint.atRestTotal}</b></p>
<p class='tooltip-body__footer'>${dataPoint.date}</p>
</div>`;
}

2
web/storagenode/src/app/components/SNOContentFilling.vue
View File

@ -114,7 +114,7 @@
<div class="chart-container__title-area disk-space-title">
<p class="chart-container__title-area__title">Disk Space Used This Month</p>
</div>
<p class="chart-container__amount disk-space-amount"><b>{{ storageSummary }}*h</b></p>
<p class="chart-container__amount disk-space-amount"><b>{{ storageSummary }}*d</b></p>
<div ref="diskSpaceChart" class="chart-container__chart" onresize="recalculateChartDimensions()">
<DiskSpaceChart :height="diskSpaceChartHeight" :width="diskSpaceChartWidth" :is-dark-mode="isDarkMode" />
</div>

2
web/storagenode/src/private/memory/size.ts
View File

@ -40,7 +40,7 @@ export class Size {
case _size >= SizeBreakpoints.KB * 2 / 3:
return `${parseFloat((size / SizeBreakpoints.KB).toFixed(decimals))}KB`;
default:
return `${size}B`;
return `${size ? size.toFixed(decimals) : size}B`;
}
}
}

4
web/storagenode/tests/unit/components/__snapshots__/DiskStatChart.spec.ts.snap
View File

@ -3,7 +3,7 @@
exports[`DiskStatChart renders correctly 1`] = `
<div class="disk-stat-area">
<p class="disk-stat-area__title">Total Disk Space</p>
<p class="disk-stat-area__amount">1B</p>
<p class="disk-stat-area__amount">1.00B</p>
<doughnutchart-stub chartid="doughnut-chart" width="400" height="400" cssclasses="" plugins="" chartdata="[object Object]" class="disk-stat-area__chart"></doughnutchart-stub>
<div class="disk-stat-area__info-area">
<div class="disk-stat-area__info-area__item">
@ -18,7 +18,7 @@ exports[`DiskStatChart renders correctly 1`] = `
<div class="disk-stat-area__info-area__item__labels-area__circle free"></div>
<p class="disk-stat-area__info-area__item__labels-area__label">Free</p>
</div>
<p class="disk-stat-area__info-area__item__labels-area__amount">1B</p>
<p class="disk-stat-area__info-area__item__labels-area__amount">1.00B</p>
</div>
<div class="disk-stat-area__info-area__item">
<div class="disk-stat-area__info-area__item__labels-area">