JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storagenode: fix B*h to bytes disk usage conversion

Browse Source 
The used space graph values are correct when a single satellite is
selected but wrong for 'All satellites'. This is related to the
queries for getting the individual disk usages for all satellites
per day and the summary and average for all satellites per day:

1. dividing the sum of at_rest_total by the total_hours is wrong.
Simply put, we were assuming that, for example (4/2)+(6/3) equals
to (4+6)/(2+3), assuming we had 4 and 6 at_rest_total values with
2 and 3 respective hours.

2. To get the average, we need to first find the sum of the
at_rest_total_bytes for each timestamp across all satellites
before taking the average of the sums instead of just taking the
average from the individual satellite values.

Closes https://github.com/storj/storj/issues/5519

Change-Id: Ib1314e238b695a6c1ecd9f9171ee86dd56bb3b24
This commit is contained in:
Clement Sam 2023-01-31 12:53:06 +00:00 committed by Storj Robot
parent 3146ad7f2e
commit 3d3f9d133a
4 changed files with 87 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
storagenode/console/service.go
View File

@ -354,15 +354,15 @@ func (s *Service) GetSatelliteData(ctx context.Context, satelliteID storj.NodeID
// Satellites represents consolidated data across all satellites. // Satellites represents consolidated data across all satellites.
type Satellites struct { type Satellites struct {
StorageDaily []storageusage.Stamp `json:"storageDaily"` StorageDaily []storageusage.StampGroup `json:"storageDaily"`
BandwidthDaily []bandwidth.UsageRollup `json:"bandwidthDaily"` BandwidthDaily []bandwidth.UsageRollup `json:"bandwidthDaily"`
StorageSummary float64 `json:"storageSummary"` StorageSummary float64 `json:"storageSummary"`
AverageUsageBytes float64 `json:"averageUsageBytes"` AverageUsageBytes float64 `json:"averageUsageBytes"`
BandwidthSummary int64 `json:"bandwidthSummary"` BandwidthSummary int64 `json:"bandwidthSummary"`
EgressSummary int64 `json:"egressSummary"` EgressSummary int64 `json:"egressSummary"`
IngressSummary int64 `json:"ingressSummary"` IngressSummary int64 `json:"ingressSummary"`
EarliestJoinedAt time.Time `json:"earliestJoinedAt"` EarliestJoinedAt time.Time `json:"earliestJoinedAt"`
Audits []Audits `json:"audits"` Audits []Audits `json:"audits"`
} }
// Audits represents audit, suspension and online scores of SNO across all satellites. // Audits represents audit, suspension and online scores of SNO across all satellites.

117
storagenode/storagenodedb/storageusage.go
View File

@ -86,7 +86,7 @@ func (db *storageUsageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID
var stamps []storageusage.Stamp var stamps []storageusage.Stamp
for rows.Next() { for rows.Next() {
var satellite storj.NodeID var satellite storj.NodeID
var atRestTotal, intervalInHours float64 var atRestTotal, intervalInHours sql.NullFloat64
var timestamp time.Time var timestamp time.Time
err = rows.Scan(&satellite, &atRestTotal, &intervalInHours, &timestamp) err = rows.Scan(&satellite, &atRestTotal, &intervalInHours, &timestamp)
@ -94,11 +94,16 @@ func (db *storageUsageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID
return nil, err return nil, err
} }
atRestTotalBytes := float64(0)
if intervalInHours.Float64 > 0 {
atRestTotalBytes = atRestTotal.Float64 / intervalInHours.Float64
}
stamps = append(stamps, storageusage.Stamp{ stamps = append(stamps, storageusage.Stamp{
SatelliteID: satellite, SatelliteID: satellite,
AtRestTotal: atRestTotal, AtRestTotal: atRestTotal.Float64,
AtRestTotalBytes: atRestTotal / intervalInHours, AtRestTotalBytes: atRestTotalBytes,
IntervalInHours: intervalInHours, IntervalInHours: intervalInHours.Float64,
IntervalStart: timestamp, IntervalStart: timestamp,
}) })
} }
@ -108,15 +113,16 @@ func (db *storageUsageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID
// GetDailyTotal returns daily storage usage stamps summed across all known satellites // GetDailyTotal returns daily storage usage stamps summed across all known satellites
// for provided time range. // for provided time range.
func (db *storageUsageDB) GetDailyTotal(ctx context.Context, from, to time.Time) (_ []storageusage.Stamp, err error) { func (db *storageUsageDB) GetDailyTotal(ctx context.Context, from, to time.Time) (_ []storageusage.StampGroup, err error) {
defer mon.Task()(&ctx)(&err) defer mon.Task()(&ctx)(&err)
// hour_interval = current row interval_end_time - previous row interval_end_time // hour_interval = current row interval_end_time - previous row interval_end_time
// Rows with 0-hour difference are assumed to be 24 hours. // Rows with 0-hour difference are assumed to be 24 hours.
query := `SELECT SUM(su3.at_rest_total), SUM(su3.hour_interval), su3.timestamp query := `SELECT SUM(su3.at_rest_total), SUM(su3.at_rest_total_bytes), su3.timestamp
FROM ( FROM (
SELECT su1.at_rest_total, SELECT
COALESCE( su1.at_rest_total,
su1.at_rest_total / COALESCE(
( (
CAST(strftime('%s', su1.interval_end_time) AS NUMERIC) CAST(strftime('%s', su1.interval_end_time) AS NUMERIC)
- -
@ -130,7 +136,7 @@ func (db *storageUsageDB) GetDailyTotal(ctx context.Context, from, to time.Time)
)) AS NUMERIC) )) AS NUMERIC)
) / 3600, ) / 3600,
24 24
) AS hour_interval, ) AS at_rest_total_bytes,
su1.timestamp su1.timestamp
FROM storage_usage su1 FROM storage_usage su1
WHERE ? <= su1.timestamp AND su1.timestamp <= ? WHERE ? <= su1.timestamp AND su1.timestamp <= ?
@ -146,20 +152,19 @@ func (db *storageUsageDB) GetDailyTotal(ctx context.Context, from, to time.Time)
err = errs.Combine(err, rows.Close()) err = errs.Combine(err, rows.Close())
}() }()
var stamps []storageusage.Stamp var stamps []storageusage.StampGroup
for rows.Next() { for rows.Next() {
var atRestTotal, intervalInHours float64 var atRestTotal, atRestTotalBytes sql.NullFloat64
var timestamp time.Time var timestamp time.Time
err = rows.Scan(&atRestTotal, &intervalInHours, &timestamp) err = rows.Scan(&atRestTotal, &atRestTotalBytes, &timestamp)
if err != nil { if err != nil {
return nil, err return nil, err
} }
stamps = append(stamps, storageusage.Stamp{ stamps = append(stamps, storageusage.StampGroup{
AtRestTotal: atRestTotal, AtRestTotal: atRestTotal.Float64,
AtRestTotalBytes: atRestTotal / intervalInHours, AtRestTotalBytes: atRestTotalBytes.Float64,
IntervalInHours: intervalInHours,
IntervalStart: timestamp, IntervalStart: timestamp,
}) })
} }
@ -172,29 +177,33 @@ func (db *storageUsageDB) Summary(ctx context.Context, from, to time.Time) (_, _
defer mon.Task()(&ctx, from, to)(&err) defer mon.Task()(&ctx, from, to)(&err)
var summary, averageUsageInBytes sql.NullFloat64 var summary, averageUsageInBytes sql.NullFloat64
query := `SELECT SUM(su3.at_rest_total), AVG(su3.at_rest_total_bytes) query := `SELECT SUM(at_rest_total), AVG(at_rest_total_bytes)
FROM ( FROM (
SELECT SELECT
at_rest_total, SUM(su1.at_rest_total) AS at_rest_total,
at_rest_total / ( SUM(
COALESCE( su1.at_rest_total / (
( COALESCE(
CAST(strftime('%s', su1.interval_end_time) AS NUMERIC) (
- CAST(strftime('%s', su1.interval_end_time) AS NUMERIC)
CAST(strftime('%s', ( -
SELECT interval_end_time CAST(strftime('%s', (
FROM storage_usage su2 SELECT interval_end_time
WHERE su2.satellite_id = su1.satellite_id FROM storage_usage su2
AND su2.timestamp < su1.timestamp WHERE su2.satellite_id = su1.satellite_id
ORDER BY su2.timestamp DESC AND su2.timestamp < su1.timestamp
LIMIT 1 ORDER BY su2.timestamp DESC
)) AS NUMERIC) LIMIT 1
) / 3600, )) AS NUMERIC)
24 ) / 3600,
24
)
) )
) AS at_rest_total_bytes ) AS at_rest_total_bytes,
su1.timestamp
FROM storage_usage su1 FROM storage_usage su1
WHERE ? <= timestamp AND timestamp <= ? WHERE ? <= su1.timestamp AND su1.timestamp <= ?
GROUP BY timestamp
) as su3` ) as su3`
err = db.QueryRowContext(ctx, query, from.UTC(), to.UTC()).Scan(&summary, &averageUsageInBytes) err = db.QueryRowContext(ctx, query, from.UTC(), to.UTC()).Scan(&summary, &averageUsageInBytes)
@ -209,27 +218,29 @@ func (db *storageUsageDB) SatelliteSummary(ctx context.Context, satelliteID stor
query := `SELECT SUM(su3.at_rest_total), AVG(su3.at_rest_total_bytes) query := `SELECT SUM(su3.at_rest_total), AVG(su3.at_rest_total_bytes)
FROM ( FROM (
SELECT SELECT
at_rest_total, su1.at_rest_total,
at_rest_total / ( (
COALESCE( su1.at_rest_total / (
( COALESCE(
CAST(strftime('%s', su1.interval_end_time) AS NUMERIC) (
- CAST(strftime('%s', su1.interval_end_time) AS NUMERIC)
CAST(strftime('%s', ( -
SELECT interval_end_time CAST(strftime('%s', (
FROM storage_usage su2 SELECT interval_end_time
WHERE su2.satellite_id = su1.satellite_id FROM storage_usage su2
AND su2.timestamp < su1.timestamp WHERE su2.satellite_id = su1.satellite_id
ORDER BY su2.timestamp DESC AND su2.timestamp < su1.timestamp
LIMIT 1 ORDER BY su2.timestamp DESC
)) AS NUMERIC) LIMIT 1
) / 3600, )) AS NUMERIC)
24 ) / 3600,
24
)
) )
) AS at_rest_total_bytes ) AS at_rest_total_bytes
FROM storage_usage su1 FROM storage_usage su1
WHERE satellite_id = ? WHERE su1.satellite_id = ?
AND ? <= timestamp AND timestamp <= ? AND ? <= su1.timestamp AND su1.timestamp <= ?
) as su3` ) as su3`
err = db.QueryRowContext(ctx, query, satelliteID, from.UTC(), to.UTC()).Scan(&summary, &averageUsageInBytes) err = db.QueryRowContext(ctx, query, satelliteID, from.UTC(), to.UTC()).Scan(&summary, &averageUsageInBytes)

14
storagenode/storageusage/storageusage.go
View File

@ -21,7 +21,7 @@ type DB interface {
GetDaily(ctx context.Context, satelliteID storj.NodeID, from, to time.Time) ([]Stamp, error) GetDaily(ctx context.Context, satelliteID storj.NodeID, from, to time.Time) ([]Stamp, error)
// GetDailyTotal returns daily storage usage stamps summed across all known satellites // GetDailyTotal returns daily storage usage stamps summed across all known satellites
// for provided time range // for provided time range
GetDailyTotal(ctx context.Context, from, to time.Time) ([]Stamp, error) GetDailyTotal(ctx context.Context, from, to time.Time) ([]StampGroup, error)
// Summary returns aggregated storage usage across all satellites. // Summary returns aggregated storage usage across all satellites.
Summary(ctx context.Context, from, to time.Time) (float64, float64, error) Summary(ctx context.Context, from, to time.Time) (float64, float64, error)
// SatelliteSummary returns aggregated storage usage for a particular satellite. // SatelliteSummary returns aggregated storage usage for a particular satellite.
@ -45,3 +45,15 @@ type Stamp struct {
// (i.e. last interval_end_time) for the day // (i.e. last interval_end_time) for the day
IntervalEndTime time.Time `json:"-"` IntervalEndTime time.Time `json:"-"`
} }
// StampGroup is storage usage stamp for all satellites from interval start till next interval
// grouped by interval_start time.
type StampGroup struct {
// AtRestTotal is the bytes*hour disk space used at the IntervalEndTime.
AtRestTotal float64 `json:"atRestTotal"`
// AtRestTotalBytes is the AtRestTotal divided by the IntervalInHours.
AtRestTotalBytes float64 `json:"atRestTotalBytes"`
// IntervalStart represents one tally day
// TODO: rename to timestamp to match DB
IntervalStart time.Time `json:"intervalStart"`
}

2
storagenode/storageusage/storageusage_test.go
View File

@ -70,7 +70,7 @@ func TestStorageUsage(t *testing.T) {
averageBySatellite[satellite] = satelliteUsageBytes / float64(len(expectedDailyStamps[satellite])) averageBySatellite[satellite] = satelliteUsageBytes / float64(len(expectedDailyStamps[satellite]))
} }
averageUsage = totalUsageBytes / totalStamps averageUsage = totalUsageBytes / float64(len(expectedDailyStampsTotals))
storagenodedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db storagenode.DB) { storagenodedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db storagenode.DB) {
storageUsageDB := db.StorageUsage() storageUsageDB := db.StorageUsage()