JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

storagenode/storageusage: add summary, rename timestamp to interval_start (#2911)

Browse Source
This commit is contained in:
Yaroslav Vorobiov 2019-09-04 17:13:43 +03:00 committed by GitHub
parent 758f7cb3dd
commit f7403f97b0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 418 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
storagenode/nodestats/service.go
View File

@ -156,9 +156,9 @@ func fromSpaceUsageResponse(resp *pb.DailyStorageUsageResponse, satelliteID stor
for _, pbUsage := range resp.GetDailyStorageUsage() {
stamps = append(stamps, storageusage.Stamp{
SatelliteID: satelliteID,
AtRestTotal: pbUsage.AtRestTotal,
Timestamp: pbUsage.Timestamp,
SatelliteID: satelliteID,
AtRestTotal: pbUsage.AtRestTotal,
IntervalStart: pbUsage.Timestamp,
})
}

13
storagenode/storagenodedb/database.go
View File

@ -663,6 +663,19 @@ func (db *DB) Migration() *migrate.Migration {
);`,
},
},
{
Description: "Empty storage_usage table, rename storage_usage.timestamp to interval_start",
Version: 20,
Action: migrate.SQL{
`DROP TABLE storage_usage`,
`CREATE TABLE storage_usage (
satellite_id BLOB NOT NULL,
at_rest_total REAL NOT NUll,
interval_start TIMESTAMP NOT NULL,
PRIMARY KEY (satellite_id, interval_start)
)`,
},
},
},
}
}

75
storagenode/storagenodedb/storageusage.go
View File

@ -36,12 +36,12 @@ func (db *storageusageDB) Store(ctx context.Context, stamps []storageusage.Stamp
return nil
}
query := `INSERT OR REPLACE INTO storage_usage(satellite_id, at_rest_total, timestamp)
query := `INSERT OR REPLACE INTO storage_usage(satellite_id, at_rest_total, interval_start)
VALUES(?,?,?)`
return db.withTx(ctx, func(tx *sql.Tx) error {
for _, stamp := range stamps {
_, err = db.ExecContext(ctx, query, stamp.SatelliteID, stamp.AtRestTotal, stamp.Timestamp.UTC())
_, err = tx.ExecContext(ctx, query, stamp.SatelliteID, stamp.AtRestTotal, stamp.IntervalStart.UTC())
if err != nil {
return err
@ -57,15 +57,14 @@ func (db *storageusageDB) Store(ctx context.Context, stamps []storageusage.Stamp
func (db *storageusageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID, from, to time.Time) (_ []storageusage.Stamp, err error) {
defer mon.Task()(&ctx)(&err)
query := `SELECT *
query := `SELECT satellite_id,
SUM(at_rest_total),
interval_start
FROM storage_usage
WHERE timestamp IN (
SELECT MAX(timestamp)
FROM storage_usage
WHERE satellite_id = ?
AND ? <= timestamp AND timestamp <= ?
GROUP BY DATE(timestamp)
)`
WHERE satellite_id = ?
AND ? <= interval_start AND interval_start <= ?
GROUP BY DATE(interval_start)
ORDER BY interval_start`
rows, err := db.QueryContext(ctx, query, satelliteID, from.UTC(), to.UTC())
if err != nil {
@ -80,17 +79,17 @@ func (db *storageusageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID
for rows.Next() {
var satellite storj.NodeID
var atRestTotal float64
var timeStamp time.Time
var intervalStart time.Time
err = rows.Scan(&satellite, &atRestTotal, &timeStamp)
err = rows.Scan(&satellite, &atRestTotal, &intervalStart)
if err != nil {
return nil, err
}
stamps = append(stamps, storageusage.Stamp{
SatelliteID: satellite,
AtRestTotal: atRestTotal,
Timestamp: timeStamp,
SatelliteID: satellite,
AtRestTotal: atRestTotal,
IntervalStart: intervalStart,
})
}
@ -102,14 +101,11 @@ func (db *storageusageDB) GetDaily(ctx context.Context, satelliteID storj.NodeID
func (db *storageusageDB) GetDailyTotal(ctx context.Context, from, to time.Time) (_ []storageusage.Stamp, err error) {
defer mon.Task()(&ctx)(&err)
query := `SELECT SUM(at_rest_total), timestamp
query := `SELECT SUM(at_rest_total), interval_start
FROM storage_usage
WHERE timestamp IN (
SELECT MAX(timestamp)
FROM storage_usage
WHERE ? <= timestamp AND timestamp <= ?
GROUP BY DATE(timestamp), satellite_id
) GROUP BY DATE(timestamp)`
WHERE ? <= interval_start AND interval_start <= ?
GROUP BY DATE(interval_start)
ORDER BY interval_start`
rows, err := db.QueryContext(ctx, query, from.UTC(), to.UTC())
if err != nil {
@ -123,22 +119,49 @@ func (db *storageusageDB) GetDailyTotal(ctx context.Context, from, to time.Time)
var stamps []storageusage.Stamp
for rows.Next() {
var atRestTotal float64
var timeStamp time.Time
var intervalStart time.Time
err = rows.Scan(&atRestTotal, &timeStamp)
err = rows.Scan(&atRestTotal, &intervalStart)
if err != nil {
return nil, err
}
stamps = append(stamps, storageusage.Stamp{
AtRestTotal: atRestTotal,
Timestamp: timeStamp,
AtRestTotal: atRestTotal,
IntervalStart: intervalStart,
})
}
return stamps, nil
}
// Summary returns aggregated storage usage across all satellites.
func (db *storageusageDB) Summary(ctx context.Context, from, to time.Time) (_ float64, err error) {
defer mon.Task()(&ctx, from, to)(&err)
var summary sql.NullFloat64
query := `SELECT SUM(at_rest_total)
FROM storage_usage
WHERE ? <= interval_start AND interval_start <= ?`
err = db.QueryRowContext(ctx, query, from.UTC(), to.UTC()).Scan(&summary)
return summary.Float64, err
}
// SatelliteSummary returns aggregated storage usage for a particular satellite.
func (db *storageusageDB) SatelliteSummary(ctx context.Context, satelliteID storj.NodeID, from, to time.Time) (_ float64, err error) {
defer mon.Task()(&ctx, satelliteID, from, to)(&err)
var summary sql.NullFloat64
query := `SELECT SUM(at_rest_total)
FROM storage_usage
WHERE satellite_id = ?
AND ? <= interval_start AND interval_start <= ?`
err = db.QueryRowContext(ctx, query, satelliteID, from.UTC(), to.UTC()).Scan(&summary)
return summary.Float64, err
}
// withTx is a helper method which executes callback in transaction scope
func (db *storageusageDB) withTx(ctx context.Context, cb func(tx *sql.Tx) error) error {
tx, err := db.Begin()

184
storagenode/storagenodedb/testdata/sqlite.v20.sql vendored Normal file
View File

@ -0,0 +1,184 @@
-- table for keeping serials that need to be verified against
CREATE TABLE used_serial_ (
satellite_id BLOB NOT NULL,
serial_number BLOB NOT NULL,
expiration TIMESTAMP NOT NULL
);
-- primary key on satellite id and serial number
CREATE UNIQUE INDEX pk_used_serial_ ON used_serial_(satellite_id, serial_number);
-- expiration index to allow fast deletion
CREATE INDEX idx_used_serial_ ON used_serial_(expiration);
-- certificate table for storing uplink/satellite certificates
CREATE TABLE certificate (
cert_id INTEGER
);
-- table for storing piece meta info
CREATE TABLE pieceinfo_ (
satellite_id BLOB NOT NULL,
piece_id BLOB NOT NULL,
piece_size BIGINT NOT NULL,
piece_expiration TIMESTAMP,
order_limit BLOB NOT NULL,
uplink_piece_hash BLOB NOT NULL,
uplink_cert_id INTEGER NOT NULL,
deletion_failed_at TIMESTAMP,
piece_creation TIMESTAMP NOT NULL,
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
);
-- primary key by satellite id and piece id
CREATE UNIQUE INDEX pk_pieceinfo_ ON pieceinfo_(satellite_id, piece_id);
-- fast queries for expiration for pieces that have one
CREATE INDEX idx_pieceinfo__expiration ON pieceinfo_(piece_expiration) WHERE piece_expiration IS NOT NULL;
-- table for storing bandwidth usage
CREATE TABLE bandwidth_usage (
satellite_id BLOB NOT NULL,
action INTEGER NOT NULL,
amount BIGINT NOT NULL,
created_at TIMESTAMP NOT NULL
);
CREATE INDEX idx_bandwidth_usage_satellite ON bandwidth_usage(satellite_id);
CREATE INDEX idx_bandwidth_usage_created ON bandwidth_usage(created_at);
-- table for storing all unsent orders
CREATE TABLE unsent_order (
satellite_id BLOB NOT NULL,
serial_number BLOB NOT NULL,
order_limit_serialized BLOB NOT NULL,
order_serialized BLOB NOT NULL,
order_limit_expiration TIMESTAMP NOT NULL,
uplink_cert_id INTEGER NOT NULL,
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
);
CREATE UNIQUE INDEX idx_orders ON unsent_order(satellite_id, serial_number);
-- table for storing all sent orders
CREATE TABLE order_archive_ (
satellite_id BLOB NOT NULL,
serial_number BLOB NOT NULL,
order_limit_serialized BLOB NOT NULL,
order_serialized BLOB NOT NULL,
uplink_cert_id INTEGER NOT NULL,
status INTEGER NOT NULL,
archived_at TIMESTAMP NOT NULL,
FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
);
CREATE TABLE bandwidth_usage_rollups (
interval_start TIMESTAMP NOT NULL,
satellite_id BLOB NOT NULL,
action INTEGER NOT NULL,
amount BIGINT NOT NULL,
PRIMARY KEY ( interval_start, satellite_id, action )
);
-- table to hold expiration data (and only expirations. no other pieceinfo)
CREATE TABLE piece_expirations (
satellite_id BLOB NOT NULL,
piece_id BLOB NOT NULL,
piece_expiration TIMESTAMP NOT NULL, -- date when it can be deleted
deletion_failed_at TIMESTAMP,
PRIMARY KEY ( satellite_id, piece_id )
);
CREATE INDEX idx_piece_expirations_piece_expiration ON piece_expirations(piece_expiration);
CREATE INDEX idx_piece_expirations_deletion_failed_at ON piece_expirations(deletion_failed_at);
-- tables to store nodestats cache
CREATE TABLE reputation (
satellite_id BLOB NOT NULL,
uptime_success_count INTEGER NOT NULL,
uptime_total_count INTEGER NOT NULL,
uptime_reputation_alpha REAL NOT NULL,
uptime_reputation_beta REAL NOT NULL,
uptime_reputation_score REAL NOT NULL,
audit_success_count INTEGER NOT NULL,
audit_total_count INTEGER NOT NULL,
audit_reputation_alpha REAL NOT NULL,
audit_reputation_beta REAL NOT NULL,
audit_reputation_score REAL NOT NULL,
disqualified TIMESTAMP,
updated_at TIMESTAMP NOT NULL,
PRIMARY KEY (satellite_id)
);
CREATE TABLE storage_usage (
satellite_id BLOB NOT NULL,
at_rest_total REAL NOT NUll,
interval_start TIMESTAMP NOT NULL,
PRIMARY KEY (satellite_id, interval_start)
);
CREATE TABLE piece_space_used (
total INTEGER NOT NULL,
satellite_id BLOB
);
CREATE UNIQUE INDEX idx_piece_space_used_satellite_id ON piece_space_used(satellite_id);
INSERT INTO unsent_order VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',X'1eddef484b4c03f01332279032796972',X'0a101eddef484b4c03f0133227903279697212202b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf410001a201968996e7ef170a402fdfd88b6753df792c063c07c555905ffac9cd3cbd1c00022200ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac30002a20d00cf14f3c68b56321ace04902dec0484eb6f9098b22b31c6b3f82db249f191630643802420c08dfeb88e50510a8c1a5b9034a0c08dfeb88e50510a8c1a5b9035246304402204df59dc6f5d1bb7217105efbc9b3604d19189af37a81efbf16258e5d7db5549e02203bb4ead16e6e7f10f658558c22b59c3339911841e8dbaae6e2dea821f7326894',X'0a101eddef484b4c03f0133227903279697210321a47304502206d4c106ddec88140414bac5979c95bdea7de2e0ecc5be766e08f7d5ea36641a7022100e932ff858f15885ffa52d07e260c2c25d3861810ea6157956c1793ad0c906284','2019-04-01 16:01:35.9254586+00:00',1);
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',0,0,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',0,0,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6,'2019-04-01 18:51:24.1074772+00:00');
INSERT INTO bandwidth_usage VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6,'2019-04-01 20:51:24.1074772+00:00');
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',0,0);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',0,0);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',1,1);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',2,2);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',2,2);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',3,3);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',3,3);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',4,4);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',4,4);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5,5);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',5,5);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 18:00:00+00:00',X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',6,6);
INSERT INTO bandwidth_usage_rollups VALUES('2019-07-12 20:00:00+00:00',X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',6,6);
INSERT INTO pieceinfo_ VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',X'd5e757fd8d207d1c46583fb58330f803dc961b71147308ff75ff1e72a0df6b0b',1000,'2019-05-09 00:00:00.000000+00:00', X'', X'0a20d5e757fd8d207d1c46583fb58330f803dc961b71147308ff75ff1e72a0df6b0b120501020304051a47304502201c16d76ecd9b208f7ad9f1edf66ce73dce50da6bde6bbd7d278415099a727421022100ca730450e7f6506c2647516f6e20d0641e47c8270f58dde2bb07d1f5a3a45673',1,NULL,'epoch');
INSERT INTO pieceinfo_ VALUES(X'2b3a5863a41f25408a8f5348839d7a1361dbd886d75786bb139a8ca0bdf41000',X'd5e757fd8d207d1c46583fb58330f803dc961b71147308ff75ff1e72a0df6b0b',337,'2019-05-09 00:00:00.000000+00:00', X'', X'0a20d5e757fd8d207d1c46583fb58330f803dc961b71147308ff75ff1e72a0df6b0b120501020304051a483046022100e623cf4705046e2c04d5b42d5edbecb81f000459713ad460c691b3361817adbf022100993da2a5298bb88de6c35b2e54009d1bf306cda5d441c228aa9eaf981ceb0f3d',2,NULL,'epoch');
INSERT INTO piece_space_used (total) VALUES (1337);
INSERT INTO piece_space_used (total, satellite_id) VALUES (1337, X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000');
INSERT INTO reputation VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',1,1,1.0,1.0,1.0,1,1,1.0,1.0,1.0,'2019-07-19 20:00:00+00:00','2019-08-23 20:00:00+00:00');
-- NEW DATA --
INSERT INTO storage_usage VALUES(X'0ed28abb2813e184a1e98b0f6605c4911ea468c7e8433eb583e0fca7ceac3000',5.0,'2019-07-19 20:00:00+00:00');

14
storagenode/storageusage/storageusage.go
View File

@ -20,13 +20,15 @@ type DB interface {
// GetDailyTotal returns daily storage usage stamps summed across all known satellites
// for provided time range
GetDailyTotal(ctx context.Context, from, to time.Time) ([]Stamp, error)
// Summary returns aggregated storage usage across all satellites.
Summary(ctx context.Context, from, to time.Time) (float64, error)
// SatelliteSummary returns aggregated storage usage for a particular satellite.
SatelliteSummary(ctx context.Context, satelliteID storj.NodeID, from, to time.Time) (float64, error)
}
// Stamp is storage usage stamp for satellite at some point in time
// Stamp is storage usage stamp for satellite from interval start till next interval.
type Stamp struct {
SatelliteID storj.NodeID `json:"-"`
AtRestTotal float64 `json:"atRestTotal"`
Timestamp time.Time `json:"timestamp"`
SatelliteID storj.NodeID `json:"-"`
AtRestTotal float64 `json:"atRestTotal"`
IntervalStart time.Time `json:"intervalStart"`
}

161
storagenode/storageusage/storageusage_test.go Normal file
View File

@ -0,0 +1,161 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package storageusage_test
import (
"math"
"testing"
"time"
"github.com/stretchr/testify/assert"
"storj.io/storj/internal/testcontext"
"storj.io/storj/internal/testrand"
"storj.io/storj/pkg/storj"
"storj.io/storj/storagenode"
"storj.io/storj/storagenode/storagenodedb/storagenodedbtest"
"storj.io/storj/storagenode/storageusage"
)
func TestStorageUsage(t *testing.T) {
const (
satelliteNum = 10
days = 30
)
now := time.Now()
var satellites []storj.NodeID
satelliteID := testrand.NodeID()
satellites = append(satellites, satelliteID)
for i := 0; i < satelliteNum-1; i++ {
satellites = append(satellites, testrand.NodeID())
}
stamps, summary := makeStorageUsageStamps(satellites, days, now)
var totalSummary float64
for _, summ := range summary {
totalSummary += summ
}
dailyStamps := make(map[time.Time]storageusage.Stamp)
dailyStampsTotals := make(map[time.Time]float64)
for _, stamp := range stamps {
if stamp.SatelliteID == satelliteID {
dailyStamps[stamp.IntervalStart.UTC()] = stamp
}
dailyStampsTotals[stamp.IntervalStart.UTC()] += stamp.AtRestTotal
}
storagenodedbtest.Run(t, func(t *testing.T, db storagenode.DB) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
storageUsageDB := db.StorageUsage()
t.Run("test store", func(t *testing.T) {
err := storageUsageDB.Store(ctx, stamps)
assert.NoError(t, err)
})
t.Run("test get daily", func(t *testing.T) {
res, err := storageUsageDB.GetDaily(ctx, satelliteID, time.Time{}, now)
assert.NoError(t, err)
assert.NotNil(t, res)
assert.Equal(t, days, len(res))
for _, stamp := range res {
assert.Equal(t, satelliteID, stamp.SatelliteID)
assert.Equal(t, dailyStamps[stamp.IntervalStart].AtRestTotal, stamp.AtRestTotal)
}
})
t.Run("test get daily total", func(t *testing.T) {
res, err := storageUsageDB.GetDailyTotal(ctx, time.Time{}, now)
assert.NoError(t, err)
assert.NotNil(t, res)
assert.Equal(t, days, len(res))
for _, stamp := range res {
assert.Equal(t, dailyStampsTotals[stamp.IntervalStart], stamp.AtRestTotal)
}
})
t.Run("test summary satellite", func(t *testing.T) {
summ, err := storageUsageDB.SatelliteSummary(ctx, satelliteID, time.Time{}, now)
assert.NoError(t, err)
assert.Equal(t, summary[satelliteID], summ)
})
t.Run("test summary", func(t *testing.T) {
summ, err := storageUsageDB.Summary(ctx, time.Time{}, now)
assert.NoError(t, err)
assert.Equal(t, totalSummary, summ)
})
})
}
func TestEmptyStorageUsage(t *testing.T) {
storagenodedbtest.Run(t, func(t *testing.T, db storagenode.DB) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
var emptySummary float64
now := time.Now()
storageUsageDB := db.StorageUsage()
t.Run("test get daily", func(t *testing.T) {
res, err := storageUsageDB.GetDaily(ctx, storj.NodeID{}, time.Time{}, now)
assert.NoError(t, err)
assert.Nil(t, res)
})
t.Run("test get daily total", func(t *testing.T) {
res, err := storageUsageDB.GetDailyTotal(ctx, time.Time{}, now)
assert.NoError(t, err)
assert.Nil(t, res)
})
t.Run("test summary satellite", func(t *testing.T) {
summ, err := storageUsageDB.SatelliteSummary(ctx, storj.NodeID{}, time.Time{}, now)
assert.NoError(t, err)
assert.Equal(t, emptySummary, summ)
})
t.Run("test summary", func(t *testing.T) {
summ, err := storageUsageDB.Summary(ctx, time.Time{}, now)
assert.NoError(t, err)
assert.Equal(t, emptySummary, summ)
})
})
}
// makeStorageUsageStamps creates storage usage stamps and expected summaries for provided satellites.
// Creates one entry per day for 30 days with last date as beginning of provided endDate.
func makeStorageUsageStamps(satellites []storj.NodeID, days int, endDate time.Time) ([]storageusage.Stamp, map[storj.NodeID]float64) {
var stamps []storageusage.Stamp
summary := make(map[storj.NodeID]float64)
startDate := time.Date(endDate.Year(), endDate.Month(), endDate.Day()-days, 0, 0, 0, 0, endDate.Location())
for _, satellite := range satellites {
for i := 0; i < days; i++ {
stamp := storageusage.Stamp{
SatelliteID: satellite,
AtRestTotal: math.Round(testrand.Float64n(1000)),
IntervalStart: startDate.Add(time.Hour * 24 * time.Duration(i)),
}
summary[satellite] += stamp.AtRestTotal
stamps = append(stamps, stamp)
}
}
return stamps, summary
}