rollup interval fixes (#1739)
* set all intervals to UTC in rollupStats map, only delete latest day after both rollups * clean up usage of interval, use intervalEndTime rather than createdAt * change some variable names, add comments * add flag for tally deletion * adds deletetallies flag to testplanet * space * Removes println: * adds test for deletes false
This commit is contained in:
Cameron
Jennifer Li Johnson
parent
92c0ab5b8d
commit
9f6b010748
@ -488,6 +488,7 @@ func (planet *Planet) newSatellites(count int) ([]*satellite.Peer, error) {
|
||||
Rollup: rollup.Config{
|
||||
Interval: 2 * time.Minute,
|
||||
MaxAlphaUsage: 25 * memory.GB,
|
||||
DeleteTallies: true,
|
||||
},
|
||||
Mail: mailservice.Config{
|
||||
SMTPServerAddress: "smtp.mail.example.com:587",
|
||||
|
||||
@ -19,21 +19,24 @@ import (
|
||||
type Config struct {
|
||||
Interval time.Duration `help:"how frequently rollup should run" devDefault:"120s" releaseDefault:"24h"`
|
||||
MaxAlphaUsage memory.Size `help:"the bandwidth and storage usage limit for the alpha release" default:"25GB"`
|
||||
DeleteTallies bool `help:"option for deleting tallies after they are rolled up" default:"true"`
|
||||
}
|
||||
|
||||
// Service is the rollup service for totalling data on storage nodes on daily intervals
|
||||
type Service struct {
|
||||
logger *zap.Logger
|
||||
ticker *time.Ticker
|
||||
db accounting.DB
|
||||
logger *zap.Logger
|
||||
ticker *time.Ticker
|
||||
db accounting.DB
|
||||
deleteTallies bool
|
||||
}
|
||||
|
||||
// New creates a new rollup service
|
||||
func New(logger *zap.Logger, db accounting.DB, interval time.Duration) *Service {
|
||||
func New(logger *zap.Logger, db accounting.DB, interval time.Duration, deleteTallies bool) *Service {
|
||||
return &Service{
|
||||
logger: logger,
|
||||
ticker: time.NewTicker(interval),
|
||||
db: db,
|
||||
logger: logger,
|
||||
ticker: time.NewTicker(interval),
|
||||
db: db,
|
||||
deleteTallies: deleteTallies,
|
||||
}
|
||||
}
|
||||
|
||||
@ -66,26 +69,33 @@ func (r *Service) Rollup(ctx context.Context) error {
|
||||
if err != nil {
|
||||
return Error.Wrap(err)
|
||||
}
|
||||
if len(rollupStats) == 0 {
|
||||
r.logger.Info("RollupStats is empty after RollupStorage")
|
||||
}
|
||||
|
||||
err = r.RollupBW(ctx, lastRollup, rollupStats)
|
||||
if err != nil {
|
||||
return Error.Wrap(err)
|
||||
}
|
||||
|
||||
//remove the latest day (which we cannot know is complete), then push to DB
|
||||
latestTally = time.Date(latestTally.Year(), latestTally.Month(), latestTally.Day(), 0, 0, 0, 0, latestTally.Location())
|
||||
delete(rollupStats, latestTally)
|
||||
if len(rollupStats) == 0 {
|
||||
r.logger.Info("RollupStats is empty after RollupBW")
|
||||
r.logger.Info("RollupStats is empty")
|
||||
return nil
|
||||
}
|
||||
|
||||
err = r.db.SaveRollup(ctx, latestTally, rollupStats)
|
||||
if err != nil {
|
||||
return Error.Wrap(err)
|
||||
}
|
||||
// Delete already rolled up tallies
|
||||
err = r.db.DeleteRawBefore(ctx, latestTally)
|
||||
if err != nil {
|
||||
return Error.Wrap(err)
|
||||
|
||||
if r.deleteTallies {
|
||||
// Delete already rolled up tallies
|
||||
err = r.db.DeleteRawBefore(ctx, latestTally)
|
||||
if err != nil {
|
||||
return Error.Wrap(err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -102,12 +112,13 @@ func (r *Service) RollupStorage(ctx context.Context, lastRollup time.Time, rollu
|
||||
//loop through tallies and build Rollup
|
||||
for _, tallyRow := range tallies {
|
||||
node := tallyRow.NodeID
|
||||
if tallyRow.CreatedAt.After(latestTally) {
|
||||
latestTally = tallyRow.CreatedAt
|
||||
// tallyEndTime is the time the at rest tally was saved
|
||||
tallyEndTime := tallyRow.IntervalEndTime.UTC()
|
||||
if tallyEndTime.After(latestTally) {
|
||||
latestTally = tallyEndTime
|
||||
}
|
||||
//create or get AccoutingRollup day entry
|
||||
iDay := tallyRow.IntervalEndTime
|
||||
iDay = time.Date(iDay.Year(), iDay.Month(), iDay.Day(), 0, 0, 0, 0, iDay.Location())
|
||||
iDay := time.Date(tallyEndTime.Year(), tallyEndTime.Month(), tallyEndTime.Day(), 0, 0, 0, 0, tallyEndTime.Location())
|
||||
if rollupStats[iDay] == nil {
|
||||
rollupStats[iDay] = make(map[storj.NodeID]*accounting.Rollup)
|
||||
}
|
||||
@ -122,13 +133,6 @@ func (r *Service) RollupStorage(ctx context.Context, lastRollup time.Time, rollu
|
||||
r.logger.Info("rollupStorage no longer supports non-accounting.AtRest datatypes")
|
||||
}
|
||||
}
|
||||
//remove the latest day (which we cannot know is complete), then push to DB
|
||||
latestTally = time.Date(latestTally.Year(), latestTally.Month(), latestTally.Day(), 0, 0, 0, 0, latestTally.Location())
|
||||
delete(rollupStats, latestTally)
|
||||
if len(rollupStats) == 0 {
|
||||
r.logger.Info("Rollup only found tallies for today")
|
||||
return time.Now(), nil
|
||||
}
|
||||
|
||||
return latestTally, nil
|
||||
}
|
||||
@ -146,11 +150,12 @@ func (r *Service) RollupBW(ctx context.Context, lastRollup time.Time, rollupStat
|
||||
}
|
||||
for _, row := range bws {
|
||||
nodeID := row.NodeID
|
||||
if row.IntervalStart.After(latestTally) {
|
||||
latestTally = row.IntervalStart
|
||||
// interval is the time the bw order was saved
|
||||
interval := row.IntervalStart.UTC()
|
||||
if interval.After(latestTally) {
|
||||
latestTally = interval
|
||||
}
|
||||
day := row.IntervalStart
|
||||
day = time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, day.Location())
|
||||
day := time.Date(interval.Year(), interval.Month(), interval.Day(), 0, 0, 0, 0, interval.Location())
|
||||
if rollupStats[day] == nil {
|
||||
rollupStats[day] = make(map[storj.NodeID]*accounting.Rollup)
|
||||
}
|
||||
@ -174,11 +179,6 @@ func (r *Service) RollupBW(ctx context.Context, lastRollup time.Time, rollupStat
|
||||
r.logger.Info("delete order type")
|
||||
}
|
||||
}
|
||||
//remove the latest day (which we cannot know is complete), then push to DB
|
||||
latestTally = time.Date(latestTally.Year(), latestTally.Month(), latestTally.Day(), 0, 0, 0, 0, latestTally.Location())
|
||||
delete(rollupStats, latestTally)
|
||||
if len(rollupStats) == 0 {
|
||||
r.logger.Info("Rollup only found data for today")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -10,12 +10,14 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
|
||||
"storj.io/storj/internal/testcontext"
|
||||
"storj.io/storj/internal/testplanet"
|
||||
"storj.io/storj/pkg/accounting"
|
||||
"storj.io/storj/pkg/pb"
|
||||
"storj.io/storj/pkg/storj"
|
||||
"storj.io/storj/satellite"
|
||||
)
|
||||
|
||||
type testData struct {
|
||||
@ -23,68 +25,128 @@ type testData struct {
|
||||
bwTotals map[storj.NodeID][]int64
|
||||
}
|
||||
|
||||
func TestRollup(t *testing.T) {
|
||||
func TestRollupNoDeletes(t *testing.T) {
|
||||
testplanet.Run(t, testplanet.Config{
|
||||
SatelliteCount: 1, StorageNodeCount: 10, UplinkCount: 0,
|
||||
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
||||
days := 5
|
||||
testData := createData(planet, days)
|
||||
Reconfigure: testplanet.Reconfigure{
|
||||
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
|
||||
config.Rollup.DeleteTallies = false
|
||||
},
|
||||
},
|
||||
},
|
||||
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
||||
days := 5
|
||||
testData := createData(planet, days)
|
||||
|
||||
// Set timestamp back by the number of days we want to save
|
||||
timestamp := time.Now().UTC().AddDate(0, 0, -1*days)
|
||||
start := timestamp
|
||||
// Set timestamp back by the number of days we want to save
|
||||
timestamp := time.Now().UTC().AddDate(0, 0, -1*days)
|
||||
start := timestamp
|
||||
|
||||
for i := 0; i < days; i++ {
|
||||
err := planet.Satellites[0].DB.Accounting().SaveAtRestRaw(ctx, timestamp, timestamp, testData[i].nodeData)
|
||||
require.NoError(t, err)
|
||||
err = saveBW(ctx, planet, testData[i].bwTotals, timestamp)
|
||||
require.NoError(t, err)
|
||||
for i := 0; i < days; i++ {
|
||||
err := planet.Satellites[0].DB.Accounting().SaveAtRestRaw(ctx, timestamp, timestamp, testData[i].nodeData)
|
||||
require.NoError(t, err)
|
||||
err = saveBW(ctx, planet, testData[i].bwTotals, timestamp)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = planet.Satellites[0].Accounting.Rollup.Rollup(ctx)
|
||||
require.NoError(t, err)
|
||||
err = planet.Satellites[0].Accounting.Rollup.Rollup(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that RollupStorage deleted all raws except for today's
|
||||
raw, err := planet.Satellites[0].DB.Accounting().GetRaw(ctx)
|
||||
require.NoError(t, err)
|
||||
for _, r := range raw {
|
||||
assert.Equal(t, r.IntervalEndTime.UTC().Truncate(time.Second), timestamp.Truncate(time.Second))
|
||||
if r.DataType == accounting.AtRest {
|
||||
assert.Equal(t, testData[i].nodeData[r.NodeID], r.DataTotal)
|
||||
} else {
|
||||
assert.Equal(t, testData[i].bwTotals[r.NodeID][r.DataType], int64(r.DataTotal))
|
||||
// Advance time by 24 hours
|
||||
timestamp = timestamp.Add(time.Hour * 24)
|
||||
end := timestamp
|
||||
|
||||
// rollup.RollupRaws cuts off the hr/min/sec before saving, we need to do the same when querying
|
||||
start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, start.Location())
|
||||
end = time.Date(end.Year(), end.Month(), end.Day(), 0, 0, 0, 0, end.Location())
|
||||
|
||||
rows, err := planet.Satellites[0].DB.Accounting().QueryPaymentInfo(ctx, start, end)
|
||||
require.NoError(t, err)
|
||||
if i == 0 { // we need at least two days for rollup to work
|
||||
assert.Equal(t, 0, len(rows))
|
||||
continue
|
||||
}
|
||||
// the number of rows should be number of nodes
|
||||
|
||||
assert.Equal(t, len(planet.StorageNodes), len(rows))
|
||||
|
||||
// verify data is correct
|
||||
for _, r := range rows {
|
||||
totals := expectedTotals(testData, r.NodeID, i)
|
||||
assert.Equal(t, int64(totals[0]), r.PutTotal)
|
||||
assert.Equal(t, int64(totals[1]), r.GetTotal)
|
||||
assert.Equal(t, int64(totals[2]), r.GetAuditTotal)
|
||||
assert.Equal(t, int64(totals[3]), r.GetRepairTotal)
|
||||
assert.Equal(t, totals[4], r.AtRestTotal)
|
||||
assert.NotEmpty(t, r.Wallet)
|
||||
}
|
||||
}
|
||||
|
||||
// Advance time by 24 hours
|
||||
timestamp = timestamp.Add(time.Hour * 24)
|
||||
end := timestamp
|
||||
|
||||
// rollup.RollupRaws cuts off the hr/min/sec before saving, we need to do the same when querying
|
||||
start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, start.Location())
|
||||
end = time.Date(end.Year(), end.Month(), end.Day(), 0, 0, 0, 0, end.Location())
|
||||
|
||||
rows, err := planet.Satellites[0].DB.Accounting().QueryPaymentInfo(ctx, start, end)
|
||||
raw, err := planet.Satellites[0].DB.Accounting().GetRaw(ctx)
|
||||
require.NoError(t, err)
|
||||
if i == 0 { // we need at least two days for rollup to work
|
||||
assert.Equal(t, 0, len(rows))
|
||||
continue
|
||||
}
|
||||
// the number of rows should be number of nodes
|
||||
assert.Equal(t, days*len(planet.StorageNodes), len(raw))
|
||||
})
|
||||
}
|
||||
func TestRollupDeletes(t *testing.T) {
|
||||
testplanet.Run(t, testplanet.Config{
|
||||
SatelliteCount: 1, StorageNodeCount: 10, UplinkCount: 0},
|
||||
func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
|
||||
days := 5
|
||||
testData := createData(planet, days)
|
||||
|
||||
assert.Equal(t, len(planet.StorageNodes), len(rows))
|
||||
// Set timestamp back by the number of days we want to save
|
||||
timestamp := time.Now().UTC().AddDate(0, 0, -1*days)
|
||||
start := timestamp
|
||||
|
||||
// verify data is correct
|
||||
for _, r := range rows {
|
||||
totals := expectedTotals(testData, r.NodeID, i)
|
||||
assert.Equal(t, int64(totals[0]), r.PutTotal)
|
||||
assert.Equal(t, int64(totals[1]), r.GetTotal)
|
||||
assert.Equal(t, int64(totals[2]), r.GetAuditTotal)
|
||||
assert.Equal(t, int64(totals[3]), r.GetRepairTotal)
|
||||
assert.Equal(t, totals[4], r.AtRestTotal)
|
||||
assert.NotEmpty(t, r.Wallet)
|
||||
for i := 0; i < days; i++ {
|
||||
err := planet.Satellites[0].DB.Accounting().SaveAtRestRaw(ctx, timestamp, timestamp, testData[i].nodeData)
|
||||
require.NoError(t, err)
|
||||
err = saveBW(ctx, planet, testData[i].bwTotals, timestamp)
|
||||
require.NoError(t, err)
|
||||
|
||||
err = planet.Satellites[0].Accounting.Rollup.Rollup(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Assert that RollupStorage deleted all raws except for today's
|
||||
raw, err := planet.Satellites[0].DB.Accounting().GetRaw(ctx)
|
||||
require.NoError(t, err)
|
||||
for _, r := range raw {
|
||||
assert.Equal(t, r.IntervalEndTime.UTC().Truncate(time.Second), timestamp.Truncate(time.Second))
|
||||
if r.DataType == accounting.AtRest {
|
||||
assert.Equal(t, testData[i].nodeData[r.NodeID], r.DataTotal)
|
||||
} else {
|
||||
assert.Equal(t, testData[i].bwTotals[r.NodeID][r.DataType], int64(r.DataTotal))
|
||||
}
|
||||
}
|
||||
|
||||
// Advance time by 24 hours
|
||||
timestamp = timestamp.Add(time.Hour * 24)
|
||||
end := timestamp
|
||||
|
||||
// rollup.RollupRaws cuts off the hr/min/sec before saving, we need to do the same when querying
|
||||
start = time.Date(start.Year(), start.Month(), start.Day(), 0, 0, 0, 0, start.Location())
|
||||
end = time.Date(end.Year(), end.Month(), end.Day(), 0, 0, 0, 0, end.Location())
|
||||
|
||||
rows, err := planet.Satellites[0].DB.Accounting().QueryPaymentInfo(ctx, start, end)
|
||||
require.NoError(t, err)
|
||||
if i == 0 { // we need at least two days for rollup to work
|
||||
assert.Equal(t, 0, len(rows))
|
||||
continue
|
||||
}
|
||||
// the number of rows should be number of nodes
|
||||
|
||||
assert.Equal(t, len(planet.StorageNodes), len(rows))
|
||||
|
||||
// verify data is correct
|
||||
for _, r := range rows {
|
||||
totals := expectedTotals(testData, r.NodeID, i)
|
||||
assert.Equal(t, int64(totals[0]), r.PutTotal)
|
||||
assert.Equal(t, int64(totals[1]), r.GetTotal)
|
||||
assert.Equal(t, int64(totals[2]), r.GetAuditTotal)
|
||||
assert.Equal(t, int64(totals[3]), r.GetRepairTotal)
|
||||
assert.Equal(t, totals[4], r.AtRestTotal)
|
||||
assert.NotEmpty(t, r.Wallet)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
// expectedTotals sums test data up to, but not including the current day's
|
||||
|
||||
@ -398,7 +398,7 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, config *Config, ve
|
||||
{ // setup accounting
|
||||
log.Debug("Setting up accounting")
|
||||
peer.Accounting.Tally = tally.New(peer.Log.Named("tally"), peer.DB.Accounting(), peer.Metainfo.Service, peer.Overlay.Service, 0, config.Tally.Interval)
|
||||
peer.Accounting.Rollup = rollup.New(peer.Log.Named("rollup"), peer.DB.Accounting(), config.Rollup.Interval)
|
||||
peer.Accounting.Rollup = rollup.New(peer.Log.Named("rollup"), peer.DB.Accounting(), config.Rollup.Interval, config.Rollup.DeleteTallies)
|
||||
}
|
||||
|
||||
{ // setup inspector
|
||||
|
||||
Loading…
Reference in New Issue
Block a user