From 34a439a99cd2faee01e67e35bae06b95ec45cf7c Mon Sep 17 00:00:00 2001 From: Cameron Date: Tue, 2 Apr 2019 23:55:24 -0400 Subject: [PATCH] Delete accounting raws after rollup (#1646) * enable deletion of raws * assert deletion of raws during TestRollupRaws * add some unit tests for DeleteRawBefore --- pkg/accounting/rollup/rollup.go | 9 +-- pkg/accounting/rollup/rollup_test.go | 85 +++++++++++++++++++--------- pkg/accounting/tally/tally_test.go | 41 ++++++++++++++ 3 files changed, 102 insertions(+), 33 deletions(-) diff --git a/pkg/accounting/rollup/rollup.go b/pkg/accounting/rollup/rollup.go index 6543433c5..adfb2751a 100644 --- a/pkg/accounting/rollup/rollup.go +++ b/pkg/accounting/rollup/rollup.go @@ -115,11 +115,6 @@ func (r *Rollup) RollupRaws(ctx context.Context) error { if err != nil { return Error.Wrap(err) } - //removed the rolled-up raws - var rolledUpRawsHaveBeenSaved bool - //todo: write files to disk or whatever we decide to do here - if rolledUpRawsHaveBeenSaved { - return Error.Wrap(r.db.DeleteRawBefore(ctx, latestTally)) - } - return nil + + return Error.Wrap(r.db.DeleteRawBefore(ctx, latestTally)) } diff --git a/pkg/accounting/rollup/rollup_test.go b/pkg/accounting/rollup/rollup_test.go index 67726e02f..efb11f113 100644 --- a/pkg/accounting/rollup/rollup_test.go +++ b/pkg/accounting/rollup/rollup_test.go @@ -8,35 +8,51 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "storj.io/storj/internal/testcontext" "storj.io/storj/internal/testplanet" + "storj.io/storj/pkg/accounting" "storj.io/storj/pkg/storj" ) +type testData struct { + nodeData map[storj.NodeID]float64 + bwTotals map[storj.NodeID][]int64 +} + func TestRollupRaws(t *testing.T) { testplanet.Run(t, testplanet.Config{ SatelliteCount: 1, StorageNodeCount: 10, UplinkCount: 0, }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { days := 5 - atRest := float64(5000) - bw := []int64{1000, 2000, 3000, 4000} - - nodeData, bwTotals := createData(planet, atRest, bw) + testData := createData(planet, days) // Set timestamp back by the number of days we want to save timestamp := time.Now().UTC().AddDate(0, 0, -1*days) start := timestamp - for i := 0; i <= days; i++ { - err := planet.Satellites[0].DB.Accounting().SaveAtRestRaw(ctx, timestamp, timestamp, nodeData) - assert.NoError(t, err) + for i := 0; i < days; i++ { + err := planet.Satellites[0].DB.Accounting().SaveAtRestRaw(ctx, timestamp, timestamp, testData[i].nodeData) + require.NoError(t, err) - err = planet.Satellites[0].DB.Accounting().SaveBWRaw(ctx, timestamp, timestamp, bwTotals) - assert.NoError(t, err) + err = planet.Satellites[0].DB.Accounting().SaveBWRaw(ctx, timestamp, timestamp, testData[i].bwTotals) + require.NoError(t, err) err = planet.Satellites[0].Accounting.Rollup.RollupRaws(ctx) - assert.NoError(t, err) + require.NoError(t, err) + + // Assert that RollupRaws deleted all raws except for today's + raw, err := planet.Satellites[0].DB.Accounting().GetRaw(ctx) + require.NoError(t, err) + for _, r := range raw { + assert.Equal(t, r.IntervalEndTime.UTC().Truncate(time.Second), timestamp.Truncate(time.Second)) + if r.DataType == accounting.AtRest { + assert.Equal(t, testData[i].nodeData[r.NodeID], r.DataTotal) + } else { + assert.Equal(t, testData[i].bwTotals[r.NodeID][r.DataType], int64(r.DataTotal)) + } + } // Advance time by 24 hours timestamp = timestamp.Add(time.Hour * 24) @@ -47,36 +63,53 @@ func TestRollupRaws(t *testing.T) { end = time.Date(end.Year(), end.Month(), end.Day(), 0, 0, 0, 0, end.Location()) rows, err := planet.Satellites[0].DB.Accounting().QueryPaymentInfo(ctx, start, end) - assert.NoError(t, err) + require.NoError(t, err) if i == 0 { // we need at least two days for rollup to work assert.Equal(t, 0, len(rows)) continue } // the number of rows should be number of nodes + assert.Equal(t, len(planet.StorageNodes), len(rows)) // verify data is correct for _, r := range rows { - i := int64(i) - assert.Equal(t, i*bw[0], r.PutTotal) - assert.Equal(t, i*bw[1], r.GetTotal) - assert.Equal(t, i*bw[2], r.GetAuditTotal) - assert.Equal(t, i*bw[3], r.GetRepairTotal) - assert.Equal(t, float64(i)*atRest, r.AtRestTotal) - assert.NotNil(t, nodeData[r.NodeID]) + totals := expectedTotals(testData, r.NodeID, i) + assert.Equal(t, int64(totals[0]), r.PutTotal) + assert.Equal(t, int64(totals[1]), r.GetTotal) + assert.Equal(t, int64(totals[2]), r.GetAuditTotal) + assert.Equal(t, int64(totals[3]), r.GetRepairTotal) + assert.Equal(t, totals[4], r.AtRestTotal) assert.NotEmpty(t, r.Wallet) } } }) } -func createData(planet *testplanet.Planet, atRest float64, bw []int64) (nodeData map[storj.NodeID]float64, bwTotals map[storj.NodeID][]int64) { - nodeData = make(map[storj.NodeID]float64) - bwTotals = make(map[storj.NodeID][]int64) - for _, n := range planet.StorageNodes { - id := n.Identity.ID - nodeData[id] = atRest - bwTotals[id] = bw +// expectedTotals sums test data up to, but not including the current day's +func expectedTotals(data []testData, id storj.NodeID, currentDay int) []float64 { + totals := make([]float64, 5) + for i := 0; i < currentDay; i++ { + totals[0] += float64(data[i].bwTotals[id][0]) + totals[1] += float64(data[i].bwTotals[id][1]) + totals[2] += float64(data[i].bwTotals[id][2]) + totals[3] += float64(data[i].bwTotals[id][3]) + totals[4] += data[i].nodeData[id] } - return nodeData, bwTotals + return totals +} + +func createData(planet *testplanet.Planet, days int) []testData { + data := make([]testData, days) + for i := 0; i < days; i++ { + i := int64(i) + data[i].nodeData = make(map[storj.NodeID]float64) + data[i].bwTotals = make(map[storj.NodeID][]int64) + for _, n := range planet.StorageNodes { + id := n.Identity.ID + data[i].nodeData[id] = float64(i * 5000) + data[i].bwTotals[id] = []int64{i * 1000, i * 2000, i * 3000, i * 4000} + } + } + return data } diff --git a/pkg/accounting/tally/tally_test.go b/pkg/accounting/tally/tally_test.go index 6247c418b..f5d81aafc 100644 --- a/pkg/accounting/tally/tally_test.go +++ b/pkg/accounting/tally/tally_test.go @@ -13,9 +13,11 @@ import ( "storj.io/storj/internal/testcontext" "storj.io/storj/internal/testplanet" + "storj.io/storj/internal/teststorj" "storj.io/storj/pkg/bwagreement/testbwagreement" "storj.io/storj/pkg/pb" "storj.io/storj/pkg/piecestore/psserver/psdb" + "storj.io/storj/pkg/storj" "storj.io/storj/satellite" ) @@ -54,6 +56,45 @@ func TestQueryWithBw(t *testing.T) { }) } +func TestDeleteRawBefore(t *testing.T) { + tests := []struct { + createdAt time.Time + eraseBefore time.Time + expectedRaws int + }{ + { + createdAt: time.Now(), + eraseBefore: time.Now(), + expectedRaws: 1, + }, + { + createdAt: time.Now(), + eraseBefore: time.Now().Add(24 * time.Hour), + expectedRaws: 0, + }, + } + + for _, tt := range tests { + testplanet.Run(t, testplanet.Config{ + SatelliteCount: 1, StorageNodeCount: 0, UplinkCount: 0, + }, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) { + id := teststorj.NodeIDFromBytes([]byte{}) + nodeData := make(map[storj.NodeID]float64) + nodeData[id] = float64(1000) + + err := planet.Satellites[0].DB.Accounting().SaveAtRestRaw(ctx, tt.createdAt, tt.createdAt, nodeData) + require.NoError(t, err) + + err = planet.Satellites[0].DB.Accounting().DeleteRawBefore(ctx, tt.eraseBefore) + require.NoError(t, err) + + raws, err := planet.Satellites[0].DB.Accounting().GetRaw(ctx) + require.NoError(t, err) + assert.Len(t, raws, tt.expectedRaws) + }) + } +} + func sendGeneratedAgreements(ctx context.Context, t *testing.T, db satellite.DB, planet *testplanet.Planet) { satID := planet.Satellites[0].Identity upID := planet.Uplinks[0].Identity