diff --git a/storagenode/storagenodedb/database.go b/storagenode/storagenodedb/database.go index 2881cedb4..0ea4964a6 100644 --- a/storagenode/storagenodedb/database.go +++ b/storagenode/storagenodedb/database.go @@ -1971,6 +1971,14 @@ func (db *DB) Migration(ctx context.Context) *migrate.Migration { return nil }), }, + { + DB: &db.payoutDB.DB, + Description: "Assume distributed == paid for paystubs before 2020-12.", + Version: 51, + Action: migrate.SQL{ + `UPDATE paystubs SET distributed = paid WHERE period < '2020-12'`, + }, + }, }, } } diff --git a/storagenode/storagenodedb/migrations_test.go b/storagenode/storagenodedb/migrations_test.go index 0a64bfa7b..478815e96 100644 --- a/storagenode/storagenodedb/migrations_test.go +++ b/storagenode/storagenodedb/migrations_test.go @@ -21,12 +21,34 @@ import ( "storj.io/storj/storagenode/storagenodedb/testdata" ) +// insertOldData will insert any OldData from the MultiDBState into the +// appropriate rawDB. This prepares the rawDB for the test comparing schema and +// data and any changes to rows. +func insertOldData(ctx context.Context, mdbs *testdata.MultiDBState, rawDBs map[string]storagenodedb.DBContainer) error { + for dbName, dbState := range mdbs.DBStates { + if dbState.OldData == "" { + continue + } + + rawDB, ok := rawDBs[dbName] + if !ok { + return errs.New("Failed to find DB %s", dbName) + } + _, err := rawDB.GetDB().ExecContext(ctx, dbState.OldData) + if err != nil { + return err + } + } + return nil +} + // insertNewData will insert any NewData from the MultiDBState into the // appropriate rawDB. This prepares the rawDB for the test comparing schema and -// data. +// data. It will not insert NewData if OldData is set: the migration is expected +// to convert OldData into what NewData would insert. func insertNewData(ctx context.Context, mdbs *testdata.MultiDBState, rawDBs map[string]storagenodedb.DBContainer) error { for dbName, dbState := range mdbs.DBStates { - if dbState.NewData == "" { + if dbState.NewData == "" || dbState.OldData != "" { continue } @@ -104,6 +126,7 @@ func TestMigrate(t *testing.T) { db, err := storagenodedb.OpenNew(ctx, log, cfg) require.NoError(t, err) defer func() { require.NoError(t, db.Close()) }() + rawDBs := db.RawDatabases() // get migration for this database migrations := db.Migration(ctx) @@ -111,15 +134,17 @@ func TestMigrate(t *testing.T) { // the schema is different when migration step is before the step, cannot test the layout tag := fmt.Sprintf("#%d - v%d", i, step.Version) - // run migration up to a specific version - err := migrations.TargetVersion(step.Version).Run(ctx, log.Named("migrate")) - require.NoError(t, err, tag) - // find the matching expected version expected, ok := testdata.States.FindVersion(step.Version) require.True(t, ok) - rawDBs := db.RawDatabases() + // insert old data for any tables + err = insertOldData(ctx, expected, rawDBs) + require.NoError(t, err, tag) + + // run migration up to a specific version + err := migrations.TargetVersion(step.Version).Run(ctx, log.Named("migrate")) + require.NoError(t, err, tag) // insert data for new tables err = insertNewData(ctx, expected, rawDBs) diff --git a/storagenode/storagenodedb/payouts.go b/storagenode/storagenodedb/payouts.go index 7232a251d..72223fac5 100644 --- a/storagenode/storagenodedb/payouts.go +++ b/storagenode/storagenodedb/payouts.go @@ -155,29 +155,29 @@ func (db *payoutDB) GetPayStub(ctx context.Context, satelliteID storj.NodeID, pe func (db *payoutDB) AllPayStubs(ctx context.Context, period string) (_ []payouts.PayStub, err error) { defer mon.Task()(&ctx)(&err) - query := `SELECT - satellite_id, - created_at, - codes, - usage_at_rest, - usage_get, - usage_put, - usage_get_repair, - usage_put_repair, - usage_get_audit, - comp_at_rest, - comp_get, - comp_put, - comp_get_repair, - comp_put_repair, - comp_get_audit, - surge_percent, - held, - owed, - disposed, - paid, - distributed - FROM paystubs WHERE period = ?` + query := `SELECT + satellite_id, + created_at, + codes, + usage_at_rest, + usage_get, + usage_put, + usage_get_repair, + usage_put_repair, + usage_get_audit, + comp_at_rest, + comp_get, + comp_put, + comp_get_repair, + comp_put_repair, + comp_get_audit, + surge_percent, + held, + owed, + disposed, + paid, + distributed + FROM paystubs WHERE period = ?` rows, err := db.QueryContext(ctx, query, period) if err != nil { @@ -230,10 +230,10 @@ func (db *payoutDB) AllPayStubs(ctx context.Context, period string) (_ []payouts func (db *payoutDB) SatellitesHeldbackHistory(ctx context.Context, id storj.NodeID) (_ []payouts.HoldForPeriod, err error) { defer mon.Task()(&ctx)(&err) - query := `SELECT - period, - held - FROM paystubs WHERE satellite_id = ? ORDER BY period ASC` + query := `SELECT + period, + held + FROM paystubs WHERE satellite_id = ? ORDER BY period ASC` rows, err := db.QueryContext(ctx, query, id) if err != nil { @@ -351,9 +351,9 @@ func (db *payoutDB) StorePayment(ctx context.Context, payment payouts.Payment) ( func (db *payoutDB) SatellitesDisposedHistory(ctx context.Context, satelliteID storj.NodeID) (_ int64, err error) { defer mon.Task()(&ctx)(&err) - query := `SELECT - disposed - FROM paystubs WHERE satellite_id = ? ORDER BY period ASC` + query := `SELECT + disposed + FROM paystubs WHERE satellite_id = ? ORDER BY period ASC` rows, err := db.QueryContext(ctx, query, satelliteID) if err != nil { diff --git a/storagenode/storagenodedb/testdata/multidbsnapshot.go b/storagenode/storagenodedb/testdata/multidbsnapshot.go index 54cfce509..02e9f6abd 100644 --- a/storagenode/storagenodedb/testdata/multidbsnapshot.go +++ b/storagenode/storagenodedb/testdata/multidbsnapshot.go @@ -65,6 +65,7 @@ var States = MultiDBStates{ &v48, &v49, &v50, + &v51, }, } @@ -96,9 +97,12 @@ type DBStates map[string]*DBState // DBState allows you to define the desired state of the DB using SQl commands. // Both the SQl and NewData fields contains SQL that will be executed to create // the expected DB. The NewData SQL additionally will be executed on the testDB -// to ensure data is consistent. +// to ensure data is consistent. If OldData is not empty, it is executed on the +// testDB before the migration is run, and NewData is not run on the testDB. This +// is used to assert that a migration that modifies data runs as expected. type DBState struct { SQL string + OldData string NewData string } @@ -125,7 +129,7 @@ type DBSnapshot struct { } // LoadMultiDBSnapshot converts a MultiDBState into a MultiDBSnapshot. It -// executes the SQL and stores the shema and data. +// executes the SQL and stores the schema and data. func LoadMultiDBSnapshot(ctx context.Context, multiDBState *MultiDBState) (*MultiDBSnapshot, error) { multiDBSnapshot := NewMultiDBSnapshot() for dbName, dbState := range multiDBState.DBStates { diff --git a/storagenode/storagenodedb/testdata/v49.go b/storagenode/storagenodedb/testdata/v49.go index 0a6217fd6..4c4318744 100644 --- a/storagenode/storagenodedb/testdata/v49.go +++ b/storagenode/storagenodedb/testdata/v49.go @@ -8,9 +8,9 @@ import "storj.io/storj/storagenode/storagenodedb" var v49 = MultiDBState{ Version: 49, DBStates: DBStates{ - storagenodedb.UsedSerialsDBName: v47.DBStates[storagenodedb.UsedSerialsDBName], - storagenodedb.StorageUsageDBName: v47.DBStates[storagenodedb.StorageUsageDBName], - storagenodedb.ReputationDBName: v48.DBStates[storagenodedb.ReputationDBName], + storagenodedb.UsedSerialsDBName: v47.DBStates[storagenodedb.UsedSerialsDBName], + storagenodedb.StorageUsageDBName: v47.DBStates[storagenodedb.StorageUsageDBName], + storagenodedb.ReputationDBName: v48.DBStates[storagenodedb.ReputationDBName], storagenodedb.PieceSpaceUsedDBName: v47.DBStates[storagenodedb.PieceSpaceUsedDBName], storagenodedb.PieceInfoDBName: v47.DBStates[storagenodedb.PieceInfoDBName], storagenodedb.PieceExpirationDBName: v47.DBStates[storagenodedb.PieceExpirationDBName], @@ -58,6 +58,6 @@ var v49 = MultiDBState{ PRIMARY KEY ( id ) );`, }, - storagenodedb.PricingDBName: v47.DBStates[storagenodedb.PricingDBName], - storagenodedb.APIKeysDBName: v47.DBStates[storagenodedb.APIKeysDBName]}, + storagenodedb.PricingDBName: v47.DBStates[storagenodedb.PricingDBName], + storagenodedb.APIKeysDBName: v47.DBStates[storagenodedb.APIKeysDBName]}, } diff --git a/storagenode/storagenodedb/testdata/v50.go b/storagenode/storagenodedb/testdata/v50.go index f61bebc2b..92c989c47 100644 --- a/storagenode/storagenodedb/testdata/v50.go +++ b/storagenode/storagenodedb/testdata/v50.go @@ -8,9 +8,9 @@ import "storj.io/storj/storagenode/storagenodedb" var v50 = MultiDBState{ Version: 50, DBStates: DBStates{ - storagenodedb.UsedSerialsDBName: v47.DBStates[storagenodedb.UsedSerialsDBName], - storagenodedb.StorageUsageDBName: v47.DBStates[storagenodedb.StorageUsageDBName], - storagenodedb.ReputationDBName: v48.DBStates[storagenodedb.ReputationDBName], + storagenodedb.UsedSerialsDBName: v47.DBStates[storagenodedb.UsedSerialsDBName], + storagenodedb.StorageUsageDBName: v47.DBStates[storagenodedb.StorageUsageDBName], + storagenodedb.ReputationDBName: v48.DBStates[storagenodedb.ReputationDBName], storagenodedb.PieceSpaceUsedDBName: v47.DBStates[storagenodedb.PieceSpaceUsedDBName], storagenodedb.PieceInfoDBName: v47.DBStates[storagenodedb.PieceInfoDBName], storagenodedb.PieceExpirationDBName: v47.DBStates[storagenodedb.PieceExpirationDBName], @@ -58,6 +58,6 @@ var v50 = MultiDBState{ PRIMARY KEY ( id ) );`, }, - storagenodedb.PricingDBName: v47.DBStates[storagenodedb.PricingDBName], - storagenodedb.APIKeysDBName: v47.DBStates[storagenodedb.APIKeysDBName]}, + storagenodedb.PricingDBName: v47.DBStates[storagenodedb.PricingDBName], + storagenodedb.APIKeysDBName: v47.DBStates[storagenodedb.APIKeysDBName]}, } diff --git a/storagenode/storagenodedb/testdata/v51.go b/storagenode/storagenodedb/testdata/v51.go new file mode 100644 index 000000000..161e56de5 --- /dev/null +++ b/storagenode/storagenodedb/testdata/v51.go @@ -0,0 +1,44 @@ +// Copyright (C) 2021 Storj Labs, Inc. +// See LICENSE for copying information. + +package testdata + +import "storj.io/storj/storagenode/storagenodedb" + +var v51 = MultiDBState{ + Version: 51, + DBStates: DBStates{ + storagenodedb.UsedSerialsDBName: v47.DBStates[storagenodedb.UsedSerialsDBName], + storagenodedb.StorageUsageDBName: v47.DBStates[storagenodedb.StorageUsageDBName], + storagenodedb.ReputationDBName: v48.DBStates[storagenodedb.ReputationDBName], + storagenodedb.PieceSpaceUsedDBName: v47.DBStates[storagenodedb.PieceSpaceUsedDBName], + storagenodedb.PieceInfoDBName: v47.DBStates[storagenodedb.PieceInfoDBName], + storagenodedb.PieceExpirationDBName: v47.DBStates[storagenodedb.PieceExpirationDBName], + storagenodedb.OrdersDBName: v47.DBStates[storagenodedb.OrdersDBName], + storagenodedb.BandwidthDBName: v47.DBStates[storagenodedb.BandwidthDBName], + storagenodedb.SatellitesDBName: v47.DBStates[storagenodedb.SatellitesDBName], + storagenodedb.DeprecatedInfoDBName: v47.DBStates[storagenodedb.DeprecatedInfoDBName], + storagenodedb.NotificationsDBName: v47.DBStates[storagenodedb.NotificationsDBName], + storagenodedb.HeldAmountDBName: &DBState{ + SQL: v50.DBStates[storagenodedb.HeldAmountDBName].SQL, + OldData: ` + -- distributed is 0 for two periods < 2020-12, and 2 periods >= 2020-12. + INSERT INTO paystubs (period, satellite_id, created_at, codes, usage_at_rest, usage_get, usage_put, usage_get_repair, usage_put_repair, usage_get_audit, comp_at_rest, comp_get, comp_put, comp_get_repair, comp_put_repair, comp_get_audit, surge_percent, held, owed, disposed, paid, distributed) VALUES + ('2020-10', 'foo', '2020-04-07T00:00:00.000000Z', 'X', 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 0), + ('2020-11', 'foo', '2020-04-07T00:00:00.000000Z', 'X', 101, 201, 301, 401, 501, 601, 701, 801, 901, 1010, 1101, 1201, 1301, 1401, 1501, 1601, 1701, 0), + ('2020-12', 'foo', '2020-04-07T00:00:00.000000Z', 'X', 102, 202, 302, 402, 502, 602, 702, 802, 902, 1020, 1102, 1202, 1302, 1402, 1502, 1602, 1702, 0), + ('2021-01', 'foo', '2020-04-07T00:00:00.000000Z', 'X', 103, 203, 303, 403, 503, 603, 703, 803, 903, 1030, 1103, 1203, 1303, 1403, 1503, 1603, 1703, 0) + `, + NewData: ` + -- distributed has been updated for the periods < 2020-12. + INSERT INTO paystubs (period, satellite_id, created_at, codes, usage_at_rest, usage_get, usage_put, usage_get_repair, usage_put_repair, usage_get_audit, comp_at_rest, comp_get, comp_put, comp_get_repair, comp_put_repair, comp_get_audit, surge_percent, held, owed, disposed, paid, distributed) VALUES + ('2020-10', 'foo', '2020-04-07T00:00:00.000000Z', 'X', 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1700), + ('2020-11', 'foo', '2020-04-07T00:00:00.000000Z', 'X', 101, 201, 301, 401, 501, 601, 701, 801, 901, 1010, 1101, 1201, 1301, 1401, 1501, 1601, 1701, 1701), + ('2020-12', 'foo', '2020-04-07T00:00:00.000000Z', 'X', 102, 202, 302, 402, 502, 602, 702, 802, 902, 1020, 1102, 1202, 1302, 1402, 1502, 1602, 1702, 0), + ('2021-01', 'foo', '2020-04-07T00:00:00.000000Z', 'X', 103, 203, 303, 403, 503, 603, 703, 803, 903, 1030, 1103, 1203, 1303, 1403, 1503, 1603, 1703, 0) + `, + }, + storagenodedb.PricingDBName: v47.DBStates[storagenodedb.PricingDBName], + storagenodedb.APIKeysDBName: v47.DBStates[storagenodedb.APIKeysDBName], + }, +}