JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

migrate: Allow work on separate dbs (#2996)

Browse Source
This commit is contained in:
Isaac Hess 2019-09-10 13:42:23 -06:00 committed by GitHub
parent 7cf5650560
commit 0b32572ae6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 120 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
internal/migrate/versions.go
View File

@ -47,6 +47,7 @@ type Migration struct {
// Step describes a single step in migration. // Step describes a single step in migration.
type Step struct { type Step struct {
DB DB // The DB to execute this step on
Description string Description string
Version int // Versions should start at 0 Version int // Versions should start at 0
Action Action Action Action
@ -90,7 +91,7 @@ func (migration *Migration) ValidateSteps() error {
} }
// Run runs the migration steps // Run runs the migration steps
func (migration *Migration) Run(log *zap.Logger, db DB) error { func (migration *Migration) Run(log *zap.Logger) error {
err := migration.ValidTableName() err := migration.ValidTableName()
if err != nil { if err != nil {
return err return err
@ -101,12 +102,17 @@ func (migration *Migration) Run(log *zap.Logger, db DB) error {
return err return err
} }
err = migration.ensureVersionTable(log, db) for _, step := range migration.Steps {
if step.DB == nil {
return Error.New("step.DB is nil for step %d", step.Version)
}
err = migration.ensureVersionTable(log, step.DB)
if err != nil { if err != nil {
return Error.New("creating version table failed: %v", err) return Error.New("creating version table failed: %v", err)
} }
version, err := migration.getLatestVersion(log, db) version, err := migration.getLatestVersion(log, step.DB)
if err != nil { if err != nil {
return Error.Wrap(err) return Error.Wrap(err)
} }
@ -117,25 +123,24 @@ func (migration *Migration) Run(log *zap.Logger, db DB) error {
log.Info("No Version") log.Info("No Version")
} }
for _, step := range migration.Steps {
if step.Version <= version { if step.Version <= version {
continue continue
} }
log := log.Named(strconv.Itoa(step.Version)) stepLog := log.Named(strconv.Itoa(step.Version))
log.Info(step.Description) stepLog.Info(step.Description)
tx, err := db.Begin() tx, err := step.DB.Begin()
if err != nil { if err != nil {
return Error.Wrap(err) return Error.Wrap(err)
} }
err = step.Action.Run(log, db, tx) err = step.Action.Run(stepLog, step.DB, tx)
if err != nil { if err != nil {
return Error.Wrap(errs.Combine(err, tx.Rollback())) return Error.Wrap(errs.Combine(err, tx.Rollback()))
} }
err = migration.addVersion(tx, db, step.Version) err = migration.addVersion(tx, step.DB, step.Version)
if err != nil { if err != nil {
return Error.Wrap(errs.Combine(err, tx.Rollback())) return Error.Wrap(errs.Combine(err, tx.Rollback()))
} }

14
internal/migrate/versions_test.go
View File

@ -62,6 +62,7 @@ func basicMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
Table: dbName, Table: dbName,
Steps: []*migrate.Step{ Steps: []*migrate.Step{
{ {
DB: testDB,
Description: "Initialize Table", Description: "Initialize Table",
Version: 1, Version: 1,
Action: migrate.SQL{ Action: migrate.SQL{
@ -70,6 +71,7 @@ func basicMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
}, },
}, },
{ {
DB: testDB,
Description: "Move files", Description: "Move files",
Version: 2, Version: 2,
Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error {
@ -79,7 +81,7 @@ func basicMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
}, },
} }
err = m.Run(zap.NewNop(), testDB) err = m.Run(zap.NewNop())
assert.NoError(t, err) assert.NoError(t, err)
var version int var version int
@ -136,6 +138,7 @@ func multipleMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
Table: dbName, Table: dbName,
Steps: []*migrate.Step{ Steps: []*migrate.Step{
{ {
DB: testDB,
Description: "Step 1", Description: "Step 1",
Version: 1, Version: 1,
Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error {
@ -144,6 +147,7 @@ func multipleMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
}), }),
}, },
{ {
DB: testDB,
Description: "Step 2", Description: "Step 2",
Version: 2, Version: 2,
Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error {
@ -154,11 +158,12 @@ func multipleMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
}, },
} }
err := m.Run(zap.NewNop(), testDB) err := m.Run(zap.NewNop())
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 2, steps) assert.Equal(t, 2, steps)
m.Steps = append(m.Steps, &migrate.Step{ m.Steps = append(m.Steps, &migrate.Step{
DB: testDB,
Description: "Step 3", Description: "Step 3",
Version: 3, Version: 3,
Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error {
@ -166,7 +171,7 @@ func multipleMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
return nil return nil
}), }),
}) })
err = m.Run(zap.NewNop(), testDB) err = m.Run(zap.NewNop())
assert.NoError(t, err) assert.NoError(t, err)
var version int var version int
@ -208,6 +213,7 @@ func failedMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
Table: dbName, Table: dbName,
Steps: []*migrate.Step{ Steps: []*migrate.Step{
{ {
DB: testDB,
Description: "Step 1", Description: "Step 1",
Version: 1, Version: 1,
Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, _ migrate.DB, tx *sql.Tx) error {
@ -217,7 +223,7 @@ func failedMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {
}, },
} }
err := m.Run(zap.NewNop(), testDB) err := m.Run(zap.NewNop())
require.Error(t, err, "migration failed") require.Error(t, err, "migration failed")
var version sql.NullInt64 var version sql.NullInt64

58
satellite/satellitedb/migrate.go
View File

@ -38,7 +38,7 @@ func (db *DB) CreateTables() error {
} }
} }
migration := db.PostgresMigration() migration := db.PostgresMigration()
return migration.Run(db.log.Named("migrate"), db.db) return migration.Run(db.log.Named("migrate"))
default: default:
return migrate.Create("database", db.db) return migrate.Create("database", db.db)
} }
@ -51,6 +51,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
Steps: []*migrate.Step{ Steps: []*migrate.Step{
{ {
// some databases may have already this done, although the version may not match // some databases may have already this done, although the version may not match
DB: db.db,
Description: "Initial setup", Description: "Initial setup",
Version: 0, Version: 0,
Action: migrate.SQL{ Action: migrate.SQL{
@ -171,6 +172,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
{ {
// some databases may have already this done, although the version may not match // some databases may have already this done, although the version may not match
DB: db.db,
Description: "Adjust table naming", Description: "Adjust table naming",
Version: 1, Version: 1,
Action: migrate.Func(func(log *zap.Logger, db migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, db migrate.DB, tx *sql.Tx) error {
@ -255,6 +257,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
{ {
// some databases may have already this done, although the version may not match // some databases may have already this done, although the version may not match
DB: db.db,
Description: "Remove bucket infos", Description: "Remove bucket infos",
Version: 2, Version: 2,
Action: migrate.SQL{ Action: migrate.SQL{
@ -263,6 +266,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
{ {
// some databases may have already this done, although the version may not match // some databases may have already this done, although the version may not match
DB: db.db,
Description: "Add certificates table", Description: "Add certificates table",
Version: 3, Version: 3,
Action: migrate.SQL{ Action: migrate.SQL{
@ -276,6 +280,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
{ {
// some databases may have already this done, although the version may not match // some databases may have already this done, although the version may not match
DB: db.db,
Description: "Adjust users table", Description: "Adjust users table",
Version: 4, Version: 4,
Action: migrate.Func(func(log *zap.Logger, db migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, db migrate.DB, tx *sql.Tx) error {
@ -322,6 +327,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}), }),
}, },
{ {
DB: db.db,
Description: "Add wallet column", Description: "Add wallet column",
Version: 5, Version: 5,
Action: migrate.SQL{ Action: migrate.SQL{
@ -334,6 +340,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add bucket usage rollup table", Description: "Add bucket usage rollup table",
Version: 6, Version: 6,
Action: migrate.SQL{ Action: migrate.SQL{
@ -356,6 +363,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add index on bwagreements", Description: "Add index on bwagreements",
Version: 7, Version: 7,
Action: migrate.SQL{ Action: migrate.SQL{
@ -363,6 +371,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add registration_tokens table", Description: "Add registration_tokens table",
Version: 8, Version: 8,
Action: migrate.SQL{ Action: migrate.SQL{
@ -377,6 +386,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add new tables for tracking used serials, bandwidth and storage", Description: "Add new tables for tracking used serials, bandwidth and storage",
Version: 9, Version: 9,
Action: migrate.SQL{ Action: migrate.SQL{
@ -445,6 +455,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "users first_name to full_name, last_name to short_name", Description: "users first_name to full_name, last_name to short_name",
Version: 10, Version: 10,
Action: migrate.SQL{ Action: migrate.SQL{
@ -454,6 +465,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "drops interval seconds from storage_rollups, renames x_storage_rollups to x_storage_tallies, adds fields to bucket_storage_tallies", Description: "drops interval seconds from storage_rollups, renames x_storage_rollups to x_storage_tallies, adds fields to bucket_storage_tallies",
Version: 11, Version: 11,
Action: migrate.SQL{ Action: migrate.SQL{
@ -481,6 +493,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Merge overlay_cache_nodes into nodes table", Description: "Merge overlay_cache_nodes into nodes table",
Version: 12, Version: 12,
Action: migrate.SQL{ Action: migrate.SQL{
@ -509,6 +522,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Change bucket_id to bucket_name and project_id", Description: "Change bucket_id to bucket_name and project_id",
Version: 13, Version: 13,
Action: migrate.SQL{ Action: migrate.SQL{
@ -545,6 +559,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add new Columns to store version information", Description: "Add new Columns to store version information",
Version: 14, Version: 14,
Action: migrate.SQL{ Action: migrate.SQL{
@ -557,6 +572,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Default Node Type should be invalid", Description: "Default Node Type should be invalid",
Version: 15, Version: 15,
Action: migrate.SQL{ Action: migrate.SQL{
@ -564,6 +580,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add path to injuredsegment to prevent duplicates", Description: "Add path to injuredsegment to prevent duplicates",
Version: 16, Version: 16,
Action: migrate.Func(func(log *zap.Logger, db migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, db migrate.DB, tx *sql.Tx) error {
@ -617,6 +634,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}), }),
}, },
{ {
DB: db.db,
Description: "Fix audit and uptime ratios for new nodes", Description: "Fix audit and uptime ratios for new nodes",
Version: 17, Version: 17,
Action: migrate.SQL{` Action: migrate.SQL{`
@ -625,6 +643,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Drops storagenode_storage_tally table, Renames accounting_raws to storagenode_storage_tally, and Drops data_type and created_at columns", Description: "Drops storagenode_storage_tally table, Renames accounting_raws to storagenode_storage_tally, and Drops data_type and created_at columns",
Version: 18, Version: 18,
Action: migrate.SQL{ Action: migrate.SQL{
@ -635,6 +654,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Added new table to store reset password tokens", Description: "Added new table to store reset password tokens",
Version: 19, Version: 19,
Action: migrate.SQL{` Action: migrate.SQL{`
@ -648,6 +668,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Adds pending_audits table, adds 'contained' column to nodes table", Description: "Adds pending_audits table, adds 'contained' column to nodes table",
Version: 20, Version: 20,
Action: migrate.SQL{ Action: migrate.SQL{
@ -667,6 +688,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add last_ip column and index", Description: "Add last_ip column and index",
Version: 21, Version: 21,
Action: migrate.SQL{ Action: migrate.SQL{
@ -677,6 +699,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Create new tables for free credits program", Description: "Create new tables for free credits program",
Version: 22, Version: 22,
Action: migrate.SQL{` Action: migrate.SQL{`
@ -698,6 +721,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Drops and recreates api key table to handle macaroons and adds revocation table", Description: "Drops and recreates api key table to handle macaroons and adds revocation table",
Version: 23, Version: 23,
Action: migrate.SQL{ Action: migrate.SQL{
@ -716,6 +740,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add usage_limit column to projects table", Description: "Add usage_limit column to projects table",
Version: 24, Version: 24,
Action: migrate.SQL{ Action: migrate.SQL{
@ -723,6 +748,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add disqualified column to nodes table", Description: "Add disqualified column to nodes table",
Version: 25, Version: 25,
Action: migrate.SQL{ Action: migrate.SQL{
@ -730,6 +756,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add invitee_credit_in_gb and award_credit_in_gb columns, delete type and credit_in_cents columns", Description: "Add invitee_credit_in_gb and award_credit_in_gb columns, delete type and credit_in_cents columns",
Version: 26, Version: 26,
Action: migrate.SQL{ Action: migrate.SQL{
@ -740,6 +767,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Create value attribution table", Description: "Create value attribution table",
Version: 27, Version: 27,
Action: migrate.SQL{ Action: migrate.SQL{
@ -752,6 +780,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Remove agreements table", Description: "Remove agreements table",
Version: 28, Version: 28,
Action: migrate.SQL{ Action: migrate.SQL{
@ -759,6 +788,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add userpaymentinfos, projectpaymentinfos, projectinvoicestamps", Description: "Add userpaymentinfos, projectpaymentinfos, projectinvoicestamps",
Version: 29, Version: 29,
Action: migrate.SQL{ Action: migrate.SQL{
@ -788,6 +818,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Alter value attribution table. Remove bucket_id. Add project_id and bucket_name as primary key", Description: "Alter value attribution table. Remove bucket_id. Add project_id and bucket_name as primary key",
Version: 30, Version: 30,
Action: migrate.SQL{ Action: migrate.SQL{
@ -801,6 +832,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add user_credit table", Description: "Add user_credit table",
Version: 31, Version: 31,
Action: migrate.SQL{ Action: migrate.SQL{
@ -818,6 +850,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Change type of disqualified column of nodes table to timestamp", Description: "Change type of disqualified column of nodes table to timestamp",
Version: 32, Version: 32,
Action: migrate.SQL{ Action: migrate.SQL{
@ -832,6 +865,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add alpha and beta columns for reputations", Description: "Add alpha and beta columns for reputations",
Version: 33, Version: 33,
Action: migrate.SQL{ Action: migrate.SQL{
@ -842,6 +876,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Remove ratio columns from node reputations", Description: "Remove ratio columns from node reputations",
Version: 34, Version: 34,
Action: migrate.SQL{ Action: migrate.SQL{
@ -850,6 +885,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Fix reputations to preserve a baseline", Description: "Fix reputations to preserve a baseline",
Version: 35, Version: 35,
Action: migrate.SQL{ Action: migrate.SQL{
@ -860,6 +896,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Update Last_IP column to be masked", Description: "Update Last_IP column to be masked",
Version: 36, Version: 36,
Action: migrate.SQL{ Action: migrate.SQL{
@ -869,6 +906,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Update project_id column from 36 byte string based UUID to 16 byte UUID", Description: "Update project_id column from 36 byte string based UUID to 16 byte UUID",
Version: 37, Version: 37,
Action: migrate.SQL{ Action: migrate.SQL{
@ -899,6 +937,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add bucket metadata table", Description: "Add bucket metadata table",
Version: 38, Version: 38,
Action: migrate.SQL{ Action: migrate.SQL{
@ -923,6 +962,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Remove disqualification flag for failing uptime checks", Description: "Remove disqualification flag for failing uptime checks",
Version: 39, Version: 39,
Action: migrate.SQL{ Action: migrate.SQL{
@ -930,6 +970,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add unique id for project payments. Add is_default property", Description: "Add unique id for project payments. Add is_default property",
Version: 40, Version: 40,
Action: migrate.SQL{ Action: migrate.SQL{
@ -946,6 +987,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Move InjuredSegment path from string to bytes", Description: "Move InjuredSegment path from string to bytes",
Version: 41, Version: 41,
Action: migrate.SQL{ Action: migrate.SQL{
@ -958,6 +1000,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Remove num_redeemed column in offers table", Description: "Remove num_redeemed column in offers table",
Version: 42, Version: 42,
Action: migrate.SQL{ Action: migrate.SQL{
@ -965,6 +1008,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Set default offer for each offer type in offers table", Description: "Set default offer for each offer type in offers table",
Version: 43, Version: 43,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1005,6 +1049,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add index on InjuredSegments attempted column", Description: "Add index on InjuredSegments attempted column",
Version: 44, Version: 44,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1012,6 +1057,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add partner id field to support OSPP", Description: "Add partner id field to support OSPP",
Version: 45, Version: 45,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1022,6 +1068,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add pending audit path", Description: "Add pending audit path",
Version: 46, Version: 46,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1031,6 +1078,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Modify default offers configuration", Description: "Modify default offers configuration",
Version: 47, Version: 47,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1050,6 +1098,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
// This partial unique index enforces uniqueness among (id, offer_id) pairs for users that have signed up // This partial unique index enforces uniqueness among (id, offer_id) pairs for users that have signed up
// but are not yet activated (credits_earned_in_cents=0). // but are not yet activated (credits_earned_in_cents=0).
// Among users that are activated, uniqueness of (id, offer_id) pairs is not required or desirable. // Among users that are activated, uniqueness of (id, offer_id) pairs is not required or desirable.
DB: db.db,
Description: "Create partial index for user_credits table", Description: "Create partial index for user_credits table",
Version: 48, Version: 48,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1058,6 +1107,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add cascade to user_id for deleting an account", Description: "Add cascade to user_id for deleting an account",
Version: 49, Version: 49,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1073,6 +1123,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Changing the primary key constraint", Description: "Changing the primary key constraint",
Version: 50, Version: 50,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1085,6 +1136,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
// Creating owner_id column for project. // Creating owner_id column for project.
// Removing projects without project members // Removing projects without project members
// And populating this column with first project member id // And populating this column with first project member id
DB: db.db,
Description: "Creating owner_id column for projects table", Description: "Creating owner_id column for projects table",
Version: 51, Version: 51,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1115,6 +1167,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Remove certRecords table", Description: "Remove certRecords table",
Version: 52, Version: 52,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1122,6 +1175,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add piece_count column to nodes table", Description: "Add piece_count column to nodes table",
Version: 53, Version: 53,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1129,6 +1183,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Add Peer Identities table", Description: "Add Peer Identities table",
Version: 54, Version: 54,
Action: migrate.SQL{ Action: migrate.SQL{
@ -1142,6 +1197,7 @@ func (db *DB) PostgresMigration() *migrate.Migration {
}, },
}, },
{ {
DB: db.db,
Description: "Added normalized_email column to users table", Description: "Added normalized_email column to users table",
Version: 55, Version: 55,
Action: migrate.SQL{ Action: migrate.SQL{

2
satellite/satellitedb/migrate_postgres_test.go
View File

@ -138,7 +138,7 @@ func TestMigratePostgres(t *testing.T) {
tag := fmt.Sprintf("#%d - v%d", i, step.Version) tag := fmt.Sprintf("#%d - v%d", i, step.Version)
// run migration up to a specific version // run migration up to a specific version
err := migrations.TargetVersion(step.Version).Run(log.Named("migrate"), rawdb) err := migrations.TargetVersion(step.Version).Run(log.Named("migrate"))
require.NoError(t, err, tag) require.NoError(t, err, tag)
// find the matching expected version // find the matching expected version

28
storagenode/storagenodedb/database.go
View File

@ -219,7 +219,7 @@ func openTestDatabase() (*sql.DB, error) {
// CreateTables creates any necessary tables. // CreateTables creates any necessary tables.
func (db *DB) CreateTables() error { func (db *DB) CreateTables() error {
migration := db.Migration() migration := db.Migration()
return migration.Run(db.log.Named("migration"), db.versionsDB) return migration.Run(db.log.Named("migration"))
} }
// Close closes any resources. // Close closes any resources.
@ -241,11 +241,6 @@ func (db *DB) Close() error {
) )
} }
// VersionsMigration returns the instance of the versions database.
func (db *DB) VersionsMigration() migrate.DB {
return db.versionsDB
}
// Versions returns the instance of the versions database. // Versions returns the instance of the versions database.
func (db *DB) Versions() SQLDB { func (db *DB) Versions() SQLDB {
return db.versionsDB return db.versionsDB
@ -307,6 +302,7 @@ func (db *DB) Migration() *migrate.Migration {
Table: "versions", Table: "versions",
Steps: []*migrate.Step{ Steps: []*migrate.Step{
{ {
DB: db.versionsDB,
Description: "Initial setup", Description: "Initial setup",
Version: 0, Version: 0,
Action: migrate.SQL{ Action: migrate.SQL{
@ -388,6 +384,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Network Wipe #2", Description: "Network Wipe #2",
Version: 1, Version: 1,
Action: migrate.SQL{ Action: migrate.SQL{
@ -395,6 +392,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Add tracking of deletion failures.", Description: "Add tracking of deletion failures.",
Version: 2, Version: 2,
Action: migrate.SQL{ Action: migrate.SQL{
@ -402,6 +400,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Add vouchersDB for storing and retrieving vouchers.", Description: "Add vouchersDB for storing and retrieving vouchers.",
Version: 3, Version: 3,
Action: migrate.SQL{ Action: migrate.SQL{
@ -413,6 +412,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Add index on pieceinfo expireation", Description: "Add index on pieceinfo expireation",
Version: 4, Version: 4,
Action: migrate.SQL{ Action: migrate.SQL{
@ -421,6 +421,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Partial Network Wipe - Tardigrade Satellites", Description: "Partial Network Wipe - Tardigrade Satellites",
Version: 5, Version: 5,
Action: migrate.SQL{ Action: migrate.SQL{
@ -432,6 +433,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Add creation date.", Description: "Add creation date.",
Version: 6, Version: 6,
Action: migrate.SQL{ Action: migrate.SQL{
@ -439,6 +441,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Drop certificate table.", Description: "Drop certificate table.",
Version: 7, Version: 7,
Action: migrate.SQL{ Action: migrate.SQL{
@ -447,6 +450,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Drop old used serials and remove pieceinfo_deletion_failed index.", Description: "Drop old used serials and remove pieceinfo_deletion_failed index.",
Version: 8, Version: 8,
Action: migrate.SQL{ Action: migrate.SQL{
@ -455,6 +459,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Add order limit table.", Description: "Add order limit table.",
Version: 9, Version: 9,
Action: migrate.SQL{ Action: migrate.SQL{
@ -462,6 +467,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Optimize index usage.", Description: "Optimize index usage.",
Version: 10, Version: 10,
Action: migrate.SQL{ Action: migrate.SQL{
@ -472,6 +478,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Create bandwidth_usage_rollup table.", Description: "Create bandwidth_usage_rollup table.",
Version: 11, Version: 11,
Action: migrate.SQL{ Action: migrate.SQL{
@ -485,6 +492,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Clear Tables from Alpha data", Description: "Clear Tables from Alpha data",
Version: 12, Version: 12,
Action: migrate.SQL{ Action: migrate.SQL{
@ -532,6 +540,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Free Storagenodes from trash data", Description: "Free Storagenodes from trash data",
Version: 13, Version: 13,
Action: migrate.Func(func(log *zap.Logger, mgdb migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, mgdb migrate.DB, tx *sql.Tx) error {
@ -561,6 +570,7 @@ func (db *DB) Migration() *migrate.Migration {
}), }),
}, },
{ {
DB: db.versionsDB,
Description: "Free Storagenodes from orphaned tmp data", Description: "Free Storagenodes from orphaned tmp data",
Version: 14, Version: 14,
Action: migrate.Func(func(log *zap.Logger, mgdb migrate.DB, tx *sql.Tx) error { Action: migrate.Func(func(log *zap.Logger, mgdb migrate.DB, tx *sql.Tx) error {
@ -578,6 +588,7 @@ func (db *DB) Migration() *migrate.Migration {
}), }),
}, },
{ {
DB: db.versionsDB,
Description: "Start piece_expirations table, deprecate pieceinfo table", Description: "Start piece_expirations table, deprecate pieceinfo table",
Version: 15, Version: 15,
Action: migrate.SQL{ Action: migrate.SQL{
@ -594,6 +605,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Add reputation and storage usage cache tables", Description: "Add reputation and storage usage cache tables",
Version: 16, Version: 16,
Action: migrate.SQL{ Action: migrate.SQL{
@ -621,6 +633,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Create piece_space_used table", Description: "Create piece_space_used table",
Version: 17, Version: 17,
Action: migrate.SQL{ Action: migrate.SQL{
@ -634,6 +647,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Drop vouchers table", Description: "Drop vouchers table",
Version: 18, Version: 18,
Action: migrate.SQL{ Action: migrate.SQL{
@ -641,6 +655,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Add disqualified field to reputation", Description: "Add disqualified field to reputation",
Version: 19, Version: 19,
Action: migrate.SQL{ Action: migrate.SQL{
@ -664,6 +679,7 @@ func (db *DB) Migration() *migrate.Migration {
}, },
}, },
{ {
DB: db.versionsDB,
Description: "Empty storage_usage table, rename storage_usage.timestamp to interval_start", Description: "Empty storage_usage table, rename storage_usage.timestamp to interval_start",
Version: 20, Version: 20,
Action: migrate.SQL{ Action: migrate.SQL{

2
storagenode/storagenodedb/migrations_test.go
View File

@ -99,7 +99,7 @@ func TestMigrate(t *testing.T) {
tag := fmt.Sprintf("#%d - v%d", i, step.Version) tag := fmt.Sprintf("#%d - v%d", i, step.Version)
// run migration up to a specific version // run migration up to a specific version
err := migrations.TargetVersion(step.Version).Run(log.Named("migrate"), db.VersionsMigration()) err := migrations.TargetVersion(step.Version).Run(log.Named("migrate"))
require.NoError(t, err, tag) require.NoError(t, err, tag)
// find the matching expected version // find the matching expected version