From 4fdea51d5cb69602c40a346c771f8bb672e04e7e Mon Sep 17 00:00:00 2001 From: Andrew Harding Date: Tue, 29 Nov 2022 13:47:02 -0700 Subject: [PATCH] storagenode/storagenodedb: faster test db init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running all of the migrations necessary to initialize a storage node database takes a significant amount of time during runs. The package current supports initializing a database from manually coalesced migration data (i.e. snapshot) which improves the situation somewhat. This change takes things a bit further by changing the snapshot code to instead hydrate the database directory from a pre-generated snapshot zip file. name old time/op new time/op delta Run_StorageNodeCount_4/Postgres-16 2.50s ± 0% 0.16s ± 0% ~ (p=1.000 n=1+1) Change-Id: I213bbba5f9199497fbe8ce889b627e853f8b29a0 --- private/testplanet/storagenode.go | 6 +- storagenode/peer.go | 3 - storagenode/storagenodedb/database.go | 12 +- storagenode/storagenodedb/snapshot.go | 394 ------------------ .../storagenodedbtest/gensnapshot/main.go | 100 +++++ .../storagenodedb/storagenodedbtest/run.go | 3 +- .../storagenodedbtest/snapshot.go | 61 +++ .../{ => storagenodedbtest}/snapshot_test.go | 26 +- .../storagenodedbtest/testdata/snapshot.zip | Bin 0 -> 10105 bytes 9 files changed, 185 insertions(+), 420 deletions(-) delete mode 100644 storagenode/storagenodedb/snapshot.go create mode 100644 storagenode/storagenodedb/storagenodedbtest/gensnapshot/main.go create mode 100644 storagenode/storagenodedb/storagenodedbtest/snapshot.go rename storagenode/storagenodedb/{ => storagenodedbtest}/snapshot_test.go (76%) create mode 100644 storagenode/storagenodedb/storagenodedbtest/testdata/snapshot.zip diff --git a/private/testplanet/storagenode.go b/private/testplanet/storagenode.go index 1b74dac77..75f49f71f 100644 --- a/private/testplanet/storagenode.go +++ b/private/testplanet/storagenode.go @@ -39,7 +39,7 @@ import ( "storj.io/storj/storagenode/piecestore" "storj.io/storj/storagenode/preflight" "storj.io/storj/storagenode/retain" - "storj.io/storj/storagenode/storagenodedb" + "storj.io/storj/storagenode/storagenodedb/storagenodedbtest" "storj.io/storj/storagenode/trust" ) @@ -227,7 +227,7 @@ func (planet *Planet) newStorageNode(ctx context.Context, prefix string, index, verisonInfo := planet.NewVersionInfo() var db storagenode.DB - db, err = storagenodedb.OpenNew(ctx, log.Named("db"), config.DatabaseConfig()) + db, err = storagenodedbtest.OpenNew(ctx, log.Named("db"), config.DatabaseConfig()) if err != nil { return nil, err } @@ -257,7 +257,7 @@ func (planet *Planet) newStorageNode(ctx context.Context, prefix string, index, // Mark the peer's PieceDeleter as in testing mode, so it is easy to wait on the deleter peer.Storage2.PieceDeleter.SetupTest() - err = db.TestMigrateToLatest(ctx) + err = db.MigrateToLatest(ctx) if err != nil { return nil, err } diff --git a/storagenode/peer.go b/storagenode/peer.go index db90652cc..fbcf97d09 100644 --- a/storagenode/peer.go +++ b/storagenode/peer.go @@ -78,9 +78,6 @@ type DB interface { // MigrateToLatest initializes the database MigrateToLatest(ctx context.Context) error - // TestMigrateToLatest is a fast migration with skipping test (not safe for production + old db state) - TestMigrateToLatest(ctx context.Context) error - // Close closes the database Close() error diff --git a/storagenode/storagenodedb/database.go b/storagenode/storagenodedb/database.go index 5218f6a70..01777920f 100644 --- a/storagenode/storagenodedb/database.go +++ b/storagenode/storagenodedb/database.go @@ -347,13 +347,6 @@ func (db *DB) MigrateToLatest(ctx context.Context) error { return migration.Run(ctx, db.log.Named("migration")) } -// TestMigrateToLatest creates any necessary tables from snapshot. -// it's faster for the testing but should be the same as MigrateToLatest. -func (db *DB) TestMigrateToLatest(ctx context.Context) error { - migration := db.Snapshot(ctx) - return migration.Run(ctx, db.log.Named("migration")) -} - // Preflight conducts a pre-flight check to ensure correct schemas and minimal read+write functionality of the database tables. func (db *DB) Preflight(ctx context.Context) (err error) { for dbName, dbContainer := range db.SQLDBs { @@ -566,6 +559,11 @@ func (db *DB) RawDatabases() map[string]DBContainer { return db.SQLDBs } +// DBDirectory returns the database directory for testing purposes. +func (db *DB) DBDirectory() string { + return db.dbDirectory +} + // migrateToDB is a helper method that performs the migration from the // deprecatedInfoDB to the specified new db. It first closes and deletes any // existing database to guarantee idempotence. After migration it also closes diff --git a/storagenode/storagenodedb/snapshot.go b/storagenode/storagenodedb/snapshot.go deleted file mode 100644 index 6e62c92e2..000000000 --- a/storagenode/storagenodedb/snapshot.go +++ /dev/null @@ -1,394 +0,0 @@ -// Copyright (C) 2022 Storj Labs, Inc. -// See LICENSE for copying information. - -package storagenodedb - -import ( - "context" - - "go.uber.org/zap" - - "storj.io/storj/private/migrate" -) - -// Snapshot supposed to generate the same database as the Migration but without the original steps. -func (db *DB) Snapshot(ctx context.Context) *migrate.Migration { - return &migrate.Migration{ - Table: VersionTable, - Steps: []*migrate.Step{ - { - DB: &db.deprecatedInfoDB.DB, - Description: "Initial setup", - Version: 0, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, DeprecatedInfoDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{}, - }, - { - DB: &db.bandwidthDB.DB, - Description: "bandwidth db snapshot", - Version: 2, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, BandwidthDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - // table for storing bandwidth usage - `CREATE TABLE bandwidth_usage ( - satellite_id BLOB NOT NULL, - action INTEGER NOT NULL, - amount BIGINT NOT NULL, - created_at TIMESTAMP NOT NULL - )`, - `CREATE INDEX idx_bandwidth_usage_satellite ON bandwidth_usage(satellite_id)`, - `CREATE INDEX idx_bandwidth_usage_created ON bandwidth_usage(created_at)`, - `CREATE TABLE bandwidth_usage_rollups ( - interval_start TIMESTAMP NOT NULL, - satellite_id BLOB NOT NULL, - action INTEGER NOT NULL, - amount BIGINT NOT NULL, - PRIMARY KEY ( interval_start, satellite_id, action ) - )`, - }, - }, - { - DB: &db.ordersDB.DB, - Description: "orders db snapshot", - Version: 3, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, OrdersDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - // table for storing all unsent orders - `CREATE TABLE unsent_order ( - satellite_id BLOB NOT NULL, - serial_number BLOB NOT NULL, - - order_limit_serialized BLOB NOT NULL, -- serialized pb.OrderLimit - order_serialized BLOB NOT NULL, -- serialized pb.Order - order_limit_expiration TIMESTAMP NOT NULL, -- when is the deadline for sending it - - uplink_cert_id INTEGER NOT NULL, - - FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id) - )`, - `CREATE UNIQUE INDEX idx_orders ON unsent_order(satellite_id, serial_number)`, - `CREATE TABLE order_archive_ ( - satellite_id BLOB NOT NULL, - serial_number BLOB NOT NULL, - - order_limit_serialized BLOB NOT NULL, - order_serialized BLOB NOT NULL, - - uplink_cert_id INTEGER NOT NULL, - - status INTEGER NOT NULL, - archived_at TIMESTAMP NOT NULL, - - FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id) - )`, - `CREATE INDEX idx_order_archived_at ON order_archive_(archived_at)`, - }, - }, - { - DB: &db.pieceExpirationDB.DB, - Description: "pieceExpiration db snapshot", - Version: 4, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, PieceExpirationDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - `CREATE TABLE piece_expirations ( - satellite_id BLOB NOT NULL, - piece_id BLOB NOT NULL, - piece_expiration TIMESTAMP NOT NULL, -- date when it can be deleted - deletion_failed_at TIMESTAMP, trash INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (satellite_id, piece_id) - )`, - `CREATE INDEX idx_piece_expirations_piece_expiration ON piece_expirations(piece_expiration)`, - `CREATE INDEX idx_piece_expirations_deletion_failed_at ON piece_expirations(deletion_failed_at)`, - `CREATE INDEX idx_piece_expirations_trashed - ON piece_expirations(satellite_id, trash) - WHERE trash = 1`, - }, - }, - { - DB: &db.v0PieceInfoDB.DB, - Description: "v0PieceInfo db snapshot", - Version: 5, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, PieceInfoDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - `CREATE TABLE pieceinfo_ ( - satellite_id BLOB NOT NULL, - piece_id BLOB NOT NULL, - piece_size BIGINT NOT NULL, - piece_expiration TIMESTAMP, - - order_limit BLOB NOT NULL, - uplink_piece_hash BLOB NOT NULL, - uplink_cert_id INTEGER NOT NULL, - - deletion_failed_at TIMESTAMP, - piece_creation TIMESTAMP NOT NULL, - - FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id) - )`, - `CREATE UNIQUE INDEX pk_pieceinfo_ ON pieceinfo_(satellite_id, piece_id)`, - `CREATE INDEX idx_pieceinfo__expiration ON pieceinfo_(piece_expiration) WHERE piece_expiration IS NOT NULL`, - }, - }, - { - DB: &db.pieceSpaceUsedDB.DB, - Description: "pieceSpaceUsed db snapshot", - Version: 6, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, PieceSpaceUsedDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - // new table to hold the most recent totals from the piece space used cache - `CREATE TABLE piece_space_used_new ( - total INTEGER NOT NULL DEFAULT 0, - content_size INTEGER NOT NULL, - satellite_id BLOB - )`, - `ALTER TABLE piece_space_used_new RENAME TO piece_space_used;`, - `CREATE UNIQUE INDEX idx_piece_space_used_satellite_id ON piece_space_used(satellite_id)`, - }, - }, - { - DB: &db.reputationDB.DB, - Description: "reputation db snapshot", - Version: 7, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, ReputationDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - ` CREATE TABLE reputation_new ( - satellite_id BLOB NOT NULL, - audit_success_count INTEGER NOT NULL, - audit_total_count INTEGER NOT NULL, - audit_reputation_alpha REAL NOT NULL, - audit_reputation_beta REAL NOT NULL, - audit_reputation_score REAL NOT NULL, - audit_unknown_reputation_alpha REAL NOT NULL, - audit_unknown_reputation_beta REAL NOT NULL, - audit_unknown_reputation_score REAL NOT NULL, - online_score REAL NOT NULL, - audit_history BLOB, - disqualified_at TIMESTAMP, - updated_at TIMESTAMP NOT NULL, - suspended_at TIMESTAMP, - offline_suspended_at TIMESTAMP, - offline_under_review_at TIMESTAMP, - joined_at TIMESTAMP NOT NULL, vetted_at TIMESTAMP, - PRIMARY KEY (satellite_id) - );`, - `ALTER TABLE reputation_new RENAME TO reputation;`, - }, - }, - { - DB: &db.storageUsageDB.DB, - Description: "storageUsage db snapshot", - Version: 8, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, StorageUsageDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - `CREATE TABLE storage_usage_new ( - timestamp TIMESTAMP NOT NULL, - satellite_id BLOB NOT NULL, - at_rest_total REAL NOT NULL, - interval_end_time TIMESTAMP NOT NULL, - PRIMARY KEY (timestamp, satellite_id) - );`, - `ALTER TABLE storage_usage_new RENAME TO storage_usage`, - }, - }, - { - DB: &db.usedSerialsDB.DB, - Description: "usedSerials db snapshot", - Version: 9, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, UsedSerialsDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{}, - }, - { - DB: &db.satellitesDB.DB, - Description: "satellites db snapshot", - Version: 10, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, SatellitesDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - `CREATE TABLE satellites_new ( - node_id BLOB NOT NULL, - added_at TIMESTAMP NOT NULL, - status INTEGER NOT NULL, address TEXT, - PRIMARY KEY (node_id) - );`, - `ALTER TABLE satellites_new RENAME TO satellites;`, - `CREATE TABLE satellite_exit_progress_new ( - satellite_id BLOB NOT NULL, - initiated_at TIMESTAMP, - finished_at TIMESTAMP, - starting_disk_usage INTEGER NOT NULL, - bytes_deleted INTEGER NOT NULL, - completion_receipt BLOB, - FOREIGN KEY (satellite_id) REFERENCES satellites (node_id) - );`, - `ALTER TABLE satellite_exit_progress_new RENAME TO satellite_exit_progress`, - }, - }, - - { - DB: &db.notificationsDB.DB, - Description: "notifications db snapshot", - Version: 11, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, NotificationsDBName); err != nil { - return ErrDatabase.Wrap(err) - } - return nil - }, - Action: migrate.SQL{ - `CREATE TABLE notifications ( - id BLOB NOT NULL, - sender_id BLOB NOT NULL, - type INTEGER NOT NULL, - title TEXT NOT NULL, - message TEXT NOT NULL, - read_at TIMESTAMP, - created_at TIMESTAMP NOT NULL, - PRIMARY KEY (id) - );`, - }, - }, - { - DB: &db.payoutDB.DB, - Description: "paystubs db snapshot", - Version: 12, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, HeldAmountDBName); err != nil { - return ErrDatabase.Wrap(err) - } - - return nil - }, - Action: migrate.SQL{ - `CREATE TABLE paystubs_new ( - period text NOT NULL, - satellite_id bytea NOT NULL, - created_at timestamp NOT NULL, - codes text NOT NULL, - usage_at_rest double precision NOT NULL, - usage_get bigint NOT NULL, - usage_put bigint NOT NULL, - usage_get_repair bigint NOT NULL, - usage_put_repair bigint NOT NULL, - usage_get_audit bigint NOT NULL, - comp_at_rest bigint NOT NULL, - comp_get bigint NOT NULL, - comp_put bigint NOT NULL, - comp_get_repair bigint NOT NULL, - comp_put_repair bigint NOT NULL, - comp_get_audit bigint NOT NULL, - surge_percent bigint NOT NULL, - held bigint NOT NULL, - owed bigint NOT NULL, - disposed bigint NOT NULL, - paid bigint NOT NULL, - distributed bigint NOT NULL, - PRIMARY KEY ( period, satellite_id ) - );`, - `CREATE TABLE payments ( - id bigserial NOT NULL, - created_at timestamp NOT NULL, - satellite_id bytea NOT NULL, - period text, - amount bigint NOT NULL, - receipt text, - notes text, - PRIMARY KEY ( id ) - );`, - `ALTER TABLE paystubs_new RENAME TO paystubs`, - }, - }, - - { - DB: &db.pricingDB.DB, - Description: "pricing db snapshot", - Version: 13, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, PricingDBName); err != nil { - return ErrDatabase.Wrap(err) - } - - return nil - }, - Action: migrate.SQL{ - `CREATE TABLE pricing ( - satellite_id BLOB NOT NULL, - egress_bandwidth_price bigint NOT NULL, - repair_bandwidth_price bigint NOT NULL, - audit_bandwidth_price bigint NOT NULL, - disk_space_price bigint NOT NULL, - PRIMARY KEY ( satellite_id ) - );`, - }, - }, - - { - DB: &db.apiKeysDB.DB, - Description: "scret db snapshot", - Version: 14, - CreateDB: func(ctx context.Context, log *zap.Logger) error { - if err := db.openDatabase(ctx, APIKeysDBName); err != nil { - return ErrDatabase.Wrap(err) - } - - return nil - }, - Action: migrate.SQL{ - `CREATE TABLE secret ( - token bytea NOT NULL, - created_at timestamp with time zone NOT NULL, - PRIMARY KEY ( token ) - );`, - }, - }, - }, - } -} diff --git a/storagenode/storagenodedb/storagenodedbtest/gensnapshot/main.go b/storagenode/storagenodedb/storagenodedbtest/gensnapshot/main.go new file mode 100644 index 000000000..d83a02e42 --- /dev/null +++ b/storagenode/storagenodedb/storagenodedbtest/gensnapshot/main.go @@ -0,0 +1,100 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "archive/zip" + "bytes" + "context" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/zeebo/errs" + "go.uber.org/zap" + + "storj.io/storj/storagenode/storagenodedb" +) + +func main() { + outFile := "snapshot.zip" + if len(os.Args) > 1 { + outFile = os.Args[1] + } + if err := run(context.Background(), outFile); err != nil { + fmt.Fprintf(os.Stderr, "Failed to generate snapshot database: %+v\n", err) + os.Exit(1) + } +} + +func run(ctx context.Context, outFile string) error { + log, err := zap.NewDevelopment() + if err != nil { + return errs.Wrap(err) + } + + tempDir, err := os.MkdirTemp("", "") + if err != nil { + return errs.Wrap(err) + } + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + log.Warn("Could not remove temp directory", zap.Error(err), zap.String("directory", tempDir)) + } + }() + + cfg := storagenodedb.Config{ + Storage: tempDir, + Info: filepath.Join(tempDir, "piecestore.db"), + Info2: filepath.Join(tempDir, "info.db"), + Pieces: tempDir, + } + + db, err := storagenodedb.OpenNew(ctx, log, cfg) + if err != nil { + return errs.Wrap(err) + } + + err = db.MigrateToLatest(ctx) + if err != nil { + return errs.Wrap(err) + } + + err = db.Close() + if err != nil { + return errs.Wrap(err) + } + + matches, err := filepath.Glob(filepath.Join(tempDir, "*.db")) + if err != nil { + return errs.Wrap(err) + } + + buf := new(bytes.Buffer) + zipWriter := zip.NewWriter(buf) + for _, match := range matches { + data, err := os.ReadFile(match) + if err != nil { + return errs.Wrap(err) + } + w, err := zipWriter.Create(filepath.Base(match)) + if err != nil { + return errs.Wrap(err) + } + _, err = io.Copy(w, bytes.NewReader(data)) + if err != nil { + return errs.Wrap(err) + } + } + if err := zipWriter.Close(); err != nil { + return errs.Wrap(err) + } + + if err := os.WriteFile(outFile, buf.Bytes(), 0644); err != nil { + return errs.Wrap(err) + } + + return nil +} diff --git a/storagenode/storagenodedb/storagenodedbtest/run.go b/storagenode/storagenodedb/storagenodedbtest/run.go index 9a024443a..2c4f1a06b 100644 --- a/storagenode/storagenodedb/storagenodedbtest/run.go +++ b/storagenode/storagenodedb/storagenodedbtest/run.go @@ -34,6 +34,7 @@ func Run(t *testing.T, test func(ctx *testcontext.Context, t *testing.T, db stor log := zaptest.NewLogger(t) storageDir := ctx.Dir("storage") + cfg := storagenodedb.Config{ Storage: storageDir, Info: filepath.Join(storageDir, "piecestore.db"), @@ -42,7 +43,7 @@ func Run(t *testing.T, test func(ctx *testcontext.Context, t *testing.T, db stor Pieces: storageDir, } - db, err := storagenodedb.OpenNew(ctx, log, cfg) + db, err := OpenNew(ctx, log, cfg) if err != nil { t.Fatal(err) } diff --git a/storagenode/storagenodedb/storagenodedbtest/snapshot.go b/storagenode/storagenodedb/storagenodedbtest/snapshot.go new file mode 100644 index 000000000..d3e609465 --- /dev/null +++ b/storagenode/storagenodedb/storagenodedbtest/snapshot.go @@ -0,0 +1,61 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package storagenodedbtest + +import ( + "archive/zip" + "bytes" + "context" + _ "embed" + "io" + "os" + "path/filepath" + + "github.com/zeebo/errs" + "go.uber.org/zap" + + "storj.io/storj/storagenode/storagenodedb" +) + +//go:generate go run ./gensnapshot testdata/snapshot.zip +//go:embed testdata/snapshot.zip +var snapshotZip []byte + +// OpenNew opens a new storage node database pre-populated with a newly +// initialized and migrated database snapshot. +func OpenNew(ctx context.Context, log *zap.Logger, config storagenodedb.Config) (*storagenodedb.DB, error) { + db, err := storagenodedb.OpenNew(ctx, log, config) + if err != nil { + return nil, err + } + if err := deploySnapshot(db.DBDirectory()); err != nil { + return nil, err + } + return db, nil +} + +func deploySnapshot(storageDir string) error { + zipReader, err := zip.NewReader(bytes.NewReader(snapshotZip), int64(len(snapshotZip))) + if err != nil { + return errs.Wrap(err) + } + + for _, f := range zipReader.File { + rc, err := f.Open() + if err != nil { + return errs.Wrap(err) + } + data, err := io.ReadAll(rc) + if err != nil { + return errs.Wrap(err) + } + if err := os.WriteFile(filepath.Join(storageDir, f.Name), data, 0644); err != nil { + return errs.Wrap(err) + } + if err := rc.Close(); err != nil { + return errs.Wrap(err) + } + } + return nil +} diff --git a/storagenode/storagenodedb/snapshot_test.go b/storagenode/storagenodedb/storagenodedbtest/snapshot_test.go similarity index 76% rename from storagenode/storagenodedb/snapshot_test.go rename to storagenode/storagenodedb/storagenodedbtest/snapshot_test.go index d352910b4..854b5667c 100644 --- a/storagenode/storagenodedb/snapshot_test.go +++ b/storagenode/storagenodedb/storagenodedbtest/snapshot_test.go @@ -1,7 +1,7 @@ // Copyright (C) 2022 Storj Labs, Inc. // See LICENSE for copying information. -package storagenodedb +package storagenodedbtest import ( "context" @@ -17,6 +17,7 @@ import ( "storj.io/common/testcontext" "storj.io/private/tagsql" "storj.io/storj/storage/filestore" + "storj.io/storj/storagenode/storagenodedb" ) // TestSnapshot tests if the snapshot migration (used for faster testplanet) is the same as the prod migration. @@ -25,24 +26,25 @@ func TestSnapshot(t *testing.T) { ctx := testcontext.New(t) defer ctx.Cleanup() - fromMigrationSteps := getSchemeSnapshot(t, ctx, "migration", func(ctx context.Context, db *DB) error { + fromMigrationSteps := getSchemeSnapshot(t, ctx, "migration", func(ctx context.Context, db *storagenodedb.DB) error { return db.MigrateToLatest(ctx) }) - fromSnapshot := getSchemeSnapshot(t, ctx, "steps", func(ctx context.Context, db *DB) error { - return db.TestMigrateToLatest(ctx) + fromSnapshot := getSchemeSnapshot(t, ctx, "steps", func(ctx context.Context, db *storagenodedb.DB) error { + if err := deploySnapshot(db.DBDirectory()); err != nil { + return err + } + return db.MigrateToLatest(ctx) }) - require.Equal(t, fromSnapshot, fromMigrationSteps, "storagenode/storagenodedb/snapshot.go produces different scheme compared to storagenode/storagenodedb/database.go. "+ - "If you changed the database.go recently, please also change snapshot.go (but instead of adding new migration step, try to modify the existing steps. snapshot.go should have "+ - "just the minimal number of migrations to keep the unit test executions fast.)") - + require.Equal(t, fromSnapshot, fromMigrationSteps, "The database snapshot produces a different scheme than the current storagenodedb migrations. "+ + "If you have introduced a new migration, please run go generate ./storagenode/storagenodedb/storagenodedbtest/testdata to update the snapshot.") } -func getSchemeSnapshot(t *testing.T, ctx *testcontext.Context, name string, init func(ctx context.Context, db *DB) error) schemeSnapshot { +func getSchemeSnapshot(t *testing.T, ctx *testcontext.Context, name string, init func(ctx context.Context, db *storagenodedb.DB) error) schemeSnapshot { log := zaptest.NewLogger(t) storageDir := ctx.Dir(name) - cfg := Config{ + cfg := storagenodedb.Config{ Pieces: storageDir, Storage: storageDir, Info: filepath.Join(storageDir, "piecestore.db"), @@ -50,7 +52,7 @@ func getSchemeSnapshot(t *testing.T, ctx *testcontext.Context, name string, init Filestore: filestore.DefaultConfig, } - db, err := OpenNew(ctx, log, cfg) + db, err := storagenodedb.OpenNew(ctx, log, cfg) if err != nil { require.NoError(t, err) } @@ -69,7 +71,7 @@ type schemeSnapshot map[string]dbScheme // dbScheme represents uniq id (table/name/...) -> sql. type dbScheme map[string]string -func getSerializedScheme(t *testing.T, ctx *testcontext.Context, db *DB) schemeSnapshot { +func getSerializedScheme(t *testing.T, ctx *testcontext.Context, db *storagenodedb.DB) schemeSnapshot { dbs := schemeSnapshot{} for dbName, db := range db.SQLDBs { s := dbScheme{} diff --git a/storagenode/storagenodedb/storagenodedbtest/testdata/snapshot.zip b/storagenode/storagenodedb/storagenodedbtest/testdata/snapshot.zip new file mode 100644 index 0000000000000000000000000000000000000000..e28c514ff3c671284360d626e74821187f327047 GIT binary patch literal 10105 zcmaia1z40@w6@0p6%Qc|0*Z7B2q>vY3Bu6bJv2kNib{hp($d}CFoe=|h@oNV9=Zo+ z?tpTR$N&E4E}jQA!(QLBH|yQ+de_>Yob-*G1lMk(zkauS=no9^t&N<4Mh<3dMg|)r zGNA){xQ#oRQ^ZE!AP?R=ZDoznwWsnp)gCgi+wJW znR z6hv&FZC527-i=y2-bofZ)jn3Nnsm5WPvZ?v75C~n^)WJGe|LQca^K3Ph|Q0%`M*4{ z#b2E>-YvikURN>@A|6}BfW&V%Ii@U`^w>73Qtz(J65`$%pdmV;Wnt)@U1SN z0m27;&a@`E`e))~-zWFq03S_7+Ru(?EYFpD?o4yXMWin*iiqtVnAEaSTPPls`y&Kn zI%*TPo__5;9g0I{nJ#5JR?fO1Pjv;is`525nQguneC;%c6RNWnQkUM93Y5H>kt4J??<>YPXsi$_8c^a1t#~zy5I%+s0R|NSw>TA-~)P>SA=wlK;^Aa!^5q z5WXBtY}ufHqEK?un74_xnT0kleD$tfll%YTNJAE3oy<1|uVGkz_iiUuNV`dpMed^iU6!m(`=;1Kxq%^y7x#f6B~3F`R@|f) z_HQXuDpv?=Hb~t*z7NlqBt$tXHm)9(J`#iQJ_y;>k!&y2jhCif8S#3yPq1hdsKm6u zC3WW!p-B5BVV#mb?Tz>i|@T>;8ES~N)cnoPGuVWy*hV&$_ zJ8V1LEI34pxK6#hUH3Yzbt;SNeq|uyz%(k8UNv)nZDDUuvBu@oW4mWupq2$jm zrdwB+!VVD-7M7G1Pq^>%PATOvqKsd@Su=R`W3yrkY==M@#6Z3Hr_UFL2KJPo+@Qx> z8V&-`J7Q^OFe+!v_ZZ159g?T1dB+yXW~w?}@=JbARSa_)---VeASJ?4W_v=}Pkv;4 z7~zh55)HGomYb%Ky|HWkgf!V~yyHdsT)-9^nePtBzldIZOdjUN<`*-pJ4QJ?~w9_yb@WQ|ZQ(Pqn~@0g`$U?PC-UOsB`B{6h} zILefmKFVfgZ;;m^w6}BqEt_&P(sKieqX`8boRiQ(c?lj9(oU?wP>OMs@gvigDPgzM zdDUVhvP+oc6R{5BAMAcYK0|cf5d*ZpU^Nc%aMZ2u2wPM>EuurTS(L*~`N_gkAyt9a z8Q{Iv&P1$$D@GBHzU{RxHJn2RhGDs_k4?aqY>;Qti=*P%N|(hV?K;pMbZ^%$5@8 zoL9lecMxic%9D2WNxLT5doL;L2h8ner)1YRHStN&Lw3{$nRR8ShWSsxX) zv4ZfNEWC~V;Gm~6ZAWuHvsG*!71v;cEIFxIp}0Ib=mYSww!i|v_g*Wu%--knEMU3S zu*sR<^G_C|1ud(|vm_YxE}mk(i9*~VTA3XzBYIA?85a1FVHU9&J(EzaiHd4NJ~+2_ zTy%D|OdEHw3fE%I*Rjd6Sa?o>OhU#I$Y+npK0iXsn!f<3uA(_o-eGNCFt)&BF^>Mw zgpaigsF5E~Fy$2nvM9c6 zy#1O|{PiZgZfk;8BYK8@;3GY{Cxavwi_^jGyh3lg@vEHBL!6n%RS}==ysr$%!(3}v zT>^Pe4z|pSBnr5@K{O6wT9CSDng>I0r&RePeVDU$WxwN87NRf|rd8YPz+2f*;fY;( z&=WW^HFqFWS*ek)k_$C2zx=V55Yuxm5!wx1+sL(vg0bpC_!>NR)=usB7fR>bGTY8q z=R*@Hk497xKD@BgJ>uLWL?JsnTd!)@vTj2DDRdVBtzAbg;k02+%^kIb1_FrP_k@>) z`x>GfxDf7#>l+~PLGO$P8SiMJ7;711^x4zrF&m*MG8CtmkWdwH^yf;3C$!wmWW3(K zby=BwWWM#x^^K$i?)5)J3(p_je2WBw2a$6=M>CSt7~&C0IC<6M?3zS(x?kPJR(d-K z;XXlrp2Gv*+`*efu|A=`dF%C2e#cSgFtg~rTkGvN(7)z&I5YFu>JeiNiycU=7I2@) zwz&Df@JYSyG}FkD$v?bMACSI+Yi#z28n3#{R0g75>=&eO+uQh%n&8n?1>Kekiywh0 z22Ou5pmumcYz#CcspiJU@#Lk38hDwIT(UhP4=XG_qng`QrFc$0;x5sUPwUfCIso&q z&tP+RTIW4cpt($f5{oD<Tl2_l3OC%z^>Rj_2Lz!>qL2eSdTJfEOzq) zZoT*WXc+Q9PJV#ZVpn`Dg*p}ImNYd@G*4bfg~6*2B+^89${gwzyIBSwZxEY)IRaCK_D6+ENuHcm9Slz6^(-QbF+TAG$ zHsf5|R6b*(JSRdl(QqgmL{Jh(Ejz!Sb~U-r3GS+WW}klFAw`?G^WRPyw`L%l*b{kEA(M%KhCrlgIo4rBMa$T8q_^)M2+;${Lo|*6Oo{ z0j5IUy~}ZmX&TD*9>`8uyJ6V!tmaOi0KPuKhn((AO5J0_Er)nkE<7gr)`pT`%&)zf z(=J}J?8nc`jfYO2Pm=OD#p*~Ka{7NaM(Wc>+-?qKVqH(F(0-@P9GHR7u_G`Ym)+Zz zLugTcrw>~ctAVVQ!*xRyYHJ?HWEpB?*s0GfA=jd^IrW;A`QB1f_vY&c86~Ql* z%7twUPZ+PDep0)2Efsq7L+~Wp&iB<^eE#CU;(iA`!3Jbx47v>XSdaOH6anUTSp4@% z4=vN>7Ft`jBMcTYa&_QZtDIRlQ`1lwx#qh_whsm}vB@PHQuE*8Y&XXDaElhCP> zg}GsPn8YxUnfuULzr3MSMW}FSQ|glEj|b~Cy9sl0K=H(oF5hN~@%qNackr9>Sau!S z6JgnyhXXIWMoaJI#PtP5J>_~AOP3A~@K-)C($6j`PS$$8+&v#9a9gYohu)jGZ85+8 z{L@QtrqTo5uN;T+JRxJ6`Z0Wn$2m{lJwv)IDxz{U=>5joYYsm>j_Q} zk67RL>Xm2V#PsWSw+@!HItY1qBLFA;(c$wlI;IMbr3%hE{|NTzNmB5?k9S@Q;_9># z%|42rsSoHe3&258+ZydB@)&Z;$LC^gf^Wapu?1DnyERi$)G;nVn9q1=7DBzag@spE zEQR>Se&*fA$>x)vc~uX|%JuaMcuP}chHpAxC1-H$Ff^hrY?~kYC2to=r9F6{kghP` zoV{CAkvg&u2Z;?jJC&`aGDVc-Wc1rjq-=sQHL7!zot+QblqnjnGJJOf``=Jjph;){ zazSBw{{>}y^aNX=v7xc9v5PGbbV*|uEH(xJkO4?Ax7FniGm#7##Nl)meaV=954)r+ zo^q^eWJVIm*~wU#zfy6|18=uAmkm6YK?t4FonpTMc)k?mVF}EV{-DWx;)PwrkLcbg z#o>y3GdjU4NxPEPpZ7ce<0?SBrh}RRE-4jHersf8Prbi%&~2%#@XQuO&|?RfSZWj^O zS`k+&Gz(L!t1|Lg51A_*fRt16!{gf&-OjYd*B6YZ2(4BtnBOI)-}m_IMR%F^kp~dm zkZk9h3|FuvqIqxE(fh54Qn=lWVmj8;j`Coex`xRlb8B6=XsQO&AI8@7+V z2BM)h#$~eb$m?#3Dpl{WPDvd15U2k4SjP`K;Evm-hu!N@b=~K*rLyYG1pZs5PnQB>L*2kXd9qQPr>h8cA??r>?J#-;x;H`_SBR$5 z{lbya!lt%iphWXy-nBCs3k@~IS>w|;T=6{_J<6R+!wJzAOS|F4v6|}Kpy%L$)&PAu zyS|L^)0E;e1CphZlcDxGl=lz!w;f0HSKMs&J#gsr1)y#J3(y1?6D|R5Z>w);Y^3XG zZ*26VCY2_?K(r~>&N7uBg1>CR2r-i$Aa3Fe&U?)`v{G?<;wJYy5u7fbn8=Stk2lNA zZFYtr`3fs>+cGYMIJ7(p!P)LBjN9j0bX9a&p=EP8=czhbW63B+aB@_) zD2<<)$x|}XeLugFeczX>H?8Wx)h+h-tFbiBOstpLK(IlPA6t9UG57quJ4&i`EEXH) z4AoB?XOhJJR&iH|HV6Ov+VSD|%ucr@ME(gzUuspVY)bEITlVk|8jiJLDldmfVcnKn z?wT^8ng>($k!`gH6kDvBURb@a>vKqiAR@##I%o&vhmns zNtV59exj?clv%l-r;?9ByCVzu?w&=d*vzT`OuuEY(~Hc zrk)4I!m`eFbhSxz7J8UD3kMRoe(d{Nw1~*<*aq%C0lBNfL($v?)%T~wApM^233Pw( zs&tVfY<}ixE6Kl(tWTaq|RF(3w|7&47kh^-8h`GX0)3`Dg zuXPpNU<3#Cm(UsUYO7ahSDN3RcL0Kh*_6MQa4|B+4`eJZ6~IspG<%%+{8!5nQh&_P zE75>P)(;I}p`O%CcY!mx|H9ctGwKp&f7dxL(Yg;U1EL&~Et6|1sdaP!6>_4ZI7x6m zO`oysxQr|p5(6${-<}FuiewjN%Cq}kYQQ&^JT(goO}nW0ff(;;cl;Ioj1fq2EOG8W zTZxCP=|atZK#ZRjHD9t%TLcUnYQK7tNo+-JUgl33U*TQ0KK)}4#}ZBevYy_qf>K2j z()E1*N^`4yt++*l^e@u|GGywd8k9r;w(A`CZn^K4A2;Ga8u2@PgLMwo#y9s6ywtc_ z+NL(0p!+3XPmuW&8v^TmwV7jP>k|s8-xMEHnbOs-t8+l>Eseuvj2y#BawGgPUA^&- z8q_n#R-NLvmU&LUt~8W`_EctV7XlStZ{^6*Q$h0|*Md3YT=s%5RbXaJuK zSl%mehPLwcBRYntIsHSFjk>rZ0UQ9w#e3c=wO?;GlS_Ls3mtCa^e?~A3V6|u#j$uE zs~+uY=9P$S!#=tqnfW6$Hh%=4lkaKqxBpWBqRZiH zwjiJ((AxA;^7mN}zQD!UF7~;rWIm;dEuS7%MU&#c;rQwm4bA<98_iZgGHgOj3w(Bc zvALUDhd8R=b8Qz`PxacH2$>{lc#}@OqNXUkD;9TvvH$}s!iKnIF5vSt3Xry2?;BAz zys6YgFG2pYSn2#qWaLo<(*&L?tm5&O0hIN}#9B9{Y45N1Y#H7doSjXiE1LG7mi! z1VnZu0S+eK(#z(L93j>k37O*s7rQg-rR8!iUZ~jh-y@XcW*CO8zt4Gu{=iqpwoaX3f^$ z(#Te+$uIx^4Dv;>05Z09bog5ojZc}+i(e@Hh)_DE{=-gHY5L?c4w-@e75g%lA)Pn3 z%fZ@h^=fyyHQbwX%4rL^14h_!Z?kN8Qt9jJB_^rXZ;I(AReF9PqL($00MS%xO0Ar{ z-1hB-@ip6Hkb=(gmaswq%`f}#)g*A_lHWpFNgmNW{-8qVee1(tlpG1{}#tomsJ?3 z)YZde9m!nIlpy%2a%6-}EMaD|$qjPu(y?Y=^Wi{#Nc+UP@lF-g8ZwaGT4#+)_a%pd zSS5C_*gzxVaNEW$mJlB*A6pWZVHqig(%diI+EX82j^Mb2V7aVHP(KJ_gEX(dq7Zf? zbk|0GUyhHYx7|z95wwn62V_R7wBc7`(*f{WYWGPik=~rnNfB?1$hOg-Ncq2nN>Jg^BZW-(kZ3V8rzcfDYdtI z0+%n%W+0cnp`!<>O?P#a{PB~hKfnpXU&aXEoC?P~cyzYFBZ)~+rn^!*v#)?I9Nd{6 zxF`C!+JbC7Z@QtiF7CVc5bs8t;6)>Q^c1&|8eedkzJ^NdcnB{uOxJk}`iISetbYo`NyhpHO|s_Xfx8~MZ}G%dUjreZB52$j0;mZl?g`qH-{ zL`OUCCOgq=%lhz!r-pQ#ryw#0>Ebb%lgr$4-t&Np^WcvXPFbP$8mgbC1KrN|tOzOa zMiG~o{}gCwP7-Fko$R*~D7X%}|PTeV*Dl#?OZ zL3PlH@muU{>`?kCWjQx^ED|`UZGiuNy-jA){fu_Z-ZE~byFaFuPWo|pY&lDtQMX

{KxVR)G_s ztl)tKprd$nGI_UlWO~1o`4{sHV|fXHQMQa40Sa1c) zWpH4iP?t9x7u!6uk-VL}m`8mvzO+ez_pK|QZ<9p$wO`a{djFN=FDSrX-@(|@66j!j zS?rJN0(Mm%U~VgJ|8ekc2I}WeCs9bFI2b8>t9$aZlDLYTbW}G>M)v-SEJl868)HZ=<6oWWO=cK;iu~}qZnB)f! zamGh9Qn%G9tuVc*@I)ddC8_tsI*=nEPFKmu`{Xn`rUsA8W6w^`TM&WYZT4a>6ThiW&NtzRvm{K9hkIE9 zBB?&O@A%JEd}}eC zO5OISAH8Iq#Z`L+S~yL&3|7(S(|R;!OPzfNLL5|C>>m;oIuPGaxtK7TZq+ccm_x{htJOJ(_2@?|NBP7mCk6Xl-fS~iO-^y z@DHEOIv-{WD>S49xsLsGmkU8(ROv)0I|Aenhwdwb7V9)rN)b63OG+|bM5_sT9h_EW zEaT@3hOP}+z!s70G=k2stf9WTaTVnNG44!8MJr|2+@)QOY!Kpc!Ek~!jH_{YcG_?kgyyH(A7ywevL z(f;zpRa`tUo2JFdxEl|gU_0_O5i`7EG3fz<+`phbpFl3TaYbX|oMzx#Y8-y7v())LTg*wHD?=6QcG^F-uSG z2;%@;+Tnu|@G-NI(G|fp3z|Ky;yj_R(zN0GsZ6naKw%cU@p9}b=e4GgvfydGB<-h$ zp7)Ju1M4&jPuWxRon5PH3}bQybqLhyM(q94$*iC+r|YQqa@GrfvTQ|LYT0LW^D zg1>k4&(Uh(Dd)@r(`v{vCZvICd(=yB!o&7~{h{qh1)bN73)mlj=p@mp=BM&U(^;vE z2=Zr9UbX6V^NQe5H_Ix+{!fi_(ZRNNumS0t8lwwk^!Lk3xyyR|1p&s4;X~2j4`3!N zCFQV?l0TH#Dw1dKD>FT0Pl?rPR2S7&vyM@TESj3@NRML+SM9>uCi4uqd)JCJt;B+6 z9Yu7H%RY}Z-jH4VaHV-SZ3Zwqv|&!r0!&!Ai#s;%YQ=3nUy|OhFLs! z6bF_A)|?M~$5lh2^DwbF@HpPJr3F^YvIlc#Wc86= z3p$PB{y45Vc9xX%nK-CLd*2anNvF%U-+2!Bm*Lcg7+;+5GdXMGI}p6Vb}Sz=sENU< zRxrr!B((lTGqh4BaKAp~xxXSGJU#LUoe@6N3&CVViRBdP(qKfgVR8CNkc+Ct8x!3cAsYZum zJdc;JFgQfMkgUf^+op0KL$xd;V@J=Z6txtgVpUGo^SIt>Xy;rQ9elLC*oEWvUR6b} z|0#vK8B9Okyg;eme=z|zdcsAEMc3XK1k|^@#E-7Upd9)JU;K#RnD0%FZ4z@Yx_{JuBpBv&6sS~5r#1@?^<`4Tre#4FoDZSBfoIK0NK_O3G|Qnk!k0?5&H^@-k{5fhMsKeZa^lNUGh$t*nns%qWW~{i@ z&7PmXa!^IcrK)(US%3t9X$hdHAfC5l4YpK8J>RuZjE;%#H+trL3XT%C=Pj0+J=LKy z7QARUT$pVl1@81!RfEK@ULZ$lsa%}n3)@V*j_3c4nXj%>>TiA-!gt+SEiOyTz9xj@H-Q1pX-RKuBBd53Km7`$wS z{BfP&+COg!{rYam)!HtW@WXz6Z%9u1-=4qTAzvu0e~baW=zqOQ^1JJ=_i{gv!%uHM zv@ZVHiuv8?*GrwNf7B1JN9a8E`vu