From 84d02f7fbfbbba2e4dbc3324543832b2335ed6c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Wed, 1 Jun 2022 12:18:56 +0200 Subject: [PATCH] metabase: use predefined snapshot for unit tests avoid migration Change-Id: I6225ae32c312030e3559d8ed0faa137ffc3cc5b7 --- cmd/metabase-orphaned-segments/main_test.go | 2 +- go.mod | 2 +- go.sum | 4 +- private/testplanet/satellite.go | 2 +- satellite/metabase/db.go | 119 ++++++++++++++++++++ satellite/metabase/metabasetest/run.go | 10 +- satellite/metabase/migrate_test.go | 61 ++++++++++ testsuite/go.mod | 2 +- testsuite/go.sum | 4 +- 9 files changed, 197 insertions(+), 9 deletions(-) create mode 100644 satellite/metabase/migrate_test.go diff --git a/cmd/metabase-orphaned-segments/main_test.go b/cmd/metabase-orphaned-segments/main_test.go index e422ebfab..48033fb0b 100644 --- a/cmd/metabase-orphaned-segments/main_test.go +++ b/cmd/metabase-orphaned-segments/main_test.go @@ -122,7 +122,7 @@ func test(t *testing.T, prepare func(t *testing.T, ctx *testcontext.Context, raw require.NoError(t, err) defer ctx.Check(metabaseDB.Close) - err = metabaseDB.MigrateToLatest(ctx) + err = metabaseDB.TestMigrateToLatest(ctx) require.NoError(t, err) prepare(t, ctx, metabaseTempDB, metabaseDB) diff --git a/go.mod b/go.mod index 14ed7fe4f..7db3eb9a7 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( storj.io/common v0.0.0-20220518091716-ec9c16f58d50 storj.io/drpc v0.0.30 storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a - storj.io/private v0.0.0-20220512110458-f0ade82ff3db + storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 ) diff --git a/go.sum b/go.sum index cdd04c6b1..7961f4424 100644 --- a/go.sum +++ b/go.sum @@ -937,7 +937,7 @@ storj.io/drpc v0.0.30 h1:jqPe4T9KEu3CDBI05A2hCMgMSHLtd/E0N0yTF9QreIE= storj.io/drpc v0.0.30/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k= storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s= -storj.io/private v0.0.0-20220512110458-f0ade82ff3db h1:JHzjxOn/8QEAh02ZDG7b/yLDfMFxzsD51vmry68v11k= -storj.io/private v0.0.0-20220512110458-f0ade82ff3db/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM= +storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 h1:wAik82Huwh8D8oy57+cQbFMU2mXWVw/3Op9od3ldtLA= +storj.io/private v0.0.0-20220525153054-e5ea0fc18d84/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM= storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk= storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85/go.mod h1:RtbTI2i9JjfWTCaSNM2bVUNP1hjAvfv9VH6D35jFh18= diff --git a/private/testplanet/satellite.go b/private/testplanet/satellite.go index b685d0a39..474dfa424 100644 --- a/private/testplanet/satellite.go +++ b/private/testplanet/satellite.go @@ -523,7 +523,7 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int return nil, err } - err = metabaseDB.MigrateToLatest(ctx) + err = metabaseDB.TestMigrateToLatest(ctx) if err != nil { return nil, err } diff --git a/satellite/metabase/db.go b/satellite/metabase/db.go index 76c0eb167..9c016d03d 100644 --- a/satellite/metabase/db.go +++ b/satellite/metabase/db.go @@ -129,6 +129,125 @@ func (db *DB) DestroyTables(ctx context.Context) error { return Error.Wrap(err) } +// TestMigrateToLatest replaces the migration steps with only one step to create metabase db. +func (db *DB) TestMigrateToLatest(ctx context.Context) error { + // First handle the idiosyncrasies of postgres and cockroach migrations. Postgres + // will need to create any schemas specified in the search path, and cockroach + // will need to create the database it was told to connect to. These things should + // not really be here, and instead should be assumed to exist. + // This is tracked in jira ticket SM-200 + switch db.impl { + case dbutil.Postgres: + schema, err := pgutil.ParseSchemaFromConnstr(db.connstr) + if err != nil { + return errs.New("error parsing schema: %+v", err) + } + + if schema != "" { + err = pgutil.CreateSchema(ctx, db.db, schema) + if err != nil { + return errs.New("error creating schema: %+v", err) + } + } + + case dbutil.Cockroach: + var dbName string + if err := db.db.QueryRowContext(ctx, `SELECT current_database();`).Scan(&dbName); err != nil { + return errs.New("error querying current database: %+v", err) + } + + _, err := db.db.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s;`, + pgutil.QuoteIdentifier(dbName))) + if err != nil { + return errs.Wrap(err) + } + } + + migration := &migrate.Migration{ + Table: "metabase_versions", + Steps: []*migrate.Step{ + { + DB: &db.db, + Description: "Test snapshot", + Version: 39, + Action: migrate.SQL{ + + `CREATE TABLE objects ( + project_id BYTEA NOT NULL, + bucket_name BYTEA NOT NULL, -- we're using bucket_name here to avoid a lookup into buckets table + object_key BYTEA NOT NULL, -- using 'object_key' instead of 'key' to avoid reserved word + version INT4 NOT NULL, + stream_id BYTEA NOT NULL, + + created_at TIMESTAMPTZ NOT NULL default now(), + expires_at TIMESTAMPTZ, + + status INT2 NOT NULL default ` + pendingStatus + `, + segment_count INT4 NOT NULL default 0, + + encrypted_metadata_nonce BYTEA default NULL, + encrypted_metadata BYTEA default NULL, + encrypted_metadata_encrypted_key BYTEA default NULL, + + total_plain_size INT8 NOT NULL default 0, -- migrated objects have this = 0 + total_encrypted_size INT8 NOT NULL default 0, + fixed_segment_size INT4 NOT NULL default 0, -- migrated objects have this = 0 + + encryption INT8 NOT NULL default 0, + + zombie_deletion_deadline TIMESTAMPTZ default now() + '1 day', + + PRIMARY KEY (project_id, bucket_name, object_key, version) + ); + CREATE TABLE segments ( + stream_id BYTEA NOT NULL, + position INT8 NOT NULL, + + root_piece_id BYTEA NOT NULL, + encrypted_key_nonce BYTEA NOT NULL, + encrypted_key BYTEA NOT NULL, + remote_alias_pieces BYTEA, + + encrypted_size INT4 NOT NULL, + plain_offset INT8 NOT NULL, -- migrated objects have this = 0 + plain_size INT4 NOT NULL, -- migrated objects have this = 0 + + redundancy INT8 NOT NULL default 0, + + inline_data BYTEA DEFAULT NULL, + + created_at TIMESTAMPTZ DEFAULT now() NOT NULL, + repaired_at TIMESTAMPTZ, + expires_at TIMESTAMPTZ, + + placement integer, + encrypted_etag BYTEA default NULL, + + PRIMARY KEY (stream_id, position) + ); + CREATE SEQUENCE node_alias_seq + INCREMENT BY 1 + MINVALUE 1 MAXVALUE 2147483647 -- MaxInt32 + START WITH 1; + CREATE TABLE node_aliases ( + node_id BYTEA NOT NULL UNIQUE, + node_alias INT4 NOT NULL UNIQUE default nextval('node_alias_seq') + ); + + CREATE TABLE segment_copies ( + stream_id BYTEA NOT NULL PRIMARY KEY, + ancestor_stream_id BYTEA NOT NULL, + + CONSTRAINT not_self_ancestor CHECK (stream_id != ancestor_stream_id) + ); + CREATE INDEX ON segment_copies (ancestor_stream_id);`, + }, + }, + }, + } + return migration.Run(ctx, db.log.Named("migrate")) +} + // MigrateToLatest migrates database to the latest version. func (db *DB) MigrateToLatest(ctx context.Context) error { // First handle the idiosyncrasies of postgres and cockroach migrations. Postgres diff --git a/satellite/metabase/metabasetest/run.go b/satellite/metabase/metabasetest/run.go index 2cab606b5..69d4924b8 100644 --- a/satellite/metabase/metabasetest/run.go +++ b/satellite/metabase/metabasetest/run.go @@ -4,6 +4,7 @@ package metabasetest import ( + "context" "testing" "github.com/spf13/pflag" @@ -19,6 +20,13 @@ import ( // RunWithConfig runs tests with specific metabase configuration. func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB)) { + RunWithConfigAndMigration(t, config, fn, func(ctx context.Context, db *metabase.DB) error { + return db.TestMigrateToLatest(ctx) + }) +} + +// RunWithConfigAndMigration runs tests with specific metabase configuration and migration type. +func RunWithConfigAndMigration(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB), migration func(ctx context.Context, db *metabase.DB) error) { for _, dbinfo := range satellitedbtest.Databases() { dbinfo := dbinfo t.Run(dbinfo.Name, func(t *testing.T) { @@ -37,7 +45,7 @@ func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontex } }() - if err := db.MigrateToLatest(ctx); err != nil { + if err := migration(ctx, db); err != nil { t.Fatal(err) } diff --git a/satellite/metabase/migrate_test.go b/satellite/metabase/migrate_test.go new file mode 100644 index 000000000..108cca33e --- /dev/null +++ b/satellite/metabase/migrate_test.go @@ -0,0 +1,61 @@ +// Copyright (C) 2021 Storj Labs, Inc. +// See LICENSE for copying information. + +package metabase_test + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "storj.io/common/testcontext" + "storj.io/private/dbutil/dbschema" + "storj.io/private/dbutil/pgutil" + "storj.io/storj/satellite/metabase" + "storj.io/storj/satellite/satellitedb/satellitedbtest" +) + +func TestMigration(t *testing.T) { + for _, dbinfo := range satellitedbtest.Databases() { + t.Run(dbinfo.Name, func(t *testing.T) { + + ctx := testcontext.NewWithTimeout(t, 8*time.Minute) + defer ctx.Cleanup() + + prodSnapshot := schemaFromMigration(t, ctx, dbinfo.MetabaseDB, func(ctx context.Context, db *metabase.DB) error { + return db.MigrateToLatest(ctx) + }) + + testSnapshot := schemaFromMigration(t, ctx, dbinfo.MetabaseDB, func(ctx context.Context, db *metabase.DB) error { + return db.TestMigrateToLatest(ctx) + }) + + prodSnapshot.DropTable("metabase_versions") + testSnapshot.DropTable("metabase_versions") + + require.Equal(t, prodSnapshot.Schema, testSnapshot.Schema, "Test snapshot scheme doesn't match the migrated scheme.") + require.Equal(t, prodSnapshot.Data, testSnapshot.Data, "Test snapshot data doesn't match the migrated data.") + + }) + } +} + +func schemaFromMigration(t *testing.T, ctx *testcontext.Context, dbinfo satellitedbtest.Database, migration func(ctx context.Context, db *metabase.DB) error) (scheme *dbschema.Snapshot) { + db, err := satellitedbtest.CreateMetabaseDB(ctx, zaptest.NewLogger(t), t.Name(), "M", 0, dbinfo, metabase.Config{ + ApplicationName: "migration", + }) + require.NoError(t, err) + + defer ctx.Check(db.Close) + + err = migration(ctx, db) + require.NoError(t, err) + + scheme, err = pgutil.QuerySnapshot(ctx, db.UnderlyingTagSQL()) + require.NoError(t, err) + + return scheme +} diff --git a/testsuite/go.mod b/testsuite/go.mod index 6cbf0ef33..b847b75d4 100644 --- a/testsuite/go.mod +++ b/testsuite/go.mod @@ -12,7 +12,7 @@ require ( go.uber.org/zap v1.17.0 storj.io/common v0.0.0-20220518091716-ec9c16f58d50 storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31 - storj.io/private v0.0.0-20220512110458-f0ade82ff3db + storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 storj.io/storj v0.12.1-0.20220325094222-4a6c2e2a9ce8 ) diff --git a/testsuite/go.sum b/testsuite/go.sum index 3b4eaa5cd..d0e42a82b 100644 --- a/testsuite/go.sum +++ b/testsuite/go.sum @@ -1486,8 +1486,8 @@ storj.io/monkit-jaeger v0.0.0-20210225162224-66fb37637bf6/go.mod h1:gj4vuCeyCRjR storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k= storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s= storj.io/private v0.0.0-20210810102517-434aeab3f17d/go.mod h1:wvudoWSyOyB2daZNaMykjjqsSUad/ttFlUwgelg9+qg= -storj.io/private v0.0.0-20220512110458-f0ade82ff3db h1:JHzjxOn/8QEAh02ZDG7b/yLDfMFxzsD51vmry68v11k= -storj.io/private v0.0.0-20220512110458-f0ade82ff3db/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM= +storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 h1:wAik82Huwh8D8oy57+cQbFMU2mXWVw/3Op9od3ldtLA= +storj.io/private v0.0.0-20220525153054-e5ea0fc18d84/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM= storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo= storj.io/uplink v1.7.1-0.20211103104100-a785482780d8/go.mod h1:pKqsMpNMIAz//2TXzUGOR6tpu3iyabvXV4VWINj4jaY= storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk=