metabase: use predefined snapshot for unit tests avoid migration

Change-Id: I6225ae32c312030e3559d8ed0faa137ffc3cc5b7
This commit is contained in:
Márton Elek 2022-06-01 12:18:56 +02:00 committed by Elek, Márton
parent 6cdd250019
commit 84d02f7fbf
9 changed files with 197 additions and 9 deletions

View File

@ -122,7 +122,7 @@ func test(t *testing.T, prepare func(t *testing.T, ctx *testcontext.Context, raw
require.NoError(t, err) require.NoError(t, err)
defer ctx.Check(metabaseDB.Close) defer ctx.Check(metabaseDB.Close)
err = metabaseDB.MigrateToLatest(ctx) err = metabaseDB.TestMigrateToLatest(ctx)
require.NoError(t, err) require.NoError(t, err)
prepare(t, ctx, metabaseTempDB, metabaseDB) prepare(t, ctx, metabaseTempDB, metabaseDB)

2
go.mod
View File

@ -54,7 +54,7 @@ require (
storj.io/common v0.0.0-20220518091716-ec9c16f58d50 storj.io/common v0.0.0-20220518091716-ec9c16f58d50
storj.io/drpc v0.0.30 storj.io/drpc v0.0.30
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a
storj.io/private v0.0.0-20220512110458-f0ade82ff3db storj.io/private v0.0.0-20220525153054-e5ea0fc18d84
storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85
) )

4
go.sum
View File

@ -937,7 +937,7 @@ storj.io/drpc v0.0.30 h1:jqPe4T9KEu3CDBI05A2hCMgMSHLtd/E0N0yTF9QreIE=
storj.io/drpc v0.0.30/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg= storj.io/drpc v0.0.30/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k= storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k=
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s= storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s=
storj.io/private v0.0.0-20220512110458-f0ade82ff3db h1:JHzjxOn/8QEAh02ZDG7b/yLDfMFxzsD51vmry68v11k= storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 h1:wAik82Huwh8D8oy57+cQbFMU2mXWVw/3Op9od3ldtLA=
storj.io/private v0.0.0-20220512110458-f0ade82ff3db/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM= storj.io/private v0.0.0-20220525153054-e5ea0fc18d84/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk= storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk=
storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85/go.mod h1:RtbTI2i9JjfWTCaSNM2bVUNP1hjAvfv9VH6D35jFh18= storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85/go.mod h1:RtbTI2i9JjfWTCaSNM2bVUNP1hjAvfv9VH6D35jFh18=

View File

@ -523,7 +523,7 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
return nil, err return nil, err
} }
err = metabaseDB.MigrateToLatest(ctx) err = metabaseDB.TestMigrateToLatest(ctx)
if err != nil { if err != nil {
return nil, err return nil, err
} }

View File

@ -129,6 +129,125 @@ func (db *DB) DestroyTables(ctx context.Context) error {
return Error.Wrap(err) return Error.Wrap(err)
} }
// TestMigrateToLatest replaces the migration steps with only one step to create metabase db.
func (db *DB) TestMigrateToLatest(ctx context.Context) error {
// First handle the idiosyncrasies of postgres and cockroach migrations. Postgres
// will need to create any schemas specified in the search path, and cockroach
// will need to create the database it was told to connect to. These things should
// not really be here, and instead should be assumed to exist.
// This is tracked in jira ticket SM-200
switch db.impl {
case dbutil.Postgres:
schema, err := pgutil.ParseSchemaFromConnstr(db.connstr)
if err != nil {
return errs.New("error parsing schema: %+v", err)
}
if schema != "" {
err = pgutil.CreateSchema(ctx, db.db, schema)
if err != nil {
return errs.New("error creating schema: %+v", err)
}
}
case dbutil.Cockroach:
var dbName string
if err := db.db.QueryRowContext(ctx, `SELECT current_database();`).Scan(&dbName); err != nil {
return errs.New("error querying current database: %+v", err)
}
_, err := db.db.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s;`,
pgutil.QuoteIdentifier(dbName)))
if err != nil {
return errs.Wrap(err)
}
}
migration := &migrate.Migration{
Table: "metabase_versions",
Steps: []*migrate.Step{
{
DB: &db.db,
Description: "Test snapshot",
Version: 39,
Action: migrate.SQL{
`CREATE TABLE objects (
project_id BYTEA NOT NULL,
bucket_name BYTEA NOT NULL, -- we're using bucket_name here to avoid a lookup into buckets table
object_key BYTEA NOT NULL, -- using 'object_key' instead of 'key' to avoid reserved word
version INT4 NOT NULL,
stream_id BYTEA NOT NULL,
created_at TIMESTAMPTZ NOT NULL default now(),
expires_at TIMESTAMPTZ,
status INT2 NOT NULL default ` + pendingStatus + `,
segment_count INT4 NOT NULL default 0,
encrypted_metadata_nonce BYTEA default NULL,
encrypted_metadata BYTEA default NULL,
encrypted_metadata_encrypted_key BYTEA default NULL,
total_plain_size INT8 NOT NULL default 0, -- migrated objects have this = 0
total_encrypted_size INT8 NOT NULL default 0,
fixed_segment_size INT4 NOT NULL default 0, -- migrated objects have this = 0
encryption INT8 NOT NULL default 0,
zombie_deletion_deadline TIMESTAMPTZ default now() + '1 day',
PRIMARY KEY (project_id, bucket_name, object_key, version)
);
CREATE TABLE segments (
stream_id BYTEA NOT NULL,
position INT8 NOT NULL,
root_piece_id BYTEA NOT NULL,
encrypted_key_nonce BYTEA NOT NULL,
encrypted_key BYTEA NOT NULL,
remote_alias_pieces BYTEA,
encrypted_size INT4 NOT NULL,
plain_offset INT8 NOT NULL, -- migrated objects have this = 0
plain_size INT4 NOT NULL, -- migrated objects have this = 0
redundancy INT8 NOT NULL default 0,
inline_data BYTEA DEFAULT NULL,
created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
repaired_at TIMESTAMPTZ,
expires_at TIMESTAMPTZ,
placement integer,
encrypted_etag BYTEA default NULL,
PRIMARY KEY (stream_id, position)
);
CREATE SEQUENCE node_alias_seq
INCREMENT BY 1
MINVALUE 1 MAXVALUE 2147483647 -- MaxInt32
START WITH 1;
CREATE TABLE node_aliases (
node_id BYTEA NOT NULL UNIQUE,
node_alias INT4 NOT NULL UNIQUE default nextval('node_alias_seq')
);
CREATE TABLE segment_copies (
stream_id BYTEA NOT NULL PRIMARY KEY,
ancestor_stream_id BYTEA NOT NULL,
CONSTRAINT not_self_ancestor CHECK (stream_id != ancestor_stream_id)
);
CREATE INDEX ON segment_copies (ancestor_stream_id);`,
},
},
},
}
return migration.Run(ctx, db.log.Named("migrate"))
}
// MigrateToLatest migrates database to the latest version. // MigrateToLatest migrates database to the latest version.
func (db *DB) MigrateToLatest(ctx context.Context) error { func (db *DB) MigrateToLatest(ctx context.Context) error {
// First handle the idiosyncrasies of postgres and cockroach migrations. Postgres // First handle the idiosyncrasies of postgres and cockroach migrations. Postgres

View File

@ -4,6 +4,7 @@
package metabasetest package metabasetest
import ( import (
"context"
"testing" "testing"
"github.com/spf13/pflag" "github.com/spf13/pflag"
@ -19,6 +20,13 @@ import (
// RunWithConfig runs tests with specific metabase configuration. // RunWithConfig runs tests with specific metabase configuration.
func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB)) { func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB)) {
RunWithConfigAndMigration(t, config, fn, func(ctx context.Context, db *metabase.DB) error {
return db.TestMigrateToLatest(ctx)
})
}
// RunWithConfigAndMigration runs tests with specific metabase configuration and migration type.
func RunWithConfigAndMigration(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB), migration func(ctx context.Context, db *metabase.DB) error) {
for _, dbinfo := range satellitedbtest.Databases() { for _, dbinfo := range satellitedbtest.Databases() {
dbinfo := dbinfo dbinfo := dbinfo
t.Run(dbinfo.Name, func(t *testing.T) { t.Run(dbinfo.Name, func(t *testing.T) {
@ -37,7 +45,7 @@ func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontex
} }
}() }()
if err := db.MigrateToLatest(ctx); err != nil { if err := migration(ctx, db); err != nil {
t.Fatal(err) t.Fatal(err)
} }

View File

@ -0,0 +1,61 @@
// Copyright (C) 2021 Storj Labs, Inc.
// See LICENSE for copying information.
package metabase_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.uber.org/zap/zaptest"
"storj.io/common/testcontext"
"storj.io/private/dbutil/dbschema"
"storj.io/private/dbutil/pgutil"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
)
func TestMigration(t *testing.T) {
for _, dbinfo := range satellitedbtest.Databases() {
t.Run(dbinfo.Name, func(t *testing.T) {
ctx := testcontext.NewWithTimeout(t, 8*time.Minute)
defer ctx.Cleanup()
prodSnapshot := schemaFromMigration(t, ctx, dbinfo.MetabaseDB, func(ctx context.Context, db *metabase.DB) error {
return db.MigrateToLatest(ctx)
})
testSnapshot := schemaFromMigration(t, ctx, dbinfo.MetabaseDB, func(ctx context.Context, db *metabase.DB) error {
return db.TestMigrateToLatest(ctx)
})
prodSnapshot.DropTable("metabase_versions")
testSnapshot.DropTable("metabase_versions")
require.Equal(t, prodSnapshot.Schema, testSnapshot.Schema, "Test snapshot scheme doesn't match the migrated scheme.")
require.Equal(t, prodSnapshot.Data, testSnapshot.Data, "Test snapshot data doesn't match the migrated data.")
})
}
}
func schemaFromMigration(t *testing.T, ctx *testcontext.Context, dbinfo satellitedbtest.Database, migration func(ctx context.Context, db *metabase.DB) error) (scheme *dbschema.Snapshot) {
db, err := satellitedbtest.CreateMetabaseDB(ctx, zaptest.NewLogger(t), t.Name(), "M", 0, dbinfo, metabase.Config{
ApplicationName: "migration",
})
require.NoError(t, err)
defer ctx.Check(db.Close)
err = migration(ctx, db)
require.NoError(t, err)
scheme, err = pgutil.QuerySnapshot(ctx, db.UnderlyingTagSQL())
require.NoError(t, err)
return scheme
}

View File

@ -12,7 +12,7 @@ require (
go.uber.org/zap v1.17.0 go.uber.org/zap v1.17.0
storj.io/common v0.0.0-20220518091716-ec9c16f58d50 storj.io/common v0.0.0-20220518091716-ec9c16f58d50
storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31 storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31
storj.io/private v0.0.0-20220512110458-f0ade82ff3db storj.io/private v0.0.0-20220525153054-e5ea0fc18d84
storj.io/storj v0.12.1-0.20220325094222-4a6c2e2a9ce8 storj.io/storj v0.12.1-0.20220325094222-4a6c2e2a9ce8
) )

View File

@ -1486,8 +1486,8 @@ storj.io/monkit-jaeger v0.0.0-20210225162224-66fb37637bf6/go.mod h1:gj4vuCeyCRjR
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k= storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k=
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s= storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s=
storj.io/private v0.0.0-20210810102517-434aeab3f17d/go.mod h1:wvudoWSyOyB2daZNaMykjjqsSUad/ttFlUwgelg9+qg= storj.io/private v0.0.0-20210810102517-434aeab3f17d/go.mod h1:wvudoWSyOyB2daZNaMykjjqsSUad/ttFlUwgelg9+qg=
storj.io/private v0.0.0-20220512110458-f0ade82ff3db h1:JHzjxOn/8QEAh02ZDG7b/yLDfMFxzsD51vmry68v11k= storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 h1:wAik82Huwh8D8oy57+cQbFMU2mXWVw/3Op9od3ldtLA=
storj.io/private v0.0.0-20220512110458-f0ade82ff3db/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM= storj.io/private v0.0.0-20220525153054-e5ea0fc18d84/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo= storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo=
storj.io/uplink v1.7.1-0.20211103104100-a785482780d8/go.mod h1:pKqsMpNMIAz//2TXzUGOR6tpu3iyabvXV4VWINj4jaY= storj.io/uplink v1.7.1-0.20211103104100-a785482780d8/go.mod h1:pKqsMpNMIAz//2TXzUGOR6tpu3iyabvXV4VWINj4jaY=
storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk= storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk=