metabase: use predefined snapshot for unit tests avoid migration
Change-Id: I6225ae32c312030e3559d8ed0faa137ffc3cc5b7
This commit is contained in:
Márton Elek
Elek, Márton
parent
6cdd250019
commit
84d02f7fbf
@ -122,7 +122,7 @@ func test(t *testing.T, prepare func(t *testing.T, ctx *testcontext.Context, raw
|
||||
require.NoError(t, err)
|
||||
defer ctx.Check(metabaseDB.Close)
|
||||
|
||||
err = metabaseDB.MigrateToLatest(ctx)
|
||||
err = metabaseDB.TestMigrateToLatest(ctx)
|
||||
require.NoError(t, err)
|
||||
|
||||
prepare(t, ctx, metabaseTempDB, metabaseDB)
|
||||
|
||||
2
go.mod
2
go.mod
@ -54,7 +54,7 @@ require (
|
||||
storj.io/common v0.0.0-20220518091716-ec9c16f58d50
|
||||
storj.io/drpc v0.0.30
|
||||
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a
|
||||
storj.io/private v0.0.0-20220512110458-f0ade82ff3db
|
||||
storj.io/private v0.0.0-20220525153054-e5ea0fc18d84
|
||||
storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85
|
||||
)
|
||||
|
||||
|
||||
4
go.sum
4
go.sum
@ -937,7 +937,7 @@ storj.io/drpc v0.0.30 h1:jqPe4T9KEu3CDBI05A2hCMgMSHLtd/E0N0yTF9QreIE=
|
||||
storj.io/drpc v0.0.30/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
|
||||
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k=
|
||||
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s=
|
||||
storj.io/private v0.0.0-20220512110458-f0ade82ff3db h1:JHzjxOn/8QEAh02ZDG7b/yLDfMFxzsD51vmry68v11k=
|
||||
storj.io/private v0.0.0-20220512110458-f0ade82ff3db/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
|
||||
storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 h1:wAik82Huwh8D8oy57+cQbFMU2mXWVw/3Op9od3ldtLA=
|
||||
storj.io/private v0.0.0-20220525153054-e5ea0fc18d84/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
|
||||
storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk=
|
||||
storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85/go.mod h1:RtbTI2i9JjfWTCaSNM2bVUNP1hjAvfv9VH6D35jFh18=
|
||||
|
||||
@ -523,7 +523,7 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = metabaseDB.MigrateToLatest(ctx)
|
||||
err = metabaseDB.TestMigrateToLatest(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@ -129,6 +129,125 @@ func (db *DB) DestroyTables(ctx context.Context) error {
|
||||
return Error.Wrap(err)
|
||||
}
|
||||
|
||||
// TestMigrateToLatest replaces the migration steps with only one step to create metabase db.
|
||||
func (db *DB) TestMigrateToLatest(ctx context.Context) error {
|
||||
// First handle the idiosyncrasies of postgres and cockroach migrations. Postgres
|
||||
// will need to create any schemas specified in the search path, and cockroach
|
||||
// will need to create the database it was told to connect to. These things should
|
||||
// not really be here, and instead should be assumed to exist.
|
||||
// This is tracked in jira ticket SM-200
|
||||
switch db.impl {
|
||||
case dbutil.Postgres:
|
||||
schema, err := pgutil.ParseSchemaFromConnstr(db.connstr)
|
||||
if err != nil {
|
||||
return errs.New("error parsing schema: %+v", err)
|
||||
}
|
||||
|
||||
if schema != "" {
|
||||
err = pgutil.CreateSchema(ctx, db.db, schema)
|
||||
if err != nil {
|
||||
return errs.New("error creating schema: %+v", err)
|
||||
}
|
||||
}
|
||||
|
||||
case dbutil.Cockroach:
|
||||
var dbName string
|
||||
if err := db.db.QueryRowContext(ctx, `SELECT current_database();`).Scan(&dbName); err != nil {
|
||||
return errs.New("error querying current database: %+v", err)
|
||||
}
|
||||
|
||||
_, err := db.db.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s;`,
|
||||
pgutil.QuoteIdentifier(dbName)))
|
||||
if err != nil {
|
||||
return errs.Wrap(err)
|
||||
}
|
||||
}
|
||||
|
||||
migration := &migrate.Migration{
|
||||
Table: "metabase_versions",
|
||||
Steps: []*migrate.Step{
|
||||
{
|
||||
DB: &db.db,
|
||||
Description: "Test snapshot",
|
||||
Version: 39,
|
||||
Action: migrate.SQL{
|
||||
|
||||
`CREATE TABLE objects (
|
||||
project_id BYTEA NOT NULL,
|
||||
bucket_name BYTEA NOT NULL, -- we're using bucket_name here to avoid a lookup into buckets table
|
||||
object_key BYTEA NOT NULL, -- using 'object_key' instead of 'key' to avoid reserved word
|
||||
version INT4 NOT NULL,
|
||||
stream_id BYTEA NOT NULL,
|
||||
|
||||
created_at TIMESTAMPTZ NOT NULL default now(),
|
||||
expires_at TIMESTAMPTZ,
|
||||
|
||||
status INT2 NOT NULL default ` + pendingStatus + `,
|
||||
segment_count INT4 NOT NULL default 0,
|
||||
|
||||
encrypted_metadata_nonce BYTEA default NULL,
|
||||
encrypted_metadata BYTEA default NULL,
|
||||
encrypted_metadata_encrypted_key BYTEA default NULL,
|
||||
|
||||
total_plain_size INT8 NOT NULL default 0, -- migrated objects have this = 0
|
||||
total_encrypted_size INT8 NOT NULL default 0,
|
||||
fixed_segment_size INT4 NOT NULL default 0, -- migrated objects have this = 0
|
||||
|
||||
encryption INT8 NOT NULL default 0,
|
||||
|
||||
zombie_deletion_deadline TIMESTAMPTZ default now() + '1 day',
|
||||
|
||||
PRIMARY KEY (project_id, bucket_name, object_key, version)
|
||||
);
|
||||
CREATE TABLE segments (
|
||||
stream_id BYTEA NOT NULL,
|
||||
position INT8 NOT NULL,
|
||||
|
||||
root_piece_id BYTEA NOT NULL,
|
||||
encrypted_key_nonce BYTEA NOT NULL,
|
||||
encrypted_key BYTEA NOT NULL,
|
||||
remote_alias_pieces BYTEA,
|
||||
|
||||
encrypted_size INT4 NOT NULL,
|
||||
plain_offset INT8 NOT NULL, -- migrated objects have this = 0
|
||||
plain_size INT4 NOT NULL, -- migrated objects have this = 0
|
||||
|
||||
redundancy INT8 NOT NULL default 0,
|
||||
|
||||
inline_data BYTEA DEFAULT NULL,
|
||||
|
||||
created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
|
||||
repaired_at TIMESTAMPTZ,
|
||||
expires_at TIMESTAMPTZ,
|
||||
|
||||
placement integer,
|
||||
encrypted_etag BYTEA default NULL,
|
||||
|
||||
PRIMARY KEY (stream_id, position)
|
||||
);
|
||||
CREATE SEQUENCE node_alias_seq
|
||||
INCREMENT BY 1
|
||||
MINVALUE 1 MAXVALUE 2147483647 -- MaxInt32
|
||||
START WITH 1;
|
||||
CREATE TABLE node_aliases (
|
||||
node_id BYTEA NOT NULL UNIQUE,
|
||||
node_alias INT4 NOT NULL UNIQUE default nextval('node_alias_seq')
|
||||
);
|
||||
|
||||
CREATE TABLE segment_copies (
|
||||
stream_id BYTEA NOT NULL PRIMARY KEY,
|
||||
ancestor_stream_id BYTEA NOT NULL,
|
||||
|
||||
CONSTRAINT not_self_ancestor CHECK (stream_id != ancestor_stream_id)
|
||||
);
|
||||
CREATE INDEX ON segment_copies (ancestor_stream_id);`,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
return migration.Run(ctx, db.log.Named("migrate"))
|
||||
}
|
||||
|
||||
// MigrateToLatest migrates database to the latest version.
|
||||
func (db *DB) MigrateToLatest(ctx context.Context) error {
|
||||
// First handle the idiosyncrasies of postgres and cockroach migrations. Postgres
|
||||
|
||||
@ -4,6 +4,7 @@
|
||||
package metabasetest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/spf13/pflag"
|
||||
@ -19,6 +20,13 @@ import (
|
||||
|
||||
// RunWithConfig runs tests with specific metabase configuration.
|
||||
func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB)) {
|
||||
RunWithConfigAndMigration(t, config, fn, func(ctx context.Context, db *metabase.DB) error {
|
||||
return db.TestMigrateToLatest(ctx)
|
||||
})
|
||||
}
|
||||
|
||||
// RunWithConfigAndMigration runs tests with specific metabase configuration and migration type.
|
||||
func RunWithConfigAndMigration(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB), migration func(ctx context.Context, db *metabase.DB) error) {
|
||||
for _, dbinfo := range satellitedbtest.Databases() {
|
||||
dbinfo := dbinfo
|
||||
t.Run(dbinfo.Name, func(t *testing.T) {
|
||||
@ -37,7 +45,7 @@ func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontex
|
||||
}
|
||||
}()
|
||||
|
||||
if err := db.MigrateToLatest(ctx); err != nil {
|
||||
if err := migration(ctx, db); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
61
satellite/metabase/migrate_test.go
Normal file
61
satellite/metabase/migrate_test.go
Normal file
@ -0,0 +1,61 @@
|
||||
// Copyright (C) 2021 Storj Labs, Inc.
|
||||
// See LICENSE for copying information.
|
||||
|
||||
package metabase_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap/zaptest"
|
||||
|
||||
"storj.io/common/testcontext"
|
||||
"storj.io/private/dbutil/dbschema"
|
||||
"storj.io/private/dbutil/pgutil"
|
||||
"storj.io/storj/satellite/metabase"
|
||||
"storj.io/storj/satellite/satellitedb/satellitedbtest"
|
||||
)
|
||||
|
||||
func TestMigration(t *testing.T) {
|
||||
for _, dbinfo := range satellitedbtest.Databases() {
|
||||
t.Run(dbinfo.Name, func(t *testing.T) {
|
||||
|
||||
ctx := testcontext.NewWithTimeout(t, 8*time.Minute)
|
||||
defer ctx.Cleanup()
|
||||
|
||||
prodSnapshot := schemaFromMigration(t, ctx, dbinfo.MetabaseDB, func(ctx context.Context, db *metabase.DB) error {
|
||||
return db.MigrateToLatest(ctx)
|
||||
})
|
||||
|
||||
testSnapshot := schemaFromMigration(t, ctx, dbinfo.MetabaseDB, func(ctx context.Context, db *metabase.DB) error {
|
||||
return db.TestMigrateToLatest(ctx)
|
||||
})
|
||||
|
||||
prodSnapshot.DropTable("metabase_versions")
|
||||
testSnapshot.DropTable("metabase_versions")
|
||||
|
||||
require.Equal(t, prodSnapshot.Schema, testSnapshot.Schema, "Test snapshot scheme doesn't match the migrated scheme.")
|
||||
require.Equal(t, prodSnapshot.Data, testSnapshot.Data, "Test snapshot data doesn't match the migrated data.")
|
||||
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func schemaFromMigration(t *testing.T, ctx *testcontext.Context, dbinfo satellitedbtest.Database, migration func(ctx context.Context, db *metabase.DB) error) (scheme *dbschema.Snapshot) {
|
||||
db, err := satellitedbtest.CreateMetabaseDB(ctx, zaptest.NewLogger(t), t.Name(), "M", 0, dbinfo, metabase.Config{
|
||||
ApplicationName: "migration",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
|
||||
defer ctx.Check(db.Close)
|
||||
|
||||
err = migration(ctx, db)
|
||||
require.NoError(t, err)
|
||||
|
||||
scheme, err = pgutil.QuerySnapshot(ctx, db.UnderlyingTagSQL())
|
||||
require.NoError(t, err)
|
||||
|
||||
return scheme
|
||||
}
|
||||
@ -12,7 +12,7 @@ require (
|
||||
go.uber.org/zap v1.17.0
|
||||
storj.io/common v0.0.0-20220518091716-ec9c16f58d50
|
||||
storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31
|
||||
storj.io/private v0.0.0-20220512110458-f0ade82ff3db
|
||||
storj.io/private v0.0.0-20220525153054-e5ea0fc18d84
|
||||
storj.io/storj v0.12.1-0.20220325094222-4a6c2e2a9ce8
|
||||
)
|
||||
|
||||
|
||||
@ -1486,8 +1486,8 @@ storj.io/monkit-jaeger v0.0.0-20210225162224-66fb37637bf6/go.mod h1:gj4vuCeyCRjR
|
||||
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k=
|
||||
storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s=
|
||||
storj.io/private v0.0.0-20210810102517-434aeab3f17d/go.mod h1:wvudoWSyOyB2daZNaMykjjqsSUad/ttFlUwgelg9+qg=
|
||||
storj.io/private v0.0.0-20220512110458-f0ade82ff3db h1:JHzjxOn/8QEAh02ZDG7b/yLDfMFxzsD51vmry68v11k=
|
||||
storj.io/private v0.0.0-20220512110458-f0ade82ff3db/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
|
||||
storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 h1:wAik82Huwh8D8oy57+cQbFMU2mXWVw/3Op9od3ldtLA=
|
||||
storj.io/private v0.0.0-20220525153054-e5ea0fc18d84/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
|
||||
storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo=
|
||||
storj.io/uplink v1.7.1-0.20211103104100-a785482780d8/go.mod h1:pKqsMpNMIAz//2TXzUGOR6tpu3iyabvXV4VWINj4jaY=
|
||||
storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk=
|
||||
|
||||
Loading…
Reference in New Issue
Block a user