metabase: use predefined snapshot for unit tests avoid migration

Change-Id: I6225ae32c312030e3559d8ed0faa137ffc3cc5b7

cmd/metabase-orphaned-segments/main_test.go

@@ -122,7 +122,7 @@ func test(t *testing.T, prepare func(t *testing.T, ctx *testcontext.Context, raw
 			require.NoError(t, err)
 			defer ctx.Check(metabaseDB.Close)
 
-			err = metabaseDB.MigrateToLatest(ctx)
+			err = metabaseDB.TestMigrateToLatest(ctx)
 			require.NoError(t, err)
 
 			prepare(t, ctx, metabaseTempDB, metabaseDB)

go.mod

@@ -54,7 +54,7 @@ require (
 	storj.io/common v0.0.0-20220518091716-ec9c16f58d50
 	storj.io/drpc v0.0.30
 	storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a
-	storj.io/private v0.0.0-20220512110458-f0ade82ff3db
+	storj.io/private v0.0.0-20220525153054-e5ea0fc18d84
 	storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85
 )
 

go.sum

@@ -937,7 +937,7 @@ storj.io/drpc v0.0.30 h1:jqPe4T9KEu3CDBI05A2hCMgMSHLtd/E0N0yTF9QreIE=
 storj.io/drpc v0.0.30/go.mod h1:6rcOyR/QQkSTX/9L5ZGtlZaE2PtXTTZl8d+ulSeeYEg=
 storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k=
 storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s=
-storj.io/private v0.0.0-20220512110458-f0ade82ff3db h1:JHzjxOn/8QEAh02ZDG7b/yLDfMFxzsD51vmry68v11k=
+storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 h1:wAik82Huwh8D8oy57+cQbFMU2mXWVw/3Op9od3ldtLA=
-storj.io/private v0.0.0-20220512110458-f0ade82ff3db/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
+storj.io/private v0.0.0-20220525153054-e5ea0fc18d84/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
 storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk=
 storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85/go.mod h1:RtbTI2i9JjfWTCaSNM2bVUNP1hjAvfv9VH6D35jFh18=

private/testplanet/satellite.go

@@ -523,7 +523,7 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
 		return nil, err
 	}
 
-	err = metabaseDB.MigrateToLatest(ctx)
+	err = metabaseDB.TestMigrateToLatest(ctx)
 	if err != nil {
 		return nil, err
 	}

satellite/metabase/db.go

@@ -129,6 +129,125 @@ func (db *DB) DestroyTables(ctx context.Context) error {
 	return Error.Wrap(err)
 }
 
+// TestMigrateToLatest replaces the migration steps with only one step to create metabase db.
+func (db *DB) TestMigrateToLatest(ctx context.Context) error {
+	// First handle the idiosyncrasies of postgres and cockroach migrations. Postgres
+	// will need to create any schemas specified in the search path, and cockroach
+	// will need to create the database it was told to connect to. These things should
+	// not really be here, and instead should be assumed to exist.
+	// This is tracked in jira ticket SM-200
+	switch db.impl {
+	case dbutil.Postgres:
+		schema, err := pgutil.ParseSchemaFromConnstr(db.connstr)
+		if err != nil {
+			return errs.New("error parsing schema: %+v", err)
+		}
+
+		if schema != "" {
+			err = pgutil.CreateSchema(ctx, db.db, schema)
+			if err != nil {
+				return errs.New("error creating schema: %+v", err)
+			}
+		}
+
+	case dbutil.Cockroach:
+		var dbName string
+		if err := db.db.QueryRowContext(ctx, `SELECT current_database();`).Scan(&dbName); err != nil {
+			return errs.New("error querying current database: %+v", err)
+		}
+
+		_, err := db.db.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s;`,
+			pgutil.QuoteIdentifier(dbName)))
+		if err != nil {
+			return errs.Wrap(err)
+		}
+	}
+
+	migration := &migrate.Migration{
+		Table: "metabase_versions",
+		Steps: []*migrate.Step{
+			{
+				DB:          &db.db,
+				Description: "Test snapshot",
+				Version:     39,
+				Action: migrate.SQL{
+
+					`CREATE TABLE objects (
+						project_id   BYTEA NOT NULL,
+						bucket_name  BYTEA NOT NULL, -- we're using bucket_name here to avoid a lookup into buckets table
+						object_key   BYTEA NOT NULL, -- using 'object_key' instead of 'key' to avoid reserved word
+						version      INT4  NOT NULL,
+						stream_id    BYTEA NOT NULL,
+
+						created_at TIMESTAMPTZ NOT NULL default now(),
+						expires_at TIMESTAMPTZ,
+
+						status         INT2 NOT NULL default ` + pendingStatus + `,
+						segment_count  INT4 NOT NULL default 0,
+
+						encrypted_metadata_nonce         BYTEA default NULL,
+						encrypted_metadata               BYTEA default NULL,
+						encrypted_metadata_encrypted_key BYTEA default NULL,
+
+						total_plain_size     INT8 NOT NULL default 0, -- migrated objects have this = 0
+						total_encrypted_size INT8 NOT NULL default 0,
+						fixed_segment_size   INT4 NOT NULL default 0, -- migrated objects have this = 0
+
+						encryption INT8 NOT NULL default 0,
+
+						zombie_deletion_deadline TIMESTAMPTZ default now() + '1 day',
+
+						PRIMARY KEY (project_id, bucket_name, object_key, version)
+					);
+					CREATE TABLE segments (
+						stream_id  BYTEA NOT NULL,
+						position   INT8  NOT NULL,
+
+						root_piece_id       BYTEA NOT NULL,
+						encrypted_key_nonce BYTEA NOT NULL,
+						encrypted_key       BYTEA NOT NULL,
+						remote_alias_pieces BYTEA,
+
+						encrypted_size INT4 NOT NULL,
+						plain_offset   INT8 NOT NULL, -- migrated objects have this = 0
+						plain_size     INT4 NOT NULL, -- migrated objects have this = 0
+
+						redundancy INT8 NOT NULL default 0,
+
+						inline_data  BYTEA DEFAULT NULL,
+
+						created_at TIMESTAMPTZ DEFAULT now() NOT NULL,
+						repaired_at TIMESTAMPTZ,
+						expires_at TIMESTAMPTZ,
+
+						placement integer,
+						encrypted_etag BYTEA default NULL,
+
+						PRIMARY KEY (stream_id, position)
+					);
+					CREATE SEQUENCE node_alias_seq
+						INCREMENT BY 1
+						MINVALUE 1 MAXVALUE 2147483647 -- MaxInt32
+						START WITH 1;
+					CREATE TABLE node_aliases (
+						node_id    BYTEA  NOT NULL UNIQUE,
+						node_alias INT4   NOT NULL UNIQUE default nextval('node_alias_seq')
+					);
+
+					CREATE TABLE segment_copies (
+						stream_id BYTEA NOT NULL PRIMARY KEY,
+						ancestor_stream_id BYTEA NOT NULL,
+
+						CONSTRAINT not_self_ancestor CHECK (stream_id != ancestor_stream_id)
+					);
+					CREATE INDEX ON segment_copies (ancestor_stream_id);`,
+				},
+			},
+		},
+	}
+	return migration.Run(ctx, db.log.Named("migrate"))
+}
+
 // MigrateToLatest migrates database to the latest version.
 func (db *DB) MigrateToLatest(ctx context.Context) error {
 	// First handle the idiosyncrasies of postgres and cockroach migrations. Postgres

satellite/metabase/metabasetest/run.go

@@ -4,6 +4,7 @@
 package metabasetest
 
 import (
+	"context"
 	"testing"
 
 	"github.com/spf13/pflag"
@@ -19,6 +20,13 @@ import (
 
 // RunWithConfig runs tests with specific metabase configuration.
 func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB)) {
+	RunWithConfigAndMigration(t, config, fn, func(ctx context.Context, db *metabase.DB) error {
+		return db.TestMigrateToLatest(ctx)
+	})
+}
+
+// RunWithConfigAndMigration runs tests with specific metabase configuration and migration type.
+func RunWithConfigAndMigration(t *testing.T, config metabase.Config, fn func(ctx *testcontext.Context, t *testing.T, db *metabase.DB), migration func(ctx context.Context, db *metabase.DB) error) {
 	for _, dbinfo := range satellitedbtest.Databases() {
 		dbinfo := dbinfo
 		t.Run(dbinfo.Name, func(t *testing.T) {
@@ -37,7 +45,7 @@ func RunWithConfig(t *testing.T, config metabase.Config, fn func(ctx *testcontex
 				}
 			}()
 
-			if err := db.MigrateToLatest(ctx); err != nil {
+			if err := migration(ctx, db); err != nil {
 				t.Fatal(err)
 			}
 

satellite/metabase/migrate_test.go

@@ -0,0 +1,61 @@
+// Copyright (C) 2021 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package metabase_test
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+
+	"storj.io/common/testcontext"
+	"storj.io/private/dbutil/dbschema"
+	"storj.io/private/dbutil/pgutil"
+	"storj.io/storj/satellite/metabase"
+	"storj.io/storj/satellite/satellitedb/satellitedbtest"
+)
+
+func TestMigration(t *testing.T) {
+	for _, dbinfo := range satellitedbtest.Databases() {
+		t.Run(dbinfo.Name, func(t *testing.T) {
+
+			ctx := testcontext.NewWithTimeout(t, 8*time.Minute)
+			defer ctx.Cleanup()
+
+			prodSnapshot := schemaFromMigration(t, ctx, dbinfo.MetabaseDB, func(ctx context.Context, db *metabase.DB) error {
+				return db.MigrateToLatest(ctx)
+			})
+
+			testSnapshot := schemaFromMigration(t, ctx, dbinfo.MetabaseDB, func(ctx context.Context, db *metabase.DB) error {
+				return db.TestMigrateToLatest(ctx)
+			})
+
+			prodSnapshot.DropTable("metabase_versions")
+			testSnapshot.DropTable("metabase_versions")
+
+			require.Equal(t, prodSnapshot.Schema, testSnapshot.Schema, "Test snapshot scheme doesn't match the migrated scheme.")
+			require.Equal(t, prodSnapshot.Data, testSnapshot.Data, "Test snapshot data doesn't match the migrated data.")
+
+		})
+	}
+}
+
+func schemaFromMigration(t *testing.T, ctx *testcontext.Context, dbinfo satellitedbtest.Database, migration func(ctx context.Context, db *metabase.DB) error) (scheme *dbschema.Snapshot) {
+	db, err := satellitedbtest.CreateMetabaseDB(ctx, zaptest.NewLogger(t), t.Name(), "M", 0, dbinfo, metabase.Config{
+		ApplicationName: "migration",
+	})
+	require.NoError(t, err)
+
+	defer ctx.Check(db.Close)
+
+	err = migration(ctx, db)
+	require.NoError(t, err)
+
+	scheme, err = pgutil.QuerySnapshot(ctx, db.UnderlyingTagSQL())
+	require.NoError(t, err)
+
+	return scheme
+}

testsuite/go.mod

@@ -12,7 +12,7 @@ require (
 	go.uber.org/zap v1.17.0
 	storj.io/common v0.0.0-20220518091716-ec9c16f58d50
 	storj.io/gateway-mt v1.18.1-0.20211210081136-cada9a567d31
-	storj.io/private v0.0.0-20220512110458-f0ade82ff3db
+	storj.io/private v0.0.0-20220525153054-e5ea0fc18d84
 	storj.io/storj v0.12.1-0.20220325094222-4a6c2e2a9ce8
 )
 

testsuite/go.sum

@@ -1486,8 +1486,8 @@ storj.io/monkit-jaeger v0.0.0-20210225162224-66fb37637bf6/go.mod h1:gj4vuCeyCRjR
 storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a h1:qads+aZlFKm5gUxobfF9s2x8/byPaPPLe2Mz+J82R+k=
 storj.io/monkit-jaeger v0.0.0-20220131130547-dc4cb5a0d97a/go.mod h1:DGEycSjvzE0JqcD3+6IjwPEK6x30oOus6AApXzl7t0s=
 storj.io/private v0.0.0-20210810102517-434aeab3f17d/go.mod h1:wvudoWSyOyB2daZNaMykjjqsSUad/ttFlUwgelg9+qg=
-storj.io/private v0.0.0-20220512110458-f0ade82ff3db h1:JHzjxOn/8QEAh02ZDG7b/yLDfMFxzsD51vmry68v11k=
+storj.io/private v0.0.0-20220525153054-e5ea0fc18d84 h1:wAik82Huwh8D8oy57+cQbFMU2mXWVw/3Op9od3ldtLA=
-storj.io/private v0.0.0-20220512110458-f0ade82ff3db/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
+storj.io/private v0.0.0-20220525153054-e5ea0fc18d84/go.mod h1:Cirl8IjFrdWi4l4oC43Va/vTvj8fr/+szpjlgj62FGM=
 storj.io/uplink v1.7.0/go.mod h1:zqj/LFDxa6RMaSRSHOmukg3mMgesOry0iHSjNldDMGo=
 storj.io/uplink v1.7.1-0.20211103104100-a785482780d8/go.mod h1:pKqsMpNMIAz//2TXzUGOR6tpu3iyabvXV4VWINj4jaY=
 storj.io/uplink v1.8.2-0.20220406151905-7305e5b6da85 h1:UBsmsfbXLHUL8NNtUGd+cnhITbVB/cSJeckE3+xUIbk=