From 7f6f7e124681720aabe49d52c65e6a4fa50f9a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rton=20Elek?= Date: Mon, 3 Oct 2022 15:32:14 +0200 Subject: [PATCH] satellite: optional migration for integration tests This patch addresses the following issues: 1. Running full migration in cockroachdb is quite slow. We already have an approach for unit tests to start from the latest snapshot. This patch makes it possible to use it for integrations tests. 2. Migration requires executing a separated command which makes it hard to run application in containerized test environments (like storj-up) or from IDE. This patch introduces a hidden flag to run migration. 3. Test user creation is painful. We do it with calling GraphQL + admin API. Providing an option with testuser makes the integration tests significant more simple (especially as the projectID -> access grant can be predictable) Change-Id: I61010728727b490ea6aac32620f2da0484966727 --- cmd/satellite/api.go | 47 +++++++++++ cmd/satellite/main.go | 1 + cmd/satellite/testdata.go | 133 ++++++++++++++++++++++++++++++ cmd/satellite/testdata_test.go | 16 ++++ satellite/metabase/db.go | 2 +- satellite/satellitedb/migrate.go | 6 +- satellite/satellitedb/projects.go | 9 +- 7 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 cmd/satellite/testdata.go create mode 100644 cmd/satellite/testdata_test.go diff --git a/cmd/satellite/api.go b/cmd/satellite/api.go index e69764bdf..2e9c52694 100644 --- a/cmd/satellite/api.go +++ b/cmd/satellite/api.go @@ -4,6 +4,8 @@ package main import ( + "strings" + "github.com/spf13/cobra" "github.com/zeebo/errs" "go.uber.org/zap" @@ -44,6 +46,31 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() + for _, migration := range strings.Split(runCfg.DatabaseOptions.Migration, ",") { + switch migration { + case fullMigration: + err = db.MigrateToLatest(ctx) + if err != nil { + return err + } + case snapshotMigration: + log.Info("Migration using latest snapshot. It's not for production", zap.String("db", "master")) + err = db.TestingMigrateToLatest(ctx) + if err != nil { + return err + } + case testDataCreation: + err := createTestData(ctx, db) + if err != nil { + return err + } + case noMigration: + // noop + default: + return errs.New("unsupported migration type: %s, please try one of the: %s", migration, strings.Join(migrationTypes, ",")) + } + } + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, metabase.Config{ ApplicationName: "satellite-api", MinPartSize: runCfg.Config.Metainfo.MinPartSize, @@ -57,6 +84,26 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, metabaseDB.Close()) }() + for _, migration := range strings.Split(runCfg.DatabaseOptions.Migration, ",") { + switch migration { + case fullMigration: + err = metabaseDB.MigrateToLatest(ctx) + if err != nil { + return err + } + case snapshotMigration: + log.Info("Migration using latest snapshot. It's not for production", zap.String("db", "master")) + err = metabaseDB.TestMigrateToLatest(ctx) + if err != nil { + return err + } + case noMigration, testDataCreation: + // noop + default: + return errs.New("unsupported migration type: %s, please try one of the: %s", migration, strings.Join(migrationTypes, ",")) + } + } + revocationDB, err := revocation.OpenDBFromCfg(ctx, runCfg.Config.Server.Config) if err != nil { return errs.New("Error creating revocation database on satellite api: %+v", err) diff --git a/cmd/satellite/main.go b/cmd/satellite/main.go index db92d72ee..9b7bbb2bc 100644 --- a/cmd/satellite/main.go +++ b/cmd/satellite/main.go @@ -60,6 +60,7 @@ type Satellite struct { Expiration time.Duration `help:"macaroon revocation cache expiration" default:"5m"` Capacity int `help:"macaroon revocation cache capacity" default:"10000"` } + Migration string `help:"coma separated migration types to run during every startup (none: no migration, snapshot: creating db from latest test snapshot (for testing only), testdata: create testuser in addition to a migration, full: do the normal migration (equals to 'satellite run migration'" default:"none" hidden:"true"` } satellite.Config diff --git a/cmd/satellite/testdata.go b/cmd/satellite/testdata.go new file mode 100644 index 000000000..4beae5561 --- /dev/null +++ b/cmd/satellite/testdata.go @@ -0,0 +1,133 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "context" + "database/sql" + "errors" + + "github.com/zeebo/errs" + "golang.org/x/crypto/bcrypt" + + "storj.io/common/macaroon" + "storj.io/common/memory" + "storj.io/common/uuid" + "storj.io/storj/satellite" + "storj.io/storj/satellite/console" + "storj.io/storj/satellite/console/consolewasm" +) + +const ( + fullMigration = "full" + snapshotMigration = "snapshot" + testDataCreation = "testdata" + noMigration = "none" +) + +var migrationTypes = []string{fullMigration, snapshotMigration, testDataCreation, noMigration} + +var ( + projectID = uuid.UUID([16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + apiKeyID = uuid.UUID([16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + head = []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, + } + secret = []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + } + password = "123a123" +) + +// createTestData creates predefined test account to make the integration tests easier. +func createTestData(ctx context.Context, db satellite.DB) error { + userID, err := uuid.FromString("be041c3c-0658-40d1-8f7c-e70a0a26cc12") + if err != nil { + return err + } + + _, err = db.Console().Users().Get(ctx, userID) + if errors.Is(err, sql.ErrNoRows) { + + hash, err := bcrypt.GenerateFromPassword([]byte(password), 0) + if err != nil { + return err + } + + _, err = db.Console().Users().Insert(ctx, &console.User{ + ID: userID, + FullName: "Hiro Protagonist", + Email: "test@storj.io", + ProjectLimit: 5, + ProjectStorageLimit: (memory.GB * 150).Int64(), + ProjectBandwidthLimit: (memory.GB * 150).Int64(), + PasswordHash: hash, + }) + if err != nil { + return err + } + + active := console.Active + err = db.Console().Users().Update(ctx, userID, console.UpdateUserRequest{ + Status: &active, + }) + if err != nil { + return err + } + } else if err != nil { + return err + } + + _, err = db.Console().Projects().Get(ctx, projectID) + if errors.Is(err, sql.ErrNoRows) { + _, err := db.Console().Projects().Insert(ctx, &console.Project{ + ID: projectID, + OwnerID: userID, + Name: "testproject", + }) + if err != nil { + return err + } + _, err = db.Console().ProjectMembers().Insert(ctx, userID, projectID) + if err != nil { + return err + } + } else if err != nil { + return err + } + + _, err = db.Console().APIKeys().GetByNameAndProjectID(ctx, "testkey", projectID) + if errors.Is(err, sql.ErrNoRows) { + _, err = db.Console().APIKeys().Create(ctx, head, console.APIKeyInfo{ + ID: apiKeyID, + ProjectID: projectID, + Name: "testkey", + Secret: secret, + }) + } else if err != nil { + return err + } + return err +} + +// GetTestApiKey can calculate an access grant for the predefined test users/project. +func GetTestApiKey(satelliteId string) (string, error) { + key, err := macaroon.FromParts(head, secret) + if err != nil { + return "", errs.Wrap(err) + } + + accessGrant, err := consolewasm.GenAccessGrant(satelliteId, key.Serialize(), password, projectID.String()) + if err != nil { + return "", errs.Wrap(err) + } + + return accessGrant, nil +} diff --git a/cmd/satellite/testdata_test.go b/cmd/satellite/testdata_test.go new file mode 100644 index 000000000..79f4c912c --- /dev/null +++ b/cmd/satellite/testdata_test.go @@ -0,0 +1,16 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetTestApiKey(t *testing.T) { + key, err := GetTestApiKey("1nxk8B3L3zJi1nGTDemMWSTiyTehu87XaMGy1f742igrTEPciS@localhost:10001") + require.NoError(t, err) + require.Equal(t, "14CuLjd1ypmCQGkwFCAWxPGVRugM52shEia6MS2ASSTJ6P2qKsW2fWEujLH8ZaycTGHdxSpXoAWxXcwCsueJdCRcj2E4MHysMiwWsmm9rLnPYjE9YwUkNgDnR4nJHNtBbBF94smgk52RQw9WHHxas6owWJJceKmTYkip3odDHW52dNVqJVFRZZqwx4WEorey9KBe95RmSNbCL3mp6LVcMTYTT8vHSzPPAaT54wGuxiSH2rFkzkHBnL7fLD1AmwUham8cM7UPCG5EnPyGHKpjD944wgEgmdohfumLqmEmmmRW", key) +} diff --git a/satellite/metabase/db.go b/satellite/metabase/db.go index 30e01c334..1815a66b6 100644 --- a/satellite/metabase/db.go +++ b/satellite/metabase/db.go @@ -171,7 +171,7 @@ func (db *DB) TestMigrateToLatest(ctx context.Context) error { { DB: &db.db, Description: "Test snapshot", - Version: 39, + Version: 15, Action: migrate.SQL{ `CREATE TABLE objects ( diff --git a/satellite/satellitedb/migrate.go b/satellite/satellitedb/migrate.go index 6fd36e86d..9495be4f1 100644 --- a/satellite/satellitedb/migrate.go +++ b/satellite/satellitedb/migrate.go @@ -119,11 +119,11 @@ func (db *satelliteDB) TestingMigrateToLatest(ctx context.Context) error { if err != nil { return ErrMigrateMinVersion.Wrap(err) } - if dbVersion > -1 { - return ErrMigrateMinVersion.New("the database must be empty, got version %d", dbVersion) - } testMigration := db.TestPostgresMigration() + if dbVersion != -1 && dbVersion != testMigration.Steps[0].Version { + return ErrMigrateMinVersion.New("the database must be empty, or be on the latest version (%d)", dbVersion) + } return testMigration.Run(ctx, db.log.Named("migrate")) default: return migrate.Create(ctx, "database", db.DB) diff --git a/satellite/satellitedb/projects.go b/satellite/satellitedb/projects.go index 281ad9905..1cc1da593 100644 --- a/satellite/satellitedb/projects.go +++ b/satellite/satellitedb/projects.go @@ -119,9 +119,12 @@ func (projects *projects) GetByPublicID(ctx context.Context, publicID uuid.UUID) func (projects *projects) Insert(ctx context.Context, project *console.Project) (_ *console.Project, err error) { defer mon.Task()(&ctx)(&err) - projectID, err := uuid.New() - if err != nil { - return nil, err + projectID := project.ID + if projectID.IsZero() { + projectID, err = uuid.New() + if err != nil { + return nil, err + } } publicID, err := uuid.New() if err != nil {