satellite: optional migration for integration tests

This patch addresses the following issues:

 1. Running full migration in cockroachdb is quite slow. We already have an approach for unit tests to start from the latest snapshot. This patch makes it possible to use it for integrations tests.
 2. Migration requires executing a separated command which makes it hard to run application in containerized test environments (like storj-up) or from IDE. This patch introduces a hidden flag to run migration.
 3. Test user creation is painful. We do it with calling GraphQL + admin API. Providing an option with testuser makes the integration tests significant more simple (especially as the projectID -> access grant can be predictable)

Change-Id: I61010728727b490ea6aac32620f2da0484966727

cmd/satellite/api.go

@@ -4,6 +4,8 @@
 package main
 
 import (
+	"strings"
+
 	"github.com/spf13/cobra"
 	"github.com/zeebo/errs"
 	"go.uber.org/zap"
@@ -44,6 +46,31 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) {
 		err = errs.Combine(err, db.Close())
 	}()
 
+	for _, migration := range strings.Split(runCfg.DatabaseOptions.Migration, ",") {
+		switch migration {
+		case fullMigration:
+			err = db.MigrateToLatest(ctx)
+			if err != nil {
+				return err
+			}
+		case snapshotMigration:
+			log.Info("Migration using latest snapshot. It's not for production", zap.String("db", "master"))
+			err = db.TestingMigrateToLatest(ctx)
+			if err != nil {
+				return err
+			}
+		case testDataCreation:
+			err := createTestData(ctx, db)
+			if err != nil {
+				return err
+			}
+		case noMigration:
+		// noop
+		default:
+			return errs.New("unsupported migration type: %s, please try one of the: %s", migration, strings.Join(migrationTypes, ","))
+		}
+	}
+
 	metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, metabase.Config{
 		ApplicationName:  "satellite-api",
 		MinPartSize:      runCfg.Config.Metainfo.MinPartSize,
@@ -57,6 +84,26 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) {
 		err = errs.Combine(err, metabaseDB.Close())
 	}()
 
+	for _, migration := range strings.Split(runCfg.DatabaseOptions.Migration, ",") {
+		switch migration {
+		case fullMigration:
+			err = metabaseDB.MigrateToLatest(ctx)
+			if err != nil {
+				return err
+			}
+		case snapshotMigration:
+			log.Info("Migration using latest snapshot. It's not for production", zap.String("db", "master"))
+			err = metabaseDB.TestMigrateToLatest(ctx)
+			if err != nil {
+				return err
+			}
+		case noMigration, testDataCreation:
+		// noop
+		default:
+			return errs.New("unsupported migration type: %s, please try one of the: %s", migration, strings.Join(migrationTypes, ","))
+		}
+	}
+
 	revocationDB, err := revocation.OpenDBFromCfg(ctx, runCfg.Config.Server.Config)
 	if err != nil {
 		return errs.New("Error creating revocation database on satellite api: %+v", err)

cmd/satellite/main.go

@@ -60,6 +60,7 @@ type Satellite struct {
 			Expiration time.Duration `help:"macaroon revocation cache expiration" default:"5m"`
 			Capacity   int           `help:"macaroon revocation cache capacity" default:"10000"`
 		}
+		Migration string `help:"coma separated migration types to run during every startup (none: no migration, snapshot: creating db from latest test snapshot (for testing only), testdata: create testuser in addition to a migration, full: do the normal migration (equals to 'satellite run migration'" default:"none" hidden:"true"`
 	}
 
 	satellite.Config

cmd/satellite/testdata.go

@@ -0,0 +1,133 @@
+// Copyright (C) 2022 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+
+	"github.com/zeebo/errs"
+	"golang.org/x/crypto/bcrypt"
+
+	"storj.io/common/macaroon"
+	"storj.io/common/memory"
+	"storj.io/common/uuid"
+	"storj.io/storj/satellite"
+	"storj.io/storj/satellite/console"
+	"storj.io/storj/satellite/console/consolewasm"
+)
+
+const (
+	fullMigration     = "full"
+	snapshotMigration = "snapshot"
+	testDataCreation  = "testdata"
+	noMigration       = "none"
+)
+
+var migrationTypes = []string{fullMigration, snapshotMigration, testDataCreation, noMigration}
+
+var (
+	projectID = uuid.UUID([16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1})
+	apiKeyID  = uuid.UUID([16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2})
+	head      = []byte{
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3,
+	}
+	secret = []byte{
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4,
+	}
+	password = "123a123"
+)
+
+// createTestData creates predefined test account to make the integration tests easier.
+func createTestData(ctx context.Context, db satellite.DB) error {
+	userID, err := uuid.FromString("be041c3c-0658-40d1-8f7c-e70a0a26cc12")
+	if err != nil {
+		return err
+	}
+
+	_, err = db.Console().Users().Get(ctx, userID)
+	if errors.Is(err, sql.ErrNoRows) {
+
+		hash, err := bcrypt.GenerateFromPassword([]byte(password), 0)
+		if err != nil {
+			return err
+		}
+
+		_, err = db.Console().Users().Insert(ctx, &console.User{
+			ID:                    userID,
+			FullName:              "Hiro Protagonist",
+			Email:                 "test@storj.io",
+			ProjectLimit:          5,
+			ProjectStorageLimit:   (memory.GB * 150).Int64(),
+			ProjectBandwidthLimit: (memory.GB * 150).Int64(),
+			PasswordHash:          hash,
+		})
+		if err != nil {
+			return err
+		}
+
+		active := console.Active
+		err = db.Console().Users().Update(ctx, userID, console.UpdateUserRequest{
+			Status: &active,
+		})
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
+	_, err = db.Console().Projects().Get(ctx, projectID)
+	if errors.Is(err, sql.ErrNoRows) {
+		_, err := db.Console().Projects().Insert(ctx, &console.Project{
+			ID:      projectID,
+			OwnerID: userID,
+			Name:    "testproject",
+		})
+		if err != nil {
+			return err
+		}
+		_, err = db.Console().ProjectMembers().Insert(ctx, userID, projectID)
+		if err != nil {
+			return err
+		}
+	} else if err != nil {
+		return err
+	}
+
+	_, err = db.Console().APIKeys().GetByNameAndProjectID(ctx, "testkey", projectID)
+	if errors.Is(err, sql.ErrNoRows) {
+		_, err = db.Console().APIKeys().Create(ctx, head, console.APIKeyInfo{
+			ID:        apiKeyID,
+			ProjectID: projectID,
+			Name:      "testkey",
+			Secret:    secret,
+		})
+	} else if err != nil {
+		return err
+	}
+	return err
+}
+
+// GetTestApiKey can calculate an access grant for the predefined test users/project.
+func GetTestApiKey(satelliteId string) (string, error) {
+	key, err := macaroon.FromParts(head, secret)
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	accessGrant, err := consolewasm.GenAccessGrant(satelliteId, key.Serialize(), password, projectID.String())
+	if err != nil {
+		return "", errs.Wrap(err)
+	}
+
+	return accessGrant, nil
+}

cmd/satellite/testdata_test.go

@@ -0,0 +1,16 @@
+// Copyright (C) 2022 Storj Labs, Inc.
+// See LICENSE for copying information.
+
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetTestApiKey(t *testing.T) {
+	key, err := GetTestApiKey("1nxk8B3L3zJi1nGTDemMWSTiyTehu87XaMGy1f742igrTEPciS@localhost:10001")
+	require.NoError(t, err)
+	require.Equal(t, "14CuLjd1ypmCQGkwFCAWxPGVRugM52shEia6MS2ASSTJ6P2qKsW2fWEujLH8ZaycTGHdxSpXoAWxXcwCsueJdCRcj2E4MHysMiwWsmm9rLnPYjE9YwUkNgDnR4nJHNtBbBF94smgk52RQw9WHHxas6owWJJceKmTYkip3odDHW52dNVqJVFRZZqwx4WEorey9KBe95RmSNbCL3mp6LVcMTYTT8vHSzPPAaT54wGuxiSH2rFkzkHBnL7fLD1AmwUham8cM7UPCG5EnPyGHKpjD944wgEgmdohfumLqmEmmmRW", key)
+}

satellite/metabase/db.go

@@ -171,7 +171,7 @@ func (db *DB) TestMigrateToLatest(ctx context.Context) error {
 			{
 				DB:          &db.db,
 				Description: "Test snapshot",
-				Version:     39,
+				Version:     15,
 				Action: migrate.SQL{
 
 					`CREATE TABLE objects (

satellite/satellitedb/migrate.go

@@ -119,11 +119,11 @@ func (db *satelliteDB) TestingMigrateToLatest(ctx context.Context) error {
 		if err != nil {
 			return ErrMigrateMinVersion.Wrap(err)
 		}
-		if dbVersion > -1 {
-			return ErrMigrateMinVersion.New("the database must be empty, got version %d", dbVersion)
-		}
 
 		testMigration := db.TestPostgresMigration()
+		if dbVersion != -1 && dbVersion != testMigration.Steps[0].Version {
+			return ErrMigrateMinVersion.New("the database must be empty, or be on the latest version (%d)", dbVersion)
+		}
 		return testMigration.Run(ctx, db.log.Named("migrate"))
 	default:
 		return migrate.Create(ctx, "database", db.DB)

satellite/satellitedb/projects.go

@@ -119,9 +119,12 @@ func (projects *projects) GetByPublicID(ctx context.Context, publicID uuid.UUID)
 func (projects *projects) Insert(ctx context.Context, project *console.Project) (_ *console.Project, err error) {
 	defer mon.Task()(&ctx)(&err)
 
-	projectID, err := uuid.New()
+	projectID := project.ID
-	if err != nil {
+	if projectID.IsZero() {
-		return nil, err
+		projectID, err = uuid.New()
+		if err != nil {
+			return nil, err
+		}
 	}
 	publicID, err := uuid.New()
 	if err != nil {