diff --git a/cmd/satellite/api.go b/cmd/satellite/api.go index e69764bdf..2e9c52694 100644 --- a/cmd/satellite/api.go +++ b/cmd/satellite/api.go @@ -4,6 +4,8 @@ package main import ( + "strings" + "github.com/spf13/cobra" "github.com/zeebo/errs" "go.uber.org/zap" @@ -44,6 +46,31 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() + for _, migration := range strings.Split(runCfg.DatabaseOptions.Migration, ",") { + switch migration { + case fullMigration: + err = db.MigrateToLatest(ctx) + if err != nil { + return err + } + case snapshotMigration: + log.Info("Migration using latest snapshot. It's not for production", zap.String("db", "master")) + err = db.TestingMigrateToLatest(ctx) + if err != nil { + return err + } + case testDataCreation: + err := createTestData(ctx, db) + if err != nil { + return err + } + case noMigration: + // noop + default: + return errs.New("unsupported migration type: %s, please try one of the: %s", migration, strings.Join(migrationTypes, ",")) + } + } + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, metabase.Config{ ApplicationName: "satellite-api", MinPartSize: runCfg.Config.Metainfo.MinPartSize, @@ -57,6 +84,26 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, metabaseDB.Close()) }() + for _, migration := range strings.Split(runCfg.DatabaseOptions.Migration, ",") { + switch migration { + case fullMigration: + err = metabaseDB.MigrateToLatest(ctx) + if err != nil { + return err + } + case snapshotMigration: + log.Info("Migration using latest snapshot. It's not for production", zap.String("db", "master")) + err = metabaseDB.TestMigrateToLatest(ctx) + if err != nil { + return err + } + case noMigration, testDataCreation: + // noop + default: + return errs.New("unsupported migration type: %s, please try one of the: %s", migration, strings.Join(migrationTypes, ",")) + } + } + revocationDB, err := revocation.OpenDBFromCfg(ctx, runCfg.Config.Server.Config) if err != nil { return errs.New("Error creating revocation database on satellite api: %+v", err) diff --git a/cmd/satellite/main.go b/cmd/satellite/main.go index db92d72ee..9b7bbb2bc 100644 --- a/cmd/satellite/main.go +++ b/cmd/satellite/main.go @@ -60,6 +60,7 @@ type Satellite struct { Expiration time.Duration `help:"macaroon revocation cache expiration" default:"5m"` Capacity int `help:"macaroon revocation cache capacity" default:"10000"` } + Migration string `help:"coma separated migration types to run during every startup (none: no migration, snapshot: creating db from latest test snapshot (for testing only), testdata: create testuser in addition to a migration, full: do the normal migration (equals to 'satellite run migration'" default:"none" hidden:"true"` } satellite.Config diff --git a/cmd/satellite/testdata.go b/cmd/satellite/testdata.go new file mode 100644 index 000000000..4beae5561 --- /dev/null +++ b/cmd/satellite/testdata.go @@ -0,0 +1,133 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "context" + "database/sql" + "errors" + + "github.com/zeebo/errs" + "golang.org/x/crypto/bcrypt" + + "storj.io/common/macaroon" + "storj.io/common/memory" + "storj.io/common/uuid" + "storj.io/storj/satellite" + "storj.io/storj/satellite/console" + "storj.io/storj/satellite/console/consolewasm" +) + +const ( + fullMigration = "full" + snapshotMigration = "snapshot" + testDataCreation = "testdata" + noMigration = "none" +) + +var migrationTypes = []string{fullMigration, snapshotMigration, testDataCreation, noMigration} + +var ( + projectID = uuid.UUID([16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}) + apiKeyID = uuid.UUID([16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}) + head = []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, + } + secret = []byte{ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, + } + password = "123a123" +) + +// createTestData creates predefined test account to make the integration tests easier. +func createTestData(ctx context.Context, db satellite.DB) error { + userID, err := uuid.FromString("be041c3c-0658-40d1-8f7c-e70a0a26cc12") + if err != nil { + return err + } + + _, err = db.Console().Users().Get(ctx, userID) + if errors.Is(err, sql.ErrNoRows) { + + hash, err := bcrypt.GenerateFromPassword([]byte(password), 0) + if err != nil { + return err + } + + _, err = db.Console().Users().Insert(ctx, &console.User{ + ID: userID, + FullName: "Hiro Protagonist", + Email: "test@storj.io", + ProjectLimit: 5, + ProjectStorageLimit: (memory.GB * 150).Int64(), + ProjectBandwidthLimit: (memory.GB * 150).Int64(), + PasswordHash: hash, + }) + if err != nil { + return err + } + + active := console.Active + err = db.Console().Users().Update(ctx, userID, console.UpdateUserRequest{ + Status: &active, + }) + if err != nil { + return err + } + } else if err != nil { + return err + } + + _, err = db.Console().Projects().Get(ctx, projectID) + if errors.Is(err, sql.ErrNoRows) { + _, err := db.Console().Projects().Insert(ctx, &console.Project{ + ID: projectID, + OwnerID: userID, + Name: "testproject", + }) + if err != nil { + return err + } + _, err = db.Console().ProjectMembers().Insert(ctx, userID, projectID) + if err != nil { + return err + } + } else if err != nil { + return err + } + + _, err = db.Console().APIKeys().GetByNameAndProjectID(ctx, "testkey", projectID) + if errors.Is(err, sql.ErrNoRows) { + _, err = db.Console().APIKeys().Create(ctx, head, console.APIKeyInfo{ + ID: apiKeyID, + ProjectID: projectID, + Name: "testkey", + Secret: secret, + }) + } else if err != nil { + return err + } + return err +} + +// GetTestApiKey can calculate an access grant for the predefined test users/project. +func GetTestApiKey(satelliteId string) (string, error) { + key, err := macaroon.FromParts(head, secret) + if err != nil { + return "", errs.Wrap(err) + } + + accessGrant, err := consolewasm.GenAccessGrant(satelliteId, key.Serialize(), password, projectID.String()) + if err != nil { + return "", errs.Wrap(err) + } + + return accessGrant, nil +} diff --git a/cmd/satellite/testdata_test.go b/cmd/satellite/testdata_test.go new file mode 100644 index 000000000..79f4c912c --- /dev/null +++ b/cmd/satellite/testdata_test.go @@ -0,0 +1,16 @@ +// Copyright (C) 2022 Storj Labs, Inc. +// See LICENSE for copying information. + +package main + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestGetTestApiKey(t *testing.T) { + key, err := GetTestApiKey("1nxk8B3L3zJi1nGTDemMWSTiyTehu87XaMGy1f742igrTEPciS@localhost:10001") + require.NoError(t, err) + require.Equal(t, "14CuLjd1ypmCQGkwFCAWxPGVRugM52shEia6MS2ASSTJ6P2qKsW2fWEujLH8ZaycTGHdxSpXoAWxXcwCsueJdCRcj2E4MHysMiwWsmm9rLnPYjE9YwUkNgDnR4nJHNtBbBF94smgk52RQw9WHHxas6owWJJceKmTYkip3odDHW52dNVqJVFRZZqwx4WEorey9KBe95RmSNbCL3mp6LVcMTYTT8vHSzPPAaT54wGuxiSH2rFkzkHBnL7fLD1AmwUham8cM7UPCG5EnPyGHKpjD944wgEgmdohfumLqmEmmmRW", key) +} diff --git a/satellite/metabase/db.go b/satellite/metabase/db.go index 30e01c334..1815a66b6 100644 --- a/satellite/metabase/db.go +++ b/satellite/metabase/db.go @@ -171,7 +171,7 @@ func (db *DB) TestMigrateToLatest(ctx context.Context) error { { DB: &db.db, Description: "Test snapshot", - Version: 39, + Version: 15, Action: migrate.SQL{ `CREATE TABLE objects ( diff --git a/satellite/satellitedb/migrate.go b/satellite/satellitedb/migrate.go index 6fd36e86d..9495be4f1 100644 --- a/satellite/satellitedb/migrate.go +++ b/satellite/satellitedb/migrate.go @@ -119,11 +119,11 @@ func (db *satelliteDB) TestingMigrateToLatest(ctx context.Context) error { if err != nil { return ErrMigrateMinVersion.Wrap(err) } - if dbVersion > -1 { - return ErrMigrateMinVersion.New("the database must be empty, got version %d", dbVersion) - } testMigration := db.TestPostgresMigration() + if dbVersion != -1 && dbVersion != testMigration.Steps[0].Version { + return ErrMigrateMinVersion.New("the database must be empty, or be on the latest version (%d)", dbVersion) + } return testMigration.Run(ctx, db.log.Named("migrate")) default: return migrate.Create(ctx, "database", db.DB) diff --git a/satellite/satellitedb/projects.go b/satellite/satellitedb/projects.go index 281ad9905..1cc1da593 100644 --- a/satellite/satellitedb/projects.go +++ b/satellite/satellitedb/projects.go @@ -119,9 +119,12 @@ func (projects *projects) GetByPublicID(ctx context.Context, publicID uuid.UUID) func (projects *projects) Insert(ctx context.Context, project *console.Project) (_ *console.Project, err error) { defer mon.Task()(&ctx)(&err) - projectID, err := uuid.New() - if err != nil { - return nil, err + projectID := project.ID + if projectID.IsZero() { + projectID, err = uuid.New() + if err != nil { + return nil, err + } } publicID, err := uuid.New() if err != nil {