JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite: optional migration for integration tests

Browse Source 
This patch addresses the following issues:

 1. Running full migration in cockroachdb is quite slow. We already have an approach for unit tests to start from the latest snapshot. This patch makes it possible to use it for integrations tests.
 2. Migration requires executing a separated command which makes it hard to run application in containerized test environments (like storj-up) or from IDE. This patch introduces a hidden flag to run migration.
 3. Test user creation is painful. We do it with calling GraphQL + admin API. Providing an option with testuser makes the integration tests significant more simple (especially as the projectID -> access grant can be predictable)

Change-Id: I61010728727b490ea6aac32620f2da0484966727
This commit is contained in:
Márton Elek 2022-10-03 15:32:14 +02:00
parent 85f9dad225
commit 7f6f7e1246
No known key found for this signature in database
GPG Key ID: D51EA8F00EE79B28
7 changed files with 207 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
cmd/satellite/api.go
View File

@ -4,6 +4,8 @@
package main
import (
"strings"
"github.com/spf13/cobra"
"github.com/zeebo/errs"
"go.uber.org/zap"
@ -44,6 +46,31 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close())
}()
for _, migration := range strings.Split(runCfg.DatabaseOptions.Migration, ",") {
switch migration {
case fullMigration:
err = db.MigrateToLatest(ctx)
if err != nil {
return err
}
case snapshotMigration:
log.Info("Migration using latest snapshot. It's not for production", zap.String("db", "master"))
err = db.TestingMigrateToLatest(ctx)
if err != nil {
return err
}
case testDataCreation:
err := createTestData(ctx, db)
if err != nil {
return err
}
case noMigration:
// noop
default:
return errs.New("unsupported migration type: %s, please try one of the: %s", migration, strings.Join(migrationTypes, ","))
}
}
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, metabase.Config{
ApplicationName: "satellite-api",
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
@ -57,6 +84,26 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, metabaseDB.Close())
}()
for _, migration := range strings.Split(runCfg.DatabaseOptions.Migration, ",") {
switch migration {
case fullMigration:
err = metabaseDB.MigrateToLatest(ctx)
if err != nil {
return err
}
case snapshotMigration:
log.Info("Migration using latest snapshot. It's not for production", zap.String("db", "master"))
err = metabaseDB.TestMigrateToLatest(ctx)
if err != nil {
return err
}
case noMigration, testDataCreation:
// noop
default:
return errs.New("unsupported migration type: %s, please try one of the: %s", migration, strings.Join(migrationTypes, ","))
}
}
revocationDB, err := revocation.OpenDBFromCfg(ctx, runCfg.Config.Server.Config)
if err != nil {
return errs.New("Error creating revocation database on satellite api: %+v", err)

1
cmd/satellite/main.go
View File

@ -60,6 +60,7 @@ type Satellite struct {
Expiration time.Duration `help:"macaroon revocation cache expiration" default:"5m"`
Capacity int `help:"macaroon revocation cache capacity" default:"10000"`
}
Migration string `help:"coma separated migration types to run during every startup (none: no migration, snapshot: creating db from latest test snapshot (for testing only), testdata: create testuser in addition to a migration, full: do the normal migration (equals to 'satellite run migration'" default:"none" hidden:"true"`
}
satellite.Config

133
cmd/satellite/testdata.go Normal file
View File

@ -0,0 +1,133 @@
// Copyright (C) 2022 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"context"
"database/sql"
"errors"
"github.com/zeebo/errs"
"golang.org/x/crypto/bcrypt"
"storj.io/common/macaroon"
"storj.io/common/memory"
"storj.io/common/uuid"
"storj.io/storj/satellite"
"storj.io/storj/satellite/console"
"storj.io/storj/satellite/console/consolewasm"
)
const (
fullMigration = "full"
snapshotMigration = "snapshot"
testDataCreation = "testdata"
noMigration = "none"
)
var migrationTypes = []string{fullMigration, snapshotMigration, testDataCreation, noMigration}
var (
projectID = uuid.UUID([16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1})
apiKeyID = uuid.UUID([16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2})
head = []byte{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3,
}
secret = []byte{
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4,
}
password = "123a123"
)
// createTestData creates predefined test account to make the integration tests easier.
func createTestData(ctx context.Context, db satellite.DB) error {
userID, err := uuid.FromString("be041c3c-0658-40d1-8f7c-e70a0a26cc12")
if err != nil {
return err
}
_, err = db.Console().Users().Get(ctx, userID)
if errors.Is(err, sql.ErrNoRows) {
hash, err := bcrypt.GenerateFromPassword([]byte(password), 0)
if err != nil {
return err
}
_, err = db.Console().Users().Insert(ctx, &console.User{
ID: userID,
FullName: "Hiro Protagonist",
Email: "test@storj.io",
ProjectLimit: 5,
ProjectStorageLimit: (memory.GB * 150).Int64(),
ProjectBandwidthLimit: (memory.GB * 150).Int64(),
PasswordHash: hash,
})
if err != nil {
return err
}
active := console.Active
err = db.Console().Users().Update(ctx, userID, console.UpdateUserRequest{
Status: &active,
})
if err != nil {
return err
}
} else if err != nil {
return err
}
_, err = db.Console().Projects().Get(ctx, projectID)
if errors.Is(err, sql.ErrNoRows) {
_, err := db.Console().Projects().Insert(ctx, &console.Project{
ID: projectID,
OwnerID: userID,
Name: "testproject",
})
if err != nil {
return err
}
_, err = db.Console().ProjectMembers().Insert(ctx, userID, projectID)
if err != nil {
return err
}
} else if err != nil {
return err
}
_, err = db.Console().APIKeys().GetByNameAndProjectID(ctx, "testkey", projectID)
if errors.Is(err, sql.ErrNoRows) {
_, err = db.Console().APIKeys().Create(ctx, head, console.APIKeyInfo{
ID: apiKeyID,
ProjectID: projectID,
Name: "testkey",
Secret: secret,
})
} else if err != nil {
return err
}
return err
}
// GetTestApiKey can calculate an access grant for the predefined test users/project.
func GetTestApiKey(satelliteId string) (string, error) {
key, err := macaroon.FromParts(head, secret)
if err != nil {
return "", errs.Wrap(err)
}
accessGrant, err := consolewasm.GenAccessGrant(satelliteId, key.Serialize(), password, projectID.String())
if err != nil {
return "", errs.Wrap(err)
}
return accessGrant, nil
}

16
cmd/satellite/testdata_test.go Normal file
View File

@ -0,0 +1,16 @@
// Copyright (C) 2022 Storj Labs, Inc.
// See LICENSE for copying information.
package main
import (
"testing"
"github.com/stretchr/testify/require"
)
func TestGetTestApiKey(t *testing.T) {
key, err := GetTestApiKey("1nxk8B3L3zJi1nGTDemMWSTiyTehu87XaMGy1f742igrTEPciS@localhost:10001")
require.NoError(t, err)
require.Equal(t, "14CuLjd1ypmCQGkwFCAWxPGVRugM52shEia6MS2ASSTJ6P2qKsW2fWEujLH8ZaycTGHdxSpXoAWxXcwCsueJdCRcj2E4MHysMiwWsmm9rLnPYjE9YwUkNgDnR4nJHNtBbBF94smgk52RQw9WHHxas6owWJJceKmTYkip3odDHW52dNVqJVFRZZqwx4WEorey9KBe95RmSNbCL3mp6LVcMTYTT8vHSzPPAaT54wGuxiSH2rFkzkHBnL7fLD1AmwUham8cM7UPCG5EnPyGHKpjD944wgEgmdohfumLqmEmmmRW", key)
}

2
satellite/metabase/db.go
View File

@ -171,7 +171,7 @@ func (db *DB) TestMigrateToLatest(ctx context.Context) error {
{
DB: &db.db,
Description: "Test snapshot",
Version: 39,
Version: 15,
Action: migrate.SQL{
`CREATE TABLE objects (

6
satellite/satellitedb/migrate.go
View File

@ -119,11 +119,11 @@ func (db *satelliteDB) TestingMigrateToLatest(ctx context.Context) error {
if err != nil {
return ErrMigrateMinVersion.Wrap(err)
}
if dbVersion > -1 {
return ErrMigrateMinVersion.New("the database must be empty, got version %d", dbVersion)
}
testMigration := db.TestPostgresMigration()
if dbVersion != -1 && dbVersion != testMigration.Steps[0].Version {
return ErrMigrateMinVersion.New("the database must be empty, or be on the latest version (%d)", dbVersion)
}
return testMigration.Run(ctx, db.log.Named("migrate"))
default:
return migrate.Create(ctx, "database", db.DB)

5
satellite/satellitedb/projects.go
View File

@ -119,10 +119,13 @@ func (projects *projects) GetByPublicID(ctx context.Context, publicID uuid.UUID)
func (projects *projects) Insert(ctx context.Context, project *console.Project) (_ *console.Project, err error) {
defer mon.Task()(&ctx)(&err)
projectID, err := uuid.New()
projectID := project.ID
if projectID.IsZero() {
projectID, err = uuid.New()
if err != nil {
return nil, err
}
}
publicID, err := uuid.New()
if err != nil {
return nil, err