JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/satellitedb: cleanup testing access

Browse Source 
Previously we were exposing the testing facilities via interface casting
the necessary parts, however, when things are not part of the main
satellite.DB interface they need to be manually propagated. Rather than
relying on using hidden methods lets expose things as long as they don't
create a direct dependency to the database driver.

Change-Id: I2eb7d8b60f4b64de1320c2d32581f7be267c0f57
This commit is contained in:
Egon Elbre 2023-02-06 14:15:36 +02:00
parent e8cd096eec
commit 3146ad7f2e
12 changed files with 113 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/satellite/api.go
View File

@ -55,7 +55,7 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) {
}
case snapshotMigration:
log.Info("MigrationUnsafe using latest snapshot. It's not for production", zap.String("db", "master"))
err = db.TestingMigrateToLatest(ctx)
err = db.Testing().TestMigrateToLatest(ctx)
if err != nil {
return err
}

2
cmd/tools/generate-missing-project-salt/main_test.go
View File

@ -108,7 +108,7 @@ func test(t *testing.T,
require.NoError(t, err)
defer ctx.Check(db.Close)
err = db.TestingMigrateToLatest(ctx)
err = db.Testing().TestMigrateToLatest(ctx)
require.NoError(t, err)
mConnStr := strings.Replace(tempDB.ConnStr, "cockroach", "postgres", 1)

2
cmd/tools/migrate-public-ids/main_test.go
View File

@ -143,7 +143,7 @@ func test(t *testing.T, prepare func(t *testing.T, ctx *testcontext.Context, raw
require.NoError(t, err)
defer ctx.Check(db.Close)
err = db.TestingMigrateToLatest(ctx)
err = db.Testing().TestMigrateToLatest(ctx)
require.NoError(t, err)
mConnStr := strings.Replace(tempDB.ConnStr, "cockroach", "postgres", 1)

6
cmd/tools/node-cleanup/main.go
View File

@ -21,7 +21,6 @@ import (
"storj.io/private/process"
"storj.io/private/tagsql"
"storj.io/storj/satellite/satellitedb"
"storj.io/storj/satellite/satellitedb/dbx"
)
var mon = monkit.Package()
@ -103,10 +102,7 @@ func Delete(ctx context.Context, log *zap.Logger, config Config) (err error) {
return errs.New("database version not correct: %w", err)
}
// TODO: ensure this works
raw := db.(interface{ TestDBAccess() *dbx.DB }).TestDBAccess()
return DeleteFromTables(ctx, log, raw.DB, config)
return DeleteFromTables(ctx, log, db.Testing().RawDB(), config)
}
var maxNodeID = (func() storj.NodeID {

2
cmd/tools/node-cleanup/main_test.go
View File

@ -23,7 +23,7 @@ import (
func TestDelete(t *testing.T) {
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
raw := db.(interface{ DebugGetDBHandle() tagsql.DB }).DebugGetDBHandle()
raw := db.Testing().RawDB()
// insert 5 nodes (4 to delete)
for i := 0; i < 5; i++ {

4
private/testplanet/run.go
View File

@ -75,9 +75,7 @@ func Run(t *testing.T, config Config, test func(t *testing.T, ctx *testcontext.C
var rawDB tagsql.DB
var queriesBefore []string
if len(planet.Satellites) > 0 && satelliteDB.Name == "Cockroach" {
db := planet.Satellites[0].DB
// not perfect but didn't find better way to do this
rawDB = db.(interface{ DebugGetDBHandle() tagsql.DB }).DebugGetDBHandle()
rawDB = planet.Satellites[0].DB.Testing().RawDB()
var err error
queriesBefore, err = satellitedbtest.FullTableScanQueries(ctx, rawDB, dbutil.Cockroach, planetConfig.applicationName)

2
private/testplanet/satellite.go
View File

@ -528,7 +528,7 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
return nil, errs.Wrap(err)
}
err = db.TestingMigrateToLatest(ctx)
err = db.Testing().TestMigrateToLatest(ctx)
if err != nil {
return nil, errs.Wrap(err)
}

22
satellite/peer.go
View File

@ -15,6 +15,8 @@ import (
"storj.io/common/identity"
"storj.io/private/debug"
"storj.io/private/tagsql"
"storj.io/storj/private/migrate"
"storj.io/storj/private/post"
"storj.io/storj/private/post/oauth2"
"storj.io/storj/private/server"
@ -84,9 +86,6 @@ type DB interface {
// Close closes the database
Close() error
// TestingMigrateToLatest initializes the database for testplanet.
TestingMigrateToLatest(ctx context.Context) error
// PeerIdentities returns a storage for peer identities
PeerIdentities() overlay.PeerIdentities
// OverlayCache returns database for caching overlay information
@ -135,6 +134,23 @@ type DB interface {
NodeAPIVersion() nodeapiversion.DB
// StorjscanPayments stores payments retrieved from storjscan.
StorjscanPayments() storjscan.PaymentsDB
// Testing provides access to testing facilities. These should not be used in production code.
Testing() TestingDB
}
// TestingDB defines access to database testing facilities.
type TestingDB interface {
// RawDB returns the underlying database connection to the primary database.
RawDB() tagsql.DB
// Schema returns the full schema for the database.
Schema() string
// TestMigrateToLatest initializes the database for testplanet.
TestMigrateToLatest(ctx context.Context) error
// ProductionMigration returns the primary migration.
ProductionMigration() *migrate.Migration
// TestMigration returns the migration used for tests.
TestMigration() *migrate.Migration
}
// Config is the global config satellite.

92
satellite/satellitedb/database.go
View File

@ -164,26 +164,6 @@ func (dbc *satelliteDBCollection) getByName(name string) *satelliteDB {
return dbc.dbs[""]
}
// TestDBAccess for raw database access,
// should not be used outside of migration tests.
func (db *satelliteDB) TestDBAccess() *dbx.DB { return db.DB }
// TestDBAccess for raw database access,
// should not be used outside of migration tests.
func (dbc *satelliteDBCollection) TestDBAccess() *dbx.DB {
return dbc.getByName("").TestDBAccess()
}
// MigrationTestingDefaultDB assists in testing migrations themselves against
// the default database.
func (dbc *satelliteDBCollection) MigrationTestingDefaultDB() interface {
TestDBAccess() *dbx.DB
TestPostgresMigration() *migrate.Migration
PostgresMigration() *migrate.Migration
} {
return dbc.getByName("")
}
// PeerIdentities returns a storage for peer identities.
func (dbc *satelliteDBCollection) PeerIdentities() overlay.PeerIdentities {
return &peerIdentities{db: dbc.getByName("peeridentities")}
@ -345,15 +325,6 @@ func (dbc *satelliteDBCollection) MigrateToLatest(ctx context.Context) error {
return eg.Err()
}
// TestingMigrateToLatest is a method for creating all tables for all database for testing.
func (dbc *satelliteDBCollection) TestingMigrateToLatest(ctx context.Context) error {
var eg errs.Group
for _, db := range dbc.dbs {
eg.Add(db.TestingMigrateToLatest(ctx))
}
return eg.Err()
}
// Close closes all satellite dbs.
func (dbc *satelliteDBCollection) Close() error {
var eg errs.Group
@ -362,3 +333,66 @@ func (dbc *satelliteDBCollection) Close() error {
}
return eg.Err()
}
// Testing provides access to testing facilities. These should not be used in production code.
func (db *satelliteDB) Testing() satellite.TestingDB {
return &satelliteDBTesting{satelliteDB: db}
}
type satelliteDBTesting struct{ *satelliteDB }
// RawDB returns the underlying database connection to the primary database.
func (db *satelliteDBTesting) RawDB() tagsql.DB {
return db.satelliteDB.DB
}
// Schema returns the full schema for the database.
func (db *satelliteDBTesting) Schema() string {
return db.satelliteDB.Schema()
}
// ProductionMigration returns the primary migration.
func (db *satelliteDBTesting) ProductionMigration() *migrate.Migration {
return db.satelliteDB.ProductionMigration()
}
// TestMigration returns the migration used for tests.
func (db *satelliteDBTesting) TestMigration() *migrate.Migration {
return db.satelliteDB.TestMigration()
}
// Testing provides access to testing facilities. These should not be used in production code.
func (dbc *satelliteDBCollection) Testing() satellite.TestingDB {
return &satelliteDBCollectionTesting{satelliteDBCollection: dbc}
}
type satelliteDBCollectionTesting struct{ *satelliteDBCollection }
// RawDB returns the underlying database connection to the primary database.
func (dbc *satelliteDBCollectionTesting) RawDB() tagsql.DB {
return dbc.getByName("").DB.DB
}
// Schema returns the full schema for the database.
func (dbc *satelliteDBCollectionTesting) Schema() string {
return dbc.getByName("").Schema()
}
// MigrateToLatest initializes the database for testplanet.
func (dbc *satelliteDBCollectionTesting) TestMigrateToLatest(ctx context.Context) error {
var eg errs.Group
for _, db := range dbc.dbs {
eg.Add(db.Testing().TestMigrateToLatest(ctx))
}
return eg.Err()
}
// ProductionMigration returns the primary migration.
func (dbc *satelliteDBCollectionTesting) ProductionMigration() *migrate.Migration {
return dbc.getByName("").ProductionMigration()
}
// TestMigration returns the migration used for tests.
func (dbc *satelliteDBCollectionTesting) TestMigration() *migrate.Migration {
return dbc.getByName("").TestMigration()
}

20
satellite/satellitedb/migrate.go
View File

@ -63,7 +63,7 @@ func (db *satelliteDB) MigrateToLatest(ctx context.Context) error {
switch db.impl {
case dbutil.Postgres, dbutil.Cockroach:
migration := db.PostgresMigration()
migration := db.ProductionMigration()
// since we merged migration steps 0-69, the current db version should never be
// less than 69 unless the migration hasn't run yet
const minDBVersion = 69
@ -83,8 +83,8 @@ func (db *satelliteDB) MigrateToLatest(ctx context.Context) error {
}
}
// TestingMigrateToLatest is a method for creating all tables for database for testing.
func (db *satelliteDB) TestingMigrateToLatest(ctx context.Context) error {
// TestMigrateToLatest is a method for creating all tables for database for testing.
func (db *satelliteDBTesting) TestMigrateToLatest(ctx context.Context) error {
switch db.impl {
case dbutil.Postgres:
schema, err := pgutil.ParseSchemaFromConnstr(db.source)
@ -113,14 +113,14 @@ func (db *satelliteDB) TestingMigrateToLatest(ctx context.Context) error {
switch db.impl {
case dbutil.Postgres, dbutil.Cockroach:
migration := db.PostgresMigration()
migration := db.ProductionMigration()
dbVersion, err := migration.CurrentVersion(ctx, db.log, db.DB)
if err != nil {
return ErrMigrateMinVersion.Wrap(err)
}
testMigration := db.TestPostgresMigration()
testMigration := db.TestMigration()
if dbVersion != -1 && dbVersion != testMigration.Steps[0].Version {
return ErrMigrateMinVersion.New("the database must be empty, or be on the latest version (%d)", dbVersion)
}
@ -134,7 +134,7 @@ func (db *satelliteDB) TestingMigrateToLatest(ctx context.Context) error {
func (db *satelliteDB) CheckVersion(ctx context.Context) error {
switch db.impl {
case dbutil.Postgres, dbutil.Cockroach:
migration := db.PostgresMigration()
migration := db.ProductionMigration()
return migration.ValidateVersions(ctx, db.log)
default:
@ -142,13 +142,13 @@ func (db *satelliteDB) CheckVersion(ctx context.Context) error {
}
}
// TestPostgresMigration returns steps needed for migrating test postgres database.
func (db *satelliteDB) TestPostgresMigration() *migrate.Migration {
// TestMigration returns steps needed for migrating test postgres database.
func (db *satelliteDB) TestMigration() *migrate.Migration {
return db.testMigration()
}
// PostgresMigration returns steps needed for migrating postgres database.
func (db *satelliteDB) PostgresMigration() *migrate.Migration {
// ProductionMigration returns steps needed for migrating postgres database.
func (db *satelliteDB) ProductionMigration() *migrate.Migration {
return &migrate.Migration{
Table: "versions",
Steps: []*migrate.Step{

37
satellite/satellitedb/migrate_test.go
View File

@ -30,8 +30,8 @@ import (
"storj.io/private/dbutil/pgutil"
"storj.io/private/dbutil/tempdb"
"storj.io/storj/private/migrate"
"storj.io/storj/satellite"
"storj.io/storj/satellite/satellitedb"
"storj.io/storj/satellite/satellitedb/dbx"
)
const maxMigrationsToTest = 10
@ -185,16 +185,6 @@ func TestMigrateCockroach(t *testing.T) {
t.Run("Generated", func(t *testing.T) { migrateGeneratedTest(t, connstr, connstr) })
}
type migrationTestingAccess interface {
// MigrationTestingDefaultDB assists in testing migrations themselves
// against the default database.
MigrationTestingDefaultDB() interface {
TestDBAccess() *dbx.DB
TestPostgresMigration() *migrate.Migration
PostgresMigration() *migrate.Migration
}
}
func migrateTest(t *testing.T, connStr string) {
ctx := testcontext.NewWithTimeout(t, 8*time.Minute)
defer ctx.Cleanup()
@ -212,15 +202,15 @@ func migrateTest(t *testing.T, connStr string) {
defer func() { require.NoError(t, db.Close()) }()
// we need raw database access unfortunately
rawdb := db.(migrationTestingAccess).MigrationTestingDefaultDB().TestDBAccess()
rawdb := db.Testing().RawDB()
loadingStart := time.Now()
snapshots, dbxschema, err := loadSnapshots(ctx, connStr, rawdb.Schema(), maxMigrationsToTest)
snapshots, dbxschema, err := loadSnapshots(ctx, connStr, db.Testing().Schema(), maxMigrationsToTest)
require.NoError(t, err)
t.Logf("snapshot loading %v", time.Since(loadingStart))
// get migration for this database
migrations := db.(migrationTestingAccess).MigrationTestingDefaultDB().PostgresMigration()
migrations := db.Testing().ProductionMigration()
// find the first matching migration step for the snapshots
firstSnapshot := snapshots.List[0]
@ -311,12 +301,12 @@ func migrateGeneratedTest(t *testing.T, connStrProd, connStrTest string) {
ctx := testcontext.NewWithTimeout(t, 8*time.Minute)
defer ctx.Cleanup()
prodVersion, prodSnapshot := schemaFromMigration(t, ctx, connStrProd, func(db migrationTestingAccess) *migrate.Migration {
return db.MigrationTestingDefaultDB().PostgresMigration()
prodVersion, prodSnapshot := schemaFromMigration(t, ctx, connStrProd, func(db satellite.DB) *migrate.Migration {
return db.Testing().ProductionMigration()
})
testVersion, testSnapshot := schemaFromMigration(t, ctx, connStrTest, func(db migrationTestingAccess) *migrate.Migration {
return db.MigrationTestingDefaultDB().TestPostgresMigration()
testVersion, testSnapshot := schemaFromMigration(t, ctx, connStrTest, func(db satellite.DB) *migrate.Migration {
return db.Testing().TestMigration()
})
assert.Equal(t, prodVersion, testVersion, "migratez version does not match migration. Run `go generate` to update.")
@ -328,7 +318,7 @@ func migrateGeneratedTest(t *testing.T, connStrProd, connStrTest string) {
require.Equal(t, prodSnapshot.Data, testSnapshot.Data, "migratez data does not match migration. Run `go generate` to update.")
}
func schemaFromMigration(t *testing.T, ctx *testcontext.Context, connStr string, getMigration func(migrationTestingAccess) *migrate.Migration) (version int, _ *dbschema.Snapshot) {
func schemaFromMigration(t *testing.T, ctx *testcontext.Context, connStr string, getMigration func(db satellite.DB) *migrate.Migration) (version int, _ *dbschema.Snapshot) {
// create tempDB
log := zaptest.NewLogger(t)
@ -343,13 +333,10 @@ func schemaFromMigration(t *testing.T, ctx *testcontext.Context, connStr string,
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
testAccess := db.(migrationTestingAccess)
migration := getMigration(testAccess)
migration := getMigration(db)
require.NoError(t, migration.Run(ctx, log))
rawdb := testAccess.MigrationTestingDefaultDB().TestDBAccess()
snapshot, err := pgutil.QuerySnapshot(ctx, rawdb)
snapshot, err := pgutil.QuerySnapshot(ctx, db.Testing().RawDB())
require.NoError(t, err)
return migration.Steps[len(migration.Steps)-1].Version, snapshot
@ -392,7 +379,7 @@ func benchmarkSetup(b *testing.B, connStr string, merged bool) {
defer func() { require.NoError(b, db.Close()) }()
if merged {
err = db.TestingMigrateToLatest(ctx)
err = db.Testing().TestMigrateToLatest(ctx)
require.NoError(b, err)
} else {
err = db.MigrateToLatest(ctx)

10
satellite/satellitedb/satellitedbtest/run.go
View File

@ -129,12 +129,6 @@ func (db *tempMasterDB) Close() error {
return errs.Combine(db.DB.Close(), db.tempDB.Close())
}
// DebugGetDBHandle exposes a handle to the raw database object. This is intended
// only for testing purposes and is temporary.
func (db *tempMasterDB) DebugGetDBHandle() tagsql.DB {
return db.tempDB.DB
}
// CreateMasterDB creates a new satellite database for testing.
func CreateMasterDB(ctx context.Context, log *zap.Logger, name string, category string, index int, dbInfo Database, applicationName string) (db satellite.DB, err error) {
if dbInfo.URL == "" {
@ -224,7 +218,7 @@ func Run(t *testing.T, test func(ctx *testcontext.Context, t *testing.T, db sate
}
}()
err = db.TestingMigrateToLatest(ctx)
err = db.Testing().TestMigrateToLatest(ctx)
if err != nil {
t.Fatal(err)
}
@ -292,7 +286,7 @@ func Bench(b *testing.B, bench func(b *testing.B, db satellite.DB)) {
// FullTableScanQueries is a helper method to list all queries which performed full table scan recently. It works only for cockroach db.
func FullTableScanQueries(ctx context.Context, db tagsql.DB, implementation dbutil.Implementation, applicationName string) (queries []string, err error) {
if implementation.String() != "cockroach" {
if implementation != dbutil.Cockroach {
return nil, nil
}