From f90ea10a4af6ef2df7f19bf8585880ccbb5b5efc Mon Sep 17 00:00:00 2001 From: Ethan Adams Date: Fri, 4 Dec 2020 05:24:39 -0500 Subject: [PATCH] Allow for DB application names per process. (#3983) --- cmd/satellite/admin.go | 1 + cmd/satellite/api.go | 3 ++- cmd/satellite/billing.go | 2 +- cmd/satellite/compensation.go | 6 +++--- cmd/satellite/gc.go | 4 ++-- cmd/satellite/gracefulexit.go | 2 +- cmd/satellite/main.go | 9 +++++---- cmd/satellite/repairer.go | 4 ++-- cmd/satellite/reports/attribution.go | 2 +- cmd/satellite/usage.go | 2 +- cmd/segment-reaper/delete.go | 2 +- cmd/segment-reaper/detect.go | 2 +- multinode/multinodedb/database.go | 5 ++++- private/dbutil/pgutil/db.go | 18 +++++++++++------- satellite/metainfo/config.go | 6 +++--- satellite/satellitedb/database.go | 6 +++++- satellite/satellitedb/migrate_test.go | 4 ++-- satellite/satellitedb/satellitedbtest/run.go | 4 ++-- storage/cockroachkv/client.go | 7 +++++-- storage/postgreskv/client.go | 7 +++++-- storage/postgreskv/client_test.go | 2 +- 21 files changed, 59 insertions(+), 39 deletions(-) diff --git a/cmd/satellite/admin.go b/cmd/satellite/admin.go index 84187a790..a7557454f 100644 --- a/cmd/satellite/admin.go +++ b/cmd/satellite/admin.go @@ -27,6 +27,7 @@ func cmdAdminRun(cmd *cobra.Command, args []string) (err error) { } db, err := satellitedb.Open(ctx, log.Named("db"), runCfg.Database, satellitedb.Options{ + ApplicationName: "satellite-admin", APIKeysLRUOptions: runCfg.APIKeysLRUOptions(), }) if err != nil { diff --git a/cmd/satellite/api.go b/cmd/satellite/api.go index c40e128f7..28171635b 100644 --- a/cmd/satellite/api.go +++ b/cmd/satellite/api.go @@ -32,6 +32,7 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) { } db, err := satellitedb.Open(ctx, log.Named("db"), runCfg.Database, satellitedb.Options{ + ApplicationName: "satellite-api", APIKeysLRUOptions: runCfg.APIKeysLRUOptions(), RevocationLRUOptions: runCfg.RevocationLRUOptions(), }) @@ -42,7 +43,7 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), runCfg.Config.Metainfo.DatabaseURL) + pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), runCfg.Config.Metainfo.DatabaseURL, "satellite-api") if err != nil { return errs.New("Error creating metainfodb connection on satellite api: %+v", err) } diff --git a/cmd/satellite/billing.go b/cmd/satellite/billing.go index ef3e050bd..6b71b8dc4 100644 --- a/cmd/satellite/billing.go +++ b/cmd/satellite/billing.go @@ -24,7 +24,7 @@ import ( func runBillingCmd(ctx context.Context, cmdFunc func(context.Context, *stripecoinpayments.Service, *dbx.DB) error) error { // Open SatelliteDB for the Payment Service logger := zap.L() - db, err := satellitedb.Open(ctx, logger.Named("db"), runCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, logger.Named("db"), runCfg.Database, satellitedb.Options{ApplicationName: "satellite-billing"}) if err != nil { return errs.New("error connecting to master database on satellite: %+v", err) } diff --git a/cmd/satellite/compensation.go b/cmd/satellite/compensation.go index 1420f2c88..01d9c8e70 100644 --- a/cmd/satellite/compensation.go +++ b/cmd/satellite/compensation.go @@ -32,7 +32,7 @@ func generateInvoicesCSV(ctx context.Context, period compensation.Period, out io WithheldPercents: generateInvoicesCfg.Compensation.WithheldPercents, } - db, err := satellitedb.Open(ctx, zap.L().Named("db"), generateInvoicesCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, zap.L().Named("db"), generateInvoicesCfg.Database, satellitedb.Options{ApplicationName: "satellite-compensation"}) if err != nil { return errs.New("error connecting to master database on satellite: %+v", err) } @@ -141,7 +141,7 @@ func recordPeriod(ctx context.Context, paystubsCSV, paymentsCSV string) (int, in return 0, 0, err } - db, err := satellitedb.Open(ctx, zap.L().Named("db"), recordPeriodCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, zap.L().Named("db"), recordPeriodCfg.Database, satellitedb.Options{ApplicationName: "satellite-compensation"}) if err != nil { return 0, 0, errs.New("error connecting to master database on satellite: %+v", err) } @@ -165,7 +165,7 @@ func recordOneOffPayments(ctx context.Context, paymentsCSV string) (int, error) return 0, err } - db, err := satellitedb.Open(ctx, zap.L().Named("db"), recordOneOffPaymentsCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, zap.L().Named("db"), recordOneOffPaymentsCfg.Database, satellitedb.Options{ApplicationName: "satellite-compensation"}) if err != nil { return 0, errs.New("error connecting to master database on satellite: %+v", err) } diff --git a/cmd/satellite/gc.go b/cmd/satellite/gc.go index 44f14fa85..4e63e28a7 100644 --- a/cmd/satellite/gc.go +++ b/cmd/satellite/gc.go @@ -28,7 +28,7 @@ func cmdGCRun(cmd *cobra.Command, args []string) (err error) { return errs.New("Failed to load identity: %+v", err) } - db, err := satellitedb.Open(ctx, log.Named("db"), runCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, log.Named("db"), runCfg.Database, satellitedb.Options{ApplicationName: "satellite-gc"}) if err != nil { return errs.New("Error starting master database on satellite GC: %+v", err) } @@ -36,7 +36,7 @@ func cmdGCRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), runCfg.Metainfo.DatabaseURL) + pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), runCfg.Metainfo.DatabaseURL, "satellite-gc") if err != nil { return errs.New("Error creating pointerDB connection GC: %+v", err) } diff --git a/cmd/satellite/gracefulexit.go b/cmd/satellite/gracefulexit.go index 1a83511af..4ca8de379 100644 --- a/cmd/satellite/gracefulexit.go +++ b/cmd/satellite/gracefulexit.go @@ -27,7 +27,7 @@ import ( // generateGracefulExitCSV creates a report with graceful exit data for exiting or exited nodes in a given period. func generateGracefulExitCSV(ctx context.Context, completed bool, start time.Time, end time.Time, output io.Writer) error { - db, err := satellitedb.Open(ctx, zap.L().Named("db"), gracefulExitCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, zap.L().Named("db"), gracefulExitCfg.Database, satellitedb.Options{ApplicationName: "satellite-gracefulexit"}) if err != nil { return errs.New("error connecting to master database on satellite: %+v", err) } diff --git a/cmd/satellite/main.go b/cmd/satellite/main.go index 1592ec00e..a4af0e63d 100644 --- a/cmd/satellite/main.go +++ b/cmd/satellite/main.go @@ -326,6 +326,7 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) { } db, err := satellitedb.Open(ctx, log.Named("db"), runCfg.Database, satellitedb.Options{ + ApplicationName: "satellite-core", ReportedRollupsReadBatchSize: runCfg.Orders.SettlementBatchSize, SaveRollupBatchSize: runCfg.Tally.SaveRollupBatchSize, ReadRollupBatchSize: runCfg.Tally.ReadRollupBatchSize, @@ -337,7 +338,7 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), runCfg.Metainfo.DatabaseURL) + pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), runCfg.Metainfo.DatabaseURL, "satellite-core") if err != nil { return errs.New("Error creating metainfodb connection: %+v", err) } @@ -401,7 +402,7 @@ func cmdMigrationRun(cmd *cobra.Command, args []string) (err error) { ctx, _ := process.Ctx(cmd) log := zap.L() - db, err := satellitedb.Open(ctx, log.Named("migration"), runCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, log.Named("migration"), runCfg.Database, satellitedb.Options{ApplicationName: "satellite-migration"}) if err != nil { return errs.New("Error creating new master database connection for satellitedb migration: %+v", err) } @@ -414,7 +415,7 @@ func cmdMigrationRun(cmd *cobra.Command, args []string) (err error) { return errs.New("Error creating tables for master database on satellite: %+v", err) } - pdb, err := metainfo.OpenStore(ctx, log.Named("migration"), runCfg.Metainfo.DatabaseURL) + pdb, err := metainfo.OpenStore(ctx, log.Named("migration"), runCfg.Metainfo.DatabaseURL, "satellite-migration") if err != nil { return errs.New("Error creating pointer database connection on satellite: %+v", err) } @@ -452,7 +453,7 @@ func cmdQDiag(cmd *cobra.Command, args []string) (err error) { ctx, _ := process.Ctx(cmd) // open the master db - database, err := satellitedb.Open(ctx, zap.L().Named("db"), qdiagCfg.Database, satellitedb.Options{}) + database, err := satellitedb.Open(ctx, zap.L().Named("db"), qdiagCfg.Database, satellitedb.Options{ApplicationName: "satellite-qdiag"}) if err != nil { return errs.New("error connecting to master database on satellite: %+v", err) } diff --git a/cmd/satellite/repairer.go b/cmd/satellite/repairer.go index 67810fe4a..839d303ec 100644 --- a/cmd/satellite/repairer.go +++ b/cmd/satellite/repairer.go @@ -31,7 +31,7 @@ func cmdRepairerRun(cmd *cobra.Command, args []string) (err error) { return errs.New("Failed to load identity: %+v", err) } - db, err := satellitedb.Open(ctx, log.Named("db"), runCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, log.Named("db"), runCfg.Database, satellitedb.Options{ApplicationName: "satellite-repairer"}) if err != nil { return errs.New("Error starting master database: %+v", err) } @@ -39,7 +39,7 @@ func cmdRepairerRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), runCfg.Metainfo.DatabaseURL) + pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), runCfg.Metainfo.DatabaseURL, "satellite-repairer") if err != nil { return errs.New("Error creating metainfo database connection: %+v", err) } diff --git a/cmd/satellite/reports/attribution.go b/cmd/satellite/reports/attribution.go index ab368c9f4..abe74fbe8 100644 --- a/cmd/satellite/reports/attribution.go +++ b/cmd/satellite/reports/attribution.go @@ -32,7 +32,7 @@ var headers = []string{ // GenerateAttributionCSV creates a report with. func GenerateAttributionCSV(ctx context.Context, database string, partnerID uuid.UUID, start time.Time, end time.Time, output io.Writer) error { log := zap.L().Named("db") - db, err := satellitedb.Open(ctx, log, database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, log, database, satellitedb.Options{ApplicationName: "satellite-attribution"}) if err != nil { return errs.New("error connecting to master database on satellite: %+v", err) } diff --git a/cmd/satellite/usage.go b/cmd/satellite/usage.go index b36a9ed66..407745229 100644 --- a/cmd/satellite/usage.go +++ b/cmd/satellite/usage.go @@ -21,7 +21,7 @@ import ( // generateNodeUsageCSV creates a report with node usage data for all nodes in a given period which can be used for payments. func generateNodeUsageCSV(ctx context.Context, start time.Time, end time.Time, output io.Writer) error { - db, err := satellitedb.Open(ctx, zap.L().Named("db"), nodeUsageCfg.Database, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, zap.L().Named("db"), nodeUsageCfg.Database, satellitedb.Options{ApplicationName: "satellite-nodeusage"}) if err != nil { return errs.New("error connecting to master database on satellite: %+v", err) } diff --git a/cmd/segment-reaper/delete.go b/cmd/segment-reaper/delete.go index 834e41f42..d4eccabd4 100644 --- a/cmd/segment-reaper/delete.go +++ b/cmd/segment-reaper/delete.go @@ -49,7 +49,7 @@ func cmdDelete(cmd *cobra.Command, args []string) (err error) { ctx, _ := process.Ctx(cmd) log := zap.L() - db, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), deleteCfg.DatabaseURL) + db, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), deleteCfg.DatabaseURL, "satellite-reaper") if err != nil { return errs.New("error connecting database: %+v", err) } diff --git a/cmd/segment-reaper/detect.go b/cmd/segment-reaper/detect.go index 54d79ff1c..2bf98f855 100644 --- a/cmd/segment-reaper/detect.go +++ b/cmd/segment-reaper/detect.go @@ -49,7 +49,7 @@ func cmdDetect(cmd *cobra.Command, args []string) (err error) { log.Warn("Failed to initialize telemetry batcher on segment reaper", zap.Error(err)) } - db, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), detectCfg.DatabaseURL) + db, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), detectCfg.DatabaseURL, "satellite-reaper") if err != nil { return errs.New("error connecting database: %+v", err) } diff --git a/multinode/multinodedb/database.go b/multinode/multinodedb/database.go index be05ac5a8..e9c651930 100644 --- a/multinode/multinodedb/database.go +++ b/multinode/multinodedb/database.go @@ -52,7 +52,10 @@ func Open(ctx context.Context, log *zap.Logger, databaseURL string) (multinode.D return nil, Error.New("unsupported driver %q", driver) } - source = pgutil.CheckApplicationName(source) + source, err = pgutil.CheckApplicationName(source, "storagenode-multinode") + if err != nil { + return nil, err + } dbxDB, err := dbx.Open(driver, source) if err != nil { diff --git a/private/dbutil/pgutil/db.go b/private/dbutil/pgutil/db.go index 1bb58b1d7..752adbf44 100644 --- a/private/dbutil/pgutil/db.go +++ b/private/dbutil/pgutil/db.go @@ -92,17 +92,21 @@ func QuerySnapshot(ctx context.Context, db dbschema.Queryer) (*dbschema.Snapshot } // CheckApplicationName ensures that the Connection String contains an application name. -func CheckApplicationName(s string) (r string) { +func CheckApplicationName(s string, app string) (string, error) { if !strings.Contains(s, "application_name") { - if !strings.Contains(s, "?") { - r = s + "?application_name=Satellite" - return + if strings.TrimSpace(app) == "" { + return s, errs.New("application name cannot be empty") } - r = s + "&application_name=Satellite" - return + + if !strings.Contains(s, "?") { + return s + "?application_name=" + app, nil + + } + + return s + "&application_name=" + app, nil } // return source as is if application_name is set - return s + return s, nil } // IsConstraintError checks if given error is about constraint violation. diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 88f6e0441..1b7663e6f 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -140,7 +140,7 @@ type PointerDB interface { } // OpenStore returns database for storing pointer data. -func OpenStore(ctx context.Context, logger *zap.Logger, dbURLString string) (db PointerDB, err error) { +func OpenStore(ctx context.Context, logger *zap.Logger, dbURLString string, app string) (db PointerDB, err error) { _, source, implementation, err := dbutil.SplitConnStr(dbURLString) if err != nil { return nil, err @@ -148,9 +148,9 @@ func OpenStore(ctx context.Context, logger *zap.Logger, dbURLString string) (db switch implementation { case dbutil.Postgres: - db, err = postgreskv.Open(ctx, source) + db, err = postgreskv.Open(ctx, source, app) case dbutil.Cockroach: - db, err = cockroachkv.Open(ctx, source) + db, err = cockroachkv.Open(ctx, source, app) default: err = Error.New("unsupported db implementation: %s", dbURLString) } diff --git a/satellite/satellitedb/database.go b/satellite/satellitedb/database.go index 0eab7f617..90221de01 100644 --- a/satellite/satellitedb/database.go +++ b/satellite/satellitedb/database.go @@ -66,6 +66,7 @@ type satelliteDB struct { // Options includes options for how a satelliteDB runs. type Options struct { + ApplicationName string APIKeysLRUOptions cache.Options RevocationLRUOptions cache.Options @@ -121,7 +122,10 @@ func open(ctx context.Context, log *zap.Logger, databaseURL string, opts Options return nil, Error.New("unsupported driver %q", driver) } - source = pgutil.CheckApplicationName(source) + source, err = pgutil.CheckApplicationName(source, opts.ApplicationName) + if err != nil { + return nil, err + } dbxDB, err := dbx.Open(driver, source) if err != nil { diff --git a/satellite/satellitedb/migrate_test.go b/satellite/satellitedb/migrate_test.go index ffe694b1d..5a2d952d7 100644 --- a/satellite/satellitedb/migrate_test.go +++ b/satellite/satellitedb/migrate_test.go @@ -157,7 +157,7 @@ func migrateTest(t *testing.T, connStr string) { defer func() { require.NoError(t, tempDB.Close()) }() // create a new satellitedb connection - db, err := satellitedb.Open(ctx, log, tempDB.ConnStr, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, log, tempDB.ConnStr, satellitedb.Options{ApplicationName: "satellite-migration-test"}) require.NoError(t, err) defer func() { require.NoError(t, db.Close()) }() @@ -249,7 +249,7 @@ func benchmarkSetup(b *testing.B, connStr string, merged bool) { defer func() { require.NoError(b, tempDB.Close()) }() // create a new satellitedb connection - db, err := satellitedb.Open(ctx, log, tempDB.ConnStr, satellitedb.Options{}) + db, err := satellitedb.Open(ctx, log, tempDB.ConnStr, satellitedb.Options{ApplicationName: "satellite-migration-test"}) require.NoError(b, err) defer func() { require.NoError(b, db.Close()) }() diff --git a/satellite/satellitedb/satellitedbtest/run.go b/satellite/satellitedb/satellitedbtest/run.go index ec9e5c02b..58ab80fb4 100644 --- a/satellite/satellitedb/satellitedbtest/run.go +++ b/satellite/satellitedb/satellitedbtest/run.go @@ -119,7 +119,7 @@ func CreateMasterDB(ctx context.Context, log *zap.Logger, name string, category // CreateMasterDBOnTopOf creates a new satellite database on top of an already existing // temporary database. func CreateMasterDBOnTopOf(ctx context.Context, log *zap.Logger, tempDB *dbutil.TempDatabase) (db satellite.DB, err error) { - masterDB, err := satellitedb.Open(ctx, log.Named("db"), tempDB.ConnStr, satellitedb.Options{}) + masterDB, err := satellitedb.Open(ctx, log.Named("db"), tempDB.ConnStr, satellitedb.Options{ApplicationName: "satellite-satellitdb-test"}) return &tempMasterDB{DB: masterDB, tempDB: tempDB}, err } @@ -156,7 +156,7 @@ func CreatePointerDB(ctx context.Context, log *zap.Logger, name string, category // CreatePointerDBOnTopOf creates a new satellite database on top of an already existing // temporary database. func CreatePointerDBOnTopOf(ctx context.Context, log *zap.Logger, tempDB *dbutil.TempDatabase) (db metainfo.PointerDB, err error) { - pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), tempDB.ConnStr) + pointerDB, err := metainfo.OpenStore(ctx, log.Named("pointerdb"), tempDB.ConnStr, "satellite-satellitdb-test") if err != nil { return nil, err } diff --git a/storage/cockroachkv/client.go b/storage/cockroachkv/client.go index 24e8d21e8..083e78a3a 100644 --- a/storage/cockroachkv/client.go +++ b/storage/cockroachkv/client.go @@ -34,8 +34,11 @@ type Client struct { } // Open connects a new cockroachkv client given db URL. -func Open(ctx context.Context, dbURL string) (*Client, error) { - dbURL = pgutil.CheckApplicationName(dbURL) +func Open(ctx context.Context, dbURL string, app string) (*Client, error) { + dbURL, err := pgutil.CheckApplicationName(dbURL, app) + if err != nil { + return nil, err + } db, err := tagsql.Open(ctx, "cockroach", dbURL) if err != nil { diff --git a/storage/postgreskv/client.go b/storage/postgreskv/client.go index ffb82f908..63bf64f3c 100644 --- a/storage/postgreskv/client.go +++ b/storage/postgreskv/client.go @@ -31,8 +31,11 @@ type Client struct { } // Open connects a new postgreskv client given db URL. -func Open(ctx context.Context, dbURL string) (*Client, error) { - dbURL = pgutil.CheckApplicationName(dbURL) +func Open(ctx context.Context, dbURL string, app string) (*Client, error) { + dbURL, err := pgutil.CheckApplicationName(dbURL, app) + if err != nil { + return nil, err + } db, err := tagsql.Open(ctx, "pgx", dbURL) if err != nil { diff --git a/storage/postgreskv/client_test.go b/storage/postgreskv/client_test.go index 1db2e2177..67bb25ecb 100644 --- a/storage/postgreskv/client_test.go +++ b/storage/postgreskv/client_test.go @@ -23,7 +23,7 @@ import ( func openTestPostgres(ctx context.Context, t testing.TB) (store *Client, cleanup func()) { connstr := pgtest.PickPostgres(t) - pgdb, err := Open(ctx, connstr) + pgdb, err := Open(ctx, connstr, "satellite-test-postgres") if err != nil { t.Fatalf("init: %v", err) }