From d21bbab2b20fd7d0d81e73446893f88ef5cb14c0 Mon Sep 17 00:00:00 2001 From: Michal Niewrzal Date: Fri, 28 Oct 2022 16:56:20 +0200 Subject: [PATCH] satellite: fix metabase configuration wiring New flag 'MultipleVersions' was not correctly passed from metainfo configuration to metabase configuration. Because configuration was set correctly for unit tests we didn't catch it and issue was found while testing on QA satellite. This change reduce number of places where new metabase flags needs to be propagated from metainfo configuration to avoid problems with setting new flags in the future. Fixes https://github.com/storj/storj/issues/5274 Change-Id: I74bc122649febefd87f665be2fba628f6bfd9044 --- cmd/satellite/admin.go | 8 ++------ cmd/satellite/api.go | 8 ++------ cmd/satellite/fetchpieces.go | 7 ++----- cmd/satellite/gc-bf.go | 8 ++------ cmd/satellite/gc.go | 8 ++------ cmd/satellite/main.go | 16 ++++------------ cmd/satellite/repair_segment.go | 7 ++----- cmd/satellite/repairer.go | 8 ++------ cmd/tools/segment-verify/main.go | 8 ++------ satellite/metainfo/config.go | 12 ++++++++++++ scripts/tests/integration/test-uplink.sh | 4 ++++ 11 files changed, 36 insertions(+), 58 deletions(-) diff --git a/cmd/satellite/admin.go b/cmd/satellite/admin.go index 0fa1d71d8..072c74067 100644 --- a/cmd/satellite/admin.go +++ b/cmd/satellite/admin.go @@ -38,12 +38,8 @@ func cmdAdminRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-admin", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-admin")) if err != nil { return errs.New("Error creating metabase connection on satellite api: %+v", err) } diff --git a/cmd/satellite/api.go b/cmd/satellite/api.go index e7d48a76b..4137f55f6 100644 --- a/cmd/satellite/api.go +++ b/cmd/satellite/api.go @@ -71,12 +71,8 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) { } } - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-api", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-api")) if err != nil { return errs.New("Error creating metabase connection on satellite api: %+v", err) } diff --git a/cmd/satellite/fetchpieces.go b/cmd/satellite/fetchpieces.go index 461b448ae..a25fac3ea 100644 --- a/cmd/satellite/fetchpieces.go +++ b/cmd/satellite/fetchpieces.go @@ -59,11 +59,8 @@ func cmdFetchPieces(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-pieces-fetcher", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-pieces-fetcher")) if err != nil { return errs.New("Error creating metabase connection: %+v", err) } diff --git a/cmd/satellite/gc-bf.go b/cmd/satellite/gc-bf.go index 8e0ac0e62..c9b53571c 100644 --- a/cmd/satellite/gc-bf.go +++ b/cmd/satellite/gc-bf.go @@ -36,12 +36,8 @@ func cmdGCBloomFilterRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-gc-bloomfilter", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-gc-bloomfilter")) if err != nil { return errs.New("Error creating metabase connection: %+v", err) } diff --git a/cmd/satellite/gc.go b/cmd/satellite/gc.go index 3aa377dca..63c169b36 100644 --- a/cmd/satellite/gc.go +++ b/cmd/satellite/gc.go @@ -36,12 +36,8 @@ func cmdGCRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-gc", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-gc")) if err != nil { return errs.New("Error creating metabase connection: %+v", err) } diff --git a/cmd/satellite/main.go b/cmd/satellite/main.go index 53f6e224d..56c894730 100644 --- a/cmd/satellite/main.go +++ b/cmd/satellite/main.go @@ -427,12 +427,8 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-core", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-core")) if err != nil { return errs.New("Error creating metabase connection: %+v", err) } @@ -516,12 +512,8 @@ func cmdMigrationRun(cmd *cobra.Command, args []string) (err error) { return errs.New("Error creating tables for master database on satellite: %+v", err) } - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-migration", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-migration")) if err != nil { return errs.New("Error creating metabase connection: %+v", err) } diff --git a/cmd/satellite/repair_segment.go b/cmd/satellite/repair_segment.go index f05de851d..8e31f498d 100644 --- a/cmd/satellite/repair_segment.go +++ b/cmd/satellite/repair_segment.go @@ -48,11 +48,8 @@ func cmdRepairSegment(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-repair-segment", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-repair-segment")) if err != nil { return errs.New("Error creating metabase connection: %+v", err) } diff --git a/cmd/satellite/repairer.go b/cmd/satellite/repairer.go index fc8b29d6c..bedc1fc40 100644 --- a/cmd/satellite/repairer.go +++ b/cmd/satellite/repairer.go @@ -39,12 +39,8 @@ func cmdRepairerRun(cmd *cobra.Command, args []string) (err error) { err = errs.Combine(err, db.Close()) }() - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-repairer", - MinPartSize: runCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts, - ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, + runCfg.Config.Metainfo.Metabase("satellite-repairer")) if err != nil { return errs.New("Error creating metabase connection: %+v", err) } diff --git a/cmd/tools/segment-verify/main.go b/cmd/tools/segment-verify/main.go index ad19f78f8..25eeaad38 100644 --- a/cmd/tools/segment-verify/main.go +++ b/cmd/tools/segment-verify/main.go @@ -106,12 +106,8 @@ func verifySegmentsRange(cmd *cobra.Command, args []string) error { }() // open metabase - metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), satelliteCfg.Metainfo.DatabaseURL, metabase.Config{ - ApplicationName: "satellite-core", - MinPartSize: satelliteCfg.Config.Metainfo.MinPartSize, - MaxNumberOfParts: satelliteCfg.Config.Metainfo.MaxNumberOfParts, - ServerSideCopy: satelliteCfg.Config.Metainfo.ServerSideCopy, - }) + metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), satelliteCfg.Metainfo.DatabaseURL, + satelliteCfg.Config.Metainfo.Metabase("satellite-core")) if err != nil { return Error.Wrap(err) } diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 2f54fac74..134f813bb 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -12,6 +12,7 @@ import ( "github.com/vivint/infectious" "storj.io/common/memory" + "storj.io/storj/satellite/metabase" "storj.io/storj/satellite/metabase/segmentloop" "storj.io/storj/satellite/metainfo/piecedeletion" "storj.io/uplink/private/eestream" @@ -142,3 +143,14 @@ type Config struct { // TODO remove when we benchmarking are done and decision is made. TestListingQuery bool `default:"false" help:"test the new query for non-recursive listing"` } + +// Metabase constructs Metabase configuration based on Metainfo configuration with specific application name. +func (c Config) Metabase(applicationName string) metabase.Config { + return metabase.Config{ + ApplicationName: applicationName, + MinPartSize: c.MinPartSize, + MaxNumberOfParts: c.MaxNumberOfParts, + ServerSideCopy: c.ServerSideCopy, + MultipleVersions: c.MultipleVersions, + } +} diff --git a/scripts/tests/integration/test-uplink.sh b/scripts/tests/integration/test-uplink.sh index 7f322a5fe..d0f9e2b4f 100755 --- a/scripts/tests/integration/test-uplink.sh +++ b/scripts/tests/integration/test-uplink.sh @@ -43,6 +43,9 @@ uplink cp "$SRC_DIR/multisegment-upload-testfile" "sj://$BUCKET/" --progress=fal uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/" --progress=false cat "$SRC_DIR/put-file" | uplink cp - "sj://$BUCKET/put-file" --progress=false +# check overwriting of existing object +uplink cp "$SRC_DIR/big-upload-testfile" "sj://$BUCKET/" --progress=false + # test parallelism to upload single object # TODO change hardcoded part size from 64MiB to 6MiB uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/diff-size-segments_upl_p2" --progress=false --parallelism 2 @@ -56,6 +59,7 @@ then echo "listing returns $FILES files" else echo "listing returns $FILES files but want $EXPECTED_FILES" + cat "$TMPDIR/list" exit 1 fi