JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite: fix metabase configuration wiring

Browse Source 
New flag 'MultipleVersions' was not correctly passed from metainfo
configuration to metabase configuration. Because configuration was
set correctly for unit tests we didn't catch it and issue was found
while testing on QA satellite.

This change reduce number of places where new metabase flags needs
to be propagated from metainfo configuration to avoid problems with
setting new flags in the future.

Fixes https://github.com/storj/storj/issues/5274

Change-Id: I74bc122649febefd87f665be2fba628f6bfd9044
This commit is contained in:
Michal Niewrzal 2022-10-28 16:56:20 +02:00 committed by Storj Robot
parent 947cdf0fb4
commit d21bbab2b2
11 changed files with 36 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/satellite/admin.go
View File

@ -38,12 +38,8 @@ func cmdAdminRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close()) err = errs.Combine(err, db.Close())
}() }()
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL,
ApplicationName: "satellite-admin", runCfg.Config.Metainfo.Metabase("satellite-admin"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection on satellite api: %+v", err) return errs.New("Error creating metabase connection on satellite api: %+v", err)
} }

8
cmd/satellite/api.go
View File

@ -71,12 +71,8 @@ func cmdAPIRun(cmd *cobra.Command, args []string) (err error) {
} }
} }
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Config.Metainfo.DatabaseURL,
ApplicationName: "satellite-api", runCfg.Config.Metainfo.Metabase("satellite-api"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection on satellite api: %+v", err) return errs.New("Error creating metabase connection on satellite api: %+v", err)
} }

7
cmd/satellite/fetchpieces.go
View File

@ -59,11 +59,8 @@ func cmdFetchPieces(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close()) err = errs.Combine(err, db.Close())
}() }()
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL,
ApplicationName: "satellite-pieces-fetcher", runCfg.Config.Metainfo.Metabase("satellite-pieces-fetcher"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection: %+v", err) return errs.New("Error creating metabase connection: %+v", err)
} }

8
cmd/satellite/gc-bf.go
View File

@ -36,12 +36,8 @@ func cmdGCBloomFilterRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close()) err = errs.Combine(err, db.Close())
}() }()
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL,
ApplicationName: "satellite-gc-bloomfilter", runCfg.Config.Metainfo.Metabase("satellite-gc-bloomfilter"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection: %+v", err) return errs.New("Error creating metabase connection: %+v", err)
} }

8
cmd/satellite/gc.go
View File

@ -36,12 +36,8 @@ func cmdGCRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close()) err = errs.Combine(err, db.Close())
}() }()
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL,
ApplicationName: "satellite-gc", runCfg.Config.Metainfo.Metabase("satellite-gc"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection: %+v", err) return errs.New("Error creating metabase connection: %+v", err)
} }

16
cmd/satellite/main.go
View File

@ -427,12 +427,8 @@ func cmdRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close()) err = errs.Combine(err, db.Close())
}() }()
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL,
ApplicationName: "satellite-core", runCfg.Config.Metainfo.Metabase("satellite-core"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection: %+v", err) return errs.New("Error creating metabase connection: %+v", err)
} }
@ -516,12 +512,8 @@ func cmdMigrationRun(cmd *cobra.Command, args []string) (err error) {
return errs.New("Error creating tables for master database on satellite: %+v", err) return errs.New("Error creating tables for master database on satellite: %+v", err)
} }
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL,
ApplicationName: "satellite-migration", runCfg.Config.Metainfo.Metabase("satellite-migration"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection: %+v", err) return errs.New("Error creating metabase connection: %+v", err)
} }

7
cmd/satellite/repair_segment.go
View File

@ -48,11 +48,8 @@ func cmdRepairSegment(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close()) err = errs.Combine(err, db.Close())
}() }()
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL,
ApplicationName: "satellite-repair-segment", runCfg.Config.Metainfo.Metabase("satellite-repair-segment"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection: %+v", err) return errs.New("Error creating metabase connection: %+v", err)
} }

8
cmd/satellite/repairer.go
View File

@ -39,12 +39,8 @@ func cmdRepairerRun(cmd *cobra.Command, args []string) (err error) {
err = errs.Combine(err, db.Close()) err = errs.Combine(err, db.Close())
}() }()
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), runCfg.Metainfo.DatabaseURL,
ApplicationName: "satellite-repairer", runCfg.Config.Metainfo.Metabase("satellite-repairer"))
MinPartSize: runCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: runCfg.Config.Metainfo.MaxNumberOfParts,
ServerSideCopy: runCfg.Config.Metainfo.ServerSideCopy,
})
if err != nil { if err != nil {
return errs.New("Error creating metabase connection: %+v", err) return errs.New("Error creating metabase connection: %+v", err)
} }

8
cmd/tools/segment-verify/main.go
View File

@ -106,12 +106,8 @@ func verifySegmentsRange(cmd *cobra.Command, args []string) error {
}() }()
// open metabase // open metabase
metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), satelliteCfg.Metainfo.DatabaseURL, metabase.Config{ metabaseDB, err := metabase.Open(ctx, log.Named("metabase"), satelliteCfg.Metainfo.DatabaseURL,
ApplicationName: "satellite-core", satelliteCfg.Config.Metainfo.Metabase("satellite-core"))
MinPartSize: satelliteCfg.Config.Metainfo.MinPartSize,
MaxNumberOfParts: satelliteCfg.Config.Metainfo.MaxNumberOfParts,
ServerSideCopy: satelliteCfg.Config.Metainfo.ServerSideCopy,
})
if err != nil { if err != nil {
return Error.Wrap(err) return Error.Wrap(err)
} }

12
satellite/metainfo/config.go
View File

@ -12,6 +12,7 @@ import (
"github.com/vivint/infectious" "github.com/vivint/infectious"
"storj.io/common/memory" "storj.io/common/memory"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metabase/segmentloop" "storj.io/storj/satellite/metabase/segmentloop"
"storj.io/storj/satellite/metainfo/piecedeletion" "storj.io/storj/satellite/metainfo/piecedeletion"
"storj.io/uplink/private/eestream" "storj.io/uplink/private/eestream"
@ -142,3 +143,14 @@ type Config struct {
// TODO remove when we benchmarking are done and decision is made. // TODO remove when we benchmarking are done and decision is made.
TestListingQuery bool `default:"false" help:"test the new query for non-recursive listing"` TestListingQuery bool `default:"false" help:"test the new query for non-recursive listing"`
} }
// Metabase constructs Metabase configuration based on Metainfo configuration with specific application name.
func (c Config) Metabase(applicationName string) metabase.Config {
return metabase.Config{
ApplicationName: applicationName,
MinPartSize: c.MinPartSize,
MaxNumberOfParts: c.MaxNumberOfParts,
ServerSideCopy: c.ServerSideCopy,
MultipleVersions: c.MultipleVersions,
}
}

4
scripts/tests/integration/test-uplink.sh
View File

@ -43,6 +43,9 @@ uplink cp "$SRC_DIR/multisegment-upload-testfile" "sj://$BUCKET/" --progress=fal
uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/" --progress=false uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/" --progress=false
cat "$SRC_DIR/put-file" | uplink cp - "sj://$BUCKET/put-file" --progress=false cat "$SRC_DIR/put-file" | uplink cp - "sj://$BUCKET/put-file" --progress=false
# check overwriting of existing object
uplink cp "$SRC_DIR/big-upload-testfile" "sj://$BUCKET/" --progress=false
# test parallelism to upload single object # test parallelism to upload single object
# TODO change hardcoded part size from 64MiB to 6MiB # TODO change hardcoded part size from 64MiB to 6MiB
uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/diff-size-segments_upl_p2" --progress=false --parallelism 2 uplink cp "$SRC_DIR/diff-size-segments" "sj://$BUCKET/diff-size-segments_upl_p2" --progress=false --parallelism 2
@ -56,6 +59,7 @@ then
echo "listing returns $FILES files" echo "listing returns $FILES files"
else else
echo "listing returns $FILES files but want $EXPECTED_FILES" echo "listing returns $FILES files but want $EXPECTED_FILES"
cat "$TMPDIR/list"
exit 1 exit 1
fi fi