From 36cbf4f0b8667fc6e639f47623120569a3b628ab Mon Sep 17 00:00:00 2001 From: Michal Niewrzal Date: Thu, 2 Feb 2023 12:34:28 +0100 Subject: [PATCH] private/testplanet: enable full table scan detection Testplanet tests will print into logs (WARN) if full table scan will be detected. Test won't be failed automatically. That's because currently we have multiple queries which are doing full table scan and it's not trivial to change. https://github.com/storj/storj/issues/5471 Change-Id: Ia2fcbfb9102424d58f95e00071329454a8c1066e --- private/testplanet/planet.go | 6 ++++ private/testplanet/run.go | 33 ++++++++++++++++++++ private/testplanet/satellite.go | 6 ++-- satellite/satellitedb/satellitedbtest/run.go | 7 +++-- 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/private/testplanet/planet.go b/private/testplanet/planet.go index e6c316617..754383979 100644 --- a/private/testplanet/planet.go +++ b/private/testplanet/planet.go @@ -63,6 +63,8 @@ type Config struct { Host string NonParallel bool Timeout time.Duration + + applicationName string } // DatabaseConfig defines connection strings for database. @@ -143,6 +145,10 @@ func NewCustom(ctx *testcontext.Context, log *zap.Logger, config Config, satelli } } + if config.applicationName == "" { + config.applicationName = "testplanet" + } + planet := &Planet{ ctx: ctx, log: log, diff --git a/private/testplanet/run.go b/private/testplanet/run.go index 27edf3676..b4b2fd1f0 100644 --- a/private/testplanet/run.go +++ b/private/testplanet/run.go @@ -8,12 +8,17 @@ import ( "runtime/pprof" "testing" + "github.com/google/go-cmp/cmp" "go.uber.org/zap" + "storj.io/common/context2" "storj.io/common/grant" "storj.io/common/storj" "storj.io/common/testcontext" + "storj.io/private/dbutil" "storj.io/private/dbutil/pgtest" + "storj.io/private/dbutil/pgutil" + "storj.io/private/tagsql" "storj.io/storj/private/testmonkit" "storj.io/storj/satellite/satellitedb/satellitedbtest" "storj.io/uplink" @@ -57,6 +62,7 @@ func Run(t *testing.T, config Config, test func(t *testing.T, ctx *testcontext.C ctx := testcontext.NewWithContextAndTimeout(parent, t, timeout) defer ctx.Cleanup() + planetConfig.applicationName = "testplanet" + pgutil.CreateRandomTestingSchemaName(6) planet, err := NewCustom(ctx, log, planetConfig, satelliteDB) if err != nil { t.Fatalf("%+v", err) @@ -66,7 +72,33 @@ func Run(t *testing.T, config Config, test func(t *testing.T, ctx *testcontext.C planet.Start(ctx) provisionUplinks(ctx, t, planet) + var rawDB tagsql.DB + var queriesBefore []string + if len(planet.Satellites) > 0 && satelliteDB.Name == "Cockroach" { + db := planet.Satellites[0].DB + // not perfect but didn't find better way to do this + rawDB = db.(interface{ DebugGetDBHandle() tagsql.DB }).DebugGetDBHandle() + + var err error + queriesBefore, err = satellitedbtest.FullTableScanQueries(ctx, rawDB, dbutil.Cockroach, planetConfig.applicationName) + if err != nil { + t.Fatalf("%+v", err) + } + } + test(t, ctx, planet) + + if rawDB != nil { + queriesAfter, err := satellitedbtest.FullTableScanQueries(context2.WithoutCancellation(ctx), rawDB, dbutil.Cockroach, planetConfig.applicationName) + if err != nil { + t.Fatalf("%+v", err) + } + + diff := cmp.Diff(queriesBefore, queriesAfter) + if diff != "" { + log.Sugar().Warnf("FULL TABLE SCAN DETECTED\n%s", diff) + } + } }) }) } @@ -106,6 +138,7 @@ func Bench(b *testing.B, config Config, bench func(b *testing.B, ctx *testcontex ctx := testcontext.NewWithContextAndTimeout(parent, b, timeout) defer ctx.Cleanup() + planetConfig.applicationName = "testplanet-bench" planet, err := NewCustom(ctx, log, planetConfig, satelliteDB) if err != nil { b.Fatalf("%+v", err) diff --git a/private/testplanet/satellite.go b/private/testplanet/satellite.go index 2fdb0319d..afe6f9f8a 100644 --- a/private/testplanet/satellite.go +++ b/private/testplanet/satellite.go @@ -356,7 +356,7 @@ func (planet *Planet) newSatellites(ctx context.Context, count int, databases sa var err error pprof.Do(ctx, pprof.Labels("peer", prefix), func(ctx context.Context) { - system, err = planet.newSatellite(ctx, prefix, index, log, databases) + system, err = planet.newSatellite(ctx, prefix, index, log, databases, planet.config.applicationName) }) if err != nil { return nil, errs.Wrap(err) @@ -370,7 +370,7 @@ func (planet *Planet) newSatellites(ctx context.Context, count int, databases sa return satellites, nil } -func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int, log *zap.Logger, databases satellitedbtest.SatelliteDatabases) (_ *Satellite, err error) { +func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int, log *zap.Logger, databases satellitedbtest.SatelliteDatabases, applicationName string) (_ *Satellite, err error) { defer mon.Task()(&ctx)(&err) storageDir := filepath.Join(planet.directory, prefix) @@ -383,7 +383,7 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int return nil, errs.Wrap(err) } - db, err := satellitedbtest.CreateMasterDB(ctx, log.Named("db"), planet.config.Name, "S", index, databases.MasterDB, "satellite-testplanet-test") + db, err := satellitedbtest.CreateMasterDB(ctx, log.Named("db"), planet.config.Name, "S", index, databases.MasterDB, applicationName) if err != nil { return nil, errs.Wrap(err) } diff --git a/satellite/satellitedb/satellitedbtest/run.go b/satellite/satellitedb/satellitedbtest/run.go index e7c419c96..9c7641fb1 100644 --- a/satellite/satellitedb/satellitedbtest/run.go +++ b/satellite/satellitedb/satellitedbtest/run.go @@ -232,7 +232,7 @@ func Run(t *testing.T, test func(ctx *testcontext.Context, t *testing.T, db sate var fullScansBefore []string tempMasterDB, ok := db.(*tempMasterDB) if ok { - fullScansBefore, err = fullTableScanQueries(ctx, tempMasterDB.tempDB.DB, tempMasterDB.tempDB.Implementation, applicationName) + fullScansBefore, err = FullTableScanQueries(ctx, tempMasterDB.tempDB.DB, tempMasterDB.tempDB.Implementation, applicationName) if err != nil { t.Fatal(err) } @@ -241,7 +241,7 @@ func Run(t *testing.T, test func(ctx *testcontext.Context, t *testing.T, db sate test(ctx, t, db) if ok { - fullScansAfter, err := fullTableScanQueries(ctx, tempMasterDB.tempDB.DB, tempMasterDB.tempDB.Implementation, applicationName) + fullScansAfter, err := FullTableScanQueries(ctx, tempMasterDB.tempDB.DB, tempMasterDB.tempDB.Implementation, applicationName) if err != nil { t.Fatal(err) } @@ -290,7 +290,8 @@ func Bench(b *testing.B, bench func(b *testing.B, db satellite.DB)) { } } -func fullTableScanQueries(ctx context.Context, db tagsql.DB, implementation dbutil.Implementation, applicationName string) (queries []string, err error) { +// FullTableScanQueries is a helper method to list all queries which performed full table scan recently. It works only for cockroach db. +func FullTableScanQueries(ctx context.Context, db tagsql.DB, implementation dbutil.Implementation, applicationName string) (queries []string, err error) { if implementation.String() != "cockroach" { return nil, nil }