JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

private/testplanet: enable full table scan detection

Browse Source 
Testplanet tests will print into logs (WARN) if full table scan will
be detected. Test won't be failed automatically. That's because
currently we have multiple queries which are doing full table scan and it's not trivial to change.

https://github.com/storj/storj/issues/5471

Change-Id: Ia2fcbfb9102424d58f95e00071329454a8c1066e
This commit is contained in:
Michal Niewrzal 2023-02-02 12:34:28 +01:00 committed by Storj Robot
parent 828654f98b
commit 36cbf4f0b8
4 changed files with 46 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
private/testplanet/planet.go
View File

@ -63,6 +63,8 @@ type Config struct {
Host string
NonParallel bool
Timeout time.Duration
applicationName string
}
// DatabaseConfig defines connection strings for database.
@ -143,6 +145,10 @@ func NewCustom(ctx *testcontext.Context, log *zap.Logger, config Config, satelli
}
}
if config.applicationName == "" {
config.applicationName = "testplanet"
}
planet := &Planet{
ctx: ctx,
log: log,

33
private/testplanet/run.go
View File

@ -8,12 +8,17 @@ import (
"runtime/pprof"
"testing"
"github.com/google/go-cmp/cmp"
"go.uber.org/zap"
"storj.io/common/context2"
"storj.io/common/grant"
"storj.io/common/storj"
"storj.io/common/testcontext"
"storj.io/private/dbutil"
"storj.io/private/dbutil/pgtest"
"storj.io/private/dbutil/pgutil"
"storj.io/private/tagsql"
"storj.io/storj/private/testmonkit"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/uplink"
@ -57,6 +62,7 @@ func Run(t *testing.T, config Config, test func(t *testing.T, ctx *testcontext.C
ctx := testcontext.NewWithContextAndTimeout(parent, t, timeout)
defer ctx.Cleanup()
planetConfig.applicationName = "testplanet" + pgutil.CreateRandomTestingSchemaName(6)
planet, err := NewCustom(ctx, log, planetConfig, satelliteDB)
if err != nil {
t.Fatalf("%+v", err)
@ -66,7 +72,33 @@ func Run(t *testing.T, config Config, test func(t *testing.T, ctx *testcontext.C
planet.Start(ctx)
provisionUplinks(ctx, t, planet)
var rawDB tagsql.DB
var queriesBefore []string
if len(planet.Satellites) > 0 && satelliteDB.Name == "Cockroach" {
db := planet.Satellites[0].DB
// not perfect but didn't find better way to do this
rawDB = db.(interface{ DebugGetDBHandle() tagsql.DB }).DebugGetDBHandle()
var err error
queriesBefore, err = satellitedbtest.FullTableScanQueries(ctx, rawDB, dbutil.Cockroach, planetConfig.applicationName)
if err != nil {
t.Fatalf("%+v", err)
}
}
test(t, ctx, planet)
if rawDB != nil {
queriesAfter, err := satellitedbtest.FullTableScanQueries(context2.WithoutCancellation(ctx), rawDB, dbutil.Cockroach, planetConfig.applicationName)
if err != nil {
t.Fatalf("%+v", err)
}
diff := cmp.Diff(queriesBefore, queriesAfter)
if diff != "" {
log.Sugar().Warnf("FULL TABLE SCAN DETECTED\n%s", diff)
}
}
})
})
}
@ -106,6 +138,7 @@ func Bench(b *testing.B, config Config, bench func(b *testing.B, ctx *testcontex
ctx := testcontext.NewWithContextAndTimeout(parent, b, timeout)
defer ctx.Cleanup()
planetConfig.applicationName = "testplanet-bench"
planet, err := NewCustom(ctx, log, planetConfig, satelliteDB)
if err != nil {
b.Fatalf("%+v", err)

6
private/testplanet/satellite.go
View File

@ -356,7 +356,7 @@ func (planet *Planet) newSatellites(ctx context.Context, count int, databases sa
var err error
pprof.Do(ctx, pprof.Labels("peer", prefix), func(ctx context.Context) {
system, err = planet.newSatellite(ctx, prefix, index, log, databases)
system, err = planet.newSatellite(ctx, prefix, index, log, databases, planet.config.applicationName)
})
if err != nil {
return nil, errs.Wrap(err)
@ -370,7 +370,7 @@ func (planet *Planet) newSatellites(ctx context.Context, count int, databases sa
return satellites, nil
}
func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int, log *zap.Logger, databases satellitedbtest.SatelliteDatabases) (_ *Satellite, err error) {
func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int, log *zap.Logger, databases satellitedbtest.SatelliteDatabases, applicationName string) (_ *Satellite, err error) {
defer mon.Task()(&ctx)(&err)
storageDir := filepath.Join(planet.directory, prefix)
@ -383,7 +383,7 @@ func (planet *Planet) newSatellite(ctx context.Context, prefix string, index int
return nil, errs.Wrap(err)
}
db, err := satellitedbtest.CreateMasterDB(ctx, log.Named("db"), planet.config.Name, "S", index, databases.MasterDB, "satellite-testplanet-test")
db, err := satellitedbtest.CreateMasterDB(ctx, log.Named("db"), planet.config.Name, "S", index, databases.MasterDB, applicationName)
if err != nil {
return nil, errs.Wrap(err)
}

7
satellite/satellitedb/satellitedbtest/run.go
View File

@ -232,7 +232,7 @@ func Run(t *testing.T, test func(ctx *testcontext.Context, t *testing.T, db sate
var fullScansBefore []string
tempMasterDB, ok := db.(*tempMasterDB)
if ok {
fullScansBefore, err = fullTableScanQueries(ctx, tempMasterDB.tempDB.DB, tempMasterDB.tempDB.Implementation, applicationName)
fullScansBefore, err = FullTableScanQueries(ctx, tempMasterDB.tempDB.DB, tempMasterDB.tempDB.Implementation, applicationName)
if err != nil {
t.Fatal(err)
}
@ -241,7 +241,7 @@ func Run(t *testing.T, test func(ctx *testcontext.Context, t *testing.T, db sate
test(ctx, t, db)
if ok {
fullScansAfter, err := fullTableScanQueries(ctx, tempMasterDB.tempDB.DB, tempMasterDB.tempDB.Implementation, applicationName)
fullScansAfter, err := FullTableScanQueries(ctx, tempMasterDB.tempDB.DB, tempMasterDB.tempDB.Implementation, applicationName)
if err != nil {
t.Fatal(err)
}
@ -290,7 +290,8 @@ func Bench(b *testing.B, bench func(b *testing.B, db satellite.DB)) {
}
}
func fullTableScanQueries(ctx context.Context, db tagsql.DB, implementation dbutil.Implementation, applicationName string) (queries []string, err error) {
// FullTableScanQueries is a helper method to list all queries which performed full table scan recently. It works only for cockroach db.
func FullTableScanQueries(ctx context.Context, db tagsql.DB, implementation dbutil.Implementation, applicationName string) (queries []string, err error) {
if implementation.String() != "cockroach" {
return nil, nil
}