JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metabase/metabasetest: detect full scan table queries

Browse Source 
This is automated test around metabase tests. It's detecting queries
which were performing full table scan during test execution.

After merging this and checking that this is not problematic in any way
we will enable this also for testplanet tests.

One query was adjusted to avoid full table scan and pass new check.

https://github.com/storj/storj/issues/5471

Change-Id: I2d176f0c25deda801e8a731b75954f83d18cc1ce
This commit is contained in:
Michal Niewrzal 2023-01-11 08:30:14 +01:00 committed by Storj Robot
parent bf5b378836
commit 8850fde9f5
3 changed files with 74 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
satellite/metabase/list_segments.go
View File

@ -107,11 +107,12 @@ func (db *DB) ListSegments(ctx context.Context, opts ListSegments) (result ListS
SELECT
root_piece_id,
remote_alias_pieces
FROM segments
FROM segments as segments
LEFT JOIN segment_copies as copies
ON copies.ancestor_stream_id = segments.stream_id
WHERE
stream_id = (SELECT ancestor_stream_id FROM segment_copies WHERE stream_id = $1)
AND position IN (SELECT position FROM UNNEST($2::INT8[]) as position)
ORDER BY stream_id, position ASC
copies.stream_id = $1 AND segments.position IN (SELECT position FROM UNNEST($2::INT8[]) as position)
ORDER BY segments.stream_id, segments.position ASC
`, opts.StreamID, pgutil.Int8Array(copiesPositions)))(func(rows tagsql.Rows) error {
for rows.Next() {

61
satellite/metabase/metabasetest/run.go
View File

@ -5,14 +5,18 @@ package metabasetest
import (
"context"
"strings"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/spf13/pflag"
"github.com/zeebo/errs"
"go.uber.org/zap/zaptest"
"storj.io/common/memory"
"storj.io/common/testcontext"
"storj.io/private/cfgstruct"
"storj.io/private/dbutil/pgutil"
"storj.io/storj/satellite/metabase"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
@ -35,6 +39,8 @@ func RunWithConfigAndMigration(t *testing.T, config metabase.Config, fn func(ctx
ctx := testcontext.New(t)
defer ctx.Cleanup()
// generate unique application name to filter out full table scan queries from other tests executions
config.ApplicationName += pgutil.CreateRandomTestingSchemaName(6)
db, err := satellitedbtest.CreateMetabaseDB(ctx, zaptest.NewLogger(t), t.Name(), "M", 0, dbinfo.MetabaseDB, config)
if err != nil {
t.Fatal(err)
@ -49,7 +55,22 @@ func RunWithConfigAndMigration(t *testing.T, config metabase.Config, fn func(ctx
t.Fatal(err)
}
fullScansBefore, err := fullTableScanQueries(ctx, db, config.ApplicationName)
if err != nil {
t.Fatal(err)
}
fn(ctx, t, db)
fullScansAfter, err := fullTableScanQueries(ctx, db, config.ApplicationName)
if err != nil {
t.Fatal(err)
}
diff := cmp.Diff(fullScansBefore, fullScansAfter)
if diff != "" {
t.Fatal(diff)
}
})
}
}
@ -62,7 +83,7 @@ func Run(t *testing.T, fn func(ctx *testcontext.Context, t *testing.T, db *metab
)
RunWithConfig(t, metabase.Config{
ApplicationName: "satellite-test",
ApplicationName: "satellite-metabase-test",
MinPartSize: config.MinPartSize,
MaxNumberOfParts: config.MaxNumberOfParts,
ServerSideCopy: config.ServerSideCopy,
@ -101,3 +122,41 @@ func Bench(b *testing.B, fn func(ctx *testcontext.Context, b *testing.B, db *met
})
}
}
func fullTableScanQueries(ctx context.Context, db *metabase.DB, applicationName string) (_ map[string]int, err error) {
if db.Implementation().String() != "cockroach" {
return nil, nil
}
rows, err := db.UnderlyingTagSQL().QueryContext(ctx,
"SELECT key, count FROM crdb_internal.node_statement_statistics WHERE full_scan = TRUE AND application_name = $1 ORDER BY count DESC",
applicationName,
)
if err != nil {
return nil, err
}
defer func() {
err = errs.Combine(err, rows.Close())
}()
result := map[string]int{}
for rows.Next() {
var query string
var count int
err := rows.Scan(&query, &count)
if err != nil {
return nil, err
}
switch {
case strings.Contains(query, "WITH testing AS (SELECT _)"):
continue
case !strings.Contains(strings.ToUpper(query), "WHERE"): // find smarter way to ignore known full table scan queries
continue
}
result[query] += count
}
return result, rows.Err()
}

14
satellite/metabase/raw.go
View File

@ -107,11 +107,12 @@ func (db *DB) TestingGetState(ctx context.Context) (_ *RawState, err error) {
// TestingDeleteAll deletes all objects and segments from the database.
func (db *DB) TestingDeleteAll(ctx context.Context) (err error) {
_, err = db.db.ExecContext(ctx, `
DELETE FROM objects;
DELETE FROM segments;
DELETE FROM segment_copies;
DELETE FROM node_aliases;
SELECT setval('node_alias_seq', 1, false);
WITH testing AS (SELECT 1) DELETE FROM objects;
WITH testing AS (SELECT 1) DELETE FROM segments;
WITH testing AS (SELECT 1) DELETE FROM segment_copies;
WITH testing AS (SELECT 1) DELETE FROM node_aliases;
WITH testing AS (SELECT 1) SELECT setval('node_alias_seq', 1, false);
`)
db.aliasCache = NewNodeAliasCache(db)
return Error.Wrap(err)
@ -122,6 +123,7 @@ func (db *DB) testingGetAllObjects(ctx context.Context) (_ []RawObject, err erro
objs := []RawObject{}
rows, err := db.db.QueryContext(ctx, `
WITH testing AS (SELECT 1)
SELECT
project_id, bucket_name, object_key, version, stream_id,
created_at, expires_at,
@ -183,6 +185,7 @@ func (db *DB) testingGetAllSegments(ctx context.Context) (_ []RawSegment, err er
segs := []RawSegment{}
rows, err := db.db.QueryContext(ctx, `
WITH testing AS (SELECT 1)
SELECT
stream_id, position,
created_at, repaired_at, expires_at,
@ -252,6 +255,7 @@ func (db *DB) testingGetAllCopies(ctx context.Context) (_ []RawCopy, err error)
copies := []RawCopy{}
rows, err := db.db.QueryContext(ctx, `
WITH testing AS (SELECT 1)
SELECT
stream_id, ancestor_stream_id
FROM segment_copies