storj/storage/postgreskv/client_test.go

150 lines
3.6 KiB
Go
Raw Normal View History

2019-01-24 20:15:10 +00:00
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package postgreskv
import (
"context"
"strings"
"testing"
"github.com/lib/pq"
"github.com/zeebo/errs"
"storj.io/common/testcontext"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/txutil"
"storj.io/storj/private/tagsql"
"storj.io/storj/storage"
"storj.io/storj/storage/testsuite"
)
func newTestPostgres(t testing.TB) (store *Client, cleanup func()) {
2019-04-26 14:39:11 +01:00
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
2019-04-26 14:39:11 +01:00
pgdb, err := New(*pgtest.ConnStr)
if err != nil {
t.Fatalf("init: %v", err)
}
return pgdb, func() {
if err := pgdb.Close(); err != nil {
t.Fatalf("failed to close db: %v", err)
}
}
}
func TestSuite(t *testing.T) {
store, cleanup := newTestPostgres(t)
defer cleanup()
Public Jenkins (#1779) * initial test * add parenthesis * remove pipeline * add few todos * use docker image for environment * use pipeline * fix * add missing steps * invoke with bash * disable protoc * try using golang image * try as root * Disable install-awscli.sh temporarily * Debugging * debugging part 2 * Set absolute path for debugging * Remove absolute path * Dont run as root * Install unzip * Dont forget to apt-get update * Put into folder that is in PATH * disable IPv6 Test * add verbose info and check protobuf * make integration non-parallel * remove -v and make checkout part of build * make a single block for linting * fix echo * update * try using things directly * try add xunit output * fix name * don't print empty lines * skip testsuites without any tests * remove coverage, because it's not showing the right thing * try using dockerfile * fix deb source * fix typos * setup postgres * use the right flag * try using postgresdb * expose different port * remove port mapping * start postgres * export * use env block * try using different host for integration tests * eat standard ports * try building images and binaries * remove if statement * add steps * do before verification * add go get goversioninfo * make separate jenkinsfile * add check * don't add empty packages * disable logging to reduce output size * add timeout * add comment about mfridman * Revert Absolute Path * Add aws to PATH * PATH Changes * Docker Env Fixes * PATH Simplification * Debugging the PATH * Debug Logs * Debugging * Update PATH Handling * Rename * revert changes to Jenkinsfile
2019-04-22 14:45:53 +01:00
// zap := zaptest.NewLogger(t)
// loggedStore := storelogger.New(zap, store)
store.SetLookupLimit(500)
Public Jenkins (#1779) * initial test * add parenthesis * remove pipeline * add few todos * use docker image for environment * use pipeline * fix * add missing steps * invoke with bash * disable protoc * try using golang image * try as root * Disable install-awscli.sh temporarily * Debugging * debugging part 2 * Set absolute path for debugging * Remove absolute path * Dont run as root * Install unzip * Dont forget to apt-get update * Put into folder that is in PATH * disable IPv6 Test * add verbose info and check protobuf * make integration non-parallel * remove -v and make checkout part of build * make a single block for linting * fix echo * update * try using things directly * try add xunit output * fix name * don't print empty lines * skip testsuites without any tests * remove coverage, because it's not showing the right thing * try using dockerfile * fix deb source * fix typos * setup postgres * use the right flag * try using postgresdb * expose different port * remove port mapping * start postgres * export * use env block * try using different host for integration tests * eat standard ports * try building images and binaries * remove if statement * add steps * do before verification * add go get goversioninfo * make separate jenkinsfile * add check * don't add empty packages * disable logging to reduce output size * add timeout * add comment about mfridman * Revert Absolute Path * Add aws to PATH * PATH Changes * Docker Env Fixes * PATH Simplification * Debugging the PATH * Debug Logs * Debugging * Update PATH Handling * Rename * revert changes to Jenkinsfile
2019-04-22 14:45:53 +01:00
testsuite.RunTests(t, store)
}
func TestThatMigrationActuallyHappened(t *testing.T) {
2020-01-16 23:48:59 +00:00
t.Skip()
ctx := testcontext.New(t)
defer ctx.Cleanup()
store, cleanup := newTestPostgres(t)
defer cleanup()
rows, err := store.db.Query(ctx, `
SELECT prosrc
FROM pg_catalog.pg_proc p,
pg_catalog.pg_namespace n
WHERE p.pronamespace = n.oid
AND p.proname = 'list_directory'
AND n.nspname = ANY(current_schemas(true))
AND p.pronargs = 4
`)
if err != nil {
t.Fatalf("failed to get list_directory source: %v", err)
}
defer func() {
if err := rows.Close(); err != nil {
t.Fatalf("failed to close rows: %v", err)
}
}()
numFound := 0
for rows.Next() {
numFound++
if numFound > 1 {
t.Fatal("there are multiple eligible list_directory() functions??")
}
var source string
if err := rows.Scan(&source); err != nil {
t.Fatalf("failed to read list_directory source: %v", err)
}
if strings.Contains(source, "distinct_prefix (truncatedpath)") {
t.Fatal("list_directory() function in pg appears to be the oldnbusted one")
}
}
}
func BenchmarkSuite(b *testing.B) {
store, cleanup := newTestPostgres(b)
defer cleanup()
testsuite.RunBenchmarks(b, store)
}
func bulkImport(ctx context.Context, db tagsql.DB, iter storage.Iterator) error {
return txutil.WithTx(ctx, db, nil, func(ctx context.Context, txn tagsql.Tx) (err error) {
stmt, err := txn.Prepare(ctx, pq.CopyIn("pathdata", "fullpath", "metadata"))
if err != nil {
return errs.New("Failed to initialize COPY FROM: %v", err)
}
defer func() {
err2 := stmt.Close()
if err2 != nil {
err = errs.Combine(err, errs.New("Failed to close COPY FROM statement: %v", err2))
}
}()
var item storage.ListItem
for iter.Next(ctx, &item) {
if _, err := stmt.Exec(ctx, []byte(item.Key), []byte(item.Value)); err != nil {
return err
}
}
if _, err = stmt.Exec(ctx); err != nil {
return errs.New("Failed to complete COPY FROM: %v", err)
}
return nil
})
}
func bulkDeleteAll(ctx context.Context, db tagsql.DB) error {
_, err := db.Exec(ctx, "TRUNCATE pathdata")
if err != nil {
return errs.New("Failed to TRUNCATE pathdata table: %v", err)
}
return nil
}
type pgLongBenchmarkStore struct {
*Client
}
func (store *pgLongBenchmarkStore) BulkImport(ctx context.Context, iter storage.Iterator) error {
2020-01-16 23:48:59 +00:00
return bulkImport(ctx, store.db, iter)
}
func (store *pgLongBenchmarkStore) BulkDeleteAll(ctx context.Context) error {
return bulkDeleteAll(ctx, store.db)
}
func BenchmarkSuiteLong(b *testing.B) {
store, cleanup := newTestPostgres(b)
defer cleanup()
testsuite.BenchmarkPathOperationsInLargeDb(b, &pgLongBenchmarkStore{store})
}