JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

private,satellite: unite all the "temp db schema" things

Browse Source 
first, so that they all work the same way, because it's getting
complicated, and second, so that we can do the appropriate thing
instead of CREATE SCHEMA for cockroachdb.

Change-Id: I27fbaeeb6223a3e06d97bcf692a2d014b31465f7
This commit is contained in:
paul cannon 2019-12-03 21:36:21 -06:00 committed by Jeff Wendling
parent 97fa000ca9
commit 378b863b2b
26 changed files with 542 additions and 434 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

78
private/dbutil/cockroachutil/db.go Normal file
View File

@ -0,0 +1,78 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cockroachutil
import (
"database/sql"
"net/url"
"strings"
"github.com/lib/pq"
"github.com/zeebo/errs"
"gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/private/dbutil"
)
var mon = monkit.Package()
// OpenUnique opens a temporary unique CockroachDB database that will be cleaned up when closed.
// It is expected that this should normally be used by way of
// "storj.io/storj/private/dbutil/tempdb".OpenUnique() instead of calling it directly.
func OpenUnique(connStr string, schemaName string) (db *dbutil.TempDatabase, err error) {
if !strings.HasPrefix(connStr, "cockroach://") {
return nil, errs.New("expected a cockroachDB URI, but got %q", connStr)
}
connStr = "postgres://" + connStr[12:]
masterDB, err := sql.Open("postgres", connStr)
if err != nil {
return nil, errs.Wrap(err)
}
defer func() {
err = errs.Combine(err, masterDB.Close())
}()
err = masterDB.Ping()
if err != nil {
return nil, errs.New("Could not open masterDB at conn %q: %v", connStr, err)
}
_, err = masterDB.Exec("CREATE DATABASE " + pq.QuoteIdentifier(schemaName))
if err != nil {
return nil, errs.Wrap(err)
}
cleanup := func(cleanupDB *sql.DB) error {
_, err := cleanupDB.Exec("DROP DATABASE " + pq.QuoteIdentifier(schemaName))
return errs.Wrap(err)
}
modifiedConnStr, err := changeDBTargetInConnStr(connStr, schemaName)
if err != nil {
return nil, errs.Combine(err, cleanup(masterDB))
}
sqlDB, err := sql.Open("postgres", modifiedConnStr)
if err != nil {
return nil, errs.Combine(errs.Wrap(err), cleanup(masterDB))
}
dbutil.Configure(sqlDB, mon)
return &dbutil.TempDatabase{
DB: sqlDB,
ConnStr: modifiedConnStr,
Schema: schemaName,
Driver: "postgres",
Implementation: dbutil.Cockroach,
Cleanup: cleanup,
}, nil
}
func changeDBTargetInConnStr(connStr string, newDBName string) (string, error) {
connURL, err := url.Parse(connStr)
if err != nil {
return "", errs.Wrap(err)
}
connURL.Path = newDBName
return connURL.String(), nil
}

63
private/dbutil/cockroachutil/openunique_test.go Normal file
View File

@ -0,0 +1,63 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package cockroachutil_test
import (
"database/sql"
"strings"
"testing"
"github.com/stretchr/testify/require"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/tempdb"
"storj.io/storj/private/testcontext"
)
func TestTempCockroachDB(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
if *pgtest.CrdbConnStr == "" {
t.Skip("CockroachDB flag missing")
}
prefix := "name#spaced/Test/DB"
testDB, err := tempdb.OpenUnique(*pgtest.CrdbConnStr, prefix)
require.NoError(t, err)
// save these so we can close testDB down below and then still try connecting to the same place
// (without requiring that the values stay intact in the testDB struct when we close it)
driverCopy := testDB.Driver
connStrCopy := testDB.ConnStr
// assert new test db exists and can be connected to again
otherConn, err := sql.Open(driverCopy, connStrCopy)
require.NoError(t, err)
defer ctx.Check(otherConn.Close)
// verify the name matches expectation
var dbName string
row := otherConn.QueryRow(`SELECT current_database()`)
err = row.Scan(&dbName)
require.NoError(t, err)
require.Truef(t, strings.HasPrefix(dbName, prefix), "Expected prefix of %q for current db name, but found %q", prefix, dbName)
// verify there is a db with such a name
var count int
row = otherConn.QueryRow(`SELECT COUNT(*) FROM pg_database WHERE datname = current_database()`)
err = row.Scan(&count)
require.NoError(t, err)
require.Equalf(t, 1, count, "Expected 1 DB with matching name, but counted %d", count)
// close testDB but leave otherConn open
err = testDB.Close()
require.NoError(t, err)
// assert new test db was deleted (we expect this connection to keep working, even though its
// database was deleted out from under it!)
row = otherConn.QueryRow(`SELECT COUNT(*) FROM pg_database WHERE datname = current_database()`)
err = row.Scan(&count)
require.NoError(t, err)
require.Equalf(t, 0, count, "Expected 0 DB with matching name, but counted %d (deletion failure?)", count)
}

89
private/dbutil/pgutil/db.go
View File

@ -15,79 +15,50 @@ import (
"storj.io/storj/private/dbutil/dbschema"
)
// DB is postgres database with schema
type DB struct {
*sql.DB
Schema string
}
var (
mon = monkit.Package()
)
// Open opens a postgres database with a schema
func Open(connstr string, schemaPrefix string) (*DB, error) {
schemaName := schemaPrefix + "-" + CreateRandomTestingSchemaName(8)
db, err := sql.Open("postgres", ConnstrWithSchema(connstr, schemaName))
if err != nil {
return nil, err
// OpenUnique opens a postgres database with a temporary unique schema, which will be cleaned up
// when closed. It is expected that this should normally be used by way of
// "storj.io/storj/private/dbutil/tempdb".OpenUnique() instead of calling it directly.
func OpenUnique(connstr string, schemaPrefix string) (*dbutil.TempDatabase, error) {
// sanity check, because you get an unhelpful error message when this happens
if strings.HasPrefix(connstr, "cockroach://") {
return nil, errs.New("can't connect to cockroach using pgutil.OpenUnique()! connstr=%q. try tempdb.OpenUnique() instead?", connstr)
}
dbutil.Configure(db, mon)
schemaName := schemaPrefix + "-" + CreateRandomTestingSchemaName(8)
connStrWithSchema := ConnstrWithSchema(connstr, schemaName)
db, err := sql.Open("postgres", connStrWithSchema)
if err == nil {
// check that connection actually worked before trying CreateSchema, to make
// troubleshooting (lots) easier
err = db.Ping()
}
if err != nil {
return nil, errs.New("failed to connect to %q with driver postgres: %v", connStrWithSchema, err)
}
err = CreateSchema(db, schemaName)
if err != nil {
return nil, errs.Combine(err, db.Close())
}
return &DB{db, schemaName}, err
cleanup := func(cleanupDB *sql.DB) error {
return DropSchema(cleanupDB, schemaName)
}
// Close closes the database and deletes the schema.
func (db *DB) Close() error {
return errs.Combine(
DropSchema(db.DB, db.Schema),
db.DB.Close(),
)
}
// LoadSchemaFromSQL inserts script into connstr and loads schema.
func LoadSchemaFromSQL(connstr, script string) (_ *dbschema.Schema, err error) {
db, err := Open(connstr, "load-schema")
if err != nil {
return nil, err
}
defer func() { err = errs.Combine(err, db.Close()) }()
_, err = db.Exec(script)
if err != nil {
return nil, err
}
return QuerySchema(db)
}
// LoadSnapshotFromSQL inserts script into connstr and loads schema.
func LoadSnapshotFromSQL(connstr, script string) (_ *dbschema.Snapshot, err error) {
db, err := Open(connstr, "load-schema")
if err != nil {
return nil, err
}
defer func() { err = errs.Combine(err, db.Close()) }()
_, err = db.Exec(script)
if err != nil {
return nil, err
}
snapshot, err := QuerySnapshot(db)
if err != nil {
return nil, err
}
snapshot.Script = script
return snapshot, nil
dbutil.Configure(db, mon)
return &dbutil.TempDatabase{
DB: db,
ConnStr: connStrWithSchema,
Schema: schemaName,
Driver: "postgres",
Implementation: dbutil.Postgres,
Cleanup: cleanup,
}, nil
}
// QuerySnapshot loads snapshot from database

58
private/dbutil/pgutil/openunique_test.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package pgutil_test
import (
"database/sql"
"strings"
"testing"
"github.com/stretchr/testify/require"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/tempdb"
"storj.io/storj/private/testcontext"
)
func TestTempPostgresDB(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
if *pgtest.ConnStr == "" {
t.Skip("PostgreSQL flag missing")
}
prefix := "name#spaced/Test/DB"
testDB, err := tempdb.OpenUnique(*pgtest.ConnStr, prefix)
require.NoError(t, err)
// assert new test db exists and can be connected to again
otherConn, err := sql.Open(testDB.Driver, testDB.ConnStr)
require.NoError(t, err)
defer ctx.Check(otherConn.Close)
// verify the name matches expectation
var name *string
row := otherConn.QueryRow(`SELECT current_schema()`)
err = row.Scan(&name)
require.NoErrorf(t, err, "connStr=%q", testDB.ConnStr)
require.NotNilf(t, name, "PG has no current_schema, which means the one we asked for doesn't exist. connStr=%q", testDB.ConnStr)
require.Truef(t, strings.HasPrefix(*name, prefix), "Expected prefix of %q for current db name, but found %q", prefix, name)
// verify there is an entry in pg_namespace with such a name
var count int
row = otherConn.QueryRow(`SELECT COUNT(*) FROM pg_namespace WHERE nspname = current_schema`)
err = row.Scan(&count)
require.NoError(t, err)
require.Equalf(t, 1, count, "Expected 1 schema with matching name, but counted %d", count)
// close testDB but leave otherConn open
err = testDB.Close()
require.NoError(t, err)
// assert new test schema was deleted
row = otherConn.QueryRow(`SELECT COUNT(*) FROM pg_namespace WHERE nspname = current_schema`)
err = row.Scan(&count)
require.NoError(t, err)
require.Equalf(t, 0, count, "Expected 0 schemas with matching name, but counted %d (deletion failure?)", count)
}

9
private/dbutil/pgutil/query.go
View File

@ -6,6 +6,7 @@ package pgutil
import (
"fmt"
"regexp"
"strings"
"github.com/lib/pq"
"github.com/zeebo/errs"
@ -139,3 +140,11 @@ var rxPostgresForeignKey = regexp.MustCompile(
`(?:\s*ON UPDATE (CASCADE|RESTRICT|SET NULL|SET DEFAULT|NO ACTION))?` +
`(?:\s*ON DELETE (CASCADE|RESTRICT|SET NULL|SET DEFAULT|NO ACTION))?$`,
)
// UnquoteIdentifier is the analog of pq.QuoteIdentifier.
func UnquoteIdentifier(quotedIdent string) string {
if len(quotedIdent) >= 2 && quotedIdent[0] == '"' && quotedIdent[len(quotedIdent)-1] == '"' {
quotedIdent = strings.ReplaceAll(quotedIdent[1:len(quotedIdent)-1], "\"\"", "\"")
}
return quotedIdent
}

33
private/dbutil/pgutil/query_test.go
View File

@ -13,6 +13,7 @@ import (
"storj.io/storj/private/dbutil/dbschema"
"storj.io/storj/private/dbutil/pgutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/tempdb"
"storj.io/storj/private/testcontext"
)
@ -21,32 +22,44 @@ const (
DefaultPostgresConn = "postgres://storj:storj-pass@test-postgres/teststorj?sslmode=disable"
)
func TestQuery(t *testing.T) {
func TestQueryPostgres(t *testing.T) {
if *pgtest.ConnStr == "" {
t.Skip("Postgres flag missing, example: -postgres-test-db=" + DefaultPostgresConn)
}
doQueryTest(t, *pgtest.ConnStr)
}
func TestQueryCockroach(t *testing.T) {
if *pgtest.CrdbConnStr == "" {
t.Skip("Cockroach flag missing, example: -cockroach-test-db=" + pgtest.DefaultCrdbConnStr)
}
doQueryTest(t, *pgtest.CrdbConnStr)
}
func doQueryTest(t *testing.T, connStr string) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
db, err := pgutil.Open(*pgtest.ConnStr, "pgutil-query")
db, err := tempdb.OpenUnique(connStr, "pgutil-query")
require.NoError(t, err)
defer ctx.Check(db.Close)
emptySchema, err := pgutil.QuerySchema(db)
assert.NoError(t, err)
require.NoError(t, err)
assert.Equal(t, &dbschema.Schema{}, emptySchema)
_, err = db.Exec(`
CREATE TABLE users (
a integer NOT NULL,
b integer NOT NULL,
a bigint NOT NULL,
b bigint NOT NULL,
c text,
UNIQUE (c),
PRIMARY KEY (a)
);
CREATE TABLE names (
users_a integer REFERENCES users( a ) ON DELETE CASCADE,
users_a bigint REFERENCES users( a ) ON DELETE CASCADE,
a text NOT NULL,
x text,
b text,
@ -58,15 +71,15 @@ func TestQuery(t *testing.T) {
require.NoError(t, err)
schema, err := pgutil.QuerySchema(db)
assert.NoError(t, err)
require.NoError(t, err)
expected := &dbschema.Schema{
Tables: []*dbschema.Table{
{
Name: "users",
Columns: []*dbschema.Column{
{Name: "a", Type: "integer", IsNullable: false, Reference: nil},
{Name: "b", Type: "integer", IsNullable: false, Reference: nil},
{Name: "a", Type: "bigint", IsNullable: false, Reference: nil},
{Name: "b", Type: "bigint", IsNullable: false, Reference: nil},
{Name: "c", Type: "text", IsNullable: true, Reference: nil},
},
PrimaryKey: []string{"a"},
@ -77,7 +90,7 @@ func TestQuery(t *testing.T) {
{
Name: "names",
Columns: []*dbschema.Column{
{Name: "users_a", Type: "integer", IsNullable: true,
{Name: "users_a", Type: "bigint", IsNullable: true,
Reference: &dbschema.Reference{
Table: "users",
Column: "a",

23
private/dbutil/pgutil/schema.go
View File

@ -9,8 +9,9 @@ import (
"encoding/hex"
"math/rand"
"net/url"
"strconv"
"strings"
"github.com/lib/pq"
)
// CreateRandomTestingSchemaName creates a random schema name string.
@ -24,8 +25,12 @@ func CreateRandomTestingSchemaName(n int) string {
// ConnstrWithSchema adds schema to a connection string
func ConnstrWithSchema(connstr, schema string) string {
schema = strings.ToLower(schema)
return connstr + "&search_path=" + url.QueryEscape(schema)
if strings.Contains(connstr, "?") {
connstr += "&options="
} else {
connstr += "?options="
}
return connstr + url.QueryEscape("--search_path="+pq.QuoteIdentifier(schema))
}
// ParseSchemaFromConnstr returns the name of the schema parsed from the
@ -36,16 +41,24 @@ func ParseSchemaFromConnstr(connstr string) (string, error) {
return "", err
}
queryValues := url.Query()
// this is the Proper™ way to encode search_path in a pq connection string
options := queryValues["options"]
for _, option := range options {
if strings.HasPrefix(option, "--search_path=") {
return UnquoteIdentifier(option[len("--search_path="):]), nil
}
}
// this is another way we've used before; supported brokenly as a kludge in github.com/lib/pq
schema := queryValues["search_path"]
if len(schema) > 0 {
return schema[0], nil
return UnquoteIdentifier(schema[0]), nil
}
return "", nil
}
// QuoteSchema quotes schema name for
func QuoteSchema(schema string) string {
return strconv.QuoteToASCII(schema)
return pq.QuoteIdentifier(schema)
}
// Execer is for executing sql

26
private/dbutil/tempdb/tempdb.go Normal file
View File

@ -0,0 +1,26 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package tempdb
import (
"strings"
"github.com/zeebo/errs"
"storj.io/storj/private/dbutil"
"storj.io/storj/private/dbutil/cockroachutil"
"storj.io/storj/private/dbutil/pgutil"
)
// OpenUnique opens a temporary, uniquely named database (or isolated database schema)
// for scratch work. When closed, this database or schema will be cleaned up and destroyed.
func OpenUnique(connURL string, namePrefix string) (*dbutil.TempDatabase, error) {
if strings.HasPrefix(connURL, "postgres://") || strings.HasPrefix(connURL, "postgresql://") {
return pgutil.OpenUnique(connURL, namePrefix)
}
if strings.HasPrefix(connURL, "cockroach://") {
return cockroachutil.OpenUnique(connURL, namePrefix)
}
return nil, errs.New("OpenUnique does not yet support the db type for %q", connURL)
}

30
private/dbutil/tempdbtype.go Normal file
View File

@ -0,0 +1,30 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package dbutil
import (
"database/sql"
"github.com/zeebo/errs"
)
// TempDatabase is a database (or something that works like an isolated database,
// such as a PostgreSQL schema) with a semi-unique name which will be cleaned up
// when closed. Mainly useful for testing purposes.
type TempDatabase struct {
*sql.DB
ConnStr string
Schema string
Driver string
Implementation Implementation
Cleanup func(*sql.DB) error
}
// Close closes the database and deletes the schema.
func (db *TempDatabase) Close() error {
return errs.Combine(
db.Cleanup(db.DB),
db.DB.Close(),
)
}

41
private/migrate/create_test.go
View File

@ -13,8 +13,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"storj.io/storj/private/dbutil/pgutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/tempdb"
"storj.io/storj/private/migrate"
)
@ -27,52 +27,57 @@ func TestCreate_Sqlite(t *testing.T) {
// should create table
err = migrate.Create("example", &sqliteDB{db, "CREATE TABLE example_table (id text)"})
assert.NoError(t, err)
require.NoError(t, err)
// shouldn't create a new table
err = migrate.Create("example", &sqliteDB{db, "CREATE TABLE example_table (id text)"})
assert.NoError(t, err)
require.NoError(t, err)
// should fail, because schema changed
err = migrate.Create("example", &sqliteDB{db, "CREATE TABLE example_table (id text, version int)"})
assert.Error(t, err)
require.Error(t, err)
// should fail, because of trying to CREATE TABLE with same name
err = migrate.Create("conflict", &sqliteDB{db, "CREATE TABLE example_table (id text, version int)"})
assert.Error(t, err)
require.Error(t, err)
}
func TestCreate_Postgres(t *testing.T) {
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
testCreateGeneric(t, *pgtest.ConnStr)
}
schema := "create-" + pgutil.CreateRandomTestingSchemaName(8)
func TestCreate_Cockroach(t *testing.T) {
if *pgtest.CrdbConnStr == "" {
t.Skip("Cockroach flag missing, example: -cockroach-test-db=" + pgtest.DefaultCrdbConnStr)
}
testCreateGeneric(t, *pgtest.CrdbConnStr)
}
db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*pgtest.ConnStr, schema))
func testCreateGeneric(t *testing.T, connStr string) {
db, err := tempdb.OpenUnique(connStr, "create-")
if err != nil {
t.Fatal(err)
}
defer func() { assert.NoError(t, db.Close()) }()
require.NoError(t, pgutil.CreateSchema(db, schema))
defer func() { assert.NoError(t, pgutil.DropSchema(db, schema)) }()
// should create table
err = migrate.Create("example", &postgresDB{db, "CREATE TABLE example_table (id text)"})
assert.NoError(t, err)
err = migrate.Create("example", &postgresDB{db.DB, "CREATE TABLE example_table (id text)"})
require.NoError(t, err)
// shouldn't create a new table
err = migrate.Create("example", &postgresDB{db, "CREATE TABLE example_table (id text)"})
assert.NoError(t, err)
err = migrate.Create("example", &postgresDB{db.DB, "CREATE TABLE example_table (id text)"})
require.NoError(t, err)
// should fail, because schema changed
err = migrate.Create("example", &postgresDB{db, "CREATE TABLE example_table (id text, version integer)"})
assert.Error(t, err)
err = migrate.Create("example", &postgresDB{db.DB, "CREATE TABLE example_table (id text, version integer)"})
require.Error(t, err)
// should fail, because of trying to CREATE TABLE with same name
err = migrate.Create("conflict", &postgresDB{db, "CREATE TABLE example_table (id text, version integer)"})
assert.Error(t, err)
err = migrate.Create("conflict", &postgresDB{db.DB, "CREATE TABLE example_table (id text, version integer)"})
require.Error(t, err)
}
type sqliteDB struct {

19
private/migrate/versions_test.go
View File

@ -16,8 +16,8 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/private/dbutil/pgutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/tempdb"
"storj.io/storj/private/migrate"
"storj.io/storj/private/testcontext"
)
@ -42,19 +42,24 @@ func TestBasicMigrationPostgres(t *testing.T) {
if *pgtest.ConnStr == "" {
t.Skipf("postgres flag missing, example:\n-postgres-test-db=%s", pgtest.DefaultConnStr)
}
testBasicMigrationGeneric(t, *pgtest.ConnStr)
}
schema := "create-" + pgutil.CreateRandomTestingSchemaName(8)
func TestBasicMigrationCockroach(t *testing.T) {
if *pgtest.CrdbConnStr == "" {
t.Skipf("cockroach flag missing, example:\n-cockroach-test-db=%s", pgtest.DefaultCrdbConnStr)
}
testBasicMigrationGeneric(t, *pgtest.CrdbConnStr)
}
db, err := sql.Open("postgres", pgutil.ConnstrWithSchema(*pgtest.ConnStr, schema))
func testBasicMigrationGeneric(t *testing.T, connStr string) {
db, err := tempdb.OpenUnique(connStr, "create-")
if err != nil {
t.Fatal(err)
}
defer func() { assert.NoError(t, db.Close()) }()
require.NoError(t, pgutil.CreateSchema(db, schema))
defer func() { assert.NoError(t, pgutil.DropSchema(db, schema)) }()
basicMigration(t, db, &postgresDB{DB: db})
basicMigration(t, db.DB, &postgresDB{DB: db.DB})
}
func basicMigration(t *testing.T, db *sql.DB, testDB migrate.DB) {

31
private/testplanet/run.go
View File

@ -6,16 +6,13 @@ package testplanet
import (
"testing"
"github.com/zeebo/errs"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
"storj.io/storj/private/dbutil/pgutil"
"storj.io/storj/private/testcontext"
"storj.io/storj/satellite"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
"storj.io/storj/storage/postgreskv"
)
// Run runs testplanet in multiple configurations.
@ -39,19 +36,7 @@ func Run(t *testing.T, config Config, test func(t *testing.T, ctx *testcontext.C
if satelliteDB.PointerDB.URL != "" {
planetConfig.Reconfigure.NewSatellitePointerDB = func(log *zap.Logger, index int) (metainfo.PointerDB, error) {
schemaSuffix := satellitedbtest.SchemaSuffix()
t.Log("schema-suffix ", schemaSuffix)
schema := satellitedbtest.SchemaName(t.Name(), "P", index, schemaSuffix)
db, err := postgreskv.New(pgutil.ConnstrWithSchema(satelliteDB.PointerDB.URL, schema))
if err != nil {
t.Fatal(err)
}
return &satellitePointerSchema{
Client: db,
schema: schema,
}, nil
return satellitedbtest.CreatePointerDB(t, "P", index, satelliteDB.PointerDB)
}
}
@ -67,17 +52,3 @@ func Run(t *testing.T, config Config, test func(t *testing.T, ctx *testcontext.C
})
}
}
// satellitePointerSchema closes database and drops the associated schema
type satellitePointerSchema struct {
*postgreskv.Client
schema string
}
// Close closes the database and drops the schema.
func (db *satellitePointerSchema) Close() error {
return errs.Combine(
db.Client.DropSchema(db.schema),
db.Client.Close(),
)
}

18
private/testplanet/satellite.go
View File

@ -5,6 +5,7 @@ package testplanet
import (
"context"
"fmt"
"net"
"os"
"path/filepath"
@ -22,7 +23,9 @@ import (
"storj.io/storj/pkg/rpc"
"storj.io/storj/pkg/server"
"storj.io/storj/pkg/storj"
"storj.io/storj/private/dbutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/tempdb"
"storj.io/storj/private/errs2"
"storj.io/storj/private/memory"
"storj.io/storj/private/version"
@ -226,12 +229,19 @@ func (planet *Planet) newSatellites(count int) ([]*SatelliteSystem, error) {
if planet.config.Reconfigure.NewSatelliteDB != nil {
db, err = planet.config.Reconfigure.NewSatelliteDB(log.Named("db"), i)
} else {
schema := satellitedbtest.SchemaName(planet.id, "S", i, "")
// TODO: This is analogous to the way we worked prior to the advent of OpenUnique,
// but it seems wrong. Tests that use planet.Start() instead of testplanet.Run()
// will not get run against both types of DB.
connStr := *pgtest.ConnStr
if *pgtest.CrdbConnStr != "" {
db, err = satellitedbtest.NewCockroach(log.Named("db"), schema)
} else {
db, err = satellitedbtest.NewPostgres(log.Named("db"), schema)
connStr = *pgtest.CrdbConnStr
}
var tempDB *dbutil.TempDatabase
tempDB, err = tempdb.OpenUnique(connStr, fmt.Sprintf("%s.%d", planet.id, i))
if err != nil {
return nil, err
}
db, err = satellitedbtest.CreateMasterDBOnTopOf(log.Named("db"), tempDB)
}
if err != nil {
return nil, err

7
satellite/peer.go
View File

@ -50,11 +50,6 @@ type DB interface {
// Close closes the database
Close() error
// CreateSchema sets the schema
CreateSchema(schema string) error
// DropSchema drops the schema
DropSchema(schema string) error
// PeerIdentities returns a storage for peer identities
PeerIdentities() overlay.PeerIdentities
// OverlayCache returns database for caching overlay information
@ -71,7 +66,7 @@ type DB interface {
Irreparable() irreparable.DB
// Console returns database for satellite console
Console() console.DB
// returns database for marketing admin GUI
// Rewards returns database for marketing admin GUI
Rewards() rewards.DB
// Orders returns database for orders
Orders() orders.DB

10
satellite/satellitedb/database.go
View File

@ -68,20 +68,10 @@ func (db *DB) Close() error {
return db.db.Close()
}
// CreateSchema creates a schema if it doesn't exist.
func (db *DB) CreateSchema(schema string) error {
return pgutil.CreateSchema(db.db, schema)
}
// TestDBAccess for raw database access,
// should not be used outside of migration tests.
func (db *DB) TestDBAccess() *dbx.DB { return db.db }
// DropSchema drops the named schema
func (db *DB) DropSchema(schema string) error {
return pgutil.DropSchema(db.db, schema)
}
// PeerIdentities returns a storage for peer identities
func (db *DB) PeerIdentities() overlay.PeerIdentities {
return &peerIdentities{db: db.db}

2
satellite/satellitedb/migrate.go
View File

@ -27,7 +27,7 @@ func (db *DB) CreateTables() error {
}
if schema != "" {
err = db.CreateSchema(schema)
err = pgutil.CreateSchema(db.db, schema)
if err != nil {
return errs.New("error creating schema: %+v", err)
}

18
satellite/satellitedb/migrate_cockroach_test.go Normal file
View File

@ -0,0 +1,18 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package satellitedb_test
import (
"testing"
"storj.io/storj/private/dbutil/pgutil/pgtest"
)
func TestMigrateCockroach(t *testing.T) {
if *pgtest.CrdbConnStr == "" {
t.Skip("Cockroach flag missing, example: -cockroach-test-db=" + pgtest.DefaultCrdbConnStr)
}
pgMigrateTest(t, *pgtest.CrdbConnStr)
}

66
satellite/satellitedb/migrate_postgres_test.go
View File

@ -12,12 +12,15 @@ import (
"sync"
"testing"
"github.com/lib/pq"
"github.com/stretchr/testify/require"
"github.com/zeebo/errs"
"go.uber.org/zap/zaptest"
"storj.io/storj/private/dbutil/dbschema"
"storj.io/storj/private/dbutil/pgutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/tempdb"
"storj.io/storj/satellite/satellitedb"
)
@ -35,16 +38,19 @@ func loadSnapshots(connstr string) (*dbschema.Snapshots, error) {
versionStr := match[19 : len(match)-4] // hack to avoid trim issues with path differences in windows/linux
version, err := strconv.Atoi(versionStr)
if err != nil {
return nil, err
return nil, errs.New("invalid testdata file %q: %v", match, err)
}
scriptData, err := ioutil.ReadFile(match)
if err != nil {
return nil, err
return nil, errs.New("could not read testdata file for version %d: %v", version, err)
}
snapshot, err := pgutil.LoadSnapshotFromSQL(connstr, string(scriptData))
snapshot, err := loadSnapshotFromSQL(connstr, string(scriptData))
if err != nil {
if pqErr, ok := err.(*pq.Error); ok && pqErr.Detail != "" {
return nil, fmt.Errorf("Version %d error: %v\nDetail: %s\nHint: %s", version, pqErr, pqErr.Detail, pqErr.Hint)
}
return nil, fmt.Errorf("Version %d error: %+v", version, err)
}
snapshot.Version = version
@ -57,6 +63,28 @@ func loadSnapshots(connstr string) (*dbschema.Snapshots, error) {
return snapshots, nil
}
// loadSnapshotFromSQL inserts script into connstr and loads schema.
func loadSnapshotFromSQL(connstr, script string) (_ *dbschema.Snapshot, err error) {
db, err := tempdb.OpenUnique(connstr, "load-schema")
if err != nil {
return nil, err
}
defer func() { err = errs.Combine(err, db.Close()) }()
_, err = db.Exec(script)
if err != nil {
return nil, err
}
snapshot, err := pgutil.QuerySnapshot(db)
if err != nil {
return nil, err
}
snapshot.Script = script
return snapshot, nil
}
const newDataSeparator = `-- NEW DATA --`
func newData(snap *dbschema.Snapshot) string {
@ -79,11 +107,27 @@ var (
// it shouldn't change during the test
func loadDBXSchema(connstr, dbxscript string) (*dbschema.Schema, error) {
dbxschema.Do(func() {
dbxschema.Schema, dbxschema.err = pgutil.LoadSchemaFromSQL(connstr, dbxscript)
dbxschema.Schema, dbxschema.err = loadSchemaFromSQL(connstr, dbxscript)
})
return dbxschema.Schema, dbxschema.err
}
// loadSchemaFromSQL inserts script into connstr and loads schema.
func loadSchemaFromSQL(connstr, script string) (_ *dbschema.Schema, err error) {
db, err := tempdb.OpenUnique(connstr, "load-schema")
if err != nil {
return nil, err
}
defer func() { err = errs.Combine(err, db.Close()) }()
_, err = db.Exec(script)
if err != nil {
return nil, err
}
return pgutil.QuerySchema(db)
}
const (
minBaseVersion = 0
maxBaseVersion = 4
@ -94,7 +138,11 @@ func TestMigratePostgres(t *testing.T) {
t.Skip("Postgres flag missing, example: -postgres-test-db=" + pgtest.DefaultConnStr)
}
snapshots, err := loadSnapshots(*pgtest.ConnStr)
pgMigrateTest(t, *pgtest.ConnStr)
}
func pgMigrateTest(t *testing.T, connStr string) {
snapshots, err := loadSnapshots(connStr)
require.NoError(t, err)
for _, snapshot := range snapshots.List {
@ -107,17 +155,13 @@ func TestMigratePostgres(t *testing.T) {
t.Run(strconv.Itoa(base.Version), func(t *testing.T) {
log := zaptest.NewLogger(t)
schemaName := "migrate/satellite/" + strconv.Itoa(base.Version) + pgutil.CreateRandomTestingSchemaName(8)
connstr := pgutil.ConnstrWithSchema(*pgtest.ConnStr, schemaName)
connstr := pgutil.ConnstrWithSchema(connStr, schemaName)
// create a new satellitedb connection
db, err := satellitedb.New(log, connstr)
require.NoError(t, err)
defer func() { require.NoError(t, db.Close()) }()
// setup our own schema to avoid collisions
require.NoError(t, db.CreateSchema(schemaName))
defer func() { require.NoError(t, db.DropSchema(schemaName)) }()
// we need raw database access unfortunately
rawdb := db.(*satellitedb.DB).TestDBAccess()
@ -171,7 +215,7 @@ func TestMigratePostgres(t *testing.T) {
}
// verify that we also match the dbx version
dbxschema, err := loadDBXSchema(*pgtest.ConnStr, rawdb.Schema())
dbxschema, err := loadDBXSchema(connStr, rawdb.Schema())
require.NoError(t, err)
require.Equal(t, dbxschema, finalSchema, "dbx")

108
satellite/satellitedb/satellitedbtest/cockroachnamespace.go
View File

@ -1,108 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package satellitedbtest
import (
"fmt"
"regexp"
"github.com/lib/pq"
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/private/dbutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/satellite"
"storj.io/storj/satellite/satellitedb"
dbx "storj.io/storj/satellite/satellitedb/dbx"
)
// NewCockroach creates a new satellite.DB that is used for testing. We create a new database with a
// unique name so that there aren't conflicts when we run tests (since we may run the tests in parallel).
// Postgres supports schemas for namespacing, but cockroachdb doesn't, so instead we use a different database for each test.
func NewCockroach(log *zap.Logger, namespacedTestDB string) (satellite.DB, error) {
if err := CockroachDefined(); err != nil {
return nil, err
}
driver, source, _, err := dbutil.SplitConnStr(*pgtest.CrdbConnStr)
if err != nil {
return nil, err
}
db, err := dbx.Open(driver, source)
if err != nil {
return nil, err
}
r := regexp.MustCompile(`\W`)
// this regex removes any non-alphanumeric character from the string
namespacedTestDB = r.ReplaceAllString(namespacedTestDB, "")
_, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s;", pq.QuoteIdentifier(namespacedTestDB)))
if err != nil {
return nil, err
}
// this regex matches substrings like this "/dbName?"
r = regexp.MustCompile("[/][a-zA-Z0-9]+[?]")
if !r.MatchString(source) {
return nil, errs.New("expecting db url format to contain a substring like '/dbName?', but got %s", source)
}
testConnURL := r.ReplaceAllString(*pgtest.CrdbConnStr, "/"+namespacedTestDB+"?")
testDB, err := satellitedb.New(log, testConnURL)
if err != nil {
return nil, err
}
return &namespacedDB{
DB: testDB,
parentRawConn: db,
namespace: namespacedTestDB,
autoDrop: true,
}, nil
}
// CockroachDefined returns an error when no database connection string is provided
func CockroachDefined() error {
if *pgtest.CrdbConnStr == "" {
return errs.New("flag --cockroach-test-db or environment variable STORJ_COCKROACH_TEST not defined for CockroachDB test database")
}
return nil
}
// namespacedDB implements namespacing for new satellite.DB databases when testing
type namespacedDB struct {
satellite.DB
parentRawConn *dbx.DB
namespace string
autoDrop bool
}
// Close closes the namespaced test database. If autoDrop is true,
// then we make a database connection to the parent db and delete the
// namespaced database that was used for testing.
func (db *namespacedDB) Close() error {
err := db.DB.Close()
if err != nil {
return err
}
var dropErr error
if db.autoDrop {
// connect to the parent db and delete the namespaced database used for the test
_, dropErr = db.parentRawConn.Exec(fmt.Sprintf("DROP DATABASE %s;", pq.QuoteIdentifier(db.namespace)))
}
return errs.Combine(dropErr, db.parentRawConn.Close())
}
// CreateTables creates table for the namespaced test database
func (db *namespacedDB) CreateTables() error {
return db.DB.CreateTables()
}
// TestDBAccess for raw database access,
// should not be used outside of migration tests.
func (db *namespacedDB) TestDBAccess() *dbx.DB {
return db.DB.(interface{ TestDBAccess() *dbx.DB }).TestDBAccess()
}

64
satellite/satellitedb/satellitedbtest/cockroachnamespace_test.go
View File

@ -1,64 +0,0 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package satellitedbtest_test
import (
"regexp"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
"storj.io/storj/private/dbutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/testcontext"
dbx "storj.io/storj/satellite/satellitedb/dbx"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
)
func TestNewCockroach(t *testing.T) {
ctx := testcontext.New(t)
defer ctx.Cleanup()
if *pgtest.CrdbConnStr == "" {
t.Skip("Cockroachdb flag missing")
}
namespacedDBName := "name#spaced/Test/DB"
testdb, err := satellitedbtest.NewCockroach(zap.L(), namespacedDBName)
require.NoError(t, err)
// assert new test db exists
driver, source, _, err := dbutil.SplitConnStr(*pgtest.CrdbConnStr)
require.NoError(t, err)
db, err := dbx.Open(driver, source)
require.NoError(t, err)
defer ctx.Check(db.Close)
// NewCockroach removes all non-alphanumeric characters from the name. We need it to match.
r := regexp.MustCompile(`\W`)
formattedName := r.ReplaceAllString(namespacedDBName, "")
var exists *bool
row := db.QueryRow(`SELECT EXISTS (
SELECT datname FROM pg_catalog.pg_database WHERE lower(datname) = lower($1)
);`, formattedName,
)
err = row.Scan(&exists)
require.NoError(t, err)
assert.True(t, *exists)
err = testdb.Close()
require.NoError(t, err)
// assert new test db was deleted
row = db.QueryRow(`SELECT EXISTS (
SELECT datname FROM pg_catalog.pg_database WHERE lower(datname) = lower($1)
);`, formattedName,
)
err = row.Scan(&exists)
require.NoError(t, err)
assert.False(t, *exists)
}

57
satellite/satellitedb/satellitedbtest/pgschema.go
View File

@ -5,33 +5,10 @@ package satellitedbtest
import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/private/dbutil/pgutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/satellite"
"storj.io/storj/satellite/satellitedb"
dbx "storj.io/storj/satellite/satellitedb/dbx"
)
// NewPostgres returns the default postgres satellite.DB for testing.
func NewPostgres(log *zap.Logger, schema string) (satellite.DB, error) {
if err := PostgresDefined(); err != nil {
return nil, err
}
db, err := satellitedb.New(log, pgutil.ConnstrWithSchema(*pgtest.ConnStr, schema))
if err != nil {
return nil, err
}
return &SchemaDB{
DB: db,
Schema: schema,
AutoDrop: true,
}, nil
}
// PostgresDefined returns an error when the --postgres-test-db or STORJ_POSTGRES_TEST is not set for tests.
func PostgresDefined() error {
if *pgtest.ConnStr == "" {
@ -39,37 +16,3 @@ func PostgresDefined() error {
}
return nil
}
// SchemaDB implements automatic schema handling for satellite.DB
type SchemaDB struct {
satellite.DB
Schema string
AutoDrop bool
}
// TestDBAccess for raw database access,
// should not be used outside of migration tests.
func (db *SchemaDB) TestDBAccess() *dbx.DB {
return db.DB.(interface{ TestDBAccess() *dbx.DB }).TestDBAccess()
}
// CreateTables creates the schema and creates tables.
func (db *SchemaDB) CreateTables() error {
err := db.DB.CreateSchema(db.Schema)
if err != nil {
return err
}
return db.DB.CreateTables()
}
// Close closes the database and drops the schema, when `AutoDrop` is set.
func (db *SchemaDB) Close() error {
var dropErr error
if db.AutoDrop {
dropErr = db.DB.DropSchema(db.Schema)
}
return errs.Combine(dropErr, db.DB.Close())
}

104
satellite/satellitedb/satellitedbtest/run.go
View File

@ -11,12 +11,18 @@ import (
"strings"
"testing"
"github.com/zeebo/errs"
"go.uber.org/zap"
"go.uber.org/zap/zaptest"
"storj.io/storj/private/dbutil"
"storj.io/storj/private/dbutil/pgutil"
"storj.io/storj/private/dbutil/pgutil/pgtest"
"storj.io/storj/private/dbutil/tempdb"
"storj.io/storj/satellite"
"storj.io/storj/satellite/metainfo"
"storj.io/storj/satellite/satellitedb"
dbx "storj.io/storj/satellite/satellitedb/dbx"
)
// SatelliteDatabases maybe name can be better
@ -73,8 +79,24 @@ func SchemaName(testname, category string, index int, schemaSuffix string) strin
return strings.ToLower(testname + "/" + schemaSuffix + "/" + category + indexStr)
}
// tempMasterDB is a satellite.DB-implementing type that cleans up after itself when closed.
type tempMasterDB struct {
satellite.DB
tempDB *dbutil.TempDatabase
}
// Close closes a tempMasterDB and cleans it up afterward.
func (db *tempMasterDB) Close() error {
return errs.Combine(db.DB.Close(), db.tempDB.Close())
}
// TestDBAccess provides a somewhat regularized access to the underlying DB
func (db *tempMasterDB) TestDBAccess() *dbx.DB {
return db.DB.(interface{ TestDBAccess() *dbx.DB }).TestDBAccess()
}
// CreateMasterDB creates a new satellite database for testing
func CreateMasterDB(t *testing.T, category string, index int, dbInfo Database) (db satellite.DB, err error) {
func CreateMasterDB(t testing.TB, category string, index int, dbInfo Database) (db satellite.DB, err error) {
if dbInfo.URL == "" {
t.Fatalf("Database %s connection string not provided. %s", dbInfo.Name, dbInfo.Message)
}
@ -85,15 +107,57 @@ func CreateMasterDB(t *testing.T, category string, index int, dbInfo Database) (
log := zaptest.NewLogger(t)
schema := SchemaName(t.Name(), category, index, schemaSuffix)
switch dbInfo.Name {
case "Postgres":
db, err = NewPostgres(log.Named("db"), schema)
case "Cockroach":
db, err = NewCockroach(log.Named("db"), schema)
default:
db, err = satellitedb.New(log.Named("db"), dbInfo.URL)
tempDB, err := tempdb.OpenUnique(dbInfo.URL, schema)
if err != nil {
return nil, err
}
return db, err
return CreateMasterDBOnTopOf(log.Named("db"), tempDB)
}
// CreateMasterDBOnTopOf creates a new satellite database on top of an already existing
// temporary database.
func CreateMasterDBOnTopOf(log *zap.Logger, tempDB *dbutil.TempDatabase) (db satellite.DB, err error) {
masterDB, err := satellitedb.New(log.Named("db"), tempDB.ConnStr)
return &tempMasterDB{DB: masterDB, tempDB: tempDB}, err
}
// tempPointerDB is a satellite.DB-implementing type that cleans up after itself when closed.
type tempPointerDB struct {
metainfo.PointerDB
tempDB *dbutil.TempDatabase
}
// Close closes a tempPointerDB and cleans it up afterward.
func (db *tempPointerDB) Close() error {
return errs.Combine(db.PointerDB.Close(), db.tempDB.Close())
}
// CreatePointerDB creates a new satellite pointer database for testing
func CreatePointerDB(t testing.TB, category string, index int, dbInfo Database) (db metainfo.PointerDB, err error) {
if dbInfo.URL == "" {
t.Fatalf("Database %s connection string not provided. %s", dbInfo.Name, dbInfo.Message)
}
schemaSuffix := SchemaSuffix()
t.Log("schema-suffix ", schemaSuffix)
log := zaptest.NewLogger(t)
schema := SchemaName(t.Name(), category, index, schemaSuffix)
tempDB, err := tempdb.OpenUnique(dbInfo.URL, schema)
if err != nil {
return nil, err
}
return CreatePointerDBOnTopOf(log.Named("pointerdb"), tempDB)
}
// CreatePointerDBOnTopOf creates a new satellite database on top of an already existing
// temporary database.
func CreatePointerDBOnTopOf(log *zap.Logger, tempDB *dbutil.TempDatabase) (db metainfo.PointerDB, err error) {
pointerDB, err := metainfo.NewStore(log.Named("pointerdb"), tempDB.ConnStr)
return &tempPointerDB{PointerDB: pointerDB, tempDB: tempDB}, err
}
// Run method will iterate over all supported databases. Will establish
@ -140,30 +204,10 @@ func Bench(b *testing.B, bench func(b *testing.B, db satellite.DB)) {
b.Skipf("Database %s connection string not provided. %s", dbInfo.MasterDB.Name, dbInfo.MasterDB.Message)
}
schemaSuffix := SchemaSuffix()
b.Log("schema-suffix ", schemaSuffix)
log := zaptest.NewLogger(b)
schema := SchemaName(b.Name(), "X", 0, schemaSuffix)
db, err := satellitedb.New(log, dbInfo.MasterDB.URL)
db, err := CreateMasterDB(b, "X", 0, dbInfo.MasterDB)
if err != nil {
b.Fatal(err)
}
if dbInfo.MasterDB.Name == "Postgres" {
pgdb, err := satellitedb.New(log, pgutil.ConnstrWithSchema(dbInfo.MasterDB.URL, schema))
if err != nil {
b.Fatal(err)
}
db = &SchemaDB{
DB: pgdb,
Schema: schema,
AutoDrop: true,
}
}
defer func() {
err := db.Close()
if err != nil {

4
satellite/satellitedb/testdata/postgres.v69.sql vendored
View File

@ -395,8 +395,8 @@ INSERT INTO "bucket_storage_tallies" ("bucket_name", "project_id", "interval_sta
INSERT INTO "reset_password_tokens" ("secret", "owner_id", "created_at") VALUES (E'\\070\\127\\144\\013\\332\\344\\102\\376\\306\\056\\303\\130\\106\\132\\321\\276\\321\\274\\170\\264\\054\\333\\221\\116\\154\\221\\335\\070\\220\\146\\344\\216'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-05-08 08:28:24.677953+00');
INSERT INTO "offers" ("name", "description", "award_credit_in_cents", "invitee_credit_in_cents", "award_credit_duration_days", "invitee_credit_duration_days", "redeemable_cap", "expires_at", "created_at", "status", "type") VALUES ('testOffer', 'Test offer 1', 0, 0, 14, 14, 50, '2019-03-14 08:28:24.636949+00', '2019-02-14 08:28:24.636949+00', 0, 0);
INSERT INTO "offers" ("name","description","award_credit_in_cents","award_credit_duration_days", "invitee_credit_in_cents","invitee_credit_duration_days", "expires_at","created_at","status","type") VALUES ('Default free credit offer','Is active when no active free credit offer',0, NULL,300, 14, '2119-03-14 08:28:24.636949+00','2019-07-14 08:28:24.636949+00',1,1);
INSERT INTO "offers" ("id","name", "description", "award_credit_in_cents", "invitee_credit_in_cents", "award_credit_duration_days", "invitee_credit_duration_days", "redeemable_cap", "expires_at", "created_at", "status", "type") VALUES (1, 'testOffer', 'Test offer 1', 0, 0, 14, 14, 50, '2019-03-14 08:28:24.636949+00', '2019-02-14 08:28:24.636949+00', 0, 0);
INSERT INTO "offers" ("id","name","description","award_credit_in_cents","award_credit_duration_days", "invitee_credit_in_cents","invitee_credit_duration_days", "expires_at","created_at","status","type") VALUES (2, 'Default free credit offer','Is active when no active free credit offer',0, NULL,300, 14, '2119-03-14 08:28:24.636949+00','2019-07-14 08:28:24.636949+00',1,1);
INSERT INTO "api_keys" ("id", "project_id", "head", "name", "secret", "partner_id", "created_at") VALUES (E'\\334/\\302;\\225\\355O\\323\\276f\\247\\354/6\\241\\033'::bytea, E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, E'\\111\\142\\147\\304\\132\\375\\070\\163\\270\\160\\251\\370\\126\\063\\351\\037\\257\\071\\143\\375\\351\\320\\253\\232\\220\\260\\075\\173\\306\\307\\115\\136'::bytea, 'key 2', E'\\254\\011\\315\\333\\273\\365\\001\\071\\024\\154\\253\\332\\301\\216\\361\\074\\221\\367\\251\\231\\274\\333\\300\\367\\001\\272\\327\\111\\315\\123\\042\\016'::bytea, NULL, '2019-02-14 08:28:24.267934+00');

4
satellite/satellitedb/testdata/postgres.v70.sql vendored
View File

@ -401,8 +401,8 @@ INSERT INTO "bucket_storage_tallies" ("bucket_name", "project_id", "interval_sta
INSERT INTO "reset_password_tokens" ("secret", "owner_id", "created_at") VALUES (E'\\070\\127\\144\\013\\332\\344\\102\\376\\306\\056\\303\\130\\106\\132\\321\\276\\321\\274\\170\\264\\054\\333\\221\\116\\154\\221\\335\\070\\220\\146\\344\\216'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-05-08 08:28:24.677953+00');
INSERT INTO "offers" ("name", "description", "award_credit_in_cents", "invitee_credit_in_cents", "award_credit_duration_days", "invitee_credit_duration_days", "redeemable_cap", "expires_at", "created_at", "status", "type") VALUES ('testOffer', 'Test offer 1', 0, 0, 14, 14, 50, '2019-03-14 08:28:24.636949+00', '2019-02-14 08:28:24.636949+00', 0, 0);
INSERT INTO "offers" ("name","description","award_credit_in_cents","award_credit_duration_days", "invitee_credit_in_cents","invitee_credit_duration_days", "expires_at","created_at","status","type") VALUES ('Default free credit offer','Is active when no active free credit offer',0, NULL,300, 14, '2119-03-14 08:28:24.636949+00','2019-07-14 08:28:24.636949+00',1,1);
INSERT INTO "offers" ("id","name", "description", "award_credit_in_cents", "invitee_credit_in_cents", "award_credit_duration_days", "invitee_credit_duration_days", "redeemable_cap", "expires_at", "created_at", "status", "type") VALUES (1, 'testOffer', 'Test offer 1', 0, 0, 14, 14, 50, '2019-03-14 08:28:24.636949+00', '2019-02-14 08:28:24.636949+00', 0, 0);
INSERT INTO "offers" ("id","name","description","award_credit_in_cents","award_credit_duration_days", "invitee_credit_in_cents","invitee_credit_duration_days", "expires_at","created_at","status","type") VALUES (2, 'Default free credit offer','Is active when no active free credit offer',0, NULL,300, 14, '2119-03-14 08:28:24.636949+00','2019-07-14 08:28:24.636949+00',1,1);
INSERT INTO "api_keys" ("id", "project_id", "head", "name", "secret", "partner_id", "created_at") VALUES (E'\\334/\\302;\\225\\355O\\323\\276f\\247\\354/6\\241\\033'::bytea, E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, E'\\111\\142\\147\\304\\132\\375\\070\\163\\270\\160\\251\\370\\126\\063\\351\\037\\257\\071\\143\\375\\351\\320\\253\\232\\220\\260\\075\\173\\306\\307\\115\\136'::bytea, 'key 2', E'\\254\\011\\315\\333\\273\\365\\001\\071\\024\\154\\253\\332\\301\\216\\361\\074\\221\\367\\251\\231\\274\\333\\300\\367\\001\\272\\327\\111\\315\\123\\042\\016'::bytea, NULL, '2019-02-14 08:28:24.267934+00');

4
satellite/satellitedb/testdata/postgres.v71.sql vendored
View File

@ -431,8 +431,8 @@ INSERT INTO "bucket_storage_tallies" ("bucket_name", "project_id", "interval_sta
INSERT INTO "reset_password_tokens" ("secret", "owner_id", "created_at") VALUES (E'\\070\\127\\144\\013\\332\\344\\102\\376\\306\\056\\303\\130\\106\\132\\321\\276\\321\\274\\170\\264\\054\\333\\221\\116\\154\\221\\335\\070\\220\\146\\344\\216'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-05-08 08:28:24.677953+00');
INSERT INTO "offers" ("name", "description", "award_credit_in_cents", "invitee_credit_in_cents", "award_credit_duration_days", "invitee_credit_duration_days", "redeemable_cap", "expires_at", "created_at", "status", "type") VALUES ('testOffer', 'Test offer 1', 0, 0, 14, 14, 50, '2019-03-14 08:28:24.636949+00', '2019-02-14 08:28:24.636949+00', 0, 0);
INSERT INTO "offers" ("name","description","award_credit_in_cents","award_credit_duration_days", "invitee_credit_in_cents","invitee_credit_duration_days", "expires_at","created_at","status","type") VALUES ('Default free credit offer','Is active when no active free credit offer',0, NULL,300, 14, '2119-03-14 08:28:24.636949+00','2019-07-14 08:28:24.636949+00',1,1);
INSERT INTO "offers" ("id","name", "description", "award_credit_in_cents", "invitee_credit_in_cents", "award_credit_duration_days", "invitee_credit_duration_days", "redeemable_cap", "expires_at", "created_at", "status", "type") VALUES (1, 'testOffer', 'Test offer 1', 0, 0, 14, 14, 50, '2019-03-14 08:28:24.636949+00', '2019-02-14 08:28:24.636949+00', 0, 0);
INSERT INTO "offers" ("id","name","description","award_credit_in_cents","award_credit_duration_days", "invitee_credit_in_cents","invitee_credit_duration_days", "expires_at","created_at","status","type") VALUES (2, 'Default free credit offer','Is active when no active free credit offer',0, NULL,300, 14, '2119-03-14 08:28:24.636949+00','2019-07-14 08:28:24.636949+00',1,1);
INSERT INTO "api_keys" ("id", "project_id", "head", "name", "secret", "partner_id", "created_at") VALUES (E'\\334/\\302;\\225\\355O\\323\\276f\\247\\354/6\\241\\033'::bytea, E'\\022\\217/\\014\\376!K\\023\\276\\031\\311}m\\236\\205\\300'::bytea, E'\\111\\142\\147\\304\\132\\375\\070\\163\\270\\160\\251\\370\\126\\063\\351\\037\\257\\071\\143\\375\\351\\320\\253\\232\\220\\260\\075\\173\\306\\307\\115\\136'::bytea, 'key 2', E'\\254\\011\\315\\333\\273\\365\\001\\071\\024\\154\\253\\332\\301\\216\\361\\074\\221\\367\\251\\231\\274\\333\\300\\367\\001\\272\\327\\111\\315\\123\\042\\016'::bytea, NULL, '2019-02-14 08:28:24.267934+00');

6
storage/postgreskv/client.go
View File

@ -12,7 +12,6 @@ import (
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/private/dbutil"
"storj.io/storj/private/dbutil/pgutil"
"storj.io/storj/storage"
"storj.io/storj/storage/postgreskv/schema"
)
@ -51,11 +50,6 @@ func New(dbURL string) (*Client, error) {
}, nil
}
// DropSchema drops the schema.
func (client *Client) DropSchema(schema string) error {
return pgutil.DropSchema(client.pgConn, schema)
}
// Put sets the value for the provided key.
func (client *Client) Put(ctx context.Context, key storage.Key, value storage.Value) (err error) {
defer mon.Task()(&ctx)(&err)