diff --git a/Jenkinsfile.public b/Jenkinsfile.public index b5916b55b..14bd158df 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -119,13 +119,13 @@ pipeline { stage('CockroachDB migration compatibility') { environment { - STORJ_COCKROACH_TEST = 'postgres://root@localhost:26257/teststorj?sslmode=disable' + STORJ_COCKROACH_TEST = 'cockroach://root@localhost:26257/teststorj?sslmode=disable' STORJ_POSTGRES_TEST = 'postgres://postgres@localhost/teststorj4?sslmode=disable' } steps { // Until we can run all the unit tests with cockroach (i.e. all the sql is fixed to be cockroachdb compatible), - // lets just run a bare bones test here that sets up testplanet and runs the database migtation + // lets just run a bare bones test here that sets up testplanet and runs the database migration sh 'psql -U postgres -c \'create database teststorj4;\'' sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database teststorj;\'' sh 'cd private/testplanet && go test ./... -run=TestRun' diff --git a/certificate/authorization/db.go b/certificate/authorization/db.go index b57b9d476..c317479bb 100644 --- a/certificate/authorization/db.go +++ b/certificate/authorization/db.go @@ -54,7 +54,7 @@ func NewDBFromCfg(config DBConfig) (*DB, error) { // NewDB creates and/or opens the authorization database. func NewDB(dbURL string, overwrite bool) (*DB, error) { - driver, source, err := dbutil.SplitConnstr(dbURL) + driver, source, _, err := dbutil.SplitConnStr(dbURL) if err != nil { return nil, extensions.ErrRevocationDB.Wrap(err) } diff --git a/pkg/revocation/common.go b/pkg/revocation/common.go index 0da90fa34..cae4bdbfc 100644 --- a/pkg/revocation/common.go +++ b/pkg/revocation/common.go @@ -23,7 +23,7 @@ func NewDBFromCfg(cfg tlsopts.Config) (*DB, error) { // NewDB returns a new revocation database given the URL func NewDB(dbURL string) (*DB, error) { - driver, source, err := dbutil.SplitConnstr(dbURL) + driver, source, _, err := dbutil.SplitConnStr(dbURL) if err != nil { return nil, extensions.ErrRevocationDB.Wrap(err) } diff --git a/private/dbutil/dbimplementation.go b/private/dbutil/dbimplementation.go new file mode 100644 index 000000000..34235ee64 --- /dev/null +++ b/private/dbutil/dbimplementation.go @@ -0,0 +1,37 @@ +// Copyright (C) 2019 Storj Labs, Inc. +// See LICENSE for copying information. + +package dbutil + +// Implementation type of valid DBs +type Implementation int + +const ( + // Unknown is an unknown db type + Unknown Implementation = iota + // Postgres is a Postgresdb type + Postgres + // Cockroach is a Cockroachdb type + Cockroach + // Bolt is a Bolt kv store + Bolt + // Redis is a Redis kv store + Redis +) + +func setImplementation(s string) Implementation { + switch s { + case "postgres": + return Postgres + case "cockroach": + return Cockroach + case "bolt": + return Bolt + case "redis": + return Redis + case "postgresql": + return Postgres + default: + return Unknown + } +} diff --git a/private/dbutil/split.go b/private/dbutil/split.go index 8a700dd4c..8f04b971c 100644 --- a/private/dbutil/split.go +++ b/private/dbutil/split.go @@ -8,15 +8,23 @@ import ( "strings" ) -// SplitConnstr returns the driver and DSN portions of a URL -func SplitConnstr(s string) (string, string, error) { +// SplitConnStr returns the driver and DSN portions of a URL, along with the db implementation. +func SplitConnStr(s string) (driver string, source string, implementation Implementation, err error) { // consider https://github.com/xo/dburl if this ends up lacking parts := strings.SplitN(s, "://", 2) if len(parts) != 2 { - return "", "", fmt.Errorf("could not parse DB URL %s", s) + return "", "", Unknown, fmt.Errorf("could not parse DB URL %s", s) } - if parts[0] == "postgres" { - parts[1] = s // postgres wants full URLS for its DSN + driver = parts[0] + source = parts[1] + implementation = setImplementation(parts[0]) + + if driver == "postgres" { + source = s // postgres wants full URLS for its DSN } - return parts[0], parts[1], nil + if driver == "cockroach" { + driver = "postgres" // cockroach's driver is actually postgres + source = "postgres://" + source + } + return driver, source, implementation, nil } diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 13c241085..48897629b 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -54,7 +54,7 @@ type PointerDB interface { // NewStore returns database for storing pointer data func NewStore(logger *zap.Logger, dbURLString string) (db PointerDB, err error) { - driver, source, err := dbutil.SplitConnstr(dbURLString) + driver, source, _, err := dbutil.SplitConnStr(dbURLString) if err != nil { return nil, err } diff --git a/satellite/repair/queue/queue2_test.go b/satellite/repair/queue/queue2_test.go index db1ca9465..f65fb81cb 100644 --- a/satellite/repair/queue/queue2_test.go +++ b/satellite/repair/queue/queue2_test.go @@ -83,7 +83,7 @@ func TestOrder(t *testing.T) { {olderRepairPath, time.Now().Add(-3 * time.Hour)}, } for _, item := range updateList { - res, err := tx.Tx.ExecContext(ctx, dbAccess.Rebind(`UPDATE injuredsegments SET attempted = timezone('utc', ?) WHERE path = ?`), item.attempted, item.path) + res, err := tx.Tx.ExecContext(ctx, dbAccess.Rebind(`UPDATE injuredsegments SET attempted = ? AT TIME ZONE 'UTC' WHERE path = ?`), item.attempted, item.path) if err != nil { return err } diff --git a/satellite/satellitedb/database.go b/satellite/satellitedb/database.go index 2f772807e..23be1fb69 100644 --- a/satellite/satellitedb/database.go +++ b/satellite/satellitedb/database.go @@ -31,15 +31,16 @@ var ( // DB contains access to different database tables type DB struct { - log *zap.Logger - db *dbx.DB - driver string - source string + log *zap.Logger + db *dbx.DB + driver string + implementation dbutil.Implementation + source string } // New creates instance of database supports postgres func New(log *zap.Logger, databaseURL string) (satellite.DB, error) { - driver, source, err := dbutil.SplitConnstr(databaseURL) + driver, source, implementation, err := dbutil.SplitConnStr(databaseURL) if err != nil { return nil, err } @@ -58,7 +59,7 @@ func New(log *zap.Logger, databaseURL string) (satellite.DB, error) { dbutil.Configure(db.DB, mon) - core := &DB{log: log, db: db, driver: driver, source: source} + core := &DB{log: log, db: db, driver: driver, source: source, implementation: implementation} return core, nil } diff --git a/satellite/satellitedb/satellitedbtest/cockroachnamespace.go b/satellite/satellitedb/satellitedbtest/cockroachnamespace.go index fa64421b6..9e7556689 100644 --- a/satellite/satellitedb/satellitedbtest/cockroachnamespace.go +++ b/satellite/satellitedb/satellitedbtest/cockroachnamespace.go @@ -26,7 +26,7 @@ func NewCockroach(log *zap.Logger, namespacedTestDB string) (satellite.DB, error return nil, err } - driver, source, err := dbutil.SplitConnstr(*pgtest.CrdbConnStr) + driver, source, _, err := dbutil.SplitConnStr(*pgtest.CrdbConnStr) if err != nil { return nil, err } @@ -48,12 +48,11 @@ func NewCockroach(log *zap.Logger, namespacedTestDB string) (satellite.DB, error if !r.MatchString(source) { return nil, errs.New("expecting db url format to contain a substring like '/dbName?', but got %s", source) } - testConnURL := r.ReplaceAllString(source, "/"+namespacedTestDB+"?") + testConnURL := r.ReplaceAllString(*pgtest.CrdbConnStr, "/"+namespacedTestDB+"?") testDB, err := satellitedb.New(log, testConnURL) if err != nil { return nil, err } - return &namespacedDB{ DB: testDB, parentRawConn: db, @@ -101,3 +100,9 @@ func (db *namespacedDB) Close() error { func (db *namespacedDB) CreateTables() error { return db.DB.CreateTables() } + +// TestDBAccess for raw database access, +// should not be used outside of migration tests. +func (db *namespacedDB) TestDBAccess() *dbx.DB { + return db.DB.(interface{ TestDBAccess() *dbx.DB }).TestDBAccess() +} diff --git a/satellite/satellitedb/satellitedbtest/cockroachnamespace_test.go b/satellite/satellitedb/satellitedbtest/cockroachnamespace_test.go index 53332fc01..ddf75a22d 100644 --- a/satellite/satellitedb/satellitedbtest/cockroachnamespace_test.go +++ b/satellite/satellitedb/satellitedbtest/cockroachnamespace_test.go @@ -30,7 +30,7 @@ func TestNewCockroach(t *testing.T) { require.NoError(t, err) // assert new test db exists - driver, source, err := dbutil.SplitConnstr(*pgtest.CrdbConnStr) + driver, source, _, err := dbutil.SplitConnStr(*pgtest.CrdbConnStr) require.NoError(t, err) db, err := dbx.Open(driver, source)