private/dbutil: distinguishes between db drivers and implementations to allow for different implementations of SQL queries.

Change-Id: I2dc8d1d371139aa8bc805e92a2b80b71f580fd64
This commit is contained in:
Jennifer Johnson 2019-12-02 16:05:21 -05:00 committed by Jennifer Li Johnson
parent b779826194
commit ecb960f506
10 changed files with 73 additions and 22 deletions

View File

@ -119,13 +119,13 @@ pipeline {
stage('CockroachDB migration compatibility') {
environment {
STORJ_COCKROACH_TEST = 'postgres://root@localhost:26257/teststorj?sslmode=disable'
STORJ_COCKROACH_TEST = 'cockroach://root@localhost:26257/teststorj?sslmode=disable'
STORJ_POSTGRES_TEST = 'postgres://postgres@localhost/teststorj4?sslmode=disable'
}
steps {
// Until we can run all the unit tests with cockroach (i.e. all the sql is fixed to be cockroachdb compatible),
// lets just run a bare bones test here that sets up testplanet and runs the database migtation
// lets just run a bare bones test here that sets up testplanet and runs the database migration
sh 'psql -U postgres -c \'create database teststorj4;\''
sh 'cockroach sql --insecure --host=localhost:26257 -e \'create database teststorj;\''
sh 'cd private/testplanet && go test ./... -run=TestRun'

View File

@ -54,7 +54,7 @@ func NewDBFromCfg(config DBConfig) (*DB, error) {
// NewDB creates and/or opens the authorization database.
func NewDB(dbURL string, overwrite bool) (*DB, error) {
driver, source, err := dbutil.SplitConnstr(dbURL)
driver, source, _, err := dbutil.SplitConnStr(dbURL)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}

View File

@ -23,7 +23,7 @@ func NewDBFromCfg(cfg tlsopts.Config) (*DB, error) {
// NewDB returns a new revocation database given the URL
func NewDB(dbURL string) (*DB, error) {
driver, source, err := dbutil.SplitConnstr(dbURL)
driver, source, _, err := dbutil.SplitConnStr(dbURL)
if err != nil {
return nil, extensions.ErrRevocationDB.Wrap(err)
}

View File

@ -0,0 +1,37 @@
// Copyright (C) 2019 Storj Labs, Inc.
// See LICENSE for copying information.
package dbutil
// Implementation type of valid DBs
type Implementation int
const (
// Unknown is an unknown db type
Unknown Implementation = iota
// Postgres is a Postgresdb type
Postgres
// Cockroach is a Cockroachdb type
Cockroach
// Bolt is a Bolt kv store
Bolt
// Redis is a Redis kv store
Redis
)
func setImplementation(s string) Implementation {
switch s {
case "postgres":
return Postgres
case "cockroach":
return Cockroach
case "bolt":
return Bolt
case "redis":
return Redis
case "postgresql":
return Postgres
default:
return Unknown
}
}

View File

@ -8,15 +8,23 @@ import (
"strings"
)
// SplitConnstr returns the driver and DSN portions of a URL
func SplitConnstr(s string) (string, string, error) {
// SplitConnStr returns the driver and DSN portions of a URL, along with the db implementation.
func SplitConnStr(s string) (driver string, source string, implementation Implementation, err error) {
// consider https://github.com/xo/dburl if this ends up lacking
parts := strings.SplitN(s, "://", 2)
if len(parts) != 2 {
return "", "", fmt.Errorf("could not parse DB URL %s", s)
return "", "", Unknown, fmt.Errorf("could not parse DB URL %s", s)
}
if parts[0] == "postgres" {
parts[1] = s // postgres wants full URLS for its DSN
driver = parts[0]
source = parts[1]
implementation = setImplementation(parts[0])
if driver == "postgres" {
source = s // postgres wants full URLS for its DSN
}
return parts[0], parts[1], nil
if driver == "cockroach" {
driver = "postgres" // cockroach's driver is actually postgres
source = "postgres://" + source
}
return driver, source, implementation, nil
}

View File

@ -54,7 +54,7 @@ type PointerDB interface {
// NewStore returns database for storing pointer data
func NewStore(logger *zap.Logger, dbURLString string) (db PointerDB, err error) {
driver, source, err := dbutil.SplitConnstr(dbURLString)
driver, source, _, err := dbutil.SplitConnStr(dbURLString)
if err != nil {
return nil, err
}

View File

@ -83,7 +83,7 @@ func TestOrder(t *testing.T) {
{olderRepairPath, time.Now().Add(-3 * time.Hour)},
}
for _, item := range updateList {
res, err := tx.Tx.ExecContext(ctx, dbAccess.Rebind(`UPDATE injuredsegments SET attempted = timezone('utc', ?) WHERE path = ?`), item.attempted, item.path)
res, err := tx.Tx.ExecContext(ctx, dbAccess.Rebind(`UPDATE injuredsegments SET attempted = ? AT TIME ZONE 'UTC' WHERE path = ?`), item.attempted, item.path)
if err != nil {
return err
}

View File

@ -34,12 +34,13 @@ type DB struct {
log *zap.Logger
db *dbx.DB
driver string
implementation dbutil.Implementation
source string
}
// New creates instance of database supports postgres
func New(log *zap.Logger, databaseURL string) (satellite.DB, error) {
driver, source, err := dbutil.SplitConnstr(databaseURL)
driver, source, implementation, err := dbutil.SplitConnStr(databaseURL)
if err != nil {
return nil, err
}
@ -58,7 +59,7 @@ func New(log *zap.Logger, databaseURL string) (satellite.DB, error) {
dbutil.Configure(db.DB, mon)
core := &DB{log: log, db: db, driver: driver, source: source}
core := &DB{log: log, db: db, driver: driver, source: source, implementation: implementation}
return core, nil
}

View File

@ -26,7 +26,7 @@ func NewCockroach(log *zap.Logger, namespacedTestDB string) (satellite.DB, error
return nil, err
}
driver, source, err := dbutil.SplitConnstr(*pgtest.CrdbConnStr)
driver, source, _, err := dbutil.SplitConnStr(*pgtest.CrdbConnStr)
if err != nil {
return nil, err
}
@ -48,12 +48,11 @@ func NewCockroach(log *zap.Logger, namespacedTestDB string) (satellite.DB, error
if !r.MatchString(source) {
return nil, errs.New("expecting db url format to contain a substring like '/dbName?', but got %s", source)
}
testConnURL := r.ReplaceAllString(source, "/"+namespacedTestDB+"?")
testConnURL := r.ReplaceAllString(*pgtest.CrdbConnStr, "/"+namespacedTestDB+"?")
testDB, err := satellitedb.New(log, testConnURL)
if err != nil {
return nil, err
}
return &namespacedDB{
DB: testDB,
parentRawConn: db,
@ -101,3 +100,9 @@ func (db *namespacedDB) Close() error {
func (db *namespacedDB) CreateTables() error {
return db.DB.CreateTables()
}
// TestDBAccess for raw database access,
// should not be used outside of migration tests.
func (db *namespacedDB) TestDBAccess() *dbx.DB {
return db.DB.(interface{ TestDBAccess() *dbx.DB }).TestDBAccess()
}

View File

@ -30,7 +30,7 @@ func TestNewCockroach(t *testing.T) {
require.NoError(t, err)
// assert new test db exists
driver, source, err := dbutil.SplitConnstr(*pgtest.CrdbConnStr)
driver, source, _, err := dbutil.SplitConnStr(*pgtest.CrdbConnStr)
require.NoError(t, err)
db, err := dbx.Open(driver, source)