JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add postgres support to storj-sim (#1908)

Browse Source 
* add flags to sotrj-sim for SA dbs

* add schema to postgres

* add createschema with parse to sa

* add metainfo db postgres support

* add kv default as bolt

* add debug log to see db source

* add env var for postgres to test-sim.sh

* fix lint errs

* dynamically add postgres to args

* add postgres to integration tests

* add sqlite and postgres integration jenkins

* fix db name

* merge integration tests into one step

* test integration tests w/psql

* try using different schema

* debug failure

* use correct host for running storj-sim

* rm sqlite integration

* add back integration
This commit is contained in:
Jess G 2019-05-14 08:13:18 -07:00 committed by GitHub
parent 78c86a8760
commit 8518618b7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 79 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Jenkinsfile.public
View File

@ -88,9 +88,12 @@ pipeline {
// use different hostname to avoid port conflicts
STORJ_NETWORK_HOST4 = '127.0.0.2'
STORJ_NETWORK_HOST6 = '127.0.0.2'
STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj2?sslmode=disable'
}
steps {
sh 'psql -U postgres -c \'create database teststorj2;\''
sh 'make test-sim'
sh 'make test-satellite-cfg-change'
}

5
cmd/storj-sim/main.go
View File

@ -23,6 +23,9 @@ type Flags struct {
IsDev bool
OnlyEnv bool // only do things necessary for loading env vars
// Connection string for the postgres database to use for storj-sim processes
Postgres string
}
var printCommands bool
@ -53,6 +56,8 @@ func main() {
rootCmd.PersistentFlags().BoolVarP(&printCommands, "print-commands", "x", false, "print commands as they are run")
rootCmd.PersistentFlags().BoolVarP(&flags.IsDev, "dev", "", false, "use configuration values tuned for development")
rootCmd.PersistentFlags().StringVarP(&flags.Postgres, "postgres", "", "", "connection string for postgres. If provided, storj-sim will use postgres for all databases that support it.")
networkCmd := &cobra.Command{
Use: "network",
Short: "local network for testing",

16
cmd/storj-sim/network.go
View File

@ -21,6 +21,7 @@ import (
"github.com/zeebo/errs"
"golang.org/x/sync/errgroup"
"storj.io/storj/internal/dbutil/pgutil"
"storj.io/storj/internal/fpath"
"storj.io/storj/internal/processgroup"
)
@ -188,7 +189,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
versioncontrol.Arguments = withCommon(versioncontrol.Directory, Arguments{
"setup": {
"--address", versioncontrol.Address,
"--debug.addr", net.JoinHostPort("127.0.0.1", port(versioncontrolPeer, 0, debugHTTP)),
"--debug.addr", net.JoinHostPort(host, port(versioncontrolPeer, 0, debugHTTP)),
},
"run": {},
})
@ -225,7 +226,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
"--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address),
"--debug.addr", net.JoinHostPort("127.0.0.1", port(bootstrapPeer, 0, debugHTTP)),
"--debug.addr", net.JoinHostPort(host, port(bootstrapPeer, 0, debugHTTP)),
},
"run": {},
})
@ -280,11 +281,18 @@ func newNetwork(flags *Flags) (*Processes, error) {
"--mail.template-path", filepath.Join(storjRoot, "web/satellite/static/emails"),
"--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address),
"--debug.addr", net.JoinHostPort("127.0.0.1", port(satellitePeer, i, debugHTTP)),
"--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugHTTP)),
},
"run": {},
})
if flags.Postgres != "" {
process.Arguments["setup"] = append(process.Arguments["setup"],
"--database", pgutil.ConnstrWithSchema(flags.Postgres, fmt.Sprintf("satellite/%d", i)),
"--metainfo.database-url", pgutil.ConnstrWithSchema(flags.Postgres, fmt.Sprintf("satellite/%d/meta", i)),
)
}
process.ExecBefore["run"] = func(process *Process) error {
return readConfigString(&process.Address, process.Directory, "server.address")
}
@ -421,7 +429,7 @@ func newNetwork(flags *Flags) (*Processes, error) {
"--storage.satellite-id-restriction=false",
"--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address),
"--debug.addr", net.JoinHostPort("127.0.0.1", port(storagenodePeer, i, debugHTTP)),
"--debug.addr", net.JoinHostPort(host, port(storagenodePeer, i, debugHTTP)),
},
"run": {},
})

15
internal/dbutil/pgutil/schema.go
View File

@ -28,6 +28,21 @@ func ConnstrWithSchema(connstr, schema string) string {
return connstr + "&search_path=" + url.QueryEscape(schema)
}
// ParseSchemaFromConnstr returns the name of the schema parsed from the
// connection string if one is provided
func ParseSchemaFromConnstr(connstr string) (string, error) {
url, err := url.Parse(connstr)
if err != nil {
return "", err
}
queryValues := url.Query()
schema := queryValues["search_path"]
if len(schema) > 0 {
return schema[0], nil
}
return "", nil
}
// QuoteSchema quotes schema name for
func QuoteSchema(schema string) string {
return strconv.QuoteToASCII(schema)

2
satellite/console/consoleweb/server.go
View File

@ -76,7 +76,7 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, mail
mailService: mailService,
}
logger.Debug("Starting Satellite UI...")
logger.Sugar().Debugf("Starting Satellite UI on %s...", server.listener.Addr().String())
if server.config.ExternalAddress != "" {
if !strings.HasSuffix(server.config.ExternalAddress, "/") {

5
satellite/metainfo/config.go
View File

@ -4,6 +4,8 @@
package metainfo
import (
"go.uber.org/zap"
"storj.io/storj/internal/dbutil"
"storj.io/storj/internal/memory"
"storj.io/storj/storage"
@ -26,7 +28,7 @@ type Config struct {
}
// NewStore returns database for storing pointer data
func NewStore(dbURLString string) (db storage.KeyValueStore, err error) {
func NewStore(logger *zap.Logger, dbURLString string) (db storage.KeyValueStore, err error) {
driver, source, err := dbutil.SplitConnstr(dbURLString)
if err != nil {
return nil, err
@ -38,5 +40,6 @@ func NewStore(dbURLString string) (db storage.KeyValueStore, err error) {
} else {
err = Error.New("unsupported db scheme: %s", driver)
}
logger.Debug("Connected to:", zap.String("db source", source))
return db, err
}

2
satellite/peer.go
View File

@ -342,7 +342,7 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, config *Config, ve
{ // setup metainfo
log.Debug("Setting up metainfo")
db, err := metainfo.NewStore(config.Metainfo.DatabaseURL)
db, err := metainfo.NewStore(peer.Log.Named("metainfo:store"), config.Metainfo.DatabaseURL)
if err != nil {
return nil, errs.Combine(err, peer.Close())
}

4
satellite/satellitedb/database.go
View File

@ -33,6 +33,7 @@ type DB struct {
log *zap.Logger
db *dbx.DB
driver string
source string
}
// New creates instance of database (supports: postgres, sqlite3)
@ -49,8 +50,9 @@ func New(log *zap.Logger, databaseURL string) (satellite.DB, error) {
return nil, Error.New("failed opening database %q, %q: %v",
driver, source, err)
}
log.Debug("Connected to:", zap.String("db source", source))
core := &DB{log: log, db: db, driver: driver}
core := &DB{log: log, db: db, driver: driver, source: source}
if driver == "sqlite3" {
return newLocked(core), nil
}

11
satellite/satellitedb/migrate.go
View File

@ -11,6 +11,7 @@ import (
"github.com/zeebo/errs"
"go.uber.org/zap"
"storj.io/storj/internal/dbutil/pgutil"
"storj.io/storj/internal/migrate"
"storj.io/storj/pkg/pb"
"storj.io/storj/satellite/console"
@ -23,6 +24,16 @@ var ErrMigrate = errs.Class("migrate")
func (db *DB) CreateTables() error {
switch db.driver {
case "postgres":
schema, err := pgutil.ParseSchemaFromConnstr(db.source)
if err != nil {
return errs.New("error parsing schema: %+v", err)
}
if schema != "" {
err = db.CreateSchema(schema)
if err != nil {
return errs.New("error creating schema: %+v", err)
}
}
migration := db.PostgresMigration()
return migration.Run(db.log.Named("migrate"), db.db)
default:

8
scripts/test-sim.sh
View File

@ -16,9 +16,15 @@ trap cleanup EXIT
export STORJ_NETWORK_DIR=$TMP
STORJ_NETWORK_HOST4=${STORJ_NETWORK_HOST4:-127.0.0.1}
STORJ_SIM_POSTGRES=${STORJ_SIM_POSTGRES:-""}
# setup the network
storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network setup
# if postgres connection string is set as STORJ_SIM_POSTGRES then use that for testing
if [ -z ${STORJ_SIM_POSTGRES} ]; then
storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network setup
else
storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network --postgres=$STORJ_SIM_POSTGRES setup
fi
# run aws-cli tests
storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-sim-aws.sh

2
storage/postgreskv/client.go
View File

@ -31,7 +31,7 @@ func New(dbURL string) (*Client, error) {
if err != nil {
return nil, err
}
err = schema.PrepareDB(pgConn)
err = schema.PrepareDB(pgConn, dbURL)
if err != nil {
return nil, err
}

17
storage/postgreskv/schema/migrate.go
View File

@ -11,11 +11,14 @@ import (
"github.com/golang-migrate/migrate/v3"
"github.com/golang-migrate/migrate/v3/database/postgres"
bindata "github.com/golang-migrate/migrate/v3/source/go_bindata"
"github.com/zeebo/errs"
"storj.io/storj/internal/dbutil/pgutil"
)
// PrepareDB applies schema migrations as necessary to the given database to
// get it up to date.
func PrepareDB(db *sql.DB) error {
func PrepareDB(db *sql.DB, dbURL string) error {
srcDriver, err := bindata.WithInstance(bindata.Resource(AssetNames(),
func(name string) ([]byte, error) {
return Asset(name)
@ -23,6 +26,18 @@ func PrepareDB(db *sql.DB) error {
if err != nil {
return err
}
schema, err := pgutil.ParseSchemaFromConnstr(dbURL)
if err != nil {
return errs.New("error parsing schema: %+v", err)
}
if schema != "" {
err := pgutil.CreateSchema(db, schema)
if err != nil {
return errs.New("error creating schema: %+v", err)
}
}
dbDriver, err := postgres.WithInstance(db, &postgres.Config{})
if err != nil {
return err