diff --git a/Jenkinsfile.public b/Jenkinsfile.public index d6c37dc25..47bad1aee 100644 --- a/Jenkinsfile.public +++ b/Jenkinsfile.public @@ -88,9 +88,12 @@ pipeline { // use different hostname to avoid port conflicts STORJ_NETWORK_HOST4 = '127.0.0.2' STORJ_NETWORK_HOST6 = '127.0.0.2' + + STORJ_SIM_POSTGRES = 'postgres://postgres@localhost/teststorj2?sslmode=disable' } steps { + sh 'psql -U postgres -c \'create database teststorj2;\'' sh 'make test-sim' sh 'make test-satellite-cfg-change' } diff --git a/cmd/storj-sim/main.go b/cmd/storj-sim/main.go index 6fbd76f2f..154bc142e 100644 --- a/cmd/storj-sim/main.go +++ b/cmd/storj-sim/main.go @@ -23,6 +23,9 @@ type Flags struct { IsDev bool OnlyEnv bool // only do things necessary for loading env vars + + // Connection string for the postgres database to use for storj-sim processes + Postgres string } var printCommands bool @@ -53,6 +56,8 @@ func main() { rootCmd.PersistentFlags().BoolVarP(&printCommands, "print-commands", "x", false, "print commands as they are run") rootCmd.PersistentFlags().BoolVarP(&flags.IsDev, "dev", "", false, "use configuration values tuned for development") + rootCmd.PersistentFlags().StringVarP(&flags.Postgres, "postgres", "", "", "connection string for postgres. If provided, storj-sim will use postgres for all databases that support it.") + networkCmd := &cobra.Command{ Use: "network", Short: "local network for testing", diff --git a/cmd/storj-sim/network.go b/cmd/storj-sim/network.go index e4aa74662..72b72de7e 100644 --- a/cmd/storj-sim/network.go +++ b/cmd/storj-sim/network.go @@ -21,6 +21,7 @@ import ( "github.com/zeebo/errs" "golang.org/x/sync/errgroup" + "storj.io/storj/internal/dbutil/pgutil" "storj.io/storj/internal/fpath" "storj.io/storj/internal/processgroup" ) @@ -188,7 +189,7 @@ func newNetwork(flags *Flags) (*Processes, error) { versioncontrol.Arguments = withCommon(versioncontrol.Directory, Arguments{ "setup": { "--address", versioncontrol.Address, - "--debug.addr", net.JoinHostPort("127.0.0.1", port(versioncontrolPeer, 0, debugHTTP)), + "--debug.addr", net.JoinHostPort(host, port(versioncontrolPeer, 0, debugHTTP)), }, "run": {}, }) @@ -225,7 +226,7 @@ func newNetwork(flags *Flags) (*Processes, error) { "--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address), - "--debug.addr", net.JoinHostPort("127.0.0.1", port(bootstrapPeer, 0, debugHTTP)), + "--debug.addr", net.JoinHostPort(host, port(bootstrapPeer, 0, debugHTTP)), }, "run": {}, }) @@ -280,11 +281,18 @@ func newNetwork(flags *Flags) (*Processes, error) { "--mail.template-path", filepath.Join(storjRoot, "web/satellite/static/emails"), "--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address), - "--debug.addr", net.JoinHostPort("127.0.0.1", port(satellitePeer, i, debugHTTP)), + "--debug.addr", net.JoinHostPort(host, port(satellitePeer, i, debugHTTP)), }, "run": {}, }) + if flags.Postgres != "" { + process.Arguments["setup"] = append(process.Arguments["setup"], + "--database", pgutil.ConnstrWithSchema(flags.Postgres, fmt.Sprintf("satellite/%d", i)), + "--metainfo.database-url", pgutil.ConnstrWithSchema(flags.Postgres, fmt.Sprintf("satellite/%d/meta", i)), + ) + } + process.ExecBefore["run"] = func(process *Process) error { return readConfigString(&process.Address, process.Directory, "server.address") } @@ -421,7 +429,7 @@ func newNetwork(flags *Flags) (*Processes, error) { "--storage.satellite-id-restriction=false", "--version.server-address", fmt.Sprintf("http://%s/", versioncontrol.Address), - "--debug.addr", net.JoinHostPort("127.0.0.1", port(storagenodePeer, i, debugHTTP)), + "--debug.addr", net.JoinHostPort(host, port(storagenodePeer, i, debugHTTP)), }, "run": {}, }) diff --git a/internal/dbutil/pgutil/schema.go b/internal/dbutil/pgutil/schema.go index ee211c8f4..566276b3d 100644 --- a/internal/dbutil/pgutil/schema.go +++ b/internal/dbutil/pgutil/schema.go @@ -28,6 +28,21 @@ func ConnstrWithSchema(connstr, schema string) string { return connstr + "&search_path=" + url.QueryEscape(schema) } +// ParseSchemaFromConnstr returns the name of the schema parsed from the +// connection string if one is provided +func ParseSchemaFromConnstr(connstr string) (string, error) { + url, err := url.Parse(connstr) + if err != nil { + return "", err + } + queryValues := url.Query() + schema := queryValues["search_path"] + if len(schema) > 0 { + return schema[0], nil + } + return "", nil +} + // QuoteSchema quotes schema name for func QuoteSchema(schema string) string { return strconv.QuoteToASCII(schema) diff --git a/satellite/console/consoleweb/server.go b/satellite/console/consoleweb/server.go index e29409edc..8aa3526ab 100644 --- a/satellite/console/consoleweb/server.go +++ b/satellite/console/consoleweb/server.go @@ -76,7 +76,7 @@ func NewServer(logger *zap.Logger, config Config, service *console.Service, mail mailService: mailService, } - logger.Debug("Starting Satellite UI...") + logger.Sugar().Debugf("Starting Satellite UI on %s...", server.listener.Addr().String()) if server.config.ExternalAddress != "" { if !strings.HasSuffix(server.config.ExternalAddress, "/") { diff --git a/satellite/metainfo/config.go b/satellite/metainfo/config.go index 9112af080..19e860bed 100644 --- a/satellite/metainfo/config.go +++ b/satellite/metainfo/config.go @@ -4,6 +4,8 @@ package metainfo import ( + "go.uber.org/zap" + "storj.io/storj/internal/dbutil" "storj.io/storj/internal/memory" "storj.io/storj/storage" @@ -26,7 +28,7 @@ type Config struct { } // NewStore returns database for storing pointer data -func NewStore(dbURLString string) (db storage.KeyValueStore, err error) { +func NewStore(logger *zap.Logger, dbURLString string) (db storage.KeyValueStore, err error) { driver, source, err := dbutil.SplitConnstr(dbURLString) if err != nil { return nil, err @@ -38,5 +40,6 @@ func NewStore(dbURLString string) (db storage.KeyValueStore, err error) { } else { err = Error.New("unsupported db scheme: %s", driver) } + logger.Debug("Connected to:", zap.String("db source", source)) return db, err } diff --git a/satellite/peer.go b/satellite/peer.go index 6708781f6..6637888ad 100644 --- a/satellite/peer.go +++ b/satellite/peer.go @@ -342,7 +342,7 @@ func New(log *zap.Logger, full *identity.FullIdentity, db DB, config *Config, ve { // setup metainfo log.Debug("Setting up metainfo") - db, err := metainfo.NewStore(config.Metainfo.DatabaseURL) + db, err := metainfo.NewStore(peer.Log.Named("metainfo:store"), config.Metainfo.DatabaseURL) if err != nil { return nil, errs.Combine(err, peer.Close()) } diff --git a/satellite/satellitedb/database.go b/satellite/satellitedb/database.go index f344a8b86..98af62193 100644 --- a/satellite/satellitedb/database.go +++ b/satellite/satellitedb/database.go @@ -33,6 +33,7 @@ type DB struct { log *zap.Logger db *dbx.DB driver string + source string } // New creates instance of database (supports: postgres, sqlite3) @@ -49,8 +50,9 @@ func New(log *zap.Logger, databaseURL string) (satellite.DB, error) { return nil, Error.New("failed opening database %q, %q: %v", driver, source, err) } + log.Debug("Connected to:", zap.String("db source", source)) - core := &DB{log: log, db: db, driver: driver} + core := &DB{log: log, db: db, driver: driver, source: source} if driver == "sqlite3" { return newLocked(core), nil } diff --git a/satellite/satellitedb/migrate.go b/satellite/satellitedb/migrate.go index 2b917a6de..ffeea6eb7 100644 --- a/satellite/satellitedb/migrate.go +++ b/satellite/satellitedb/migrate.go @@ -11,6 +11,7 @@ import ( "github.com/zeebo/errs" "go.uber.org/zap" + "storj.io/storj/internal/dbutil/pgutil" "storj.io/storj/internal/migrate" "storj.io/storj/pkg/pb" "storj.io/storj/satellite/console" @@ -23,6 +24,16 @@ var ErrMigrate = errs.Class("migrate") func (db *DB) CreateTables() error { switch db.driver { case "postgres": + schema, err := pgutil.ParseSchemaFromConnstr(db.source) + if err != nil { + return errs.New("error parsing schema: %+v", err) + } + if schema != "" { + err = db.CreateSchema(schema) + if err != nil { + return errs.New("error creating schema: %+v", err) + } + } migration := db.PostgresMigration() return migration.Run(db.log.Named("migrate"), db.db) default: diff --git a/scripts/test-sim.sh b/scripts/test-sim.sh index b188e8e26..8bfca0f55 100755 --- a/scripts/test-sim.sh +++ b/scripts/test-sim.sh @@ -16,9 +16,15 @@ trap cleanup EXIT export STORJ_NETWORK_DIR=$TMP STORJ_NETWORK_HOST4=${STORJ_NETWORK_HOST4:-127.0.0.1} +STORJ_SIM_POSTGRES=${STORJ_SIM_POSTGRES:-""} # setup the network -storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network setup +# if postgres connection string is set as STORJ_SIM_POSTGRES then use that for testing +if [ -z ${STORJ_SIM_POSTGRES} ]; then + storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network setup +else + storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network --postgres=$STORJ_SIM_POSTGRES setup +fi # run aws-cli tests storj-sim -x --satellites 2 --host $STORJ_NETWORK_HOST4 network test bash "$SCRIPTDIR"/test-sim-aws.sh diff --git a/storage/postgreskv/client.go b/storage/postgreskv/client.go index 058b8107a..76f5524bb 100644 --- a/storage/postgreskv/client.go +++ b/storage/postgreskv/client.go @@ -31,7 +31,7 @@ func New(dbURL string) (*Client, error) { if err != nil { return nil, err } - err = schema.PrepareDB(pgConn) + err = schema.PrepareDB(pgConn, dbURL) if err != nil { return nil, err } diff --git a/storage/postgreskv/schema/migrate.go b/storage/postgreskv/schema/migrate.go index 8d35bbc42..14ee26647 100644 --- a/storage/postgreskv/schema/migrate.go +++ b/storage/postgreskv/schema/migrate.go @@ -11,11 +11,14 @@ import ( "github.com/golang-migrate/migrate/v3" "github.com/golang-migrate/migrate/v3/database/postgres" bindata "github.com/golang-migrate/migrate/v3/source/go_bindata" + "github.com/zeebo/errs" + + "storj.io/storj/internal/dbutil/pgutil" ) // PrepareDB applies schema migrations as necessary to the given database to // get it up to date. -func PrepareDB(db *sql.DB) error { +func PrepareDB(db *sql.DB, dbURL string) error { srcDriver, err := bindata.WithInstance(bindata.Resource(AssetNames(), func(name string) ([]byte, error) { return Asset(name) @@ -23,6 +26,18 @@ func PrepareDB(db *sql.DB) error { if err != nil { return err } + + schema, err := pgutil.ParseSchemaFromConnstr(dbURL) + if err != nil { + return errs.New("error parsing schema: %+v", err) + } + if schema != "" { + err := pgutil.CreateSchema(db, schema) + if err != nil { + return errs.New("error creating schema: %+v", err) + } + } + dbDriver, err := postgres.WithInstance(db, &postgres.Config{}) if err != nil { return err