db: set max open conns, conn max lifetime, add db stat monitoring (#2117)

internal/dbutil/defaults.go

@@ -3,5 +3,32 @@
 
 package dbutil
 
-// DefaultMaxIdleConns default value for database connections
-const DefaultMaxIdleConns = 100
+import (
+	"database/sql"
+	"flag"
+
+	monkit "gopkg.in/spacemonkeygo/monkit.v2"
+)
+
+var (
+	maxIdleConns    = flag.Int("db.max_idle_conns", 50, "default value for database connections, -1 means the stdlib default")
+	maxOpenConns    = flag.Int("db.max_open_conns", 100, "default value for database connections, -1 means the stdlib default")
+	connMaxLifetime = flag.Duration("db.conn_max_lifetime", -1, "default value for database connections, -1 means the stdlib default")
+)
+
+// Configure Sets Connection Boundaries and adds db_stats monitoring to monkit
+func Configure(db *sql.DB, mon *monkit.Scope) {
+	if *maxIdleConns >= 0 {
+		db.SetMaxIdleConns(*maxIdleConns)
+	}
+	if *maxOpenConns >= 0 {
+		db.SetMaxOpenConns(*maxOpenConns)
+	}
+	if *connMaxLifetime >= 0 {
+		db.SetConnMaxLifetime(*connMaxLifetime)
+	}
+	mon.Chain("db_stats", monkit.StatSourceFunc(
+		func(cb func(name string, val float64)) {
+			monkit.StatSourceFromStruct(db.Stats()).Stats(cb)
+		}))
+}

internal/dbutil/pgutil/db.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/lib/pq"
 	"github.com/zeebo/errs"
+	monkit "gopkg.in/spacemonkeygo/monkit.v2"
 
 	"storj.io/storj/internal/dbutil"
 	"storj.io/storj/internal/dbutil/dbschema"
@@ -21,6 +22,10 @@ type DB struct {
 	Schema string
 }
 
+var (
+	mon = monkit.Package()
+)
+
 // Open opens a postgres database with a schema
 func Open(connstr string, schemaPrefix string) (*DB, error) {
 	schemaName := schemaPrefix + "-" + CreateRandomTestingSchemaName(8)
@@ -30,7 +35,7 @@ func Open(connstr string, schemaPrefix string) (*DB, error) {
 		return nil, err
 	}
 
-	db.SetMaxIdleConns(dbutil.DefaultMaxIdleConns)
+	dbutil.Configure(db, mon)
 
 	err = CreateSchema(db, schemaName)
 	if err != nil {

satellite/satellitedb/database.go

@@ -54,7 +54,7 @@ func New(log *zap.Logger, databaseURL string) (satellite.DB, error) {
 	}
 	log.Debug("Connected to:", zap.String("db source", source))
 
-	db.SetMaxIdleConns(dbutil.DefaultMaxIdleConns)
+	dbutil.Configure(db.DB, mon)
 
 	core := &DB{log: log, db: db, driver: driver, source: source}
 	if driver == "sqlite3" {

scripts/testdata/satellite-config.yaml.lock

@@ -40,6 +40,15 @@
 # satellite database connection string
 # database: "postgres://"
 
+# default value for database connections, -1 means the stdlib default
+# db.conn_max_lifetime: -1ns
+
+# default value for database connections, -1 means the stdlib default
+# db.max_idle_conns: 50
+
+# default value for database connections, -1 means the stdlib default
+# db.max_open_conns: 100
+
 # address to listen on for debug endpoints
 # debug.addr: "127.0.0.1:0"
 

storage/postgreskv/client.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/lib/pq"
 	"github.com/zeebo/errs"
+	monkit "gopkg.in/spacemonkeygo/monkit.v2"
 
 	"storj.io/storj/internal/dbutil"
 	"storj.io/storj/storage"
@@ -20,6 +21,10 @@ const (
 	defaultBucket    = ""
 )
 
+var (
+	mon = monkit.Package()
+)
+
 // Client is the entrypoint into a postgreskv data store
 type Client struct {
 	URL    string
@@ -33,7 +38,7 @@ func New(dbURL string) (*Client, error) {
 		return nil, err
 	}
 
-	pgConn.SetMaxIdleConns(dbutil.DefaultMaxIdleConns)
+	dbutil.Configure(pgConn, mon)
 
 	err = schema.PrepareDB(pgConn, dbURL)
 	if err != nil {

storagenode/storagenodedb/infodb.go

@@ -36,7 +36,7 @@ func newInfo(path string) (*InfoDB, error) {
 		return nil, ErrInfo.Wrap(err)
 	}
 
-	db.SetMaxIdleConns(dbutil.DefaultMaxIdleConns)
+	dbutil.Configure(db, mon)
 
 	return &InfoDB{db: db}, nil
 }
@@ -48,7 +48,7 @@ func NewInfoInMemory() (*InfoDB, error) {
 		return nil, ErrInfo.Wrap(err)
 	}
 
-	db.SetMaxIdleConns(dbutil.DefaultMaxIdleConns)
+	dbutil.Configure(db, mon)
 
 	return &InfoDB{db: db}, nil
 }
@@ -153,15 +153,15 @@ func (db *InfoDB) Migration() *migrate.Migration {
 					`CREATE TABLE order_archive (
 						satellite_id  BLOB NOT NULL,
 						serial_number BLOB NOT NULL,
-						
+
 						order_limit_serialized BLOB NOT NULL, -- serialized pb.OrderLimit
 						order_serialized       BLOB NOT NULL, -- serialized pb.Order
-						
+
 						uplink_cert_id INTEGER NOT NULL,
-						
+
 						status      INTEGER   NOT NULL, -- accepted, rejected, confirmed
 						archived_at TIMESTAMP NOT NULL, -- when was it rejected
-						
+
 						FOREIGN KEY(uplink_cert_id) REFERENCES certificate(cert_id)
 					)`,
 					`CREATE INDEX idx_order_archive_satellite ON order_archive(satellite_id)`,