JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

better database error handling (#784)

Browse Source 
* better database error handling

Change-Id: I28dbd69cf6c2fa268e02405521ff6e6c1a68a702

* missing comments added

* missing comment added
This commit is contained in:
JT Olio 2018-12-07 07:46:42 -07:00 committed by GitHub
parent 023eb95a9b
commit 1c96db01ba
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 109 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/accounting/db.go
View File

@ -4,17 +4,20 @@
package accounting
import (
"github.com/zeebo/errs"
"storj.io/storj/internal/migrate"
dbx "storj.io/storj/pkg/accounting/dbx"
"storj.io/storj/pkg/utils"
)
// LastBandwidthTally is a name in the accounting timestamps database
var LastBandwidthTally dbx.Timestamps_Name_Field
var (
// Error is the default accountingdb errs class
Error = errs.Class("accountingdb")
func init() {
// LastBandwidthTally is a name in the accounting timestamps database
LastBandwidthTally = dbx.Timestamps_Name("LastBandwidthTally")
}
)
// NewDb - constructor for DB
func NewDb(databaseURL string) (*dbx.DB, error) {
@ -24,7 +27,8 @@ func NewDb(databaseURL string) (*dbx.DB, error) {
}
db, err := dbx.Open(dbURL.Scheme, dbURL.Path)
if err != nil {
return nil, err
return nil, Error.New("failed opening database %q, %q: %v",
dbURL.Scheme, dbURL.Path, err)
}
err = migrate.Create("accounting", db)
if err != nil {

10
pkg/accounting/dbx/gen.go
View File

@ -5,3 +5,13 @@ package dbx
// go:generate dbx.v1 schema -d postgres -d sqlite3 accounting.dbx .
// go:generate dbx.v1 golang -d postgres -d sqlite3 -p dbx accounting.dbx .
import (
"github.com/zeebo/errs"
)
func init() {
// catch dbx errors
c := errs.Class("accountingdb")
WrapErr = func(e *Error) error { return c.Wrap(e) }
}

10
pkg/datarepair/irreparabledb/dbx/gen.go
View File

@ -5,3 +5,13 @@ package irreparabledb
//go:generate dbx.v1 schema -d postgres -d sqlite3 irreparabledb.dbx .
//go:generate dbx.v1 golang -d postgres -d sqlite3 irreparabledb.dbx .
import (
"github.com/zeebo/errs"
)
func init() {
// catch dbx errors
c := errs.Class("irreparabledb")
WrapErr = func(e *Error) error { return c.Wrap(e) }
}

3
pkg/datarepair/irreparabledb/irreparabledb.go
View File

@ -39,7 +39,8 @@ func New(source string) (*Database, error) {
db, err := dbx.Open(u.Scheme, u.Path)
if err != nil {
return nil, err
return nil, Error.New("failed opening database %q, %q: %v",
u.Scheme, u.Path, err)
}
err = migrate.Create("irreparabledb", db)

2
pkg/kademlia/kademlia_test.go
View File

@ -18,7 +18,7 @@ import (
"github.com/stretchr/testify/assert"
"google.golang.org/grpc"
"storj.io/storj/internal/identity"
testidentity "storj.io/storj/internal/identity"
"storj.io/storj/internal/teststorj"
"storj.io/storj/pkg/node"
"storj.io/storj/pkg/pb"

2
pkg/miniogw/config.go
View File

@ -18,7 +18,7 @@ import (
"storj.io/storj/pkg/pointerdb/pdbclient"
"storj.io/storj/pkg/provider"
"storj.io/storj/pkg/storage/buckets"
"storj.io/storj/pkg/storage/ec"
ecclient "storj.io/storj/pkg/storage/ec"
"storj.io/storj/pkg/storage/segments"
"storj.io/storj/pkg/storage/streams"
"storj.io/storj/pkg/storj"

10
pkg/piecestore/psserver/psdb/psdb.go
View File

@ -16,17 +16,21 @@ import (
"github.com/gogo/protobuf/proto"
_ "github.com/mattn/go-sqlite3" // register sqlite to sql
"github.com/zeebo/errs"
"go.uber.org/zap"
monkit "gopkg.in/spacemonkeygo/monkit.v2"
"storj.io/storj/pkg/pb"
"storj.io/storj/pkg/piecestore"
pstore "storj.io/storj/pkg/piecestore"
"storj.io/storj/pkg/storj"
"storj.io/storj/pkg/utils"
)
var (
mon = monkit.Package()
mon = monkit.Package()
// Error is the default psdb errs class
Error = errs.Class("psdb")
defaultCheckInterval = flag.Duration("piecestore.ttl.check_interval", time.Hour, "number of seconds to sleep between ttl checks")
)
@ -54,7 +58,7 @@ func Open(ctx context.Context, dataPath, DBPath string) (db *DB, err error) {
sqlite, err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc&mutex=full", DBPath))
if err != nil {
return nil, err
return nil, Error.Wrap(err)
}
db = &DB{
DB: sqlite,

11
pkg/satellite/satellitedb/db.go
View File

@ -4,11 +4,18 @@
package satellitedb
import (
"github.com/zeebo/errs"
"storj.io/storj/internal/migrate"
"storj.io/storj/pkg/satellite"
"storj.io/storj/pkg/satellite/satellitedb/dbx"
)
var (
// Error is the default satellitedb errs class
Error = errs.Class("satellitedb")
)
// Database contains access to different satellite databases
type Database struct {
db *dbx.DB
@ -17,9 +24,9 @@ type Database struct {
// New - constructor for DB
func New(driver, source string) (satellite.DB, error) {
db, err := dbx.Open(driver, source)
if err != nil {
return nil, err
return nil, Error.New("failed opening database %q, %q: %v",
driver, source, err)
}
database := &Database{

10
pkg/satellite/satellitedb/dbx/gen.go
View File

@ -5,3 +5,13 @@ package dbx
//go:generate dbx.v1 golang -d sqlite3 -p dbx satellitedb.dbx .
//go:generate dbx.v1 schema -d sqlite3 satellitedb.dbx .
import (
"github.com/zeebo/errs"
)
func init() {
// catch dbx errors
c := errs.Class("satellitedb")
WrapErr = func(e *Error) error { return c.Wrap(e) }
}

10
pkg/statdb/dbx/gen.go
View File

@ -5,3 +5,13 @@ package statdb
// go:generate dbx.v1 schema -d postgres -d sqlite3 statdb.dbx .
// go:generate dbx.v1 golang -d postgres -d sqlite3 statdb.dbx .
import (
"github.com/zeebo/errs"
)
func init() {
// catch dbx errors
c := errs.Class("statdb")
WrapErr = func(e *Error) error { return c.Wrap(e) }
}

7
pkg/statdb/statdb.go
View File

@ -36,7 +36,8 @@ type StatDB struct {
func NewStatDB(driver, source string, log *zap.Logger) (*StatDB, error) {
db, err := dbx.Open(driver, source)
if err != nil {
return nil, err
return nil, Error.New("failed opening database %q, %q: %v",
driver, source, err)
}
err = migrate.Create("statdb", db)
@ -176,8 +177,8 @@ func (s *StatDB) findInvalidNodesQuery(nodeIds storj.NodeIDList, auditSuccess, u
}
args = append(args, auditSuccess, uptime)
rows, err := s.DB.Query(`SELECT nodes.id, nodes.total_audit_count,
nodes.total_uptime_count, nodes.audit_success_ratio,
rows, err := s.DB.Query(`SELECT nodes.id, nodes.total_audit_count,
nodes.total_uptime_count, nodes.audit_success_ratio,
nodes.uptime_ratio
FROM nodes
WHERE nodes.id IN (?`+strings.Repeat(", ?", len(nodeIds)-1)+`)

10
satellite/satellitedb/database.go
View File

@ -4,12 +4,19 @@
package satellitedb
import (
"github.com/zeebo/errs"
"storj.io/storj/internal/migrate"
"storj.io/storj/pkg/bwagreement"
"storj.io/storj/pkg/utils"
dbx "storj.io/storj/satellite/satellitedb/dbx"
)
var (
// Error is the default satellitedb errs class
Error = errs.Class("satellitedb")
)
// DB contains access to different database tables
type DB struct {
db *dbx.DB
@ -27,7 +34,8 @@ func NewDB(databaseURL string) (*DB, error) {
}
db, err := dbx.Open(dbURL.Scheme, source)
if err != nil {
return nil, err
return nil, Error.New("failed opening database %q, %q: %v",
dbURL.Scheme, source, err)
}
return &DB{db: db}, nil

10
satellite/satellitedb/dbx/gen.go
View File

@ -5,3 +5,13 @@ package satellitedb
//go:generate dbx.v1 golang -d postgres -d sqlite3 satellitedb.dbx .
//go:generate dbx.v1 schema -d postgres -d sqlite3 satellitedb.dbx .
import (
"github.com/zeebo/errs"
)
func init() {
// catch dbx errors
c := errs.Class("satellitedb")
WrapErr = func(e *Error) error { return c.Wrap(e) }
}

34
storage/boltdb/client.go
View File

@ -33,16 +33,15 @@ const (
func New(path, bucket string) (*Client, error) {
db, err := bolt.Open(path, fileMode, &bolt.Options{Timeout: defaultTimeout})
if err != nil {
return nil, err
return nil, Error.Wrap(err)
}
err = db.Update(func(tx *bolt.Tx) error {
err = Error.Wrap(db.Update(func(tx *bolt.Tx) error {
_, err = tx.CreateBucketIfNotExists([]byte(bucket))
return err
})
}))
if err != nil {
if closeErr := db.Close(); closeErr != nil {
if closeErr := Error.Wrap(db.Close()); closeErr != nil {
return nil, utils.CombineErrors(err, closeErr)
}
return nil, err
@ -63,10 +62,10 @@ func New(path, bucket string) (*Client, error) {
func NewShared(path string, buckets ...string) ([]*Client, error) {
db, err := bolt.Open(path, fileMode, &bolt.Options{Timeout: defaultTimeout})
if err != nil {
return nil, err
return nil, Error.Wrap(err)
}
err = db.Update(func(tx *bolt.Tx) error {
err = Error.Wrap(db.Update(func(tx *bolt.Tx) error {
for _, bucket := range buckets {
_, err := tx.CreateBucketIfNotExists([]byte(bucket))
if err != nil {
@ -74,10 +73,9 @@ func NewShared(path string, buckets ...string) ([]*Client, error) {
}
}
return err
})
}))
if err != nil {
if closeErr := db.Close(); closeErr != nil {
if closeErr := Error.Wrap(db.Close()); closeErr != nil {
return nil, utils.CombineErrors(err, closeErr)
}
return nil, err
@ -100,15 +98,15 @@ func NewShared(path string, buckets ...string) ([]*Client, error) {
}
func (client *Client) update(fn func(*bolt.Bucket) error) error {
return client.db.Update(func(tx *bolt.Tx) error {
return Error.Wrap(client.db.Update(func(tx *bolt.Tx) error {
return fn(tx.Bucket(client.Bucket))
})
}))
}
func (client *Client) view(fn func(*bolt.Bucket) error) error {
return client.db.View(func(tx *bolt.Tx) error {
return Error.Wrap(client.db.View(func(tx *bolt.Tx) error {
return fn(tx.Bucket(client.Bucket))
})
}))
}
// Put adds a value to the provided key in boltdb, returning an error on failure.
@ -153,19 +151,21 @@ func (client *Client) Delete(key storage.Key) error {
// List returns either a list of keys for which boltdb has values or an error.
func (client *Client) List(first storage.Key, limit int) (storage.Keys, error) {
return storage.ListKeys(client, first, limit)
rv, err := storage.ListKeys(client, first, limit)
return rv, Error.Wrap(err)
}
// ReverseList returns either a list of keys for which boltdb has values or an error.
// Starts from first and iterates backwards
func (client *Client) ReverseList(first storage.Key, limit int) (storage.Keys, error) {
return storage.ReverseListKeys(client, first, limit)
rv, err := storage.ReverseListKeys(client, first, limit)
return rv, Error.Wrap(err)
}
// Close closes a BoltDB client
func (client *Client) Close() error {
if atomic.AddInt32(client.referenceCount, -1) == 0 {
return client.db.Close()
return Error.Wrap(client.db.Close())
}
return nil
}