JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/metainfo: Handle "server is not accepting clients" error during CRDB node rejoins

Browse Source 
https: //storjlabs.atlassian.net/browse/SM-1035
Change-Id: I27243b0d8fc3250916c86ceb915f973cbf80f656
This commit is contained in:
Ethan 2020-05-29 10:45:46 -04:00 committed by Natalie Villasana
parent d5c86aa987
commit b1bb665c78
1 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
private/dbutil/cockroachutil/driver.go
View File

@ -90,6 +90,7 @@ func (c *cockroachConn) Close() error {
func (c *cockroachConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { func (c *cockroachConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) {
result, err := c.underlying.ExecContext(ctx, query, args) result, err := c.underlying.ExecContext(ctx, query, args)
for err != nil && !c.isInTransaction() && NeedsRetry(err) { for err != nil && !c.isInTransaction() && NeedsRetry(err) {
mon.Event("needed_retry")
result, err = c.underlying.ExecContext(ctx, query, args) result, err = c.underlying.ExecContext(ctx, query, args)
} }
return result, err return result, err
@ -149,6 +150,7 @@ func (c *cockroachConn) QueryContext(ctx context.Context, query string, args []d
if c.isInTransaction() { if c.isInTransaction() {
return nil, err return nil, err
} }
mon.Event("needed_retry")
continue continue
} }
return nil, err return nil, err
@ -162,6 +164,7 @@ func (c *cockroachConn) QueryContext(ctx context.Context, query string, args []d
if c.isInTransaction() { if c.isInTransaction() {
return nil, err return nil, err
} }
mon.Event("needed_retry")
continue continue
} }
return nil, err return nil, err
@ -248,6 +251,7 @@ func (stmt *cockroachStmt) Exec(args []driver.Value) (driver.Result, error) {
} }
result, err := stmt.underlyingStmt.ExecContext(context.Background(), namedArgs) result, err := stmt.underlyingStmt.ExecContext(context.Background(), namedArgs)
for err != nil && !stmt.conn.isInTransaction() && NeedsRetry(err) { for err != nil && !stmt.conn.isInTransaction() && NeedsRetry(err) {
mon.Event("needed_retry")
result, err = stmt.underlyingStmt.ExecContext(context.Background(), namedArgs) result, err = stmt.underlyingStmt.ExecContext(context.Background(), namedArgs)
} }
return result, err return result, err
@ -268,6 +272,7 @@ func (stmt *cockroachStmt) Query(args []driver.Value) (driver.Rows, error) {
func (stmt *cockroachStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { func (stmt *cockroachStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) {
result, err := stmt.underlyingStmt.ExecContext(ctx, args) result, err := stmt.underlyingStmt.ExecContext(ctx, args)
for err != nil && !stmt.conn.isInTransaction() && NeedsRetry(err) { for err != nil && !stmt.conn.isInTransaction() && NeedsRetry(err) {
mon.Event("needed_retry")
result, err = stmt.underlyingStmt.ExecContext(ctx, args) result, err = stmt.underlyingStmt.ExecContext(ctx, args)
} }
return result, err return result, err
@ -283,6 +288,7 @@ func (stmt *cockroachStmt) QueryContext(ctx context.Context, args []driver.Named
if stmt.conn.isInTransaction() { if stmt.conn.isInTransaction() {
return nil, err return nil, err
} }
mon.Event("needed_retry")
continue continue
} }
return nil, err return nil, err
@ -296,6 +302,7 @@ func (stmt *cockroachStmt) QueryContext(ctx context.Context, args []driver.Named
if stmt.conn.isInTransaction() { if stmt.conn.isInTransaction() {
return nil, err return nil, err
} }
mon.Event("needed_retry")
continue continue
} }
return nil, err return nil, err
@ -317,7 +324,12 @@ func translateName(name string) string {
// borrowed from code in crdb. // borrowed from code in crdb.
func NeedsRetry(err error) bool { func NeedsRetry(err error) bool {
code := errCode(err) code := errCode(err)
return code == "40001" || code == "CR000"
// 57P01 occurs when a CRDB node rejoins the cluster but is not ready to accept connections
// CRDB support recommended a retry at this point
// Support ticket: https://support.cockroachlabs.com/hc/en-us/requests/5510
// TODO re-evaluate this if support provides a better solution
return code == "40001" || code == "CR000" || code == "57P01"
} }
// borrowed from crdb // borrowed from crdb