satellite/reputation: add disqualification reason for status update

Set disqualification reason when reputations stats are updated on DB.Update.
Added tests for DisqualifyNode and for disqualification cases which happens during Update.

Change-Id: I00130ab5d9722422805159ad2f183c205de60f7e
This commit is contained in:
Yaroslav Vorobiov 2021-10-25 23:40:41 +03:00
parent 1422a1ff19
commit 4223fa01f8
7 changed files with 251 additions and 86 deletions

View File

@ -67,7 +67,7 @@ type DB interface {
// Reliable returns all nodes that are reliable // Reliable returns all nodes that are reliable
Reliable(context.Context, *NodeCriteria) (storj.NodeIDList, error) Reliable(context.Context, *NodeCriteria) (storj.NodeIDList, error)
// UpdateReputation updates the DB columns for all reputation fields in ReputationStatus. // UpdateReputation updates the DB columns for all reputation fields in ReputationStatus.
UpdateReputation(ctx context.Context, id storj.NodeID, request *ReputationStatus) error UpdateReputation(ctx context.Context, id storj.NodeID, request ReputationUpdate) error
// UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version. // UpdateNodeInfo updates node dossier with info requested from the node itself like node type, email, wallet, capacity, and version.
UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *InfoResponse) (stats *NodeDossier, err error) UpdateNodeInfo(ctx context.Context, node storj.NodeID, nodeInfo *InfoResponse) (stats *NodeDossier, err error)
// UpdateCheckIn updates a single storagenode's check-in stats. // UpdateCheckIn updates a single storagenode's check-in stats.
@ -119,6 +119,22 @@ type DB interface {
IterateAllNodeDossiers(context.Context, func(context.Context, *NodeDossier) error) error IterateAllNodeDossiers(context.Context, func(context.Context, *NodeDossier) error) error
} }
// DisqualificationReason is disqualification reason enum type.
type DisqualificationReason int
const (
// DisqualificationReasonUnknown denotes undetermined disqualification reason.
DisqualificationReasonUnknown DisqualificationReason = 0
// DisqualificationReasonAuditFailure denotes disqualification due to audit score falling below threshold.
DisqualificationReasonAuditFailure DisqualificationReason = 1
// DisqualificationReasonSuspension denotes disqualification due to unknown audit failure after grace period for unknown audits
// has elapsed.
DisqualificationReasonSuspension DisqualificationReason = 2
// DisqualificationReasonNodeOffline denotes disqualification due to node's online score falling below threshold after tracking
// period has elapsed.
DisqualificationReasonNodeOffline DisqualificationReason = 3
)
// NodeCheckInInfo contains all the info that will be updated when a node checkins. // NodeCheckInInfo contains all the info that will be updated when a node checkins.
type NodeCheckInInfo struct { type NodeCheckInInfo struct {
NodeID storj.NodeID NodeID storj.NodeID
@ -163,52 +179,20 @@ type NodeCriteria struct {
// ReputationStatus indicates current reputation status for a node. // ReputationStatus indicates current reputation status for a node.
type ReputationStatus struct { type ReputationStatus struct {
Contained bool // TODO: check to see if this column is still used. Disqualified *time.Time
Disqualified *time.Time DisqualificationReason *DisqualificationReason
UnknownAuditSuspended *time.Time UnknownAuditSuspended *time.Time
OfflineSuspended *time.Time OfflineSuspended *time.Time
VettedAt *time.Time VettedAt *time.Time
} }
// Equal checks if two ReputationStatus contains the same value. // ReputationUpdate contains reputation update data for a node.
func (status ReputationStatus) Equal(value ReputationStatus) bool { type ReputationUpdate struct {
if status.Contained != value.Contained { Disqualified *time.Time
return false DisqualificationReason DisqualificationReason
} UnknownAuditSuspended *time.Time
OfflineSuspended *time.Time
if status.Disqualified != nil && value.Disqualified != nil { VettedAt *time.Time
if !status.Disqualified.Equal(*value.Disqualified) {
return false
}
} else if !(status.Disqualified == nil && value.Disqualified == nil) {
return false
}
if status.UnknownAuditSuspended != nil && value.UnknownAuditSuspended != nil {
if !status.UnknownAuditSuspended.Equal(*value.UnknownAuditSuspended) {
return false
}
} else if !(status.UnknownAuditSuspended == nil && value.UnknownAuditSuspended == nil) {
return false
}
if status.OfflineSuspended != nil && value.OfflineSuspended != nil {
if !status.OfflineSuspended.Equal(*value.OfflineSuspended) {
return false
}
} else if !(status.OfflineSuspended == nil && value.OfflineSuspended == nil) {
return false
}
if status.VettedAt != nil && value.VettedAt != nil {
if !status.VettedAt.Equal(*value.VettedAt) {
return false
}
} else if !(status.VettedAt == nil && value.VettedAt == nil) {
return false
}
return true
} }
// ExitStatus is used for reading graceful exit status. // ExitStatus is used for reading graceful exit status.
@ -520,7 +504,7 @@ func (service *Service) Reliable(ctx context.Context) (nodes storj.NodeIDList, e
} }
// UpdateReputation updates the DB columns for any of the reputation fields. // UpdateReputation updates the DB columns for any of the reputation fields.
func (service *Service) UpdateReputation(ctx context.Context, id storj.NodeID, request *ReputationStatus) (err error) { func (service *Service) UpdateReputation(ctx context.Context, id storj.NodeID, request ReputationUpdate) (err error) {
defer mon.Task()(&ctx)(&err) defer mon.Task()(&ctx)(&err)
return service.db.UpdateReputation(ctx, id, request) return service.db.UpdateReputation(ctx, id, request)
} }

View File

@ -726,7 +726,7 @@ func TestUpdateReputation(t *testing.T) {
t2 := t0.Add(2 * time.Hour) t2 := t0.Add(2 * time.Hour)
t3 := t0.Add(3 * time.Hour) t3 := t0.Add(3 * time.Hour)
reputationChange := &overlay.ReputationStatus{ reputationChange := overlay.ReputationUpdate{
Disqualified: nil, Disqualified: nil,
UnknownAuditSuspended: &t1, UnknownAuditSuspended: &t1,
OfflineSuspended: &t2, OfflineSuspended: &t2,

View File

@ -12,9 +12,12 @@ import (
"go.uber.org/zap" "go.uber.org/zap"
"storj.io/common/testcontext" "storj.io/common/testcontext"
"storj.io/common/testrand"
"storj.io/storj/private/testplanet" "storj.io/storj/private/testplanet"
"storj.io/storj/satellite" "storj.io/storj/satellite"
"storj.io/storj/satellite/overlay"
"storj.io/storj/satellite/reputation" "storj.io/storj/satellite/reputation"
"storj.io/storj/satellite/satellitedb/satellitedbtest"
) )
func TestUpdate(t *testing.T) { func TestUpdate(t *testing.T) {
@ -60,6 +63,132 @@ func TestUpdate(t *testing.T) {
}) })
} }
func TestDBDisqualifyNode(t *testing.T) {
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
reputationDB := db.Reputation()
nodeID := testrand.NodeID()
now := time.Now().Truncate(time.Second).UTC()
err := reputationDB.DisqualifyNode(ctx, nodeID, now)
require.NoError(t, err)
info, err := reputationDB.Get(ctx, nodeID)
require.NoError(t, err)
require.NotNil(t, info.Disqualified)
require.Equal(t, now, info.Disqualified.UTC())
})
}
func TestDBDisqualificationAuditFailure(t *testing.T) {
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
reputationDB := db.Reputation()
nodeID := testrand.NodeID()
now := time.Now()
updateReq := reputation.UpdateRequest{
NodeID: nodeID,
AuditOutcome: reputation.AuditFailure,
AuditCount: 0,
AuditLambda: 1,
AuditWeight: 1,
AuditDQ: 0.99,
SuspensionGracePeriod: 0,
SuspensionDQEnabled: false,
AuditsRequiredForVetting: 0,
AuditHistory: reputation.AuditHistoryConfig{},
}
status, err := reputationDB.Update(ctx, updateReq, now)
require.NoError(t, err)
require.NotNil(t, status.Disqualified)
assert.WithinDuration(t, now, *status.Disqualified, time.Microsecond)
assert.Equal(t, overlay.DisqualificationReasonAuditFailure, status.DisqualificationReason)
})
}
func TestDBDisqualificationSuspension(t *testing.T) {
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
reputationDB := db.Reputation()
nodeID := testrand.NodeID()
now := time.Now().Truncate(time.Second).UTC()
updateReq := reputation.UpdateRequest{
NodeID: nodeID,
AuditOutcome: reputation.AuditUnknown,
AuditCount: 0,
AuditLambda: 1,
AuditWeight: 1,
AuditDQ: 0.99,
SuspensionGracePeriod: 0,
SuspensionDQEnabled: true,
AuditsRequiredForVetting: 0,
AuditHistory: reputation.AuditHistoryConfig{},
}
// suspend node due to failed unknown audit
err := reputationDB.SuspendNodeUnknownAudit(ctx, nodeID, now.Add(-time.Second))
require.NoError(t, err)
// disqualify node after failed unknown audit when node is suspended
status, err := reputationDB.Update(ctx, updateReq, now)
require.NoError(t, err)
require.NotNil(t, status.Disqualified)
assert.Nil(t, status.UnknownAuditSuspended)
assert.Equal(t, now, status.Disqualified.UTC())
assert.Equal(t, overlay.DisqualificationReasonSuspension, status.DisqualificationReason)
})
}
func TestDBDisqualificationNodeOffline(t *testing.T) {
satellitedbtest.Run(t, func(ctx *testcontext.Context, t *testing.T, db satellite.DB) {
reputationDB := db.Reputation()
nodeID := testrand.NodeID()
now := time.Now().Truncate(time.Second).UTC()
updateReq := reputation.UpdateRequest{
NodeID: nodeID,
AuditOutcome: reputation.AuditOffline,
AuditCount: 0,
AuditLambda: 0,
AuditWeight: 0,
AuditDQ: 0,
SuspensionGracePeriod: 0,
SuspensionDQEnabled: false,
AuditsRequiredForVetting: 0,
AuditHistory: reputation.AuditHistoryConfig{
WindowSize: 0,
TrackingPeriod: 1 * time.Second,
GracePeriod: 0,
OfflineThreshold: 1,
OfflineDQEnabled: true,
OfflineSuspensionEnabled: true,
},
}
// first window always returns perfect score
_, err := reputationDB.Update(ctx, updateReq, now)
require.NoError(t, err)
// put node to offline suspension
suspendedAt := now.Add(time.Second)
status, err := reputationDB.Update(ctx, updateReq, suspendedAt)
require.NoError(t, err)
require.Equal(t, suspendedAt, status.OfflineSuspended.UTC())
// should have at least 2 windows in audit history after earliest window is removed
_, err = reputationDB.Update(ctx, updateReq, now.Add(2*time.Second))
require.NoError(t, err)
// disqualify node
disqualifiedAt := now.Add(3 * time.Second)
status, err = reputationDB.Update(ctx, updateReq, disqualifiedAt)
require.NoError(t, err)
require.NotNil(t, status.Disqualified)
assert.Equal(t, disqualifiedAt, status.Disqualified.UTC())
assert.Equal(t, overlay.DisqualificationReasonNodeOffline, status.DisqualificationReason)
})
}
func testAuditHistoryConfig() reputation.AuditHistoryConfig { func testAuditHistoryConfig() reputation.AuditHistoryConfig {
return reputation.AuditHistoryConfig{ return reputation.AuditHistoryConfig{
WindowSize: time.Hour, WindowSize: time.Hour,

View File

@ -15,13 +15,13 @@ import (
// DB is an interface for storing reputation data. // DB is an interface for storing reputation data.
type DB interface { type DB interface {
Update(ctx context.Context, request UpdateRequest, now time.Time) (_ *overlay.ReputationStatus, err error) Update(ctx context.Context, request UpdateRequest, now time.Time) (_ *overlay.ReputationUpdate, err error)
Get(ctx context.Context, nodeID storj.NodeID) (*Info, error) Get(ctx context.Context, nodeID storj.NodeID) (*Info, error)
// UnsuspendNodeUnknownAudit unsuspends a storage node for unknown audits. // UnsuspendNodeUnknownAudit unsuspends a storage node for unknown audits.
UnsuspendNodeUnknownAudit(ctx context.Context, nodeID storj.NodeID) (err error) UnsuspendNodeUnknownAudit(ctx context.Context, nodeID storj.NodeID) (err error)
// DisqualifyNode disqualifies a storage node. // DisqualifyNode disqualifies a storage node.
DisqualifyNode(ctx context.Context, nodeID storj.NodeID) (err error) DisqualifyNode(ctx context.Context, nodeID storj.NodeID, disqualifiedAt time.Time) (err error)
// SuspendNodeUnknownAudit suspends a storage node for unknown audits. // SuspendNodeUnknownAudit suspends a storage node for unknown audits.
SuspendNodeUnknownAudit(ctx context.Context, nodeID storj.NodeID, suspendedAt time.Time) (err error) SuspendNodeUnknownAudit(ctx context.Context, nodeID storj.NodeID, suspendedAt time.Time) (err error)
// UpdateAuditHistory updates a node's audit history // UpdateAuditHistory updates a node's audit history
@ -33,10 +33,10 @@ type Info struct {
AuditSuccessCount int64 AuditSuccessCount int64
TotalAuditCount int64 TotalAuditCount int64
VettedAt *time.Time VettedAt *time.Time
Disqualified *time.Time
UnknownAuditSuspended *time.Time UnknownAuditSuspended *time.Time
OfflineSuspended *time.Time OfflineSuspended *time.Time
UnderReview *time.Time UnderReview *time.Time
Disqualified *time.Time
OnlineScore float64 OnlineScore float64
AuditHistory AuditHistory AuditHistory AuditHistory
AuditReputationAlpha float64 AuditReputationAlpha float64
@ -92,7 +92,7 @@ func (service *Service) ApplyAudit(ctx context.Context, nodeID storj.NodeID, rep
// Due to inconsistencies in the precision of time.Now() on different platforms and databases, the time comparison // Due to inconsistencies in the precision of time.Now() on different platforms and databases, the time comparison
// for the VettedAt status is done using time values that are truncated to second precision. // for the VettedAt status is done using time values that are truncated to second precision.
if hasReputationChanged(*statusUpdate, reputation, now) { if hasReputationChanged(*statusUpdate, reputation, now) {
err = service.overlay.UpdateReputation(ctx, nodeID, statusUpdate) err = service.overlay.UpdateReputation(ctx, nodeID, *statusUpdate)
if err != nil { if err != nil {
return err return err
} }
@ -138,7 +138,7 @@ func (service *Service) TestSuspendNodeUnknownAudit(ctx context.Context, nodeID
// TestDisqualifyNode disqualifies a storage node. // TestDisqualifyNode disqualifies a storage node.
func (service *Service) TestDisqualifyNode(ctx context.Context, nodeID storj.NodeID) (err error) { func (service *Service) TestDisqualifyNode(ctx context.Context, nodeID storj.NodeID) (err error) {
err = service.db.DisqualifyNode(ctx, nodeID) err = service.db.DisqualifyNode(ctx, nodeID, time.Now())
if err != nil { if err != nil {
return err return err
} }
@ -161,7 +161,7 @@ func (service *Service) Close() error { return nil }
// hasReputationChanged determines if the current node reputation is different from the newly updated reputation. This // hasReputationChanged determines if the current node reputation is different from the newly updated reputation. This
// function will only consider the Disqualified, UnknownAudiSuspended and OfflineSuspended statuses for changes. // function will only consider the Disqualified, UnknownAudiSuspended and OfflineSuspended statuses for changes.
func hasReputationChanged(updated, current overlay.ReputationStatus, now time.Time) bool { func hasReputationChanged(updated overlay.ReputationUpdate, current overlay.ReputationStatus, now time.Time) bool {
if statusChanged(current.Disqualified, updated.Disqualified) || if statusChanged(current.Disqualified, updated.Disqualified) ||
statusChanged(current.UnknownAuditSuspended, updated.UnknownAuditSuspended) || statusChanged(current.UnknownAuditSuspended, updated.UnknownAuditSuspended) ||
statusChanged(current.OfflineSuspended, updated.OfflineSuspended) { statusChanged(current.OfflineSuspended, updated.OfflineSuspended) {

View File

@ -6,6 +6,7 @@ package reputation_test
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"go.uber.org/zap" "go.uber.org/zap"
"golang.org/x/sync/errgroup" "golang.org/x/sync/errgroup"
@ -144,3 +145,44 @@ func TestGet(t *testing.T) {
require.EqualValues(t, 1, newNode.OnlineScore) require.EqualValues(t, 1, newNode.OnlineScore)
}) })
} }
func TestDisqualificationAuditFailure(t *testing.T) {
testplanet.Run(t, testplanet.Config{
SatelliteCount: 1, StorageNodeCount: 1, UplinkCount: 0,
Reconfigure: testplanet.Reconfigure{
Satellite: func(log *zap.Logger, index int, config *satellite.Config) {
config.Reputation.AuditLambda = 1
config.Reputation.AuditWeight = 1
config.Reputation.AuditDQ = 0.4
},
},
}, func(t *testing.T, ctx *testcontext.Context, planet *testplanet.Planet) {
satel := planet.Satellites[0]
nodeID := planet.StorageNodes[0].ID()
nodeInfo, err := satel.Overlay.Service.Get(ctx, nodeID)
require.NoError(t, err)
assert.Nil(t, nodeInfo.Disqualified)
err = satel.Reputation.Service.ApplyAudit(ctx, nodeID, nodeInfo.Reputation.Status, reputation.AuditFailure)
require.NoError(t, err)
// node is not disqualified after failed audit if score is above threshold
repInfo, err := satel.Reputation.Service.Get(ctx, nodeID)
require.NoError(t, err)
assert.Nil(t, repInfo.Disqualified)
nodeInfo, err = satel.Overlay.Service.Get(ctx, nodeID)
require.NoError(t, err)
assert.Nil(t, nodeInfo.Disqualified)
err = satel.Reputation.Service.ApplyAudit(ctx, nodeID, nodeInfo.Reputation.Status, reputation.AuditFailure)
require.NoError(t, err)
repInfo, err = satel.Reputation.Service.Get(ctx, nodeID)
require.NoError(t, err)
assert.NotNil(t, repInfo.Disqualified)
nodeInfo, err = satel.Overlay.Service.Get(ctx, nodeID)
require.NoError(t, err)
assert.NotNil(t, nodeInfo.Disqualified)
})
}

View File

@ -636,8 +636,8 @@ func (cache *overlaycache) reliable(ctx context.Context, criteria *overlay.NodeC
return nodes, Error.Wrap(rows.Err()) return nodes, Error.Wrap(rows.Err())
} }
// UpdateReputation updates the DB columns for any of the reputation fields in UpdateReputationRequest. // UpdateReputation updates the DB columns for any of the reputation fields in ReputationUpdate.
func (cache *overlaycache) UpdateReputation(ctx context.Context, id storj.NodeID, request *overlay.ReputationStatus) (err error) { func (cache *overlaycache) UpdateReputation(ctx context.Context, id storj.NodeID, request overlay.ReputationUpdate) (err error) {
defer mon.Task()(&ctx)(&err) defer mon.Task()(&ctx)(&err)
updateFields := dbx.Node_Update_Fields{} updateFields := dbx.Node_Update_Fields{}

View File

@ -34,7 +34,7 @@ type reputations struct {
// 2. Depends on the result of the first step, // 2. Depends on the result of the first step,
// a. if existing row is returned, do compare-and-swap. // a. if existing row is returned, do compare-and-swap.
// b. if no row found, insert a new row. // b. if no row found, insert a new row.
func (reputations *reputations) Update(ctx context.Context, updateReq reputation.UpdateRequest, now time.Time) (_ *overlay.ReputationStatus, err error) { func (reputations *reputations) Update(ctx context.Context, updateReq reputation.UpdateRequest, now time.Time) (_ *overlay.ReputationUpdate, err error) {
defer mon.Task()(&ctx)(&err) defer mon.Task()(&ctx)(&err)
for { for {
@ -51,21 +51,23 @@ func (reputations *reputations) Update(ctx context.Context, updateReq reputation
return nil, Error.Wrap(err) return nil, Error.Wrap(err)
} }
auditHistoryResponse, err := reputations.UpdateAuditHistory(ctx, historyBytes, updateReq, now)
if err != nil {
return nil, Error.Wrap(err)
}
// set default reputation stats for new node // set default reputation stats for new node
newNode := dbx.Reputation{ newNode := dbx.Reputation{
Id: updateReq.NodeID.Bytes(), Id: updateReq.NodeID.Bytes(),
UnknownAuditReputationAlpha: 1, UnknownAuditReputationAlpha: 1,
AuditReputationAlpha: 1, AuditReputationAlpha: 1,
OnlineScore: 1, OnlineScore: 1,
AuditHistory: auditHistoryResponse.History, AuditHistory: historyBytes,
} }
createFields := reputations.populateCreateFields(&newNode, updateReq, auditHistoryResponse, now) auditHistoryResponse, err := reputations.UpdateAuditHistory(ctx, historyBytes, updateReq, now)
if err != nil {
return nil, Error.Wrap(err)
}
update := reputations.populateUpdateNodeStats(&newNode, updateReq, auditHistoryResponse, now)
createFields := reputations.populateCreateFields(update)
stats, err := reputations.db.Create_Reputation(ctx, dbx.Reputation_Id(updateReq.NodeID.Bytes()), dbx.Reputation_AuditHistory(auditHistoryResponse.History), createFields) stats, err := reputations.db.Create_Reputation(ctx, dbx.Reputation_Id(updateReq.NodeID.Bytes()), dbx.Reputation_AuditHistory(auditHistoryResponse.History), createFields)
if err != nil { if err != nil {
// if node has been added into the table during a concurrent // if node has been added into the table during a concurrent
@ -79,8 +81,15 @@ func (reputations *reputations) Update(ctx context.Context, updateReq reputation
return nil, Error.Wrap(err) return nil, Error.Wrap(err)
} }
rep := getNodeStatus(stats) status := getNodeStatus(stats)
return &rep, nil repUpdate := overlay.ReputationUpdate{
Disqualified: status.Disqualified,
DisqualificationReason: update.DisqualificationReason,
UnknownAuditSuspended: status.UnknownAuditSuspended,
OfflineSuspended: status.OfflineSuspended,
VettedAt: status.VettedAt,
}
return &repUpdate, nil
} }
auditHistoryResponse, err := reputations.UpdateAuditHistory(ctx, dbNode.AuditHistory, updateReq, now) auditHistoryResponse, err := reputations.UpdateAuditHistory(ctx, dbNode.AuditHistory, updateReq, now)
@ -88,7 +97,9 @@ func (reputations *reputations) Update(ctx context.Context, updateReq reputation
return nil, Error.Wrap(err) return nil, Error.Wrap(err)
} }
updateFields := reputations.populateUpdateFields(dbNode, updateReq, auditHistoryResponse, now) update := reputations.populateUpdateNodeStats(dbNode, updateReq, auditHistoryResponse, now)
updateFields := reputations.populateUpdateFields(update, auditHistoryResponse.History)
oldAuditHistory := dbx.Reputation_AuditHistory(dbNode.AuditHistory) oldAuditHistory := dbx.Reputation_AuditHistory(dbNode.AuditHistory)
dbNode, err = reputations.db.Update_Reputation_By_Id_And_AuditHistory(ctx, dbx.Reputation_Id(updateReq.NodeID.Bytes()), oldAuditHistory, updateFields) dbNode, err = reputations.db.Update_Reputation_By_Id_And_AuditHistory(ctx, dbx.Reputation_Id(updateReq.NodeID.Bytes()), oldAuditHistory, updateFields)
if err != nil && !errors.Is(err, sql.ErrNoRows) { if err != nil && !errors.Is(err, sql.ErrNoRows) {
@ -102,10 +113,16 @@ func (reputations *reputations) Update(ctx context.Context, updateReq reputation
continue continue
} }
newStats := getNodeStatus(dbNode) status := getNodeStatus(dbNode)
return &newStats, nil repUpdate := overlay.ReputationUpdate{
Disqualified: status.Disqualified,
DisqualificationReason: update.DisqualificationReason,
UnknownAuditSuspended: status.UnknownAuditSuspended,
OfflineSuspended: status.OfflineSuspended,
VettedAt: status.VettedAt,
}
return &repUpdate, nil
} }
} }
func (reputations *reputations) Get(ctx context.Context, nodeID storj.NodeID) (*reputation.Info, error) { func (reputations *reputations) Get(ctx context.Context, nodeID storj.NodeID) (*reputation.Info, error) {
@ -126,10 +143,10 @@ func (reputations *reputations) Get(ctx context.Context, nodeID storj.NodeID) (*
AuditSuccessCount: res.AuditSuccessCount, AuditSuccessCount: res.AuditSuccessCount,
TotalAuditCount: res.TotalAuditCount, TotalAuditCount: res.TotalAuditCount,
VettedAt: res.VettedAt, VettedAt: res.VettedAt,
Disqualified: res.Disqualified,
UnknownAuditSuspended: res.UnknownAuditSuspended, UnknownAuditSuspended: res.UnknownAuditSuspended,
OfflineSuspended: res.OfflineSuspended, OfflineSuspended: res.OfflineSuspended,
UnderReview: res.UnderReview, UnderReview: res.UnderReview,
Disqualified: res.Disqualified,
OnlineScore: res.OnlineScore, OnlineScore: res.OnlineScore,
AuditHistory: *history, AuditHistory: *history,
AuditReputationAlpha: res.AuditReputationAlpha, AuditReputationAlpha: res.AuditReputationAlpha,
@ -140,7 +157,7 @@ func (reputations *reputations) Get(ctx context.Context, nodeID storj.NodeID) (*
} }
// DisqualifyNode disqualifies a storage node. // DisqualifyNode disqualifies a storage node.
func (reputations *reputations) DisqualifyNode(ctx context.Context, nodeID storj.NodeID) (err error) { func (reputations *reputations) DisqualifyNode(ctx context.Context, nodeID storj.NodeID, disqualifiedAt time.Time) (err error) {
defer mon.Task()(&ctx)(&err) defer mon.Task()(&ctx)(&err)
err = reputations.db.WithTx(ctx, func(ctx context.Context, tx *dbx.Tx) (err error) { err = reputations.db.WithTx(ctx, func(ctx context.Context, tx *dbx.Tx) (err error) {
@ -169,7 +186,7 @@ func (reputations *reputations) DisqualifyNode(ctx context.Context, nodeID storj
} }
updateFields := dbx.Reputation_Update_Fields{} updateFields := dbx.Reputation_Update_Fields{}
updateFields.Disqualified = dbx.Reputation_Disqualified(time.Now().UTC()) updateFields.Disqualified = dbx.Reputation_Disqualified(disqualifiedAt.UTC())
_, err = tx.Update_Reputation_By_Id(ctx, dbx.Reputation_Id(nodeID.Bytes()), updateFields) _, err = tx.Update_Reputation_By_Id(ctx, dbx.Reputation_Id(nodeID.Bytes()), updateFields)
return err return err
@ -254,9 +271,7 @@ func (reputations *reputations) UnsuspendNodeUnknownAudit(ctx context.Context, n
return Error.Wrap(err) return Error.Wrap(err)
} }
func (reputations *reputations) populateCreateFields(dbNode *dbx.Reputation, updateReq reputation.UpdateRequest, auditHistoryResponse *reputation.UpdateAuditHistoryResponse, now time.Time) dbx.Reputation_Create_Fields { func (reputations *reputations) populateCreateFields(update updateNodeStats) dbx.Reputation_Create_Fields {
update := reputations.populateUpdateNodeStats(dbNode, updateReq, auditHistoryResponse, now)
createFields := dbx.Reputation_Create_Fields{} createFields := dbx.Reputation_Create_Fields{}
if update.VettedAt.set { if update.VettedAt.set {
@ -290,9 +305,6 @@ func (reputations *reputations) populateCreateFields(dbNode *dbx.Reputation, upd
if update.AuditSuccessCount.set { if update.AuditSuccessCount.set {
createFields.AuditSuccessCount = dbx.Reputation_AuditSuccessCount(update.AuditSuccessCount.value) createFields.AuditSuccessCount = dbx.Reputation_AuditSuccessCount(update.AuditSuccessCount.value)
} }
if updateReq.AuditOutcome == reputation.AuditSuccess {
createFields.AuditSuccessCount = dbx.Reputation_AuditSuccessCount(dbNode.AuditSuccessCount + 1)
}
if update.OnlineScore.set { if update.OnlineScore.set {
createFields.OnlineScore = dbx.Reputation_OnlineScore(update.OnlineScore.value) createFields.OnlineScore = dbx.Reputation_OnlineScore(update.OnlineScore.value)
@ -314,11 +326,9 @@ func (reputations *reputations) populateCreateFields(dbNode *dbx.Reputation, upd
return createFields return createFields
} }
func (reputations *reputations) populateUpdateFields(dbNode *dbx.Reputation, updateReq reputation.UpdateRequest, auditHistoryResponse *reputation.UpdateAuditHistoryResponse, now time.Time) dbx.Reputation_Update_Fields { func (reputations *reputations) populateUpdateFields(update updateNodeStats, history []byte) dbx.Reputation_Update_Fields {
update := reputations.populateUpdateNodeStats(dbNode, updateReq, auditHistoryResponse, now)
updateFields := dbx.Reputation_Update_Fields{ updateFields := dbx.Reputation_Update_Fields{
AuditHistory: dbx.Reputation_AuditHistory(auditHistoryResponse.History), AuditHistory: dbx.Reputation_AuditHistory(history),
} }
if update.VettedAt.set { if update.VettedAt.set {
updateFields.VettedAt = dbx.Reputation_VettedAt(update.VettedAt.value) updateFields.VettedAt = dbx.Reputation_VettedAt(update.VettedAt.value)
@ -351,9 +361,6 @@ func (reputations *reputations) populateUpdateFields(dbNode *dbx.Reputation, upd
if update.AuditSuccessCount.set { if update.AuditSuccessCount.set {
updateFields.AuditSuccessCount = dbx.Reputation_AuditSuccessCount(update.AuditSuccessCount.value) updateFields.AuditSuccessCount = dbx.Reputation_AuditSuccessCount(update.AuditSuccessCount.value)
} }
if updateReq.AuditOutcome == reputation.AuditSuccess {
updateFields.AuditSuccessCount = dbx.Reputation_AuditSuccessCount(dbNode.AuditSuccessCount + 1)
}
if update.OnlineScore.set { if update.OnlineScore.set {
updateFields.OnlineScore = dbx.Reputation_OnlineScore(update.OnlineScore.value) updateFields.OnlineScore = dbx.Reputation_OnlineScore(update.OnlineScore.value)
@ -467,6 +474,7 @@ func (reputations *reputations) populateUpdateNodeStats(dbNode *dbx.Reputation,
reputations.db.log.Info("Disqualified", zap.String("DQ type", "audit failure"), zap.String("Node ID", updateReq.NodeID.String())) reputations.db.log.Info("Disqualified", zap.String("DQ type", "audit failure"), zap.String("Node ID", updateReq.NodeID.String()))
mon.Meter("bad_audit_dqs").Mark(1) //mon:locked mon.Meter("bad_audit_dqs").Mark(1) //mon:locked
updateFields.Disqualified = timeField{set: true, value: now} updateFields.Disqualified = timeField{set: true, value: now}
updateFields.DisqualificationReason = overlay.DisqualificationReasonAuditFailure
} }
// if unknown audit rep goes below threshold, suspend node. Otherwise unsuspend node. // if unknown audit rep goes below threshold, suspend node. Otherwise unsuspend node.
@ -492,6 +500,7 @@ func (reputations *reputations) populateUpdateNodeStats(dbNode *dbx.Reputation,
reputations.db.log.Info("Disqualified", zap.String("DQ type", "suspension grace period expired for unknown audits"), zap.String("Node ID", updateReq.NodeID.String())) reputations.db.log.Info("Disqualified", zap.String("DQ type", "suspension grace period expired for unknown audits"), zap.String("Node ID", updateReq.NodeID.String()))
mon.Meter("unknown_suspension_dqs").Mark(1) //mon:locked mon.Meter("unknown_suspension_dqs").Mark(1) //mon:locked
updateFields.Disqualified = timeField{set: true, value: now} updateFields.Disqualified = timeField{set: true, value: now}
updateFields.DisqualificationReason = overlay.DisqualificationReasonSuspension
updateFields.UnknownAuditSuspended = timeField{set: true, isNil: true} updateFields.UnknownAuditSuspended = timeField{set: true, isNil: true}
} }
} }
@ -549,6 +558,7 @@ func (reputations *reputations) populateUpdateNodeStats(dbNode *dbx.Reputation,
reputations.db.log.Info("Disqualified", zap.String("DQ type", "node offline"), zap.String("Node ID", updateReq.NodeID.String())) reputations.db.log.Info("Disqualified", zap.String("DQ type", "node offline"), zap.String("Node ID", updateReq.NodeID.String()))
mon.Meter("offline_dqs").Mark(1) //mon:locked mon.Meter("offline_dqs").Mark(1) //mon:locked
updateFields.Disqualified = timeField{set: true, value: now} updateFields.Disqualified = timeField{set: true, value: now}
updateFields.DisqualificationReason = overlay.DisqualificationReasonNodeOffline
} }
} else { } else {
updateFields.OfflineUnderReview = timeField{set: true, isNil: true} updateFields.OfflineUnderReview = timeField{set: true, isNil: true}
@ -592,6 +602,7 @@ type updateNodeStats struct {
AuditReputationAlpha float64Field AuditReputationAlpha float64Field
AuditReputationBeta float64Field AuditReputationBeta float64Field
Disqualified timeField Disqualified timeField
DisqualificationReason overlay.DisqualificationReason
UnknownAuditReputationAlpha float64Field UnknownAuditReputationAlpha float64Field
UnknownAuditReputationBeta float64Field UnknownAuditReputationBeta float64Field
UnknownAuditSuspended timeField UnknownAuditSuspended timeField
@ -611,7 +622,6 @@ func getNodeStatus(dbNode *dbx.Reputation) overlay.ReputationStatus {
UnknownAuditSuspended: dbNode.UnknownAuditSuspended, UnknownAuditSuspended: dbNode.UnknownAuditSuspended,
OfflineSuspended: dbNode.OfflineSuspended, OfflineSuspended: dbNode.OfflineSuspended,
} }
} }
// updateReputation uses the Beta distribution model to determine a node's reputation. // updateReputation uses the Beta distribution model to determine a node's reputation.