JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

satellite/satellitedb,private: add initial value on testplanet startup

Browse Source 
Currently, reputation table is only populated when a node has been
audited. This is ok in production, however a lot of our tests doesn't
upload any data or trigger audits.
This PR adds an initialization step in testplanet to populate reputation
table with zero value for nodes reputation.

Change-Id: I11b381236669db346dc68a48a6d4a27334a0a8b8
This commit is contained in:
Yingrong Zhao 2021-07-12 17:12:11 -04:00
parent 6c7bf357cd
commit 55a77d04bc
6 changed files with 240 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
private/testplanet/planet.go
View File

@ -213,6 +213,19 @@ func (planet *Planet) Start(ctx context.Context) {
})
})
}
if !planet.config.Reconfigure.DisableInitReputationDB {
for _, peer := range planet.StorageNodes {
peer := peer
for _, sat := range planet.Satellites {
sat := sat
pprof.Do(ctx, pprof.Labels("peer", peer.Label(), "startup", "reputation"), func(ctx context.Context) {
group.Go(func() error {
return sat.DB.Reputation().Init(ctx, peer.ID())
})
})
}
}
}
_ = group.Wait()
planet.started = true

7
private/testplanet/reconfigure.go
View File

@ -23,9 +23,10 @@ type Reconfigure struct {
SatelliteMetabaseDB func(log *zap.Logger, index int, db *metabase.DB) (*metabase.DB, error)
Satellite func(log *zap.Logger, index int, config *satellite.Config)
StorageNodeDB func(index int, db storagenode.DB, log *zap.Logger) (storagenode.DB, error)
StorageNode func(index int, config *storagenode.Config)
UniqueIPCount int
StorageNodeDB func(index int, db storagenode.DB, log *zap.Logger) (storagenode.DB, error)
StorageNode func(index int, config *storagenode.Config)
UniqueIPCount int
DisableInitReputationDB bool
VersionControl func(config *versioncontrol.Config)

1
satellite/reputation/service.go
View File

@ -21,6 +21,7 @@ type DB interface {
Update(ctx context.Context, request UpdateRequest, now time.Time) (_ *overlay.ReputationStatus, changed bool, err error)
SetNodeStatus(ctx context.Context, id storj.NodeID, status overlay.ReputationStatus) error
Get(ctx context.Context, nodeID storj.NodeID) (*Info, error)
Init(ctx context.Context, nodeID storj.NodeID) error
// UnsuspendNodeUnknownAudit unsuspends a storage node for unknown audits.
UnsuspendNodeUnknownAudit(ctx context.Context, nodeID storj.NodeID) (_ *overlay.ReputationStatus, err error)

2
satellite/satellitedb/dbx/satellitedb.dbx
View File

@ -291,6 +291,8 @@ model reputation (
field unknown_audit_reputation_beta float64 ( updatable, default 0 )
)
create reputation ()
update reputation ( where reputation.id = ? )
update reputation (
where reputation.id = ?

209
satellite/satellitedb/dbx/satellitedb.dbx.go
View File

@ -10969,6 +10969,101 @@ func (obj *pgxImpl) Create_AuditHistory(ctx context.Context,
}
func (obj *pgxImpl) Create_Reputation(ctx context.Context,
reputation_id Reputation_Id_Field,
reputation_audit_history Reputation_AuditHistory_Field,
optional Reputation_Create_Fields) (
reputation *Reputation, err error) {
defer mon.Task()(&ctx)(&err)
__id_val := reputation_id.value()
__vetted_at_val := optional.VettedAt.value()
__disqualified_val := optional.Disqualified.value()
__suspended_val := optional.Suspended.value()
__unknown_audit_suspended_val := optional.UnknownAuditSuspended.value()
__offline_suspended_val := optional.OfflineSuspended.value()
__under_review_val := optional.UnderReview.value()
__audit_history_val := reputation_audit_history.value()
var __columns = &__sqlbundle_Hole{SQL: __sqlbundle_Literal("id, vetted_at, disqualified, suspended, unknown_audit_suspended, offline_suspended, under_review, audit_history")}
var __placeholders = &__sqlbundle_Hole{SQL: __sqlbundle_Literal("?, ?, ?, ?, ?, ?, ?, ?")}
var __clause = &__sqlbundle_Hole{SQL: __sqlbundle_Literals{Join: "", SQLs: []__sqlbundle_SQL{__sqlbundle_Literal("("), __columns, __sqlbundle_Literal(") VALUES ("), __placeholders, __sqlbundle_Literal(")")}}}
var __embed_stmt = __sqlbundle_Literals{Join: "", SQLs: []__sqlbundle_SQL{__sqlbundle_Literal("INSERT INTO reputations "), __clause, __sqlbundle_Literal(" RETURNING reputations.id, reputations.audit_success_count, reputations.total_audit_count, reputations.vetted_at, reputations.created_at, reputations.updated_at, reputations.contained, reputations.disqualified, reputations.suspended, reputations.unknown_audit_suspended, reputations.offline_suspended, reputations.under_review, reputations.online_score, reputations.audit_history, reputations.audit_reputation_alpha, reputations.audit_reputation_beta, reputations.unknown_audit_reputation_alpha, reputations.unknown_audit_reputation_beta")}}
var __values []interface{}
__values = append(__values, __id_val, __vetted_at_val, __disqualified_val, __suspended_val, __unknown_audit_suspended_val, __offline_suspended_val, __under_review_val, __audit_history_val)
__optional_columns := __sqlbundle_Literals{Join: ", "}
__optional_placeholders := __sqlbundle_Literals{Join: ", "}
if optional.AuditSuccessCount._set {
__values = append(__values, optional.AuditSuccessCount.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_success_count"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.TotalAuditCount._set {
__values = append(__values, optional.TotalAuditCount.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("total_audit_count"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.Contained._set {
__values = append(__values, optional.Contained.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("contained"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.OnlineScore._set {
__values = append(__values, optional.OnlineScore.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("online_score"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.AuditReputationAlpha._set {
__values = append(__values, optional.AuditReputationAlpha.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_reputation_alpha"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.AuditReputationBeta._set {
__values = append(__values, optional.AuditReputationBeta.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_reputation_beta"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.UnknownAuditReputationAlpha._set {
__values = append(__values, optional.UnknownAuditReputationAlpha.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("unknown_audit_reputation_alpha"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.UnknownAuditReputationBeta._set {
__values = append(__values, optional.UnknownAuditReputationBeta.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("unknown_audit_reputation_beta"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if len(__optional_columns.SQLs) == 0 {
if __columns.SQL == nil {
__clause.SQL = __sqlbundle_Literal("DEFAULT VALUES")
}
} else {
__columns.SQL = __sqlbundle_Literals{Join: ", ", SQLs: []__sqlbundle_SQL{__columns.SQL, __optional_columns}}
__placeholders.SQL = __sqlbundle_Literals{Join: ", ", SQLs: []__sqlbundle_SQL{__placeholders.SQL, __optional_placeholders}}
}
var __stmt = __sqlbundle_Render(obj.dialect, __embed_stmt)
obj.logStmt(__stmt, __values...)
reputation = &Reputation{}
err = obj.queryRowContext(ctx, __stmt, __values...).Scan(&reputation.Id, &reputation.AuditSuccessCount, &reputation.TotalAuditCount, &reputation.VettedAt, &reputation.CreatedAt, &reputation.UpdatedAt, &reputation.Contained, &reputation.Disqualified, &reputation.Suspended, &reputation.UnknownAuditSuspended, &reputation.OfflineSuspended, &reputation.UnderReview, &reputation.OnlineScore, &reputation.AuditHistory, &reputation.AuditReputationAlpha, &reputation.AuditReputationBeta, &reputation.UnknownAuditReputationAlpha, &reputation.UnknownAuditReputationBeta)
if err != nil {
return nil, obj.makeErr(err)
}
return reputation, nil
}
func (obj *pgxImpl) Create_User(ctx context.Context,
user_id User_Id_Field,
user_email User_Email_Field,
@ -17019,6 +17114,101 @@ func (obj *pgxcockroachImpl) Create_AuditHistory(ctx context.Context,
}
func (obj *pgxcockroachImpl) Create_Reputation(ctx context.Context,
reputation_id Reputation_Id_Field,
reputation_audit_history Reputation_AuditHistory_Field,
optional Reputation_Create_Fields) (
reputation *Reputation, err error) {
defer mon.Task()(&ctx)(&err)
__id_val := reputation_id.value()
__vetted_at_val := optional.VettedAt.value()
__disqualified_val := optional.Disqualified.value()
__suspended_val := optional.Suspended.value()
__unknown_audit_suspended_val := optional.UnknownAuditSuspended.value()
__offline_suspended_val := optional.OfflineSuspended.value()
__under_review_val := optional.UnderReview.value()
__audit_history_val := reputation_audit_history.value()
var __columns = &__sqlbundle_Hole{SQL: __sqlbundle_Literal("id, vetted_at, disqualified, suspended, unknown_audit_suspended, offline_suspended, under_review, audit_history")}
var __placeholders = &__sqlbundle_Hole{SQL: __sqlbundle_Literal("?, ?, ?, ?, ?, ?, ?, ?")}
var __clause = &__sqlbundle_Hole{SQL: __sqlbundle_Literals{Join: "", SQLs: []__sqlbundle_SQL{__sqlbundle_Literal("("), __columns, __sqlbundle_Literal(") VALUES ("), __placeholders, __sqlbundle_Literal(")")}}}
var __embed_stmt = __sqlbundle_Literals{Join: "", SQLs: []__sqlbundle_SQL{__sqlbundle_Literal("INSERT INTO reputations "), __clause, __sqlbundle_Literal(" RETURNING reputations.id, reputations.audit_success_count, reputations.total_audit_count, reputations.vetted_at, reputations.created_at, reputations.updated_at, reputations.contained, reputations.disqualified, reputations.suspended, reputations.unknown_audit_suspended, reputations.offline_suspended, reputations.under_review, reputations.online_score, reputations.audit_history, reputations.audit_reputation_alpha, reputations.audit_reputation_beta, reputations.unknown_audit_reputation_alpha, reputations.unknown_audit_reputation_beta")}}
var __values []interface{}
__values = append(__values, __id_val, __vetted_at_val, __disqualified_val, __suspended_val, __unknown_audit_suspended_val, __offline_suspended_val, __under_review_val, __audit_history_val)
__optional_columns := __sqlbundle_Literals{Join: ", "}
__optional_placeholders := __sqlbundle_Literals{Join: ", "}
if optional.AuditSuccessCount._set {
__values = append(__values, optional.AuditSuccessCount.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_success_count"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.TotalAuditCount._set {
__values = append(__values, optional.TotalAuditCount.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("total_audit_count"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.Contained._set {
__values = append(__values, optional.Contained.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("contained"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.OnlineScore._set {
__values = append(__values, optional.OnlineScore.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("online_score"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.AuditReputationAlpha._set {
__values = append(__values, optional.AuditReputationAlpha.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_reputation_alpha"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.AuditReputationBeta._set {
__values = append(__values, optional.AuditReputationBeta.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_reputation_beta"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.UnknownAuditReputationAlpha._set {
__values = append(__values, optional.UnknownAuditReputationAlpha.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("unknown_audit_reputation_alpha"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if optional.UnknownAuditReputationBeta._set {
__values = append(__values, optional.UnknownAuditReputationBeta.value())
__optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("unknown_audit_reputation_beta"))
__optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?"))
}
if len(__optional_columns.SQLs) == 0 {
if __columns.SQL == nil {
__clause.SQL = __sqlbundle_Literal("DEFAULT VALUES")
}
} else {
__columns.SQL = __sqlbundle_Literals{Join: ", ", SQLs: []__sqlbundle_SQL{__columns.SQL, __optional_columns}}
__placeholders.SQL = __sqlbundle_Literals{Join: ", ", SQLs: []__sqlbundle_SQL{__placeholders.SQL, __optional_placeholders}}
}
var __stmt = __sqlbundle_Render(obj.dialect, __embed_stmt)
obj.logStmt(__stmt, __values...)
reputation = &Reputation{}
err = obj.queryRowContext(ctx, __stmt, __values...).Scan(&reputation.Id, &reputation.AuditSuccessCount, &reputation.TotalAuditCount, &reputation.VettedAt, &reputation.CreatedAt, &reputation.UpdatedAt, &reputation.Contained, &reputation.Disqualified, &reputation.Suspended, &reputation.UnknownAuditSuspended, &reputation.OfflineSuspended, &reputation.UnderReview, &reputation.OnlineScore, &reputation.AuditHistory, &reputation.AuditReputationAlpha, &reputation.AuditReputationBeta, &reputation.UnknownAuditReputationAlpha, &reputation.UnknownAuditReputationBeta)
if err != nil {
return nil, obj.makeErr(err)
}
return reputation, nil
}
func (obj *pgxcockroachImpl) Create_User(ctx context.Context,
user_id User_Id_Field,
user_email User_Email_Field,
@ -23427,6 +23617,19 @@ func (rx *Rx) Create_RegistrationToken(ctx context.Context,
}
func (rx *Rx) Create_Reputation(ctx context.Context,
reputation_id Reputation_Id_Field,
reputation_audit_history Reputation_AuditHistory_Field,
optional Reputation_Create_Fields) (
reputation *Reputation, err error) {
var tx *Tx
if tx, err = rx.getTx(ctx); err != nil {
return
}
return tx.Create_Reputation(ctx, reputation_id, reputation_audit_history, optional)
}
func (rx *Rx) Create_ResetPasswordToken(ctx context.Context,
reset_password_token_secret ResetPasswordToken_Secret_Field,
reset_password_token_owner_id ResetPasswordToken_OwnerId_Field) (
@ -24778,6 +24981,12 @@ type Methods interface {
optional RegistrationToken_Create_Fields) (
registration_token *RegistrationToken, err error)
Create_Reputation(ctx context.Context,
reputation_id Reputation_Id_Field,
reputation_audit_history Reputation_AuditHistory_Field,
optional Reputation_Create_Fields) (
reputation *Reputation, err error)
Create_ResetPasswordToken(ctx context.Context,
reset_password_token_secret ResetPasswordToken_Secret_Field,
reset_password_token_owner_id ResetPasswordToken_OwnerId_Field) (

11
satellite/satellitedb/reputations.go
View File

@ -369,6 +369,17 @@ func (reputations *reputations) UnsuspendNodeUnknownAudit(ctx context.Context, n
}, nil
}
// Init creates an entry for a node with its reputaion in zero value.
func (reputations *reputations) Init(ctx context.Context, nodeID storj.NodeID) error {
historyBytes, err := pb.Marshal(&internalpb.AuditHistory{})
if err != nil {
return err
}
_, err = reputations.db.Create_Reputation(ctx, dbx.Reputation_Id(nodeID.Bytes()), dbx.Reputation_AuditHistory(historyBytes), dbx.Reputation_Create_Fields{})
return Error.Wrap(err)
}
func (reputations *reputations) populateUpdateFields(dbNode *dbx.Reputation, updateReq reputation.UpdateRequest, auditHistoryResponse *reputation.UpdateAuditHistoryResponse, now time.Time) dbx.Reputation_Update_Fields {
update := reputations.populateUpdateNodeStats(dbNode, updateReq, auditHistoryResponse, now)