From 55a77d04bc0e5bd87d7495473f2f72b600cd010d Mon Sep 17 00:00:00 2001 From: Yingrong Zhao Date: Mon, 12 Jul 2021 17:12:11 -0400 Subject: [PATCH] satellite/satellitedb,private: add initial value on testplanet startup Currently, reputation table is only populated when a node has been audited. This is ok in production, however a lot of our tests doesn't upload any data or trigger audits. This PR adds an initialization step in testplanet to populate reputation table with zero value for nodes reputation. Change-Id: I11b381236669db346dc68a48a6d4a27334a0a8b8 --- private/testplanet/planet.go | 13 ++ private/testplanet/reconfigure.go | 7 +- satellite/reputation/service.go | 1 + satellite/satellitedb/dbx/satellitedb.dbx | 2 + satellite/satellitedb/dbx/satellitedb.dbx.go | 209 +++++++++++++++++++ satellite/satellitedb/reputations.go | 11 + 6 files changed, 240 insertions(+), 3 deletions(-) diff --git a/private/testplanet/planet.go b/private/testplanet/planet.go index 812a19a24..adf7ed9a0 100644 --- a/private/testplanet/planet.go +++ b/private/testplanet/planet.go @@ -213,6 +213,19 @@ func (planet *Planet) Start(ctx context.Context) { }) }) } + if !planet.config.Reconfigure.DisableInitReputationDB { + for _, peer := range planet.StorageNodes { + peer := peer + for _, sat := range planet.Satellites { + sat := sat + pprof.Do(ctx, pprof.Labels("peer", peer.Label(), "startup", "reputation"), func(ctx context.Context) { + group.Go(func() error { + return sat.DB.Reputation().Init(ctx, peer.ID()) + }) + }) + } + } + } _ = group.Wait() planet.started = true diff --git a/private/testplanet/reconfigure.go b/private/testplanet/reconfigure.go index 019867f17..fd0cef093 100644 --- a/private/testplanet/reconfigure.go +++ b/private/testplanet/reconfigure.go @@ -23,9 +23,10 @@ type Reconfigure struct { SatelliteMetabaseDB func(log *zap.Logger, index int, db *metabase.DB) (*metabase.DB, error) Satellite func(log *zap.Logger, index int, config *satellite.Config) - StorageNodeDB func(index int, db storagenode.DB, log *zap.Logger) (storagenode.DB, error) - StorageNode func(index int, config *storagenode.Config) - UniqueIPCount int + StorageNodeDB func(index int, db storagenode.DB, log *zap.Logger) (storagenode.DB, error) + StorageNode func(index int, config *storagenode.Config) + UniqueIPCount int + DisableInitReputationDB bool VersionControl func(config *versioncontrol.Config) diff --git a/satellite/reputation/service.go b/satellite/reputation/service.go index aad23cfa9..07791516d 100644 --- a/satellite/reputation/service.go +++ b/satellite/reputation/service.go @@ -21,6 +21,7 @@ type DB interface { Update(ctx context.Context, request UpdateRequest, now time.Time) (_ *overlay.ReputationStatus, changed bool, err error) SetNodeStatus(ctx context.Context, id storj.NodeID, status overlay.ReputationStatus) error Get(ctx context.Context, nodeID storj.NodeID) (*Info, error) + Init(ctx context.Context, nodeID storj.NodeID) error // UnsuspendNodeUnknownAudit unsuspends a storage node for unknown audits. UnsuspendNodeUnknownAudit(ctx context.Context, nodeID storj.NodeID) (_ *overlay.ReputationStatus, err error) diff --git a/satellite/satellitedb/dbx/satellitedb.dbx b/satellite/satellitedb/dbx/satellitedb.dbx index 3eb15c90b..9d6d47289 100644 --- a/satellite/satellitedb/dbx/satellitedb.dbx +++ b/satellite/satellitedb/dbx/satellitedb.dbx @@ -291,6 +291,8 @@ model reputation ( field unknown_audit_reputation_beta float64 ( updatable, default 0 ) ) +create reputation () + update reputation ( where reputation.id = ? ) update reputation ( where reputation.id = ? diff --git a/satellite/satellitedb/dbx/satellitedb.dbx.go b/satellite/satellitedb/dbx/satellitedb.dbx.go index 7a6ae7fb2..ee5b1d5b6 100644 --- a/satellite/satellitedb/dbx/satellitedb.dbx.go +++ b/satellite/satellitedb/dbx/satellitedb.dbx.go @@ -10969,6 +10969,101 @@ func (obj *pgxImpl) Create_AuditHistory(ctx context.Context, } +func (obj *pgxImpl) Create_Reputation(ctx context.Context, + reputation_id Reputation_Id_Field, + reputation_audit_history Reputation_AuditHistory_Field, + optional Reputation_Create_Fields) ( + reputation *Reputation, err error) { + defer mon.Task()(&ctx)(&err) + __id_val := reputation_id.value() + __vetted_at_val := optional.VettedAt.value() + __disqualified_val := optional.Disqualified.value() + __suspended_val := optional.Suspended.value() + __unknown_audit_suspended_val := optional.UnknownAuditSuspended.value() + __offline_suspended_val := optional.OfflineSuspended.value() + __under_review_val := optional.UnderReview.value() + __audit_history_val := reputation_audit_history.value() + + var __columns = &__sqlbundle_Hole{SQL: __sqlbundle_Literal("id, vetted_at, disqualified, suspended, unknown_audit_suspended, offline_suspended, under_review, audit_history")} + var __placeholders = &__sqlbundle_Hole{SQL: __sqlbundle_Literal("?, ?, ?, ?, ?, ?, ?, ?")} + var __clause = &__sqlbundle_Hole{SQL: __sqlbundle_Literals{Join: "", SQLs: []__sqlbundle_SQL{__sqlbundle_Literal("("), __columns, __sqlbundle_Literal(") VALUES ("), __placeholders, __sqlbundle_Literal(")")}}} + + var __embed_stmt = __sqlbundle_Literals{Join: "", SQLs: []__sqlbundle_SQL{__sqlbundle_Literal("INSERT INTO reputations "), __clause, __sqlbundle_Literal(" RETURNING reputations.id, reputations.audit_success_count, reputations.total_audit_count, reputations.vetted_at, reputations.created_at, reputations.updated_at, reputations.contained, reputations.disqualified, reputations.suspended, reputations.unknown_audit_suspended, reputations.offline_suspended, reputations.under_review, reputations.online_score, reputations.audit_history, reputations.audit_reputation_alpha, reputations.audit_reputation_beta, reputations.unknown_audit_reputation_alpha, reputations.unknown_audit_reputation_beta")}} + + var __values []interface{} + __values = append(__values, __id_val, __vetted_at_val, __disqualified_val, __suspended_val, __unknown_audit_suspended_val, __offline_suspended_val, __under_review_val, __audit_history_val) + + __optional_columns := __sqlbundle_Literals{Join: ", "} + __optional_placeholders := __sqlbundle_Literals{Join: ", "} + + if optional.AuditSuccessCount._set { + __values = append(__values, optional.AuditSuccessCount.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_success_count")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.TotalAuditCount._set { + __values = append(__values, optional.TotalAuditCount.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("total_audit_count")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.Contained._set { + __values = append(__values, optional.Contained.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("contained")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.OnlineScore._set { + __values = append(__values, optional.OnlineScore.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("online_score")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.AuditReputationAlpha._set { + __values = append(__values, optional.AuditReputationAlpha.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_reputation_alpha")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.AuditReputationBeta._set { + __values = append(__values, optional.AuditReputationBeta.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_reputation_beta")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.UnknownAuditReputationAlpha._set { + __values = append(__values, optional.UnknownAuditReputationAlpha.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("unknown_audit_reputation_alpha")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.UnknownAuditReputationBeta._set { + __values = append(__values, optional.UnknownAuditReputationBeta.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("unknown_audit_reputation_beta")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if len(__optional_columns.SQLs) == 0 { + if __columns.SQL == nil { + __clause.SQL = __sqlbundle_Literal("DEFAULT VALUES") + } + } else { + __columns.SQL = __sqlbundle_Literals{Join: ", ", SQLs: []__sqlbundle_SQL{__columns.SQL, __optional_columns}} + __placeholders.SQL = __sqlbundle_Literals{Join: ", ", SQLs: []__sqlbundle_SQL{__placeholders.SQL, __optional_placeholders}} + } + var __stmt = __sqlbundle_Render(obj.dialect, __embed_stmt) + obj.logStmt(__stmt, __values...) + + reputation = &Reputation{} + err = obj.queryRowContext(ctx, __stmt, __values...).Scan(&reputation.Id, &reputation.AuditSuccessCount, &reputation.TotalAuditCount, &reputation.VettedAt, &reputation.CreatedAt, &reputation.UpdatedAt, &reputation.Contained, &reputation.Disqualified, &reputation.Suspended, &reputation.UnknownAuditSuspended, &reputation.OfflineSuspended, &reputation.UnderReview, &reputation.OnlineScore, &reputation.AuditHistory, &reputation.AuditReputationAlpha, &reputation.AuditReputationBeta, &reputation.UnknownAuditReputationAlpha, &reputation.UnknownAuditReputationBeta) + if err != nil { + return nil, obj.makeErr(err) + } + return reputation, nil + +} + func (obj *pgxImpl) Create_User(ctx context.Context, user_id User_Id_Field, user_email User_Email_Field, @@ -17019,6 +17114,101 @@ func (obj *pgxcockroachImpl) Create_AuditHistory(ctx context.Context, } +func (obj *pgxcockroachImpl) Create_Reputation(ctx context.Context, + reputation_id Reputation_Id_Field, + reputation_audit_history Reputation_AuditHistory_Field, + optional Reputation_Create_Fields) ( + reputation *Reputation, err error) { + defer mon.Task()(&ctx)(&err) + __id_val := reputation_id.value() + __vetted_at_val := optional.VettedAt.value() + __disqualified_val := optional.Disqualified.value() + __suspended_val := optional.Suspended.value() + __unknown_audit_suspended_val := optional.UnknownAuditSuspended.value() + __offline_suspended_val := optional.OfflineSuspended.value() + __under_review_val := optional.UnderReview.value() + __audit_history_val := reputation_audit_history.value() + + var __columns = &__sqlbundle_Hole{SQL: __sqlbundle_Literal("id, vetted_at, disqualified, suspended, unknown_audit_suspended, offline_suspended, under_review, audit_history")} + var __placeholders = &__sqlbundle_Hole{SQL: __sqlbundle_Literal("?, ?, ?, ?, ?, ?, ?, ?")} + var __clause = &__sqlbundle_Hole{SQL: __sqlbundle_Literals{Join: "", SQLs: []__sqlbundle_SQL{__sqlbundle_Literal("("), __columns, __sqlbundle_Literal(") VALUES ("), __placeholders, __sqlbundle_Literal(")")}}} + + var __embed_stmt = __sqlbundle_Literals{Join: "", SQLs: []__sqlbundle_SQL{__sqlbundle_Literal("INSERT INTO reputations "), __clause, __sqlbundle_Literal(" RETURNING reputations.id, reputations.audit_success_count, reputations.total_audit_count, reputations.vetted_at, reputations.created_at, reputations.updated_at, reputations.contained, reputations.disqualified, reputations.suspended, reputations.unknown_audit_suspended, reputations.offline_suspended, reputations.under_review, reputations.online_score, reputations.audit_history, reputations.audit_reputation_alpha, reputations.audit_reputation_beta, reputations.unknown_audit_reputation_alpha, reputations.unknown_audit_reputation_beta")}} + + var __values []interface{} + __values = append(__values, __id_val, __vetted_at_val, __disqualified_val, __suspended_val, __unknown_audit_suspended_val, __offline_suspended_val, __under_review_val, __audit_history_val) + + __optional_columns := __sqlbundle_Literals{Join: ", "} + __optional_placeholders := __sqlbundle_Literals{Join: ", "} + + if optional.AuditSuccessCount._set { + __values = append(__values, optional.AuditSuccessCount.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_success_count")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.TotalAuditCount._set { + __values = append(__values, optional.TotalAuditCount.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("total_audit_count")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.Contained._set { + __values = append(__values, optional.Contained.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("contained")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.OnlineScore._set { + __values = append(__values, optional.OnlineScore.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("online_score")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.AuditReputationAlpha._set { + __values = append(__values, optional.AuditReputationAlpha.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_reputation_alpha")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.AuditReputationBeta._set { + __values = append(__values, optional.AuditReputationBeta.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("audit_reputation_beta")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.UnknownAuditReputationAlpha._set { + __values = append(__values, optional.UnknownAuditReputationAlpha.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("unknown_audit_reputation_alpha")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if optional.UnknownAuditReputationBeta._set { + __values = append(__values, optional.UnknownAuditReputationBeta.value()) + __optional_columns.SQLs = append(__optional_columns.SQLs, __sqlbundle_Literal("unknown_audit_reputation_beta")) + __optional_placeholders.SQLs = append(__optional_placeholders.SQLs, __sqlbundle_Literal("?")) + } + + if len(__optional_columns.SQLs) == 0 { + if __columns.SQL == nil { + __clause.SQL = __sqlbundle_Literal("DEFAULT VALUES") + } + } else { + __columns.SQL = __sqlbundle_Literals{Join: ", ", SQLs: []__sqlbundle_SQL{__columns.SQL, __optional_columns}} + __placeholders.SQL = __sqlbundle_Literals{Join: ", ", SQLs: []__sqlbundle_SQL{__placeholders.SQL, __optional_placeholders}} + } + var __stmt = __sqlbundle_Render(obj.dialect, __embed_stmt) + obj.logStmt(__stmt, __values...) + + reputation = &Reputation{} + err = obj.queryRowContext(ctx, __stmt, __values...).Scan(&reputation.Id, &reputation.AuditSuccessCount, &reputation.TotalAuditCount, &reputation.VettedAt, &reputation.CreatedAt, &reputation.UpdatedAt, &reputation.Contained, &reputation.Disqualified, &reputation.Suspended, &reputation.UnknownAuditSuspended, &reputation.OfflineSuspended, &reputation.UnderReview, &reputation.OnlineScore, &reputation.AuditHistory, &reputation.AuditReputationAlpha, &reputation.AuditReputationBeta, &reputation.UnknownAuditReputationAlpha, &reputation.UnknownAuditReputationBeta) + if err != nil { + return nil, obj.makeErr(err) + } + return reputation, nil + +} + func (obj *pgxcockroachImpl) Create_User(ctx context.Context, user_id User_Id_Field, user_email User_Email_Field, @@ -23427,6 +23617,19 @@ func (rx *Rx) Create_RegistrationToken(ctx context.Context, } +func (rx *Rx) Create_Reputation(ctx context.Context, + reputation_id Reputation_Id_Field, + reputation_audit_history Reputation_AuditHistory_Field, + optional Reputation_Create_Fields) ( + reputation *Reputation, err error) { + var tx *Tx + if tx, err = rx.getTx(ctx); err != nil { + return + } + return tx.Create_Reputation(ctx, reputation_id, reputation_audit_history, optional) + +} + func (rx *Rx) Create_ResetPasswordToken(ctx context.Context, reset_password_token_secret ResetPasswordToken_Secret_Field, reset_password_token_owner_id ResetPasswordToken_OwnerId_Field) ( @@ -24778,6 +24981,12 @@ type Methods interface { optional RegistrationToken_Create_Fields) ( registration_token *RegistrationToken, err error) + Create_Reputation(ctx context.Context, + reputation_id Reputation_Id_Field, + reputation_audit_history Reputation_AuditHistory_Field, + optional Reputation_Create_Fields) ( + reputation *Reputation, err error) + Create_ResetPasswordToken(ctx context.Context, reset_password_token_secret ResetPasswordToken_Secret_Field, reset_password_token_owner_id ResetPasswordToken_OwnerId_Field) ( diff --git a/satellite/satellitedb/reputations.go b/satellite/satellitedb/reputations.go index 9b7027631..3b594f077 100644 --- a/satellite/satellitedb/reputations.go +++ b/satellite/satellitedb/reputations.go @@ -369,6 +369,17 @@ func (reputations *reputations) UnsuspendNodeUnknownAudit(ctx context.Context, n }, nil } +// Init creates an entry for a node with its reputaion in zero value. +func (reputations *reputations) Init(ctx context.Context, nodeID storj.NodeID) error { + historyBytes, err := pb.Marshal(&internalpb.AuditHistory{}) + if err != nil { + return err + } + + _, err = reputations.db.Create_Reputation(ctx, dbx.Reputation_Id(nodeID.Bytes()), dbx.Reputation_AuditHistory(historyBytes), dbx.Reputation_Create_Fields{}) + return Error.Wrap(err) +} + func (reputations *reputations) populateUpdateFields(dbNode *dbx.Reputation, updateReq reputation.UpdateRequest, auditHistoryResponse *reputation.UpdateAuditHistoryResponse, now time.Time) dbx.Reputation_Update_Fields { update := reputations.populateUpdateNodeStats(dbNode, updateReq, auditHistoryResponse, now)