diff --git a/cmd/tools/node-cleanup/main.go b/cmd/tools/node-cleanup/main.go index 2ede9de77..96f10717d 100644 --- a/cmd/tools/node-cleanup/main.go +++ b/cmd/tools/node-cleanup/main.go @@ -51,6 +51,7 @@ func init() { type Config struct { SatelliteDB string Limit int + CreatedAt string MaxIterations int } @@ -59,6 +60,7 @@ type Config struct { func (config *Config) BindFlags(flag *flag.FlagSet) { flag.StringVar(&config.SatelliteDB, "satellitedb", "", "connection URL for satelliteDB") flag.IntVar(&config.Limit, "limit", 1000, "number of deletes to perform at once") + flag.StringVar(&config.CreatedAt, "created-at", "", "latest node creation date for which to delete in iso8601 format YYYY-MM-DD") flag.IntVar(&config.MaxIterations, "max-iterations", -1, "number of maximum iterations (negative is unlimited)") } @@ -154,7 +156,9 @@ func DeleteFromTables(ctx context.Context, log *zap.Logger, db tagsql.DB, config err = db.QueryRowContext(ctx, ` WITH deleted_nodes AS ( DELETE FROM nodes - WHERE id > $1 AND id <= $2 AND email LIKE '%@atredis.com' + WHERE id > $1 AND id <= $2 + AND last_contact_success = '0001-01-01 00:00:00+00' + AND created_at <= $3 RETURNING id ), deleted_paystubs AS ( @@ -177,7 +181,7 @@ func DeleteFromTables(ctx context.Context, log *zap.Logger, db tagsql.DB, config (select count(*) from deleted_paystubs), (select count(*) from deleted_peer_identities), (select count(*) from deleted_node_api_versions) - `, cursor, batchEnd).Scan(&deletedNodes, &deletedPaystubs, &deletedPeerIdentities, &deletedNodeAPIVersions) + `, cursor, batchEnd, config.CreatedAt).Scan(&deletedNodes, &deletedPaystubs, &deletedPeerIdentities, &deletedNodeAPIVersions) if err != nil { return errs.New("batch deletion failed: %w", err) } diff --git a/cmd/tools/node-cleanup/main_test.go b/cmd/tools/node-cleanup/main_test.go index 3bd75a6b6..16214d7ab 100644 --- a/cmd/tools/node-cleanup/main_test.go +++ b/cmd/tools/node-cleanup/main_test.go @@ -38,19 +38,20 @@ func TestDelete(t *testing.T) { } requireTableCount("nodes", 5) - requireTableCount("nodes WHERE email LIKE '%@atredis.com'", 4) + requireTableCount("nodes WHERE last_contact_success = '0001-01-01 00:00:00+00' AND created_at <= '2022-09-01'", 4) requireTableCount("storagenode_paystubs", 5) requireTableCount("peer_identities", 5) requireTableCount("node_api_versions", 5) err := nodecleanup.DeleteFromTables(ctx, zaptest.NewLogger(t), raw, nodecleanup.Config{ Limit: 2, + CreatedAt: "2022-09-01", MaxIterations: -1, }) require.NoError(t, err) requireTableCount("nodes", 1) - requireTableCount("nodes WHERE email LIKE '%@atredis.com'", 0) + requireTableCount("nodes WHERE last_contact_success = '0001-01-01 00:00:00+00' AND created_at <= '2022-09-01'", 0) requireTableCount("storagenode_paystubs", 1) requireTableCount("peer_identities", 1) requireTableCount("node_api_versions", 1) @@ -88,19 +89,20 @@ func TestLargeDelete(t *testing.T) { } requireTableCount("nodes", valid+problematic) - requireTableCount("nodes WHERE email LIKE '%@atredis.com'", problematic) + requireTableCount("nodes WHERE last_contact_success = '0001-01-01 00:00:00+00' AND created_at <= '2022-09-01'", problematic) requireTableCount("storagenode_paystubs", valid+problematic) requireTableCount("peer_identities", valid+problematic) requireTableCount("node_api_versions", valid+problematic) err := nodecleanup.DeleteFromTables(ctx, zaptest.NewLogger(t), raw, nodecleanup.Config{ Limit: 1000, + CreatedAt: "2022-09-01", MaxIterations: -1, }) require.NoError(t, err) requireTableCount("nodes", valid) - requireTableCount("nodes WHERE email LIKE '%@atredis.com'", 0) + requireTableCount("nodes WHERE last_contact_success = '0001-01-01 00:00:00+00' AND created_at <= '2022-09-01'", 0) requireTableCount("storagenode_paystubs", valid) requireTableCount("peer_identities", valid) requireTableCount("node_api_versions", valid) @@ -115,34 +117,40 @@ func insertNode(ctx context.Context, t *testing.T, raw tagsql.DB, index int, pro address := fmt.Sprintf("127.0.0.1:100%02d", index) - var email string + var created_at string + var updated_at string + var last_contact_success string if problematic { - email = fmt.Sprintf("node%d@atredis.com", index) + created_at = "2022-08-31 23:59:31.028103+00" + updated_at = "2022-10-01 01:08:31.028103+00" + last_contact_success = "0001-01-01 00:00:00+00" } else { - email = fmt.Sprintf("node%d@anything.com", index) + created_at = "2022-09-01 23:59:31.028103+00" + updated_at = "2022-10-01 01:08:31.028103+00" + last_contact_success = "2022-10-01 01:08:31.028103+00" } _, err := raw.ExecContext(ctx, ` - INSERT INTO "nodes" ("id", "address", "last_net", "email", "wallet", "created_at", "updated_at", "last_contact_success", "exit_success") VALUES - ($1, $2, '', $3, '', '2019-02-14 08:07:31.108963+00', '2019-02-14 08:07:31.108963+00', '0001-01-01 00:00:00+00', false) - `, nodeid, address, email) + INSERT INTO "nodes" ("id", "address", "email", "last_net", "wallet", "created_at", "updated_at", "last_contact_success", "exit_success") VALUES + ($1, $2, '', '', '', $3, $4, $5, false) + `, nodeid, address, created_at, updated_at, last_contact_success) require.NoError(t, err) _, err = raw.ExecContext(ctx, ` INSERT INTO "storagenode_paystubs"("period", "node_id", "created_at", "codes", "usage_at_rest", "usage_get", "usage_put", "usage_get_repair", "usage_put_repair", "usage_get_audit", "comp_at_rest", "comp_get", "comp_put", "comp_get_repair", "comp_put_repair", "comp_get_audit", "surge_percent", "held", "owed", "disposed", "paid", "distributed") VALUES - ('2020-01', $1, '2020-04-07T20:14:21.479141Z', '', 1327959864508416, 294054066688, 159031363328, 226751, 0, 836608, 2861984, 5881081, 0, 226751, 0, 8, 300, 0, 26909472, 0, 26909472, 0) + ('2022-08', $1, '2022-09-07T20:14:21.479141Z', '', 1327959864508416, 294054066688, 159031363328, 226751, 0, 836608, 2861984, 5881081, 0, 226751, 0, 8, 300, 0, 26909472, 0, 26909472, 0) `, nodeid) require.NoError(t, err) _, err = raw.ExecContext(ctx, ` INSERT INTO "peer_identities"("node_id", "leaf_serial_number","chain", "updated_at") VALUES - ($1, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-02-14 08:07:31.335028+00') + ($1, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2022-09-07 08:07:31.335028+00') `, nodeid) require.NoError(t, err) _, err = raw.ExecContext(ctx, ` INSERT INTO "node_api_versions"("id", "api_version", "created_at", "updated_at") VALUES - ($1, 1, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00'); - `, nodeid) + ($1, 1, $2, $3); + `, nodeid, created_at, updated_at) require.NoError(t, err) }