JakeHillion/storj
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

cmd/tools/node-cleanup: swap deletion for atredis emails with last contact and created at timestamps

Browse Source 
Change-Id: I4acde09e00ea4e5450dcb0fd26bbb2c3afa767f6
This commit is contained in:
Jennifer Johnson 2022-10-11 22:49:12 -04:00 committed by Storj Robot
parent 982b9a1844
commit 1da7520a88
2 changed files with 28 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
cmd/tools/node-cleanup/main.go
View File

@ -51,6 +51,7 @@ func init() {
type Config struct {
SatelliteDB string
Limit int
CreatedAt string
MaxIterations int
}
@ -59,6 +60,7 @@ type Config struct {
func (config *Config) BindFlags(flag *flag.FlagSet) {
flag.StringVar(&config.SatelliteDB, "satellitedb", "", "connection URL for satelliteDB")
flag.IntVar(&config.Limit, "limit", 1000, "number of deletes to perform at once")
flag.StringVar(&config.CreatedAt, "created-at", "", "latest node creation date for which to delete in iso8601 format YYYY-MM-DD")
flag.IntVar(&config.MaxIterations, "max-iterations", -1, "number of maximum iterations (negative is unlimited)")
}
@ -154,7 +156,9 @@ func DeleteFromTables(ctx context.Context, log *zap.Logger, db tagsql.DB, config
err = db.QueryRowContext(ctx, `
WITH deleted_nodes AS (
DELETE FROM nodes
WHERE id > $1 AND id <= $2 AND email LIKE '%@atredis.com'
WHERE id > $1 AND id <= $2
AND last_contact_success = '0001-01-01 00:00:00+00'
AND created_at <= $3
RETURNING id
),
deleted_paystubs AS (
@ -177,7 +181,7 @@ func DeleteFromTables(ctx context.Context, log *zap.Logger, db tagsql.DB, config
(select count(*) from deleted_paystubs),
(select count(*) from deleted_peer_identities),
(select count(*) from deleted_node_api_versions)
`, cursor, batchEnd).Scan(&deletedNodes, &deletedPaystubs, &deletedPeerIdentities, &deletedNodeAPIVersions)
`, cursor, batchEnd, config.CreatedAt).Scan(&deletedNodes, &deletedPaystubs, &deletedPeerIdentities, &deletedNodeAPIVersions)
if err != nil {
return errs.New("batch deletion failed: %w", err)
}

36
cmd/tools/node-cleanup/main_test.go
View File

@ -38,19 +38,20 @@ func TestDelete(t *testing.T) {
}
requireTableCount("nodes", 5)
requireTableCount("nodes WHERE email LIKE '%@atredis.com'", 4)
requireTableCount("nodes WHERE last_contact_success = '0001-01-01 00:00:00+00' AND created_at <= '2022-09-01'", 4)
requireTableCount("storagenode_paystubs", 5)
requireTableCount("peer_identities", 5)
requireTableCount("node_api_versions", 5)
err := nodecleanup.DeleteFromTables(ctx, zaptest.NewLogger(t), raw, nodecleanup.Config{
Limit: 2,
CreatedAt: "2022-09-01",
MaxIterations: -1,
})
require.NoError(t, err)
requireTableCount("nodes", 1)
requireTableCount("nodes WHERE email LIKE '%@atredis.com'", 0)
requireTableCount("nodes WHERE last_contact_success = '0001-01-01 00:00:00+00' AND created_at <= '2022-09-01'", 0)
requireTableCount("storagenode_paystubs", 1)
requireTableCount("peer_identities", 1)
requireTableCount("node_api_versions", 1)
@ -88,19 +89,20 @@ func TestLargeDelete(t *testing.T) {
}
requireTableCount("nodes", valid+problematic)
requireTableCount("nodes WHERE email LIKE '%@atredis.com'", problematic)
requireTableCount("nodes WHERE last_contact_success = '0001-01-01 00:00:00+00' AND created_at <= '2022-09-01'", problematic)
requireTableCount("storagenode_paystubs", valid+problematic)
requireTableCount("peer_identities", valid+problematic)
requireTableCount("node_api_versions", valid+problematic)
err := nodecleanup.DeleteFromTables(ctx, zaptest.NewLogger(t), raw, nodecleanup.Config{
Limit: 1000,
CreatedAt: "2022-09-01",
MaxIterations: -1,
})
require.NoError(t, err)
requireTableCount("nodes", valid)
requireTableCount("nodes WHERE email LIKE '%@atredis.com'", 0)
requireTableCount("nodes WHERE last_contact_success = '0001-01-01 00:00:00+00' AND created_at <= '2022-09-01'", 0)
requireTableCount("storagenode_paystubs", valid)
requireTableCount("peer_identities", valid)
requireTableCount("node_api_versions", valid)
@ -115,34 +117,40 @@ func insertNode(ctx context.Context, t *testing.T, raw tagsql.DB, index int, pro
address := fmt.Sprintf("127.0.0.1:100%02d", index)
var email string
var created_at string
var updated_at string
var last_contact_success string
if problematic {
email = fmt.Sprintf("node%d@atredis.com", index)
created_at = "2022-08-31 23:59:31.028103+00"
updated_at = "2022-10-01 01:08:31.028103+00"
last_contact_success = "0001-01-01 00:00:00+00"
} else {
email = fmt.Sprintf("node%d@anything.com", index)
created_at = "2022-09-01 23:59:31.028103+00"
updated_at = "2022-10-01 01:08:31.028103+00"
last_contact_success = "2022-10-01 01:08:31.028103+00"
}
_, err := raw.ExecContext(ctx, `
INSERT INTO "nodes" ("id", "address", "last_net", "email", "wallet", "created_at", "updated_at", "last_contact_success", "exit_success") VALUES
($1, $2, '', $3, '', '2019-02-14 08:07:31.108963+00', '2019-02-14 08:07:31.108963+00', '0001-01-01 00:00:00+00', false)
`, nodeid, address, email)
INSERT INTO "nodes" ("id", "address", "email", "last_net", "wallet", "created_at", "updated_at", "last_contact_success", "exit_success") VALUES
($1, $2, '', '', '', $3, $4, $5, false)
`, nodeid, address, created_at, updated_at, last_contact_success)
require.NoError(t, err)
_, err = raw.ExecContext(ctx, `
INSERT INTO "storagenode_paystubs"("period", "node_id", "created_at", "codes", "usage_at_rest", "usage_get", "usage_put", "usage_get_repair", "usage_put_repair", "usage_get_audit", "comp_at_rest", "comp_get", "comp_put", "comp_get_repair", "comp_put_repair", "comp_get_audit", "surge_percent", "held", "owed", "disposed", "paid", "distributed") VALUES
('2020-01', $1, '2020-04-07T20:14:21.479141Z', '', 1327959864508416, 294054066688, 159031363328, 226751, 0, 836608, 2861984, 5881081, 0, 226751, 0, 8, 300, 0, 26909472, 0, 26909472, 0)
('2022-08', $1, '2022-09-07T20:14:21.479141Z', '', 1327959864508416, 294054066688, 159031363328, 226751, 0, 836608, 2861984, 5881081, 0, 226751, 0, 8, 300, 0, 26909472, 0, 26909472, 0)
`, nodeid)
require.NoError(t, err)
_, err = raw.ExecContext(ctx, `
INSERT INTO "peer_identities"("node_id", "leaf_serial_number","chain", "updated_at") VALUES
($1, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2019-02-14 08:07:31.335028+00')
($1, E'\\363\\342\\363\\371>+F\\256\\263\\300\\273|\\342N\\347\\014'::bytea, E'\\363\\311\\033w\\222\\303Ci\\265\\343U\\303\\312\\204",'::bytea, '2022-09-07 08:07:31.335028+00')
`, nodeid)
require.NoError(t, err)
_, err = raw.ExecContext(ctx, `
INSERT INTO "node_api_versions"("id", "api_version", "created_at", "updated_at") VALUES
($1, 1, '2019-02-14 08:07:31.028103+00', '2019-02-14 08:07:31.108963+00');
`, nodeid)
($1, 1, $2, $3);
`, nodeid, created_at, updated_at)
require.NoError(t, err)
}