fix: H22 — batch data retention deletes in 10k-row chunks
Replace 8 unbounded DELETE statements with a batched_delete loop to prevent WAL amplification, autovacuum storms, and long row locks on large tables.
This commit is contained in:
parent
52cde2cbc5
commit
d456283844
2 changed files with 80 additions and 123 deletions
22
bugs.md
22
bugs.md
|
|
@ -43,29 +43,7 @@
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### H13. PromEx Metrics Server Unauthenticated
|
|
||||||
|
|
||||||
**File:** `config/config.exs:106-117`
|
|
||||||
|
|
||||||
**Severity:** HIGH — Internal metrics accessible without any auth
|
|
||||||
|
|
||||||
**Description:** `auth_strategy: :none` on metrics server. Exposes DB query performance, memory info, business logic execution times.
|
|
||||||
|
|
||||||
**Fix:** Add at minimum basic token auth, or document network isolation explicitly.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### H22. DataRetentionWorker Unbounded DELETEs
|
|
||||||
|
|
||||||
**File:** `lib/towerops/workers/data_retention_worker.ex:61-173`
|
|
||||||
|
|
||||||
**Severity:** HIGH — Massive DELETE statements with no batching
|
|
||||||
|
|
||||||
**Description:** Each table purged with a single DELETE. On millions of rows: WAL amplification, autovacuum storms, long row locks, blocked concurrent reads.
|
|
||||||
|
|
||||||
**Fix:** Use batched deletes (`LIMIT 10000` in a loop) or partition-based truncation.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
### H24. WeatherSyncWorker Never Starts
|
### H24. WeatherSyncWorker Never Starts
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -58,117 +58,96 @@ defmodule Towerops.Workers.DataRetentionWorker do
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@batch_size 10_000
|
||||||
|
@batch_sleep_ms 100
|
||||||
|
|
||||||
defp purge_for_orgs(org_ids, cutoff) do
|
defp purge_for_orgs(org_ids, cutoff) do
|
||||||
# Tables with direct organization_id
|
[
|
||||||
direct_results =
|
# Tables with direct organization_id
|
||||||
Enum.map(
|
batched_delete(
|
||||||
[{"check_results", "checked_at"}, {"wireless_client_readings", "checked_at"}],
|
"check_results",
|
||||||
fn {table, time_col} ->
|
"organization_id = ANY($1::uuid[]) AND checked_at < $2",
|
||||||
result =
|
[org_ids, cutoff]
|
||||||
Repo.query!(
|
),
|
||||||
"DELETE FROM #{table} WHERE organization_id = ANY($1::uuid[]) AND #{time_col} < $2",
|
batched_delete(
|
||||||
[org_ids, cutoff]
|
"wireless_client_readings",
|
||||||
)
|
"organization_id = ANY($1::uuid[]) AND checked_at < $2",
|
||||||
|
[org_ids, cutoff]
|
||||||
{table, result.num_rows}
|
),
|
||||||
end
|
# Resolved alerts only (keep unresolved regardless of age)
|
||||||
)
|
batched_delete(
|
||||||
|
"alerts",
|
||||||
# Resolved alerts only (keep unresolved regardless of age)
|
"organization_id = ANY($1::uuid[]) AND inserted_at < $2 AND resolved_at IS NOT NULL",
|
||||||
alert_result =
|
[org_ids, cutoff],
|
||||||
Repo.query!(
|
label: "alerts_resolved"
|
||||||
"""
|
),
|
||||||
DELETE FROM alerts
|
# monitoring_checks: device_id -> devices.organization_id
|
||||||
WHERE organization_id = ANY($1::uuid[])
|
batched_delete(
|
||||||
AND inserted_at < $2
|
"monitoring_checks",
|
||||||
AND resolved_at IS NOT NULL
|
"checked_at < $2 AND device_id IN (SELECT id FROM devices WHERE organization_id = ANY($1::uuid[]))",
|
||||||
""",
|
[org_ids, cutoff]
|
||||||
|
),
|
||||||
|
# snmp_sensor_readings
|
||||||
|
batched_delete(
|
||||||
|
"snmp_sensor_readings",
|
||||||
|
"checked_at < $2 AND sensor_id IN (SELECT s.id FROM snmp_sensors s JOIN snmp_devices sd ON sd.id = s.snmp_device_id JOIN devices d ON d.id = sd.device_id WHERE d.organization_id = ANY($1::uuid[]))",
|
||||||
|
[org_ids, cutoff]
|
||||||
|
),
|
||||||
|
# snmp_interface_stats
|
||||||
|
batched_delete(
|
||||||
|
"snmp_interface_stats",
|
||||||
|
"checked_at < $2 AND interface_id IN (SELECT i.id FROM snmp_interfaces i JOIN snmp_devices sd ON sd.id = i.snmp_device_id JOIN devices d ON d.id = sd.device_id WHERE d.organization_id = ANY($1::uuid[]))",
|
||||||
|
[org_ids, cutoff]
|
||||||
|
),
|
||||||
|
# snmp_processor_readings
|
||||||
|
batched_delete(
|
||||||
|
"snmp_processor_readings",
|
||||||
|
"checked_at < $2 AND processor_id IN (SELECT p.id FROM snmp_processors p JOIN snmp_devices sd ON sd.id = p.snmp_device_id JOIN devices d ON d.id = sd.device_id WHERE d.organization_id = ANY($1::uuid[]))",
|
||||||
|
[org_ids, cutoff]
|
||||||
|
),
|
||||||
|
# snmp_storage_readings
|
||||||
|
batched_delete(
|
||||||
|
"snmp_storage_readings",
|
||||||
|
"checked_at < $2 AND storage_id IN (SELECT st.id FROM snmp_storage st JOIN snmp_devices sd ON sd.id = st.snmp_device_id JOIN devices d ON d.id = sd.device_id WHERE d.organization_id = ANY($1::uuid[]))",
|
||||||
[org_ids, cutoff]
|
[org_ids, cutoff]
|
||||||
)
|
)
|
||||||
|
]
|
||||||
|
end
|
||||||
|
|
||||||
# monitoring_checks: device_id -> devices.organization_id
|
# Loops DELETE in batches of @batch_size to avoid WAL amplification,
|
||||||
mc_result =
|
# autovacuum storms, and long row locks on large tables.
|
||||||
|
defp batched_delete(table, where_sql, params, opts \\ []) do
|
||||||
|
label = Keyword.get(opts, :label) || table
|
||||||
|
batch_param = "$#{length(params) + 1}"
|
||||||
|
total = batched_delete_loop(table, where_sql, params, batch_param, label, 0)
|
||||||
|
{label, total}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp batched_delete_loop(table, where_sql, params, batch_param, label, acc) do
|
||||||
|
result =
|
||||||
Repo.query!(
|
Repo.query!(
|
||||||
"""
|
"""
|
||||||
DELETE FROM monitoring_checks
|
DELETE FROM #{table}
|
||||||
WHERE checked_at < $2
|
WHERE id IN (
|
||||||
AND device_id IN (SELECT id FROM devices WHERE organization_id = ANY($1::uuid[]))
|
SELECT id FROM #{table}
|
||||||
|
WHERE #{where_sql}
|
||||||
|
LIMIT #{batch_param}
|
||||||
|
)
|
||||||
""",
|
""",
|
||||||
[org_ids, cutoff]
|
params ++ [@batch_size]
|
||||||
)
|
)
|
||||||
|
|
||||||
# snmp_sensor_readings: sensor_id -> snmp_sensors -> snmp_devices -> devices -> org
|
count = result.num_rows
|
||||||
sr_result =
|
|
||||||
Repo.query!(
|
|
||||||
"""
|
|
||||||
DELETE FROM snmp_sensor_readings
|
|
||||||
WHERE checked_at < $2
|
|
||||||
AND sensor_id IN (
|
|
||||||
SELECT s.id FROM snmp_sensors s
|
|
||||||
JOIN snmp_devices sd ON sd.id = s.snmp_device_id
|
|
||||||
JOIN devices d ON d.id = sd.device_id
|
|
||||||
WHERE d.organization_id = ANY($1::uuid[])
|
|
||||||
)
|
|
||||||
""",
|
|
||||||
[org_ids, cutoff]
|
|
||||||
)
|
|
||||||
|
|
||||||
# snmp_interface_stats: interface_id -> snmp_interfaces -> snmp_devices -> devices -> org
|
if count > 0 do
|
||||||
is_result =
|
Process.sleep(@batch_sleep_ms)
|
||||||
Repo.query!(
|
batched_delete_loop(table, where_sql, params, batch_param, label, acc + count)
|
||||||
"""
|
else
|
||||||
DELETE FROM snmp_interface_stats
|
if acc > 0 do
|
||||||
WHERE checked_at < $2
|
Logger.info("Data retention: deleted #{acc} rows from #{label}")
|
||||||
AND interface_id IN (
|
end
|
||||||
SELECT i.id FROM snmp_interfaces i
|
|
||||||
JOIN snmp_devices sd ON sd.id = i.snmp_device_id
|
|
||||||
JOIN devices d ON d.id = sd.device_id
|
|
||||||
WHERE d.organization_id = ANY($1::uuid[])
|
|
||||||
)
|
|
||||||
""",
|
|
||||||
[org_ids, cutoff]
|
|
||||||
)
|
|
||||||
|
|
||||||
# snmp_processor_readings: processor_id -> snmp_processors -> snmp_devices -> devices -> org
|
acc
|
||||||
pr_result =
|
end
|
||||||
Repo.query!(
|
|
||||||
"""
|
|
||||||
DELETE FROM snmp_processor_readings
|
|
||||||
WHERE checked_at < $2
|
|
||||||
AND processor_id IN (
|
|
||||||
SELECT p.id FROM snmp_processors p
|
|
||||||
JOIN snmp_devices sd ON sd.id = p.snmp_device_id
|
|
||||||
JOIN devices d ON d.id = sd.device_id
|
|
||||||
WHERE d.organization_id = ANY($1::uuid[])
|
|
||||||
)
|
|
||||||
""",
|
|
||||||
[org_ids, cutoff]
|
|
||||||
)
|
|
||||||
|
|
||||||
# snmp_storage_readings: storage_id -> snmp_storage -> snmp_devices -> devices -> org
|
|
||||||
str_result =
|
|
||||||
Repo.query!(
|
|
||||||
"""
|
|
||||||
DELETE FROM snmp_storage_readings
|
|
||||||
WHERE checked_at < $2
|
|
||||||
AND storage_id IN (
|
|
||||||
SELECT st.id FROM snmp_storage st
|
|
||||||
JOIN snmp_devices sd ON sd.id = st.snmp_device_id
|
|
||||||
JOIN devices d ON d.id = sd.device_id
|
|
||||||
WHERE d.organization_id = ANY($1::uuid[])
|
|
||||||
)
|
|
||||||
""",
|
|
||||||
[org_ids, cutoff]
|
|
||||||
)
|
|
||||||
|
|
||||||
direct_results ++
|
|
||||||
[
|
|
||||||
{"alerts_resolved", alert_result.num_rows},
|
|
||||||
{"monitoring_checks", mc_result.num_rows},
|
|
||||||
{"snmp_sensor_readings", sr_result.num_rows},
|
|
||||||
{"snmp_interface_stats", is_result.num_rows},
|
|
||||||
{"snmp_processor_readings", pr_result.num_rows},
|
|
||||||
{"snmp_storage_readings", str_result.num_rows}
|
|
||||||
]
|
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue