fix(alerts): resolve stuck alerts on every successful poll, not just status change

Previous fix only resolved alerts when device status changed from down to up.
This didn't help devices already marked as 'up' with stuck unresolved alerts.

Now:
- Always check for stuck device_down alerts after successful SNMP poll
- Also check for stuck device_up alerts (should always be auto-resolved)
- Logs when stuck alerts are found and resolved
- Works for existing stuck alerts without waiting for status to change

This fixes the 8 stuck alerts in production - they'll auto-resolve on next poll.
This commit is contained in:
Graham McIntire 2026-03-05 16:54:00 -06:00
parent 94099745d4
commit 192cf5b064
No known key found for this signature in database

View file

@ -1484,6 +1484,65 @@ defmodule ToweropsWeb.AgentChannel do
)
end
# Resolve any stuck alerts for a device that's currently up
# This handles edge cases where device is UP but alerts didn't resolve (e.g., due to bugs)
defp resolve_any_stuck_down_alerts(device) do
now = DateTime.truncate(DateTime.utc_now(), :second)
# Resolve stuck device_down alerts
case Alerts.get_active_alert(device.id, :device_down) do
nil ->
:ok
alert ->
Logger.info(
"Resolving stuck device_down alert for #{device.name} - device is UP but alert was not resolved",
device_id: device.id,
alert_id: alert.id
)
# Create device_up alert (informational)
message =
if device.snmp_enabled,
do: "Device is now responding to SNMP",
else: "Device is now responding to ping"
Alerts.create_alert(%{
device_id: device.id,
alert_type: :device_up,
triggered_at: now,
resolved_at: now,
message: message
})
# Resolve the stuck alert
Alerts.resolve_alert(alert)
Phoenix.PubSub.broadcast(
Towerops.PubSub,
"alerts:org:#{device.organization_id}:resolved",
{:alert_resolved, device.id, :device_down}
)
end
# Also resolve any stuck device_up alerts (should have been created with resolved_at)
case Alerts.get_active_alert(device.id, :device_up) do
nil ->
:ok
alert ->
Logger.info(
"Resolving stuck device_up alert for #{device.name} - should have been auto-resolved",
device_id: device.id,
alert_id: alert.id
)
Alerts.resolve_alert(alert)
end
:ok
end
defp fetch_device(device_id) do
case Devices.get_device_with_details(device_id) do
nil -> {:error, :device_not_found}
@ -1605,13 +1664,13 @@ defmodule ToweropsWeb.AgentChannel do
defp process_polling_result(device, result, _socket) do
Devices.update_snmp_poll_time(device)
# Successful SNMP poll means device is up - check for status change and resolve alerts
# Successful SNMP poll means device is up - update status and resolve any stuck alerts
old_status = device.status
new_status = :up
# Update device status if needed
if old_status != new_status do
Devices.update_device_status(device, new_status)
handle_agent_status_change(device, old_status, new_status)
Phoenix.PubSub.broadcast(
Towerops.PubSub,
@ -1620,6 +1679,10 @@ defmodule ToweropsWeb.AgentChannel do
)
end
# Always check for and resolve stuck device_down alerts after successful poll
# This handles edge cases where alerts got stuck (device already UP but alert not resolved)
resolve_any_stuck_down_alerts(device)
snmp_device = device.snmp_device
oid_values = normalize_oid_keys(result.oid_values)
# Use server time to avoid clock drift issues between agents