fix(alerts): resolve stuck alerts on every successful poll, not just status change
Previous fix only resolved alerts when device status changed from down to up. This didn't help devices already marked as 'up' with stuck unresolved alerts. Now: - Always check for stuck device_down alerts after successful SNMP poll - Also check for stuck device_up alerts (should always be auto-resolved) - Logs when stuck alerts are found and resolved - Works for existing stuck alerts without waiting for status to change This fixes the 8 stuck alerts in production - they'll auto-resolve on next poll.
This commit is contained in:
parent
94099745d4
commit
192cf5b064
1 changed files with 65 additions and 2 deletions
|
|
@ -1484,6 +1484,65 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
)
|
||||
end
|
||||
|
||||
# Resolve any stuck alerts for a device that's currently up
|
||||
# This handles edge cases where device is UP but alerts didn't resolve (e.g., due to bugs)
|
||||
defp resolve_any_stuck_down_alerts(device) do
|
||||
now = DateTime.truncate(DateTime.utc_now(), :second)
|
||||
|
||||
# Resolve stuck device_down alerts
|
||||
case Alerts.get_active_alert(device.id, :device_down) do
|
||||
nil ->
|
||||
:ok
|
||||
|
||||
alert ->
|
||||
Logger.info(
|
||||
"Resolving stuck device_down alert for #{device.name} - device is UP but alert was not resolved",
|
||||
device_id: device.id,
|
||||
alert_id: alert.id
|
||||
)
|
||||
|
||||
# Create device_up alert (informational)
|
||||
message =
|
||||
if device.snmp_enabled,
|
||||
do: "Device is now responding to SNMP",
|
||||
else: "Device is now responding to ping"
|
||||
|
||||
Alerts.create_alert(%{
|
||||
device_id: device.id,
|
||||
alert_type: :device_up,
|
||||
triggered_at: now,
|
||||
resolved_at: now,
|
||||
message: message
|
||||
})
|
||||
|
||||
# Resolve the stuck alert
|
||||
Alerts.resolve_alert(alert)
|
||||
|
||||
Phoenix.PubSub.broadcast(
|
||||
Towerops.PubSub,
|
||||
"alerts:org:#{device.organization_id}:resolved",
|
||||
{:alert_resolved, device.id, :device_down}
|
||||
)
|
||||
end
|
||||
|
||||
# Also resolve any stuck device_up alerts (should have been created with resolved_at)
|
||||
case Alerts.get_active_alert(device.id, :device_up) do
|
||||
nil ->
|
||||
:ok
|
||||
|
||||
alert ->
|
||||
Logger.info(
|
||||
"Resolving stuck device_up alert for #{device.name} - should have been auto-resolved",
|
||||
device_id: device.id,
|
||||
alert_id: alert.id
|
||||
)
|
||||
|
||||
Alerts.resolve_alert(alert)
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp fetch_device(device_id) do
|
||||
case Devices.get_device_with_details(device_id) do
|
||||
nil -> {:error, :device_not_found}
|
||||
|
|
@ -1605,13 +1664,13 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
defp process_polling_result(device, result, _socket) do
|
||||
Devices.update_snmp_poll_time(device)
|
||||
|
||||
# Successful SNMP poll means device is up - check for status change and resolve alerts
|
||||
# Successful SNMP poll means device is up - update status and resolve any stuck alerts
|
||||
old_status = device.status
|
||||
new_status = :up
|
||||
|
||||
# Update device status if needed
|
||||
if old_status != new_status do
|
||||
Devices.update_device_status(device, new_status)
|
||||
handle_agent_status_change(device, old_status, new_status)
|
||||
|
||||
Phoenix.PubSub.broadcast(
|
||||
Towerops.PubSub,
|
||||
|
|
@ -1620,6 +1679,10 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
)
|
||||
end
|
||||
|
||||
# Always check for and resolve stuck device_down alerts after successful poll
|
||||
# This handles edge cases where alerts got stuck (device already UP but alert not resolved)
|
||||
resolve_any_stuck_down_alerts(device)
|
||||
|
||||
snmp_device = device.snmp_device
|
||||
oid_values = normalize_oid_keys(result.oid_values)
|
||||
# Use server time to avoid clock drift issues between agents
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue