diff --git a/lib/towerops_web/channels/agent_channel.ex b/lib/towerops_web/channels/agent_channel.ex index f6f27720..92150f83 100644 --- a/lib/towerops_web/channels/agent_channel.ex +++ b/lib/towerops_web/channels/agent_channel.ex @@ -1484,6 +1484,65 @@ defmodule ToweropsWeb.AgentChannel do ) end + # Resolve any stuck alerts for a device that's currently up + # This handles edge cases where device is UP but alerts didn't resolve (e.g., due to bugs) + defp resolve_any_stuck_down_alerts(device) do + now = DateTime.truncate(DateTime.utc_now(), :second) + + # Resolve stuck device_down alerts + case Alerts.get_active_alert(device.id, :device_down) do + nil -> + :ok + + alert -> + Logger.info( + "Resolving stuck device_down alert for #{device.name} - device is UP but alert was not resolved", + device_id: device.id, + alert_id: alert.id + ) + + # Create device_up alert (informational) + message = + if device.snmp_enabled, + do: "Device is now responding to SNMP", + else: "Device is now responding to ping" + + Alerts.create_alert(%{ + device_id: device.id, + alert_type: :device_up, + triggered_at: now, + resolved_at: now, + message: message + }) + + # Resolve the stuck alert + Alerts.resolve_alert(alert) + + Phoenix.PubSub.broadcast( + Towerops.PubSub, + "alerts:org:#{device.organization_id}:resolved", + {:alert_resolved, device.id, :device_down} + ) + end + + # Also resolve any stuck device_up alerts (should have been created with resolved_at) + case Alerts.get_active_alert(device.id, :device_up) do + nil -> + :ok + + alert -> + Logger.info( + "Resolving stuck device_up alert for #{device.name} - should have been auto-resolved", + device_id: device.id, + alert_id: alert.id + ) + + Alerts.resolve_alert(alert) + end + + :ok + end + defp fetch_device(device_id) do case Devices.get_device_with_details(device_id) do nil -> {:error, :device_not_found} @@ -1605,13 +1664,13 @@ defmodule ToweropsWeb.AgentChannel do defp process_polling_result(device, result, _socket) do Devices.update_snmp_poll_time(device) - # Successful SNMP poll means device is up - check for status change and resolve alerts + # Successful SNMP poll means device is up - update status and resolve any stuck alerts old_status = device.status new_status = :up + # Update device status if needed if old_status != new_status do Devices.update_device_status(device, new_status) - handle_agent_status_change(device, old_status, new_status) Phoenix.PubSub.broadcast( Towerops.PubSub, @@ -1620,6 +1679,10 @@ defmodule ToweropsWeb.AgentChannel do ) end + # Always check for and resolve stuck device_down alerts after successful poll + # This handles edge cases where alerts got stuck (device already UP but alert not resolved) + resolve_any_stuck_down_alerts(device) + snmp_device = device.snmp_device oid_values = normalize_oid_keys(result.oid_values) # Use server time to avoid clock drift issues between agents