From 192cf5b06494af7d6884c3a030473f54df0463dd Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Thu, 5 Mar 2026 16:54:00 -0600
Subject: [PATCH] fix(alerts): resolve stuck alerts on every successful poll,
 not just status change

Previous fix only resolved alerts when device status changed from down to up.
This didn't help devices already marked as 'up' with stuck unresolved alerts.

Now:
- Always check for stuck device_down alerts after successful SNMP poll
- Also check for stuck device_up alerts (should always be auto-resolved)
- Logs when stuck alerts are found and resolved
- Works for existing stuck alerts without waiting for status to change

This fixes the 8 stuck alerts in production - they'll auto-resolve on next poll.
---
 lib/towerops_web/channels/agent_channel.ex | 67 +++++++++++++++++++++-
 1 file changed, 65 insertions(+), 2 deletions(-)

diff --git a/lib/towerops_web/channels/agent_channel.ex b/lib/towerops_web/channels/agent_channel.ex
index f6f27720..92150f83 100644
--- a/lib/towerops_web/channels/agent_channel.ex
+++ b/lib/towerops_web/channels/agent_channel.ex
@@ -1484,6 +1484,65 @@ defmodule ToweropsWeb.AgentChannel do
     )
   end
 
+  # Resolve any stuck alerts for a device that's currently up
+  # This handles edge cases where device is UP but alerts didn't resolve (e.g., due to bugs)
+  defp resolve_any_stuck_down_alerts(device) do
+    now = DateTime.truncate(DateTime.utc_now(), :second)
+
+    # Resolve stuck device_down alerts
+    case Alerts.get_active_alert(device.id, :device_down) do
+      nil ->
+        :ok
+
+      alert ->
+        Logger.info(
+          "Resolving stuck device_down alert for #{device.name} - device is UP but alert was not resolved",
+          device_id: device.id,
+          alert_id: alert.id
+        )
+
+        # Create device_up alert (informational)
+        message =
+          if device.snmp_enabled,
+            do: "Device is now responding to SNMP",
+            else: "Device is now responding to ping"
+
+        Alerts.create_alert(%{
+          device_id: device.id,
+          alert_type: :device_up,
+          triggered_at: now,
+          resolved_at: now,
+          message: message
+        })
+
+        # Resolve the stuck alert
+        Alerts.resolve_alert(alert)
+
+        Phoenix.PubSub.broadcast(
+          Towerops.PubSub,
+          "alerts:org:#{device.organization_id}:resolved",
+          {:alert_resolved, device.id, :device_down}
+        )
+    end
+
+    # Also resolve any stuck device_up alerts (should have been created with resolved_at)
+    case Alerts.get_active_alert(device.id, :device_up) do
+      nil ->
+        :ok
+
+      alert ->
+        Logger.info(
+          "Resolving stuck device_up alert for #{device.name} - should have been auto-resolved",
+          device_id: device.id,
+          alert_id: alert.id
+        )
+
+        Alerts.resolve_alert(alert)
+    end
+
+    :ok
+  end
+
   defp fetch_device(device_id) do
     case Devices.get_device_with_details(device_id) do
       nil -> {:error, :device_not_found}
@@ -1605,13 +1664,13 @@ defmodule ToweropsWeb.AgentChannel do
   defp process_polling_result(device, result, _socket) do
     Devices.update_snmp_poll_time(device)
 
-    # Successful SNMP poll means device is up - check for status change and resolve alerts
+    # Successful SNMP poll means device is up - update status and resolve any stuck alerts
     old_status = device.status
     new_status = :up
 
+    # Update device status if needed
     if old_status != new_status do
       Devices.update_device_status(device, new_status)
-      handle_agent_status_change(device, old_status, new_status)
 
       Phoenix.PubSub.broadcast(
         Towerops.PubSub,
@@ -1620,6 +1679,10 @@ defmodule ToweropsWeb.AgentChannel do
       )
     end
 
+    # Always check for and resolve stuck device_down alerts after successful poll
+    # This handles edge cases where alerts got stuck (device already UP but alert not resolved)
+    resolve_any_stuck_down_alerts(device)
+
     snmp_device = device.snmp_device
     oid_values = normalize_oid_keys(result.oid_values)
     # Use server time to avoid clock drift issues between agents