fix(alerts): auto-resolve device_down alerts when agent polls device successfully

When agents poll devices via SNMP and get successful results, the code was
only processing sensor/interface data without updating device status or
resolving alerts. This caused device_down alerts to persist even after
devices came back online.

This fix:
- Updates device status to 'up' when SNMP polling succeeds
- Auto-resolves any active device_down alerts
- Creates device_up alert (matching DeviceMonitorWorker behavior)
- Broadcasts device_status_changed via PubSub

Also fixes device_up alert creation to include resolved_at timestamp
(matching DeviceMonitorWorker behavior).

Test coverage: Added test verifying auto-resolution when device recovers.
This commit is contained in:
Graham McIntire 2026-03-05 16:47:48 -06:00
parent 04564a402f
commit 3f1d97218d
No known key found for this signature in database
2 changed files with 70 additions and 0 deletions

View file

@ -1468,6 +1468,7 @@ defmodule ToweropsWeb.AgentChannel do
device_id: device.id,
alert_type: :device_up,
triggered_at: now,
resolved_at: now,
message: message
})
@ -1604,6 +1605,21 @@ defmodule ToweropsWeb.AgentChannel do
defp process_polling_result(device, result, _socket) do
Devices.update_snmp_poll_time(device)
# Successful SNMP poll means device is up - check for status change and resolve alerts
old_status = device.status
new_status = :up
if old_status != new_status do
Devices.update_device_status(device, new_status)
handle_agent_status_change(device, old_status, new_status)
Phoenix.PubSub.broadcast(
Towerops.PubSub,
"device:#{device.id}",
{:device_status_changed, device.id, new_status, nil}
)
end
snmp_device = device.snmp_device
oid_values = normalize_oid_keys(result.oid_values)
# Use server time to avoid clock drift issues between agents

View file

@ -916,6 +916,60 @@ defmodule ToweropsWeb.AgentChannelTest do
updated_sensor = Towerops.Repo.get!(Sensor, sensor_with_divisor.id)
assert updated_sensor.last_value == 12.0
end
test "poll result auto-resolves device_down alert when device comes back up", %{
socket: socket,
discovered_device: device,
sensor: sensor
} do
# Set device status to down
{:ok, device} = Towerops.Devices.update_device_status(device, :down)
# Create a device_down alert
{:ok, alert} =
Towerops.Alerts.create_alert(%{
device_id: device.id,
alert_type: "device_down",
triggered_at: DateTime.utc_now(),
message: "Device is not responding to SNMP"
})
assert is_nil(alert.resolved_at)
assert device.status == :down
# Send successful SNMP polling result
result = %SnmpResult{
device_id: device.id,
job_type: :POLL,
job_id: "poll:#{device.id}",
timestamp: DateTime.to_unix(DateTime.utc_now()),
oid_values: %{
sensor.sensor_oid => "45"
}
}
push(socket, "result", encode_payload(result))
Process.sleep(50)
# Verify device status updated to up
updated_device = Towerops.Devices.get_device(device.id)
assert updated_device.status == :up
# Verify device_down alert was auto-resolved
updated_alert = Towerops.Alerts.get_alert(alert.id)
assert updated_alert.resolved_at
# Verify device_up alert was created
up_alerts =
device.id
|> Towerops.Alerts.list_devices_alerts()
|> Enum.filter(&(&1.alert_type == "device_up"))
assert length(up_alerts) == 1
up_alert = hd(up_alerts)
assert up_alert.message == "Device is now responding to SNMP"
assert up_alert.resolved_at
end
end
# ── Stage 10: SNMPv3 credential path ────────────────────────────────