fix(alerts): auto-resolve device_down alerts when agent polls device successfully
When agents poll devices via SNMP and get successful results, the code was only processing sensor/interface data without updating device status or resolving alerts. This caused device_down alerts to persist even after devices came back online. This fix: - Updates device status to 'up' when SNMP polling succeeds - Auto-resolves any active device_down alerts - Creates device_up alert (matching DeviceMonitorWorker behavior) - Broadcasts device_status_changed via PubSub Also fixes device_up alert creation to include resolved_at timestamp (matching DeviceMonitorWorker behavior). Test coverage: Added test verifying auto-resolution when device recovers.
This commit is contained in:
parent
04564a402f
commit
3f1d97218d
2 changed files with 70 additions and 0 deletions
|
|
@ -1468,6 +1468,7 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
device_id: device.id,
|
||||
alert_type: :device_up,
|
||||
triggered_at: now,
|
||||
resolved_at: now,
|
||||
message: message
|
||||
})
|
||||
|
||||
|
|
@ -1604,6 +1605,21 @@ defmodule ToweropsWeb.AgentChannel do
|
|||
defp process_polling_result(device, result, _socket) do
|
||||
Devices.update_snmp_poll_time(device)
|
||||
|
||||
# Successful SNMP poll means device is up - check for status change and resolve alerts
|
||||
old_status = device.status
|
||||
new_status = :up
|
||||
|
||||
if old_status != new_status do
|
||||
Devices.update_device_status(device, new_status)
|
||||
handle_agent_status_change(device, old_status, new_status)
|
||||
|
||||
Phoenix.PubSub.broadcast(
|
||||
Towerops.PubSub,
|
||||
"device:#{device.id}",
|
||||
{:device_status_changed, device.id, new_status, nil}
|
||||
)
|
||||
end
|
||||
|
||||
snmp_device = device.snmp_device
|
||||
oid_values = normalize_oid_keys(result.oid_values)
|
||||
# Use server time to avoid clock drift issues between agents
|
||||
|
|
|
|||
|
|
@ -916,6 +916,60 @@ defmodule ToweropsWeb.AgentChannelTest do
|
|||
updated_sensor = Towerops.Repo.get!(Sensor, sensor_with_divisor.id)
|
||||
assert updated_sensor.last_value == 12.0
|
||||
end
|
||||
|
||||
test "poll result auto-resolves device_down alert when device comes back up", %{
|
||||
socket: socket,
|
||||
discovered_device: device,
|
||||
sensor: sensor
|
||||
} do
|
||||
# Set device status to down
|
||||
{:ok, device} = Towerops.Devices.update_device_status(device, :down)
|
||||
|
||||
# Create a device_down alert
|
||||
{:ok, alert} =
|
||||
Towerops.Alerts.create_alert(%{
|
||||
device_id: device.id,
|
||||
alert_type: "device_down",
|
||||
triggered_at: DateTime.utc_now(),
|
||||
message: "Device is not responding to SNMP"
|
||||
})
|
||||
|
||||
assert is_nil(alert.resolved_at)
|
||||
assert device.status == :down
|
||||
|
||||
# Send successful SNMP polling result
|
||||
result = %SnmpResult{
|
||||
device_id: device.id,
|
||||
job_type: :POLL,
|
||||
job_id: "poll:#{device.id}",
|
||||
timestamp: DateTime.to_unix(DateTime.utc_now()),
|
||||
oid_values: %{
|
||||
sensor.sensor_oid => "45"
|
||||
}
|
||||
}
|
||||
|
||||
push(socket, "result", encode_payload(result))
|
||||
Process.sleep(50)
|
||||
|
||||
# Verify device status updated to up
|
||||
updated_device = Towerops.Devices.get_device(device.id)
|
||||
assert updated_device.status == :up
|
||||
|
||||
# Verify device_down alert was auto-resolved
|
||||
updated_alert = Towerops.Alerts.get_alert(alert.id)
|
||||
assert updated_alert.resolved_at
|
||||
|
||||
# Verify device_up alert was created
|
||||
up_alerts =
|
||||
device.id
|
||||
|> Towerops.Alerts.list_devices_alerts()
|
||||
|> Enum.filter(&(&1.alert_type == "device_up"))
|
||||
|
||||
assert length(up_alerts) == 1
|
||||
up_alert = hd(up_alerts)
|
||||
assert up_alert.message == "Device is now responding to SNMP"
|
||||
assert up_alert.resolved_at
|
||||
end
|
||||
end
|
||||
|
||||
# ── Stage 10: SNMPv3 credential path ────────────────────────────────
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue