221 lines
5.5 KiB
Elixir
221 lines
5.5 KiB
Elixir
defmodule Towerops.Monitoring.DeviceMonitor do
|
|
@moduledoc """
|
|
GenServer that monitors a single piece of device by periodically pinging it.
|
|
"""
|
|
use GenServer
|
|
|
|
alias Towerops.Alerts
|
|
alias Towerops.Devices
|
|
alias Towerops.Monitoring
|
|
|
|
require Logger
|
|
|
|
# Allow dependency injection for testing
|
|
@ping_module Application.compile_env(:towerops, :ping_module, Towerops.Monitoring.Ping)
|
|
@poller_module Application.compile_env(:towerops, :poller_module, Towerops.Snmp.Poller)
|
|
|
|
# Suppress warnings for Mox modules that are defined at runtime during tests
|
|
@compile {:no_warn_undefined, [Towerops.Monitoring.PingMock, Towerops.Snmp.PollerMock]}
|
|
|
|
# Client API
|
|
|
|
@doc """
|
|
Starts a monitor for the given device ID.
|
|
"""
|
|
def start_link(device_id) do
|
|
GenServer.start_link(__MODULE__, device_id, name: via_tuple(device_id))
|
|
end
|
|
|
|
@doc """
|
|
Triggers an immediate check for the device.
|
|
"""
|
|
def trigger_check(device_id) do
|
|
# Check if monitor process is running
|
|
case Registry.lookup(Towerops.Monitoring.Registry, device_id) do
|
|
[{_pid, _}] ->
|
|
# Process exists, send cast
|
|
GenServer.cast(via_tuple(device_id), :check_now)
|
|
|
|
[] ->
|
|
# Process doesn't exist (monitoring disabled), perform check directly
|
|
Task.start(fn -> perform_check(device_id) end)
|
|
end
|
|
end
|
|
|
|
# Server Callbacks
|
|
|
|
@impl true
|
|
def init(device_id) do
|
|
device = Devices.get_device!(device_id)
|
|
|
|
if device.monitoring_enabled do
|
|
# Perform immediate check when monitoring starts
|
|
send(self(), :check_device)
|
|
end
|
|
|
|
{:ok, %{device_id: device_id}}
|
|
end
|
|
|
|
@impl true
|
|
def handle_info(:check_device, state) do
|
|
_ = perform_check(state.device_id)
|
|
{:noreply, state}
|
|
end
|
|
|
|
@impl true
|
|
def handle_cast(:check_now, state) do
|
|
_ = perform_check(state.device_id)
|
|
{:noreply, state}
|
|
end
|
|
|
|
# Private Functions
|
|
|
|
defp perform_check(device_id) do
|
|
device = Devices.get_device!(device_id)
|
|
|
|
# Use SNMP if enabled, otherwise fallback to ping
|
|
check_result =
|
|
if device.snmp_enabled do
|
|
client_opts = @poller_module.build_client_opts(device)
|
|
@poller_module.check_device(client_opts)
|
|
else
|
|
@ping_module.ping(device.ip_address)
|
|
end
|
|
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
status =
|
|
case check_result do
|
|
{:ok, _time} -> :success
|
|
{:error, _reason} -> :failure
|
|
end
|
|
|
|
# Save the check result
|
|
case Monitoring.create_check(%{
|
|
device_id: device_id,
|
|
status: status,
|
|
response_time_ms: nil,
|
|
checked_at: now
|
|
}) do
|
|
{:ok, _check} ->
|
|
:ok
|
|
|
|
{:error, changeset} ->
|
|
Logger.error("Failed to create monitoring check for device #{device_id}: #{inspect(changeset.errors)}")
|
|
end
|
|
|
|
# device status if it changed
|
|
new_status = if status == :success, do: :up, else: :down
|
|
old_status = device.status
|
|
|
|
_ = Devices.update_device_status(device, new_status)
|
|
|
|
# Create alerts if status changed
|
|
_ =
|
|
if old_status != new_status do
|
|
handle_status_change(device, old_status, new_status)
|
|
end
|
|
|
|
# Broadcast status change via PubSub
|
|
_ =
|
|
Phoenix.PubSub.broadcast(
|
|
Towerops.PubSub,
|
|
"device:#{device_id}",
|
|
{:device_status_changed, device_id, new_status, nil}
|
|
)
|
|
|
|
# Only schedule next check if monitoring is enabled
|
|
if device.monitoring_enabled do
|
|
schedule_next_check(device.check_interval_seconds)
|
|
end
|
|
end
|
|
|
|
defp handle_status_change(device, old_status, new_status) do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
case {old_status, new_status} do
|
|
{_, :down} ->
|
|
handle_equipment_down(device, now)
|
|
|
|
{_, :up} ->
|
|
handle_equipment_up(device, now)
|
|
end
|
|
end
|
|
|
|
defp handle_equipment_down(device, now) do
|
|
if Alerts.has_active_alert?(device.id, :device_down) do
|
|
:ok
|
|
else
|
|
create_device_down_alert(device, now)
|
|
end
|
|
end
|
|
|
|
defp create_device_down_alert(device, now) do
|
|
alert_message = get_down_alert_message(device)
|
|
|
|
{:ok, _alert} =
|
|
Alerts.create_alert(%{
|
|
device_id: device.id,
|
|
alert_type: :device_down,
|
|
triggered_at: now,
|
|
message: alert_message
|
|
})
|
|
|
|
Phoenix.PubSub.broadcast(
|
|
Towerops.PubSub,
|
|
"alerts:new",
|
|
{:new_alert, device.id, :device_down}
|
|
)
|
|
end
|
|
|
|
defp get_down_alert_message(device) do
|
|
if device.snmp_enabled do
|
|
"Device is not responding to SNMP"
|
|
else
|
|
"Device is not responding to ping"
|
|
end
|
|
end
|
|
|
|
defp handle_equipment_up(device, now) do
|
|
recovery_message = get_recovery_message(device)
|
|
|
|
{:ok, _alert} =
|
|
Alerts.create_alert(%{
|
|
device_id: device.id,
|
|
alert_type: :device_up,
|
|
triggered_at: now,
|
|
message: recovery_message
|
|
})
|
|
|
|
resolve_down_alert(device)
|
|
|
|
Phoenix.PubSub.broadcast(
|
|
Towerops.PubSub,
|
|
"alerts:resolved",
|
|
{:alert_resolved, device.id, :device_down}
|
|
)
|
|
end
|
|
|
|
defp get_recovery_message(device) do
|
|
if device.snmp_enabled do
|
|
"Device is now responding to SNMP"
|
|
else
|
|
"Device is now responding to ping"
|
|
end
|
|
end
|
|
|
|
defp resolve_down_alert(device) do
|
|
case Alerts.get_active_alert(device.id, :device_down) do
|
|
nil -> :ok
|
|
alert -> Alerts.resolve_alert(alert)
|
|
end
|
|
end
|
|
|
|
defp schedule_next_check(interval_seconds) do
|
|
Process.send_after(self(), :check_device, interval_seconds * 1000)
|
|
end
|
|
|
|
defp via_tuple(device_id) do
|
|
{:via, Registry, {Towerops.Monitoring.Registry, device_id}}
|
|
end
|
|
end
|