towerops/lib/towerops/monitoring/supervisor.ex
Graham McIntire 91759ac16c
Add error handling and logging to monitoring supervisor
Improved startup reliability and debugging for SNMP pollers:
- Added try/rescue blocks around monitor and poller startup
- Added logging for successful startup and errors
- Added error logging to PollerWorker's perform_poll
- Added debug logging for successful sensor/interface polls

This will help identify issues when pollers fail to start or
encounter errors during polling, which was causing silent failures
where sensors weren't being collected.
2026-01-04 13:24:33 -06:00

126 lines
3.5 KiB
Elixir

defmodule Towerops.Monitoring.Supervisor do
@moduledoc """
Supervisor for managing equipment monitoring workers.
"""
use Supervisor
alias Towerops.Monitoring.EquipmentMonitor
alias Towerops.Snmp.PollerRegistry
alias Towerops.Snmp.PollerSupervisor
alias Towerops.Snmp.PollerWorker
require Logger
def start_link(init_arg) do
Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__)
end
@impl true
def init(_init_arg) do
children = [
# Registry for naming monitor processes
{Registry, keys: :unique, name: Towerops.Monitoring.Registry},
# Registry for naming SNMP poller processes
{Registry, keys: :unique, name: PollerRegistry},
# DynamicSupervisor for monitor workers
{DynamicSupervisor, name: Towerops.Monitoring.DynamicSupervisor, strategy: :one_for_one},
# DynamicSupervisor for SNMP poller workers
{DynamicSupervisor, name: PollerSupervisor, strategy: :one_for_one}
]
# Start all monitors after supervisor is initialized (but not in test mode)
# In test mode, the Repo uses Sandbox pool which we can detect
_ =
if !test_mode?() do
Task.start(fn ->
# Wait briefly for the supervisor tree to be fully initialized
Process.sleep(100)
try do
start_all_monitors()
Logger.info("Started all equipment monitors")
rescue
error ->
Logger.error("Failed to start monitors: #{inspect(error)}")
end
try do
start_all_snmp_pollers()
Logger.info("Started all SNMP pollers")
rescue
error ->
Logger.error("Failed to start SNMP pollers: #{inspect(error)}")
end
end)
end
Supervisor.init(children, strategy: :one_for_one)
end
@doc """
Starts monitoring for a specific equipment.
"""
def start_monitor(equipment_id) do
spec = {EquipmentMonitor, equipment_id}
DynamicSupervisor.start_child(Towerops.Monitoring.DynamicSupervisor, spec)
end
@doc """
Stops monitoring for a specific equipment.
"""
def stop_monitor(equipment_id) do
case Registry.lookup(Towerops.Monitoring.Registry, equipment_id) do
[{pid, _}] ->
DynamicSupervisor.terminate_child(Towerops.Monitoring.DynamicSupervisor, pid)
[] ->
:ok
end
end
@doc """
Starts monitors for all equipment that has monitoring enabled.
"""
def start_all_monitors do
Enum.each(Towerops.Equipment.list_monitored_equipment(), fn equipment ->
start_monitor(equipment.id)
end)
end
@doc """
Starts an SNMP poller for a specific equipment.
"""
def start_snmp_poller(equipment_id) do
spec = {PollerWorker, equipment_id}
DynamicSupervisor.start_child(PollerSupervisor, spec)
end
@doc """
Stops an SNMP poller for a specific equipment.
"""
def stop_snmp_poller(equipment_id) do
case Registry.lookup(PollerRegistry, equipment_id) do
[{pid, _}] ->
DynamicSupervisor.terminate_child(PollerSupervisor, pid)
[] ->
:ok
end
end
@doc """
Starts SNMP pollers for all equipment that has SNMP enabled.
"""
def start_all_snmp_pollers do
Enum.each(Towerops.Equipment.list_snmp_enabled_equipment(), fn equipment ->
start_snmp_poller(equipment.id)
end)
end
# Check if running in test mode by inspecting the Repo pool configuration
defp test_mode? do
config = Application.get_env(:towerops, Towerops.Repo, [])
Keyword.get(config, :pool) == Ecto.Adapters.SQL.Sandbox
end
end