towerops/lib/towerops/workers/check_executor_worker.ex
Graham McIntire d7741cdc0e
feat: implement unified checks system (Phase 1-4)
This commit implements the unified checks architecture that consolidates
SNMP monitoring with HTTP/TCP/DNS checks under a single "check" abstraction,
enabling consistent graphing, alerting, and management across all check types.

Database Changes:
- Add source_type and source_id to checks table for tracking auto-discovered
  vs manually created checks
- Add value field to check_results for storing numeric sensor readings
- Maintain backward compatibility with existing check_results data

New SNMP Executors:
- SnmpSensorExecutor: Poll sensor OIDs and return formatted values with
  status determination (OK/WARNING/CRITICAL based on limits)
- SnmpInterfaceExecutor: Poll interface stats (bandwidth, packets, errors)
- SnmpProcessorExecutor: Poll CPU/processor usage
- SnmpStorageExecutor: Poll disk/memory usage with percentage calculations

Check Execution Worker:
- CheckExecutorWorker: Unified Oban worker that dispatches to appropriate
  executor based on check_type (snmp_sensor, snmp_interface, http, tcp, etc.)
- Self-schedules next execution with distributed polling offsets
- Records results in check_results TimescaleDB hypertable
- Updates check state (OK/WARNING/CRITICAL/UNKNOWN)

Discovery Integration:
- Auto-creates checks during SNMP discovery for sensors, interfaces,
  processors, and storage
- Links checks to source entities via source_id for data lookup
- Enables/disables checks based on discovery results

UI Enhancements:
- Checks tab on device detail page with grouped display
- FormComponent for adding manual HTTP/TCP/DNS checks
- Empty state with "Run Discovery" prompt
- Check status badges and last checked times

Graphing:
- Update GraphLive to accept check_id parameter
- Query check_results table for time-series data
- Support all check types (SNMP, HTTP response times, etc.)

Testing:
- Comprehensive test suite for SnmpSensorExecutor (5 tests)
- Test suite for CheckExecutorWorker (7 tests)
- Test coverage for discovery check creation (6 tests)
- Remove deprecated monitoring_test.exs testing old API

Bug Fixes:
- Fix SNMP executors reading credentials from correct Device schema fields
  (device.snmp_version instead of device.snmp_device.version)
- Update agent channel test to query MonitoringCheck table directly

Code Quality:
- Extract add_snmp_credentials helper to reduce cyclomatic complexity
- Use map-based lookups for sensor formatting and check type grouping
- Apply pattern matching in dispatcher to reduce complexity
- All credo checks passing with no issues

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-02-12 16:58:40 -06:00

165 lines
5.2 KiB
Elixir

defmodule Towerops.Workers.CheckExecutorWorker do
@moduledoc """
Unified worker for executing all check types.
Dispatches to appropriate executor based on check_type:
- snmp_sensor → SnmpSensorExecutor
- snmp_interface → SnmpInterfaceExecutor
- snmp_processor → SnmpProcessorExecutor
- snmp_storage → SnmpStorageExecutor
- http → HttpExecutor
- tcp → TcpExecutor
- dns → DnsExecutor
- ping → PingExecutor (future)
Records results in check_results TimescaleDB hypertable and updates
check state (OK/WARNING/CRITICAL/UNKNOWN).
Self-schedules next execution based on check interval with distributed
polling offsets to prevent thundering herd.
"""
use Oban.Worker,
queue: :check_executors,
max_attempts: 3
alias Towerops.Monitoring
alias Towerops.Monitoring.Executors.DnsExecutor
alias Towerops.Monitoring.Executors.HttpExecutor
alias Towerops.Monitoring.Executors.SnmpInterfaceExecutor
alias Towerops.Monitoring.Executors.SnmpProcessorExecutor
alias Towerops.Monitoring.Executors.SnmpSensorExecutor
alias Towerops.Monitoring.Executors.SnmpStorageExecutor
alias Towerops.Monitoring.Executors.TcpExecutor
alias Towerops.Workers.PollingOffset
require Logger
@impl Oban.Worker
def perform(%Oban.Job{args: %{"check_id" => check_id}}) do
case Monitoring.get_check(check_id) do
nil ->
Logger.debug("Check #{check_id} deleted, skipping execution")
:ok
check ->
if check.enabled do
execute_and_record(check)
schedule_next_check(check)
else
Logger.debug("Check #{check_id} disabled, skipping execution")
end
:ok
end
end
defp execute_and_record(check) do
Logger.debug("Executing check #{check.id} (#{check.check_type})")
check
|> dispatch_executor()
|> record_result(check)
end
defp dispatch_executor(%{check_type: "snmp_sensor"} = check), do: SnmpSensorExecutor.execute(check)
defp dispatch_executor(%{check_type: "snmp_interface"} = check), do: SnmpInterfaceExecutor.execute(check)
defp dispatch_executor(%{check_type: "snmp_processor"} = check), do: SnmpProcessorExecutor.execute(check)
defp dispatch_executor(%{check_type: "snmp_storage"} = check), do: SnmpStorageExecutor.execute(check)
defp dispatch_executor(%{check_type: "http"} = check), do: execute_http_check(check)
defp dispatch_executor(%{check_type: "tcp"} = check), do: execute_tcp_check(check)
defp dispatch_executor(%{check_type: "dns"} = check), do: execute_dns_check(check)
defp dispatch_executor(%{check_type: "ping"}), do: {:error, "Ping executor not yet implemented"}
defp dispatch_executor(%{check_type: type}), do: {:error, "Unknown check type: #{type}"}
defp record_result({:ok, %{value: value, status: status, output: output, response_time_ms: time}}, check) do
Monitoring.create_check_result(%{
check_id: check.id,
organization_id: check.organization_id,
value: value,
status: status,
output: output,
response_time_ms: time,
checked_at: DateTime.utc_now(),
agent_token_id: check.agent_token_id
})
Monitoring.update_check_state(check, status, output)
Logger.debug("Check #{check.id} completed: status=#{status}, value=#{value}, output=#{output}")
end
defp record_result({:error, reason}, check) do
Logger.warning("Check #{check.id} failed: #{inspect(reason)}")
Monitoring.create_check_result(%{
check_id: check.id,
organization_id: check.organization_id,
status: 3,
output: "Error: #{inspect(reason)}",
checked_at: DateTime.utc_now(),
agent_token_id: check.agent_token_id
})
Monitoring.update_check_state(check, 3, "Error: #{inspect(reason)}")
end
# Adapter for HTTP executor which returns different format
defp execute_http_check(check) do
case HttpExecutor.execute(check.config, check.timeout_ms) do
{:ok, response_time, output} ->
{:ok,
%{
value: nil,
status: 0,
output: output,
response_time_ms: response_time
}}
{:error, reason} ->
{:error, reason}
end
end
# Adapter for TCP executor which returns different format
defp execute_tcp_check(check) do
case TcpExecutor.execute(check.config, check.timeout_ms) do
{:ok, response_time, output} ->
{:ok,
%{
value: nil,
status: 0,
output: output,
response_time_ms: response_time
}}
{:error, reason} ->
{:error, reason}
end
end
# Adapter for DNS executor which returns different format
defp execute_dns_check(check) do
case DnsExecutor.execute(check.config, check.timeout_ms) do
{:ok, response_time, output} ->
{:ok,
%{
value: nil,
status: 0,
output: output,
response_time_ms: response_time
}}
{:error, reason} ->
{:error, reason}
end
end
defp schedule_next_check(check) do
# Calculate staggered offset based on check ID
offset = PollingOffset.calculate_offset(check.id, check.interval_seconds)
%{check_id: check.id}
|> new(schedule_in: offset)
|> Oban.insert()
end
end