towerops/lib/towerops/monitoring/check.ex
Graham McIntire d7741cdc0e
feat: implement unified checks system (Phase 1-4)
This commit implements the unified checks architecture that consolidates
SNMP monitoring with HTTP/TCP/DNS checks under a single "check" abstraction,
enabling consistent graphing, alerting, and management across all check types.

Database Changes:
- Add source_type and source_id to checks table for tracking auto-discovered
  vs manually created checks
- Add value field to check_results for storing numeric sensor readings
- Maintain backward compatibility with existing check_results data

New SNMP Executors:
- SnmpSensorExecutor: Poll sensor OIDs and return formatted values with
  status determination (OK/WARNING/CRITICAL based on limits)
- SnmpInterfaceExecutor: Poll interface stats (bandwidth, packets, errors)
- SnmpProcessorExecutor: Poll CPU/processor usage
- SnmpStorageExecutor: Poll disk/memory usage with percentage calculations

Check Execution Worker:
- CheckExecutorWorker: Unified Oban worker that dispatches to appropriate
  executor based on check_type (snmp_sensor, snmp_interface, http, tcp, etc.)
- Self-schedules next execution with distributed polling offsets
- Records results in check_results TimescaleDB hypertable
- Updates check state (OK/WARNING/CRITICAL/UNKNOWN)

Discovery Integration:
- Auto-creates checks during SNMP discovery for sensors, interfaces,
  processors, and storage
- Links checks to source entities via source_id for data lookup
- Enables/disables checks based on discovery results

UI Enhancements:
- Checks tab on device detail page with grouped display
- FormComponent for adding manual HTTP/TCP/DNS checks
- Empty state with "Run Discovery" prompt
- Check status badges and last checked times

Graphing:
- Update GraphLive to accept check_id parameter
- Query check_results table for time-series data
- Support all check types (SNMP, HTTP response times, etc.)

Testing:
- Comprehensive test suite for SnmpSensorExecutor (5 tests)
- Test suite for CheckExecutorWorker (7 tests)
- Test coverage for discovery check creation (6 tests)
- Remove deprecated monitoring_test.exs testing old API

Bug Fixes:
- Fix SNMP executors reading credentials from correct Device schema fields
  (device.snmp_version instead of device.snmp_device.version)
- Update agent channel test to query MonitoringCheck table directly

Code Quality:
- Extract add_snmp_credentials helper to reduce cyclomatic complexity
- Use map-based lookups for sensor formatting and check type grouping
- Apply pattern matching in dispatcher to reduce complexity
- All credo checks passing with no issues

Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-02-12 16:58:40 -06:00

186 lines
5.6 KiB
Elixir

defmodule Towerops.Monitoring.Check do
@moduledoc """
Generic check schema following Icinga2's Checkable pattern.
Supports multiple check types (http, tcp, dns, passive, ping) with
type-specific configuration stored in JSONB `config` field.
Following Icinga2's design:
- Generic check fields (interval, retry, max_attempts, state tracking)
- Check-specific config in `config` map (like Icinga2's vars)
- Flapping detection
- Soft/hard state transitions
- Optional device association (can be standalone)
"""
use Ecto.Schema
import Ecto.Changeset
alias Towerops.Agents.AgentToken
alias Towerops.Devices.Device
alias Towerops.Organizations.Organization
@primary_key {:id, :binary_id, autogenerate: true}
@foreign_key_type :binary_id
@check_types ~w(http tcp dns passive ping snmp_sensor snmp_interface snmp_processor snmp_storage)
schema "checks" do
field :name, :string
field :check_type, :string
field :description, :string
# Source tracking for auto-discovered checks
field :source_type, :string
field :source_id, :binary_id
# Generic check settings (from Icinga2's Checkable)
field :interval_seconds, :integer, default: 60
field :retry_interval_seconds, :integer, default: 30
field :max_check_attempts, :integer, default: 3
field :timeout_ms, :integer, default: 5000
field :enabled, :boolean, default: true
# Check-specific configuration (JSONB, like Icinga2's vars)
field :config, :map, default: %{}
# State tracking (soft/hard states like Icinga2)
field :current_state, :integer, default: 3
field :current_state_type, :string, default: "soft"
field :current_check_attempt, :integer, default: 1
field :last_check_at, :utc_datetime
field :last_state_change_at, :utc_datetime
field :last_hard_state_change_at, :utc_datetime
# Flapping detection
field :enable_flapping, :boolean, default: false
field :flapping_threshold_low, :float, default: 25.0
field :flapping_threshold_high, :float, default: 30.0
field :is_flapping, :boolean, default: false
field :flapping_state_changes, :integer, default: 0
field :flapping_window_start_at, :utc_datetime
# Passive checks
field :enable_passive_checks, :boolean, default: false
field :freshness_threshold_seconds, :integer
field :check_key, :string
belongs_to :organization, Organization
belongs_to :device, Device
belongs_to :agent_token, AgentToken
timestamps(type: :utc_datetime)
end
def changeset(check, attrs) do
check
|> cast(attrs, [
:name,
:check_type,
:description,
:source_type,
:source_id,
:interval_seconds,
:retry_interval_seconds,
:max_check_attempts,
:timeout_ms,
:enabled,
:config,
:enable_flapping,
:flapping_threshold_low,
:flapping_threshold_high,
:enable_passive_checks,
:freshness_threshold_seconds,
:check_key,
:organization_id,
:device_id,
:agent_token_id
])
|> validate_required([:name, :check_type, :organization_id, :config])
|> validate_inclusion(:check_type, @check_types)
|> validate_source_type()
|> validate_number(:interval_seconds, greater_than: 0)
|> validate_number(:retry_interval_seconds, greater_than: 0)
|> validate_number(:max_check_attempts, greater_than: 0)
|> validate_number(:timeout_ms, greater_than: 0)
|> unique_constraint(:check_key)
|> foreign_key_constraint(:organization_id)
|> foreign_key_constraint(:device_id)
|> foreign_key_constraint(:agent_token_id)
|> validate_config_by_type()
|> generate_check_key_if_passive()
end
defp validate_config_by_type(changeset) do
check_type = get_field(changeset, :check_type)
config = get_field(changeset, :config) || %{}
case check_type do
"http" -> validate_http_config(changeset, config)
"tcp" -> validate_tcp_config(changeset, config)
"dns" -> validate_dns_config(changeset, config)
_ -> changeset
end
end
defp validate_http_config(changeset, config) do
if Map.has_key?(config, "url") do
changeset
else
add_error(changeset, :config, "HTTP check requires 'url' in config")
end
end
defp validate_tcp_config(changeset, config) do
cond do
not Map.has_key?(config, "host") ->
add_error(changeset, :config, "TCP check requires 'host' in config")
not Map.has_key?(config, "port") ->
add_error(changeset, :config, "TCP check requires 'port' in config")
true ->
changeset
end
end
defp validate_dns_config(changeset, config) do
if Map.has_key?(config, "hostname") do
changeset
else
add_error(changeset, :config, "DNS check requires 'hostname' in config")
end
end
defp validate_source_type(changeset) do
source_type = get_field(changeset, :source_type)
if source_type && source_type not in ["auto_discovery", "manual"] do
add_error(changeset, :source_type, "must be auto_discovery or manual")
else
changeset
end
end
defp generate_check_key_if_passive(changeset) do
check_type = get_field(changeset, :check_type)
check_key = get_field(changeset, :check_key)
if check_type == "passive" and is_nil(check_key) do
put_change(changeset, :check_key, generate_random_key())
else
changeset
end
end
defp generate_random_key do
32
|> :crypto.strong_rand_bytes()
|> Base.url_encode64(padding: false)
end
def state_label(0), do: "OK"
def state_label(1), do: "WARNING"
def state_label(2), do: "CRITICAL"
def state_label(3), do: "UNKNOWN"
end