This commit implements the unified checks architecture that consolidates SNMP monitoring with HTTP/TCP/DNS checks under a single "check" abstraction, enabling consistent graphing, alerting, and management across all check types. Database Changes: - Add source_type and source_id to checks table for tracking auto-discovered vs manually created checks - Add value field to check_results for storing numeric sensor readings - Maintain backward compatibility with existing check_results data New SNMP Executors: - SnmpSensorExecutor: Poll sensor OIDs and return formatted values with status determination (OK/WARNING/CRITICAL based on limits) - SnmpInterfaceExecutor: Poll interface stats (bandwidth, packets, errors) - SnmpProcessorExecutor: Poll CPU/processor usage - SnmpStorageExecutor: Poll disk/memory usage with percentage calculations Check Execution Worker: - CheckExecutorWorker: Unified Oban worker that dispatches to appropriate executor based on check_type (snmp_sensor, snmp_interface, http, tcp, etc.) - Self-schedules next execution with distributed polling offsets - Records results in check_results TimescaleDB hypertable - Updates check state (OK/WARNING/CRITICAL/UNKNOWN) Discovery Integration: - Auto-creates checks during SNMP discovery for sensors, interfaces, processors, and storage - Links checks to source entities via source_id for data lookup - Enables/disables checks based on discovery results UI Enhancements: - Checks tab on device detail page with grouped display - FormComponent for adding manual HTTP/TCP/DNS checks - Empty state with "Run Discovery" prompt - Check status badges and last checked times Graphing: - Update GraphLive to accept check_id parameter - Query check_results table for time-series data - Support all check types (SNMP, HTTP response times, etc.) Testing: - Comprehensive test suite for SnmpSensorExecutor (5 tests) - Test suite for CheckExecutorWorker (7 tests) - Test coverage for discovery check creation (6 tests) - Remove deprecated monitoring_test.exs testing old API Bug Fixes: - Fix SNMP executors reading credentials from correct Device schema fields (device.snmp_version instead of device.snmp_device.version) - Update agent channel test to query MonitoringCheck table directly Code Quality: - Extract add_snmp_credentials helper to reduce cyclomatic complexity - Use map-based lookups for sensor formatting and check type grouping - Apply pattern matching in dispatcher to reduce complexity - All credo checks passing with no issues Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
186 lines
5.6 KiB
Elixir
186 lines
5.6 KiB
Elixir
defmodule Towerops.Monitoring.Check do
|
|
@moduledoc """
|
|
Generic check schema following Icinga2's Checkable pattern.
|
|
|
|
Supports multiple check types (http, tcp, dns, passive, ping) with
|
|
type-specific configuration stored in JSONB `config` field.
|
|
|
|
Following Icinga2's design:
|
|
- Generic check fields (interval, retry, max_attempts, state tracking)
|
|
- Check-specific config in `config` map (like Icinga2's vars)
|
|
- Flapping detection
|
|
- Soft/hard state transitions
|
|
- Optional device association (can be standalone)
|
|
"""
|
|
use Ecto.Schema
|
|
|
|
import Ecto.Changeset
|
|
|
|
alias Towerops.Agents.AgentToken
|
|
alias Towerops.Devices.Device
|
|
alias Towerops.Organizations.Organization
|
|
|
|
@primary_key {:id, :binary_id, autogenerate: true}
|
|
@foreign_key_type :binary_id
|
|
|
|
@check_types ~w(http tcp dns passive ping snmp_sensor snmp_interface snmp_processor snmp_storage)
|
|
|
|
schema "checks" do
|
|
field :name, :string
|
|
field :check_type, :string
|
|
field :description, :string
|
|
|
|
# Source tracking for auto-discovered checks
|
|
field :source_type, :string
|
|
field :source_id, :binary_id
|
|
|
|
# Generic check settings (from Icinga2's Checkable)
|
|
field :interval_seconds, :integer, default: 60
|
|
field :retry_interval_seconds, :integer, default: 30
|
|
field :max_check_attempts, :integer, default: 3
|
|
field :timeout_ms, :integer, default: 5000
|
|
field :enabled, :boolean, default: true
|
|
|
|
# Check-specific configuration (JSONB, like Icinga2's vars)
|
|
field :config, :map, default: %{}
|
|
|
|
# State tracking (soft/hard states like Icinga2)
|
|
field :current_state, :integer, default: 3
|
|
field :current_state_type, :string, default: "soft"
|
|
field :current_check_attempt, :integer, default: 1
|
|
field :last_check_at, :utc_datetime
|
|
field :last_state_change_at, :utc_datetime
|
|
field :last_hard_state_change_at, :utc_datetime
|
|
|
|
# Flapping detection
|
|
field :enable_flapping, :boolean, default: false
|
|
field :flapping_threshold_low, :float, default: 25.0
|
|
field :flapping_threshold_high, :float, default: 30.0
|
|
field :is_flapping, :boolean, default: false
|
|
field :flapping_state_changes, :integer, default: 0
|
|
field :flapping_window_start_at, :utc_datetime
|
|
|
|
# Passive checks
|
|
field :enable_passive_checks, :boolean, default: false
|
|
field :freshness_threshold_seconds, :integer
|
|
field :check_key, :string
|
|
|
|
belongs_to :organization, Organization
|
|
belongs_to :device, Device
|
|
belongs_to :agent_token, AgentToken
|
|
|
|
timestamps(type: :utc_datetime)
|
|
end
|
|
|
|
def changeset(check, attrs) do
|
|
check
|
|
|> cast(attrs, [
|
|
:name,
|
|
:check_type,
|
|
:description,
|
|
:source_type,
|
|
:source_id,
|
|
:interval_seconds,
|
|
:retry_interval_seconds,
|
|
:max_check_attempts,
|
|
:timeout_ms,
|
|
:enabled,
|
|
:config,
|
|
:enable_flapping,
|
|
:flapping_threshold_low,
|
|
:flapping_threshold_high,
|
|
:enable_passive_checks,
|
|
:freshness_threshold_seconds,
|
|
:check_key,
|
|
:organization_id,
|
|
:device_id,
|
|
:agent_token_id
|
|
])
|
|
|> validate_required([:name, :check_type, :organization_id, :config])
|
|
|> validate_inclusion(:check_type, @check_types)
|
|
|> validate_source_type()
|
|
|> validate_number(:interval_seconds, greater_than: 0)
|
|
|> validate_number(:retry_interval_seconds, greater_than: 0)
|
|
|> validate_number(:max_check_attempts, greater_than: 0)
|
|
|> validate_number(:timeout_ms, greater_than: 0)
|
|
|> unique_constraint(:check_key)
|
|
|> foreign_key_constraint(:organization_id)
|
|
|> foreign_key_constraint(:device_id)
|
|
|> foreign_key_constraint(:agent_token_id)
|
|
|> validate_config_by_type()
|
|
|> generate_check_key_if_passive()
|
|
end
|
|
|
|
defp validate_config_by_type(changeset) do
|
|
check_type = get_field(changeset, :check_type)
|
|
config = get_field(changeset, :config) || %{}
|
|
|
|
case check_type do
|
|
"http" -> validate_http_config(changeset, config)
|
|
"tcp" -> validate_tcp_config(changeset, config)
|
|
"dns" -> validate_dns_config(changeset, config)
|
|
_ -> changeset
|
|
end
|
|
end
|
|
|
|
defp validate_http_config(changeset, config) do
|
|
if Map.has_key?(config, "url") do
|
|
changeset
|
|
else
|
|
add_error(changeset, :config, "HTTP check requires 'url' in config")
|
|
end
|
|
end
|
|
|
|
defp validate_tcp_config(changeset, config) do
|
|
cond do
|
|
not Map.has_key?(config, "host") ->
|
|
add_error(changeset, :config, "TCP check requires 'host' in config")
|
|
|
|
not Map.has_key?(config, "port") ->
|
|
add_error(changeset, :config, "TCP check requires 'port' in config")
|
|
|
|
true ->
|
|
changeset
|
|
end
|
|
end
|
|
|
|
defp validate_dns_config(changeset, config) do
|
|
if Map.has_key?(config, "hostname") do
|
|
changeset
|
|
else
|
|
add_error(changeset, :config, "DNS check requires 'hostname' in config")
|
|
end
|
|
end
|
|
|
|
defp validate_source_type(changeset) do
|
|
source_type = get_field(changeset, :source_type)
|
|
|
|
if source_type && source_type not in ["auto_discovery", "manual"] do
|
|
add_error(changeset, :source_type, "must be auto_discovery or manual")
|
|
else
|
|
changeset
|
|
end
|
|
end
|
|
|
|
defp generate_check_key_if_passive(changeset) do
|
|
check_type = get_field(changeset, :check_type)
|
|
check_key = get_field(changeset, :check_key)
|
|
|
|
if check_type == "passive" and is_nil(check_key) do
|
|
put_change(changeset, :check_key, generate_random_key())
|
|
else
|
|
changeset
|
|
end
|
|
end
|
|
|
|
defp generate_random_key do
|
|
32
|
|
|> :crypto.strong_rand_bytes()
|
|
|> Base.url_encode64(padding: false)
|
|
end
|
|
|
|
def state_label(0), do: "OK"
|
|
def state_label(1), do: "WARNING"
|
|
def state_label(2), do: "CRITICAL"
|
|
def state_label(3), do: "UNKNOWN"
|
|
end
|