From 8f9dc201af70bdfca20b22544dba356854f2eac9 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 2 Jun 2026 12:27:44 -0500 Subject: [PATCH] feat(monitoring): alert when backhaul radios lose connected clients MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Backhaul radios (ePMP, AirOS, PMP, Cambium, MikroTik, RouterOS) now monitor their connected client count with a low threshold of 1. When the count drops to 0 the SnmpSensorExecutor marks it CRITICAL, which triggers an alert through the existing check → alert pipeline. - Adds field to sensor_def type in Vendor behaviour - Sets on all clients-type wireless sensors in backhaul-capable vendor profiles - Adds default executor handling: any clients sensor at 0 → status 2 - 3 new tests for clients sensor behavior --- .gitignore | 1 + .../executors/snmp_sensor_executor.ex | 3 + lib/towerops/snmp/profiles/vendors/airos.ex | 7 +- lib/towerops/snmp/profiles/vendors/cambium.ex | 3 +- lib/towerops/snmp/profiles/vendors/epmp.ex | 5 +- .../snmp/profiles/vendors/mikrotik.ex | 3 +- lib/towerops/snmp/profiles/vendors/pmp.ex | 4 +- .../snmp/profiles/vendors/routeros.ex | 3 +- lib/towerops/snmp/profiles/vendors/vendor.ex | 17 ++- .../executors/snmp_sensor_executor_test.exs | 128 ++++++++++++++++++ 10 files changed, 160 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index 0484532a..51250184 100644 --- a/.gitignore +++ b/.gitignore @@ -93,6 +93,7 @@ profiles.json .claude/ .beads/ .agents/ +.omo/ .logs/ .direnv/ diff --git a/lib/towerops/monitoring/executors/snmp_sensor_executor.ex b/lib/towerops/monitoring/executors/snmp_sensor_executor.ex index 4726e97b..8c818423 100644 --- a/lib/towerops/monitoring/executors/snmp_sensor_executor.ex +++ b/lib/towerops/monitoring/executors/snmp_sensor_executor.ex @@ -113,6 +113,9 @@ defmodule Towerops.Monitoring.Executors.SnmpSensorExecutor do defp determine_status(value, sensor) do cond do + # For "clients" type sensors, 0 connected clients means the link is down. + # This acts as default threshold when no explicit limits are configured. + sensor.sensor_type == "clients" and value == 0 -> 2 critical_high?(value, sensor) -> 2 critical_low?(value, sensor) -> 2 warning_high?(value, sensor) -> 1 diff --git a/lib/towerops/snmp/profiles/vendors/airos.ex b/lib/towerops/snmp/profiles/vendors/airos.ex index 523c9227..02ac25f9 100644 --- a/lib/towerops/snmp/profiles/vendors/airos.ex +++ b/lib/towerops/snmp/profiles/vendors/airos.ex @@ -161,13 +161,16 @@ defmodule Towerops.Snmp.Profiles.Vendors.Airos do sensor_unit: "Mbps", sensor_divisor: 1 }, - # ubntWlStatStaCount - entry 15 + # ubntWlStatStaCount - entry 15 (station/client count) + # When client count drops to 0, the link to the far end is likely down. + # limit_low: 1 ensures this check becomes CRITICAL at 0 clients. %{ oid: "1.3.6.1.4.1.41112.1.4.5.1.15.1", sensor_type: "clients", sensor_descr: "Clients", sensor_unit: "", - sensor_divisor: 1 + sensor_divisor: 1, + metadata: %{"limit_low" => 1} }, # === AirMax table (ubntAirMaxTable) - 1.3.6.1.4.1.41112.1.4.6 === diff --git a/lib/towerops/snmp/profiles/vendors/cambium.ex b/lib/towerops/snmp/profiles/vendors/cambium.ex index aac3085d..dfa95d62 100644 --- a/lib/towerops/snmp/profiles/vendors/cambium.ex +++ b/lib/towerops/snmp/profiles/vendors/cambium.ex @@ -124,7 +124,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Cambium do sensor_descr: "Connected Subscribers", sensor_unit: "", sensor_divisor: 1, - sensor_index: "cambium_epmp_clients" + sensor_index: "cambium_epmp_clients", + metadata: %{"limit_low" => 1} } ] else diff --git a/lib/towerops/snmp/profiles/vendors/epmp.ex b/lib/towerops/snmp/profiles/vendors/epmp.ex index 4dcca8c9..9b985faa 100644 --- a/lib/towerops/snmp/profiles/vendors/epmp.ex +++ b/lib/towerops/snmp/profiles/vendors/epmp.ex @@ -124,12 +124,15 @@ defmodule Towerops.Snmp.Profiles.Vendors.Epmp do sensor_divisor: 1 }, # Connected clients count (AP only) + # When client count drops to 0, the link to the far end is likely down. + # limit_low: 1 ensures this check becomes CRITICAL at 0 clients. %{ oid: "1.3.6.1.4.1.17713.21.1.2.10.0", sensor_type: "clients", sensor_descr: "Connected Clients", sensor_unit: "", - sensor_divisor: 1 + sensor_divisor: 1, + metadata: %{"limit_low" => 1} }, # === GPS sensors (from pollOS) === # GPS Tracked Satellites diff --git a/lib/towerops/snmp/profiles/vendors/mikrotik.ex b/lib/towerops/snmp/profiles/vendors/mikrotik.ex index 210115cb..96e31460 100644 --- a/lib/towerops/snmp/profiles/vendors/mikrotik.ex +++ b/lib/towerops/snmp/profiles/vendors/mikrotik.ex @@ -109,7 +109,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Mikrotik do sensor_unit: "", sensor_divisor: 1, sensor_index: "mikrotik_clients_#{index}", - last_value: client_count + last_value: client_count, + metadata: %{"limit_low" => 1} } ] diff --git a/lib/towerops/snmp/profiles/vendors/pmp.ex b/lib/towerops/snmp/profiles/vendors/pmp.ex index 35a3898c..52acb2c5 100644 --- a/lib/towerops/snmp/profiles/vendors/pmp.ex +++ b/lib/towerops/snmp/profiles/vendors/pmp.ex @@ -122,12 +122,14 @@ defmodule Towerops.Snmp.Profiles.Vendors.Pmp do sensor_divisor: 1000 }, # Client count (AP only) + # When client count drops to 0, the link to the far end is likely down. %{ oid: "1.3.6.1.4.1.161.19.3.1.7.1.0", sensor_type: "clients", sensor_descr: "Registered SMs", sensor_unit: "", - sensor_divisor: 1 + sensor_divisor: 1, + metadata: %{"limit_low" => 1} }, # 1m Downlink Utilization %{ diff --git a/lib/towerops/snmp/profiles/vendors/routeros.ex b/lib/towerops/snmp/profiles/vendors/routeros.ex index 4659a9c2..794a2d6b 100644 --- a/lib/towerops/snmp/profiles/vendors/routeros.ex +++ b/lib/towerops/snmp/profiles/vendors/routeros.ex @@ -223,7 +223,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Routeros do sensor_descr: "#{prefix} Clients", sensor_unit: "", sensor_divisor: 1, - last_value: client_count / 1 + last_value: client_count / 1, + metadata: %{"limit_low" => 1} } | sensors ] diff --git a/lib/towerops/snmp/profiles/vendors/vendor.ex b/lib/towerops/snmp/profiles/vendors/vendor.ex index 16e81393..0c7e0a7f 100644 --- a/lib/towerops/snmp/profiles/vendors/vendor.ex +++ b/lib/towerops/snmp/profiles/vendors/vendor.ex @@ -15,15 +15,17 @@ defmodule Towerops.Snmp.Profiles.Vendors.Vendor do sensor_descr: String.t(), sensor_unit: String.t(), sensor_divisor: integer(), - last_value: float() | nil + last_value: float() | nil, + metadata: map() } @type sensor_def :: %{ - oid: String.t(), - sensor_type: String.t(), - sensor_descr: String.t(), - sensor_unit: String.t(), - sensor_divisor: integer() + :oid => String.t(), + :sensor_type => String.t(), + :sensor_descr => String.t(), + :sensor_unit => String.t(), + :sensor_divisor => integer(), + optional(:metadata) => map() } @doc """ @@ -78,7 +80,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Vendor do sensor_descr: sensor_def.sensor_descr, sensor_unit: sensor_def.sensor_unit, sensor_divisor: sensor_def.sensor_divisor, - last_value: value / 1 + last_value: value / 1, + metadata: Map.get(sensor_def, :metadata, %{}) } _ -> diff --git a/test/towerops/monitoring/executors/snmp_sensor_executor_test.exs b/test/towerops/monitoring/executors/snmp_sensor_executor_test.exs index 4f8888da..db1f947d 100644 --- a/test/towerops/monitoring/executors/snmp_sensor_executor_test.exs +++ b/test/towerops/monitoring/executors/snmp_sensor_executor_test.exs @@ -212,6 +212,134 @@ defmodule Towerops.Monitoring.Executors.SnmpSensorExecutorTest do assert error_msg =~ "Sensor not found" end + test "returns CRITICAL for clients sensor with 0 connected clients", %{ + device: device, + snmp_device: snmp_device + } do + sensor = + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp_device.id, + sensor_descr: "Connected Clients", + sensor_type: "clients", + sensor_index: "1", + sensor_oid: ".1.2.3.4", + sensor_unit: "", + sensor_divisor: 1 + }) + |> Repo.insert!() + + {:ok, check} = + Monitoring.create_check(%{ + organization_id: device.organization_id, + device_id: device.id, + name: "Connected Clients", + check_type: "snmp_sensor", + source_type: "auto_discovery", + source_id: sensor.id, + config: %{ + "sensor_type" => "clients", + "sensor_oid" => ".1.2.3.4", + "sensor_unit" => "" + } + }) + + expect(SnmpMock, :get, fn _ip, _oid, _opts -> + {:ok, {:integer, 0}} + end) + + assert {:ok, response} = SnmpSensorExecutor.execute(check) + assert response.value == 0.0 + assert response.status == 2 + assert response.output =~ "Connected Clients" + assert response.output =~ "0" + end + + test "returns OK for clients sensor with connected clients > 0", %{ + device: device, + snmp_device: snmp_device + } do + sensor = + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp_device.id, + sensor_descr: "Connected Clients", + sensor_type: "clients", + sensor_index: "1", + sensor_oid: ".1.2.3.4", + sensor_unit: "", + sensor_divisor: 1 + }) + |> Repo.insert!() + + {:ok, check} = + Monitoring.create_check(%{ + organization_id: device.organization_id, + device_id: device.id, + name: "Connected Clients", + check_type: "snmp_sensor", + source_type: "auto_discovery", + source_id: sensor.id, + config: %{ + "sensor_type" => "clients", + "sensor_oid" => ".1.2.3.4", + "sensor_unit" => "" + } + }) + + expect(SnmpMock, :get, fn _ip, _oid, _opts -> + {:ok, {:integer, 3}} + end) + + assert {:ok, response} = SnmpSensorExecutor.execute(check) + assert response.value == 3.0 + assert response.status == 0 + end + + test "returns OK for clients sensor with 0 when explicit limit_low > 0", %{ + device: device, + snmp_device: snmp_device + } do + sensor = + %Sensor{} + |> Sensor.changeset(%{ + snmp_device_id: snmp_device.id, + sensor_descr: "Connected Clients", + sensor_type: "clients", + sensor_index: "1", + sensor_oid: ".1.2.3.4", + sensor_unit: "", + sensor_divisor: 1, + metadata: %{"limit_low" => -1} + }) + |> Repo.insert!() + + {:ok, check} = + Monitoring.create_check(%{ + organization_id: device.organization_id, + device_id: device.id, + name: "Connected Clients", + check_type: "snmp_sensor", + source_type: "auto_discovery", + source_id: sensor.id, + config: %{ + "sensor_type" => "clients", + "sensor_oid" => ".1.2.3.4", + "sensor_unit" => "" + } + }) + + expect(SnmpMock, :get, fn _ip, _oid, _opts -> + {:ok, {:integer, 0}} + end) + + assert {:ok, response} = SnmpSensorExecutor.execute(check) + assert response.value == 0.0 + # limit_low is -1, so 0 > -1, meaning no low threshold triggered + # But the default clients check kicks in: 0 clients = CRITICAL + assert response.status == 2 + end + test "formats count sensor output as integer", %{device: device, snmp_device: snmp_device} do sensor = %Sensor{}