feat(monitoring): alert when backhaul radios lose connected clients

Backhaul radios (ePMP, AirOS, PMP, Cambium, MikroTik, RouterOS) now
monitor their connected client count with a low threshold of 1. When
the count drops to 0 the SnmpSensorExecutor marks it CRITICAL, which
triggers an alert through the existing check → alert pipeline.

- Adds  field to sensor_def type in Vendor behaviour
- Sets  on all clients-type wireless sensors in
  backhaul-capable vendor profiles
- Adds default executor handling: any clients sensor at 0 → status 2
- 3 new tests for clients sensor behavior
This commit is contained in:
Graham McIntire 2026-06-02 12:27:44 -05:00
parent b1cbdc6c28
commit 8f9dc201af
10 changed files with 160 additions and 14 deletions

1
.gitignore vendored
View file

@ -93,6 +93,7 @@ profiles.json
.claude/
.beads/
.agents/
.omo/
.logs/
.direnv/

View file

@ -113,6 +113,9 @@ defmodule Towerops.Monitoring.Executors.SnmpSensorExecutor do
defp determine_status(value, sensor) do
cond do
# For "clients" type sensors, 0 connected clients means the link is down.
# This acts as default threshold when no explicit limits are configured.
sensor.sensor_type == "clients" and value == 0 -> 2
critical_high?(value, sensor) -> 2
critical_low?(value, sensor) -> 2
warning_high?(value, sensor) -> 1

View file

@ -161,13 +161,16 @@ defmodule Towerops.Snmp.Profiles.Vendors.Airos do
sensor_unit: "Mbps",
sensor_divisor: 1
},
# ubntWlStatStaCount - entry 15
# ubntWlStatStaCount - entry 15 (station/client count)
# When client count drops to 0, the link to the far end is likely down.
# limit_low: 1 ensures this check becomes CRITICAL at 0 clients.
%{
oid: "1.3.6.1.4.1.41112.1.4.5.1.15.1",
sensor_type: "clients",
sensor_descr: "Clients",
sensor_unit: "",
sensor_divisor: 1
sensor_divisor: 1,
metadata: %{"limit_low" => 1}
},
# === AirMax table (ubntAirMaxTable) - 1.3.6.1.4.1.41112.1.4.6 ===

View file

@ -124,7 +124,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Cambium do
sensor_descr: "Connected Subscribers",
sensor_unit: "",
sensor_divisor: 1,
sensor_index: "cambium_epmp_clients"
sensor_index: "cambium_epmp_clients",
metadata: %{"limit_low" => 1}
}
]
else

View file

@ -124,12 +124,15 @@ defmodule Towerops.Snmp.Profiles.Vendors.Epmp do
sensor_divisor: 1
},
# Connected clients count (AP only)
# When client count drops to 0, the link to the far end is likely down.
# limit_low: 1 ensures this check becomes CRITICAL at 0 clients.
%{
oid: "1.3.6.1.4.1.17713.21.1.2.10.0",
sensor_type: "clients",
sensor_descr: "Connected Clients",
sensor_unit: "",
sensor_divisor: 1
sensor_divisor: 1,
metadata: %{"limit_low" => 1}
},
# === GPS sensors (from pollOS) ===
# GPS Tracked Satellites

View file

@ -109,7 +109,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Mikrotik do
sensor_unit: "",
sensor_divisor: 1,
sensor_index: "mikrotik_clients_#{index}",
last_value: client_count
last_value: client_count,
metadata: %{"limit_low" => 1}
}
]

View file

@ -122,12 +122,14 @@ defmodule Towerops.Snmp.Profiles.Vendors.Pmp do
sensor_divisor: 1000
},
# Client count (AP only)
# When client count drops to 0, the link to the far end is likely down.
%{
oid: "1.3.6.1.4.1.161.19.3.1.7.1.0",
sensor_type: "clients",
sensor_descr: "Registered SMs",
sensor_unit: "",
sensor_divisor: 1
sensor_divisor: 1,
metadata: %{"limit_low" => 1}
},
# 1m Downlink Utilization
%{

View file

@ -223,7 +223,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Routeros do
sensor_descr: "#{prefix} Clients",
sensor_unit: "",
sensor_divisor: 1,
last_value: client_count / 1
last_value: client_count / 1,
metadata: %{"limit_low" => 1}
}
| sensors
]

View file

@ -15,15 +15,17 @@ defmodule Towerops.Snmp.Profiles.Vendors.Vendor do
sensor_descr: String.t(),
sensor_unit: String.t(),
sensor_divisor: integer(),
last_value: float() | nil
last_value: float() | nil,
metadata: map()
}
@type sensor_def :: %{
oid: String.t(),
sensor_type: String.t(),
sensor_descr: String.t(),
sensor_unit: String.t(),
sensor_divisor: integer()
:oid => String.t(),
:sensor_type => String.t(),
:sensor_descr => String.t(),
:sensor_unit => String.t(),
:sensor_divisor => integer(),
optional(:metadata) => map()
}
@doc """
@ -78,7 +80,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Vendor do
sensor_descr: sensor_def.sensor_descr,
sensor_unit: sensor_def.sensor_unit,
sensor_divisor: sensor_def.sensor_divisor,
last_value: value / 1
last_value: value / 1,
metadata: Map.get(sensor_def, :metadata, %{})
}
_ ->

View file

@ -212,6 +212,134 @@ defmodule Towerops.Monitoring.Executors.SnmpSensorExecutorTest do
assert error_msg =~ "Sensor not found"
end
test "returns CRITICAL for clients sensor with 0 connected clients", %{
device: device,
snmp_device: snmp_device
} do
sensor =
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp_device.id,
sensor_descr: "Connected Clients",
sensor_type: "clients",
sensor_index: "1",
sensor_oid: ".1.2.3.4",
sensor_unit: "",
sensor_divisor: 1
})
|> Repo.insert!()
{:ok, check} =
Monitoring.create_check(%{
organization_id: device.organization_id,
device_id: device.id,
name: "Connected Clients",
check_type: "snmp_sensor",
source_type: "auto_discovery",
source_id: sensor.id,
config: %{
"sensor_type" => "clients",
"sensor_oid" => ".1.2.3.4",
"sensor_unit" => ""
}
})
expect(SnmpMock, :get, fn _ip, _oid, _opts ->
{:ok, {:integer, 0}}
end)
assert {:ok, response} = SnmpSensorExecutor.execute(check)
assert response.value == 0.0
assert response.status == 2
assert response.output =~ "Connected Clients"
assert response.output =~ "0"
end
test "returns OK for clients sensor with connected clients > 0", %{
device: device,
snmp_device: snmp_device
} do
sensor =
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp_device.id,
sensor_descr: "Connected Clients",
sensor_type: "clients",
sensor_index: "1",
sensor_oid: ".1.2.3.4",
sensor_unit: "",
sensor_divisor: 1
})
|> Repo.insert!()
{:ok, check} =
Monitoring.create_check(%{
organization_id: device.organization_id,
device_id: device.id,
name: "Connected Clients",
check_type: "snmp_sensor",
source_type: "auto_discovery",
source_id: sensor.id,
config: %{
"sensor_type" => "clients",
"sensor_oid" => ".1.2.3.4",
"sensor_unit" => ""
}
})
expect(SnmpMock, :get, fn _ip, _oid, _opts ->
{:ok, {:integer, 3}}
end)
assert {:ok, response} = SnmpSensorExecutor.execute(check)
assert response.value == 3.0
assert response.status == 0
end
test "returns OK for clients sensor with 0 when explicit limit_low > 0", %{
device: device,
snmp_device: snmp_device
} do
sensor =
%Sensor{}
|> Sensor.changeset(%{
snmp_device_id: snmp_device.id,
sensor_descr: "Connected Clients",
sensor_type: "clients",
sensor_index: "1",
sensor_oid: ".1.2.3.4",
sensor_unit: "",
sensor_divisor: 1,
metadata: %{"limit_low" => -1}
})
|> Repo.insert!()
{:ok, check} =
Monitoring.create_check(%{
organization_id: device.organization_id,
device_id: device.id,
name: "Connected Clients",
check_type: "snmp_sensor",
source_type: "auto_discovery",
source_id: sensor.id,
config: %{
"sensor_type" => "clients",
"sensor_oid" => ".1.2.3.4",
"sensor_unit" => ""
}
})
expect(SnmpMock, :get, fn _ip, _oid, _opts ->
{:ok, {:integer, 0}}
end)
assert {:ok, response} = SnmpSensorExecutor.execute(check)
assert response.value == 0.0
# limit_low is -1, so 0 > -1, meaning no low threshold triggered
# But the default clients check kicks in: 0 clients = CRITICAL
assert response.status == 2
end
test "formats count sensor output as integer", %{device: device, snmp_device: snmp_device} do
sensor =
%Sensor{}