feat(monitoring): alert when backhaul radios lose connected clients
Backhaul radios (ePMP, AirOS, PMP, Cambium, MikroTik, RouterOS) now monitor their connected client count with a low threshold of 1. When the count drops to 0 the SnmpSensorExecutor marks it CRITICAL, which triggers an alert through the existing check → alert pipeline. - Adds field to sensor_def type in Vendor behaviour - Sets on all clients-type wireless sensors in backhaul-capable vendor profiles - Adds default executor handling: any clients sensor at 0 → status 2 - 3 new tests for clients sensor behavior
This commit is contained in:
parent
b1cbdc6c28
commit
8f9dc201af
10 changed files with 160 additions and 14 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -93,6 +93,7 @@ profiles.json
|
|||
.claude/
|
||||
.beads/
|
||||
.agents/
|
||||
.omo/
|
||||
.logs/
|
||||
.direnv/
|
||||
|
||||
|
|
|
|||
|
|
@ -113,6 +113,9 @@ defmodule Towerops.Monitoring.Executors.SnmpSensorExecutor do
|
|||
|
||||
defp determine_status(value, sensor) do
|
||||
cond do
|
||||
# For "clients" type sensors, 0 connected clients means the link is down.
|
||||
# This acts as default threshold when no explicit limits are configured.
|
||||
sensor.sensor_type == "clients" and value == 0 -> 2
|
||||
critical_high?(value, sensor) -> 2
|
||||
critical_low?(value, sensor) -> 2
|
||||
warning_high?(value, sensor) -> 1
|
||||
|
|
|
|||
7
lib/towerops/snmp/profiles/vendors/airos.ex
vendored
7
lib/towerops/snmp/profiles/vendors/airos.ex
vendored
|
|
@ -161,13 +161,16 @@ defmodule Towerops.Snmp.Profiles.Vendors.Airos do
|
|||
sensor_unit: "Mbps",
|
||||
sensor_divisor: 1
|
||||
},
|
||||
# ubntWlStatStaCount - entry 15
|
||||
# ubntWlStatStaCount - entry 15 (station/client count)
|
||||
# When client count drops to 0, the link to the far end is likely down.
|
||||
# limit_low: 1 ensures this check becomes CRITICAL at 0 clients.
|
||||
%{
|
||||
oid: "1.3.6.1.4.1.41112.1.4.5.1.15.1",
|
||||
sensor_type: "clients",
|
||||
sensor_descr: "Clients",
|
||||
sensor_unit: "",
|
||||
sensor_divisor: 1
|
||||
sensor_divisor: 1,
|
||||
metadata: %{"limit_low" => 1}
|
||||
},
|
||||
|
||||
# === AirMax table (ubntAirMaxTable) - 1.3.6.1.4.1.41112.1.4.6 ===
|
||||
|
|
|
|||
|
|
@ -124,7 +124,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Cambium do
|
|||
sensor_descr: "Connected Subscribers",
|
||||
sensor_unit: "",
|
||||
sensor_divisor: 1,
|
||||
sensor_index: "cambium_epmp_clients"
|
||||
sensor_index: "cambium_epmp_clients",
|
||||
metadata: %{"limit_low" => 1}
|
||||
}
|
||||
]
|
||||
else
|
||||
|
|
|
|||
5
lib/towerops/snmp/profiles/vendors/epmp.ex
vendored
5
lib/towerops/snmp/profiles/vendors/epmp.ex
vendored
|
|
@ -124,12 +124,15 @@ defmodule Towerops.Snmp.Profiles.Vendors.Epmp do
|
|||
sensor_divisor: 1
|
||||
},
|
||||
# Connected clients count (AP only)
|
||||
# When client count drops to 0, the link to the far end is likely down.
|
||||
# limit_low: 1 ensures this check becomes CRITICAL at 0 clients.
|
||||
%{
|
||||
oid: "1.3.6.1.4.1.17713.21.1.2.10.0",
|
||||
sensor_type: "clients",
|
||||
sensor_descr: "Connected Clients",
|
||||
sensor_unit: "",
|
||||
sensor_divisor: 1
|
||||
sensor_divisor: 1,
|
||||
metadata: %{"limit_low" => 1}
|
||||
},
|
||||
# === GPS sensors (from pollOS) ===
|
||||
# GPS Tracked Satellites
|
||||
|
|
|
|||
|
|
@ -109,7 +109,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Mikrotik do
|
|||
sensor_unit: "",
|
||||
sensor_divisor: 1,
|
||||
sensor_index: "mikrotik_clients_#{index}",
|
||||
last_value: client_count
|
||||
last_value: client_count,
|
||||
metadata: %{"limit_low" => 1}
|
||||
}
|
||||
]
|
||||
|
||||
|
|
|
|||
4
lib/towerops/snmp/profiles/vendors/pmp.ex
vendored
4
lib/towerops/snmp/profiles/vendors/pmp.ex
vendored
|
|
@ -122,12 +122,14 @@ defmodule Towerops.Snmp.Profiles.Vendors.Pmp do
|
|||
sensor_divisor: 1000
|
||||
},
|
||||
# Client count (AP only)
|
||||
# When client count drops to 0, the link to the far end is likely down.
|
||||
%{
|
||||
oid: "1.3.6.1.4.1.161.19.3.1.7.1.0",
|
||||
sensor_type: "clients",
|
||||
sensor_descr: "Registered SMs",
|
||||
sensor_unit: "",
|
||||
sensor_divisor: 1
|
||||
sensor_divisor: 1,
|
||||
metadata: %{"limit_low" => 1}
|
||||
},
|
||||
# 1m Downlink Utilization
|
||||
%{
|
||||
|
|
|
|||
|
|
@ -223,7 +223,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Routeros do
|
|||
sensor_descr: "#{prefix} Clients",
|
||||
sensor_unit: "",
|
||||
sensor_divisor: 1,
|
||||
last_value: client_count / 1
|
||||
last_value: client_count / 1,
|
||||
metadata: %{"limit_low" => 1}
|
||||
}
|
||||
| sensors
|
||||
]
|
||||
|
|
|
|||
17
lib/towerops/snmp/profiles/vendors/vendor.ex
vendored
17
lib/towerops/snmp/profiles/vendors/vendor.ex
vendored
|
|
@ -15,15 +15,17 @@ defmodule Towerops.Snmp.Profiles.Vendors.Vendor do
|
|||
sensor_descr: String.t(),
|
||||
sensor_unit: String.t(),
|
||||
sensor_divisor: integer(),
|
||||
last_value: float() | nil
|
||||
last_value: float() | nil,
|
||||
metadata: map()
|
||||
}
|
||||
|
||||
@type sensor_def :: %{
|
||||
oid: String.t(),
|
||||
sensor_type: String.t(),
|
||||
sensor_descr: String.t(),
|
||||
sensor_unit: String.t(),
|
||||
sensor_divisor: integer()
|
||||
:oid => String.t(),
|
||||
:sensor_type => String.t(),
|
||||
:sensor_descr => String.t(),
|
||||
:sensor_unit => String.t(),
|
||||
:sensor_divisor => integer(),
|
||||
optional(:metadata) => map()
|
||||
}
|
||||
|
||||
@doc """
|
||||
|
|
@ -78,7 +80,8 @@ defmodule Towerops.Snmp.Profiles.Vendors.Vendor do
|
|||
sensor_descr: sensor_def.sensor_descr,
|
||||
sensor_unit: sensor_def.sensor_unit,
|
||||
sensor_divisor: sensor_def.sensor_divisor,
|
||||
last_value: value / 1
|
||||
last_value: value / 1,
|
||||
metadata: Map.get(sensor_def, :metadata, %{})
|
||||
}
|
||||
|
||||
_ ->
|
||||
|
|
|
|||
|
|
@ -212,6 +212,134 @@ defmodule Towerops.Monitoring.Executors.SnmpSensorExecutorTest do
|
|||
assert error_msg =~ "Sensor not found"
|
||||
end
|
||||
|
||||
test "returns CRITICAL for clients sensor with 0 connected clients", %{
|
||||
device: device,
|
||||
snmp_device: snmp_device
|
||||
} do
|
||||
sensor =
|
||||
%Sensor{}
|
||||
|> Sensor.changeset(%{
|
||||
snmp_device_id: snmp_device.id,
|
||||
sensor_descr: "Connected Clients",
|
||||
sensor_type: "clients",
|
||||
sensor_index: "1",
|
||||
sensor_oid: ".1.2.3.4",
|
||||
sensor_unit: "",
|
||||
sensor_divisor: 1
|
||||
})
|
||||
|> Repo.insert!()
|
||||
|
||||
{:ok, check} =
|
||||
Monitoring.create_check(%{
|
||||
organization_id: device.organization_id,
|
||||
device_id: device.id,
|
||||
name: "Connected Clients",
|
||||
check_type: "snmp_sensor",
|
||||
source_type: "auto_discovery",
|
||||
source_id: sensor.id,
|
||||
config: %{
|
||||
"sensor_type" => "clients",
|
||||
"sensor_oid" => ".1.2.3.4",
|
||||
"sensor_unit" => ""
|
||||
}
|
||||
})
|
||||
|
||||
expect(SnmpMock, :get, fn _ip, _oid, _opts ->
|
||||
{:ok, {:integer, 0}}
|
||||
end)
|
||||
|
||||
assert {:ok, response} = SnmpSensorExecutor.execute(check)
|
||||
assert response.value == 0.0
|
||||
assert response.status == 2
|
||||
assert response.output =~ "Connected Clients"
|
||||
assert response.output =~ "0"
|
||||
end
|
||||
|
||||
test "returns OK for clients sensor with connected clients > 0", %{
|
||||
device: device,
|
||||
snmp_device: snmp_device
|
||||
} do
|
||||
sensor =
|
||||
%Sensor{}
|
||||
|> Sensor.changeset(%{
|
||||
snmp_device_id: snmp_device.id,
|
||||
sensor_descr: "Connected Clients",
|
||||
sensor_type: "clients",
|
||||
sensor_index: "1",
|
||||
sensor_oid: ".1.2.3.4",
|
||||
sensor_unit: "",
|
||||
sensor_divisor: 1
|
||||
})
|
||||
|> Repo.insert!()
|
||||
|
||||
{:ok, check} =
|
||||
Monitoring.create_check(%{
|
||||
organization_id: device.organization_id,
|
||||
device_id: device.id,
|
||||
name: "Connected Clients",
|
||||
check_type: "snmp_sensor",
|
||||
source_type: "auto_discovery",
|
||||
source_id: sensor.id,
|
||||
config: %{
|
||||
"sensor_type" => "clients",
|
||||
"sensor_oid" => ".1.2.3.4",
|
||||
"sensor_unit" => ""
|
||||
}
|
||||
})
|
||||
|
||||
expect(SnmpMock, :get, fn _ip, _oid, _opts ->
|
||||
{:ok, {:integer, 3}}
|
||||
end)
|
||||
|
||||
assert {:ok, response} = SnmpSensorExecutor.execute(check)
|
||||
assert response.value == 3.0
|
||||
assert response.status == 0
|
||||
end
|
||||
|
||||
test "returns OK for clients sensor with 0 when explicit limit_low > 0", %{
|
||||
device: device,
|
||||
snmp_device: snmp_device
|
||||
} do
|
||||
sensor =
|
||||
%Sensor{}
|
||||
|> Sensor.changeset(%{
|
||||
snmp_device_id: snmp_device.id,
|
||||
sensor_descr: "Connected Clients",
|
||||
sensor_type: "clients",
|
||||
sensor_index: "1",
|
||||
sensor_oid: ".1.2.3.4",
|
||||
sensor_unit: "",
|
||||
sensor_divisor: 1,
|
||||
metadata: %{"limit_low" => -1}
|
||||
})
|
||||
|> Repo.insert!()
|
||||
|
||||
{:ok, check} =
|
||||
Monitoring.create_check(%{
|
||||
organization_id: device.organization_id,
|
||||
device_id: device.id,
|
||||
name: "Connected Clients",
|
||||
check_type: "snmp_sensor",
|
||||
source_type: "auto_discovery",
|
||||
source_id: sensor.id,
|
||||
config: %{
|
||||
"sensor_type" => "clients",
|
||||
"sensor_oid" => ".1.2.3.4",
|
||||
"sensor_unit" => ""
|
||||
}
|
||||
})
|
||||
|
||||
expect(SnmpMock, :get, fn _ip, _oid, _opts ->
|
||||
{:ok, {:integer, 0}}
|
||||
end)
|
||||
|
||||
assert {:ok, response} = SnmpSensorExecutor.execute(check)
|
||||
assert response.value == 0.0
|
||||
# limit_low is -1, so 0 > -1, meaning no low threshold triggered
|
||||
# But the default clients check kicks in: 0 clients = CRITICAL
|
||||
assert response.status == 2
|
||||
end
|
||||
|
||||
test "formats count sensor output as integer", %{device: device, snmp_device: snmp_device} do
|
||||
sensor =
|
||||
%Sensor{}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue