319 lines
10 KiB
Elixir
319 lines
10 KiB
Elixir
defmodule Towerops.Workers.WirelessInsightWorker do
|
|
@moduledoc """
|
|
Oban cron worker that generates wireless client health insights.
|
|
|
|
Runs every 5 minutes to detect:
|
|
- `wireless_signal_weak`: Clients with poor signal strength
|
|
- `wireless_snr_low`: Clients with low SNR (signal-to-noise ratio)
|
|
- `wireless_ap_overloaded`: Access points with excessive client load
|
|
- `wireless_client_missing`: Expected subscribers not connecting
|
|
|
|
Auto-resolves insights when conditions improve (with hysteresis to prevent flapping).
|
|
"""
|
|
use Oban.Pro.Worker, queue: :maintenance
|
|
|
|
import Ecto.Query
|
|
|
|
alias Towerops.Gaiia
|
|
alias Towerops.Organizations.Organization
|
|
alias Towerops.Preseem.Insight
|
|
alias Towerops.Preseem.Insights
|
|
alias Towerops.Repo
|
|
alias Towerops.Snmp
|
|
alias Towerops.Snmp.WirelessClient
|
|
|
|
# Alert thresholds
|
|
@signal_warning_threshold -75
|
|
@signal_critical_threshold -85
|
|
@signal_recovery_threshold -70
|
|
|
|
@snr_warning_threshold 15
|
|
@snr_critical_threshold 10
|
|
@snr_recovery_threshold 18
|
|
|
|
@ap_warning_threshold 50
|
|
@ap_critical_threshold 75
|
|
@ap_recovery_threshold 40
|
|
|
|
@impl Oban.Pro.Worker
|
|
def process(%Oban.Job{}) do
|
|
# Use streaming to avoid loading all org IDs into memory
|
|
Repo.transaction(fn ->
|
|
Organization
|
|
|> Repo.stream()
|
|
|> Enum.each(&process_organization/1)
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
defp process_organization(organization) do
|
|
check_weak_signals(organization)
|
|
check_low_snr(organization)
|
|
check_overloaded_aps(organization)
|
|
check_missing_clients(organization)
|
|
|
|
# Auto-resolve insights that no longer apply
|
|
auto_resolve_weak_signals(organization)
|
|
auto_resolve_low_snr(organization)
|
|
auto_resolve_overloaded_aps(organization)
|
|
end
|
|
|
|
# --- Weak Signal Detection ---
|
|
|
|
defp check_weak_signals(organization) do
|
|
# Critical: < -85 dBm
|
|
organization.id
|
|
|> Snmp.list_weak_signal_clients(@signal_critical_threshold)
|
|
|> Enum.each(fn client ->
|
|
generate_signal_insight(organization, client, "critical")
|
|
end)
|
|
|
|
# Warning: < -75 dBm (but not critical)
|
|
organization.id
|
|
|> Snmp.list_weak_signal_clients(@signal_warning_threshold)
|
|
|> Enum.reject(&(&1.signal_strength < @signal_critical_threshold))
|
|
|> Enum.each(fn client ->
|
|
generate_signal_insight(organization, client, "warning")
|
|
end)
|
|
end
|
|
|
|
defp generate_signal_insight(organization, client, urgency) do
|
|
dedup_key = "#{client.device_id}_#{client.mac_address}"
|
|
|
|
Insights.insert_insight_if_new(%{
|
|
organization_id: organization.id,
|
|
device_id: client.device_id,
|
|
type: "wireless_signal_weak",
|
|
urgency: urgency,
|
|
channel: "proactive",
|
|
source: "snmp",
|
|
title: "Weak signal detected: #{client.mac_address} (#{client.signal_strength} dBm)",
|
|
description:
|
|
"Wireless client #{client.mac_address} on #{client.device.name} has weak signal strength " <>
|
|
"(#{client.signal_strength} dBm). This may cause connectivity issues and poor performance.",
|
|
metadata: %{
|
|
"dedup_key" => dedup_key,
|
|
"device_id" => client.device_id,
|
|
"mac_address" => client.mac_address,
|
|
"signal_strength" => client.signal_strength,
|
|
"threshold" => if(urgency == "critical", do: @signal_critical_threshold, else: @signal_warning_threshold)
|
|
}
|
|
})
|
|
end
|
|
|
|
defp auto_resolve_weak_signals(organization) do
|
|
# Get all active weak signal insights
|
|
active_insights =
|
|
Repo.all(
|
|
from(i in Insight,
|
|
where: i.organization_id == ^organization.id,
|
|
where: i.type == "wireless_signal_weak",
|
|
where: i.status == "active"
|
|
)
|
|
)
|
|
|
|
# Check current state and resolve if improved
|
|
Enum.each(active_insights, fn insight ->
|
|
dedup_key = insight.metadata["dedup_key"]
|
|
[device_id, mac_address] = String.split(dedup_key, "_", parts: 2)
|
|
|
|
# Get current client state
|
|
client =
|
|
Repo.one(
|
|
from(c in WirelessClient,
|
|
where: c.device_id == ^device_id,
|
|
where: fragment("LOWER(?)", c.mac_address) == ^String.downcase(mac_address)
|
|
)
|
|
)
|
|
|
|
# Resolve if signal improved (with hysteresis) or client disconnected
|
|
if is_nil(client) or
|
|
(client.signal_strength && client.signal_strength >= @signal_recovery_threshold) do
|
|
insight |> Insight.changeset(%{status: "resolved"}) |> Repo.update()
|
|
end
|
|
end)
|
|
end
|
|
|
|
# --- Low SNR Detection ---
|
|
|
|
defp check_low_snr(organization) do
|
|
# Critical: < 10 dB
|
|
organization.id
|
|
|> Snmp.list_low_snr_clients(@snr_critical_threshold)
|
|
|> Enum.each(fn client ->
|
|
generate_snr_insight(organization, client, "critical")
|
|
end)
|
|
|
|
# Warning: < 15 dB (but not critical)
|
|
organization.id
|
|
|> Snmp.list_low_snr_clients(@snr_warning_threshold)
|
|
|> Enum.reject(&(&1.snr < @snr_critical_threshold))
|
|
|> Enum.each(fn client ->
|
|
generate_snr_insight(organization, client, "warning")
|
|
end)
|
|
end
|
|
|
|
defp generate_snr_insight(organization, client, urgency) do
|
|
dedup_key = "#{client.device_id}_#{client.mac_address}"
|
|
|
|
Insights.insert_insight_if_new(%{
|
|
organization_id: organization.id,
|
|
device_id: client.device_id,
|
|
type: "wireless_snr_low",
|
|
urgency: urgency,
|
|
channel: "proactive",
|
|
source: "snmp",
|
|
title: "Low SNR detected: #{client.mac_address} (#{client.snr} dB)",
|
|
description:
|
|
"Wireless client #{client.mac_address} on #{client.device.name} has low signal-to-noise ratio " <>
|
|
"(#{client.snr} dB). This indicates interference or noise affecting signal quality.",
|
|
metadata: %{
|
|
"dedup_key" => dedup_key,
|
|
"device_id" => client.device_id,
|
|
"mac_address" => client.mac_address,
|
|
"snr" => client.snr,
|
|
"threshold" => if(urgency == "critical", do: @snr_critical_threshold, else: @snr_warning_threshold)
|
|
}
|
|
})
|
|
end
|
|
|
|
defp auto_resolve_low_snr(organization) do
|
|
active_insights =
|
|
Repo.all(
|
|
from(i in Insight,
|
|
where: i.organization_id == ^organization.id,
|
|
where: i.type == "wireless_snr_low",
|
|
where: i.status == "active"
|
|
)
|
|
)
|
|
|
|
Enum.each(active_insights, fn insight ->
|
|
dedup_key = insight.metadata["dedup_key"]
|
|
[device_id, mac_address] = String.split(dedup_key, "_", parts: 2)
|
|
|
|
client =
|
|
Repo.one(
|
|
from(c in WirelessClient,
|
|
where: c.device_id == ^device_id,
|
|
where: fragment("LOWER(?)", c.mac_address) == ^String.downcase(mac_address)
|
|
)
|
|
)
|
|
|
|
# Resolve if SNR improved (with hysteresis) or client disconnected
|
|
if is_nil(client) or (client.snr && client.snr >= @snr_recovery_threshold) do
|
|
insight |> Insight.changeset(%{status: "resolved"}) |> Repo.update()
|
|
end
|
|
end)
|
|
end
|
|
|
|
# --- Overloaded AP Detection ---
|
|
|
|
defp check_overloaded_aps(organization) do
|
|
# Critical: > 75 clients
|
|
organization.id
|
|
|> Snmp.list_overloaded_aps(@ap_critical_threshold)
|
|
|> Enum.each(fn {device, count} ->
|
|
generate_overload_insight(organization, device, count, "critical")
|
|
end)
|
|
|
|
# Warning: > 50 clients (but not critical)
|
|
organization.id
|
|
|> Snmp.list_overloaded_aps(@ap_warning_threshold)
|
|
|> Enum.reject(fn {_, count} -> count > @ap_critical_threshold end)
|
|
|> Enum.each(fn {device, count} ->
|
|
generate_overload_insight(organization, device, count, "warning")
|
|
end)
|
|
end
|
|
|
|
defp generate_overload_insight(organization, device, count, urgency) do
|
|
Insights.insert_insight_if_new(%{
|
|
organization_id: organization.id,
|
|
device_id: device.id,
|
|
type: "wireless_ap_overloaded",
|
|
urgency: urgency,
|
|
channel: "proactive",
|
|
source: "snmp",
|
|
title: "Access point overloaded: #{device.name} (#{count} clients)",
|
|
description:
|
|
"Access point #{device.name} has #{count} connected clients. " <>
|
|
"This may cause performance degradation. Consider load balancing or adding capacity.",
|
|
metadata: %{
|
|
"dedup_key" => device.id,
|
|
"device_id" => device.id,
|
|
"client_count" => count,
|
|
"threshold" => if(urgency == "critical", do: @ap_critical_threshold, else: @ap_warning_threshold)
|
|
}
|
|
})
|
|
end
|
|
|
|
defp auto_resolve_overloaded_aps(organization) do
|
|
active_insights =
|
|
Repo.all(
|
|
from(i in Insight,
|
|
where: i.organization_id == ^organization.id,
|
|
where: i.type == "wireless_ap_overloaded",
|
|
where: i.status == "active"
|
|
)
|
|
)
|
|
|
|
# Get current client counts by device
|
|
client_counts = Snmp.get_wireless_client_count_by_device(organization.id)
|
|
|
|
Enum.each(active_insights, fn insight ->
|
|
device_id = insight.metadata["device_id"]
|
|
current_count = Map.get(client_counts, device_id, 0)
|
|
|
|
# Resolve if load decreased below recovery threshold
|
|
if current_count <= @ap_recovery_threshold do
|
|
insight |> Insight.changeset(%{status: "resolved"}) |> Repo.update()
|
|
end
|
|
end)
|
|
end
|
|
|
|
# --- Missing Client Detection ---
|
|
|
|
defp check_missing_clients(organization) do
|
|
organization.id
|
|
|> Gaiia.list_missing_subscribers()
|
|
|> Enum.each(fn entry ->
|
|
generate_missing_client_insight(organization, entry)
|
|
end)
|
|
end
|
|
|
|
defp generate_missing_client_insight(organization, entry) do
|
|
dedup_key = "#{entry.device_id}_#{entry.mac}"
|
|
days_missing = DateTime.diff(DateTime.utc_now(), entry.last_seen, :day)
|
|
|
|
urgency =
|
|
if days_missing >= 5 do
|
|
"warning"
|
|
else
|
|
"info"
|
|
end
|
|
|
|
Insights.insert_insight_if_new(%{
|
|
organization_id: organization.id,
|
|
device_id: entry.device_id,
|
|
type: "wireless_client_missing",
|
|
urgency: urgency,
|
|
channel: "proactive",
|
|
source: "snmp",
|
|
title: "Expected subscriber not connected: #{entry.account.account_name}",
|
|
description:
|
|
"Subscriber #{entry.account.account_name} (#{entry.mac}) is expected to be connected to " <>
|
|
"#{entry.device.name} but has not been seen for #{days_missing} days. " <>
|
|
"Last seen: #{Calendar.strftime(entry.last_seen, "%Y-%m-%d %H:%M UTC")}",
|
|
metadata: %{
|
|
"dedup_key" => dedup_key,
|
|
"device_id" => entry.device_id,
|
|
"mac_address" => entry.mac,
|
|
"account_id" => entry.account.id,
|
|
"account_name" => entry.account.account_name,
|
|
"confidence" => entry.confidence,
|
|
"last_seen" => DateTime.to_iso8601(entry.last_seen),
|
|
"days_missing" => days_missing
|
|
}
|
|
})
|
|
end
|
|
end
|