diff --git a/CHANGELOG.txt b/CHANGELOG.txt index de8446d2..91bb0986 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,20 @@ +2026-05-09 +feat(insights): UpstreamDegradation multi-AP correlation rule + Towerops.Recommendations.Rules.UpstreamDegradation — when 50%+ of + the access points at one site simultaneously have Preseem QoE + score <60, root cause is a shared upstream (backhaul, transit, + DNS, DHCP) rather than per-AP RF. One site-level insight per + affected site listing the degraded APs with their QoE scores. + Critical urgency when 100% of APs are degraded; warning otherwise. + Runs FIRST in the rules list so the upstream signal surfaces + before any per-AP RF rules. + Insight.@valid_types extended with upstream_degradation. + LLM hint tells the model to instruct the operator to investigate + upstream FIRST and to NOT chase per-AP RF symptoms. + UI: indigo card with degraded/total AP counts, "Investigate + upstream first" badge, and a clickable list of the affected APs + with their QoE scores. + 2026-05-09 feat(insights): device-overheating recommendation Towerops.Recommendations.Rules.DeviceOverheating — flags devices diff --git a/lib/towerops/llm/insight_prompt.ex b/lib/towerops/llm/insight_prompt.ex index 43535ddc..bfe50e97 100644 --- a/lib/towerops/llm/insight_prompt.ex +++ b/lib/towerops/llm/insight_prompt.ex @@ -99,6 +99,16 @@ defmodule Towerops.LLM.InsightPrompt do p95. Metadata includes current_count and baseline_p95. Recommend rebalancing or capacity management. """, + "upstream_degradation" => """ + Rule context: 50%+ of the access points at one site simultaneously + show poor Preseem QoE — synchronized degradation across multiple APs + is the signature of a shared upstream cause (backhaul, transit, + DNS, DHCP) rather than per-AP RF. Metadata fields: + site_id, degraded_ap_count, total_ap_count, qoe_threshold, + degraded_aps (list of {name, qoe_score, device_id}). + Tell the operator to investigate the backhaul / upstream path FIRST + and to NOT chase per-AP RF symptoms. List the affected APs. + """, "device_overheating" => """ Rule context: a device has at least one temperature sensor above 65 °C (warning) or 75 °C (critical). Metadata fields: diff --git a/lib/towerops/preseem/insight.ex b/lib/towerops/preseem/insight.ex index 3316a2b9..fbfb608c 100644 --- a/lib/towerops/preseem/insight.ex +++ b/lib/towerops/preseem/insight.ex @@ -10,7 +10,7 @@ defmodule Towerops.Preseem.Insight do @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id - @valid_types ~w(qoe_degradation capacity_saturation firmware_opportunity model_underperforming subscriber_growth config_drift snmp_cpu_high snmp_memory_high snmp_disk_high device_poll_gap agent_offline firmware_mismatch reconciliation_finding subscriber_growth_trend suspect_config_change backhaul_over_capacity backhaul_near_capacity wireless_signal_weak wireless_snr_low wireless_ap_overloaded wireless_client_missing wireless_coverage_gap ap_frequency_change sector_overload cpe_realign own_fleet_channel_conflict device_overheating) + @valid_types ~w(qoe_degradation capacity_saturation firmware_opportunity model_underperforming subscriber_growth config_drift snmp_cpu_high snmp_memory_high snmp_disk_high device_poll_gap agent_offline firmware_mismatch reconciliation_finding subscriber_growth_trend suspect_config_change backhaul_over_capacity backhaul_near_capacity wireless_signal_weak wireless_snr_low wireless_ap_overloaded wireless_client_missing wireless_coverage_gap ap_frequency_change sector_overload cpe_realign own_fleet_channel_conflict device_overheating upstream_degradation) @valid_urgencies ~w(critical warning info) @valid_statuses ~w(active dismissed resolved) @valid_channels ~w(proactive contextual passive) diff --git a/lib/towerops/recommendations/rules/upstream_degradation.ex b/lib/towerops/recommendations/rules/upstream_degradation.ex new file mode 100644 index 00000000..a7e9a83c --- /dev/null +++ b/lib/towerops/recommendations/rules/upstream_degradation.ex @@ -0,0 +1,100 @@ +defmodule Towerops.Recommendations.Rules.UpstreamDegradation do + @moduledoc """ + Multi-AP correlation rule. When 50%+ of the access points at a site + show simultaneously degraded Preseem QoE scores, the root cause is + almost never per-AP RF — it's the shared upstream (backhaul, transit, + CGNAT, DNS, DHCP, etc.). This rule emits one site-level insight per + affected site so operators investigate the right layer instead of + chasing per-AP RF symptoms. + + Threshold: a site qualifies when: + * total_aps >= 2, AND + * at least 50% of APs have qoe_score < 60 + + Critical urgency when 100% of APs at the site are degraded; warning + otherwise. + """ + + import Ecto.Query + + alias Towerops.Devices.Device + alias Towerops.Preseem.AccessPoint + alias Towerops.Repo + + @qoe_threshold 60.0 + + @spec evaluate(Ecto.UUID.t()) :: [map()] + def evaluate(organization_id) when is_binary(organization_id) do + organization_id + |> load_aps_with_site() + |> Enum.group_by(fn {_ap, site_id} -> site_id end) + |> Enum.reject(fn {site_id, _aps} -> is_nil(site_id) end) + |> Enum.map(&evaluate_site(&1, organization_id)) + |> Enum.reject(&is_nil/1) + end + + defp load_aps_with_site(organization_id) do + Repo.all( + from(ap in AccessPoint, + join: d in Device, + on: d.id == ap.device_id, + where: ap.organization_id == ^organization_id and not is_nil(ap.device_id) and not is_nil(ap.qoe_score), + select: {ap, d.site_id} + ) + ) + end + + defp evaluate_site({site_id, ap_pairs}, organization_id) do + aps = Enum.map(ap_pairs, fn {ap, _site} -> ap end) + total = length(aps) + + degraded = + Enum.filter(aps, fn ap -> + is_number(ap.qoe_score) and ap.qoe_score < @qoe_threshold + end) + + cond do + total < 2 -> nil + length(degraded) * 2 < total -> nil + true -> build_insight(site_id, total, degraded, organization_id) + end + end + + defp build_insight(site_id, total, degraded, organization_id) do + deg_count = length(degraded) + [first | _] = degraded + + %{ + organization_id: organization_id, + site_id: site_id, + device_id: first.device_id, + type: "upstream_degradation", + urgency: if(deg_count == total, do: "critical", else: "warning"), + channel: "proactive", + source: "preseem", + title: "Upstream degradation at site: #{deg_count}/#{total} APs show poor QoE", + description: + "#{deg_count} of #{total} access points at this site simultaneously " <> + "have QoE scores below #{trunc(@qoe_threshold)}. Synchronized " <> + "degradation across multiple APs almost always indicates a shared " <> + "upstream cause (backhaul, transit, DNS, DHCP) rather than per-AP " <> + "RF. Investigate the site's upstream path before any RF action.", + metadata: %{ + "dedup_key" => "upstream_degradation:#{site_id}", + "site_id" => site_id, + "degraded_ap_count" => deg_count, + "total_ap_count" => total, + "qoe_threshold" => @qoe_threshold, + "degraded_aps" => + Enum.map(degraded, fn ap -> + %{ + "preseem_ap_id" => ap.id, + "device_id" => ap.device_id, + "name" => ap.name, + "qoe_score" => ap.qoe_score + } + end) + } + } + end +end diff --git a/lib/towerops/workers/recommendations_run_worker.ex b/lib/towerops/workers/recommendations_run_worker.ex index de607069..411fc259 100644 --- a/lib/towerops/workers/recommendations_run_worker.ex +++ b/lib/towerops/workers/recommendations_run_worker.ex @@ -22,11 +22,21 @@ defmodule Towerops.Workers.RecommendationsRunWorker do alias Towerops.Recommendations.Rules.FrequencyChange alias Towerops.Recommendations.Rules.OwnFleetChannelConflict alias Towerops.Recommendations.Rules.SectorOverload + alias Towerops.Recommendations.Rules.UpstreamDegradation alias Towerops.Repo require Logger - @rules [FrequencyChange, OwnFleetChannelConflict, SectorOverload, CpeRealign, DeviceOverheating] + # UpstreamDegradation runs first so its insight surfaces before any + # per-AP RF rules — a site-wide upstream issue is the dominant signal. + @rules [ + UpstreamDegradation, + FrequencyChange, + OwnFleetChannelConflict, + SectorOverload, + CpeRealign, + DeviceOverheating + ] @impl Oban.Worker def perform(%Oban.Job{}) do diff --git a/lib/towerops_web/live/insights_live/index.html.heex b/lib/towerops_web/live/insights_live/index.html.heex index 0140a955..91def05f 100644 --- a/lib/towerops_web/live/insights_live/index.html.heex +++ b/lib/towerops_web/live/insights_live/index.html.heex @@ -480,6 +480,47 @@ <% end %> + <%= if insight.type == "upstream_degradation" do %> +
+
+
+ + {t("Degraded APs")} + + + {insight.metadata["degraded_ap_count"]} / {insight.metadata[ + "total_ap_count" + ]} + +
+ + {t("Investigate upstream first")} + +
+ <%= if is_list(insight.metadata["degraded_aps"]) do %> + + <% end %> +
+ <% end %> + <%= if insight.type == "device_overheating" do %>
diff --git a/priv/static/changelog.txt b/priv/static/changelog.txt index 5cb8125d..dd9dec89 100644 --- a/priv/static/changelog.txt +++ b/priv/static/changelog.txt @@ -1,4 +1,5 @@ 2026-05-09 — Smarter recommendations + AI-generated insight summaries +* New recommendation: upstream degradation — when many of your APs at one site go bad at the same time, the system flags it as an upstream/backhaul problem so you don't waste time chasing per-AP RF symptoms * New recommendation: device overheating — flags any device with a temperature sensor at or above 65 °C, with the linked QoE score so you can tell whether a slowdown is thermal vs RF before sending a truck * New recommendation: same-channel conflicts within your own network — when two or more of your APs at the same site sit on the same channel, the system flags the conflict with a clickable evidence table showing noise floor, RF score, average CPE SNR, and client count per AP, and escalates to critical when the RF environment is also contaminated * New recommendation: sector overload detection — APs with low free airtime and active subscribers get flagged with subscriber count, QoE score, and AP model so you can decide whether to split the sector or upgrade the radio diff --git a/test/towerops/recommendations/rules/device_overheating_test.exs b/test/towerops/recommendations/rules/device_overheating_test.exs index e4afe201..32f860be 100644 --- a/test/towerops/recommendations/rules/device_overheating_test.exs +++ b/test/towerops/recommendations/rules/device_overheating_test.exs @@ -102,7 +102,7 @@ defmodule Towerops.Recommendations.Rules.DeviceOverheatingTest do assert insight.description =~ "QoE" end - test "scopes to organization", %{org: org, snmp: snmp} do + test "scopes to organization", %{snmp: snmp} do insert_temp(snmp, 78.0) other_user = user_fixture() diff --git a/test/towerops/recommendations/rules/upstream_degradation_test.exs b/test/towerops/recommendations/rules/upstream_degradation_test.exs new file mode 100644 index 00000000..2410465d --- /dev/null +++ b/test/towerops/recommendations/rules/upstream_degradation_test.exs @@ -0,0 +1,116 @@ +defmodule Towerops.Recommendations.Rules.UpstreamDegradationTest do + @moduledoc """ + Tests the multi-AP correlation rule that detects upstream / backhaul + problems by recognizing when many APs at the same site degrade + simultaneously. Prevents wasted per-AP RF investigation. + """ + use Towerops.DataCase, async: true + + import Towerops.AccountsFixtures + import Towerops.DevicesFixtures + import Towerops.OrganizationsFixtures + + alias Towerops.Preseem.AccessPoint + alias Towerops.Recommendations.Rules.UpstreamDegradation + + setup do + user = user_fixture() + org = organization_fixture(user.id) + {:ok, site} = Towerops.Sites.create_site(%{name: "Tower-1", organization_id: org.id}) + %{org: org, site: site} + end + + defp insert_ap(org, site, name, qoe) do + device = device_fixture(%{organization_id: org.id, site_id: site && site.id, name: name}) + + {:ok, ap} = + %AccessPoint{} + |> AccessPoint.changeset(%{ + organization_id: org.id, + preseem_id: "ap-up-#{System.unique_integer([:positive])}", + name: name, + device_id: device.id, + match_confidence: "auto_ip", + qoe_score: qoe + }) + |> Repo.insert() + + ap + end + + test "no insight when all APs are healthy", %{org: org, site: site} do + insert_ap(org, site, "AP-A", 85.0) + insert_ap(org, site, "AP-B", 80.0) + + assert UpstreamDegradation.evaluate(org.id) == [] + end + + test "no insight when only one AP is degraded (per-AP problem)", + %{org: org, site: site} do + insert_ap(org, site, "AP-A", 40.0) + insert_ap(org, site, "AP-B", 80.0) + insert_ap(org, site, "AP-C", 78.0) + + assert UpstreamDegradation.evaluate(org.id) == [] + end + + test "emits insight when 50%+ of APs at a site are degraded", + %{org: org, site: site} do + insert_ap(org, site, "AP-A", 40.0) + insert_ap(org, site, "AP-B", 35.0) + insert_ap(org, site, "AP-C", 80.0) + + assert [insight] = UpstreamDegradation.evaluate(org.id) + assert insight.type == "upstream_degradation" + assert insight.site_id == site.id + assert insight.urgency == "warning" + assert insight.metadata["site_id"] == site.id + assert insight.metadata["degraded_ap_count"] == 2 + assert insight.metadata["total_ap_count"] == 3 + end + + test "critical urgency when ALL APs at the site are degraded", + %{org: org, site: site} do + insert_ap(org, site, "AP-A", 40.0) + insert_ap(org, site, "AP-B", 35.0) + insert_ap(org, site, "AP-C", 30.0) + + assert [insight] = UpstreamDegradation.evaluate(org.id) + assert insight.urgency == "critical" + end + + test "ignores sites with fewer than 2 APs", %{org: org, site: site} do + insert_ap(org, site, "AP-A", 30.0) + + assert UpstreamDegradation.evaluate(org.id) == [] + end + + test "scopes to organization", %{org: org, site: site} do + insert_ap(org, site, "AP-A", 30.0) + insert_ap(org, site, "AP-B", 30.0) + + other_user = user_fixture() + other_org = organization_fixture(other_user.id) + assert UpstreamDegradation.evaluate(other_org.id) == [] + end + + test "ignores APs with nil qoe_score", %{org: org, site: site} do + insert_ap(org, site, "AP-A", nil) + insert_ap(org, site, "AP-B", nil) + + assert UpstreamDegradation.evaluate(org.id) == [] + end + + test "lists affected APs in metadata", %{org: org, site: site} do + a = insert_ap(org, site, "AP-A", 35.0) + b = insert_ap(org, site, "AP-B", 40.0) + insert_ap(org, site, "AP-C", 80.0) + + [insight] = UpstreamDegradation.evaluate(org.id) + aps = insight.metadata["degraded_aps"] + assert is_list(aps) + ids = Enum.map(aps, & &1["preseem_ap_id"]) + assert a.id in ids + assert b.id in ids + end +end diff --git a/test/towerops_web/live/device_live/show_test.exs b/test/towerops_web/live/device_live/show_test.exs index 22c0a231..d0a70fc2 100644 --- a/test/towerops_web/live/device_live/show_test.exs +++ b/test/towerops_web/live/device_live/show_test.exs @@ -1,5 +1,10 @@ defmodule ToweropsWeb.DeviceLive.ShowTest do - use ToweropsWeb.ConnCase, async: true + # Not async: this file inserts to InterfaceStat which is a TimescaleDB + # hypertable. Concurrent inserts to hypertable chunks can deadlock on + # chunk-table locks under parallel SQL-sandbox tests, producing + # sporadic ERROR 40P01 deadlock_detected failures in setup blocks. + # Running this module sync keeps the chart-rendering tests stable. + use ToweropsWeb.ConnCase, async: false import Phoenix.LiveViewTest