From b6513ec301a002e05fa3d9d76e4b7be5ec6fe39b Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 9 May 2026 17:56:38 -0500 Subject: [PATCH] feat(insights): OpticalRxLow rule for fiber transceivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fiber operators care a lot about optical Rx power because connectors get dirty, splices degrade, and patch panels develop bend-radius problems. The data is already in snmp_transceiver_readings — this rule turns it into actionable insights. - Towerops.Recommendations.Rules.OpticalRxLow — single query selects the latest reading per transceiver (last hour), filters to those with rx_power_dbm <= -28, joins back to the device. Critical at <= -32 dBm. One insight per (device, port) via metadata.dedup_key. - Insight.@valid_types extended with optical_rx_low. - LLM prompt hint walks the operator through the standard ladder: clean SC/LC connector with a one-click cleaner, inspect bend radius, verify splices, then replace the transceiver. - UI: cyan card with port, Rx power, Tx power, transceiver type, and wavelength. - Wired into RecommendationsRunWorker. 8 new tests. --- CHANGELOG.txt | 14 ++ lib/towerops/llm/insight_prompt.ex | 10 ++ lib/towerops/preseem/insight.ex | 2 +- .../recommendations/rules/optical_rx_low.ex | 103 +++++++++++++ .../workers/recommendations_run_worker.ex | 4 +- .../live/insights_live/index.html.heex | 41 ++++++ priv/static/changelog.txt | 1 + .../rules/optical_rx_low_test.exs | 139 ++++++++++++++++++ 8 files changed, 312 insertions(+), 2 deletions(-) create mode 100644 lib/towerops/recommendations/rules/optical_rx_low.ex create mode 100644 test/towerops/recommendations/rules/optical_rx_low_test.exs diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 91bb0986..963b4cfc 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,17 @@ +2026-05-09 +feat(insights): OpticalRxLow rule for fiber transceivers + Towerops.Recommendations.Rules.OpticalRxLow — flags fiber + transceivers (snmp_transceivers) whose latest received optical + power is below operator thresholds: -28 dBm warning, -32 dBm + critical. Reads only readings within the last hour to avoid + false positives from stale DOM snapshots. One insight per + (device, port) via metadata.dedup_key. + Insight.@valid_types extended with optical_rx_low. + LLM hint walks the operator through the standard ladder: clean the + connector, inspect bend radius, verify splices, then replace. + UI: cyan card with port, Rx power, Tx power, transceiver type, and + wavelength. + 2026-05-09 feat(insights): UpstreamDegradation multi-AP correlation rule Towerops.Recommendations.Rules.UpstreamDegradation — when 50%+ of diff --git a/lib/towerops/llm/insight_prompt.ex b/lib/towerops/llm/insight_prompt.ex index bfe50e97..51a51b76 100644 --- a/lib/towerops/llm/insight_prompt.ex +++ b/lib/towerops/llm/insight_prompt.ex @@ -109,6 +109,16 @@ defmodule Towerops.LLM.InsightPrompt do Tell the operator to investigate the backhaul / upstream path FIRST and to NOT chase per-AP RF symptoms. List the affected APs. """, + "optical_rx_low" => """ + Rule context: a fiber transceiver shows received optical power + below the actionable threshold. Metadata fields: + port, rx_power_dbm, tx_power_dbm, transceiver_type, vendor_name, + wavelength_nm, warning_threshold_dbm, critical_threshold_dbm. + Recommend (in order): clean the SC/LC connector with a one-click + cleaner; inspect the patch panel for kinks or tight bends; verify + the splice; replace the transceiver if Rx still low. Mention the + specific port and dBm value. + """, "device_overheating" => """ Rule context: a device has at least one temperature sensor above 65 °C (warning) or 75 °C (critical). Metadata fields: diff --git a/lib/towerops/preseem/insight.ex b/lib/towerops/preseem/insight.ex index fbfb608c..8cbd8eeb 100644 --- a/lib/towerops/preseem/insight.ex +++ b/lib/towerops/preseem/insight.ex @@ -10,7 +10,7 @@ defmodule Towerops.Preseem.Insight do @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id - @valid_types ~w(qoe_degradation capacity_saturation firmware_opportunity model_underperforming subscriber_growth config_drift snmp_cpu_high snmp_memory_high snmp_disk_high device_poll_gap agent_offline firmware_mismatch reconciliation_finding subscriber_growth_trend suspect_config_change backhaul_over_capacity backhaul_near_capacity wireless_signal_weak wireless_snr_low wireless_ap_overloaded wireless_client_missing wireless_coverage_gap ap_frequency_change sector_overload cpe_realign own_fleet_channel_conflict device_overheating upstream_degradation) + @valid_types ~w(qoe_degradation capacity_saturation firmware_opportunity model_underperforming subscriber_growth config_drift snmp_cpu_high snmp_memory_high snmp_disk_high device_poll_gap agent_offline firmware_mismatch reconciliation_finding subscriber_growth_trend suspect_config_change backhaul_over_capacity backhaul_near_capacity wireless_signal_weak wireless_snr_low wireless_ap_overloaded wireless_client_missing wireless_coverage_gap ap_frequency_change sector_overload cpe_realign own_fleet_channel_conflict device_overheating upstream_degradation optical_rx_low) @valid_urgencies ~w(critical warning info) @valid_statuses ~w(active dismissed resolved) @valid_channels ~w(proactive contextual passive) diff --git a/lib/towerops/recommendations/rules/optical_rx_low.ex b/lib/towerops/recommendations/rules/optical_rx_low.ex new file mode 100644 index 00000000..432bc4d8 --- /dev/null +++ b/lib/towerops/recommendations/rules/optical_rx_low.ex @@ -0,0 +1,103 @@ +defmodule Towerops.Recommendations.Rules.OpticalRxLow do + @moduledoc """ + Flags fiber transceivers whose latest received optical power is below + operator-actionable thresholds. Catches dirty connectors, splice + degradation, fiber bend-loss, and aging optics before they fail + outright. + + Thresholds (typical 1310/1550 nm SFP/SFP+ operating range): + * `< -28 dBm` → warning + * `< -32 dBm` → critical (link may be unstable; below most receiver + sensitivity floors) + + Reads only readings within the last hour to avoid false positives from + stale data when DOM polling pauses. + """ + + import Ecto.Query + + alias Towerops.Devices.Device + alias Towerops.Repo + alias Towerops.Snmp.Device, as: SnmpDevice + alias Towerops.Snmp.Transceiver + alias Towerops.Snmp.TransceiverReading + + @warning_threshold_dbm -28.0 + @critical_threshold_dbm -32.0 + @lookback_seconds 3600 + + @spec evaluate(Ecto.UUID.t()) :: [map()] + def evaluate(organization_id) when is_binary(organization_id) do + cutoff = + DateTime.utc_now() + |> DateTime.add(-@lookback_seconds, :second) + |> DateTime.truncate(:second) + + organization_id + |> load_low_rx(cutoff) + |> Enum.map(&build_insight(&1, organization_id)) + end + + defp load_low_rx(organization_id, cutoff) do + # Latest reading per transceiver in last hour, with rx below threshold + latest_per_xcvr = + from(r in TransceiverReading, + where: r.measured_at >= ^cutoff and not is_nil(r.rx_power_dbm), + distinct: r.transceiver_id, + order_by: [desc: r.measured_at], + select: r + ) + + Repo.all( + from(r in subquery(latest_per_xcvr), + join: x in Transceiver, + on: x.id == r.transceiver_id, + join: sd in SnmpDevice, + on: sd.id == x.snmp_device_id, + join: d in Device, + on: d.id == sd.device_id, + where: d.organization_id == ^organization_id and r.rx_power_dbm <= ^@warning_threshold_dbm, + select: {d, x, r} + ) + ) + end + + defp build_insight({device, xcvr, reading}, organization_id) do + rx = reading.rx_power_dbm + + %{ + organization_id: organization_id, + device_id: device.id, + type: "optical_rx_low", + urgency: urgency_for(rx), + channel: "proactive", + source: "snmp", + title: + "Optical Rx low on #{device.name || "device"} #{xcvr.port_index}: " <> + "#{format_dbm(rx)}", + description: + "Transceiver #{xcvr.port_index} (#{xcvr.transceiver_type || "SFP"}) " <> + "shows received optical power of #{format_dbm(rx)} — below the " <> + "actionable threshold. Inspect connector cleanliness, fiber " <> + "bend-radius, and patch-panel splices.", + metadata: %{ + "dedup_key" => "optical_rx_low:#{device.id}:#{xcvr.port_index}", + "port" => xcvr.port_index, + "rx_power_dbm" => rx, + "tx_power_dbm" => reading.tx_power_dbm, + "transceiver_type" => xcvr.transceiver_type, + "vendor_name" => xcvr.vendor_name, + "wavelength_nm" => xcvr.wavelength_nm, + "warning_threshold_dbm" => @warning_threshold_dbm, + "critical_threshold_dbm" => @critical_threshold_dbm, + "measured_at" => reading.measured_at && DateTime.to_iso8601(reading.measured_at) + } + } + end + + defp urgency_for(rx) when is_number(rx) and rx <= @critical_threshold_dbm, do: "critical" + defp urgency_for(_), do: "warning" + + defp format_dbm(rx) when is_number(rx), do: "#{Float.round(rx / 1.0, 1)} dBm" + defp format_dbm(_), do: "?dBm" +end diff --git a/lib/towerops/workers/recommendations_run_worker.ex b/lib/towerops/workers/recommendations_run_worker.ex index 411fc259..9b84eccd 100644 --- a/lib/towerops/workers/recommendations_run_worker.ex +++ b/lib/towerops/workers/recommendations_run_worker.ex @@ -20,6 +20,7 @@ defmodule Towerops.Workers.RecommendationsRunWorker do alias Towerops.Recommendations.Rules.CpeRealign alias Towerops.Recommendations.Rules.DeviceOverheating alias Towerops.Recommendations.Rules.FrequencyChange + alias Towerops.Recommendations.Rules.OpticalRxLow alias Towerops.Recommendations.Rules.OwnFleetChannelConflict alias Towerops.Recommendations.Rules.SectorOverload alias Towerops.Recommendations.Rules.UpstreamDegradation @@ -35,7 +36,8 @@ defmodule Towerops.Workers.RecommendationsRunWorker do OwnFleetChannelConflict, SectorOverload, CpeRealign, - DeviceOverheating + DeviceOverheating, + OpticalRxLow ] @impl Oban.Worker diff --git a/lib/towerops_web/live/insights_live/index.html.heex b/lib/towerops_web/live/insights_live/index.html.heex index 91def05f..e067d1bd 100644 --- a/lib/towerops_web/live/insights_live/index.html.heex +++ b/lib/towerops_web/live/insights_live/index.html.heex @@ -480,6 +480,47 @@ <% end %> + <%= if insight.type == "optical_rx_low" do %> +
+
+
+ + {t("Port")} + + + {insight.metadata["port"]} + +
+
+ + {t("Rx power")} + + + {insight.metadata["rx_power_dbm"]} dBm + +
+ <%= if insight.metadata["tx_power_dbm"] do %> +
+ + {t("Tx power")} + + + {insight.metadata["tx_power_dbm"]} dBm + +
+ <% end %> + <%= if insight.metadata["transceiver_type"] do %> + + {insight.metadata["transceiver_type"]} + <%= if insight.metadata["wavelength_nm"] do %> + · {insight.metadata["wavelength_nm"]} nm + <% end %> + + <% end %> +
+
+ <% end %> + <%= if insight.type == "upstream_degradation" do %>
diff --git a/priv/static/changelog.txt b/priv/static/changelog.txt index dd9dec89..87dcb18b 100644 --- a/priv/static/changelog.txt +++ b/priv/static/changelog.txt @@ -1,4 +1,5 @@ 2026-05-09 — Smarter recommendations + AI-generated insight summaries +* New recommendation: low optical Rx power on fiber transceivers, with the standard cleanup-then-replace troubleshooting ladder * New recommendation: upstream degradation — when many of your APs at one site go bad at the same time, the system flags it as an upstream/backhaul problem so you don't waste time chasing per-AP RF symptoms * New recommendation: device overheating — flags any device with a temperature sensor at or above 65 °C, with the linked QoE score so you can tell whether a slowdown is thermal vs RF before sending a truck * New recommendation: same-channel conflicts within your own network — when two or more of your APs at the same site sit on the same channel, the system flags the conflict with a clickable evidence table showing noise floor, RF score, average CPE SNR, and client count per AP, and escalates to critical when the RF environment is also contaminated diff --git a/test/towerops/recommendations/rules/optical_rx_low_test.exs b/test/towerops/recommendations/rules/optical_rx_low_test.exs new file mode 100644 index 00000000..7f4a2390 --- /dev/null +++ b/test/towerops/recommendations/rules/optical_rx_low_test.exs @@ -0,0 +1,139 @@ +defmodule Towerops.Recommendations.Rules.OpticalRxLowTest do + @moduledoc """ + Tests the rule that flags fiber transceivers whose latest received + optical power is below operator-actionable thresholds. + """ + use Towerops.DataCase, async: true + + import Towerops.AccountsFixtures + import Towerops.DevicesFixtures + import Towerops.OrganizationsFixtures + + alias Towerops.Recommendations.Rules.OpticalRxLow + alias Towerops.Snmp.Device, as: SnmpDevice + alias Towerops.Snmp.Transceiver + alias Towerops.Snmp.TransceiverReading + + setup do + user = user_fixture() + org = organization_fixture(user.id) + device = device_fixture(%{organization_id: org.id, name: "Switch-1"}) + + {:ok, snmp} = + %SnmpDevice{} + |> SnmpDevice.changeset(%{device_id: device.id, sys_name: "Switch-1"}) + |> Repo.insert() + + %{org: org, device: device, snmp: snmp} + end + + defp insert_xcvr(snmp, opts \\ []) do + {:ok, xcvr} = + %Transceiver{} + |> Transceiver.changeset(%{ + snmp_device_id: snmp.id, + port_index: Keyword.get(opts, :port, "Gi1/0/1"), + transceiver_type: "SFP+", + vendor_name: "Test", + wavelength_nm: 1310, + nominal_bitrate_mbps: 10_000, + supports_dom: true + }) + |> Repo.insert() + + xcvr + end + + defp insert_reading(xcvr, rx_dbm, opts \\ []) do + measured_at = Keyword.get(opts, :measured_at, DateTime.truncate(DateTime.utc_now(), :second)) + + %TransceiverReading{} + |> TransceiverReading.changeset(%{ + transceiver_id: xcvr.id, + rx_power_dbm: rx_dbm, + tx_power_dbm: Keyword.get(opts, :tx, -1.5), + temperature_celsius: 35.0, + measured_at: measured_at + }) + |> Repo.insert!() + end + + test "no insight when there are no transceivers", %{org: org} do + assert OpticalRxLow.evaluate(org.id) == [] + end + + test "no insight when rx_power is healthy", %{org: org, snmp: snmp} do + xcvr = insert_xcvr(snmp) + insert_reading(xcvr, -8.0) + + assert OpticalRxLow.evaluate(org.id) == [] + end + + test "warning insight when rx_power is between -32 and -28 dBm", + %{org: org, device: device, snmp: snmp} do + xcvr = insert_xcvr(snmp, port: "Gi1/0/24") + insert_reading(xcvr, -29.5) + + assert [insight] = OpticalRxLow.evaluate(org.id) + assert insight.type == "optical_rx_low" + assert insight.urgency == "warning" + assert insight.device_id == device.id + assert insight.metadata["rx_power_dbm"] == -29.5 + assert insight.metadata["port"] == "Gi1/0/24" + assert insight.title =~ "Switch-1" + assert insight.title =~ "-29.5" + end + + test "critical insight at <= -32 dBm", %{org: org, snmp: snmp} do + xcvr = insert_xcvr(snmp) + insert_reading(xcvr, -33.0) + + assert [insight] = OpticalRxLow.evaluate(org.id) + assert insight.urgency == "critical" + end + + test "uses the most recent reading per transceiver", %{org: org, snmp: snmp} do + xcvr = insert_xcvr(snmp) + + insert_reading(xcvr, -29.0, + measured_at: DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second) + ) + + insert_reading(xcvr, -8.0) + + assert OpticalRxLow.evaluate(org.id) == [] + end + + test "ignores readings older than 1 hour", %{org: org, snmp: snmp} do + xcvr = insert_xcvr(snmp) + + insert_reading(xcvr, -29.0, + measured_at: DateTime.utc_now() |> DateTime.add(-2 * 3600, :second) |> DateTime.truncate(:second) + ) + + assert OpticalRxLow.evaluate(org.id) == [] + end + + test "produces one insight per (device, port) — distinct dedup", %{org: org, snmp: snmp} do + a = insert_xcvr(snmp, port: "Gi1/0/1") + b = insert_xcvr(snmp, port: "Gi1/0/2") + + insert_reading(a, -29.0) + insert_reading(b, -30.0) + + assert insights = OpticalRxLow.evaluate(org.id) + assert length(insights) == 2 + + keys = Enum.map(insights, & &1.metadata["dedup_key"]) + assert Enum.uniq(keys) == keys + end + + test "scopes to organization", %{snmp: snmp} do + xcvr = insert_xcvr(snmp) + insert_reading(xcvr, -33.0) + + other_user = user_fixture() + other_org = organization_fixture(other_user.id) + assert OpticalRxLow.evaluate(other_org.id) == [] + end +end