From 03f364a95660e4668aeaf1e9a4d91bfc1e9a80f2 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 9 May 2026 16:41:48 -0500 Subject: [PATCH] feat(insights): LLM-powered insight enrichment Adds an optional plain-language summary and recommended action to every active insight. A new Oban cron worker runs every 5 minutes, picks up unenriched insights, and asks the configured LLM to restate the finding and suggest one specific action grounded in the structured metadata. - Migration: adds llm_summary, recommended_action, llm_model, llm_enriched_at columns to preseem_insights - Towerops.LLM context with swappable behaviour; real client uses Req with the standard Req.Test plug for test isolation - MockClient + InsightPrompt module (builds chat messages, parses JSON with code-fence stripping and a raw-text fallback) - Worker logs and skips on rate limits / missing key / parse errors so insights still render without an AI summary when the LLM is unavailable - Optional towerops-llm k8s secret with example template; deployment.yaml references it as optional in both init and main containers - UI renders summary + recommended action callout on /insights and on the org-level insights page --- CHANGELOG.txt | 15 + config/runtime.exs | 10 + config/test.exs | 5 + ...-llm-recommendations-and-frequency-rule.md | 959 ++++++++++++++++++ k8s/README.md | 13 + k8s/deployment.yaml | 24 + k8s/towerops-llm-secret.example.yaml | 27 + lib/towerops/llm.ex | 21 + lib/towerops/llm/behaviour.ex | 13 + lib/towerops/llm/deepseek.ex | 86 ++ lib/towerops/llm/insight_prompt.ex | 66 ++ lib/towerops/llm/mock_client.ex | 15 + lib/towerops/preseem/insight.ex | 10 +- lib/towerops/preseem/insights.ex | 32 + .../workers/insight_llm_enrichment_worker.ex | 43 + .../live/insights_live/index.html.heex | 16 + .../live/org/preseem_insights_live.html.heex | 16 + ...449_add_llm_fields_to_preseem_insights.exs | 14 + priv/static/changelog.txt | 4 + test/towerops/llm/deepseek_test.exs | 73 ++ test/towerops/llm/insight_prompt_test.exs | 65 ++ test/towerops/preseem/insight_test.exs | 23 + test/towerops/preseem/insights_test.exs | 97 ++ .../insight_llm_enrichment_worker_test.exs | 121 +++ .../live/insights_live_events_test.exs | 37 + 25 files changed, 1804 insertions(+), 1 deletion(-) create mode 100644 docs/plans/2026-05-09-llm-recommendations-and-frequency-rule.md create mode 100644 k8s/towerops-llm-secret.example.yaml create mode 100644 lib/towerops/llm.ex create mode 100644 lib/towerops/llm/behaviour.ex create mode 100644 lib/towerops/llm/deepseek.ex create mode 100644 lib/towerops/llm/insight_prompt.ex create mode 100644 lib/towerops/llm/mock_client.ex create mode 100644 lib/towerops/workers/insight_llm_enrichment_worker.ex create mode 100644 priv/repo/migrations/20260509212449_add_llm_fields_to_preseem_insights.exs create mode 100644 test/towerops/llm/deepseek_test.exs create mode 100644 test/towerops/llm/insight_prompt_test.exs create mode 100644 test/towerops/workers/insight_llm_enrichment_worker_test.exs diff --git a/CHANGELOG.txt b/CHANGELOG.txt index be0f0347..22d648d7 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,18 @@ +2026-05-09 +feat(insights): LLM-powered insight enrichment + Schema: added llm_summary, recommended_action, llm_model, llm_enriched_at + to preseem_insights (migration 20260509212449). + New module Towerops.LLM with swappable behaviour (real client + MockClient). + New worker Towerops.Workers.InsightLlmEnrichmentWorker — Oban cron every + 5 min — populates active insights with plain-language summaries and + recommended actions. Failures (missing key, rate limit, parse) are + logged and skipped; insights still display without AI text. + k8s: optional towerops-llm secret referenced from deployment.yaml as + optional in both init and main containers; example template at + k8s/towerops-llm-secret.example.yaml. + UI: insights LiveView (/insights) renders summary + recommended action + in a styled callout when present. + 2026-05-06 feat: optionally attach a coverage to a specific device at the chosen site Schema: added nullable `device_id` FK to coverages (migration diff --git a/config/runtime.exs b/config/runtime.exs index f879b1a6..71545722 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -20,6 +20,14 @@ if System.get_env("PHX_SERVER") do config :towerops, ToweropsWeb.Endpoint, server: true end +# DeepSeek LLM (used by Towerops.LLM.DeepSeek for insight enrichment). +# DEEPSEEK_API_KEY is optional — when missing, enrichment is skipped and +# insights still display without an LLM-generated summary. +config :towerops, Towerops.LLM.DeepSeek, + api_key: System.get_env("DEEPSEEK_API_KEY"), + base_url: System.get_env("DEEPSEEK_BASE_URL") || "https://api.deepseek.com/v1", + model: System.get_env("DEEPSEEK_MODEL") || "deepseek-v4-pro" + config :towerops, ToweropsWeb.Endpoint, http: [port: String.to_integer(System.get_env("PORT", "4000"))] # Configure Google Maps API key for geocoding (available in all environments) @@ -250,6 +258,8 @@ if config_env() == :prod do {"*/5 * * * *", Towerops.Workers.SystemInsightWorker}, # Wireless client health insights every 5 minutes {"*/5 * * * *", Towerops.Workers.WirelessInsightWorker}, + # Enrich active insights with LLM-generated summaries every 5 minutes + {"*/5 * * * *", Towerops.Workers.InsightLlmEnrichmentWorker}, # Gaiia reconciliation insights nightly at 4:30 AM {"30 4 * * *", Towerops.Workers.GaiiaInsightWorker}, # Backhaul capacity utilization insights every 15 minutes diff --git a/config/test.exs b/config/test.exs index 2013f9aa..00d2c19a 100644 --- a/config/test.exs +++ b/config/test.exs @@ -82,6 +82,11 @@ else queue_interval: 5_000 end +# Use mock LLM in tests by default. Tests that exercise the real DeepSeek +# client (e.g. test/towerops/llm/deepseek_test.exs) call the module +# directly and bypass this configuration. +config :towerops, Towerops.LLM, impl: Towerops.LLM.MockClient + # Configure Cloak encryption for testing # Use a fixed test key (same as dev for simplicity) config :towerops, Towerops.Vault, diff --git a/docs/plans/2026-05-09-llm-recommendations-and-frequency-rule.md b/docs/plans/2026-05-09-llm-recommendations-and-frequency-rule.md new file mode 100644 index 00000000..1a4164bd --- /dev/null +++ b/docs/plans/2026-05-09-llm-recommendations-and-frequency-rule.md @@ -0,0 +1,959 @@ +# LLM-Enriched Recommendations + AP Frequency-Change Rule + +> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task. + +**Goal:** Make towerops produce concrete, actionable recommendations ("AP X needs a frequency change") backed by structured evidence and explained in natural language by an LLM (DeepSeek). Phase 1 ships the LLM-enrichment foundation against existing insights; Phase 2 adds the first new multi-source rule (frequency change) on top. + +**Architecture:** +- Reuse the existing `preseem_insights` table (already a general insight store written to by 4+ workers). Add an `llm_summary` text field and a `recommended_action` text field. Keep deterministic rules generating structured findings; LLM only paraphrases/explains. +- DeepSeek client lives in `lib/towerops/llm/deepseek.ex`, fronted by a thin `Towerops.LLM` context. API key from `DEEPSEEK_API_KEY` env (system-wide, not per-org). HTTP via `:req`. +- A new Oban worker (`InsightLlmEnrichmentWorker`) listens for new active insights without an `llm_summary` and fills it in. Failures are non-fatal (insight still shown, just no LLM text). +- Phase 2 adds a `wireless_neighbor_scans` table populated by SNMP wireless tables (vendor profiles), and a `FrequencyChangeRule` that emits `ap_frequency_change` insights with structured evidence (current channel, top neighbors on same/adjacent channel, recommended channel). + +**Tech Stack:** +- Elixir 1.17 / Phoenix 1.8 / Ecto / Oban (Pro vendored) +- HTTP: `:req` (only approved client) +- DeepSeek API: OpenAI-compatible `/chat/completions` at `https://api.deepseek.com/v1` +- Tests: ExUnit + Mox for HTTP boundary +- TDD: tests first (per project AGENTS.md) + +**Out of scope (deferred):** +- Per-org DeepSeek keys (system-wide for now) +- Streaming LLM responses +- Other rules (sector overload, GPS sync, thermal, backhaul) — same pattern, follow-on plans +- LLM Q&A / RAG over insights + +--- + +## Phase 1 — LLM Enrichment MVP + +### Task 1: Add llm_summary + recommended_action columns to preseem_insights + +**Files:** +- Create: `priv/repo/migrations/_add_llm_fields_to_preseem_insights.exs` +- Modify: `lib/towerops/preseem/insight.ex` +- Test: `test/towerops/preseem/insight_test.exs` (existing — extend) + +**Step 1: Generate migration** + +```bash +cd /Users/graham/dev/towerops/towerops-web && mix ecto.gen.migration add_llm_fields_to_preseem_insights +``` + +**Step 2: Write migration** + +```elixir +defmodule Towerops.Repo.Migrations.AddLlmFieldsToPreseemInsights do + use Ecto.Migration + + def change do + alter table(:preseem_insights) do + add :llm_summary, :text + add :recommended_action, :text + add :llm_model, :string + add :llm_enriched_at, :utc_datetime + end + + create index(:preseem_insights, [:llm_enriched_at]) + end +end +``` + +**Step 3: Add a failing test that round-trips llm_summary** + +In `test/towerops/preseem/insight_test.exs`, add: + +```elixir +test "changeset accepts llm_summary, recommended_action, llm_model, llm_enriched_at" do + attrs = %{ + organization_id: Ecto.UUID.generate(), + type: "wireless_signal_weak", + urgency: "warning", + channel: "proactive", + title: "weak signal", + llm_summary: "AP overloaded; consider rebalancing", + recommended_action: "Move 3 clients to neighboring AP", + llm_model: "deepseek-chat", + llm_enriched_at: ~U[2026-05-09 12:00:00Z] + } + + changeset = Insight.changeset(%Insight{}, attrs) + assert changeset.valid? + assert get_change(changeset, :llm_summary) == "AP overloaded; consider rebalancing" + assert get_change(changeset, :recommended_action) == "Move 3 clients to neighboring AP" +end +``` + +**Step 4: Run the test, expect FAIL** + +```bash +cd /Users/graham/dev/towerops/towerops-web && mix test test/towerops/preseem/insight_test.exs --only line: +``` +Expected: FAIL — fields not in cast list. + +**Step 5: Update Insight schema** + +In `lib/towerops/preseem/insight.ex`: + +```elixir +schema "preseem_insights" do + # ... existing fields ... + field :llm_summary, :string + field :recommended_action, :string + field :llm_model, :string + field :llm_enriched_at, :utc_datetime + # ... existing belongs_to and timestamps ... +end + +def changeset(insight, attrs) do + insight + |> cast(attrs, [ + # ... existing fields ..., + :llm_summary, + :recommended_action, + :llm_model, + :llm_enriched_at + ]) + # ... existing validations ... +end +``` + +**Step 6: Run migration and tests** + +```bash +mix ecto.migrate +mix test test/towerops/preseem/insight_test.exs +``` +Expected: PASS. + +**Step 7: Commit** + +```bash +git add priv/repo/migrations/*add_llm_fields* lib/towerops/preseem/insight.ex test/towerops/preseem/insight_test.exs +git commit -m "feat(insights): add llm_summary, recommended_action, llm_model, llm_enriched_at" +``` + +--- + +### Task 2: DeepSeek client module (boundary, mockable) + +**Files:** +- Create: `lib/towerops/llm.ex` (context with `complete/2` callback) +- Create: `lib/towerops/llm/deepseek.ex` (real implementation) +- Create: `lib/towerops/llm/behaviour.ex` (`@callback complete(prompt, opts) :: {:ok, String.t()} | {:error, term()}`) +- Modify: `config/config.exs`, `config/test.exs`, `config/runtime.exs` +- Test: `test/towerops/llm/deepseek_test.exs` + +**Step 1: Write the behaviour** + +```elixir +# lib/towerops/llm/behaviour.ex +defmodule Towerops.LLM.Behaviour do + @moduledoc "Behaviour for LLM clients used in recommendation enrichment." + @callback complete(messages :: list(map()), opts :: keyword()) :: + {:ok, %{content: String.t(), model: String.t()}} | {:error, term()} +end +``` + +**Step 2: Write the context** + +```elixir +# lib/towerops/llm.ex +defmodule Towerops.LLM do + @moduledoc "Public LLM API. Delegates to the configured implementation." + + def complete(messages, opts \\ []) do + impl().complete(messages, opts) + end + + defp impl, do: Application.get_env(:towerops, __MODULE__)[:impl] || Towerops.LLM.DeepSeek +end +``` + +**Step 3: Write the failing client test** + +```elixir +# test/towerops/llm/deepseek_test.exs +defmodule Towerops.LLM.DeepSeekTest do + use ExUnit.Case, async: true + alias Towerops.LLM.DeepSeek + + setup do + bypass = Bypass.open() + Application.put_env(:towerops, DeepSeek, base_url: "http://localhost:#{bypass.port}", api_key: "test-key") + on_exit(fn -> Application.delete_env(:towerops, DeepSeek) end) + {:ok, bypass: bypass} + end + + test "complete/2 posts to /chat/completions and parses content", %{bypass: bypass} do + Bypass.expect_once(bypass, "POST", "/chat/completions", fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + assert {:ok, %{"messages" => [%{"role" => "user", "content" => "hello"}], "model" => "deepseek-chat"}} = Jason.decode(body) + auth = Plug.Conn.get_req_header(conn, "authorization") |> List.first() + assert auth == "Bearer test-key" + + Plug.Conn.resp(conn, 200, Jason.encode!(%{ + "model" => "deepseek-chat", + "choices" => [%{"message" => %{"content" => "hi there"}}] + })) + end) + + assert {:ok, %{content: "hi there", model: "deepseek-chat"}} = + DeepSeek.complete([%{role: "user", content: "hello"}], []) + end + + test "complete/2 returns {:error, :missing_api_key} when key missing" do + Application.put_env(:towerops, DeepSeek, base_url: "http://example.invalid") + assert {:error, :missing_api_key} = DeepSeek.complete([%{role: "user", content: "x"}], []) + end + + test "complete/2 returns {:error, {:http, status, body}} on non-2xx", %{bypass: bypass} do + Bypass.expect_once(bypass, "POST", "/chat/completions", fn conn -> + Plug.Conn.resp(conn, 500, ~s({"error":"boom"})) + end) + + assert {:error, {:http, 500, _}} = + DeepSeek.complete([%{role: "user", content: "x"}], []) + end +end +``` + +NOTE: Bypass is in deps already (verify with `mix deps | grep bypass`). If absent, add it under `Mix.env() in [:test]`. + +**Step 4: Run the test, expect FAIL** + +```bash +mix test test/towerops/llm/deepseek_test.exs +``` +Expected: FAIL — module DeepSeek not defined. + +**Step 5: Implement DeepSeek client** + +```elixir +# lib/towerops/llm/deepseek.ex +defmodule Towerops.LLM.DeepSeek do + @moduledoc """ + DeepSeek chat completion client (OpenAI-compatible). + + Configuration in `config/runtime.exs`: + + config :towerops, Towerops.LLM.DeepSeek, + api_key: System.get_env("DEEPSEEK_API_KEY"), + base_url: System.get_env("DEEPSEEK_BASE_URL") || "https://api.deepseek.com/v1", + model: "deepseek-chat" + """ + @behaviour Towerops.LLM.Behaviour + + @default_base_url "https://api.deepseek.com/v1" + @default_model "deepseek-chat" + @receive_timeout 30_000 + + @impl true + def complete(messages, opts) do + cfg = Application.get_env(:towerops, __MODULE__, []) + api_key = opts[:api_key] || cfg[:api_key] + base_url = opts[:base_url] || cfg[:base_url] || @default_base_url + model = opts[:model] || cfg[:model] || @default_model + + cond do + is_nil(api_key) or api_key == "" -> + {:error, :missing_api_key} + + true -> + request(base_url, api_key, model, messages, opts) + end + end + + defp request(base_url, api_key, model, messages, opts) do + body = %{ + model: model, + messages: messages, + temperature: opts[:temperature] || 0.2, + max_tokens: opts[:max_tokens] || 400 + } + + case Req.post(base_url <> "/chat/completions", + json: body, + headers: [{"authorization", "Bearer " <> api_key}], + receive_timeout: @receive_timeout + ) do + {:ok, %{status: status, body: %{"choices" => [%{"message" => %{"content" => content}} | _], "model" => actual_model}}} when status in 200..299 -> + {:ok, %{content: content, model: actual_model}} + + {:ok, %{status: status, body: body}} -> + {:error, {:http, status, body}} + + {:error, reason} -> + {:error, reason} + end + end +end +``` + +**Step 6: Wire config** + +`config/runtime.exs` (production runtime block): +```elixir +config :towerops, Towerops.LLM.DeepSeek, + api_key: System.get_env("DEEPSEEK_API_KEY"), + base_url: System.get_env("DEEPSEEK_BASE_URL") || "https://api.deepseek.com/v1" +``` + +`config/test.exs`: +```elixir +config :towerops, Towerops.LLM, impl: Towerops.LLM.MockClient +config :towerops, Towerops.LLM.DeepSeek, base_url: "http://localhost", api_key: nil +``` + +`config/dev.exs` (so a dev with the env var set can try it): +```elixir +config :towerops, Towerops.LLM.DeepSeek, + api_key: System.get_env("DEEPSEEK_API_KEY"), + base_url: "https://api.deepseek.com/v1" +``` + +**Step 7: Add Mock client for tests** + +```elixir +# lib/towerops/llm/mock_client.ex +defmodule Towerops.LLM.MockClient do + @moduledoc "Test stand-in for Towerops.LLM. Returns canned responses configurable via Process dict." + @behaviour Towerops.LLM.Behaviour + + @impl true + def complete(_messages, _opts) do + Process.get(:llm_mock_response, {:ok, %{content: "MOCK_RESPONSE", model: "mock"}}) + end +end +``` + +Tests can then `Process.put(:llm_mock_response, {:ok, %{content: "...", model: "deepseek-chat"}})` per-test. + +**Step 8: Run all tests, expect PASS** + +```bash +mix test test/towerops/llm/ +``` + +**Step 9: Commit** + +```bash +git add lib/towerops/llm.ex lib/towerops/llm/ config/runtime.exs config/dev.exs config/test.exs test/towerops/llm/ +git commit -m "feat(llm): add DeepSeek client behind Towerops.LLM context" +``` + +--- + +### Task 3: Insights public API for LLM enrichment + +**Files:** +- Modify: `lib/towerops/preseem/insights.ex` +- Test: `test/towerops/preseem/insights_test.exs` (existing — extend) + +**Step 1: Add failing test for `list_unenriched_insights/1` and `apply_llm_enrichment/3`** + +```elixir +test "list_unenriched_insights/1 returns active insights with nil llm_enriched_at, oldest first" do + org = insert(:organization) + device = insert(:device, organization: org) + i1 = insert(:preseem_insight, organization: org, device: device, status: "active", + inserted_at: ~U[2026-05-01 00:00:00Z]) + _i2 = insert(:preseem_insight, organization: org, status: "active", + llm_enriched_at: ~U[2026-05-08 12:00:00Z]) + _i3 = insert(:preseem_insight, organization: org, status: "dismissed") + i4 = insert(:preseem_insight, organization: org, status: "active", + inserted_at: ~U[2026-05-02 00:00:00Z]) + + assert [%{id: id1}, %{id: id4}] = Insights.list_unenriched_insights(limit: 10) + assert id1 == i1.id + assert id4 == i4.id +end + +test "apply_llm_enrichment/3 sets llm_summary, recommended_action, llm_model, llm_enriched_at" do + insight = insert(:preseem_insight, status: "active") + assert {:ok, updated} = Insights.apply_llm_enrichment(insight, + %{summary: "AP overloaded.", action: "Rebalance 3 clients."}, + "deepseek-chat") + assert updated.llm_summary == "AP overloaded." + assert updated.recommended_action == "Rebalance 3 clients." + assert updated.llm_model == "deepseek-chat" + refute is_nil(updated.llm_enriched_at) +end +``` + +You'll likely need to add a `preseem_insight` factory in `test/support/factories.ex` (or wherever ex_machina lives — check existing factories first). If factories aren't used, build attrs inline via the existing `Insights.insert_insight_if_new/1`. + +**Step 2: Run the test, expect FAIL** + +```bash +mix test test/towerops/preseem/insights_test.exs +``` + +**Step 3: Implement** + +In `lib/towerops/preseem/insights.ex`: + +```elixir +import Ecto.Query + +def list_unenriched_insights(opts \\ []) do + limit = Keyword.get(opts, :limit, 50) + + from(i in Insight, + where: i.status == "active" and is_nil(i.llm_enriched_at), + order_by: [asc: i.inserted_at], + limit: ^limit + ) + |> Repo.all() +end + +def apply_llm_enrichment(%Insight{} = insight, %{summary: summary, action: action}, model) do + insight + |> Insight.changeset(%{ + llm_summary: summary, + recommended_action: action, + llm_model: model, + llm_enriched_at: DateTime.utc_now() |> DateTime.truncate(:second) + }) + |> Repo.update() +end +``` + +**Step 4: Run tests, expect PASS** + +```bash +mix test test/towerops/preseem/insights_test.exs +``` + +**Step 5: Commit** + +```bash +git add lib/towerops/preseem/insights.ex test/towerops/preseem/insights_test.exs test/support/ +git commit -m "feat(insights): add list_unenriched_insights/1 and apply_llm_enrichment/3" +``` + +--- + +### Task 4: InsightLlmEnrichmentWorker (Oban) + +**Files:** +- Create: `lib/towerops/workers/insight_llm_enrichment_worker.ex` +- Create: `lib/towerops/llm/insight_prompt.ex` (prompt builder, pure) +- Test: `test/towerops/llm/insight_prompt_test.exs` +- Test: `test/towerops/workers/insight_llm_enrichment_worker_test.exs` +- Modify: `config/runtime.exs` cron + queue list + +**Step 1: Write a failing pure-function test for the prompt builder** + +```elixir +# test/towerops/llm/insight_prompt_test.exs +defmodule Towerops.LLM.InsightPromptTest do + use ExUnit.Case, async: true + alias Towerops.LLM.InsightPrompt + alias Towerops.Preseem.Insight + + test "build/1 returns system + user messages and includes structured metadata as JSON" do + insight = %Insight{ + type: "wireless_signal_weak", + urgency: "warning", + title: "Weak signal: AA:BB:CC (-78 dBm)", + description: "Wireless client AA:BB:CC has weak signal.", + metadata: %{"signal_strength" => -78, "threshold" => -75, "device_id" => "abc"} + } + + messages = InsightPrompt.build(insight) + assert [%{role: "system", content: sys}, %{role: "user", content: user}] = messages + assert sys =~ "WISP/network operations assistant" + assert user =~ "wireless_signal_weak" + assert user =~ "-78" + assert user =~ "JSON" + end + + test "parse/1 extracts summary and action from JSON response" do + json = ~s|{"summary": "AP X overloaded.", "action": "Move clients to AP Y."}| + assert {:ok, %{summary: "AP X overloaded.", action: "Move clients to AP Y."}} = + InsightPrompt.parse(json) + end + + test "parse/1 falls back when content is not JSON" do + assert {:ok, %{summary: "raw text", action: nil}} = InsightPrompt.parse("raw text") + end +end +``` + +**Step 2: Run, FAIL** + +**Step 3: Implement prompt + parser** + +```elixir +# lib/towerops/llm/insight_prompt.ex +defmodule Towerops.LLM.InsightPrompt do + @moduledoc """ + Builds chat messages for enriching a network-operations insight, and + parses the LLM's JSON response back into a {summary, action} pair. + """ + alias Towerops.Preseem.Insight + + @system """ + You are a WISP/network operations assistant. The user will give you a structured + insight detected by an automated rule. Your job: + + 1. Restate the finding in one or two plain sentences a network operator can act on. + 2. Suggest one specific next action they should take, citing the numbers in the data. + + Do NOT invent numbers. Only reference values present in the structured data. + Respond ONLY with JSON: {"summary": "...", "action": "..."}. + """ + + @spec build(Insight.t()) :: [map()] + def build(%Insight{} = insight) do + payload = + %{ + type: insight.type, + urgency: insight.urgency, + title: insight.title, + description: insight.description, + metadata: insight.metadata || %{} + } + |> Jason.encode!(pretty: true) + + [ + %{role: "system", content: @system}, + %{role: "user", content: "Insight (JSON):\n" <> payload} + ] + end + + @spec parse(String.t()) :: {:ok, %{summary: String.t(), action: String.t() | nil}} + def parse(content) when is_binary(content) do + cleaned = content |> String.trim() |> strip_code_fences() + + case Jason.decode(cleaned) do + {:ok, %{"summary" => s} = m} when is_binary(s) -> + {:ok, %{summary: s, action: m["action"]}} + + _ -> + {:ok, %{summary: cleaned, action: nil}} + end + end + + defp strip_code_fences(text) do + text + |> String.replace(~r/^```(json)?\s*/m, "") + |> String.replace(~r/```\s*$/m, "") + |> String.trim() + end +end +``` + +**Step 4: Run, PASS** + +**Step 5: Write a failing worker test** + +```elixir +# test/towerops/workers/insight_llm_enrichment_worker_test.exs +defmodule Towerops.Workers.InsightLlmEnrichmentWorkerTest do + use Towerops.DataCase, async: false + alias Towerops.Workers.InsightLlmEnrichmentWorker + alias Towerops.Preseem.{Insight, Insights} + alias Towerops.Repo + + setup do + Application.put_env(:towerops, Towerops.LLM, impl: Towerops.LLM.MockClient) + :ok + end + + test "perform/1 enriches active, unenriched insights" do + org = insert(:organization) + insight = insert(:preseem_insight, organization: org, status: "active", + llm_enriched_at: nil, + type: "wireless_signal_weak", + metadata: %{"signal_strength" => -82}) + + Process.put(:llm_mock_response, {:ok, %{ + content: ~s|{"summary":"Weak link to CPE","action":"Re-aim CPE antenna"}|, + model: "deepseek-chat" + }}) + + assert :ok = perform_job(InsightLlmEnrichmentWorker, %{}) + + updated = Repo.get!(Insight, insight.id) + assert updated.llm_summary == "Weak link to CPE" + assert updated.recommended_action == "Re-aim CPE antenna" + assert updated.llm_model == "deepseek-chat" + refute is_nil(updated.llm_enriched_at) + end + + test "perform/1 leaves insight unenriched on LLM error and does not crash" do + insight = insert(:preseem_insight, status: "active", llm_enriched_at: nil) + Process.put(:llm_mock_response, {:error, :missing_api_key}) + + assert :ok = perform_job(InsightLlmEnrichmentWorker, %{}) + + refreshed = Repo.get!(Insight, insight.id) + assert is_nil(refreshed.llm_summary) + assert is_nil(refreshed.llm_enriched_at) + end + + test "perform/1 skips already-enriched insights" do + insert(:preseem_insight, status: "active", + llm_enriched_at: ~U[2026-05-01 00:00:00Z], + llm_summary: "existing") + + Process.put(:llm_mock_response, {:ok, %{content: "should not be called", model: "x"}}) + + assert :ok = perform_job(InsightLlmEnrichmentWorker, %{}) + end +end +``` + +**Step 6: Run, FAIL** + +**Step 7: Implement the worker** + +```elixir +# lib/towerops/workers/insight_llm_enrichment_worker.ex +defmodule Towerops.Workers.InsightLlmEnrichmentWorker do + @moduledoc """ + Oban cron worker that fills in `llm_summary` / `recommended_action` for + active insights that have no LLM enrichment yet. + + Run frequency is intentionally modest (every 5 min). Failures (rate limits, + missing key, parse errors) are logged and skipped — the insight is still + visible to the operator without LLM text. + """ + use Oban.Worker, queue: :maintenance, max_attempts: 3 + require Logger + + alias Towerops.LLM + alias Towerops.LLM.InsightPrompt + alias Towerops.Preseem.Insights + + @batch_size 25 + + @impl Oban.Worker + def perform(%Oban.Job{}) do + Insights.list_unenriched_insights(limit: @batch_size) + |> Enum.each(&enrich_one/1) + + :ok + end + + defp enrich_one(insight) do + messages = InsightPrompt.build(insight) + + case LLM.complete(messages, max_tokens: 400, temperature: 0.2) do + {:ok, %{content: content, model: model}} -> + case InsightPrompt.parse(content) do + {:ok, parsed} -> + Insights.apply_llm_enrichment(insight, parsed, model) + + other -> + Logger.warning("LLM parse failed for insight #{insight.id}: #{inspect(other)}") + end + + {:error, reason} -> + Logger.warning("LLM enrichment failed for insight #{insight.id}: #{inspect(reason)}") + end + end +end +``` + +**Step 8: Add cron entry in `config/runtime.exs`** + +In the `:crontab` block, add: +```elixir +{"*/5 * * * *", Towerops.Workers.InsightLlmEnrichmentWorker} +``` + +**Step 9: Run tests, PASS** + +```bash +mix test test/towerops/workers/insight_llm_enrichment_worker_test.exs +mix test test/towerops/llm/ +``` + +**Step 10: Commit** + +```bash +git add lib/towerops/llm/insight_prompt.ex lib/towerops/workers/insight_llm_enrichment_worker.ex test/towerops/llm/insight_prompt_test.exs test/towerops/workers/insight_llm_enrichment_worker_test.exs config/runtime.exs +git commit -m "feat(insights): LLM enrichment worker via DeepSeek" +``` + +--- + +### Task 5: Show llm_summary + recommended_action in Preseem Insights LiveView + +**Files:** +- Modify: `lib/towerops_web/live/org/preseem_insights_live.ex` (template / render block) +- Modify: `lib/towerops_web/live/device_live/show.html.heex` (Preseem tab insight cards) +- Test: `test/towerops_web/live/insights_live_events_test.exs` (existing, extend) + +**Step 1: Add failing LiveView render test** + +```elixir +test "renders llm_summary and recommended_action when present", %{conn: conn, org: org, user: user} do + insight = insert(:preseem_insight, organization: org, status: "active", + llm_summary: "AP overloaded with 80 clients", + recommended_action: "Move 10 clients to neighboring AP") + conn = log_in_user(conn, user) + {:ok, _view, html} = live(conn, ~p"/orgs/#{org.slug}/settings/integrations/preseem/insights") + + assert html =~ "AP overloaded with 80 clients" + assert html =~ "Move 10 clients to neighboring AP" +end +``` + +**Step 2: Run, FAIL** + +**Step 3: Update the LiveView template** (locate the insight card render block; render `@insight.llm_summary` in a soft-bg box and `@insight.recommended_action` as a prominent "Recommended action" line). Add a small "AI-generated" note + the model name. + +A concrete pattern (HEEX): +```heex +
+

Plain-language summary

+

{@insight.llm_summary}

+

+ Recommended action: {@insight.recommended_action} +

+

AI-generated by {@insight.llm_model}

+
+``` + +**Step 4: Run, PASS** + +**Step 5: Mirror the same render block in `device_live/show.html.heex` Preseem tab** (look for where current insights are listed) + +**Step 6: Smoke run server in dev manually if convenient (optional — `mix phx.server`).** UI changes auto-reload. Verify visually if dev server is running. + +**Step 7: Commit** + +```bash +git add lib/towerops_web/live/org/preseem_insights_live.ex lib/towerops_web/live/device_live/show.html.heex test/towerops_web/live/insights_live_events_test.exs +git commit -m "feat(insights): render LLM summary and recommended action in UI" +``` + +--- + +### Task 6: Phase 1 verification + manual smoke + +**Files:** none (verification step) + +**Step 1: Run full precommit** + +```bash +cd /Users/graham/dev/towerops/towerops-web && mix precommit +``` +Expected: PASS. + +**Step 2: Manual smoke (only if user has DEEPSEEK_API_KEY in env)** + +```bash +DEEPSEEK_API_KEY=... iex -S mix +iex> Towerops.LLM.complete([%{role: "user", content: "Say 'hello'"}]) +{:ok, %{content: "hello", model: "deepseek-chat"}} +``` + +**Step 3: Update `priv/static/changelog.txt` and `CHANGELOG.txt`** + +`priv/static/changelog.txt` (top): +``` +2026-05-09 +* AI-generated summaries and recommended actions for network insights +``` + +`CHANGELOG.txt` (top): +``` +2026-05-09: feat(insights): LLM enrichment via DeepSeek +- Adds llm_summary, recommended_action, llm_model, llm_enriched_at to preseem_insights +- New Towerops.LLM context with DeepSeek implementation +- InsightLlmEnrichmentWorker (every 5 min) populates summaries for active insights +- LiveView renders LLM summary + recommendation on insights and device pages +``` + +**Step 4: Commit** + +```bash +git add priv/static/changelog.txt CHANGELOG.txt +git commit -m "docs: changelog entries for LLM enrichment" +``` + +--- + +## Phase 2 — AP Frequency-Change Rule + +**Note:** Phase 1 is independently shippable. Phase 2 builds on it: same insight table, automatic LLM enrichment will apply to the new `ap_frequency_change` type with no extra wiring. + +### Task 7: Add wireless_neighbor_scans schema + +**Files:** +- Create: `priv/repo/migrations/_create_wireless_neighbor_scans.exs` +- Create: `lib/towerops/snmp/wireless_neighbor_scan.ex` +- Test: `test/towerops/snmp/wireless_neighbor_scan_test.exs` + +**Schema fields (one row per neighbor seen by an AP per scan window):** +- `id` binary_id +- `device_id` (FK devices, the observing AP) +- `bssid` string (neighbor MAC) +- `ssid` string (nullable) +- `channel` integer +- `frequency_mhz` integer +- `channel_width_mhz` integer (nullable, default 20) +- `rssi_dbm` integer +- `is_own_fleet` boolean (set true if `bssid` matches another device in same org) +- `seen_at` utc_datetime +- `inserted_at` (no updated_at — append-only) +- Indexes: `[device_id, seen_at]`, `[bssid]`, `[device_id, channel]` + +**Step 1: Write failing schema test** (`changeset/2` validates required fields + accepts optional ones). + +**Step 2: Generate migration + write schema** following same TDD cycle as Task 1. + +**Step 3: Commit:** `feat(snmp): wireless_neighbor_scans schema for RF environment scanning` + +--- + +### Task 8: Per-device current channel storage + +**Files:** +- Modify: `lib/towerops/snmp/device.ex` (add `current_channel`, `current_frequency_mhz`, `current_channel_width_mhz`, `last_radio_seen_at`) +- Migration: alter `snmp_devices` +- Test: changeset coverage + +These are populated by vendor profiles. We're only adding storage here; the actual polling extraction is Task 9. + +**TDD cycle as Task 1.** Commit: `feat(snmp): persist AP current radio channel/frequency on snmp_devices` + +--- + +### Task 9: Vendor profile extraction (Cambium + UniFi + RouterOS) + +**Files (modify):** +- `lib/towerops/snmp/profiles/vendors/cambium.ex` +- `lib/towerops/snmp/profiles/vendors/unifi.ex` +- `lib/towerops/snmp/profiles/vendors/mikrotik.ex` + +For each, add OID extraction for current channel + neighbor scan table where supported: +- Cambium: `cambium450MWirelessChannel`, `whispRfFrequency` etc. — check the vendor MIBs in `priv/mibs/` +- UniFi: `UBNT-UniFi-MIB` channel/frequency leaves +- RouterOS: `mtxrWlChannel`, `mtxrWlSurveyTable` for neighbor scan + +**TDD cycle:** +1. Test parses a sample SNMP varbind list into `%{current_channel: ..., neighbors: [...]}` +2. Implement minimal walk callback +3. Persist via `Snmp` context + +If a vendor doesn't have a public neighbor-scan MIB, skip it and mark as a follow-up. + +**Commit:** `feat(snmp): extract AP channel + neighbor scans for cambium/unifi/routeros` + +--- + +### Task 10: FrequencyChangeRule + +**Files:** +- Create: `lib/towerops/recommendations/rules/frequency_change.ex` +- Create: `lib/towerops/recommendations/rule_behaviour.ex` (`@callback evaluate(org_id) :: [Insight changeset]`) +- Modify: `lib/towerops/preseem/insight.ex` — add `ap_frequency_change` to `@valid_types` +- Test: `test/towerops/recommendations/rules/frequency_change_test.exs` + +**Rule logic (deterministic):** +- Input: a `Towerops.Snmp.Device` with non-nil `current_channel` +- For each AP, query its `wireless_neighbor_scans` from last 24h +- Find neighbors on **same channel** with RSSI ≥ −75 dBm OR neighbors on **adjacent channel** (±1 in 5GHz UNII grid, or ±5 MHz overlap) with RSSI ≥ −70 dBm +- If ≥ 1 strong same-channel neighbor exists, recommend a channel change +- Recommended channel = pick the channel in the AP's allowed channel list (configurable per band) with the lowest aggregate interference score: `Σ 10^((rssi+95)/10)` over neighbors observed on that channel in last 24h +- Emit insight with metadata: + ``` + %{ + "current_channel" => 149, + "current_frequency_mhz" => 5745, + "channel_width_mhz" => 80, + "top_offenders" => [ + %{"bssid" => "...", "ssid" => "...", "rssi_dbm" => -62, "is_own_fleet" => true}, + ... + ], + "recommended_channel" => 161, + "recommended_score" => 3.14, + "current_score" => 42.7, + "improvement_db" => 11.4 + } + ``` +- Title: `"AP recommend channel change: 149 → 161 (-11 dB interference)"` +- Urgency: `warning` (or `critical` if current_score > 100) +- channel: `proactive`, source: `system` + +**Step 1: Write failing tests** for `FrequencyChange.evaluate/1`: +- empty fixture → `[]` +- AP with no neighbors → no recommendation +- AP with one −62dBm same-channel neighbor → recommendation present +- Recommended channel beats current by ≥ 6 dB +- `top_offenders` sorted by RSSI desc, capped at 5 + +**Step 2: Implement.** + +**Step 3: Run, PASS.** + +**Step 4: Commit:** `feat(rules): frequency-change recommendation rule` + +--- + +### Task 11: RecommendationsRunWorker (cron, hourly) + +**Files:** +- Create: `lib/towerops/workers/recommendations_run_worker.ex` +- Test: `test/towerops/workers/recommendations_run_worker_test.exs` +- Modify: `config/runtime.exs` cron + +Worker iterates orgs, runs all rules, upserts insights via `Insights.insert_insight_if_new/1`. Cron `@hourly`. **Same TDD cycle.** + +**Commit:** `feat(workers): hourly recommendations run worker` + +--- + +### Task 12: UI evidence card for ap_frequency_change + +**Files:** +- Modify: `lib/towerops_web/live/org/preseem_insights_live.ex` (or extract a `RecommendationCard` component) +- Possibly add: `lib/towerops_web/live/components/frequency_change_card.ex` +- Test: e2e view test + +**Render:** +- Header: "Frequency change recommended" +- Current vs recommended channel side-by-side +- List of top offenders (BSSID/SSID/RSSI/own-fleet badge) +- Improvement in dB (interference score delta) +- LLM summary box (auto-populated by Phase 1 worker — no extra wiring) +- Actions: Dismiss, Mark Actioned, Snooze 7 days + +**TDD: render test asserts current_channel and recommended_channel appear.** + +**Commit:** `feat(ui): frequency-change recommendation card` + +--- + +### Task 13: Phase 2 verification + +**Step 1:** `mix precommit`. PASS. + +**Step 2:** Update both changelog files. + +**Step 3:** Final commit. + +--- + +## Out-of-band manual checks (operator-side, post-merge) + +- Set `DEEPSEEK_API_KEY` in production secrets (`towerops-secrets`). +- Verify `InsightLlmEnrichmentWorker` is in Oban dashboard. +- Watch the first hour of cron runs; if rate limits hit, lower batch_size. +- Pick one production AP with known co-channel interference, verify recommendation surfaces with reasonable evidence. + +## Future work (out of scope) + +- Sector overload rule (uses Preseem airtime + own SNMP) +- GPS sync drift rule (vendor-specific) +- Thermal correlation rule +- Backhaul saturation rule +- Per-CPE re-aim rule (needs SNR margin computation against MCS table) +- LLM Q&A over insights (RAG) +- LLM-proposed novel rules (offline analysis of unexplained QoE drops) diff --git a/k8s/README.md b/k8s/README.md index 399315aa..02b60167 100644 --- a/k8s/README.md +++ b/k8s/README.md @@ -10,6 +10,19 @@ Required secrets in the `towerops` namespace: - `towerops-db` - Database connection credentials - `towerops-aws` - AWS credentials (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_REGION) +Optional secrets: +- `towerops-llm` - DeepSeek API credentials for LLM-powered insight enrichment. + Optional — when missing, insights still display without an AI summary. + See `k8s/towerops-llm-secret.example.yaml` for the template, or create with: + + ```bash + kubectl create secret generic towerops-llm \ + --from-literal=DEEPSEEK_API_KEY="" \ + --from-literal=DEEPSEEK_MODEL="deepseek-v4-pro" \ + -n towerops + kubectl rollout restart deployment/towerops -n towerops + ``` + For local development, the project root `.envrc` is used by direnv. ## Deployment Timestamp diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index c5f1df98..822d94fb 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -91,6 +91,18 @@ spec: name: towerops-billing key: STRIPE_METER_ID optional: true + - name: DEEPSEEK_API_KEY + valueFrom: + secretKeyRef: + name: towerops-llm + key: DEEPSEEK_API_KEY + optional: true + - name: DEEPSEEK_MODEL + valueFrom: + secretKeyRef: + name: towerops-llm + key: DEEPSEEK_MODEL + optional: true envFrom: - secretRef: name: towerops-db @@ -200,6 +212,18 @@ spec: name: towerops-billing key: STRIPE_METER_ID optional: true + - name: DEEPSEEK_API_KEY + valueFrom: + secretKeyRef: + name: towerops-llm + key: DEEPSEEK_API_KEY + optional: true + - name: DEEPSEEK_MODEL + valueFrom: + secretKeyRef: + name: towerops-llm + key: DEEPSEEK_MODEL + optional: true envFrom: # Redis connection configured via towerops-redis secret - secretRef: diff --git a/k8s/towerops-llm-secret.example.yaml b/k8s/towerops-llm-secret.example.yaml new file mode 100644 index 00000000..2cf2e87d --- /dev/null +++ b/k8s/towerops-llm-secret.example.yaml @@ -0,0 +1,27 @@ +# Template for the towerops-llm secret. Apply with values filled in: +# +# kubectl create secret generic towerops-llm \ +# --from-literal=DEEPSEEK_API_KEY="" \ +# --from-literal=DEEPSEEK_MODEL="deepseek-v4-pro" \ +# -n towerops +# +# Or apply this file directly after replacing the placeholder strings with +# your real values, then run: kubectl apply -f k8s/towerops-llm-secret.example.yaml +# +# This file is committed as a placeholder — DO NOT put real keys in here. +# Real values live in 1Password and are created in the cluster manually. +# +# The deployment references this secret with `optional: true`, so the app +# will start fine without it (LLM enrichment just stays no-op). +--- +apiVersion: v1 +kind: Secret +metadata: + name: towerops-llm + namespace: towerops +type: Opaque +stringData: + # DeepSeek API key — get one at https://platform.deepseek.com/ + DEEPSEEK_API_KEY: "" + # Optional: override the default model (default: deepseek-v4-pro) + DEEPSEEK_MODEL: "" diff --git a/lib/towerops/llm.ex b/lib/towerops/llm.ex new file mode 100644 index 00000000..b9d254cc --- /dev/null +++ b/lib/towerops/llm.ex @@ -0,0 +1,21 @@ +defmodule Towerops.LLM do + @moduledoc """ + Public LLM API. Delegates to the configured implementation. + + Configure via `:towerops, Towerops.LLM` keyword `:impl`. Defaults to + `Towerops.LLM.DeepSeek`. Tests typically swap in `Towerops.LLM.MockClient`. + """ + + @behaviour Towerops.LLM.Behaviour + + @impl true + def complete(messages, opts \\ []) do + impl().complete(messages, opts) + end + + defp impl do + :towerops + |> Application.get_env(__MODULE__, []) + |> Keyword.get(:impl, Towerops.LLM.DeepSeek) + end +end diff --git a/lib/towerops/llm/behaviour.ex b/lib/towerops/llm/behaviour.ex new file mode 100644 index 00000000..be269ae1 --- /dev/null +++ b/lib/towerops/llm/behaviour.ex @@ -0,0 +1,13 @@ +defmodule Towerops.LLM.Behaviour do + @moduledoc """ + Behaviour for chat-completion LLM clients used in recommendation enrichment. + + The expectation is OpenAI-style chat messages — `[%{role: "system" | "user" | "assistant", content: "..."}]`. + """ + + @type message :: %{role: String.t(), content: String.t()} + @type response :: %{content: String.t(), model: String.t()} + + @callback complete(messages :: [message()], opts :: keyword()) :: + {:ok, response()} | {:error, term()} +end diff --git a/lib/towerops/llm/deepseek.ex b/lib/towerops/llm/deepseek.ex new file mode 100644 index 00000000..c105c4c0 --- /dev/null +++ b/lib/towerops/llm/deepseek.ex @@ -0,0 +1,86 @@ +defmodule Towerops.LLM.DeepSeek do + @moduledoc """ + DeepSeek chat-completion client (OpenAI-compatible). + + Configure in `config/runtime.exs`: + + config :towerops, Towerops.LLM.DeepSeek, + api_key: System.get_env("DEEPSEEK_API_KEY"), + base_url: System.get_env("DEEPSEEK_BASE_URL") || "https://api.deepseek.com/v1", + model: "deepseek-chat" + """ + + @behaviour Towerops.LLM.Behaviour + + @default_base_url "https://api.deepseek.com/v1" + @default_model "deepseek-v4-pro" + @receive_timeout 30_000 + + @impl true + def complete(messages, opts \\ []) do + cfg = Application.get_env(:towerops, __MODULE__, []) + api_key = opts[:api_key] || cfg[:api_key] + base_url = opts[:base_url] || cfg[:base_url] || @default_base_url + model = opts[:model] || cfg[:model] || @default_model + + if blank?(api_key) do + {:error, :missing_api_key} + else + request(base_url, api_key, model, messages, opts) + end + end + + defp blank?(nil), do: true + defp blank?(""), do: true + defp blank?(s) when is_binary(s), do: String.trim(s) == "" + + defp request(base_url, api_key, model, messages, opts) do + body = %{ + model: model, + messages: messages, + temperature: opts[:temperature] || 0.2, + max_tokens: opts[:max_tokens] || 400 + } + + req_opts = + maybe_inject_test_plug( + method: :post, + url: base_url <> "/chat/completions", + json: body, + headers: [{"authorization", "Bearer " <> api_key}], + receive_timeout: @receive_timeout + ) + + case Req.request(req_opts) do + {:ok, %{status: status, body: response_body}} when status in 200..299 -> + parse_response(response_body) + + {:ok, %{status: status, body: response_body}} -> + {:error, {:http, status, response_body}} + + {:error, reason} -> + {:error, reason} + end + rescue + exception -> + {:error, Exception.message(exception)} + end + + defp maybe_inject_test_plug(req_opts) do + if Application.get_env(:towerops, :env) == :test do + req_opts + |> Keyword.put(:plug, {Req.Test, __MODULE__}) + |> Keyword.put_new(:retry, false) + else + req_opts + end + end + + defp parse_response(%{"choices" => [%{"message" => %{"content" => content}} | _]} = body) do + {:ok, %{content: content, model: body["model"] || @default_model}} + end + + defp parse_response(body) do + {:error, {:unexpected_response, body}} + end +end diff --git a/lib/towerops/llm/insight_prompt.ex b/lib/towerops/llm/insight_prompt.ex new file mode 100644 index 00000000..902f0f6d --- /dev/null +++ b/lib/towerops/llm/insight_prompt.ex @@ -0,0 +1,66 @@ +defmodule Towerops.LLM.InsightPrompt do + @moduledoc """ + Builds chat messages for enriching a network-operations insight, and + parses the LLM's JSON response back into a `{summary, action}` pair. + + Tries hard to never crash on the LLM's output — if the response isn't + valid JSON we still return `{:ok, ...}` so the worker can store the raw + text as the summary instead of dropping the enrichment entirely. + """ + + alias Towerops.Preseem.Insight + + @system """ + You are a WISP/network-operations assistant. The user will give you a + structured insight detected by an automated rule. Your job: + + 1. Restate the finding in one or two plain sentences a network operator + can act on. Reference specific numbers from the structured data. + 2. Suggest one specific next action they should take, citing the values + in the data. + + Do NOT invent numbers. Only reference values present in the structured + data. Respond ONLY with JSON: {"summary": "...", "action": "..."}. + """ + + @spec build(Insight.t()) :: [map()] + def build(%Insight{} = insight) do + payload = + Jason.encode!( + %{ + type: insight.type, + urgency: insight.urgency, + title: insight.title, + description: insight.description, + metadata: insight.metadata || %{} + }, + pretty: true + ) + + [ + %{role: "system", content: @system}, + %{role: "user", content: "Insight (JSON):\n" <> payload <> "\n\nRespond with JSON."} + ] + end + + @spec parse(String.t()) :: {:ok, %{summary: String.t(), action: String.t() | nil}} + def parse(content) when is_binary(content) do + cleaned = content |> String.trim() |> strip_code_fences() + + case Jason.decode(cleaned) do + {:ok, %{"summary" => s} = m} when is_binary(s) -> + action = m["action"] + {:ok, %{summary: s, action: if(is_binary(action), do: action)}} + + _ -> + {:ok, %{summary: cleaned, action: nil}} + end + end + + defp strip_code_fences(text) do + text + |> String.replace(~r/^```(?:json)?\s*/m, "") + |> String.replace(~r/```\s*$/m, "") + |> String.trim() + end +end diff --git a/lib/towerops/llm/mock_client.ex b/lib/towerops/llm/mock_client.ex new file mode 100644 index 00000000..e2d0d0ed --- /dev/null +++ b/lib/towerops/llm/mock_client.ex @@ -0,0 +1,15 @@ +defmodule Towerops.LLM.MockClient do + @moduledoc """ + Test stand-in for `Towerops.LLM`. Returns a canned response that tests + configure via the process dictionary key `:llm_mock_response`. + + Default response returns `{:ok, %{content: "MOCK_RESPONSE", model: "mock"}}`. + """ + + @behaviour Towerops.LLM.Behaviour + + @impl true + def complete(_messages, _opts) do + Process.get(:llm_mock_response, {:ok, %{content: "MOCK_RESPONSE", model: "mock"}}) + end +end diff --git a/lib/towerops/preseem/insight.ex b/lib/towerops/preseem/insight.ex index 1e149f6c..bdf04d98 100644 --- a/lib/towerops/preseem/insight.ex +++ b/lib/towerops/preseem/insight.ex @@ -26,6 +26,10 @@ defmodule Towerops.Preseem.Insight do field :metadata, :map field :dismissed_at, :utc_datetime field :source, :string, default: "preseem" + field :llm_summary, :string + field :recommended_action, :string + field :llm_model, :string + field :llm_enriched_at, :utc_datetime belongs_to :organization, Towerops.Organizations.Organization belongs_to :preseem_access_point, Towerops.Preseem.AccessPoint @@ -52,7 +56,11 @@ defmodule Towerops.Preseem.Insight do :description, :metadata, :dismissed_at, - :source + :source, + :llm_summary, + :recommended_action, + :llm_model, + :llm_enriched_at ]) |> validate_required([:organization_id, :type, :urgency, :channel, :title]) |> validate_inclusion(:type, @valid_types) diff --git a/lib/towerops/preseem/insights.ex b/lib/towerops/preseem/insights.ex index 5d17d416..28055c01 100644 --- a/lib/towerops/preseem/insights.ex +++ b/lib/towerops/preseem/insights.ex @@ -88,6 +88,38 @@ defmodule Towerops.Preseem.Insights do end end + @doc """ + List active insights that have not been LLM-enriched yet. + + Returns insights ordered by `inserted_at` ascending so the oldest get + enriched first. The `:limit` option caps the batch size (default 50). + """ + def list_unenriched_insights(opts \\ []) do + limit = Keyword.get(opts, :limit, 50) + + Insight + |> where([i], i.status == "active" and is_nil(i.llm_enriched_at)) + |> order_by([i], asc: i.inserted_at) + |> limit(^limit) + |> Repo.all() + end + + @doc """ + Apply LLM-generated enrichment to an insight. Sets `llm_summary`, + `recommended_action`, `llm_model`, and `llm_enriched_at` (now). + """ + def apply_llm_enrichment(%Insight{} = insight, %{summary: summary} = parsed, model) + when is_binary(summary) and is_binary(model) do + insight + |> Insight.changeset(%{ + llm_summary: summary, + recommended_action: Map.get(parsed, :action), + llm_model: model, + llm_enriched_at: Towerops.Time.now() + }) + |> Repo.update() + end + @doc "Bulk dismiss insights by IDs." def dismiss_insights(insight_ids) when is_list(insight_ids) do now = Towerops.Time.now() diff --git a/lib/towerops/workers/insight_llm_enrichment_worker.ex b/lib/towerops/workers/insight_llm_enrichment_worker.ex new file mode 100644 index 00000000..e2af32dc --- /dev/null +++ b/lib/towerops/workers/insight_llm_enrichment_worker.ex @@ -0,0 +1,43 @@ +defmodule Towerops.Workers.InsightLlmEnrichmentWorker do + @moduledoc """ + Oban cron worker that fills in `llm_summary` and `recommended_action` for + active insights that have no LLM enrichment yet. + + Failures (rate limits, missing API key, parse errors) are logged and the + insight is left unenriched — the operator still sees the underlying finding + in the UI, just without a plain-language summary. + """ + + use Oban.Worker, queue: :maintenance, max_attempts: 3 + + alias Towerops.LLM + alias Towerops.LLM.InsightPrompt + alias Towerops.Preseem.Insights + + require Logger + + @batch_size 25 + + @impl Oban.Worker + def perform(%Oban.Job{}) do + [limit: @batch_size] + |> Insights.list_unenriched_insights() + |> Enum.each(&enrich_one/1) + + :ok + end + + defp enrich_one(insight) do + messages = InsightPrompt.build(insight) + + case LLM.complete(messages, max_tokens: 400, temperature: 0.2) do + {:ok, %{content: content, model: model}} -> + {:ok, parsed} = InsightPrompt.parse(content) + Insights.apply_llm_enrichment(insight, parsed, model) + + {:error, reason} -> + Logger.warning("LLM enrichment failed for insight #{insight.id}: #{inspect(reason)}") + :ok + end + end +end diff --git a/lib/towerops_web/live/insights_live/index.html.heex b/lib/towerops_web/live/insights_live/index.html.heex index 5491ab01..7f174176 100644 --- a/lib/towerops_web/live/insights_live/index.html.heex +++ b/lib/towerops_web/live/insights_live/index.html.heex @@ -274,6 +274,22 @@

<% end %> + <%= if insight.llm_summary do %> +
+

{t("Summary")}

+

{insight.llm_summary}

+ <%= if insight.recommended_action do %> +

+ {t("Recommended action:")} + {insight.recommended_action} +

+ <% end %> +

+ AI-generated{if insight.llm_model, do: " · " <> insight.llm_model, else: ""} +

+
+ <% end %> + <%!-- Affected devices list (from metadata) --%> <%= if insight.metadata["device_names"] && length(insight.metadata["device_names"]) > 0 do %>
diff --git a/lib/towerops_web/live/org/preseem_insights_live.html.heex b/lib/towerops_web/live/org/preseem_insights_live.html.heex index d15f86eb..818a95d3 100644 --- a/lib/towerops_web/live/org/preseem_insights_live.html.heex +++ b/lib/towerops_web/live/org/preseem_insights_live.html.heex @@ -253,6 +253,22 @@

<% end %> + <%= if insight.llm_summary do %> +
+

{t("Summary")}

+

{insight.llm_summary}

+ <%= if insight.recommended_action do %> +

+ {t("Recommended action:")} + {insight.recommended_action} +

+ <% end %> +

+ AI-generated{if insight.llm_model, do: " · " <> insight.llm_model, else: ""} +

+
+ <% end %> +
<%!-- Access point --%> <%= if insight.preseem_access_point do %> diff --git a/priv/repo/migrations/20260509212449_add_llm_fields_to_preseem_insights.exs b/priv/repo/migrations/20260509212449_add_llm_fields_to_preseem_insights.exs new file mode 100644 index 00000000..f1deb03c --- /dev/null +++ b/priv/repo/migrations/20260509212449_add_llm_fields_to_preseem_insights.exs @@ -0,0 +1,14 @@ +defmodule Towerops.Repo.Migrations.AddLlmFieldsToPreseemInsights do + use Ecto.Migration + + def change do + alter table(:preseem_insights) do + add :llm_summary, :text + add :recommended_action, :text + add :llm_model, :string + add :llm_enriched_at, :utc_datetime + end + + create index(:preseem_insights, [:llm_enriched_at]) + end +end diff --git a/priv/static/changelog.txt b/priv/static/changelog.txt index 71110789..2c38a482 100644 --- a/priv/static/changelog.txt +++ b/priv/static/changelog.txt @@ -1,3 +1,7 @@ +2026-05-09 — AI-generated insight summaries +* Active insights now include a plain-language summary and a specific recommended action +* AI summaries refresh automatically — no setup required for users + 2026-04-28 — Drag-and-drop reorder + restrictions + new-schedule polish * Drag levels and layers to reorder them on escalation policy and schedule pages (up/down arrow buttons still work) * Added on-call shift restrictions on schedule layers (time of day, time of week) diff --git a/test/towerops/llm/deepseek_test.exs b/test/towerops/llm/deepseek_test.exs new file mode 100644 index 00000000..829e428b --- /dev/null +++ b/test/towerops/llm/deepseek_test.exs @@ -0,0 +1,73 @@ +defmodule Towerops.LLM.DeepSeekTest do + use ExUnit.Case, async: true + + alias Towerops.LLM.DeepSeek + + setup do + # Each test sets its own Req.Test stub. Req.Test plug is wired in + # `DeepSeek` itself for the :test env. + :ok + end + + describe "complete/2" do + test "posts chat-completions and parses content" do + Req.Test.stub(DeepSeek, fn conn -> + assert conn.method == "POST" + assert conn.request_path == "/v1/chat/completions" + auth = conn |> Plug.Conn.get_req_header("authorization") |> List.first() + assert auth == "Bearer test-key" + + {:ok, body, conn} = Plug.Conn.read_body(conn) + assert {:ok, decoded} = Jason.decode(body) + assert decoded["model"] == "deepseek-v4-pro" + assert [%{"role" => "user", "content" => "hello"}] = decoded["messages"] + + Req.Test.json(conn, %{ + "model" => "deepseek-v4-pro", + "choices" => [%{"message" => %{"content" => "hi there"}}] + }) + end) + + assert {:ok, %{content: "hi there", model: "deepseek-v4-pro"}} = + DeepSeek.complete([%{role: "user", content: "hello"}], api_key: "test-key") + end + + test "returns {:error, :missing_api_key} when api_key is nil" do + assert {:error, :missing_api_key} = + DeepSeek.complete([%{role: "user", content: "x"}], api_key: nil) + end + + test "returns {:error, :missing_api_key} when api_key is empty" do + assert {:error, :missing_api_key} = + DeepSeek.complete([%{role: "user", content: "x"}], api_key: "") + end + + test "returns {:error, {:http, status, body}} on non-2xx" do + Req.Test.stub(DeepSeek, fn conn -> + Plug.Conn.send_resp(conn, 500, ~s({"error":"boom"})) + end) + + assert {:error, {:http, 500, _body}} = + DeepSeek.complete([%{role: "user", content: "x"}], api_key: "test-key") + end + + test "honors custom model and temperature" do + Req.Test.stub(DeepSeek, fn conn -> + {:ok, body, conn} = Plug.Conn.read_body(conn) + assert {:ok, %{"model" => "deepseek-reasoner", "temperature" => 0.7}} = Jason.decode(body) + + Req.Test.json(conn, %{ + "model" => "deepseek-reasoner", + "choices" => [%{"message" => %{"content" => "ok"}}] + }) + end) + + assert {:ok, %{model: "deepseek-reasoner"}} = + DeepSeek.complete([%{role: "user", content: "hi"}], + api_key: "test-key", + model: "deepseek-reasoner", + temperature: 0.7 + ) + end + end +end diff --git a/test/towerops/llm/insight_prompt_test.exs b/test/towerops/llm/insight_prompt_test.exs new file mode 100644 index 00000000..770b1124 --- /dev/null +++ b/test/towerops/llm/insight_prompt_test.exs @@ -0,0 +1,65 @@ +defmodule Towerops.LLM.InsightPromptTest do + use ExUnit.Case, async: true + + alias Towerops.LLM.InsightPrompt + alias Towerops.Preseem.Insight + + describe "build/1" do + test "returns system + user messages and includes structured metadata as JSON" do + insight = %Insight{ + type: "wireless_signal_weak", + urgency: "warning", + title: "Weak signal: AA:BB:CC (-78 dBm)", + description: "Wireless client AA:BB:CC has weak signal.", + metadata: %{"signal_strength" => -78, "threshold" => -75, "device_id" => "abc"} + } + + messages = InsightPrompt.build(insight) + + assert [%{role: "system", content: sys}, %{role: "user", content: user}] = messages + assert sys =~ "WISP" or sys =~ "network" + assert user =~ "wireless_signal_weak" + assert user =~ "-78" + assert user =~ "JSON" + end + + test "tolerates nil metadata and description" do + insight = %Insight{ + type: "agent_offline", + urgency: "critical", + title: "Agent offline", + description: nil, + metadata: nil + } + + assert [_system, %{role: "user", content: user}] = InsightPrompt.build(insight) + assert user =~ "agent_offline" + end + end + + describe "parse/1" do + test "extracts summary and action from a JSON object" do + json = ~s|{"summary": "AP X overloaded.", "action": "Move clients to AP Y."}| + assert {:ok, %{summary: "AP X overloaded.", action: "Move clients to AP Y."}} = InsightPrompt.parse(json) + end + + test "strips ```json fences before parsing" do + content = """ + ```json + {"summary": "Bufferbloat detected.", "action": "Enable AQM on AP-7."} + ``` + """ + + assert {:ok, %{summary: "Bufferbloat detected.", action: "Enable AQM on AP-7."}} = + InsightPrompt.parse(content) + end + + test "falls back to {summary: raw, action: nil} when content is not JSON" do + assert {:ok, %{summary: "raw text", action: nil}} = InsightPrompt.parse("raw text") + end + + test "falls back when JSON has no summary key" do + assert {:ok, %{summary: _, action: nil}} = InsightPrompt.parse(~s|{"foo": "bar"}|) + end + end +end diff --git a/test/towerops/preseem/insight_test.exs b/test/towerops/preseem/insight_test.exs index 4b668468..ee2cd23b 100644 --- a/test/towerops/preseem/insight_test.exs +++ b/test/towerops/preseem/insight_test.exs @@ -370,5 +370,28 @@ defmodule Towerops.Preseem.InsightTest do assert reloaded.source == "snmp" assert reloaded.site_id == site.id end + + test "accepts and round-trips llm enrichment fields", %{organization: org} do + attrs = %{ + organization_id: org.id, + type: "wireless_signal_weak", + urgency: "warning", + channel: "proactive", + title: "Weak signal", + source: "snmp", + llm_summary: "AP overloaded; consider rebalancing.", + recommended_action: "Move 3 clients to neighboring AP.", + llm_model: "deepseek-chat", + llm_enriched_at: ~U[2026-05-09 12:00:00Z] + } + + {:ok, insight} = %Insight{} |> Insight.changeset(attrs) |> Repo.insert() + reloaded = Repo.get!(Insight, insight.id) + + assert reloaded.llm_summary == "AP overloaded; consider rebalancing." + assert reloaded.recommended_action == "Move 3 clients to neighboring AP." + assert reloaded.llm_model == "deepseek-chat" + assert reloaded.llm_enriched_at == ~U[2026-05-09 12:00:00Z] + end end end diff --git a/test/towerops/preseem/insights_test.exs b/test/towerops/preseem/insights_test.exs index fd7a0a3d..3f9f6577 100644 --- a/test/towerops/preseem/insights_test.exs +++ b/test/towerops/preseem/insights_test.exs @@ -520,6 +520,103 @@ defmodule Towerops.Preseem.InsightsTest do end end + describe "list_unenriched_insights/1" do + test "returns active insights with no llm_enriched_at, oldest first", %{org: org, access_point: ap} do + {:ok, oldest} = + insert_insight(org, ap, %{ + type: "qoe_degradation", + urgency: "warning", + title: "Old", + inserted_at: ~U[2026-05-01 00:00:00Z] + }) + + {:ok, _enriched} = + insert_insight(org, ap, %{ + type: "qoe_degradation", + urgency: "warning", + title: "Already enriched", + llm_enriched_at: ~U[2026-05-08 12:00:00Z], + llm_summary: "summary", + llm_model: "deepseek-chat" + }) + + {:ok, _dismissed} = + insert_insight(org, ap, %{ + type: "qoe_degradation", + urgency: "warning", + title: "Dismissed", + status: "dismissed" + }) + + {:ok, newer} = + insert_insight(org, ap, %{ + type: "qoe_degradation", + urgency: "warning", + title: "Newer", + inserted_at: ~U[2026-05-02 00:00:00Z] + }) + + results = Insights.list_unenriched_insights(limit: 10) + + ids = Enum.map(results, & &1.id) + assert oldest.id in ids + assert newer.id in ids + assert length(results) == 2 + # Oldest first + assert hd(results).id == oldest.id + end + + test "respects limit option", %{org: org, access_point: ap} do + for i <- 1..5 do + insert_insight(org, ap, %{ + type: "qoe_degradation", + urgency: "warning", + title: "Insight #{i}" + }) + end + + assert length(Insights.list_unenriched_insights(limit: 3)) == 3 + end + end + + describe "apply_llm_enrichment/3" do + test "sets llm_summary, recommended_action, llm_model, and llm_enriched_at", %{org: org, access_point: ap} do + {:ok, insight} = + insert_insight(org, ap, %{ + type: "wireless_signal_weak", + urgency: "warning", + title: "Weak signal" + }) + + assert {:ok, updated} = + Insights.apply_llm_enrichment( + insight, + %{summary: "AP overloaded.", action: "Rebalance 3 clients."}, + "deepseek-v4-pro" + ) + + assert updated.llm_summary == "AP overloaded." + assert updated.recommended_action == "Rebalance 3 clients." + assert updated.llm_model == "deepseek-v4-pro" + assert updated.llm_enriched_at + end + + test "accepts a parsed map missing :action and stores nil", %{org: org, access_point: ap} do + {:ok, insight} = + insert_insight(org, ap, %{ + type: "wireless_signal_weak", + urgency: "warning", + title: "Weak signal" + }) + + assert {:ok, updated} = + Insights.apply_llm_enrichment(insight, %{summary: "Just summary"}, "deepseek-v4-pro") + + assert updated.llm_summary == "Just summary" + assert is_nil(updated.recommended_action) + end + end + # Helpers defp insert_insight(org, ap, attrs) do diff --git a/test/towerops/workers/insight_llm_enrichment_worker_test.exs b/test/towerops/workers/insight_llm_enrichment_worker_test.exs new file mode 100644 index 00000000..522a9424 --- /dev/null +++ b/test/towerops/workers/insight_llm_enrichment_worker_test.exs @@ -0,0 +1,121 @@ +defmodule Towerops.Workers.InsightLlmEnrichmentWorkerTest do + use Towerops.DataCase, async: false + + alias Towerops.Preseem.AccessPoint + alias Towerops.Preseem.Insight + alias Towerops.Repo + alias Towerops.Workers.InsightLlmEnrichmentWorker + + setup do + # Force the LLM context to use our mock client for these tests + prev = Application.get_env(:towerops, Towerops.LLM) + Application.put_env(:towerops, Towerops.LLM, impl: Towerops.LLM.MockClient) + on_exit(fn -> Application.put_env(:towerops, Towerops.LLM, prev || []) end) + + user = Towerops.AccountsFixtures.user_fixture(enable_totp: true) + {:ok, organization} = Towerops.Organizations.create_organization(%{name: "Enrich Org"}, user.id) + + {:ok, ap} = + %AccessPoint{} + |> AccessPoint.changeset(%{ + organization_id: organization.id, + preseem_id: "ap-enrich-#{System.unique_integer([:positive])}", + name: "Enrich AP" + }) + |> Repo.insert() + + %{organization: organization, access_point: ap} + end + + defp insert_active_insight(org, ap, attrs \\ %{}) do + full = + Map.merge( + %{ + organization_id: org.id, + preseem_access_point_id: ap.id, + type: "wireless_signal_weak", + urgency: "warning", + channel: "proactive", + status: "active", + title: "Weak signal", + metadata: %{"signal_strength" => -82, "threshold" => -75} + }, + attrs + ) + + {:ok, insight} = + %Insight{} + |> Insight.changeset(full) + |> Repo.insert() + + insight + end + + test "perform/1 enriches active, unenriched insights using the mock LLM", %{organization: org, access_point: ap} do + insight = insert_active_insight(org, ap) + + Process.put( + :llm_mock_response, + {:ok, + %{ + content: ~s|{"summary":"Weak link to CPE","action":"Re-aim CPE antenna"}|, + model: "deepseek-v4-pro" + }} + ) + + assert :ok = InsightLlmEnrichmentWorker.perform(%Oban.Job{}) + + updated = Repo.get!(Insight, insight.id) + assert updated.llm_summary == "Weak link to CPE" + assert updated.recommended_action == "Re-aim CPE antenna" + assert updated.llm_model == "deepseek-v4-pro" + assert updated.llm_enriched_at + end + + test "perform/1 leaves insight unenriched on LLM error and does not crash", %{organization: org, access_point: ap} do + insight = insert_active_insight(org, ap) + + Process.put(:llm_mock_response, {:error, :missing_api_key}) + + assert :ok = InsightLlmEnrichmentWorker.perform(%Oban.Job{}) + + refreshed = Repo.get!(Insight, insight.id) + assert is_nil(refreshed.llm_summary) + assert is_nil(refreshed.llm_enriched_at) + end + + test "perform/1 skips already-enriched insights", %{organization: org, access_point: ap} do + insight = + insert_active_insight(org, ap, %{ + llm_enriched_at: ~U[2026-05-01 00:00:00Z], + llm_summary: "previously generated", + llm_model: "deepseek-v4-pro" + }) + + Process.put( + :llm_mock_response, + {:ok, %{content: "should not be applied", model: "mock"}} + ) + + assert :ok = InsightLlmEnrichmentWorker.perform(%Oban.Job{}) + + refreshed = Repo.get!(Insight, insight.id) + assert refreshed.llm_summary == "previously generated" + end + + test "perform/1 falls back to raw text as summary when LLM returns non-JSON", %{organization: org, access_point: ap} do + insight = insert_active_insight(org, ap) + + Process.put( + :llm_mock_response, + {:ok, %{content: "AP is overloaded; suggest splitting the sector.", model: "deepseek-v4-pro"}} + ) + + assert :ok = InsightLlmEnrichmentWorker.perform(%Oban.Job{}) + + updated = Repo.get!(Insight, insight.id) + assert updated.llm_summary == "AP is overloaded; suggest splitting the sector." + assert is_nil(updated.recommended_action) + assert updated.llm_enriched_at + end +end diff --git a/test/towerops_web/live/insights_live_events_test.exs b/test/towerops_web/live/insights_live_events_test.exs index 3026d09a..b0dc95fd 100644 --- a/test/towerops_web/live/insights_live_events_test.exs +++ b/test/towerops_web/live/insights_live_events_test.exs @@ -99,4 +99,41 @@ defmodule ToweropsWeb.InsightsLive.IndexEventsTest do assert Repo.get!(Insight, i2.id).status == "dismissed" end end + + describe "LLM enrichment rendering" do + test "shows llm_summary, recommended_action, and model when present", %{ + conn: conn, + organization: organization + } do + {:ok, _enriched} = + Repo.insert( + Insight.changeset(%Insight{}, %{ + organization_id: organization.id, + type: "wireless_signal_weak", + urgency: "warning", + status: "active", + channel: "proactive", + title: "Enriched insight", + source: "snmp", + llm_summary: "AP has 3 clients pinned to MCS0; downstream is congested.", + recommended_action: "Re-aim CPE at -82 dBm or move it to AP-7.", + llm_model: "deepseek-v4-pro", + llm_enriched_at: ~U[2026-05-09 12:00:00Z] + }) + ) + + {:ok, _view, html} = live(conn, ~p"/insights") + + assert html =~ "AP has 3 clients pinned to MCS0; downstream is congested." + assert html =~ "Re-aim CPE at -82 dBm or move it to AP-7." + assert html =~ "deepseek-v4-pro" + end + + test "does not render the summary block when llm_summary is nil", %{conn: conn} do + {:ok, _view, html} = live(conn, ~p"/insights") + + refute html =~ "Recommended action:" + refute html =~ "AI-generated" + end + end end