From dbd4183f37e72763e9847f2a1c35c2570dec7a81 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 10 May 2026 13:03:25 -0500 Subject: [PATCH] fix(llm): drop model name from footer, harden JSON parser MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UI: the AI-generated footer dropped " · deepseek-v4-pro" — model identity is irrelevant to operators reviewing the recommendation. Parser: stop letting raw JSON leak into the summary text when the LLM returns an almost-but-not-quite-valid response. - Strip ```json fences (already done) PLUS try a balanced-brace extraction so a "Here's the JSON: {…}" preamble doesn't break Jason.decode - When all decode paths fail, regex-extract `"summary"` and `"action"` fields from the text rather than dumping the whole blob with braces intact - Last-ditch sanitiser strips remaining brace/quote noise so the rendered summary never shows literal `"summary"` keys Maintenance: `Towerops.Maintenance.RepairLlmSummaries.run()` clears `llm_enriched_at` on already-stored rows whose summaries contain JSON artifacts so the next worker cycle re-enriches them with the new parser. Test: insights_live_events_test now asserts the AI-generated label is present *and* that the model name does not leak into the rendered HTML. --- lib/towerops/llm/insight_prompt.ex | 93 ++++++++++++++++++- .../maintenance/repair_llm_summaries.ex | 49 ++++++++++ .../live/insights_live/index.html.heex | 2 +- test/towerops/llm/insight_prompt_test.exs | 36 +++++++ .../live/insights_live_events_test.exs | 6 +- 5 files changed, 179 insertions(+), 7 deletions(-) create mode 100644 lib/towerops/maintenance/repair_llm_summaries.ex diff --git a/lib/towerops/llm/insight_prompt.ex b/lib/towerops/llm/insight_prompt.ex index 51a51b76..9265ffdd 100644 --- a/lib/towerops/llm/insight_prompt.ex +++ b/lib/towerops/llm/insight_prompt.ex @@ -162,16 +162,101 @@ defmodule Towerops.LLM.InsightPrompt do def parse(content) when is_binary(content) do cleaned = content |> String.trim() |> strip_code_fences() - case Jason.decode(cleaned) do + cleaned + |> try_decode() + |> case do {:ok, %{"summary" => s} = m} when is_binary(s) -> - action = m["action"] - {:ok, %{summary: s, action: if(is_binary(action), do: action)}} + {:ok, %{summary: s, action: optional_string(m["action"])}} _ -> - {:ok, %{summary: cleaned, action: nil}} + # Fallback: extract `summary` and `action` via regex from the raw + # text rather than dumping the whole thing — that way a stray + # leading sentence or trailing prose doesn't drag JSON braces + # into the user-visible summary. + {:ok, + %{ + summary: extract_field(cleaned, "summary") || strip_json_remnants(cleaned), + action: extract_field(cleaned, "action") + }} end end + # Try Jason.decode on the trimmed text, then on the first balanced + # `{...}` substring (handles "Here's the JSON: {…}" style preambles). + defp try_decode(text) do + case Jason.decode(text) do + {:ok, _} = ok -> + ok + + _ -> + case extract_braced(text) do + nil -> :error + json -> Jason.decode(json) + end + end + end + + # Extract the first balanced top-level `{...}` in the text, depth-aware + # so a string containing `}` doesn't truncate it. Returns nil if none. + defp extract_braced(text) do + case String.split(text, "{", parts: 2) do + [_, rest] -> walk_braced("{" <> rest, 0, []) + _ -> nil + end + end + + defp walk_braced("", _depth, _acc), do: nil + defp walk_braced(<<"\\", c, rest::binary>>, depth, acc), do: walk_braced(rest, depth, [c, "\\" | acc]) + defp walk_braced(<<"{", rest::binary>>, depth, acc), do: walk_braced(rest, depth + 1, ["{" | acc]) + + defp walk_braced(<<"}", rest::binary>>, depth, acc) do + new_acc = ["}" | acc] + + if depth == 1 do + new_acc |> Enum.reverse() |> IO.iodata_to_binary() + else + walk_braced(rest, depth - 1, new_acc) + end + end + + defp walk_braced(<>, depth, acc), do: walk_braced(rest, depth, [c | acc]) + + # Pull just the value out of a `"summary": "..."` (or `"action": "..."`) + # pair. Stops at the first unescaped closing quote. + defp extract_field(text, key) do + pattern = ~r/"#{key}"\s*:\s*"((?:[^"\\]|\\.)*)"/ + + case Regex.run(pattern, text) do + [_, value] -> value |> unescape_json_string() |> String.trim() |> nil_if_blank() + _ -> nil + end + end + + defp unescape_json_string(s) do + s + |> String.replace("\\\"", "\"") + |> String.replace("\\n", "\n") + |> String.replace("\\\\", "\\") + end + + defp nil_if_blank(""), do: nil + defp nil_if_blank(s), do: s + + defp optional_string(s) when is_binary(s) and s != "", do: s + defp optional_string(_), do: nil + + # Last-ditch: strip braces, key labels, and surrounding quotes so a + # totally malformed JSON-ish blob still renders as readable prose. + defp strip_json_remnants(text) do + text + |> String.replace(~r/^[\s\{\[]+/, "") + |> String.replace(~r/[\s\}\]]+$/, "") + |> String.replace(~r/"(summary|action)"\s*:\s*"?/i, "") + |> String.replace(~r/"\s*,\s*"action"\s*:\s*"/, " — ") + |> String.replace(~r/"\s*$/, "") + |> String.trim() + end + defp strip_code_fences(text) do text |> String.replace(~r/^```(?:json)?\s*/m, "") diff --git a/lib/towerops/maintenance/repair_llm_summaries.ex b/lib/towerops/maintenance/repair_llm_summaries.ex new file mode 100644 index 00000000..bc362e8c --- /dev/null +++ b/lib/towerops/maintenance/repair_llm_summaries.ex @@ -0,0 +1,49 @@ +defmodule Towerops.Maintenance.RepairLlmSummaries do + @moduledoc """ + Resets `llm_enriched_at` (and clears the cached summary fields) on any + insight whose stored summary still contains JSON syntax that leaked + through an earlier enrichment run. The next + `InsightLlmEnrichmentWorker` cycle will re-enrich those rows with the + hardened parser. + + Run from the prod release: + + /app/bin/towerops eval 'Towerops.Maintenance.RepairLlmSummaries.run()' + """ + + import Ecto.Query + + alias Towerops.Preseem.Insight + alias Towerops.Repo + + require Logger + + @spec run() :: :ok + def run do + {count, _} = + Repo.update_all( + from(i in Insight, + where: + not is_nil(i.llm_enriched_at) and + (like(i.llm_summary, "%\"summary\"%") or like(i.llm_summary, "%```%") or like(i.llm_summary, "{%")), + update: [ + set: [ + llm_summary: nil, + recommended_action: nil, + llm_enriched_at: nil, + llm_prompt_tokens: nil, + llm_completion_tokens: nil + ] + ] + ), + [] + ) + + msg = + "LLM summary repair: cleared #{count} rows so the next enrichment cycle reprocesses them." + + IO.puts(msg) + Logger.info(msg) + :ok + end +end diff --git a/lib/towerops_web/live/insights_live/index.html.heex b/lib/towerops_web/live/insights_live/index.html.heex index fa336345..d6b73bbc 100644 --- a/lib/towerops_web/live/insights_live/index.html.heex +++ b/lib/towerops_web/live/insights_live/index.html.heex @@ -292,7 +292,7 @@

<% end %>

- AI-generated{if insight.llm_model, do: " · " <> insight.llm_model, else: ""} + AI-generated

<% end %> diff --git a/test/towerops/llm/insight_prompt_test.exs b/test/towerops/llm/insight_prompt_test.exs index 7ab39312..eb0214ca 100644 --- a/test/towerops/llm/insight_prompt_test.exs +++ b/test/towerops/llm/insight_prompt_test.exs @@ -92,5 +92,41 @@ defmodule Towerops.LLM.InsightPromptTest do test "falls back when JSON has no summary key" do assert {:ok, %{summary: _, action: nil}} = InsightPrompt.parse(~s|{"foo": "bar"}|) end + + test "extracts JSON embedded in surrounding prose (preamble)" do + content = ~s|Here is the response: {"summary": "Sector congested.", "action": "Split."}| + assert {:ok, %{summary: "Sector congested.", action: "Split."}} = InsightPrompt.parse(content) + end + + test "extracts JSON when wrapped in extra commentary on both sides" do + content = """ + Sure, based on the data: + {"summary": "Backhaul saturated at 95%.", "action": "Procure capacity now."} + Hope this helps. + """ + + assert {:ok, %{summary: "Backhaul saturated at 95%.", action: "Procure capacity now."}} = + InsightPrompt.parse(content) + end + + test "regex-extracts summary/action when JSON parse fails on a trailing comma" do + # LLM returned an almost-JSON blob with a trailing comma (Jason + # rejects, but the field-level regex still picks up the values). + content = ~s|{"summary": "Sector congested at 95%.", "action": "Split sector.",}| + + {:ok, parsed} = InsightPrompt.parse(content) + assert parsed.summary == "Sector congested at 95%." + assert parsed.action == "Split sector." + end + + test "stripped JSON remnants never leak '\"summary\"' into the rendered text" do + # The kind of garbage that was leaking into the UI before — JSON + # that's almost-but-not-quite-valid. The fallback should sanitize. + garbage = ~s|{"summary": "AP X is hot| + + {:ok, parsed} = InsightPrompt.parse(garbage) + refute parsed.summary =~ ~s|"summary"| + refute parsed.summary =~ ~s|{| + end end end diff --git a/test/towerops_web/live/insights_live_events_test.exs b/test/towerops_web/live/insights_live_events_test.exs index fd4af27b..c83213fb 100644 --- a/test/towerops_web/live/insights_live_events_test.exs +++ b/test/towerops_web/live/insights_live_events_test.exs @@ -101,7 +101,7 @@ defmodule ToweropsWeb.InsightsLive.IndexEventsTest do end describe "LLM enrichment rendering" do - test "shows llm_summary, recommended_action, and model when present", %{ + test "shows llm_summary, recommended_action, and AI-generated label when present", %{ conn: conn, organization: organization } do @@ -126,7 +126,9 @@ defmodule ToweropsWeb.InsightsLive.IndexEventsTest do assert html =~ "AP has 3 clients pinned to MCS0; downstream is congested." assert html =~ "Re-aim CPE at -82 dBm or move it to AP-7." - assert html =~ "deepseek-v4-pro" + assert html =~ "AI-generated" + # Model identity is intentionally not exposed in the UI. + refute html =~ "deepseek-v4-pro" end test "does not render the summary block when llm_summary is nil", %{conn: conn} do