From 4cfb97a7416f94cc52be2906017f74300c70fe4e Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 11 May 2026 09:49:30 -0500 Subject: [PATCH] fix(llm): handle reasoning models, bump token limits to 20k deepseek-v4-pro is a reasoning model that burns tokens on reasoning_content before producing content. When max_tokens was too low (400/1500), all tokens went to reasoning and content was empty. - parse_response now falls back to reasoning_content when content is blank - max_tokens raised to 20_000 across all call sites and default - improved parse logging shows raw/cleaned byte sizes --- lib/towerops/llm/deepseek.ex | 14 ++++++++++++-- lib/towerops/llm/network_insight_prompt.ex | 2 +- lib/towerops/workers/ai_network_insight_worker.ex | 2 +- .../workers/insight_llm_enrichment_worker.ex | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/towerops/llm/deepseek.ex b/lib/towerops/llm/deepseek.ex index 38a4e182..594c71dc 100644 --- a/lib/towerops/llm/deepseek.ex +++ b/lib/towerops/llm/deepseek.ex @@ -46,7 +46,7 @@ defmodule Towerops.LLM.DeepSeek do model: model, messages: messages, temperature: opts[:temperature] || 0.2, - max_tokens: opts[:max_tokens] || 400 + max_tokens: opts[:max_tokens] || 20_000 } req_opts = @@ -83,9 +83,19 @@ defmodule Towerops.LLM.DeepSeek do end end - defp parse_response(%{"choices" => [%{"message" => %{"content" => content}} | _]} = body) do + defp parse_response(%{"choices" => [%{"message" => message} | _]} = body) do usage = body["usage"] || %{} + content = + case message do + %{"content" => c} when is_binary(c) and c != "" -> c + # Reasoning models (deepseek-v4-pro) may exhaust max_tokens during + # reasoning and leave content empty; fall back to reasoning_content. + %{"reasoning_content" => rc} when is_binary(rc) -> rc + %{"content" => c} when is_binary(c) -> c + _ -> "" + end + {:ok, %{ content: content, diff --git a/lib/towerops/llm/network_insight_prompt.ex b/lib/towerops/llm/network_insight_prompt.ex index 3cde1c31..96bb6159 100644 --- a/lib/towerops/llm/network_insight_prompt.ex +++ b/lib/towerops/llm/network_insight_prompt.ex @@ -85,7 +85,7 @@ defmodule Towerops.LLM.NetworkInsightPrompt do _ -> Logger.warning( - "AI network insight: could not decode LLM response, content preview: #{String.slice(cleaned, 0, 200)}" + "AI network insight: could not decode LLM response, raw_bytes=#{byte_size(content)} cleaned_bytes=#{byte_size(cleaned)}, raw preview: #{String.slice(String.trim(content), 0, 300)}" ) [] diff --git a/lib/towerops/workers/ai_network_insight_worker.ex b/lib/towerops/workers/ai_network_insight_worker.ex index 8b8a7b87..fb5915a7 100644 --- a/lib/towerops/workers/ai_network_insight_worker.ex +++ b/lib/towerops/workers/ai_network_insight_worker.ex @@ -46,7 +46,7 @@ defmodule Towerops.Workers.AiNetworkInsightWorker do snapshot = NetworkSnapshot.build(organization_id) messages = NetworkInsightPrompt.build(snapshot) - case LLM.complete(messages, max_tokens: 1500, temperature: 0.3) do + case LLM.complete(messages, max_tokens: 20_000, temperature: 0.3) do {:ok, %{content: content, model: model} = response} -> {:ok, observations} = NetworkInsightPrompt.parse(content) diff --git a/lib/towerops/workers/insight_llm_enrichment_worker.ex b/lib/towerops/workers/insight_llm_enrichment_worker.ex index 155a88ed..eb6819e3 100644 --- a/lib/towerops/workers/insight_llm_enrichment_worker.ex +++ b/lib/towerops/workers/insight_llm_enrichment_worker.ex @@ -31,7 +31,7 @@ defmodule Towerops.Workers.InsightLlmEnrichmentWorker do defp enrich_one(insight) do messages = InsightPrompt.build(insight) - case LLM.complete(messages, max_tokens: 400, temperature: 0.2) do + case LLM.complete(messages, max_tokens: 20_000, temperature: 0.2) do {:ok, %{content: content, model: model} = response} -> {:ok, parsed} = InsightPrompt.parse(content)