fix(llm): handle reasoning models, bump token limits to 20k

deepseek-v4-pro is a reasoning model that burns tokens on
reasoning_content before producing content. When max_tokens was too low
(400/1500), all tokens went to reasoning and content was empty.

- parse_response now falls back to reasoning_content when content is blank
- max_tokens raised to 20_000 across all call sites and default
- improved parse logging shows raw/cleaned byte sizes
This commit is contained in:
Graham McIntire 2026-05-11 09:49:30 -05:00
parent d90d24198c
commit 4cfb97a741
4 changed files with 15 additions and 5 deletions

View file

@ -46,7 +46,7 @@ defmodule Towerops.LLM.DeepSeek do
model: model,
messages: messages,
temperature: opts[:temperature] || 0.2,
max_tokens: opts[:max_tokens] || 400
max_tokens: opts[:max_tokens] || 20_000
}
req_opts =
@ -83,9 +83,19 @@ defmodule Towerops.LLM.DeepSeek do
end
end
defp parse_response(%{"choices" => [%{"message" => %{"content" => content}} | _]} = body) do
defp parse_response(%{"choices" => [%{"message" => message} | _]} = body) do
usage = body["usage"] || %{}
content =
case message do
%{"content" => c} when is_binary(c) and c != "" -> c
# Reasoning models (deepseek-v4-pro) may exhaust max_tokens during
# reasoning and leave content empty; fall back to reasoning_content.
%{"reasoning_content" => rc} when is_binary(rc) -> rc
%{"content" => c} when is_binary(c) -> c
_ -> ""
end
{:ok,
%{
content: content,

View file

@ -85,7 +85,7 @@ defmodule Towerops.LLM.NetworkInsightPrompt do
_ ->
Logger.warning(
"AI network insight: could not decode LLM response, content preview: #{String.slice(cleaned, 0, 200)}"
"AI network insight: could not decode LLM response, raw_bytes=#{byte_size(content)} cleaned_bytes=#{byte_size(cleaned)}, raw preview: #{String.slice(String.trim(content), 0, 300)}"
)
[]

View file

@ -46,7 +46,7 @@ defmodule Towerops.Workers.AiNetworkInsightWorker do
snapshot = NetworkSnapshot.build(organization_id)
messages = NetworkInsightPrompt.build(snapshot)
case LLM.complete(messages, max_tokens: 1500, temperature: 0.3) do
case LLM.complete(messages, max_tokens: 20_000, temperature: 0.3) do
{:ok, %{content: content, model: model} = response} ->
{:ok, observations} = NetworkInsightPrompt.parse(content)

View file

@ -31,7 +31,7 @@ defmodule Towerops.Workers.InsightLlmEnrichmentWorker do
defp enrich_one(insight) do
messages = InsightPrompt.build(insight)
case LLM.complete(messages, max_tokens: 400, temperature: 0.2) do
case LLM.complete(messages, max_tokens: 20_000, temperature: 0.2) do
{:ok, %{content: content, model: model} = response} ->
{:ok, parsed} = InsightPrompt.parse(content)