fix(llm): handle reasoning models, bump token limits to 20k
deepseek-v4-pro is a reasoning model that burns tokens on reasoning_content before producing content. When max_tokens was too low (400/1500), all tokens went to reasoning and content was empty. - parse_response now falls back to reasoning_content when content is blank - max_tokens raised to 20_000 across all call sites and default - improved parse logging shows raw/cleaned byte sizes
This commit is contained in:
parent
d90d24198c
commit
4cfb97a741
4 changed files with 15 additions and 5 deletions
|
|
@ -46,7 +46,7 @@ defmodule Towerops.LLM.DeepSeek do
|
|||
model: model,
|
||||
messages: messages,
|
||||
temperature: opts[:temperature] || 0.2,
|
||||
max_tokens: opts[:max_tokens] || 400
|
||||
max_tokens: opts[:max_tokens] || 20_000
|
||||
}
|
||||
|
||||
req_opts =
|
||||
|
|
@ -83,9 +83,19 @@ defmodule Towerops.LLM.DeepSeek do
|
|||
end
|
||||
end
|
||||
|
||||
defp parse_response(%{"choices" => [%{"message" => %{"content" => content}} | _]} = body) do
|
||||
defp parse_response(%{"choices" => [%{"message" => message} | _]} = body) do
|
||||
usage = body["usage"] || %{}
|
||||
|
||||
content =
|
||||
case message do
|
||||
%{"content" => c} when is_binary(c) and c != "" -> c
|
||||
# Reasoning models (deepseek-v4-pro) may exhaust max_tokens during
|
||||
# reasoning and leave content empty; fall back to reasoning_content.
|
||||
%{"reasoning_content" => rc} when is_binary(rc) -> rc
|
||||
%{"content" => c} when is_binary(c) -> c
|
||||
_ -> ""
|
||||
end
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
content: content,
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ defmodule Towerops.LLM.NetworkInsightPrompt do
|
|||
|
||||
_ ->
|
||||
Logger.warning(
|
||||
"AI network insight: could not decode LLM response, content preview: #{String.slice(cleaned, 0, 200)}"
|
||||
"AI network insight: could not decode LLM response, raw_bytes=#{byte_size(content)} cleaned_bytes=#{byte_size(cleaned)}, raw preview: #{String.slice(String.trim(content), 0, 300)}"
|
||||
)
|
||||
|
||||
[]
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ defmodule Towerops.Workers.AiNetworkInsightWorker do
|
|||
snapshot = NetworkSnapshot.build(organization_id)
|
||||
messages = NetworkInsightPrompt.build(snapshot)
|
||||
|
||||
case LLM.complete(messages, max_tokens: 1500, temperature: 0.3) do
|
||||
case LLM.complete(messages, max_tokens: 20_000, temperature: 0.3) do
|
||||
{:ok, %{content: content, model: model} = response} ->
|
||||
{:ok, observations} = NetworkInsightPrompt.parse(content)
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ defmodule Towerops.Workers.InsightLlmEnrichmentWorker do
|
|||
defp enrich_one(insight) do
|
||||
messages = InsightPrompt.build(insight)
|
||||
|
||||
case LLM.complete(messages, max_tokens: 400, temperature: 0.2) do
|
||||
case LLM.complete(messages, max_tokens: 20_000, temperature: 0.2) do
|
||||
{:ok, %{content: content, model: model} = response} ->
|
||||
{:ok, parsed} = InsightPrompt.parse(content)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue