From e18d8776956ccae6c907d1acbd5b70f499031a97 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 10 May 2026 11:50:54 -0500 Subject: [PATCH] feat(insights): persist LLM prompt + completion token counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track DeepSeek token spend per enriched insight so cost can be rolled up by SQL — `SELECT date_trunc('day', llm_enriched_at), organization_id, SUM(llm_prompt_tokens), SUM(llm_completion_tokens) FROM preseem_insights GROUP BY 1, 2`. - Migration adds llm_prompt_tokens + llm_completion_tokens to preseem_insights - DeepSeek client surfaces the API's `usage` block (prompt/completion/ total tokens) on the success response - apply_llm_enrichment/4 takes a usage map and persists the two int columns - Worker plumbs the usage from the response through to the context --- lib/towerops/llm/deepseek.ex | 11 ++++++- lib/towerops/preseem/insight.ex | 6 +++- lib/towerops/preseem/insights.ex | 9 +++-- .../workers/insight_llm_enrichment_worker.ex | 10 ++++-- ...dd_llm_token_usage_to_preseem_insights.exs | 10 ++++++ test/towerops/llm/deepseek_test.exs | 33 +++++++++++++++++++ .../insight_llm_enrichment_worker_test.exs | 23 +++++++++++++ 7 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 priv/repo/migrations/20260510165000_add_llm_token_usage_to_preseem_insights.exs diff --git a/lib/towerops/llm/deepseek.ex b/lib/towerops/llm/deepseek.ex index c105c4c0..af63b1df 100644 --- a/lib/towerops/llm/deepseek.ex +++ b/lib/towerops/llm/deepseek.ex @@ -77,7 +77,16 @@ defmodule Towerops.LLM.DeepSeek do end defp parse_response(%{"choices" => [%{"message" => %{"content" => content}} | _]} = body) do - {:ok, %{content: content, model: body["model"] || @default_model}} + usage = body["usage"] || %{} + + {:ok, + %{ + content: content, + model: body["model"] || @default_model, + prompt_tokens: usage["prompt_tokens"], + completion_tokens: usage["completion_tokens"], + total_tokens: usage["total_tokens"] + }} end defp parse_response(body) do diff --git a/lib/towerops/preseem/insight.ex b/lib/towerops/preseem/insight.ex index 8cbd8eeb..613ec8b3 100644 --- a/lib/towerops/preseem/insight.ex +++ b/lib/towerops/preseem/insight.ex @@ -30,6 +30,8 @@ defmodule Towerops.Preseem.Insight do field :recommended_action, :string field :llm_model, :string field :llm_enriched_at, :utc_datetime + field :llm_prompt_tokens, :integer + field :llm_completion_tokens, :integer belongs_to :organization, Towerops.Organizations.Organization belongs_to :preseem_access_point, Towerops.Preseem.AccessPoint @@ -60,7 +62,9 @@ defmodule Towerops.Preseem.Insight do :llm_summary, :recommended_action, :llm_model, - :llm_enriched_at + :llm_enriched_at, + :llm_prompt_tokens, + :llm_completion_tokens ]) |> validate_required([:organization_id, :type, :urgency, :channel, :title]) |> validate_inclusion(:type, @valid_types) diff --git a/lib/towerops/preseem/insights.ex b/lib/towerops/preseem/insights.ex index 861b739d..782f3d78 100644 --- a/lib/towerops/preseem/insights.ex +++ b/lib/towerops/preseem/insights.ex @@ -106,16 +106,19 @@ defmodule Towerops.Preseem.Insights do @doc """ Apply LLM-generated enrichment to an insight. Sets `llm_summary`, - `recommended_action`, `llm_model`, and `llm_enriched_at` (now). + `recommended_action`, `llm_model`, `llm_enriched_at` (now), and the + prompt/completion token counts when the LLM client surfaces them. """ - def apply_llm_enrichment(%Insight{} = insight, %{summary: summary} = parsed, model) + def apply_llm_enrichment(%Insight{} = insight, %{summary: summary} = parsed, model, usage \\ %{}) when is_binary(summary) and is_binary(model) do insight |> Insight.changeset(%{ llm_summary: summary, recommended_action: Map.get(parsed, :action), llm_model: model, - llm_enriched_at: Towerops.Time.now() + llm_enriched_at: Towerops.Time.now(), + llm_prompt_tokens: Map.get(usage, :prompt_tokens), + llm_completion_tokens: Map.get(usage, :completion_tokens) }) |> Repo.update() end diff --git a/lib/towerops/workers/insight_llm_enrichment_worker.ex b/lib/towerops/workers/insight_llm_enrichment_worker.ex index e2af32dc..ba414822 100644 --- a/lib/towerops/workers/insight_llm_enrichment_worker.ex +++ b/lib/towerops/workers/insight_llm_enrichment_worker.ex @@ -31,9 +31,15 @@ defmodule Towerops.Workers.InsightLlmEnrichmentWorker do messages = InsightPrompt.build(insight) case LLM.complete(messages, max_tokens: 400, temperature: 0.2) do - {:ok, %{content: content, model: model}} -> + {:ok, %{content: content, model: model} = response} -> {:ok, parsed} = InsightPrompt.parse(content) - Insights.apply_llm_enrichment(insight, parsed, model) + + usage = %{ + prompt_tokens: Map.get(response, :prompt_tokens), + completion_tokens: Map.get(response, :completion_tokens) + } + + Insights.apply_llm_enrichment(insight, parsed, model, usage) {:error, reason} -> Logger.warning("LLM enrichment failed for insight #{insight.id}: #{inspect(reason)}") diff --git a/priv/repo/migrations/20260510165000_add_llm_token_usage_to_preseem_insights.exs b/priv/repo/migrations/20260510165000_add_llm_token_usage_to_preseem_insights.exs new file mode 100644 index 00000000..a1d6a9e0 --- /dev/null +++ b/priv/repo/migrations/20260510165000_add_llm_token_usage_to_preseem_insights.exs @@ -0,0 +1,10 @@ +defmodule Towerops.Repo.Migrations.AddLlmTokenUsageToPreseemInsights do + use Ecto.Migration + + def change do + alter table(:preseem_insights) do + add :llm_prompt_tokens, :integer + add :llm_completion_tokens, :integer + end + end +end diff --git a/test/towerops/llm/deepseek_test.exs b/test/towerops/llm/deepseek_test.exs index 829e428b..3ead738e 100644 --- a/test/towerops/llm/deepseek_test.exs +++ b/test/towerops/llm/deepseek_test.exs @@ -32,6 +32,39 @@ defmodule Towerops.LLM.DeepSeekTest do DeepSeek.complete([%{role: "user", content: "hello"}], api_key: "test-key") end + test "surfaces prompt/completion/total tokens from the API usage block" do + Req.Test.stub(DeepSeek, fn conn -> + Req.Test.json(conn, %{ + "model" => "deepseek-v4-pro", + "choices" => [%{"message" => %{"content" => "hello"}}], + "usage" => %{"prompt_tokens" => 42, "completion_tokens" => 17, "total_tokens" => 59} + }) + end) + + assert {:ok, response} = + DeepSeek.complete([%{role: "user", content: "x"}], api_key: "test-key") + + assert response.prompt_tokens == 42 + assert response.completion_tokens == 17 + assert response.total_tokens == 59 + end + + test "returns nil token counts when the API omits the usage block" do + Req.Test.stub(DeepSeek, fn conn -> + Req.Test.json(conn, %{ + "model" => "deepseek-v4-pro", + "choices" => [%{"message" => %{"content" => "hello"}}] + }) + end) + + assert {:ok, response} = + DeepSeek.complete([%{role: "user", content: "x"}], api_key: "test-key") + + assert is_nil(response.prompt_tokens) + assert is_nil(response.completion_tokens) + assert is_nil(response.total_tokens) + end + test "returns {:error, :missing_api_key} when api_key is nil" do assert {:error, :missing_api_key} = DeepSeek.complete([%{role: "user", content: "x"}], api_key: nil) diff --git a/test/towerops/workers/insight_llm_enrichment_worker_test.exs b/test/towerops/workers/insight_llm_enrichment_worker_test.exs index 522a9424..fcd525d7 100644 --- a/test/towerops/workers/insight_llm_enrichment_worker_test.exs +++ b/test/towerops/workers/insight_llm_enrichment_worker_test.exs @@ -72,6 +72,29 @@ defmodule Towerops.Workers.InsightLlmEnrichmentWorkerTest do assert updated.llm_enriched_at end + test "perform/1 persists prompt + completion token counts when the LLM returns usage", + %{organization: org, access_point: ap} do + insight = insert_active_insight(org, ap) + + Process.put( + :llm_mock_response, + {:ok, + %{ + content: ~s|{"summary":"x","action":"y"}|, + model: "deepseek-v4-pro", + prompt_tokens: 123, + completion_tokens: 45, + total_tokens: 168 + }} + ) + + assert :ok = InsightLlmEnrichmentWorker.perform(%Oban.Job{}) + + updated = Repo.get!(Insight, insight.id) + assert updated.llm_prompt_tokens == 123 + assert updated.llm_completion_tokens == 45 + end + test "perform/1 leaves insight unenriched on LLM error and does not crash", %{organization: org, access_point: ap} do insight = insert_active_insight(org, ap)