feat(insights): auto-resolve stale insights after 7 days

If a rule keeps re-firing the underlying condition, the dedup query
suppresses duplicate inserts but `inserted_at` of the live row stays
old — so a real 6-month-old issue and a one-off blip both linger
forever. Auto-resolving any active insight older than 7 days clears
the dashboard. Still-real issues will be re-created by the next rule
pass (dedup only checks `status: "active"`), so the operator sees
fresh evidence rather than stale text.

- Towerops.Workers.InsightExpiryWorker (queue: :maintenance) flips
  `status` to "resolved" on rows where `inserted_at < now - 7d`.
- Window is configurable via `:max_age_days` for ops who want a
  different cadence per env.
- Oban cron: 04:00 UTC nightly, after the 02:00 rule pass and 03:00
  LLM enrichment.
This commit is contained in:
Graham McIntire 2026-05-10 13:06:35 -05:00
parent dbd4183f37
commit fd3c9691f2
3 changed files with 138 additions and 0 deletions

View file

@ -276,6 +276,10 @@ if config_env() == :prod do
# API spend isn't justified by minute-by-minute freshness when
# ops review the dashboard in the morning.
{"0 3 * * *", Towerops.Workers.InsightLlmEnrichmentWorker},
# Auto-resolve insights older than 7 days. Nightly at 04:00 UTC,
# after the rule pass + LLM enrichment have finished. Still-real
# issues will be re-created on the next rule run.
{"0 4 * * *", Towerops.Workers.InsightExpiryWorker},
# Gaiia reconciliation insights nightly at 4:30 AM
{"30 4 * * *", Towerops.Workers.GaiiaInsightWorker},
# Backhaul capacity utilization insights every 15 minutes

View file

@ -0,0 +1,52 @@
defmodule Towerops.Workers.InsightExpiryWorker do
@moduledoc """
Auto-resolves active `Preseem.Insight` rows that haven't been
reaffirmed in `:max_age_days` (default 7).
Why this is safe: every rule's dedup query filters by `status:
"active"`, so once an insight is flipped to `resolved` the next rule
pass will create a fresh one but only if the underlying condition
is *still* present. Stale insights that no longer reflect reality
simply stay resolved and disappear from the dashboard.
Runs nightly from the Oban cron. The default window can be overridden
per environment via:
config :towerops, Towerops.Workers.InsightExpiryWorker,
max_age_days: 14
"""
use Oban.Worker, queue: :maintenance, max_attempts: 3
import Ecto.Query
alias Towerops.Preseem.Insight
alias Towerops.Repo
require Logger
@default_max_age_days 7
@impl Oban.Worker
def perform(%Oban.Job{} = _job) do
cutoff = DateTime.add(DateTime.utc_now(), -max_age_days() * 86_400, :second)
{count, _} =
Repo.update_all(
from(i in Insight, where: i.status == "active" and i.inserted_at < ^cutoff, update: [set: [status: "resolved"]]),
[]
)
if count > 0 do
Logger.info("InsightExpiryWorker resolved #{count} stale insights older than #{max_age_days()}d")
end
:ok
end
defp max_age_days do
:towerops
|> Application.get_env(__MODULE__, [])
|> Keyword.get(:max_age_days, @default_max_age_days)
end
end

View file

@ -0,0 +1,82 @@
defmodule Towerops.Workers.InsightExpiryWorkerTest do
use Towerops.DataCase, async: false
import Towerops.AccountsFixtures
import Towerops.OrganizationsFixtures
alias Towerops.Preseem.Insight
alias Towerops.Workers.InsightExpiryWorker
setup do
user = user_fixture()
org = organization_fixture(user.id)
%{org: org}
end
defp insert_insight(org, attrs) do
full =
Map.merge(
%{
organization_id: org.id,
type: "qoe_degradation",
urgency: "warning",
channel: "proactive",
title: "T",
source: "preseem",
status: "active"
},
attrs
)
{:ok, insight} =
%Insight{}
|> Insight.changeset(full)
|> Repo.insert()
# Force `inserted_at` because Ecto sets it automatically.
if ts = attrs[:inserted_at] do
Repo.update_all(from(i in Insight, where: i.id == ^insight.id), set: [inserted_at: ts])
Repo.get!(Insight, insight.id)
else
insight
end
end
test "resolves active insights older than the configured window", %{org: org} do
eight_days_ago = DateTime.utc_now() |> DateTime.add(-8 * 86_400, :second) |> DateTime.truncate(:second)
fresh = DateTime.utc_now() |> DateTime.add(-1 * 86_400, :second) |> DateTime.truncate(:second)
stale = insert_insight(org, %{title: "stale", inserted_at: eight_days_ago})
fresh_insight = insert_insight(org, %{title: "fresh", inserted_at: fresh})
_already_resolved = insert_insight(org, %{title: "already", status: "resolved", inserted_at: eight_days_ago})
assert :ok = InsightExpiryWorker.perform(%Oban.Job{})
assert Repo.get!(Insight, stale.id).status == "resolved"
# Fresh ones stay active
assert Repo.get!(Insight, fresh_insight.id).status == "active"
end
test "honours the :max_age_days config override", %{org: org} do
previous = Application.get_env(:towerops, InsightExpiryWorker, [])
Application.put_env(:towerops, InsightExpiryWorker, max_age_days: 30)
on_exit(fn ->
Application.put_env(:towerops, InsightExpiryWorker, previous)
end)
twenty_days_ago = DateTime.utc_now() |> DateTime.add(-20 * 86_400, :second) |> DateTime.truncate(:second)
insight = insert_insight(org, %{title: "20-day-old", inserted_at: twenty_days_ago})
assert :ok = InsightExpiryWorker.perform(%Oban.Job{})
# Window is 30d, the row is 20d old — should still be active.
assert Repo.get!(Insight, insight.id).status == "active"
end
test "is a no-op when no stale insights exist", %{org: org} do
insert_insight(org, %{title: "fresh"})
assert :ok = InsightExpiryWorker.perform(%Oban.Job{})
end
end