feat(insights): auto-resolve stale insights after 7 days
If a rule keeps re-firing the underlying condition, the dedup query suppresses duplicate inserts but `inserted_at` of the live row stays old — so a real 6-month-old issue and a one-off blip both linger forever. Auto-resolving any active insight older than 7 days clears the dashboard. Still-real issues will be re-created by the next rule pass (dedup only checks `status: "active"`), so the operator sees fresh evidence rather than stale text. - Towerops.Workers.InsightExpiryWorker (queue: :maintenance) flips `status` to "resolved" on rows where `inserted_at < now - 7d`. - Window is configurable via `:max_age_days` for ops who want a different cadence per env. - Oban cron: 04:00 UTC nightly, after the 02:00 rule pass and 03:00 LLM enrichment.
This commit is contained in:
parent
dbd4183f37
commit
fd3c9691f2
3 changed files with 138 additions and 0 deletions
|
|
@ -276,6 +276,10 @@ if config_env() == :prod do
|
|||
# API spend isn't justified by minute-by-minute freshness when
|
||||
# ops review the dashboard in the morning.
|
||||
{"0 3 * * *", Towerops.Workers.InsightLlmEnrichmentWorker},
|
||||
# Auto-resolve insights older than 7 days. Nightly at 04:00 UTC,
|
||||
# after the rule pass + LLM enrichment have finished. Still-real
|
||||
# issues will be re-created on the next rule run.
|
||||
{"0 4 * * *", Towerops.Workers.InsightExpiryWorker},
|
||||
# Gaiia reconciliation insights nightly at 4:30 AM
|
||||
{"30 4 * * *", Towerops.Workers.GaiiaInsightWorker},
|
||||
# Backhaul capacity utilization insights every 15 minutes
|
||||
|
|
|
|||
52
lib/towerops/workers/insight_expiry_worker.ex
Normal file
52
lib/towerops/workers/insight_expiry_worker.ex
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
defmodule Towerops.Workers.InsightExpiryWorker do
|
||||
@moduledoc """
|
||||
Auto-resolves active `Preseem.Insight` rows that haven't been
|
||||
reaffirmed in `:max_age_days` (default 7).
|
||||
|
||||
Why this is safe: every rule's dedup query filters by `status:
|
||||
"active"`, so once an insight is flipped to `resolved` the next rule
|
||||
pass will create a fresh one — but only if the underlying condition
|
||||
is *still* present. Stale insights that no longer reflect reality
|
||||
simply stay resolved and disappear from the dashboard.
|
||||
|
||||
Runs nightly from the Oban cron. The default window can be overridden
|
||||
per environment via:
|
||||
|
||||
config :towerops, Towerops.Workers.InsightExpiryWorker,
|
||||
max_age_days: 14
|
||||
"""
|
||||
|
||||
use Oban.Worker, queue: :maintenance, max_attempts: 3
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Towerops.Preseem.Insight
|
||||
alias Towerops.Repo
|
||||
|
||||
require Logger
|
||||
|
||||
@default_max_age_days 7
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{} = _job) do
|
||||
cutoff = DateTime.add(DateTime.utc_now(), -max_age_days() * 86_400, :second)
|
||||
|
||||
{count, _} =
|
||||
Repo.update_all(
|
||||
from(i in Insight, where: i.status == "active" and i.inserted_at < ^cutoff, update: [set: [status: "resolved"]]),
|
||||
[]
|
||||
)
|
||||
|
||||
if count > 0 do
|
||||
Logger.info("InsightExpiryWorker resolved #{count} stale insights older than #{max_age_days()}d")
|
||||
end
|
||||
|
||||
:ok
|
||||
end
|
||||
|
||||
defp max_age_days do
|
||||
:towerops
|
||||
|> Application.get_env(__MODULE__, [])
|
||||
|> Keyword.get(:max_age_days, @default_max_age_days)
|
||||
end
|
||||
end
|
||||
82
test/towerops/workers/insight_expiry_worker_test.exs
Normal file
82
test/towerops/workers/insight_expiry_worker_test.exs
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
defmodule Towerops.Workers.InsightExpiryWorkerTest do
|
||||
use Towerops.DataCase, async: false
|
||||
|
||||
import Towerops.AccountsFixtures
|
||||
import Towerops.OrganizationsFixtures
|
||||
|
||||
alias Towerops.Preseem.Insight
|
||||
alias Towerops.Workers.InsightExpiryWorker
|
||||
|
||||
setup do
|
||||
user = user_fixture()
|
||||
org = organization_fixture(user.id)
|
||||
%{org: org}
|
||||
end
|
||||
|
||||
defp insert_insight(org, attrs) do
|
||||
full =
|
||||
Map.merge(
|
||||
%{
|
||||
organization_id: org.id,
|
||||
type: "qoe_degradation",
|
||||
urgency: "warning",
|
||||
channel: "proactive",
|
||||
title: "T",
|
||||
source: "preseem",
|
||||
status: "active"
|
||||
},
|
||||
attrs
|
||||
)
|
||||
|
||||
{:ok, insight} =
|
||||
%Insight{}
|
||||
|> Insight.changeset(full)
|
||||
|> Repo.insert()
|
||||
|
||||
# Force `inserted_at` because Ecto sets it automatically.
|
||||
if ts = attrs[:inserted_at] do
|
||||
Repo.update_all(from(i in Insight, where: i.id == ^insight.id), set: [inserted_at: ts])
|
||||
Repo.get!(Insight, insight.id)
|
||||
else
|
||||
insight
|
||||
end
|
||||
end
|
||||
|
||||
test "resolves active insights older than the configured window", %{org: org} do
|
||||
eight_days_ago = DateTime.utc_now() |> DateTime.add(-8 * 86_400, :second) |> DateTime.truncate(:second)
|
||||
fresh = DateTime.utc_now() |> DateTime.add(-1 * 86_400, :second) |> DateTime.truncate(:second)
|
||||
|
||||
stale = insert_insight(org, %{title: "stale", inserted_at: eight_days_ago})
|
||||
fresh_insight = insert_insight(org, %{title: "fresh", inserted_at: fresh})
|
||||
_already_resolved = insert_insight(org, %{title: "already", status: "resolved", inserted_at: eight_days_ago})
|
||||
|
||||
assert :ok = InsightExpiryWorker.perform(%Oban.Job{})
|
||||
|
||||
assert Repo.get!(Insight, stale.id).status == "resolved"
|
||||
# Fresh ones stay active
|
||||
assert Repo.get!(Insight, fresh_insight.id).status == "active"
|
||||
end
|
||||
|
||||
test "honours the :max_age_days config override", %{org: org} do
|
||||
previous = Application.get_env(:towerops, InsightExpiryWorker, [])
|
||||
Application.put_env(:towerops, InsightExpiryWorker, max_age_days: 30)
|
||||
|
||||
on_exit(fn ->
|
||||
Application.put_env(:towerops, InsightExpiryWorker, previous)
|
||||
end)
|
||||
|
||||
twenty_days_ago = DateTime.utc_now() |> DateTime.add(-20 * 86_400, :second) |> DateTime.truncate(:second)
|
||||
|
||||
insight = insert_insight(org, %{title: "20-day-old", inserted_at: twenty_days_ago})
|
||||
|
||||
assert :ok = InsightExpiryWorker.perform(%Oban.Job{})
|
||||
|
||||
# Window is 30d, the row is 20d old — should still be active.
|
||||
assert Repo.get!(Insight, insight.id).status == "active"
|
||||
end
|
||||
|
||||
test "is a no-op when no stale insights exist", %{org: org} do
|
||||
insert_insight(org, %{title: "fresh"})
|
||||
assert :ok = InsightExpiryWorker.perform(%Oban.Job{})
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue