diff --git a/config/runtime.exs b/config/runtime.exs index 9f03bc1b..f69a8f91 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -276,6 +276,10 @@ if config_env() == :prod do # API spend isn't justified by minute-by-minute freshness when # ops review the dashboard in the morning. {"0 3 * * *", Towerops.Workers.InsightLlmEnrichmentWorker}, + # Auto-resolve insights older than 7 days. Nightly at 04:00 UTC, + # after the rule pass + LLM enrichment have finished. Still-real + # issues will be re-created on the next rule run. + {"0 4 * * *", Towerops.Workers.InsightExpiryWorker}, # Gaiia reconciliation insights nightly at 4:30 AM {"30 4 * * *", Towerops.Workers.GaiiaInsightWorker}, # Backhaul capacity utilization insights every 15 minutes diff --git a/lib/towerops/workers/insight_expiry_worker.ex b/lib/towerops/workers/insight_expiry_worker.ex new file mode 100644 index 00000000..0bd0508e --- /dev/null +++ b/lib/towerops/workers/insight_expiry_worker.ex @@ -0,0 +1,52 @@ +defmodule Towerops.Workers.InsightExpiryWorker do + @moduledoc """ + Auto-resolves active `Preseem.Insight` rows that haven't been + reaffirmed in `:max_age_days` (default 7). + + Why this is safe: every rule's dedup query filters by `status: + "active"`, so once an insight is flipped to `resolved` the next rule + pass will create a fresh one — but only if the underlying condition + is *still* present. Stale insights that no longer reflect reality + simply stay resolved and disappear from the dashboard. + + Runs nightly from the Oban cron. The default window can be overridden + per environment via: + + config :towerops, Towerops.Workers.InsightExpiryWorker, + max_age_days: 14 + """ + + use Oban.Worker, queue: :maintenance, max_attempts: 3 + + import Ecto.Query + + alias Towerops.Preseem.Insight + alias Towerops.Repo + + require Logger + + @default_max_age_days 7 + + @impl Oban.Worker + def perform(%Oban.Job{} = _job) do + cutoff = DateTime.add(DateTime.utc_now(), -max_age_days() * 86_400, :second) + + {count, _} = + Repo.update_all( + from(i in Insight, where: i.status == "active" and i.inserted_at < ^cutoff, update: [set: [status: "resolved"]]), + [] + ) + + if count > 0 do + Logger.info("InsightExpiryWorker resolved #{count} stale insights older than #{max_age_days()}d") + end + + :ok + end + + defp max_age_days do + :towerops + |> Application.get_env(__MODULE__, []) + |> Keyword.get(:max_age_days, @default_max_age_days) + end +end diff --git a/test/towerops/workers/insight_expiry_worker_test.exs b/test/towerops/workers/insight_expiry_worker_test.exs new file mode 100644 index 00000000..1a99a0ec --- /dev/null +++ b/test/towerops/workers/insight_expiry_worker_test.exs @@ -0,0 +1,82 @@ +defmodule Towerops.Workers.InsightExpiryWorkerTest do + use Towerops.DataCase, async: false + + import Towerops.AccountsFixtures + import Towerops.OrganizationsFixtures + + alias Towerops.Preseem.Insight + alias Towerops.Workers.InsightExpiryWorker + + setup do + user = user_fixture() + org = organization_fixture(user.id) + %{org: org} + end + + defp insert_insight(org, attrs) do + full = + Map.merge( + %{ + organization_id: org.id, + type: "qoe_degradation", + urgency: "warning", + channel: "proactive", + title: "T", + source: "preseem", + status: "active" + }, + attrs + ) + + {:ok, insight} = + %Insight{} + |> Insight.changeset(full) + |> Repo.insert() + + # Force `inserted_at` because Ecto sets it automatically. + if ts = attrs[:inserted_at] do + Repo.update_all(from(i in Insight, where: i.id == ^insight.id), set: [inserted_at: ts]) + Repo.get!(Insight, insight.id) + else + insight + end + end + + test "resolves active insights older than the configured window", %{org: org} do + eight_days_ago = DateTime.utc_now() |> DateTime.add(-8 * 86_400, :second) |> DateTime.truncate(:second) + fresh = DateTime.utc_now() |> DateTime.add(-1 * 86_400, :second) |> DateTime.truncate(:second) + + stale = insert_insight(org, %{title: "stale", inserted_at: eight_days_ago}) + fresh_insight = insert_insight(org, %{title: "fresh", inserted_at: fresh}) + _already_resolved = insert_insight(org, %{title: "already", status: "resolved", inserted_at: eight_days_ago}) + + assert :ok = InsightExpiryWorker.perform(%Oban.Job{}) + + assert Repo.get!(Insight, stale.id).status == "resolved" + # Fresh ones stay active + assert Repo.get!(Insight, fresh_insight.id).status == "active" + end + + test "honours the :max_age_days config override", %{org: org} do + previous = Application.get_env(:towerops, InsightExpiryWorker, []) + Application.put_env(:towerops, InsightExpiryWorker, max_age_days: 30) + + on_exit(fn -> + Application.put_env(:towerops, InsightExpiryWorker, previous) + end) + + twenty_days_ago = DateTime.utc_now() |> DateTime.add(-20 * 86_400, :second) |> DateTime.truncate(:second) + + insight = insert_insight(org, %{title: "20-day-old", inserted_at: twenty_days_ago}) + + assert :ok = InsightExpiryWorker.perform(%Oban.Job{}) + + # Window is 30d, the row is 20d old — should still be active. + assert Repo.get!(Insight, insight.id).status == "active" + end + + test "is a no-op when no stale insights exist", %{org: org} do + insert_insight(org, %{title: "fresh"}) + assert :ok = InsightExpiryWorker.perform(%Oban.Job{}) + end +end