93 lines
2.7 KiB
Elixir
93 lines
2.7 KiB
Elixir
defmodule Towerops.Workers.SystemInsightWorker do
|
|
@moduledoc """
|
|
Oban cron worker that generates system-level insights.
|
|
|
|
Runs every 5 minutes to detect:
|
|
- `agent_offline`: Agents not seen in over 10 minutes
|
|
- Auto-resolves agent_offline insights when agents come back online
|
|
"""
|
|
use Oban.Worker, queue: :maintenance
|
|
|
|
import Ecto.Query
|
|
|
|
alias Towerops.Agents.AgentToken
|
|
alias Towerops.Organizations.Organization
|
|
alias Towerops.Preseem.Insight
|
|
alias Towerops.Preseem.Insights
|
|
alias Towerops.Repo
|
|
|
|
@stale_threshold_minutes 10
|
|
|
|
@impl Oban.Worker
|
|
def perform(%Oban.Job{}) do
|
|
# Use streaming to avoid loading all org IDs into memory
|
|
Repo.transaction(fn ->
|
|
Organization
|
|
|> select([o], o.id)
|
|
|> Repo.stream()
|
|
|> Enum.each(fn org_id ->
|
|
offline_agents = list_offline_agents(org_id)
|
|
offline_ids = MapSet.new(offline_agents, & &1.id)
|
|
|
|
Enum.each(offline_agents, &generate_agent_offline_insight/1)
|
|
auto_resolve_recovered_agents(org_id, offline_ids)
|
|
end)
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
defp list_offline_agents(organization_id) do
|
|
cutoff = DateTime.add(DateTime.utc_now(), -@stale_threshold_minutes, :minute)
|
|
|
|
AgentToken
|
|
|> where([a], a.organization_id == ^organization_id)
|
|
|> where([a], a.enabled == true)
|
|
|> where([a], not is_nil(a.last_seen_at))
|
|
|> where([a], a.last_seen_at < ^cutoff)
|
|
|> Repo.all()
|
|
end
|
|
|
|
defp generate_agent_offline_insight(agent) do
|
|
minutes_offline = DateTime.diff(DateTime.utc_now(), agent.last_seen_at, :minute)
|
|
|
|
Insights.insert_insight_if_new(%{
|
|
organization_id: agent.organization_id,
|
|
agent_token_id: agent.id,
|
|
type: "agent_offline",
|
|
urgency: "critical",
|
|
channel: "proactive",
|
|
source: "system",
|
|
title: "Agent '#{agent.name}' is offline",
|
|
description:
|
|
"Agent #{agent.name} has not been seen for #{minutes_offline} minutes. " <>
|
|
"Check agent connectivity and host status.",
|
|
metadata: %{
|
|
"agent_name" => agent.name,
|
|
"last_seen_at" => DateTime.to_iso8601(agent.last_seen_at),
|
|
"minutes_offline" => minutes_offline
|
|
}
|
|
})
|
|
end
|
|
|
|
defp auto_resolve_recovered_agents(organization_id, currently_offline_ids) do
|
|
active_agent_insights =
|
|
Insight
|
|
|> where(
|
|
[i],
|
|
i.organization_id == ^organization_id and
|
|
i.type == "agent_offline" and
|
|
i.status == "active"
|
|
)
|
|
|> where([i], not is_nil(i.agent_token_id))
|
|
|> Repo.all()
|
|
|
|
Enum.each(active_agent_insights, fn insight ->
|
|
if !MapSet.member?(currently_offline_ids, insight.agent_token_id) do
|
|
insight
|
|
|> Insight.changeset(%{status: "resolved"})
|
|
|> Repo.update()
|
|
end
|
|
end)
|
|
end
|
|
end
|