Graham McIntie
7b72ed3164
- POST /api/v1/webhooks/pagerduty/:organization_id endpoint - HMAC-SHA256 signature verification (optional, via webhook_secret) - incident.resolved → resolve_alert_silent (no re-notify to PagerDuty) - incident.acknowledged → acknowledge_alert_silent (no re-notify) - Matches alerts via dedup_key format: towerops-alert-<uuid> - Webhook secret configurable in org settings PagerDuty panel - Setup instructions in UI for PagerDuty Generic Webhooks (v3)
333 lines
8.8 KiB
Elixir
333 lines
8.8 KiB
Elixir
defmodule Towerops.Alerts do
|
|
@moduledoc """
|
|
The Alerts context.
|
|
"""
|
|
|
|
import Ecto.Query
|
|
|
|
alias Towerops.Alerts.Alert
|
|
alias Towerops.Gaiia.ImpactAnalysis
|
|
alias Towerops.PagerDuty.Notifier
|
|
alias Towerops.Repo
|
|
|
|
@doc """
|
|
Creates an alert for device status change.
|
|
|
|
For device_down alerts, automatically computes and stores Gaiia impact analysis
|
|
if the device belongs to an organization with a Gaiia integration.
|
|
"""
|
|
@spec create_alert(map()) :: {:ok, Alert.t()} | {:error, Ecto.Changeset.t()}
|
|
def create_alert(attrs) do
|
|
with {:ok, alert} <- Repo.insert(Alert.changeset(%Alert{}, attrs)) do
|
|
maybe_compute_gaiia_impact(alert)
|
|
{:ok, alert}
|
|
end
|
|
end
|
|
|
|
defp maybe_compute_gaiia_impact(%Alert{alert_type: :device_down, device_id: device_id} = alert)
|
|
when not is_nil(device_id) do
|
|
device = Towerops.Devices.get_device(device_id)
|
|
|
|
if device do
|
|
impact = ImpactAnalysis.analyze_device_impact(device.organization_id, device_id)
|
|
|
|
if impact.total_subscribers > 0 do
|
|
store_gaiia_impact(alert, ImpactAnalysis.to_json(impact))
|
|
end
|
|
end
|
|
rescue
|
|
_ -> :ok
|
|
end
|
|
|
|
defp maybe_compute_gaiia_impact(_alert), do: :ok
|
|
|
|
@doc """
|
|
Returns the list of alerts for an device.
|
|
"""
|
|
@spec list_devices_alerts(String.t(), integer()) :: [Alert.t()]
|
|
def list_devices_alerts(device_id, limit \\ 100) do
|
|
Repo.all(
|
|
from(a in Alert,
|
|
where: a.device_id == ^device_id,
|
|
order_by: [desc: a.triggered_at],
|
|
limit: ^limit,
|
|
preload: [:device, :acknowledged_by]
|
|
)
|
|
)
|
|
end
|
|
|
|
@doc """
|
|
Returns the list of active (unresolved) alerts for an organization.
|
|
Only returns equipment_down alerts, as equipment_up alerts are informational.
|
|
"""
|
|
@spec list_organization_active_alerts(String.t()) :: [Alert.t()]
|
|
def list_organization_active_alerts(organization_id) do
|
|
Repo.all(
|
|
from(a in Alert,
|
|
join: e in assoc(a, :device),
|
|
join: s in assoc(e, :site),
|
|
where: s.organization_id == ^organization_id,
|
|
where: a.alert_type == :device_down,
|
|
where: is_nil(a.resolved_at),
|
|
order_by: [desc: a.triggered_at],
|
|
preload: [:acknowledged_by, device: {e, site: s}]
|
|
)
|
|
)
|
|
end
|
|
|
|
@doc """
|
|
Returns the count of active (unresolved) alerts for an organization.
|
|
"""
|
|
@spec count_active_alerts(String.t()) :: integer()
|
|
def count_active_alerts(organization_id) do
|
|
Repo.aggregate(
|
|
from(a in Alert,
|
|
join: e in assoc(a, :device),
|
|
join: s in assoc(e, :site),
|
|
where: s.organization_id == ^organization_id,
|
|
where: a.alert_type == :device_down,
|
|
where: is_nil(a.resolved_at)
|
|
),
|
|
:count
|
|
)
|
|
end
|
|
|
|
@doc "Returns the count of active alerts for devices at a specific site."
|
|
def count_site_active_alerts(site_id) do
|
|
Repo.aggregate(
|
|
from(a in Alert,
|
|
join: e in assoc(a, :device),
|
|
where: e.site_id == ^site_id,
|
|
where: a.alert_type == :device_down,
|
|
where: is_nil(a.resolved_at)
|
|
),
|
|
:count
|
|
)
|
|
end
|
|
|
|
@doc "Returns active (unresolved) alerts for devices at a specific site."
|
|
def list_site_active_alerts(site_id) do
|
|
Repo.all(
|
|
from(a in Alert,
|
|
join: e in assoc(a, :device),
|
|
where: e.site_id == ^site_id,
|
|
where: a.alert_type == :device_down,
|
|
where: is_nil(a.resolved_at),
|
|
order_by: [desc: a.triggered_at],
|
|
preload: [device: e]
|
|
)
|
|
)
|
|
end
|
|
|
|
@doc """
|
|
Returns the list of all alerts for an organization.
|
|
|
|
Accepts either an integer limit or a map of filters:
|
|
- limit (integer): Max number of alerts to return
|
|
- filters (map):
|
|
- status: Filter by status ("active", "acknowledged", "resolved")
|
|
- limit: Max number of alerts to return
|
|
"""
|
|
@spec list_organization_alerts(String.t(), integer() | map()) :: [Alert.t()]
|
|
def list_organization_alerts(organization_id, limit) when is_integer(limit) do
|
|
list_organization_alerts(organization_id, %{"limit" => limit})
|
|
end
|
|
|
|
def list_organization_alerts(organization_id, filters) when is_map(filters) do
|
|
limit = filters["limit"] || 100
|
|
|
|
query =
|
|
from(a in Alert,
|
|
join: e in assoc(a, :device),
|
|
join: s in assoc(e, :site),
|
|
where: s.organization_id == ^organization_id,
|
|
order_by: [desc: a.triggered_at],
|
|
limit: ^limit,
|
|
preload: [device: {e, site: s}, acknowledged_by: []]
|
|
)
|
|
|
|
query =
|
|
case filters["status"] do
|
|
"active" -> where(query, [a], is_nil(a.resolved_at) and is_nil(a.acknowledged_at))
|
|
"acknowledged" -> where(query, [a], is_nil(a.resolved_at) and not is_nil(a.acknowledged_at))
|
|
"resolved" -> where(query, [a], not is_nil(a.resolved_at))
|
|
_ -> query
|
|
end
|
|
|
|
Repo.all(query)
|
|
end
|
|
|
|
@doc """
|
|
Returns alerts for multiple organizations (for GDPR data access).
|
|
Accepts a `since` option to filter alerts created after a specific date.
|
|
"""
|
|
def list_alerts_for_organizations(organization_ids, opts \\ []) when is_list(organization_ids) do
|
|
since = Keyword.get(opts, :since)
|
|
|
|
query =
|
|
from(a in Alert,
|
|
join: e in assoc(a, :device),
|
|
join: s in assoc(e, :site),
|
|
where: s.organization_id in ^organization_ids,
|
|
order_by: [desc: a.triggered_at],
|
|
preload: [device: {e, site: s}]
|
|
)
|
|
|
|
query =
|
|
if since do
|
|
where(query, [a], a.inserted_at >= ^since)
|
|
else
|
|
query
|
|
end
|
|
|
|
Repo.all(query)
|
|
end
|
|
|
|
@doc """
|
|
Gets a single alert.
|
|
where: s.organization_id == ^organization_id,
|
|
order_by: [desc: a.triggered_at],
|
|
limit: ^limit,
|
|
preload: [device: {e, site: s}, acknowledged_by: []]
|
|
)
|
|
)
|
|
end
|
|
|
|
@doc \"""
|
|
Gets a single alert.
|
|
"""
|
|
def get_alert!(id) do
|
|
Alert
|
|
|> Repo.get!(id)
|
|
|> Repo.preload([:device, :acknowledged_by])
|
|
end
|
|
|
|
@doc """
|
|
Gets a single alert. Returns nil if not found.
|
|
"""
|
|
def get_alert(id) do
|
|
case Repo.get(Alert, id) do
|
|
nil -> nil
|
|
alert -> Repo.preload(alert, [:device, :acknowledged_by])
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Acknowledges an alert.
|
|
"""
|
|
def acknowledge_alert(alert, user_id) do
|
|
result =
|
|
alert
|
|
|> Alert.changeset(%{
|
|
acknowledged_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
acknowledged_by_id: user_id
|
|
})
|
|
|> Repo.update()
|
|
|
|
case result do
|
|
{:ok, updated_alert} ->
|
|
Task.start(fn -> Notifier.notify_acknowledge(updated_alert) end)
|
|
{:ok, updated_alert}
|
|
|
|
error ->
|
|
error
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Resolves an alert.
|
|
"""
|
|
def resolve_alert(alert) do
|
|
result =
|
|
alert
|
|
|> Alert.changeset(%{resolved_at: DateTime.truncate(DateTime.utc_now(), :second)})
|
|
|> Repo.update()
|
|
|
|
case result do
|
|
{:ok, updated_alert} ->
|
|
Task.start(fn -> Notifier.notify_resolve(updated_alert) end)
|
|
{:ok, updated_alert}
|
|
|
|
error ->
|
|
error
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Resolves an alert without notifying PagerDuty (used when PagerDuty is the source).
|
|
"""
|
|
def resolve_alert_silent(alert) do
|
|
alert
|
|
|> Alert.changeset(%{resolved_at: DateTime.truncate(DateTime.utc_now(), :second)})
|
|
|> Repo.update()
|
|
end
|
|
|
|
@doc """
|
|
Acknowledges an alert without notifying PagerDuty (used when PagerDuty is the source).
|
|
"""
|
|
def acknowledge_alert_silent(alert) do
|
|
alert
|
|
|> Alert.changeset(%{acknowledged_at: DateTime.truncate(DateTime.utc_now(), :second)})
|
|
|> Repo.update()
|
|
end
|
|
|
|
@doc """
|
|
Stores Gaiia impact analysis data on an alert record.
|
|
"""
|
|
def store_gaiia_impact(%Alert{} = alert, impact_data) when is_map(impact_data) do
|
|
alert
|
|
|> Alert.changeset(%{gaiia_impact: impact_data})
|
|
|> Repo.update()
|
|
end
|
|
|
|
@doc """
|
|
Checks if there's an active alert of the same type for the device.
|
|
"""
|
|
def has_active_alert?(device_id, alert_type) do
|
|
Repo.exists?(
|
|
from(a in Alert,
|
|
where: a.device_id == ^device_id,
|
|
where: a.alert_type == ^alert_type,
|
|
where: is_nil(a.resolved_at)
|
|
)
|
|
)
|
|
end
|
|
|
|
@doc """
|
|
Gets the latest active alert of a specific type for device.
|
|
"""
|
|
def get_active_alert(device_id, alert_type) do
|
|
Repo.one(
|
|
from(a in Alert,
|
|
where: a.device_id == ^device_id,
|
|
where: a.alert_type == ^alert_type,
|
|
where: is_nil(a.resolved_at),
|
|
order_by: [desc: a.triggered_at],
|
|
limit: 1
|
|
)
|
|
)
|
|
end
|
|
|
|
@doc """
|
|
Checks if there's an active alert for a check.
|
|
"""
|
|
def has_active_check_alert?(check_id) do
|
|
Repo.exists?(
|
|
from(a in Alert,
|
|
where: a.check_id == ^check_id,
|
|
where: is_nil(a.resolved_at)
|
|
)
|
|
)
|
|
end
|
|
|
|
@doc """
|
|
Resolves all active alerts for a check.
|
|
"""
|
|
def resolve_check_alerts(check_id) do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
Repo.update_all(from(a in Alert, where: a.check_id == ^check_id, where: is_nil(a.resolved_at)),
|
|
set: [resolved_at: now]
|
|
)
|
|
end
|
|
end
|