Graham McIntie
90af7ac0d1
- StormDetector GenServer: sliding-window per org, >10 alerts/min triggers suppression mode with single summary notification - Site correlation: >3 devices at same site down within 120s creates single 'Site Outage' alert instead of N individual alerts - Notification rate limiter: max 5 per user per 15min window, excess batched into digest via AlertDigestWorker - PagerDuty client: retry with exponential backoff on 429, respects Retry-After header - Batch maintenance checks: new Maintenance.devices_in_maintenance/1 takes list of device_ids, returns MapSet (single query vs N queries) - Migration: alert_storms, site_outages, notification_digests tables plus site_outage_id/storm_suppressed on alerts - DeviceMonitorWorker integrates storm detection and site correlation before creating alerts - Tests for StormDetector, SiteCorrelation, NotificationRateLimiter, and batch maintenance checks
121 lines
3.8 KiB
Elixir
121 lines
3.8 KiB
Elixir
defmodule Towerops.Alerts.Alert do
|
|
@moduledoc """
|
|
Alert schema for device monitoring alerts.
|
|
|
|
Tracks device_up, device_down, and monitoring check alerts with
|
|
acknowledgement status and auto-resolution.
|
|
"""
|
|
use Ecto.Schema
|
|
|
|
import Ecto.Changeset
|
|
|
|
alias Ecto.Association.NotLoaded
|
|
alias Towerops.Accounts.User
|
|
alias Towerops.Alerts.SiteOutage
|
|
alias Towerops.Devices.Device
|
|
alias Towerops.Monitoring.Check
|
|
alias Towerops.Organizations.Organization
|
|
|
|
@primary_key {:id, :binary_id, autogenerate: true}
|
|
@foreign_key_type :binary_id
|
|
schema "alerts" do
|
|
# Alert type: device_down, device_up for device alerts
|
|
# For check alerts, uses string like "check_http", "check_ping", etc.
|
|
field :alert_type, :string
|
|
field :triggered_at, :utc_datetime
|
|
field :acknowledged_at, :utc_datetime
|
|
field :resolved_at, :utc_datetime
|
|
field :email_sent_at, :utc_datetime
|
|
field :message, :string
|
|
field :gaiia_impact, :map
|
|
|
|
# Storm correlation
|
|
field :storm_suppressed, :boolean, default: false
|
|
belongs_to :site_outage, SiteOutage
|
|
|
|
# Check-related fields (added in ExtendAlertsForChecks migration)
|
|
field :severity, :integer
|
|
field :notification_sent, :boolean, default: false
|
|
field :notification_sent_at, :utc_datetime
|
|
field :output, :string
|
|
|
|
belongs_to :device, Device
|
|
belongs_to :check, Check
|
|
belongs_to :acknowledged_by, User
|
|
belongs_to :organization, Organization
|
|
|
|
timestamps(type: :utc_datetime)
|
|
end
|
|
|
|
@type t :: %__MODULE__{
|
|
id: Ecto.UUID.t(),
|
|
alert_type: String.t(),
|
|
triggered_at: DateTime.t(),
|
|
acknowledged_at: DateTime.t() | nil,
|
|
resolved_at: DateTime.t() | nil,
|
|
email_sent_at: DateTime.t() | nil,
|
|
message: String.t() | nil,
|
|
gaiia_impact: map() | nil,
|
|
severity: integer() | nil,
|
|
notification_sent: boolean(),
|
|
notification_sent_at: DateTime.t() | nil,
|
|
output: String.t() | nil,
|
|
device_id: Ecto.UUID.t() | nil,
|
|
device: NotLoaded.t() | Device.t() | nil,
|
|
check_id: Ecto.UUID.t() | nil,
|
|
check: NotLoaded.t() | Check.t() | nil,
|
|
acknowledged_by_id: Ecto.UUID.t() | nil,
|
|
acknowledged_by: NotLoaded.t() | User.t() | nil,
|
|
organization_id: Ecto.UUID.t() | nil,
|
|
organization: NotLoaded.t() | Organization.t() | nil,
|
|
site_outage_id: Ecto.UUID.t() | nil,
|
|
site_outage: NotLoaded.t() | SiteOutage.t() | nil,
|
|
storm_suppressed: boolean(),
|
|
inserted_at: DateTime.t(),
|
|
updated_at: DateTime.t()
|
|
}
|
|
|
|
@doc false
|
|
def changeset(alert, attrs) do
|
|
alert
|
|
|> cast(attrs, [
|
|
:device_id,
|
|
:check_id,
|
|
:alert_type,
|
|
:triggered_at,
|
|
:acknowledged_at,
|
|
:acknowledged_by_id,
|
|
:resolved_at,
|
|
:email_sent_at,
|
|
:message,
|
|
:gaiia_impact,
|
|
:severity,
|
|
:notification_sent,
|
|
:notification_sent_at,
|
|
:output,
|
|
:organization_id,
|
|
:site_outage_id,
|
|
:storm_suppressed
|
|
])
|
|
|> validate_required([:alert_type, :triggered_at])
|
|
|> validate_inclusion(:severity, [1, 2])
|
|
|> validate_device_or_check()
|
|
|> foreign_key_constraint(:device_id)
|
|
|> foreign_key_constraint(:check_id)
|
|
|> foreign_key_constraint(:acknowledged_by_id)
|
|
|> foreign_key_constraint(:organization_id)
|
|
|> foreign_key_constraint(:site_outage_id)
|
|
end
|
|
|
|
# Ensure either device_id or check_id is present (but not required for flexibility)
|
|
defp validate_device_or_check(changeset) do
|
|
device_id = get_field(changeset, :device_id)
|
|
check_id = get_field(changeset, :check_id)
|
|
|
|
if is_nil(device_id) and is_nil(check_id) do
|
|
add_error(changeset, :base, "either device_id or check_id must be present")
|
|
else
|
|
changeset
|
|
end
|
|
end
|
|
end
|