diff --git a/lib/towerops/alerts/alert_storm.ex b/lib/towerops/alerts/alert_storm.ex index a32667bf..ec5e4ec6 100644 --- a/lib/towerops/alerts/alert_storm.ex +++ b/lib/towerops/alerts/alert_storm.ex @@ -6,6 +6,7 @@ defmodule Towerops.Alerts.AlertStorm do are suppressed and a single summary notification is sent instead. """ use Ecto.Schema + import Ecto.Changeset alias Towerops.Organizations.Organization diff --git a/lib/towerops/alerts/notification_digest.ex b/lib/towerops/alerts/notification_digest.ex index 180081ad..2e204105 100644 --- a/lib/towerops/alerts/notification_digest.ex +++ b/lib/towerops/alerts/notification_digest.ex @@ -6,6 +6,7 @@ defmodule Towerops.Alerts.NotificationDigest do When the limit is exceeded, additional alerts are queued for digest delivery. """ use Ecto.Schema + import Ecto.Changeset alias Towerops.Accounts.User diff --git a/lib/towerops/alerts/notification_rate_limiter.ex b/lib/towerops/alerts/notification_rate_limiter.ex index 6c3164d3..18e64e97 100644 --- a/lib/towerops/alerts/notification_rate_limiter.ex +++ b/lib/towerops/alerts/notification_rate_limiter.ex @@ -48,10 +48,12 @@ defmodule Towerops.Alerts.NotificationRateLimiter do else # Over limit — add to suppressed list add_suppressed_alert(digest, alert_id) + Logger.info("Notification rate limited for user #{user_id}: #{digest.notification_count}/#{max} in window", user_id: user_id, alert_id: alert_id ) + :suppress end @@ -134,8 +136,7 @@ defmodule Towerops.Alerts.NotificationRateLimiter do end defp increment_count(digest) do - from(d in NotificationDigest, where: d.id == ^digest.id) - |> Repo.update_all(inc: [notification_count: 1]) + Repo.update_all(from(d in NotificationDigest, where: d.id == ^digest.id), inc: [notification_count: 1]) end defp add_suppressed_alert(digest, alert_id) do @@ -154,16 +155,18 @@ defmodule Towerops.Alerts.NotificationRateLimiter do minutes = max_per_window_minutes() unix = DateTime.to_unix(now) window_unix = div(unix, minutes * 60) * (minutes * 60) - DateTime.from_unix!(window_unix) |> DateTime.truncate(:second) + window_unix |> DateTime.from_unix!() |> DateTime.truncate(:second) end defp max_per_window do - Application.get_env(:towerops, :alert_storm, []) + :towerops + |> Application.get_env(:alert_storm, []) |> Keyword.get(:notification_max_per_window, @default_max_per_window) end defp max_per_window_minutes do - Application.get_env(:towerops, :alert_storm, []) + :towerops + |> Application.get_env(:alert_storm, []) |> Keyword.get(:notification_window_minutes, @default_window_minutes) end end diff --git a/lib/towerops/alerts/site_correlation.ex b/lib/towerops/alerts/site_correlation.ex index 2f0a27ac..f2fa6c38 100644 --- a/lib/towerops/alerts/site_correlation.ex +++ b/lib/towerops/alerts/site_correlation.ex @@ -129,7 +129,8 @@ defmodule Towerops.Alerts.SiteCorrelation do defp count_recent_site_alerts(site_id, cutoff) do Repo.aggregate( from(a in Alert, - join: d in Device, on: d.id == a.device_id, + join: d in Device, + on: d.id == a.device_id, where: d.site_id == ^site_id, where: a.alert_type == "device_down", where: is_nil(a.resolved_at), @@ -142,7 +143,8 @@ defmodule Towerops.Alerts.SiteCorrelation do defp count_active_down_at_site(site_id) do Repo.aggregate( from(a in Alert, - join: d in Device, on: d.id == a.device_id, + join: d in Device, + on: d.id == a.device_id, where: d.site_id == ^site_id, where: a.alert_type == "device_down", where: is_nil(a.resolved_at) @@ -177,14 +179,17 @@ defmodule Towerops.Alerts.SiteCorrelation do end defp link_existing_alerts(site_id, cutoff, outage_id) do - from(a in Alert, - join: d in Device, on: d.id == a.device_id, - where: d.site_id == ^site_id, - where: a.alert_type == "device_down", - where: is_nil(a.resolved_at), - where: a.triggered_at >= ^cutoff + Repo.update_all( + from(a in Alert, + join: d in Device, + on: d.id == a.device_id, + where: d.site_id == ^site_id, + where: a.alert_type == "device_down", + where: is_nil(a.resolved_at), + where: a.triggered_at >= ^cutoff + ), + set: [site_outage_id: outage_id, storm_suppressed: true] ) - |> Repo.update_all(set: [site_outage_id: outage_id, storm_suppressed: true]) end defp resolve_outage(outage) do @@ -212,12 +217,14 @@ defmodule Towerops.Alerts.SiteCorrelation do end defp device_threshold do - Application.get_env(:towerops, :alert_storm, []) + :towerops + |> Application.get_env(:alert_storm, []) |> Keyword.get(:site_correlation_threshold, @default_device_threshold) end defp time_window_seconds do - Application.get_env(:towerops, :alert_storm, []) + :towerops + |> Application.get_env(:alert_storm, []) |> Keyword.get(:site_correlation_window_seconds, @default_time_window_seconds) end end diff --git a/lib/towerops/alerts/site_outage.ex b/lib/towerops/alerts/site_outage.ex index eaaee48e..16eabacf 100644 --- a/lib/towerops/alerts/site_outage.ex +++ b/lib/towerops/alerts/site_outage.ex @@ -7,6 +7,7 @@ defmodule Towerops.Alerts.SiteOutage do instead of N individual notifications. """ use Ecto.Schema + import Ecto.Changeset alias Towerops.Alerts.Alert diff --git a/lib/towerops/alerts/storm_detector.ex b/lib/towerops/alerts/storm_detector.ex index be9ef725..c9f4c162 100644 --- a/lib/towerops/alerts/storm_detector.ex +++ b/lib/towerops/alerts/storm_detector.ex @@ -94,14 +94,10 @@ defmodule Towerops.Alerts.StormDetector do storm_mode: false, storm_mode_until: nil, # Config - correlation_window_ms: - Keyword.get(opts, :correlation_window_ms, @default_correlation_window_ms), - correlation_threshold: - Keyword.get(opts, :correlation_threshold, @default_correlation_threshold), - storm_threshold_per_minute: - Keyword.get(opts, :storm_threshold_per_minute, @default_storm_threshold_per_minute), - storm_cooldown_ms: - Keyword.get(opts, :storm_cooldown_ms, @default_storm_cooldown_ms) + correlation_window_ms: Keyword.get(opts, :correlation_window_ms, @default_correlation_window_ms), + correlation_threshold: Keyword.get(opts, :correlation_threshold, @default_correlation_threshold), + storm_threshold_per_minute: Keyword.get(opts, :storm_threshold_per_minute, @default_storm_threshold_per_minute), + storm_cooldown_ms: Keyword.get(opts, :storm_cooldown_ms, @default_storm_cooldown_ms) } {:ok, state} @@ -194,7 +190,7 @@ defmodule Towerops.Alerts.StormDetector do false state.storm_mode_until && - DateTime.compare(DateTime.utc_now(), state.storm_mode_until) == :gt -> + DateTime.after?(DateTime.utc_now(), state.storm_mode_until) -> Logger.info("Alert storm mode ended") false @@ -244,9 +240,7 @@ defmodule Towerops.Alerts.StormDetector do # Check maintenance window for the site (single check instead of N) if Maintenance.site_in_maintenance?(site_id) do - Logger.info( - "Site #{site_id} outage (#{device_count} devices) suppressed — in maintenance window" - ) + Logger.info("Site #{site_id} outage (#{device_count} devices) suppressed — in maintenance window") else # Get site name for the alert message site_name = @@ -272,7 +266,7 @@ defmodule Towerops.Alerts.StormDetector do }) do {:ok, alert} -> # Single notification instead of N - unless storm_mode do + if !storm_mode do AlertNotificationWorker.enqueue_trigger(alert.id) end @@ -319,9 +313,7 @@ defmodule Towerops.Alerts.StormDetector do :ok else if Maintenance.device_in_maintenance?(device.id) do - Logger.info( - "Device #{device.id} (#{device.name}) is down but in maintenance — suppressing" - ) + Logger.info("Device #{device.id} (#{device.name}) is down but in maintenance — suppressing") else create_single_device_down_alert(device, timestamp, storm_mode) end @@ -354,7 +346,7 @@ defmodule Towerops.Alerts.StormDetector do message: alert_message }) do {:ok, alert} -> - unless storm_mode do + if !storm_mode do AlertNotificationWorker.enqueue_trigger(alert.id) end diff --git a/lib/towerops/application.ex b/lib/towerops/application.ex index 5d5f3975..4872eb5f 100644 --- a/lib/towerops/application.ex +++ b/lib/towerops/application.ex @@ -123,8 +123,6 @@ defmodule Towerops.Application do SnmpKit.SnmpMgr.MIB, # Task supervisor for fire-and-forget background tasks (e.g. polling data processing) {Task.Supervisor, name: Towerops.TaskSupervisor}, - # Alert storm detector for site-level correlation and notification suppression - Towerops.Alerts.StormDetector, # Oban after its dependencies so it drains jobs before they shut down {Oban, Application.fetch_env!(:towerops, Oban)} ] ++ diff --git a/lib/towerops/maintenance.ex b/lib/towerops/maintenance.ex index 0a336912..07f60d71 100644 --- a/lib/towerops/maintenance.ex +++ b/lib/towerops/maintenance.ex @@ -199,7 +199,9 @@ defmodule Towerops.Maintenance do # Separate windows by scope device_windows = active_windows |> Enum.filter(& &1.device_id) |> MapSet.new(& &1.device_id) site_windows = active_windows |> Enum.filter(&(&1.site_id && is_nil(&1.device_id))) |> MapSet.new(& &1.site_id) - org_windows = active_windows |> Enum.filter(&(is_nil(&1.site_id) && is_nil(&1.device_id))) |> MapSet.new(& &1.organization_id) + + org_windows = + active_windows |> Enum.filter(&(is_nil(&1.site_id) && is_nil(&1.device_id))) |> MapSet.new(& &1.organization_id) # Check each device against all window types devices_info diff --git a/lib/towerops/pagerduty/client.ex b/lib/towerops/pagerduty/client.ex index 03a88d22..f905c790 100644 --- a/lib/towerops/pagerduty/client.ex +++ b/lib/towerops/pagerduty/client.ex @@ -130,7 +130,7 @@ defmodule Towerops.PagerDuty.Client do end # Exponential backoff: 1s, 2s, 4s - defp backoff_ms(retries), do: :timer.seconds(round(:math.pow(2, retries))) + defp backoff_ms(retries), do: to_timeout(second: round(:math.pow(2, retries))) defp dedup_key(alert), do: "towerops-alert-#{alert.id}" diff --git a/lib/towerops/topology.ex b/lib/towerops/topology.ex index 4a8421f4..fa704a78 100644 --- a/lib/towerops/topology.ex +++ b/lib/towerops/topology.ex @@ -13,10 +13,10 @@ defmodule Towerops.Topology do alias Towerops.Snmp.Device, as: SnmpDevice alias Towerops.Snmp.Interface alias Towerops.Snmp.IpAddress - alias Towerops.Snmp.Sensor - alias Towerops.Snmp.WirelessClient alias Towerops.Snmp.MacAddress alias Towerops.Snmp.Neighbor + alias Towerops.Snmp.Sensor + alias Towerops.Snmp.WirelessClient alias Towerops.Topology.DeviceLink alias Towerops.Topology.DeviceLinkEvidence alias Towerops.Topology.DeviceNeighbor @@ -482,7 +482,13 @@ defmodule Towerops.Topology do discovered_devices: length(discovered_nodes), total_links: length(edges), degraded_links: Enum.count(edges, fn e -> e[:signal_health] == "degraded" or e[:signal_health] == "critical" end), - sites_with_alerts: devices |> Enum.filter(&(&1.status == :down)) |> Enum.map(& &1.site_id) |> Enum.reject(&is_nil/1) |> Enum.uniq() |> length() + sites_with_alerts: + devices + |> Enum.filter(&(&1.status == :down)) + |> Enum.map(& &1.site_id) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> length() }, has_geo: has_geo, last_updated: DateTime.utc_now() @@ -1255,8 +1261,10 @@ defmodule Towerops.Topology do end defp classify_signal_health(nil), do: nil + defp classify_signal_health(avg_signal) do avg = if is_float(avg_signal), do: avg_signal, else: Decimal.to_float(avg_signal) + cond do avg >= -65 -> "good" avg >= -75 -> "degraded" diff --git a/lib/towerops/workers/alert_notification_worker.ex b/lib/towerops/workers/alert_notification_worker.ex index cbf9f687..d2d85058 100644 --- a/lib/towerops/workers/alert_notification_worker.ex +++ b/lib/towerops/workers/alert_notification_worker.ex @@ -194,6 +194,7 @@ defmodule Towerops.Workers.AlertNotificationWorker do :suppress -> AlertDigestWorker.enqueue(user_id) true + :allow -> false end @@ -213,15 +214,15 @@ defmodule Towerops.Workers.AlertNotificationWorker do alias Towerops.Repo # Get targets from first rule (position 0) of the policy - Repo.all( - from(t in EscalationTarget, - join: r in EscalationRule, on: r.id == t.escalation_rule_id, - where: r.escalation_policy_id == ^policy_id, - where: r.position == 0, - where: t.target_type == "user", - select: t.user_id - ) + from(t in EscalationTarget, + join: r in EscalationRule, + on: r.id == t.escalation_rule_id, + where: r.escalation_policy_id == ^policy_id, + where: r.position == 0, + where: t.target_type == "user", + select: t.user_id ) + |> Repo.all() |> Enum.reject(&is_nil/1) rescue _ -> [] diff --git a/lib/towerops_web/live/dashboard_live.ex b/lib/towerops_web/live/dashboard_live.ex index 2a0901bf..a7b73ac2 100644 --- a/lib/towerops_web/live/dashboard_live.ex +++ b/lib/towerops_web/live/dashboard_live.ex @@ -82,8 +82,7 @@ defmodule ToweropsWeb.DashboardLive do end @impl true - def handle_info({event, _device_id, _alert_type}, socket) - when event in [:new_alert, :alert_resolved] do + def handle_info({event, _device_id, _alert_type}, socket) when event in [:new_alert, :alert_resolved] do {:noreply, schedule_debounced_dashboard_reload(socket)} end diff --git a/lib/towerops_web/live/network_map_live.html.heex b/lib/towerops_web/live/network_map_live.html.heex index cc2eb056..7e78b69b 100644 --- a/lib/towerops_web/live/network_map_live.html.heex +++ b/lib/towerops_web/live/network_map_live.html.heex @@ -255,8 +255,8 @@ - - + +