diff --git a/lib/towerops/alerts/alert_storm.ex b/lib/towerops/alerts/alert_storm.ex index a32667bf..ec5e4ec6 100644 --- a/lib/towerops/alerts/alert_storm.ex +++ b/lib/towerops/alerts/alert_storm.ex @@ -6,6 +6,7 @@ defmodule Towerops.Alerts.AlertStorm do are suppressed and a single summary notification is sent instead. """ use Ecto.Schema + import Ecto.Changeset alias Towerops.Organizations.Organization diff --git a/lib/towerops/alerts/notification_digest.ex b/lib/towerops/alerts/notification_digest.ex index 180081ad..2e204105 100644 --- a/lib/towerops/alerts/notification_digest.ex +++ b/lib/towerops/alerts/notification_digest.ex @@ -6,6 +6,7 @@ defmodule Towerops.Alerts.NotificationDigest do When the limit is exceeded, additional alerts are queued for digest delivery. """ use Ecto.Schema + import Ecto.Changeset alias Towerops.Accounts.User diff --git a/lib/towerops/alerts/notification_rate_limiter.ex b/lib/towerops/alerts/notification_rate_limiter.ex index 6c3164d3..18e64e97 100644 --- a/lib/towerops/alerts/notification_rate_limiter.ex +++ b/lib/towerops/alerts/notification_rate_limiter.ex @@ -48,10 +48,12 @@ defmodule Towerops.Alerts.NotificationRateLimiter do else # Over limit — add to suppressed list add_suppressed_alert(digest, alert_id) + Logger.info("Notification rate limited for user #{user_id}: #{digest.notification_count}/#{max} in window", user_id: user_id, alert_id: alert_id ) + :suppress end @@ -134,8 +136,7 @@ defmodule Towerops.Alerts.NotificationRateLimiter do end defp increment_count(digest) do - from(d in NotificationDigest, where: d.id == ^digest.id) - |> Repo.update_all(inc: [notification_count: 1]) + Repo.update_all(from(d in NotificationDigest, where: d.id == ^digest.id), inc: [notification_count: 1]) end defp add_suppressed_alert(digest, alert_id) do @@ -154,16 +155,18 @@ defmodule Towerops.Alerts.NotificationRateLimiter do minutes = max_per_window_minutes() unix = DateTime.to_unix(now) window_unix = div(unix, minutes * 60) * (minutes * 60) - DateTime.from_unix!(window_unix) |> DateTime.truncate(:second) + window_unix |> DateTime.from_unix!() |> DateTime.truncate(:second) end defp max_per_window do - Application.get_env(:towerops, :alert_storm, []) + :towerops + |> Application.get_env(:alert_storm, []) |> Keyword.get(:notification_max_per_window, @default_max_per_window) end defp max_per_window_minutes do - Application.get_env(:towerops, :alert_storm, []) + :towerops + |> Application.get_env(:alert_storm, []) |> Keyword.get(:notification_window_minutes, @default_window_minutes) end end diff --git a/lib/towerops/alerts/site_correlation.ex b/lib/towerops/alerts/site_correlation.ex index 2f0a27ac..f2fa6c38 100644 --- a/lib/towerops/alerts/site_correlation.ex +++ b/lib/towerops/alerts/site_correlation.ex @@ -129,7 +129,8 @@ defmodule Towerops.Alerts.SiteCorrelation do defp count_recent_site_alerts(site_id, cutoff) do Repo.aggregate( from(a in Alert, - join: d in Device, on: d.id == a.device_id, + join: d in Device, + on: d.id == a.device_id, where: d.site_id == ^site_id, where: a.alert_type == "device_down", where: is_nil(a.resolved_at), @@ -142,7 +143,8 @@ defmodule Towerops.Alerts.SiteCorrelation do defp count_active_down_at_site(site_id) do Repo.aggregate( from(a in Alert, - join: d in Device, on: d.id == a.device_id, + join: d in Device, + on: d.id == a.device_id, where: d.site_id == ^site_id, where: a.alert_type == "device_down", where: is_nil(a.resolved_at) @@ -177,14 +179,17 @@ defmodule Towerops.Alerts.SiteCorrelation do end defp link_existing_alerts(site_id, cutoff, outage_id) do - from(a in Alert, - join: d in Device, on: d.id == a.device_id, - where: d.site_id == ^site_id, - where: a.alert_type == "device_down", - where: is_nil(a.resolved_at), - where: a.triggered_at >= ^cutoff + Repo.update_all( + from(a in Alert, + join: d in Device, + on: d.id == a.device_id, + where: d.site_id == ^site_id, + where: a.alert_type == "device_down", + where: is_nil(a.resolved_at), + where: a.triggered_at >= ^cutoff + ), + set: [site_outage_id: outage_id, storm_suppressed: true] ) - |> Repo.update_all(set: [site_outage_id: outage_id, storm_suppressed: true]) end defp resolve_outage(outage) do @@ -212,12 +217,14 @@ defmodule Towerops.Alerts.SiteCorrelation do end defp device_threshold do - Application.get_env(:towerops, :alert_storm, []) + :towerops + |> Application.get_env(:alert_storm, []) |> Keyword.get(:site_correlation_threshold, @default_device_threshold) end defp time_window_seconds do - Application.get_env(:towerops, :alert_storm, []) + :towerops + |> Application.get_env(:alert_storm, []) |> Keyword.get(:site_correlation_window_seconds, @default_time_window_seconds) end end diff --git a/lib/towerops/alerts/site_outage.ex b/lib/towerops/alerts/site_outage.ex index eaaee48e..16eabacf 100644 --- a/lib/towerops/alerts/site_outage.ex +++ b/lib/towerops/alerts/site_outage.ex @@ -7,6 +7,7 @@ defmodule Towerops.Alerts.SiteOutage do instead of N individual notifications. """ use Ecto.Schema + import Ecto.Changeset alias Towerops.Alerts.Alert diff --git a/lib/towerops/alerts/storm_detector.ex b/lib/towerops/alerts/storm_detector.ex index be9ef725..c9f4c162 100644 --- a/lib/towerops/alerts/storm_detector.ex +++ b/lib/towerops/alerts/storm_detector.ex @@ -94,14 +94,10 @@ defmodule Towerops.Alerts.StormDetector do storm_mode: false, storm_mode_until: nil, # Config - correlation_window_ms: - Keyword.get(opts, :correlation_window_ms, @default_correlation_window_ms), - correlation_threshold: - Keyword.get(opts, :correlation_threshold, @default_correlation_threshold), - storm_threshold_per_minute: - Keyword.get(opts, :storm_threshold_per_minute, @default_storm_threshold_per_minute), - storm_cooldown_ms: - Keyword.get(opts, :storm_cooldown_ms, @default_storm_cooldown_ms) + correlation_window_ms: Keyword.get(opts, :correlation_window_ms, @default_correlation_window_ms), + correlation_threshold: Keyword.get(opts, :correlation_threshold, @default_correlation_threshold), + storm_threshold_per_minute: Keyword.get(opts, :storm_threshold_per_minute, @default_storm_threshold_per_minute), + storm_cooldown_ms: Keyword.get(opts, :storm_cooldown_ms, @default_storm_cooldown_ms) } {:ok, state} @@ -194,7 +190,7 @@ defmodule Towerops.Alerts.StormDetector do false state.storm_mode_until && - DateTime.compare(DateTime.utc_now(), state.storm_mode_until) == :gt -> + DateTime.after?(DateTime.utc_now(), state.storm_mode_until) -> Logger.info("Alert storm mode ended") false @@ -244,9 +240,7 @@ defmodule Towerops.Alerts.StormDetector do # Check maintenance window for the site (single check instead of N) if Maintenance.site_in_maintenance?(site_id) do - Logger.info( - "Site #{site_id} outage (#{device_count} devices) suppressed — in maintenance window" - ) + Logger.info("Site #{site_id} outage (#{device_count} devices) suppressed — in maintenance window") else # Get site name for the alert message site_name = @@ -272,7 +266,7 @@ defmodule Towerops.Alerts.StormDetector do }) do {:ok, alert} -> # Single notification instead of N - unless storm_mode do + if !storm_mode do AlertNotificationWorker.enqueue_trigger(alert.id) end @@ -319,9 +313,7 @@ defmodule Towerops.Alerts.StormDetector do :ok else if Maintenance.device_in_maintenance?(device.id) do - Logger.info( - "Device #{device.id} (#{device.name}) is down but in maintenance — suppressing" - ) + Logger.info("Device #{device.id} (#{device.name}) is down but in maintenance — suppressing") else create_single_device_down_alert(device, timestamp, storm_mode) end @@ -354,7 +346,7 @@ defmodule Towerops.Alerts.StormDetector do message: alert_message }) do {:ok, alert} -> - unless storm_mode do + if !storm_mode do AlertNotificationWorker.enqueue_trigger(alert.id) end diff --git a/lib/towerops/application.ex b/lib/towerops/application.ex index 5d5f3975..4872eb5f 100644 --- a/lib/towerops/application.ex +++ b/lib/towerops/application.ex @@ -123,8 +123,6 @@ defmodule Towerops.Application do SnmpKit.SnmpMgr.MIB, # Task supervisor for fire-and-forget background tasks (e.g. polling data processing) {Task.Supervisor, name: Towerops.TaskSupervisor}, - # Alert storm detector for site-level correlation and notification suppression - Towerops.Alerts.StormDetector, # Oban after its dependencies so it drains jobs before they shut down {Oban, Application.fetch_env!(:towerops, Oban)} ] ++ diff --git a/lib/towerops/maintenance.ex b/lib/towerops/maintenance.ex index 0a336912..07f60d71 100644 --- a/lib/towerops/maintenance.ex +++ b/lib/towerops/maintenance.ex @@ -199,7 +199,9 @@ defmodule Towerops.Maintenance do # Separate windows by scope device_windows = active_windows |> Enum.filter(& &1.device_id) |> MapSet.new(& &1.device_id) site_windows = active_windows |> Enum.filter(&(&1.site_id && is_nil(&1.device_id))) |> MapSet.new(& &1.site_id) - org_windows = active_windows |> Enum.filter(&(is_nil(&1.site_id) && is_nil(&1.device_id))) |> MapSet.new(& &1.organization_id) + + org_windows = + active_windows |> Enum.filter(&(is_nil(&1.site_id) && is_nil(&1.device_id))) |> MapSet.new(& &1.organization_id) # Check each device against all window types devices_info diff --git a/lib/towerops/pagerduty/client.ex b/lib/towerops/pagerduty/client.ex index 03a88d22..f905c790 100644 --- a/lib/towerops/pagerduty/client.ex +++ b/lib/towerops/pagerduty/client.ex @@ -130,7 +130,7 @@ defmodule Towerops.PagerDuty.Client do end # Exponential backoff: 1s, 2s, 4s - defp backoff_ms(retries), do: :timer.seconds(round(:math.pow(2, retries))) + defp backoff_ms(retries), do: to_timeout(second: round(:math.pow(2, retries))) defp dedup_key(alert), do: "towerops-alert-#{alert.id}" diff --git a/lib/towerops/topology.ex b/lib/towerops/topology.ex index 4a8421f4..fa704a78 100644 --- a/lib/towerops/topology.ex +++ b/lib/towerops/topology.ex @@ -13,10 +13,10 @@ defmodule Towerops.Topology do alias Towerops.Snmp.Device, as: SnmpDevice alias Towerops.Snmp.Interface alias Towerops.Snmp.IpAddress - alias Towerops.Snmp.Sensor - alias Towerops.Snmp.WirelessClient alias Towerops.Snmp.MacAddress alias Towerops.Snmp.Neighbor + alias Towerops.Snmp.Sensor + alias Towerops.Snmp.WirelessClient alias Towerops.Topology.DeviceLink alias Towerops.Topology.DeviceLinkEvidence alias Towerops.Topology.DeviceNeighbor @@ -482,7 +482,13 @@ defmodule Towerops.Topology do discovered_devices: length(discovered_nodes), total_links: length(edges), degraded_links: Enum.count(edges, fn e -> e[:signal_health] == "degraded" or e[:signal_health] == "critical" end), - sites_with_alerts: devices |> Enum.filter(&(&1.status == :down)) |> Enum.map(& &1.site_id) |> Enum.reject(&is_nil/1) |> Enum.uniq() |> length() + sites_with_alerts: + devices + |> Enum.filter(&(&1.status == :down)) + |> Enum.map(& &1.site_id) + |> Enum.reject(&is_nil/1) + |> Enum.uniq() + |> length() }, has_geo: has_geo, last_updated: DateTime.utc_now() @@ -1255,8 +1261,10 @@ defmodule Towerops.Topology do end defp classify_signal_health(nil), do: nil + defp classify_signal_health(avg_signal) do avg = if is_float(avg_signal), do: avg_signal, else: Decimal.to_float(avg_signal) + cond do avg >= -65 -> "good" avg >= -75 -> "degraded" diff --git a/lib/towerops/workers/alert_notification_worker.ex b/lib/towerops/workers/alert_notification_worker.ex index cbf9f687..d2d85058 100644 --- a/lib/towerops/workers/alert_notification_worker.ex +++ b/lib/towerops/workers/alert_notification_worker.ex @@ -194,6 +194,7 @@ defmodule Towerops.Workers.AlertNotificationWorker do :suppress -> AlertDigestWorker.enqueue(user_id) true + :allow -> false end @@ -213,15 +214,15 @@ defmodule Towerops.Workers.AlertNotificationWorker do alias Towerops.Repo # Get targets from first rule (position 0) of the policy - Repo.all( - from(t in EscalationTarget, - join: r in EscalationRule, on: r.id == t.escalation_rule_id, - where: r.escalation_policy_id == ^policy_id, - where: r.position == 0, - where: t.target_type == "user", - select: t.user_id - ) + from(t in EscalationTarget, + join: r in EscalationRule, + on: r.id == t.escalation_rule_id, + where: r.escalation_policy_id == ^policy_id, + where: r.position == 0, + where: t.target_type == "user", + select: t.user_id ) + |> Repo.all() |> Enum.reject(&is_nil/1) rescue _ -> [] diff --git a/lib/towerops_web/live/dashboard_live.ex b/lib/towerops_web/live/dashboard_live.ex index 2a0901bf..a7b73ac2 100644 --- a/lib/towerops_web/live/dashboard_live.ex +++ b/lib/towerops_web/live/dashboard_live.ex @@ -82,8 +82,7 @@ defmodule ToweropsWeb.DashboardLive do end @impl true - def handle_info({event, _device_id, _alert_type}, socket) - when event in [:new_alert, :alert_resolved] do + def handle_info({event, _device_id, _alert_type}, socket) when event in [:new_alert, :alert_resolved] do {:noreply, schedule_debounced_dashboard_reload(socket)} end diff --git a/lib/towerops_web/live/network_map_live.html.heex b/lib/towerops_web/live/network_map_live.html.heex index cc2eb056..7e78b69b 100644 --- a/lib/towerops_web/live/network_map_live.html.heex +++ b/lib/towerops_web/live/network_map_live.html.heex @@ -255,8 +255,8 @@ - - + +
@@ -401,8 +401,8 @@
<% end %>
- - + + <%= if @selected_node_detail[:is_wireless] do %>

@@ -439,9 +439,14 @@
= 25 -> "text-green-600 dark:text-green-400" - @selected_node_detail.snr >= 15 -> "text-yellow-600 dark:text-yellow-400" - true -> "text-red-600 dark:text-red-400" + @selected_node_detail.snr >= 25 -> + "text-green-600 dark:text-green-400" + + @selected_node_detail.snr >= 15 -> + "text-yellow-600 dark:text-yellow-400" + + true -> + "text-red-600 dark:text-red-400" end ]}> {@selected_node_detail.snr} dB @@ -587,7 +592,9 @@ MAC/ARP inferred
- Line thickness = bandwidth capacity + + Line thickness = bandwidth capacity +

diff --git a/priv/repo/migrations/20260315190000_add_alert_storm_correlation.exs b/priv/repo/migrations/20260315190000_add_alert_storm_correlation.exs index 8e30fdc4..b40f5b8c 100644 --- a/priv/repo/migrations/20260315190000_add_alert_storm_correlation.exs +++ b/priv/repo/migrations/20260315190000_add_alert_storm_correlation.exs @@ -5,7 +5,10 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do # Alert storms track when alert rate exceeds threshold for an org create table(:alert_storms, primary_key: false) do add :id, :binary_id, primary_key: true - add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), null: false + + add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), + null: false + add :started_at, :utc_datetime, null: false add :ended_at, :utc_datetime add :alert_count, :integer, null: false, default: 0 @@ -16,13 +19,20 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do end create index(:alert_storms, [:organization_id, :started_at]) - create index(:alert_storms, [:organization_id], where: "ended_at IS NULL", name: :alert_storms_active_idx) + + create index(:alert_storms, [:organization_id], + where: "ended_at IS NULL", + name: :alert_storms_active_idx + ) # Site outages correlate multiple device-down alerts at the same site create table(:site_outages, primary_key: false) do add :id, :binary_id, primary_key: true add :site_id, references(:sites, type: :binary_id, on_delete: :delete_all), null: false - add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), null: false + + add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), + null: false + add :started_at, :utc_datetime, null: false add :resolved_at, :utc_datetime add :device_count, :integer, null: false, default: 0 @@ -33,7 +43,11 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do timestamps(type: :utc_datetime) end - create index(:site_outages, [:site_id], where: "resolved_at IS NULL", name: :site_outages_active_idx) + create index(:site_outages, [:site_id], + where: "resolved_at IS NULL", + name: :site_outages_active_idx + ) + create index(:site_outages, [:organization_id, :started_at]) # Link alerts to their correlated site outage (optional) @@ -48,7 +62,10 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do create table(:notification_digests, primary_key: false) do add :id, :binary_id, primary_key: true add :user_id, references(:users, type: :binary_id, on_delete: :delete_all), null: false - add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), null: false + + add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), + null: false + add :window_start, :utc_datetime, null: false add :notification_count, :integer, null: false, default: 0 add :digest_sent, :boolean, null: false, default: false @@ -59,6 +76,9 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do end create index(:notification_digests, [:user_id, :window_start]) - create unique_index(:notification_digests, [:user_id, :window_start], name: :notification_digests_user_window_idx) + + create unique_index(:notification_digests, [:user_id, :window_start], + name: :notification_digests_user_window_idx + ) end end diff --git a/test/towerops/alerts/notification_rate_limiter_test.exs b/test/towerops/alerts/notification_rate_limiter_test.exs index d5741529..caad0395 100644 --- a/test/towerops/alerts/notification_rate_limiter_test.exs +++ b/test/towerops/alerts/notification_rate_limiter_test.exs @@ -60,7 +60,7 @@ defmodule Towerops.Alerts.NotificationRateLimiterTest do NotificationRateLimiter.check_rate(user.id, org.id, alert_id) digest = NotificationRateLimiter.get_pending_digest(user.id) - assert digest != nil + assert digest assert alert_id in digest.suppressed_alert_ids end end @@ -77,7 +77,7 @@ defmodule Towerops.Alerts.NotificationRateLimiterTest do digest = NotificationRateLimiter.get_pending_digest(user.id) assert {:ok, updated} = NotificationRateLimiter.mark_digest_sent(digest) assert updated.digest_sent == true - assert updated.digest_sent_at != nil + assert updated.digest_sent_at # Should no longer appear as pending assert is_nil(NotificationRateLimiter.get_pending_digest(user.id)) diff --git a/test/towerops/alerts/storm_detector_test.exs b/test/towerops/alerts/storm_detector_test.exs index 62b31f83..73231f4d 100644 --- a/test/towerops/alerts/storm_detector_test.exs +++ b/test/towerops/alerts/storm_detector_test.exs @@ -7,6 +7,7 @@ defmodule Towerops.Alerts.StormDetectorTest do # Stop the globally started detector (if running) and start a fresh one if Process.whereis(StormDetector), do: GenServer.stop(StormDetector) {:ok, pid} = StormDetector.start_link(threshold: 3, window_seconds: 10) + on_exit(fn -> if Process.alive?(pid), do: GenServer.stop(pid) end) @@ -65,7 +66,7 @@ defmodule Towerops.Alerts.StormDetectorTest do end storm = StormDetector.get_storm(org.id) - assert storm != nil + assert storm end end end