graham/towerops
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fix: remove duplicate StormDetector child spec

Browse source 
The StormDetector was being added twice:
1. In the main children list (line 127)
2. In background_workers() (line 176)

This caused the supervisor to fail with 'duplicate_child_name' error.

Removed the duplicate from main children list since it's correctly
placed in background_workers() which also handles test env properly.
This commit is contained in:
Graham McIntire 2026-03-15 16:05:30 -05:00
parent 44fc017653
commit acf70b3bee
No known key found for this signature in database
16 changed files with 108 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
lib/towerops/alerts/alert_storm.ex
View file

@ -6,6 +6,7 @@ defmodule Towerops.Alerts.AlertStorm do
are suppressed and a single summary notification is sent instead.
"""
use Ecto.Schema
import Ecto.Changeset
alias Towerops.Organizations.Organization

1
lib/towerops/alerts/notification_digest.ex
View file

@ -6,6 +6,7 @@ defmodule Towerops.Alerts.NotificationDigest do
When the limit is exceeded, additional alerts are queued for digest delivery.
"""
use Ecto.Schema
import Ecto.Changeset
alias Towerops.Accounts.User

13
lib/towerops/alerts/notification_rate_limiter.ex
View file

@ -48,10 +48,12 @@ defmodule Towerops.Alerts.NotificationRateLimiter do
else
# Over limit — add to suppressed list
add_suppressed_alert(digest, alert_id)
Logger.info("Notification rate limited for user #{user_id}: #{digest.notification_count}/#{max} in window",
user_id: user_id,
alert_id: alert_id
)
:suppress
end
@ -134,8 +136,7 @@ defmodule Towerops.Alerts.NotificationRateLimiter do
end
defp increment_count(digest) do
from(d in NotificationDigest, where: d.id == ^digest.id)
|> Repo.update_all(inc: [notification_count: 1])
Repo.update_all(from(d in NotificationDigest, where: d.id == ^digest.id), inc: [notification_count: 1])
end
defp add_suppressed_alert(digest, alert_id) do
@ -154,16 +155,18 @@ defmodule Towerops.Alerts.NotificationRateLimiter do
minutes = max_per_window_minutes()
unix = DateTime.to_unix(now)
window_unix = div(unix, minutes * 60) * (minutes * 60)
DateTime.from_unix!(window_unix) |> DateTime.truncate(:second)
window_unix |> DateTime.from_unix!() |> DateTime.truncate(:second)
end
defp max_per_window do
Application.get_env(:towerops, :alert_storm, [])
:towerops
|> Application.get_env(:alert_storm, [])
|> Keyword.get(:notification_max_per_window, @default_max_per_window)
end
defp max_per_window_minutes do
Application.get_env(:towerops, :alert_storm, [])
:towerops
|> Application.get_env(:alert_storm, [])
|> Keyword.get(:notification_window_minutes, @default_window_minutes)
end
end

29
lib/towerops/alerts/site_correlation.ex
View file

@ -129,7 +129,8 @@ defmodule Towerops.Alerts.SiteCorrelation do
defp count_recent_site_alerts(site_id, cutoff) do
Repo.aggregate(
from(a in Alert,
join: d in Device, on: d.id == a.device_id,
join: d in Device,
on: d.id == a.device_id,
where: d.site_id == ^site_id,
where: a.alert_type == "device_down",
where: is_nil(a.resolved_at),
@ -142,7 +143,8 @@ defmodule Towerops.Alerts.SiteCorrelation do
defp count_active_down_at_site(site_id) do
Repo.aggregate(
from(a in Alert,
join: d in Device, on: d.id == a.device_id,
join: d in Device,
on: d.id == a.device_id,
where: d.site_id == ^site_id,
where: a.alert_type == "device_down",
where: is_nil(a.resolved_at)
@ -177,14 +179,17 @@ defmodule Towerops.Alerts.SiteCorrelation do
end
defp link_existing_alerts(site_id, cutoff, outage_id) do
from(a in Alert,
join: d in Device, on: d.id == a.device_id,
where: d.site_id == ^site_id,
where: a.alert_type == "device_down",
where: is_nil(a.resolved_at),
where: a.triggered_at >= ^cutoff
Repo.update_all(
from(a in Alert,
join: d in Device,
on: d.id == a.device_id,
where: d.site_id == ^site_id,
where: a.alert_type == "device_down",
where: is_nil(a.resolved_at),
where: a.triggered_at >= ^cutoff
),
set: [site_outage_id: outage_id, storm_suppressed: true]
)
|> Repo.update_all(set: [site_outage_id: outage_id, storm_suppressed: true])
end
defp resolve_outage(outage) do
@ -212,12 +217,14 @@ defmodule Towerops.Alerts.SiteCorrelation do
end
defp device_threshold do
Application.get_env(:towerops, :alert_storm, [])
:towerops
|> Application.get_env(:alert_storm, [])
|> Keyword.get(:site_correlation_threshold, @default_device_threshold)
end
defp time_window_seconds do
Application.get_env(:towerops, :alert_storm, [])
:towerops
|> Application.get_env(:alert_storm, [])
|> Keyword.get(:site_correlation_window_seconds, @default_time_window_seconds)
end
end

1
lib/towerops/alerts/site_outage.ex
View file

@ -7,6 +7,7 @@ defmodule Towerops.Alerts.SiteOutage do
instead of N individual notifications.
"""
use Ecto.Schema
import Ecto.Changeset
alias Towerops.Alerts.Alert

26
lib/towerops/alerts/storm_detector.ex
View file

@ -94,14 +94,10 @@ defmodule Towerops.Alerts.StormDetector do
storm_mode: false,
storm_mode_until: nil,
# Config
correlation_window_ms:
Keyword.get(opts, :correlation_window_ms, @default_correlation_window_ms),
correlation_threshold:
Keyword.get(opts, :correlation_threshold, @default_correlation_threshold),
storm_threshold_per_minute:
Keyword.get(opts, :storm_threshold_per_minute, @default_storm_threshold_per_minute),
storm_cooldown_ms:
Keyword.get(opts, :storm_cooldown_ms, @default_storm_cooldown_ms)
correlation_window_ms: Keyword.get(opts, :correlation_window_ms, @default_correlation_window_ms),
correlation_threshold: Keyword.get(opts, :correlation_threshold, @default_correlation_threshold),
storm_threshold_per_minute: Keyword.get(opts, :storm_threshold_per_minute, @default_storm_threshold_per_minute),
storm_cooldown_ms: Keyword.get(opts, :storm_cooldown_ms, @default_storm_cooldown_ms)
}
{:ok, state}
@ -194,7 +190,7 @@ defmodule Towerops.Alerts.StormDetector do
false
state.storm_mode_until &&
DateTime.compare(DateTime.utc_now(), state.storm_mode_until) == :gt ->
DateTime.after?(DateTime.utc_now(), state.storm_mode_until) ->
Logger.info("Alert storm mode ended")
false
@ -244,9 +240,7 @@ defmodule Towerops.Alerts.StormDetector do
# Check maintenance window for the site (single check instead of N)
if Maintenance.site_in_maintenance?(site_id) do
Logger.info(
"Site #{site_id} outage (#{device_count} devices) suppressed — in maintenance window"
)
Logger.info("Site #{site_id} outage (#{device_count} devices) suppressed — in maintenance window")
else
# Get site name for the alert message
site_name =
@ -272,7 +266,7 @@ defmodule Towerops.Alerts.StormDetector do
}) do
{:ok, alert} ->
# Single notification instead of N
unless storm_mode do
if !storm_mode do
AlertNotificationWorker.enqueue_trigger(alert.id)
end
@ -319,9 +313,7 @@ defmodule Towerops.Alerts.StormDetector do
:ok
else
if Maintenance.device_in_maintenance?(device.id) do
Logger.info(
"Device #{device.id} (#{device.name}) is down but in maintenance — suppressing"
)
Logger.info("Device #{device.id} (#{device.name}) is down but in maintenance — suppressing")
else
create_single_device_down_alert(device, timestamp, storm_mode)
end
@ -354,7 +346,7 @@ defmodule Towerops.Alerts.StormDetector do
message: alert_message
}) do
{:ok, alert} ->
unless storm_mode do
if !storm_mode do
AlertNotificationWorker.enqueue_trigger(alert.id)
end

2
lib/towerops/application.ex
View file

@ -123,8 +123,6 @@ defmodule Towerops.Application do
SnmpKit.SnmpMgr.MIB,
# Task supervisor for fire-and-forget background tasks (e.g. polling data processing)
{Task.Supervisor, name: Towerops.TaskSupervisor},
# Alert storm detector for site-level correlation and notification suppression
Towerops.Alerts.StormDetector,
# Oban after its dependencies so it drains jobs before they shut down
{Oban, Application.fetch_env!(:towerops, Oban)}
] ++

4
lib/towerops/maintenance.ex
View file

@ -199,7 +199,9 @@ defmodule Towerops.Maintenance do
# Separate windows by scope
device_windows = active_windows |> Enum.filter(& &1.device_id) |> MapSet.new(& &1.device_id)
site_windows = active_windows |> Enum.filter(&(&1.site_id && is_nil(&1.device_id))) |> MapSet.new(& &1.site_id)
org_windows = active_windows |> Enum.filter(&(is_nil(&1.site_id) && is_nil(&1.device_id))) |> MapSet.new(& &1.organization_id)
org_windows =
active_windows |> Enum.filter(&(is_nil(&1.site_id) && is_nil(&1.device_id))) |> MapSet.new(& &1.organization_id)
# Check each device against all window types
devices_info

2
lib/towerops/pagerduty/client.ex
View file

@ -130,7 +130,7 @@ defmodule Towerops.PagerDuty.Client do
end
# Exponential backoff: 1s, 2s, 4s
defp backoff_ms(retries), do: :timer.seconds(round(:math.pow(2, retries)))
defp backoff_ms(retries), do: to_timeout(second: round(:math.pow(2, retries)))
defp dedup_key(alert), do: "towerops-alert-#{alert.id}"

14
lib/towerops/topology.ex
View file

@ -13,10 +13,10 @@ defmodule Towerops.Topology do
alias Towerops.Snmp.Device, as: SnmpDevice
alias Towerops.Snmp.Interface
alias Towerops.Snmp.IpAddress
alias Towerops.Snmp.Sensor
alias Towerops.Snmp.WirelessClient
alias Towerops.Snmp.MacAddress
alias Towerops.Snmp.Neighbor
alias Towerops.Snmp.Sensor
alias Towerops.Snmp.WirelessClient
alias Towerops.Topology.DeviceLink
alias Towerops.Topology.DeviceLinkEvidence
alias Towerops.Topology.DeviceNeighbor
@ -482,7 +482,13 @@ defmodule Towerops.Topology do
discovered_devices: length(discovered_nodes),
total_links: length(edges),
degraded_links: Enum.count(edges, fn e -> e[:signal_health] == "degraded" or e[:signal_health] == "critical" end),
sites_with_alerts: devices |> Enum.filter(&(&1.status == :down)) |> Enum.map(& &1.site_id) |> Enum.reject(&is_nil/1) |> Enum.uniq() |> length()
sites_with_alerts:
devices
|> Enum.filter(&(&1.status == :down))
|> Enum.map(& &1.site_id)
|> Enum.reject(&is_nil/1)
|> Enum.uniq()
|> length()
},
has_geo: has_geo,
last_updated: DateTime.utc_now()
@ -1255,8 +1261,10 @@ defmodule Towerops.Topology do
end
defp classify_signal_health(nil), do: nil
defp classify_signal_health(avg_signal) do
avg = if is_float(avg_signal), do: avg_signal, else: Decimal.to_float(avg_signal)
cond do
avg >= -65 -> "good"
avg >= -75 -> "degraded"

17
lib/towerops/workers/alert_notification_worker.ex
View file

@ -194,6 +194,7 @@ defmodule Towerops.Workers.AlertNotificationWorker do
:suppress ->
AlertDigestWorker.enqueue(user_id)
true
:allow ->
false
end
@ -213,15 +214,15 @@ defmodule Towerops.Workers.AlertNotificationWorker do
alias Towerops.Repo
# Get targets from first rule (position 0) of the policy
Repo.all(
from(t in EscalationTarget,
join: r in EscalationRule, on: r.id == t.escalation_rule_id,
where: r.escalation_policy_id == ^policy_id,
where: r.position == 0,
where: t.target_type == "user",
select: t.user_id
)
from(t in EscalationTarget,
join: r in EscalationRule,
on: r.id == t.escalation_rule_id,
where: r.escalation_policy_id == ^policy_id,
where: r.position == 0,
where: t.target_type == "user",
select: t.user_id
)
|> Repo.all()
|> Enum.reject(&is_nil/1)
rescue
_ -> []

3
lib/towerops_web/live/dashboard_live.ex
View file

@ -82,8 +82,7 @@ defmodule ToweropsWeb.DashboardLive do
end
@impl true
def handle_info({event, _device_id, _alert_type}, socket)
when event in [:new_alert, :alert_resolved] do
def handle_info({event, _device_id, _alert_type}, socket) when event in [:new_alert, :alert_resolved] do
{:noreply, schedule_debounced_dashboard_reload(socket)}
end

23
lib/towerops_web/live/network_map_live.html.heex
View file

@ -255,8 +255,8 @@
</div>
</div>
</div>
<!-- Network Map Container -->
<!-- Network Map Container -->
<div class="bg-white dark:bg-gray-800 rounded-lg border border-gray-200 dark:border-white/10 overflow-hidden">
<div class="p-4 border-b border-gray-200 dark:border-white/10">
<div class="flex items-center justify-between">
@ -401,8 +401,8 @@
</div>
<% end %>
</div>
<!-- Wireless Stats (for APs, CPEs, backhaul radios) -->
<!-- Wireless Stats (for APs, CPEs, backhaul radios) -->
<%= if @selected_node_detail[:is_wireless] do %>
<div class="mt-4 pt-3 border-t border-gray-200 dark:border-white/10">
<h4 class="text-xs font-medium text-gray-500 dark:text-gray-400 uppercase tracking-wider mb-2">
@ -439,9 +439,14 @@
<div class={[
"text-sm font-semibold",
cond do
@selected_node_detail.snr >= 25 -> "text-green-600 dark:text-green-400"
@selected_node_detail.snr >= 15 -> "text-yellow-600 dark:text-yellow-400"
true -> "text-red-600 dark:text-red-400"
@selected_node_detail.snr >= 25 ->
"text-green-600 dark:text-green-400"
@selected_node_detail.snr >= 15 ->
"text-yellow-600 dark:text-yellow-400"
true ->
"text-red-600 dark:text-red-400"
end
]}>
{@selected_node_detail.snr} dB
@ -587,7 +592,9 @@
<span class="text-gray-700 dark:text-gray-300">MAC/ARP inferred</span>
</div>
<div class="flex items-center space-x-1.5">
<span class="text-gray-500 dark:text-gray-400">Line thickness = bandwidth capacity</span>
<span class="text-gray-500 dark:text-gray-400">
Line thickness = bandwidth capacity
</span>
</div>
</div>
</div>

32
priv/repo/migrations/20260315190000_add_alert_storm_correlation.exs
View file

@ -5,7 +5,10 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do
# Alert storms track when alert rate exceeds threshold for an org
create table(:alert_storms, primary_key: false) do
add :id, :binary_id, primary_key: true
add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), null: false
add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all),
null: false
add :started_at, :utc_datetime, null: false
add :ended_at, :utc_datetime
add :alert_count, :integer, null: false, default: 0
@ -16,13 +19,20 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do
end
create index(:alert_storms, [:organization_id, :started_at])
create index(:alert_storms, [:organization_id], where: "ended_at IS NULL", name: :alert_storms_active_idx)
create index(:alert_storms, [:organization_id],
where: "ended_at IS NULL",
name: :alert_storms_active_idx
)
# Site outages correlate multiple device-down alerts at the same site
create table(:site_outages, primary_key: false) do
add :id, :binary_id, primary_key: true
add :site_id, references(:sites, type: :binary_id, on_delete: :delete_all), null: false
add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), null: false
add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all),
null: false
add :started_at, :utc_datetime, null: false
add :resolved_at, :utc_datetime
add :device_count, :integer, null: false, default: 0
@ -33,7 +43,11 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do
timestamps(type: :utc_datetime)
end
create index(:site_outages, [:site_id], where: "resolved_at IS NULL", name: :site_outages_active_idx)
create index(:site_outages, [:site_id],
where: "resolved_at IS NULL",
name: :site_outages_active_idx
)
create index(:site_outages, [:organization_id, :started_at])
# Link alerts to their correlated site outage (optional)
@ -48,7 +62,10 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do
create table(:notification_digests, primary_key: false) do
add :id, :binary_id, primary_key: true
add :user_id, references(:users, type: :binary_id, on_delete: :delete_all), null: false
add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all), null: false
add :organization_id, references(:organizations, type: :binary_id, on_delete: :delete_all),
null: false
add :window_start, :utc_datetime, null: false
add :notification_count, :integer, null: false, default: 0
add :digest_sent, :boolean, null: false, default: false
@ -59,6 +76,9 @@ defmodule Towerops.Repo.Migrations.AddAlertStormCorrelation do
end
create index(:notification_digests, [:user_id, :window_start])
create unique_index(:notification_digests, [:user_id, :window_start], name: :notification_digests_user_window_idx)
create unique_index(:notification_digests, [:user_id, :window_start],
name: :notification_digests_user_window_idx
)
end
end

4
test/towerops/alerts/notification_rate_limiter_test.exs
View file

@ -60,7 +60,7 @@ defmodule Towerops.Alerts.NotificationRateLimiterTest do
NotificationRateLimiter.check_rate(user.id, org.id, alert_id)
digest = NotificationRateLimiter.get_pending_digest(user.id)
assert digest != nil
assert digest
assert alert_id in digest.suppressed_alert_ids
end
end
@ -77,7 +77,7 @@ defmodule Towerops.Alerts.NotificationRateLimiterTest do
digest = NotificationRateLimiter.get_pending_digest(user.id)
assert {:ok, updated} = NotificationRateLimiter.mark_digest_sent(digest)
assert updated.digest_sent == true
assert updated.digest_sent_at != nil
assert updated.digest_sent_at
# Should no longer appear as pending
assert is_nil(NotificationRateLimiter.get_pending_digest(user.id))

3
test/towerops/alerts/storm_detector_test.exs
View file

@ -7,6 +7,7 @@ defmodule Towerops.Alerts.StormDetectorTest do
# Stop the globally started detector (if running) and start a fresh one
if Process.whereis(StormDetector), do: GenServer.stop(StormDetector)
{:ok, pid} = StormDetector.start_link(threshold: 3, window_seconds: 10)
on_exit(fn ->
if Process.alive?(pid), do: GenServer.stop(pid)
end)
@ -65,7 +66,7 @@ defmodule Towerops.Alerts.StormDetectorTest do
end
storm = StormDetector.get_storm(org.id)
assert storm != nil
assert storm
end
end
end