From bb4ff232e3a3db1ee8637cad4e8917ea72dc12a9 Mon Sep 17 00:00:00 2001 From: Graham McIntie Date: Sat, 14 Feb 2026 11:18:04 -0600 Subject: [PATCH] feat: add PagerDuty Events API v2 integration with 2-way alert sync --- lib/towerops/alerts.ex | 38 ++++-- lib/towerops/integrations/integration.ex | 2 +- lib/towerops/pagerduty/client.ex | 112 ++++++++++++++++++ lib/towerops/pagerduty/notifier.ex | 84 +++++++++++++ lib/towerops/workers/device_monitor_worker.ex | 8 +- lib/towerops_web/live/help_live/index.ex | 77 ++++++++++++ .../live/org/integrations_live.ex | 7 ++ lib/towerops_web/live/org/settings_live.ex | 11 ++ .../live/org/settings_live.html.heex | 39 +++--- test/towerops/pagerduty/client_test.exs | 45 +++++++ 10 files changed, 397 insertions(+), 26 deletions(-) create mode 100644 lib/towerops/pagerduty/client.ex create mode 100644 lib/towerops/pagerduty/notifier.ex create mode 100644 test/towerops/pagerduty/client_test.exs diff --git a/lib/towerops/alerts.ex b/lib/towerops/alerts.ex index d2069274..b2f4d717 100644 --- a/lib/towerops/alerts.ex +++ b/lib/towerops/alerts.ex @@ -215,21 +215,41 @@ defmodule Towerops.Alerts do Acknowledges an alert. """ def acknowledge_alert(alert, user_id) do - alert - |> Alert.changeset(%{ - acknowledged_at: DateTime.truncate(DateTime.utc_now(), :second), - acknowledged_by_id: user_id - }) - |> Repo.update() + result = + alert + |> Alert.changeset(%{ + acknowledged_at: DateTime.truncate(DateTime.utc_now(), :second), + acknowledged_by_id: user_id + }) + |> Repo.update() + + case result do + {:ok, updated_alert} -> + Task.start(fn -> Towerops.PagerDuty.Notifier.notify_acknowledge(updated_alert) end) + {:ok, updated_alert} + + error -> + error + end end @doc """ Resolves an alert. """ def resolve_alert(alert) do - alert - |> Alert.changeset(%{resolved_at: DateTime.truncate(DateTime.utc_now(), :second)}) - |> Repo.update() + result = + alert + |> Alert.changeset(%{resolved_at: DateTime.truncate(DateTime.utc_now(), :second)}) + |> Repo.update() + + case result do + {:ok, updated_alert} -> + Task.start(fn -> Towerops.PagerDuty.Notifier.notify_resolve(updated_alert) end) + {:ok, updated_alert} + + error -> + error + end end @doc """ diff --git a/lib/towerops/integrations/integration.ex b/lib/towerops/integrations/integration.ex index cd95a041..f40b480f 100644 --- a/lib/towerops/integrations/integration.ex +++ b/lib/towerops/integrations/integration.ex @@ -14,7 +14,7 @@ defmodule Towerops.Integrations.Integration do @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id - @valid_providers ~w(preseem gaiia) + @valid_providers ~w(preseem gaiia pagerduty) @valid_sync_statuses ~w(never success partial failed) schema "integrations" do diff --git a/lib/towerops/pagerduty/client.ex b/lib/towerops/pagerduty/client.ex new file mode 100644 index 00000000..42e7564a --- /dev/null +++ b/lib/towerops/pagerduty/client.ex @@ -0,0 +1,112 @@ +defmodule Towerops.PagerDuty.Client do + @moduledoc "PagerDuty Events API v2 client." + + require Logger + + @events_url "https://events.pagerduty.com/v2/enqueue" + + def trigger(routing_key, alert, device, site_name, towerops_url) do + send_event(%{ + routing_key: routing_key, + event_action: "trigger", + dedup_key: dedup_key(alert), + payload: %{ + summary: alert_summary(alert, device), + source: "towerops", + severity: severity(alert.alert_type), + timestamp: DateTime.to_iso8601(alert.triggered_at), + component: device.hostname || device.ip_address, + group: site_name, + class: to_string(alert.alert_type), + custom_details: %{ + device_id: device.id, + device_name: device.hostname || device.ip_address, + ip_address: device.ip_address, + site_name: site_name, + alert_type: to_string(alert.alert_type), + message: alert.message, + towerops_url: towerops_url + } + }, + links: [%{href: towerops_url, text: "View in TowerOps"}] + }) + end + + def acknowledge(routing_key, alert) do + send_event(%{ + routing_key: routing_key, + event_action: "acknowledge", + dedup_key: dedup_key(alert) + }) + end + + def resolve(routing_key, alert) do + send_event(%{ + routing_key: routing_key, + event_action: "resolve", + dedup_key: dedup_key(alert) + }) + end + + def test_connection(routing_key) do + test_dedup = "towerops-test-#{System.unique_integer([:positive])}" + + case send_event(%{ + routing_key: routing_key, + event_action: "trigger", + dedup_key: test_dedup, + payload: %{ + summary: "TowerOps integration test", + source: "towerops", + severity: "info", + timestamp: DateTime.to_iso8601(DateTime.utc_now()) + } + }) do + {:ok, _} -> + send_event(%{ + routing_key: routing_key, + event_action: "resolve", + dedup_key: test_dedup + }) + + error -> + error + end + end + + defp send_event(body) do + case Req.post(@events_url, json: body) do + {:ok, %{status: status, body: resp_body}} when status in [200, 202] -> + {:ok, resp_body} + + {:ok, %{status: 400, body: resp_body}} -> + {:error, {:bad_request, resp_body["message"] || "Invalid event"}} + + {:ok, %{status: 429}} -> + {:error, :rate_limited} + + {:ok, %{status: status}} -> + {:error, {:unexpected_status, status}} + + {:error, error} -> + Logger.warning("PagerDuty API error: #{inspect(error)}") + {:error, error} + end + end + + defp dedup_key(alert), do: "towerops-alert-#{alert.id}" + + defp alert_summary(alert, device) do + name = device.hostname || device.ip_address + + case alert.alert_type do + :device_down -> "Device down: #{name}" + :device_up -> "Device recovered: #{name}" + other -> "Alert: #{other} on #{name}" + end + end + + defp severity(:device_down), do: "critical" + defp severity(:device_up), do: "info" + defp severity(_), do: "warning" +end diff --git a/lib/towerops/pagerduty/notifier.ex b/lib/towerops/pagerduty/notifier.ex new file mode 100644 index 00000000..4909dccb --- /dev/null +++ b/lib/towerops/pagerduty/notifier.ex @@ -0,0 +1,84 @@ +defmodule Towerops.PagerDuty.Notifier do + @moduledoc "Sends alert events to PagerDuty when integration is configured." + + alias Towerops.Integrations + alias Towerops.PagerDuty.Client + alias Towerops.Repo + + require Logger + + def notify_trigger(alert, device) do + with {:ok, routing_key} <- get_routing_key(device.organization_id) do + site_name = get_site_name(device) + towerops_url = device_url(device) + + case Client.trigger(routing_key, alert, device, site_name, towerops_url) do + {:ok, _} -> + Logger.info("PagerDuty incident triggered for alert #{alert.id}") + :ok + + {:error, reason} -> + Logger.warning("PagerDuty trigger failed for alert #{alert.id}: #{inspect(reason)}") + :error + end + end + end + + def notify_acknowledge(alert) do + alert = Repo.preload(alert, device: :site) + + with {:ok, routing_key} <- get_routing_key(alert.device.organization_id) do + case Client.acknowledge(routing_key, alert) do + {:ok, _} -> + Logger.info("PagerDuty incident acknowledged for alert #{alert.id}") + :ok + + {:error, reason} -> + Logger.warning( + "PagerDuty acknowledge failed for alert #{alert.id}: #{inspect(reason)}" + ) + + :error + end + end + end + + def notify_resolve(alert) do + alert = Repo.preload(alert, device: :site) + + with {:ok, routing_key} <- get_routing_key(alert.device.organization_id) do + case Client.resolve(routing_key, alert) do + {:ok, _} -> + Logger.info("PagerDuty incident resolved for alert #{alert.id}") + :ok + + {:error, reason} -> + Logger.warning("PagerDuty resolve failed for alert #{alert.id}: #{inspect(reason)}") + :error + end + end + end + + defp get_routing_key(organization_id) do + case Integrations.get_integration(organization_id, "pagerduty") do + {:ok, %{enabled: true, credentials: %{"api_key" => key}}} when key != "" -> + {:ok, key} + + _ -> + {:error, :not_configured} + end + end + + defp get_site_name(device) do + device = Repo.preload(device, :site) + + case device.site do + nil -> "Unknown Site" + site -> site.name + end + end + + defp device_url(device) do + ToweropsWeb.Endpoint.url() <> "/devices/#{device.id}" + end +end diff --git a/lib/towerops/workers/device_monitor_worker.ex b/lib/towerops/workers/device_monitor_worker.ex index e88bf8f6..ba25d396 100644 --- a/lib/towerops/workers/device_monitor_worker.ex +++ b/lib/towerops/workers/device_monitor_worker.ex @@ -202,7 +202,7 @@ defmodule Towerops.Workers.DeviceMonitorWorker do defp create_device_down_alert(device, now) do alert_message = get_down_alert_message(device) - {:ok, _alert} = + {:ok, alert} = Alerts.create_alert(%{ device_id: device.id, alert_type: :device_down, @@ -210,6 +210,8 @@ defmodule Towerops.Workers.DeviceMonitorWorker do message: alert_message }) + Task.start(fn -> Towerops.PagerDuty.Notifier.notify_trigger(alert, device) end) + _ = Phoenix.PubSub.broadcast( Towerops.PubSub, @@ -229,7 +231,7 @@ defmodule Towerops.Workers.DeviceMonitorWorker do defp handle_equipment_up(device, now) do recovery_message = get_recovery_message(device) - {:ok, _alert} = + {:ok, alert} = Alerts.create_alert(%{ device_id: device.id, alert_type: :device_up, @@ -237,6 +239,8 @@ defmodule Towerops.Workers.DeviceMonitorWorker do message: recovery_message }) + Task.start(fn -> Towerops.PagerDuty.Notifier.notify_trigger(alert, device) end) + resolve_down_alert(device) _ = diff --git a/lib/towerops_web/live/help_live/index.ex b/lib/towerops_web/live/help_live/index.ex index 8c2c7b20..c9dd1f0d 100644 --- a/lib/towerops_web/live/help_live/index.ex +++ b/lib/towerops_web/live/help_live/index.ex @@ -1756,6 +1756,83 @@ defmodule ToweropsWeb.HelpLive.Index do +

+ PagerDuty +

+ +

+ PagerDuty + is an incident management and on-call alerting platform. The PagerDuty integration provides + 2-way alert sync — when a device goes down in Towerops, a PagerDuty incident is automatically + triggered. Acknowledging or resolving the alert in Towerops updates the PagerDuty incident as well. +

+ +

+ How It Works +

+ + + +

+ Configuration +

+ +
+
+
+
+ 1 +
+
+
+

+ Create an Events API v2 Integration in PagerDuty +

+

+ In PagerDuty, go to <.code>Services → select your service → <.code>Integrations tab → <.code>Add Integration → choose <.code>Events API v2. Copy the <.code>Integration Key (routing key). +

+
+
+ +
+
+
+ 2 +
+
+
+

+ Configure in Towerops +

+

+ Navigate to <.code>Organization Settings → <.code>Integrations tab → click <.code>Configure on PagerDuty. Paste your integration key and test the connection. +

+
+
+
+ +
+
+ <.icon + name="hero-information-circle" + class="h-5 w-5 text-blue-600 dark:text-blue-400 flex-shrink-0 mt-0.5" + /> +
+

+ Event-Driven +

+

+ Unlike other integrations, PagerDuty is event-driven — there is no periodic sync. + Alerts are sent to PagerDuty in real time as they occur. No polling or sync interval is needed. +

+
+
+
+

General Integration Features

diff --git a/lib/towerops_web/live/org/integrations_live.ex b/lib/towerops_web/live/org/integrations_live.ex index 1df6bc6f..c5218c3c 100644 --- a/lib/towerops_web/live/org/integrations_live.ex +++ b/lib/towerops_web/live/org/integrations_live.ex @@ -22,6 +22,13 @@ defmodule ToweropsWeb.Org.IntegrationsLive do description: "Billing and subscriber management. Enables outage impact analysis, inventory reconciliation, and subscriber-aware monitoring.", icon: "hero-user-group" + }, + %{ + id: "pagerduty", + name: "PagerDuty", + description: + "Incident management and on-call alerting. Automatically triggers, acknowledges, and resolves PagerDuty incidents from TowerOps alerts.", + icon: "hero-bell-alert" } ] diff --git a/lib/towerops_web/live/org/settings_live.ex b/lib/towerops_web/live/org/settings_live.ex index 3576b410..de2427e3 100644 --- a/lib/towerops_web/live/org/settings_live.ex +++ b/lib/towerops_web/live/org/settings_live.ex @@ -26,6 +26,13 @@ defmodule ToweropsWeb.Org.SettingsLive do description: "Billing and subscriber management. Enables outage impact analysis, inventory reconciliation, and subscriber-aware monitoring.", icon: "hero-user-group" + }, + %{ + id: "pagerduty", + name: "PagerDuty", + description: + "Incident management and on-call alerting. Automatically triggers, acknowledges, and resolves PagerDuty incidents from TowerOps alerts.", + icon: "hero-bell-alert" } ] @@ -395,6 +402,10 @@ defmodule ToweropsWeb.Org.SettingsLive do defp test_provider_connection("preseem", api_key), do: PreseemClient.test_connection(api_key) defp test_provider_connection("gaiia", api_key), do: GaiiaClient.test_connection(api_key) + + defp test_provider_connection("pagerduty", api_key), + do: Towerops.PagerDuty.Client.test_connection(api_key) + defp test_provider_connection(_, _api_key), do: {:error, "Unknown provider"} defp load_integrations(organization_id) do diff --git a/lib/towerops_web/live/org/settings_live.html.heex b/lib/towerops_web/live/org/settings_live.html.heex index 5d61d02b..774ffa86 100644 --- a/lib/towerops_web/live/org/settings_live.html.heex +++ b/lib/towerops_web/live/org/settings_live.html.heex @@ -840,6 +840,9 @@ <%= if provider.id == "gaiia" do %> Syncs every {integration.sync_interval_minutes} minutes. Reconciliation runs nightly. Also receives real-time webhook updates. <% end %> + <%= if provider.id == "pagerduty" do %> + Event-driven — alerts are sent to PagerDuty in real time when devices go down, are acknowledged, or recover. + <% end %>

<% end %> @@ -894,24 +897,32 @@ <.input field={@integration_form[:api_key]} type="password" - label="API Key" - placeholder={"Enter your #{provider.name} API key"} + label={if(provider.id == "pagerduty", do: "Integration Key (Routing Key)", else: "API Key")} + placeholder={if(provider.id == "pagerduty", do: "Enter your PagerDuty integration key", else: "Enter your #{provider.name} API key")} autocomplete="off" value={get_credential(@integrations[provider.id], "api_key")} /> - <.input - field={@integration_form[:sync_interval_minutes]} - type="number" - label="Sync interval (minutes)" - min="5" - value={ - if(@integrations[provider.id], - do: @integrations[provider.id].sync_interval_minutes, - else: if(provider.id == "gaiia", do: 15, else: 10) - ) - } - /> + <%= if provider.id == "pagerduty" do %> +

+ Find this in PagerDuty → Services → Your Service → Integrations → Events API v2 +

+ <% end %> + + <%= if provider.id != "pagerduty" do %> + <.input + field={@integration_form[:sync_interval_minutes]} + type="number" + label="Sync interval (minutes)" + min="5" + value={ + if(@integrations[provider.id], + do: @integrations[provider.id].sync_interval_minutes, + else: if(provider.id == "gaiia", do: 15, else: 10) + ) + } + /> + <% end %> <%= if @test_result do %>