From 7b72ed316401c04e030cb58aaf672c0f7a5f268f Mon Sep 17 00:00:00 2001 From: Graham McIntie Date: Sat, 14 Feb 2026 14:58:58 -0600 Subject: [PATCH] PagerDuty two-way sync: webhook receiver for resolve/acknowledge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - POST /api/v1/webhooks/pagerduty/:organization_id endpoint - HMAC-SHA256 signature verification (optional, via webhook_secret) - incident.resolved → resolve_alert_silent (no re-notify to PagerDuty) - incident.acknowledged → acknowledge_alert_silent (no re-notify) - Matches alerts via dedup_key format: towerops-alert- - Webhook secret configurable in org settings PagerDuty panel - Setup instructions in UI for PagerDuty Generic Webhooks (v3) --- lib/towerops/alerts.ex | 18 ++ .../api/v1/pagerduty_webhook_controller.ex | 155 ++++++++++++++++++ lib/towerops_web/live/org/settings_live.ex | 13 +- .../live/org/settings_live.html.heex | 55 +++++++ lib/towerops_web/router.ex | 1 + 5 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 lib/towerops_web/controllers/api/v1/pagerduty_webhook_controller.ex diff --git a/lib/towerops/alerts.ex b/lib/towerops/alerts.ex index 2fe32ab0..7e1b939b 100644 --- a/lib/towerops/alerts.ex +++ b/lib/towerops/alerts.ex @@ -253,6 +253,24 @@ defmodule Towerops.Alerts do end end + @doc """ + Resolves an alert without notifying PagerDuty (used when PagerDuty is the source). + """ + def resolve_alert_silent(alert) do + alert + |> Alert.changeset(%{resolved_at: DateTime.truncate(DateTime.utc_now(), :second)}) + |> Repo.update() + end + + @doc """ + Acknowledges an alert without notifying PagerDuty (used when PagerDuty is the source). + """ + def acknowledge_alert_silent(alert) do + alert + |> Alert.changeset(%{acknowledged_at: DateTime.truncate(DateTime.utc_now(), :second)}) + |> Repo.update() + end + @doc """ Stores Gaiia impact analysis data on an alert record. """ diff --git a/lib/towerops_web/controllers/api/v1/pagerduty_webhook_controller.ex b/lib/towerops_web/controllers/api/v1/pagerduty_webhook_controller.ex new file mode 100644 index 00000000..d5883abb --- /dev/null +++ b/lib/towerops_web/controllers/api/v1/pagerduty_webhook_controller.ex @@ -0,0 +1,155 @@ +defmodule ToweropsWeb.Api.V1.PagerdutyWebhookController do + @moduledoc """ + Handles inbound PagerDuty V3 webhook events. + + When an incident is resolved or acknowledged in PagerDuty, this controller + updates the corresponding TowerOps alert so we don't re-alert on it. + + PagerDuty signs webhooks with HMAC-SHA256 using a per-integration secret. + The signature is in the `X-PagerDuty-Signature` header as `v1=`. + """ + use ToweropsWeb, :controller + + alias Towerops.Alerts + alias Towerops.Integrations + + require Logger + + def create(conn, %{"organization_id" => organization_id} = params) do + with {:ok, raw_body} <- get_raw_body(conn), + {:ok, integration} <- get_pagerduty_integration(organization_id), + :ok <- verify_signature(conn, raw_body, integration) do + # params already parsed by Phoenix JSON parser + process_webhook(params, organization_id) + json(conn, %{status: "accepted"}) + else + {:error, :not_configured} -> + conn |> put_status(404) |> json(%{error: "Integration not found"}) + + {:error, :invalid_signature} -> + conn |> put_status(401) |> json(%{error: "Invalid signature"}) + + {:error, _reason} -> + conn |> put_status(500) |> json(%{error: "Internal error"}) + end + end + + defp get_raw_body(conn) do + case conn.private[:raw_body] do + body when is_binary(body) and body != "" -> {:ok, body} + _ -> {:error, :no_body} + end + end + + defp get_pagerduty_integration(organization_id) do + case Integrations.get_integration(organization_id, "pagerduty") do + {:ok, %{enabled: true} = integration} -> {:ok, integration} + _ -> {:error, :not_configured} + end + end + + defp verify_signature(conn, raw_body, integration) do + webhook_secret = get_in(integration.credentials, ["webhook_secret"]) + + if is_nil(webhook_secret) or webhook_secret == "" do + # No webhook secret configured — accept without verification + # (user hasn't set up the V3 webhook extension yet) + :ok + else + signatures = + conn + |> Plug.Conn.get_req_header("x-pagerduty-signature") + |> Enum.flat_map(&String.split(&1, ",")) + |> Enum.map(&String.trim/1) + + expected = + :hmac + |> :crypto.mac(:sha256, webhook_secret, raw_body) + |> Base.encode16(case: :lower) + + if Enum.any?(signatures, fn sig -> + case sig do + "v1=" <> hex -> Plug.Crypto.secure_compare(hex, expected) + _ -> false + end + end) do + :ok + else + Logger.warning("PagerDuty webhook signature mismatch for org #{integration.organization_id}") + + {:error, :invalid_signature} + end + end + end + + defp process_webhook(%{"event" => %{"event_type" => event_type, "data" => data}}, organization_id) do + case event_type do + "incident.resolved" -> + resolve_from_pagerduty(data, organization_id) + + "incident.acknowledged" -> + acknowledge_from_pagerduty(data, organization_id) + + other -> + Logger.debug("Ignoring PagerDuty webhook event: #{other}") + :ok + end + end + + defp process_webhook(_payload, _organization_id), do: :ok + + defp resolve_from_pagerduty(data, _organization_id) do + case extract_alert_id(data) do + nil -> + Logger.debug("PagerDuty resolve: no TowerOps alert ID found in incident") + :ok + + alert_id -> + case Alerts.get_alert(alert_id) do + nil -> + Logger.warning("PagerDuty resolve: alert #{alert_id} not found") + + %{resolved_at: resolved_at} when not is_nil(resolved_at) -> + Logger.debug("PagerDuty resolve: alert #{alert_id} already resolved") + + alert -> + Logger.info("Resolving alert #{alert_id} from PagerDuty") + Alerts.resolve_alert_silent(alert) + end + end + end + + defp acknowledge_from_pagerduty(data, _organization_id) do + case extract_alert_id(data) do + nil -> + :ok + + alert_id -> + case Alerts.get_alert(alert_id) do + nil -> + Logger.warning("PagerDuty acknowledge: alert #{alert_id} not found") + + %{acknowledged_at: ack} when not is_nil(ack) -> + Logger.debug("PagerDuty acknowledge: alert #{alert_id} already acknowledged") + + alert -> + Logger.info("Acknowledging alert #{alert_id} from PagerDuty") + Alerts.acknowledge_alert_silent(alert) + end + end + end + + # PagerDuty includes our dedup_key as the alert key. + # Our dedup_key format is "towerops-alert-" + defp extract_alert_id(data) do + # Try the first alert's dedup key + alerts = data["alerts"] || [] + + Enum.find_value(alerts, fn alert -> + case alert["alert_key"] do + "towerops-alert-" <> id -> id + _ -> nil + end + end) + end +end diff --git a/lib/towerops_web/live/org/settings_live.ex b/lib/towerops_web/live/org/settings_live.ex index 6981c7fd..aec61e1f 100644 --- a/lib/towerops_web/live/org/settings_live.ex +++ b/lib/towerops_web/live/org/settings_live.ex @@ -507,13 +507,20 @@ defmodule ToweropsWeb.Org.SettingsLive do api_key = Map.get(params, "api_key", "") sync_interval = Map.get(params, "sync_interval_minutes") + # Use form value if provided, otherwise preserve existing webhook_secret = - case existing_integration do - %Integration{credentials: %{"webhook_secret" => secret}} when is_binary(secret) -> + case Map.get(params, "webhook_secret") do + secret when is_binary(secret) and secret != "" -> secret _ -> - "" + case existing_integration do + %Integration{credentials: %{"webhook_secret" => secret}} when is_binary(secret) -> + secret + + _ -> + "" + end end attrs = %{credentials: %{"api_key" => api_key, "webhook_secret" => webhook_secret}} diff --git a/lib/towerops_web/live/org/settings_live.html.heex b/lib/towerops_web/live/org/settings_live.html.heex index 1700df20..9d79fb85 100644 --- a/lib/towerops_web/live/org/settings_live.html.heex +++ b/lib/towerops_web/live/org/settings_live.html.heex @@ -1502,6 +1502,61 @@ When connected, TowerOps will automatically trigger, acknowledge, and resolve PagerDuty incidents in sync with your alerts.

+ +
+
+ <.icon name="hero-arrow-path" class="h-4 w-4 text-gray-400" /> +

+ Two-Way Sync (Webhooks) +

+
+

+ When someone resolves or acknowledges an incident directly in PagerDuty, + TowerOps will automatically update the corresponding alert. +

+
+ +
+ + {ToweropsWeb.Endpoint.url()}/api/v1/webhooks/pagerduty/{@current_scope.organization.id} + +
+
+ <.input + name="integration[webhook_secret]" + type="password" + label="Webhook Signing Secret" + value={ + if(@integrations[provider.id], + do: + get_in(@integrations[provider.id].credentials, ["webhook_secret"]) || + "", + else: "" + ) + } + placeholder="Optional — paste from PagerDuty webhook extension" + /> +
+
    +
  1. + In PagerDuty, go to Integrations + → Generic Webhooks (v3) +
    2. +
  2. + Add a subscription with the Webhook URL above +
    3. +
  3. + Select events: incident.resolved + and incident.acknowledged +
    4. +
  4. + Copy the Signing Secret and paste it above +
    5. +
+
+
<% end %> <%= if provider.id != "pagerduty" do %> diff --git a/lib/towerops_web/router.ex b/lib/towerops_web/router.ex index 3e456d2b..eb98cc95 100644 --- a/lib/towerops_web/router.ex +++ b/lib/towerops_web/router.ex @@ -175,6 +175,7 @@ defmodule ToweropsWeb.Router do pipe_through [:api, :rate_limit_api] post "/gaiia/:organization_id", GaiiaWebhookController, :create + post "/pagerduty/:organization_id", PagerdutyWebhookController, :create end # Admin API routes (requires superuser API token)