PagerDuty two-way sync: webhook receiver for resolve/acknowledge

- POST /api/v1/webhooks/pagerduty/:organization_id endpoint
- HMAC-SHA256 signature verification (optional, via webhook_secret)
- incident.resolved → resolve_alert_silent (no re-notify to PagerDuty)
- incident.acknowledged → acknowledge_alert_silent (no re-notify)
- Matches alerts via dedup_key format: towerops-alert-<uuid>
- Webhook secret configurable in org settings PagerDuty panel
- Setup instructions in UI for PagerDuty Generic Webhooks (v3)

lib/towerops/alerts.ex

@@ -253,6 +253,24 @@ defmodule Towerops.Alerts do
     end
   end
 
+  @doc """
+  Resolves an alert without notifying PagerDuty (used when PagerDuty is the source).
+  """
+  def resolve_alert_silent(alert) do
+    alert
+    |> Alert.changeset(%{resolved_at: DateTime.truncate(DateTime.utc_now(), :second)})
+    |> Repo.update()
+  end
+
+  @doc """
+  Acknowledges an alert without notifying PagerDuty (used when PagerDuty is the source).
+  """
+  def acknowledge_alert_silent(alert) do
+    alert
+    |> Alert.changeset(%{acknowledged_at: DateTime.truncate(DateTime.utc_now(), :second)})
+    |> Repo.update()
+  end
+
   @doc """
   Stores Gaiia impact analysis data on an alert record.
   """

lib/towerops_web/controllers/api/v1/pagerduty_webhook_controller.ex

@@ -0,0 +1,155 @@
+defmodule ToweropsWeb.Api.V1.PagerdutyWebhookController do
+  @moduledoc """
+  Handles inbound PagerDuty V3 webhook events.
+
+  When an incident is resolved or acknowledged in PagerDuty, this controller
+  updates the corresponding TowerOps alert so we don't re-alert on it.
+
+  PagerDuty signs webhooks with HMAC-SHA256 using a per-integration secret.
+  The signature is in the `X-PagerDuty-Signature` header as `v1=<hex_digest>`.
+  """
+  use ToweropsWeb, :controller
+
+  alias Towerops.Alerts
+  alias Towerops.Integrations
+
+  require Logger
+
+  def create(conn, %{"organization_id" => organization_id} = params) do
+    with {:ok, raw_body} <- get_raw_body(conn),
+         {:ok, integration} <- get_pagerduty_integration(organization_id),
+         :ok <- verify_signature(conn, raw_body, integration) do
+      # params already parsed by Phoenix JSON parser
+      process_webhook(params, organization_id)
+      json(conn, %{status: "accepted"})
+    else
+      {:error, :not_configured} ->
+        conn |> put_status(404) |> json(%{error: "Integration not found"})
+
+      {:error, :invalid_signature} ->
+        conn |> put_status(401) |> json(%{error: "Invalid signature"})
+
+      {:error, _reason} ->
+        conn |> put_status(500) |> json(%{error: "Internal error"})
+    end
+  end
+
+  defp get_raw_body(conn) do
+    case conn.private[:raw_body] do
+      body when is_binary(body) and body != "" -> {:ok, body}
+      _ -> {:error, :no_body}
+    end
+  end
+
+  defp get_pagerduty_integration(organization_id) do
+    case Integrations.get_integration(organization_id, "pagerduty") do
+      {:ok, %{enabled: true} = integration} -> {:ok, integration}
+      _ -> {:error, :not_configured}
+    end
+  end
+
+  defp verify_signature(conn, raw_body, integration) do
+    webhook_secret = get_in(integration.credentials, ["webhook_secret"])
+
+    if is_nil(webhook_secret) or webhook_secret == "" do
+      # No webhook secret configured — accept without verification
+      # (user hasn't set up the V3 webhook extension yet)
+      :ok
+    else
+      signatures =
+        conn
+        |> Plug.Conn.get_req_header("x-pagerduty-signature")
+        |> Enum.flat_map(&String.split(&1, ","))
+        |> Enum.map(&String.trim/1)
+
+      expected =
+        :hmac
+        |> :crypto.mac(:sha256, webhook_secret, raw_body)
+        |> Base.encode16(case: :lower)
+
+      if Enum.any?(signatures, fn sig ->
+           case sig do
+             "v1=" <> hex -> Plug.Crypto.secure_compare(hex, expected)
+             _ -> false
+           end
+         end) do
+        :ok
+      else
+        Logger.warning("PagerDuty webhook signature mismatch for org #{integration.organization_id}")
+
+        {:error, :invalid_signature}
+      end
+    end
+  end
+
+  defp process_webhook(%{"event" => %{"event_type" => event_type, "data" => data}}, organization_id) do
+    case event_type do
+      "incident.resolved" ->
+        resolve_from_pagerduty(data, organization_id)
+
+      "incident.acknowledged" ->
+        acknowledge_from_pagerduty(data, organization_id)
+
+      other ->
+        Logger.debug("Ignoring PagerDuty webhook event: #{other}")
+        :ok
+    end
+  end
+
+  defp process_webhook(_payload, _organization_id), do: :ok
+
+  defp resolve_from_pagerduty(data, _organization_id) do
+    case extract_alert_id(data) do
+      nil ->
+        Logger.debug("PagerDuty resolve: no TowerOps alert ID found in incident")
+        :ok
+
+      alert_id ->
+        case Alerts.get_alert(alert_id) do
+          nil ->
+            Logger.warning("PagerDuty resolve: alert #{alert_id} not found")
+
+          %{resolved_at: resolved_at} when not is_nil(resolved_at) ->
+            Logger.debug("PagerDuty resolve: alert #{alert_id} already resolved")
+
+          alert ->
+            Logger.info("Resolving alert #{alert_id} from PagerDuty")
+            Alerts.resolve_alert_silent(alert)
+        end
+    end
+  end
+
+  defp acknowledge_from_pagerduty(data, _organization_id) do
+    case extract_alert_id(data) do
+      nil ->
+        :ok
+
+      alert_id ->
+        case Alerts.get_alert(alert_id) do
+          nil ->
+            Logger.warning("PagerDuty acknowledge: alert #{alert_id} not found")
+
+          %{acknowledged_at: ack} when not is_nil(ack) ->
+            Logger.debug("PagerDuty acknowledge: alert #{alert_id} already acknowledged")
+
+          alert ->
+            Logger.info("Acknowledging alert #{alert_id} from PagerDuty")
+            Alerts.acknowledge_alert_silent(alert)
+        end
+    end
+  end
+
+  # PagerDuty includes our dedup_key as the alert key.
+  # Our dedup_key format is "towerops-alert-<uuid>"
+  defp extract_alert_id(data) do
+    # Try the first alert's dedup key
+    alerts = data["alerts"] || []
+
+    Enum.find_value(alerts, fn alert ->
+      case alert["alert_key"] do
+        "towerops-alert-" <> id -> id
+        _ -> nil
+      end
+    end)
+  end
+end

lib/towerops_web/live/org/settings_live.ex

@@ -507,13 +507,20 @@ defmodule ToweropsWeb.Org.SettingsLive do
     api_key = Map.get(params, "api_key", "")
     sync_interval = Map.get(params, "sync_interval_minutes")
 
+    # Use form value if provided, otherwise preserve existing
     webhook_secret =
-      case existing_integration do
-        %Integration{credentials: %{"webhook_secret" => secret}} when is_binary(secret) ->
+      case Map.get(params, "webhook_secret") do
+        secret when is_binary(secret) and secret != "" ->
           secret
 
         _ ->
-          ""
+          case existing_integration do
+            %Integration{credentials: %{"webhook_secret" => secret}} when is_binary(secret) ->
+              secret
+
+            _ ->
+              ""
+          end
       end
 
     attrs = %{credentials: %{"api_key" => api_key, "webhook_secret" => webhook_secret}}

lib/towerops_web/live/org/settings_live.html.heex

@@ -1502,6 +1502,61 @@
                         When connected, TowerOps will automatically trigger, acknowledge, and resolve PagerDuty incidents in sync with your alerts.
                       </p>
                     </div>
+
+                    <div class="border-t border-gray-200 pt-5 mt-5 dark:border-white/10">
+                      <div class="flex items-center gap-2 mb-4">
+                        <.icon name="hero-arrow-path" class="h-4 w-4 text-gray-400" />
+                        <h4 class="text-sm font-semibold text-gray-900 dark:text-white">
+                          Two-Way Sync (Webhooks)
+                        </h4>
+                      </div>
+                      <p class="text-xs text-gray-500 dark:text-gray-400 mb-3">
+                        When someone resolves or acknowledges an incident directly in PagerDuty,
+                        TowerOps will automatically update the corresponding alert.
+                      </p>
+                      <div class="mb-3">
+                        <label class="block text-xs font-medium text-gray-700 dark:text-gray-300 mb-1">
+                          Webhook URL
+                        </label>
+                        <div class="flex items-center gap-2">
+                          <code class="text-xs bg-gray-100 dark:bg-gray-800 text-gray-800 dark:text-gray-200 px-2.5 py-1.5 rounded-md break-all">
+                            {ToweropsWeb.Endpoint.url()}/api/v1/webhooks/pagerduty/{@current_scope.organization.id}
+                          </code>
+                        </div>
+                      </div>
+                      <.input
+                        name="integration[webhook_secret]"
+                        type="password"
+                        label="Webhook Signing Secret"
+                        value={
+                          if(@integrations[provider.id],
+                            do:
+                              get_in(@integrations[provider.id].credentials, ["webhook_secret"]) ||
+                                "",
+                            else: ""
+                          )
+                        }
+                        placeholder="Optional — paste from PagerDuty webhook extension"
+                      />
+                      <div class="rounded-lg bg-blue-50 dark:bg-blue-900/20 border border-blue-200 dark:border-blue-800/50 p-3 mt-3">
+                        <ol class="list-decimal list-inside space-y-1 text-xs text-blue-800 dark:text-blue-300 ml-1">
+                          <li>
+                            In PagerDuty, go to <strong>Integrations</strong>
+                            → <strong>Generic Webhooks (v3)</strong>
+                          </li>
+                          <li>
+                            Add a subscription with the Webhook URL above
+                          </li>
+                          <li>
+                            Select events: <strong>incident.resolved</strong>
+                            and <strong>incident.acknowledged</strong>
+                          </li>
+                          <li>
+                            Copy the <strong>Signing Secret</strong> and paste it above
+                          </li>
+                        </ol>
+                      </div>
+                    </div>
                   <% end %>
 
                   <%= if provider.id != "pagerduty" do %>

lib/towerops_web/router.ex

@@ -175,6 +175,7 @@ defmodule ToweropsWeb.Router do
     pipe_through [:api, :rate_limit_api]
 
     post "/gaiia/:organization_id", GaiiaWebhookController, :create
+    post "/pagerduty/:organization_id", PagerdutyWebhookController, :create
   end
 
   # Admin API routes (requires superuser API token)