towerops/lib/towerops_web/controllers/api/v1/pagerduty_webhook_controller.ex
Graham McIntire 3b0961dfc0 refactor(webhooks): enqueue Oban jobs immediately, ACK fast
All 4 webhook controllers now verify the signature, insert an Oban job,
and return immediately. Previously, Stripe and PagerDuty did DB queries +
writes inline; Gaiia ran upserts; Agent Release broadcast to all connected
WebSockets — all in the request path.

New workers:
- StripeWebhookWorker — delegates to WebhookProcessor.process/1
- GaiiaWebhookWorker — delegates to Webhooks.process_event/3
- PagerdutyWebhookWorker — resolve/acknowledge alerts by dedup key
- AgentReleaseWebhookWorker — broadcast mass update to agents

Also fix: log error when resolve_alert fails in agent_channel (was silently
discarded, causing "Resolving stuck device_down alert" to repeat in logs).
2026-05-10 17:18:23 -05:00

130 lines
4 KiB
Elixir

defmodule ToweropsWeb.Api.V1.PagerdutyWebhookController do
@moduledoc """
Handles inbound PagerDuty V3 webhook events.
Verifies the signature, enqueues an Oban job, and ACKs immediately.
All processing happens in `Towerops.Workers.PagerdutyWebhookWorker`.
"""
use ToweropsWeb, :controller
alias Towerops.Integrations
alias Towerops.Workers.PagerdutyWebhookWorker
require Logger
def create(conn, %{"organization_id" => organization_id} = params) do
with {:ok, raw_body} <- get_raw_body(conn),
{:ok, integration} <- get_pagerduty_integration(organization_id),
:ok <- validate_organization_match(integration, organization_id),
:ok <- verify_signature(conn, raw_body, integration),
{:ok, event_type, data} <- extract_event(params) do
case Oban.insert(
PagerdutyWebhookWorker.new(%{
organization_id: organization_id,
event_type: event_type,
data: data
})
) do
{:ok, _job} ->
json(conn, %{status: "accepted"})
{:error, changeset} ->
Logger.error("Failed to enqueue PagerDuty webhook: #{inspect(changeset.errors)}")
conn |> put_status(500) |> json(%{error: "Failed to enqueue"})
end
else
{:error, :no_body} ->
conn |> put_status(400) |> json(%{error: "Missing body"})
{:error, :organization_mismatch} ->
conn |> put_status(403) |> json(%{error: "Organization mismatch"})
{:error, :not_configured} ->
conn |> put_status(404) |> json(%{error: "Integration not found"})
{:error, :invalid_signature} ->
conn |> put_status(401) |> json(%{error: "Invalid signature"})
{:error, :no_secret_configured} ->
conn |> put_status(403) |> json(%{error: "Webhook secret not configured"})
{:error, :unhandled_event} ->
json(conn, %{status: "accepted"})
{:error, _reason} ->
conn |> put_status(500) |> json(%{error: "Internal error"})
end
end
defp extract_event(%{"event" => %{"event_type" => event_type, "data" => data}}) do
case event_type do
type when type in ~w(incident.resolved incident.acknowledged) ->
{:ok, type, data}
_ ->
{:error, :unhandled_event}
end
end
defp extract_event(_), do: {:error, :unhandled_event}
defp get_raw_body(conn) do
case conn.private[:raw_body] do
body when is_binary(body) and body != "" -> {:ok, body}
_ -> {:error, :no_body}
end
end
defp get_pagerduty_integration(organization_id) do
case Integrations.get_integration(organization_id, "pagerduty") do
{:ok, %{enabled: true} = integration} -> {:ok, integration}
_ -> {:error, :not_configured}
end
end
defp validate_organization_match(integration, url_organization_id) do
if integration.organization_id == url_organization_id do
:ok
else
{:error, :organization_mismatch}
end
end
defp verify_signature(conn, raw_body, integration) do
webhook_secret = get_in(integration.credentials, ["webhook_secret"])
if is_nil(webhook_secret) or webhook_secret == "" do
Logger.warning("PagerDuty webhook: rejected - no webhook secret configured")
{:error, :no_secret_configured}
else
check_pagerduty_hmac(conn, raw_body, webhook_secret, integration.organization_id)
end
end
defp check_pagerduty_hmac(conn, raw_body, secret, org_id) do
signatures =
conn
|> Plug.Conn.get_req_header("x-pagerduty-signature")
|> Enum.flat_map(&String.split(&1, ","))
|> Enum.map(&String.trim/1)
expected =
:hmac
|> :crypto.mac(:sha256, secret, raw_body)
|> Base.encode16(case: :lower)
if signature_matches?(signatures, expected) do
:ok
else
Logger.warning("PagerDuty webhook signature mismatch for org #{org_id}")
{:error, :invalid_signature}
end
end
defp signature_matches?(signatures, expected) do
Enum.any?(signatures, fn
"v1=" <> hex -> Plug.Crypto.secure_compare(hex, expected)
_ -> false
end)
end
end