All 4 webhook controllers now verify the signature, insert an Oban job, and return immediately. Previously, Stripe and PagerDuty did DB queries + writes inline; Gaiia ran upserts; Agent Release broadcast to all connected WebSockets — all in the request path. New workers: - StripeWebhookWorker — delegates to WebhookProcessor.process/1 - GaiiaWebhookWorker — delegates to Webhooks.process_event/3 - PagerdutyWebhookWorker — resolve/acknowledge alerts by dedup key - AgentReleaseWebhookWorker — broadcast mass update to agents Also fix: log error when resolve_alert fails in agent_channel (was silently discarded, causing "Resolving stuck device_down alert" to repeat in logs).
130 lines
4 KiB
Elixir
130 lines
4 KiB
Elixir
defmodule ToweropsWeb.Api.V1.PagerdutyWebhookController do
|
|
@moduledoc """
|
|
Handles inbound PagerDuty V3 webhook events.
|
|
|
|
Verifies the signature, enqueues an Oban job, and ACKs immediately.
|
|
All processing happens in `Towerops.Workers.PagerdutyWebhookWorker`.
|
|
"""
|
|
use ToweropsWeb, :controller
|
|
|
|
alias Towerops.Integrations
|
|
alias Towerops.Workers.PagerdutyWebhookWorker
|
|
|
|
require Logger
|
|
|
|
def create(conn, %{"organization_id" => organization_id} = params) do
|
|
with {:ok, raw_body} <- get_raw_body(conn),
|
|
{:ok, integration} <- get_pagerduty_integration(organization_id),
|
|
:ok <- validate_organization_match(integration, organization_id),
|
|
:ok <- verify_signature(conn, raw_body, integration),
|
|
{:ok, event_type, data} <- extract_event(params) do
|
|
case Oban.insert(
|
|
PagerdutyWebhookWorker.new(%{
|
|
organization_id: organization_id,
|
|
event_type: event_type,
|
|
data: data
|
|
})
|
|
) do
|
|
{:ok, _job} ->
|
|
json(conn, %{status: "accepted"})
|
|
|
|
{:error, changeset} ->
|
|
Logger.error("Failed to enqueue PagerDuty webhook: #{inspect(changeset.errors)}")
|
|
conn |> put_status(500) |> json(%{error: "Failed to enqueue"})
|
|
end
|
|
else
|
|
{:error, :no_body} ->
|
|
conn |> put_status(400) |> json(%{error: "Missing body"})
|
|
|
|
{:error, :organization_mismatch} ->
|
|
conn |> put_status(403) |> json(%{error: "Organization mismatch"})
|
|
|
|
{:error, :not_configured} ->
|
|
conn |> put_status(404) |> json(%{error: "Integration not found"})
|
|
|
|
{:error, :invalid_signature} ->
|
|
conn |> put_status(401) |> json(%{error: "Invalid signature"})
|
|
|
|
{:error, :no_secret_configured} ->
|
|
conn |> put_status(403) |> json(%{error: "Webhook secret not configured"})
|
|
|
|
{:error, :unhandled_event} ->
|
|
json(conn, %{status: "accepted"})
|
|
|
|
{:error, _reason} ->
|
|
conn |> put_status(500) |> json(%{error: "Internal error"})
|
|
end
|
|
end
|
|
|
|
defp extract_event(%{"event" => %{"event_type" => event_type, "data" => data}}) do
|
|
case event_type do
|
|
type when type in ~w(incident.resolved incident.acknowledged) ->
|
|
{:ok, type, data}
|
|
|
|
_ ->
|
|
{:error, :unhandled_event}
|
|
end
|
|
end
|
|
|
|
defp extract_event(_), do: {:error, :unhandled_event}
|
|
|
|
defp get_raw_body(conn) do
|
|
case conn.private[:raw_body] do
|
|
body when is_binary(body) and body != "" -> {:ok, body}
|
|
_ -> {:error, :no_body}
|
|
end
|
|
end
|
|
|
|
defp get_pagerduty_integration(organization_id) do
|
|
case Integrations.get_integration(organization_id, "pagerduty") do
|
|
{:ok, %{enabled: true} = integration} -> {:ok, integration}
|
|
_ -> {:error, :not_configured}
|
|
end
|
|
end
|
|
|
|
defp validate_organization_match(integration, url_organization_id) do
|
|
if integration.organization_id == url_organization_id do
|
|
:ok
|
|
else
|
|
{:error, :organization_mismatch}
|
|
end
|
|
end
|
|
|
|
defp verify_signature(conn, raw_body, integration) do
|
|
webhook_secret = get_in(integration.credentials, ["webhook_secret"])
|
|
|
|
if is_nil(webhook_secret) or webhook_secret == "" do
|
|
Logger.warning("PagerDuty webhook: rejected - no webhook secret configured")
|
|
{:error, :no_secret_configured}
|
|
else
|
|
check_pagerduty_hmac(conn, raw_body, webhook_secret, integration.organization_id)
|
|
end
|
|
end
|
|
|
|
defp check_pagerduty_hmac(conn, raw_body, secret, org_id) do
|
|
signatures =
|
|
conn
|
|
|> Plug.Conn.get_req_header("x-pagerduty-signature")
|
|
|> Enum.flat_map(&String.split(&1, ","))
|
|
|> Enum.map(&String.trim/1)
|
|
|
|
expected =
|
|
:hmac
|
|
|> :crypto.mac(:sha256, secret, raw_body)
|
|
|> Base.encode16(case: :lower)
|
|
|
|
if signature_matches?(signatures, expected) do
|
|
:ok
|
|
else
|
|
Logger.warning("PagerDuty webhook signature mismatch for org #{org_id}")
|
|
{:error, :invalid_signature}
|
|
end
|
|
end
|
|
|
|
defp signature_matches?(signatures, expected) do
|
|
Enum.any?(signatures, fn
|
|
"v1=" <> hex -> Plug.Crypto.secure_compare(hex, expected)
|
|
_ -> false
|
|
end)
|
|
end
|
|
end
|