176 lines
5.4 KiB
Elixir
176 lines
5.4 KiB
Elixir
defmodule ToweropsWeb.Api.V1.PagerdutyWebhookController do
|
|
@moduledoc """
|
|
Handles inbound PagerDuty V3 webhook events.
|
|
|
|
When an incident is resolved or acknowledged in PagerDuty, this controller
|
|
updates the corresponding TowerOps alert so we don't re-alert on it.
|
|
|
|
PagerDuty signs webhooks with HMAC-SHA256 using a per-integration secret.
|
|
The signature is in the `X-PagerDuty-Signature` header as `v1=<hex_digest>`.
|
|
"""
|
|
use ToweropsWeb, :controller
|
|
|
|
alias Towerops.Alerts
|
|
alias Towerops.Integrations
|
|
|
|
require Logger
|
|
|
|
def create(conn, %{"organization_id" => organization_id} = params) do
|
|
with {:ok, raw_body} <- get_raw_body(conn),
|
|
{:ok, integration} <- get_pagerduty_integration(organization_id),
|
|
:ok <- validate_organization_match(integration, organization_id),
|
|
:ok <- verify_signature(conn, raw_body, integration) do
|
|
# params already parsed by Phoenix JSON parser
|
|
process_webhook(params, organization_id)
|
|
json(conn, %{status: "accepted"})
|
|
else
|
|
{:error, :organization_mismatch} ->
|
|
Logger.warning("PagerDuty webhook: rejected - organization ID mismatch (URL vs integration)")
|
|
|
|
conn |> put_status(403) |> json(%{error: "Organization mismatch"})
|
|
|
|
{:error, :not_configured} ->
|
|
conn |> put_status(404) |> json(%{error: "Integration not found"})
|
|
|
|
{:error, :invalid_signature} ->
|
|
conn |> put_status(401) |> json(%{error: "Invalid signature"})
|
|
|
|
{:error, :no_secret_configured} ->
|
|
conn |> put_status(403) |> json(%{error: "Webhook secret not configured"})
|
|
|
|
{:error, _reason} ->
|
|
conn |> put_status(500) |> json(%{error: "Internal error"})
|
|
end
|
|
end
|
|
|
|
defp get_raw_body(conn) do
|
|
case conn.private[:raw_body] do
|
|
body when is_binary(body) and body != "" -> {:ok, body}
|
|
_ -> {:error, :no_body}
|
|
end
|
|
end
|
|
|
|
defp get_pagerduty_integration(organization_id) do
|
|
case Integrations.get_integration(organization_id, "pagerduty") do
|
|
{:ok, %{enabled: true} = integration} -> {:ok, integration}
|
|
_ -> {:error, :not_configured}
|
|
end
|
|
end
|
|
|
|
defp validate_organization_match(integration, url_organization_id) do
|
|
if integration.organization_id == url_organization_id do
|
|
:ok
|
|
else
|
|
{:error, :organization_mismatch}
|
|
end
|
|
end
|
|
|
|
defp verify_signature(conn, raw_body, integration) do
|
|
webhook_secret = get_in(integration.credentials, ["webhook_secret"])
|
|
|
|
if is_nil(webhook_secret) or webhook_secret == "" do
|
|
Logger.warning("PagerDuty webhook: rejected - no webhook secret configured")
|
|
{:error, :no_secret_configured}
|
|
else
|
|
check_pagerduty_hmac(conn, raw_body, webhook_secret, integration.organization_id)
|
|
end
|
|
end
|
|
|
|
defp check_pagerduty_hmac(conn, raw_body, secret, org_id) do
|
|
signatures =
|
|
conn
|
|
|> Plug.Conn.get_req_header("x-pagerduty-signature")
|
|
|> Enum.flat_map(&String.split(&1, ","))
|
|
|> Enum.map(&String.trim/1)
|
|
|
|
expected =
|
|
:hmac
|
|
|> :crypto.mac(:sha256, secret, raw_body)
|
|
|> Base.encode16(case: :lower)
|
|
|
|
if signature_matches?(signatures, expected) do
|
|
:ok
|
|
else
|
|
Logger.warning("PagerDuty webhook signature mismatch for org #{org_id}")
|
|
{:error, :invalid_signature}
|
|
end
|
|
end
|
|
|
|
defp signature_matches?(signatures, expected) do
|
|
Enum.any?(signatures, fn
|
|
"v1=" <> hex -> Plug.Crypto.secure_compare(hex, expected)
|
|
_ -> false
|
|
end)
|
|
end
|
|
|
|
defp process_webhook(%{"event" => %{"event_type" => event_type, "data" => data}}, organization_id) do
|
|
case event_type do
|
|
"incident.resolved" ->
|
|
resolve_from_pagerduty(data, organization_id)
|
|
|
|
"incident.acknowledged" ->
|
|
acknowledge_from_pagerduty(data, organization_id)
|
|
|
|
other ->
|
|
Logger.debug("Ignoring PagerDuty webhook event: #{other}")
|
|
:ok
|
|
end
|
|
end
|
|
|
|
defp process_webhook(_payload, _organization_id), do: :ok
|
|
|
|
defp resolve_from_pagerduty(data, _organization_id) do
|
|
case extract_alert_id(data) do
|
|
nil ->
|
|
Logger.debug("PagerDuty resolve: no TowerOps alert ID found in incident")
|
|
:ok
|
|
|
|
alert_id ->
|
|
case Alerts.get_alert(alert_id) do
|
|
nil ->
|
|
Logger.warning("PagerDuty resolve: alert #{alert_id} not found")
|
|
|
|
%{resolved_at: resolved_at} when not is_nil(resolved_at) ->
|
|
Logger.debug("PagerDuty resolve: alert #{alert_id} already resolved")
|
|
|
|
alert ->
|
|
Logger.info("Resolving alert #{alert_id} from PagerDuty")
|
|
Alerts.resolve_alert_silent(alert)
|
|
end
|
|
end
|
|
end
|
|
|
|
defp acknowledge_from_pagerduty(data, _organization_id) do
|
|
case extract_alert_id(data) do
|
|
nil ->
|
|
:ok
|
|
|
|
alert_id ->
|
|
case Alerts.get_alert(alert_id) do
|
|
nil ->
|
|
Logger.warning("PagerDuty acknowledge: alert #{alert_id} not found")
|
|
|
|
%{acknowledged_at: ack} when not is_nil(ack) ->
|
|
Logger.debug("PagerDuty acknowledge: alert #{alert_id} already acknowledged")
|
|
|
|
alert ->
|
|
Logger.info("Acknowledging alert #{alert_id} from PagerDuty")
|
|
Alerts.acknowledge_alert_silent(alert)
|
|
end
|
|
end
|
|
end
|
|
|
|
# PagerDuty includes our dedup_key as the alert key.
|
|
# Our dedup_key format is "towerops-alert-<uuid>"
|
|
defp extract_alert_id(data) do
|
|
# Try the first alert's dedup key
|
|
alerts = data["alerts"] || []
|
|
|
|
Enum.find_value(alerts, fn alert ->
|
|
case alert["alert_key"] do
|
|
"towerops-alert-" <> id -> id
|
|
_ -> nil
|
|
end
|
|
end)
|
|
end
|
|
end
|