diff --git a/lib/towerops/alerts.ex b/lib/towerops/alerts.ex index d2069274..b2f4d717 100644 --- a/lib/towerops/alerts.ex +++ b/lib/towerops/alerts.ex @@ -215,21 +215,41 @@ defmodule Towerops.Alerts do Acknowledges an alert. """ def acknowledge_alert(alert, user_id) do - alert - |> Alert.changeset(%{ - acknowledged_at: DateTime.truncate(DateTime.utc_now(), :second), - acknowledged_by_id: user_id - }) - |> Repo.update() + result = + alert + |> Alert.changeset(%{ + acknowledged_at: DateTime.truncate(DateTime.utc_now(), :second), + acknowledged_by_id: user_id + }) + |> Repo.update() + + case result do + {:ok, updated_alert} -> + Task.start(fn -> Towerops.PagerDuty.Notifier.notify_acknowledge(updated_alert) end) + {:ok, updated_alert} + + error -> + error + end end @doc """ Resolves an alert. """ def resolve_alert(alert) do - alert - |> Alert.changeset(%{resolved_at: DateTime.truncate(DateTime.utc_now(), :second)}) - |> Repo.update() + result = + alert + |> Alert.changeset(%{resolved_at: DateTime.truncate(DateTime.utc_now(), :second)}) + |> Repo.update() + + case result do + {:ok, updated_alert} -> + Task.start(fn -> Towerops.PagerDuty.Notifier.notify_resolve(updated_alert) end) + {:ok, updated_alert} + + error -> + error + end end @doc """ diff --git a/lib/towerops/integrations/integration.ex b/lib/towerops/integrations/integration.ex index cd95a041..f40b480f 100644 --- a/lib/towerops/integrations/integration.ex +++ b/lib/towerops/integrations/integration.ex @@ -14,7 +14,7 @@ defmodule Towerops.Integrations.Integration do @primary_key {:id, :binary_id, autogenerate: true} @foreign_key_type :binary_id - @valid_providers ~w(preseem gaiia) + @valid_providers ~w(preseem gaiia pagerduty) @valid_sync_statuses ~w(never success partial failed) schema "integrations" do diff --git a/lib/towerops/pagerduty/client.ex b/lib/towerops/pagerduty/client.ex new file mode 100644 index 00000000..42e7564a --- /dev/null +++ b/lib/towerops/pagerduty/client.ex @@ -0,0 +1,112 @@ +defmodule Towerops.PagerDuty.Client do + @moduledoc "PagerDuty Events API v2 client." + + require Logger + + @events_url "https://events.pagerduty.com/v2/enqueue" + + def trigger(routing_key, alert, device, site_name, towerops_url) do + send_event(%{ + routing_key: routing_key, + event_action: "trigger", + dedup_key: dedup_key(alert), + payload: %{ + summary: alert_summary(alert, device), + source: "towerops", + severity: severity(alert.alert_type), + timestamp: DateTime.to_iso8601(alert.triggered_at), + component: device.hostname || device.ip_address, + group: site_name, + class: to_string(alert.alert_type), + custom_details: %{ + device_id: device.id, + device_name: device.hostname || device.ip_address, + ip_address: device.ip_address, + site_name: site_name, + alert_type: to_string(alert.alert_type), + message: alert.message, + towerops_url: towerops_url + } + }, + links: [%{href: towerops_url, text: "View in TowerOps"}] + }) + end + + def acknowledge(routing_key, alert) do + send_event(%{ + routing_key: routing_key, + event_action: "acknowledge", + dedup_key: dedup_key(alert) + }) + end + + def resolve(routing_key, alert) do + send_event(%{ + routing_key: routing_key, + event_action: "resolve", + dedup_key: dedup_key(alert) + }) + end + + def test_connection(routing_key) do + test_dedup = "towerops-test-#{System.unique_integer([:positive])}" + + case send_event(%{ + routing_key: routing_key, + event_action: "trigger", + dedup_key: test_dedup, + payload: %{ + summary: "TowerOps integration test", + source: "towerops", + severity: "info", + timestamp: DateTime.to_iso8601(DateTime.utc_now()) + } + }) do + {:ok, _} -> + send_event(%{ + routing_key: routing_key, + event_action: "resolve", + dedup_key: test_dedup + }) + + error -> + error + end + end + + defp send_event(body) do + case Req.post(@events_url, json: body) do + {:ok, %{status: status, body: resp_body}} when status in [200, 202] -> + {:ok, resp_body} + + {:ok, %{status: 400, body: resp_body}} -> + {:error, {:bad_request, resp_body["message"] || "Invalid event"}} + + {:ok, %{status: 429}} -> + {:error, :rate_limited} + + {:ok, %{status: status}} -> + {:error, {:unexpected_status, status}} + + {:error, error} -> + Logger.warning("PagerDuty API error: #{inspect(error)}") + {:error, error} + end + end + + defp dedup_key(alert), do: "towerops-alert-#{alert.id}" + + defp alert_summary(alert, device) do + name = device.hostname || device.ip_address + + case alert.alert_type do + :device_down -> "Device down: #{name}" + :device_up -> "Device recovered: #{name}" + other -> "Alert: #{other} on #{name}" + end + end + + defp severity(:device_down), do: "critical" + defp severity(:device_up), do: "info" + defp severity(_), do: "warning" +end diff --git a/lib/towerops/pagerduty/notifier.ex b/lib/towerops/pagerduty/notifier.ex new file mode 100644 index 00000000..4909dccb --- /dev/null +++ b/lib/towerops/pagerduty/notifier.ex @@ -0,0 +1,84 @@ +defmodule Towerops.PagerDuty.Notifier do + @moduledoc "Sends alert events to PagerDuty when integration is configured." + + alias Towerops.Integrations + alias Towerops.PagerDuty.Client + alias Towerops.Repo + + require Logger + + def notify_trigger(alert, device) do + with {:ok, routing_key} <- get_routing_key(device.organization_id) do + site_name = get_site_name(device) + towerops_url = device_url(device) + + case Client.trigger(routing_key, alert, device, site_name, towerops_url) do + {:ok, _} -> + Logger.info("PagerDuty incident triggered for alert #{alert.id}") + :ok + + {:error, reason} -> + Logger.warning("PagerDuty trigger failed for alert #{alert.id}: #{inspect(reason)}") + :error + end + end + end + + def notify_acknowledge(alert) do + alert = Repo.preload(alert, device: :site) + + with {:ok, routing_key} <- get_routing_key(alert.device.organization_id) do + case Client.acknowledge(routing_key, alert) do + {:ok, _} -> + Logger.info("PagerDuty incident acknowledged for alert #{alert.id}") + :ok + + {:error, reason} -> + Logger.warning( + "PagerDuty acknowledge failed for alert #{alert.id}: #{inspect(reason)}" + ) + + :error + end + end + end + + def notify_resolve(alert) do + alert = Repo.preload(alert, device: :site) + + with {:ok, routing_key} <- get_routing_key(alert.device.organization_id) do + case Client.resolve(routing_key, alert) do + {:ok, _} -> + Logger.info("PagerDuty incident resolved for alert #{alert.id}") + :ok + + {:error, reason} -> + Logger.warning("PagerDuty resolve failed for alert #{alert.id}: #{inspect(reason)}") + :error + end + end + end + + defp get_routing_key(organization_id) do + case Integrations.get_integration(organization_id, "pagerduty") do + {:ok, %{enabled: true, credentials: %{"api_key" => key}}} when key != "" -> + {:ok, key} + + _ -> + {:error, :not_configured} + end + end + + defp get_site_name(device) do + device = Repo.preload(device, :site) + + case device.site do + nil -> "Unknown Site" + site -> site.name + end + end + + defp device_url(device) do + ToweropsWeb.Endpoint.url() <> "/devices/#{device.id}" + end +end diff --git a/lib/towerops/workers/device_monitor_worker.ex b/lib/towerops/workers/device_monitor_worker.ex index e88bf8f6..ba25d396 100644 --- a/lib/towerops/workers/device_monitor_worker.ex +++ b/lib/towerops/workers/device_monitor_worker.ex @@ -202,7 +202,7 @@ defmodule Towerops.Workers.DeviceMonitorWorker do defp create_device_down_alert(device, now) do alert_message = get_down_alert_message(device) - {:ok, _alert} = + {:ok, alert} = Alerts.create_alert(%{ device_id: device.id, alert_type: :device_down, @@ -210,6 +210,8 @@ defmodule Towerops.Workers.DeviceMonitorWorker do message: alert_message }) + Task.start(fn -> Towerops.PagerDuty.Notifier.notify_trigger(alert, device) end) + _ = Phoenix.PubSub.broadcast( Towerops.PubSub, @@ -229,7 +231,7 @@ defmodule Towerops.Workers.DeviceMonitorWorker do defp handle_equipment_up(device, now) do recovery_message = get_recovery_message(device) - {:ok, _alert} = + {:ok, alert} = Alerts.create_alert(%{ device_id: device.id, alert_type: :device_up, @@ -237,6 +239,8 @@ defmodule Towerops.Workers.DeviceMonitorWorker do message: recovery_message }) + Task.start(fn -> Towerops.PagerDuty.Notifier.notify_trigger(alert, device) end) + resolve_down_alert(device) _ = diff --git a/lib/towerops_web/live/help_live/index.ex b/lib/towerops_web/live/help_live/index.ex index 8c2c7b20..c9dd1f0d 100644 --- a/lib/towerops_web/live/help_live/index.ex +++ b/lib/towerops_web/live/help_live/index.ex @@ -1756,6 +1756,83 @@ defmodule ToweropsWeb.HelpLive.Index do +
+ PagerDuty + is an incident management and on-call alerting platform. The PagerDuty integration provides + 2-way alert sync — when a device goes down in Towerops, a PagerDuty incident is automatically + triggered. Acknowledging or resolving the alert in Towerops updates the PagerDuty incident as well. +
+ ++ In PagerDuty, go to <.code>Services → select your service → <.code>Integrations tab → <.code>Add Integration → choose <.code>Events API v2. Copy the <.code>Integration Key (routing key). +
++ Navigate to <.code>Organization Settings → <.code>Integrations tab → click <.code>Configure on PagerDuty. Paste your integration key and test the connection. +
++ Unlike other integrations, PagerDuty is event-driven — there is no periodic sync. + Alerts are sent to PagerDuty in real time as they occur. No polling or sync interval is needed. +
++ Find this in PagerDuty → Services → Your Service → Integrations → Events API v2 +
+ <% end %> + + <%= if provider.id != "pagerduty" do %> + <.input + field={@integration_form[:sync_interval_minutes]} + type="number" + label="Sync interval (minutes)" + min="5" + value={ + if(@integrations[provider.id], + do: @integrations[provider.id].sync_interval_minutes, + else: if(provider.id == "gaiia", do: 15, else: 10) + ) + } + /> + <% end %> <%= if @test_result do %>