towerops/lib/towerops/workers/alert_notification_worker.ex

defmodule Towerops.Workers.AlertNotificationWorker do
  @moduledoc """
  Oban worker for sending alert notifications reliably.

  Handles trigger, acknowledge, and resolve notifications with automatic
  retries and error tracking.
  """
  use Oban.Pro.Worker,
    queue: :notifications,
    max_attempts: 3,
    priority: 1

  alias Towerops.Alerts
  alias Towerops.Alerts.NotificationRateLimiter
  alias Towerops.Devices
  alias Towerops.OnCall.Escalation
  alias Towerops.PagerDuty.Notifier
  alias Towerops.Workers.AlertDigestWorker

  require Logger

  @impl Oban.Pro.Worker
  def process(%Oban.Job{args: %{"action" => "trigger", "alert_id" => alert_id}}) do
    with {:ok, alert} <- fetch_alert(alert_id),
         {:ok, device} <- fetch_device(alert.device_id) do
      handle_trigger_notification(alert, device, alert_id)
    else
      {:error, :alert_not_found} ->
        Logger.warning("Alert #{alert_id} not found for notification")
        :ok

      {:error, :device_not_found} ->
        Logger.warning("Device not found for alert #{alert_id}")
        :ok
    end
  end

  def process(%Oban.Job{args: %{"action" => "acknowledge", "alert_id" => alert_id}}) do
    case fetch_alert(alert_id) do
      {:ok, alert} ->
        routing = alert_routing_for_alert(alert)

        if routing in ["builtin", "both"], do: maybe_acknowledge_incident(alert)

        if routing in ["pagerduty", "both"] do
          notify_pagerduty_acknowledge(alert, alert_id)
        else
          :ok
        end

      {:error, :alert_not_found} ->
        Logger.warning("Alert #{alert_id} not found for acknowledge notification")
        :ok
    end
  end

  def process(%Oban.Job{args: %{"action" => "resolve", "alert_id" => alert_id}}) do
    case fetch_alert(alert_id) do
      {:ok, alert} ->
        routing = alert_routing_for_alert(alert)

        if routing in ["builtin", "both"], do: maybe_resolve_incident(alert)

        if routing in ["pagerduty", "both"] do
          notify_pagerduty_resolve(alert, alert_id)
        else
          :ok
        end

      {:error, :alert_not_found} ->
        Logger.warning("Alert #{alert_id} not found for resolve notification")
        :ok
    end
  end

  # Helper to enqueue notification jobs
  def enqueue_trigger(alert_id) do
    %{action: "trigger", alert_id: alert_id}
    |> new()
    |> Oban.insert()
  end

  def enqueue_acknowledge(alert_id) do
    %{action: "acknowledge", alert_id: alert_id}
    |> new()
    |> Oban.insert()
  end

  def enqueue_resolve(alert_id) do
    %{action: "resolve", alert_id: alert_id}
    |> new()
    |> Oban.insert()
  end

  defp handle_trigger_notification(alert, device, alert_id) do
    if alert.storm_suppressed do
      Logger.debug("Skipping notification for storm-suppressed alert #{alert_id}")
      :ok
    else
      send_notifications(alert, device, alert_id)
    end
  end

  defp send_notifications(alert, device, alert_id) do
    routing = alert_routing(device)

    # Apply per-user rate limiting for built-in notifications
    rate_limited_users =
      if routing in ["builtin", "both"] do
        check_and_apply_rate_limits(alert, device)
      else
        MapSet.new()
      end

    pd_result =
      if routing in ["pagerduty", "both"] do
        notify_pagerduty_trigger(alert, device, alert_id)
      else
        :ok
      end

    if routing in ["builtin", "both"] do
      maybe_start_escalation(alert, device, rate_limited_users)
    end

    pd_result
  end

  defp notify_pagerduty_trigger(alert, device, alert_id) do
    case Notifier.notify_trigger(alert, device) do
      {:error, :not_configured} ->
        Logger.debug("PagerDuty not configured for alert #{alert_id}, skipping")
        :ok

      result ->
        result
    end
  end

  defp notify_pagerduty_acknowledge(alert, alert_id) do
    case Notifier.notify_acknowledge(alert) do
      {:error, :not_configured} ->
        Logger.debug("PagerDuty not configured for alert #{alert_id}, skipping")
        :ok

      result ->
        result
    end
  end

  defp notify_pagerduty_resolve(alert, alert_id) do
    case Notifier.notify_resolve(alert) do
      {:error, :not_configured} ->
        Logger.debug("PagerDuty not configured for alert #{alert_id}, skipping")
        :ok

      result ->
        result
    end
  end

  defp maybe_start_escalation(alert, device, _rate_limited_users) do
    policy_id = device.escalation_policy_id || device.organization.default_escalation_policy_id

    if policy_id do
      case Escalation.start_escalation(alert, policy_id) do
        {:ok, _incident} ->
          Logger.info("Started built-in escalation for alert #{alert.id}")

        {:error, reason} ->
          Logger.warning("Failed to start escalation for alert #{alert.id}: #{inspect(reason)}")
      end
    end
  end

  # Check rate limits for users who would receive this notification.
  # Returns a MapSet of user IDs that are rate-limited (notifications suppressed).
  defp check_and_apply_rate_limits(alert, device) do
    org_id = device.organization_id
    policy_id = device.escalation_policy_id || device.organization.default_escalation_policy_id

    policy_id
    |> case do
      nil -> []
      id -> resolve_escalation_targets(id)
    end
    |> Enum.filter(&should_suppress_for_user?(&1, org_id, alert.id))
    |> MapSet.new()
  end

  defp should_suppress_for_user?(user_id, org_id, alert_id) do
    case NotificationRateLimiter.check_rate(user_id, org_id, alert_id) do
      :suppress ->
        _ = AlertDigestWorker.enqueue(user_id)
        true

      :allow ->
        false
    end
  end

  # Resolve escalation policy targets to user IDs
  defp resolve_escalation_targets(policy_id) do
    import Ecto.Query

    alias Towerops.OnCall.EscalationRule
    alias Towerops.OnCall.EscalationTarget
    alias Towerops.Repo

    # Get targets from first rule (position 0) of the policy
    from(t in EscalationTarget,
      join: r in EscalationRule,
      on: r.id == t.escalation_rule_id,
      where: r.escalation_policy_id == ^policy_id,
      where: r.position == 0,
      where: t.target_type == "user",
      select: t.user_id
    )
    |> Repo.all()
    |> Enum.reject(&is_nil/1)
  rescue
    _ -> []
  end

  defp maybe_acknowledge_incident(alert) do
    case Escalation.find_incident_for_alert(alert.id) do
      nil ->
        :ok

      incident ->
        case Escalation.acknowledge_incident(incident.id, nil) do
          {:ok, _} -> Logger.info("Acknowledged incident #{incident.id} for alert #{alert.id}")
          {:error, reason} -> Logger.warning("Failed to acknowledge incident: #{inspect(reason)}")
        end
    end
  end

  defp maybe_resolve_incident(alert) do
    case Escalation.find_incident_for_alert(alert.id) do
      nil ->
        :ok

      incident ->
        case Escalation.resolve_incident(incident.id, nil) do
          {:ok, _} -> Logger.info("Resolved incident #{incident.id} for alert #{alert.id}")
          {:error, reason} -> Logger.warning("Failed to resolve incident: #{inspect(reason)}")
        end
    end
  end

  # Get alert routing from the device's organization (preloaded)
  defp alert_routing(%{organization: %{alert_routing: routing}}) when is_binary(routing), do: routing
  defp alert_routing(_device), do: "builtin"

  # Get alert routing for acknowledge/resolve (need to look up from alert's device)
  defp alert_routing_for_alert(alert) do
    case fetch_device(alert.device_id) do
      {:ok, device} -> alert_routing(device)
      _ -> "builtin"
    end
  end

  defp fetch_alert(alert_id) do
    case Alerts.get_alert(alert_id) do
      nil -> {:error, :alert_not_found}
      alert -> {:ok, alert}
    end
  end

  defp fetch_device(device_id) do
    case Devices.get_device(device_id) do
      nil -> {:error, :device_not_found}
      device -> {:ok, device}
    end
  end
end