prop/lib/microwaveprop/oban_error_reporter.ex

defmodule Microwaveprop.ObanErrorReporter do
  @moduledoc """
  Structured logging + per-worker recovery tap for Oban job exceptions.

  `PromEx.Plugins.Oban` already counts job exceptions as a Prometheus
  counter, but a counter alone can't tell us *which* worker failed, why,
  or whether the job is about to be retried. This module closes that
  gap by attaching a `:telemetry` handler to `[:oban, :job, :exception]`
  that:

    * logs a single `Logger.error` line with structured metadata —
      `worker`, `queue`, `attempt`, `max_attempts`, `retry_exhausted?`,
      `kind`, `reason` class, `job_id`, and a safe-for-log args summary
      (keys only, no values that might contain secrets or bodies); and

    * dispatches to opt-in per-worker callbacks for recovery actions:

          def on_permanent_failure(%Oban.Job{} = job), do: ...
          def on_transient_failure(%Oban.Job{} = job), do: ...

      Callback failures are caught so a bug in one worker's recovery
      path cannot detach the telemetry handler or crash another job's
      logging.

  The reporter does not emit a second telemetry event — PromEx counts
  come from Oban's own `[:oban, :job, :exception]` event, so emitting
  anything here would double-count.
  """

  require Logger

  @handler_id "microwaveprop-oban-error-reporter"
  @event [:oban, :job, :exception]

  @doc """
  Attach the reporter to Oban's exception telemetry event.

  Safe to call more than once — a pre-existing attachment is treated
  as success.
  """
  @spec attach() :: :ok
  def attach do
    case :telemetry.attach(@handler_id, @event, &__MODULE__.handle_event/4, %{}) do
      :ok -> :ok
      {:error, :already_exists} -> :ok
    end
  end

  @doc """
  Detach the reporter. Primarily useful for tests.
  """
  @spec detach() :: :ok
  def detach do
    _ = :telemetry.detach(@handler_id)
    :ok
  end

  @doc false
  @spec handle_event(:telemetry.event(), map(), map(), any()) :: :ok
  def handle_event(@event, measurements, metadata, _config) do
    do_handle(measurements, metadata)
  rescue
    error ->
      # A bug in the reporter must never detach the handler.
      Logger.error("ObanErrorReporter handler crashed: " <> Exception.format(:error, error, __STACKTRACE__))
  end

  defp do_handle(_measurements, metadata) do
    ctx = context(metadata)
    log_exception(ctx, metadata)
    dispatch_callback(ctx, metadata)
    :ok
  end

  # ——— logging ————————————————————————————————————————————————————————

  defp log_exception(ctx, metadata) do
    kind = Map.get(metadata, :kind, :error)
    reason = Map.get(metadata, :reason)

    Logger.error(
      "Oban job exception " <>
        "worker=#{ctx.worker} " <>
        "queue=#{ctx.queue} " <>
        "job_id=#{ctx.job_id} " <>
        "attempt=#{ctx.attempt} " <>
        "max_attempts=#{ctx.max_attempts} " <>
        "retry_exhausted?=#{ctx.retry_exhausted?} " <>
        "kind=#{kind} " <>
        "reason_class=#{reason_class(reason)} " <>
        "reason=#{format_reason(reason)} " <>
        "args_keys=#{inspect(ctx.args_summary)}"
    )
  end

  defp format_reason(%{__exception__: true} = ex) do
    "#{inspect(ex.__struct__)}: #{Exception.message(ex)}"
  end

  defp format_reason(other), do: inspect(other)

  defp reason_class(%{__exception__: true} = ex), do: inspect(ex.__struct__)
  defp reason_class(other) when is_atom(other), do: inspect(other)
  defp reason_class(_), do: "term"

  # ——— context extraction ——————————————————————————————————————————————

  defp context(%{job: job} = metadata) when is_map(job) do
    attempt = Map.get(job, :attempt, 0)
    max_attempts = Map.get(job, :max_attempts, 0)
    worker = Map.get(job, :worker) || Map.get(metadata, :worker) || "unknown"
    queue = Map.get(job, :queue) || Map.get(metadata, :queue) || "unknown"

    %{
      worker: worker,
      queue: queue,
      job_id: Map.get(job, :id, 0),
      attempt: attempt,
      max_attempts: max_attempts,
      retry_exhausted?: retry_exhausted?(attempt, max_attempts),
      args_summary: args_summary(Map.get(job, :args))
    }
  end

  defp context(metadata) do
    %{
      worker: Map.get(metadata, :worker, "unknown"),
      queue: Map.get(metadata, :queue, "unknown"),
      job_id: 0,
      attempt: 0,
      max_attempts: 0,
      retry_exhausted?: false,
      args_summary: []
    }
  end

  defp retry_exhausted?(attempt, max) when is_integer(attempt) and is_integer(max) and max > 0, do: attempt >= max

  defp retry_exhausted?(_, _), do: false

  # Never log args *values* — they can contain bodies, tokens, or PII.
  # A sorted key list is enough to identify "which shape of payload".
  defp args_summary(args) when is_map(args) do
    args |> Map.keys() |> Enum.map(&to_string/1) |> Enum.sort()
  end

  defp args_summary(_), do: []

  # ——— per-worker callbacks ———————————————————————————————————————————

  defp dispatch_callback(%{retry_exhausted?: exhausted?} = ctx, metadata) do
    job = Map.get(metadata, :job)
    fun = if exhausted?, do: :on_permanent_failure, else: :on_transient_failure

    case worker_module(ctx.worker) do
      nil ->
        :ok

      mod ->
        if function_exported?(mod, fun, 1) do
          safely_invoke(mod, fun, job)
        else
          :ok
        end
    end
  end

  # `Module.safe_concat/1` avoids creating an atom for unknown workers,
  # and returns an existing atom otherwise. If the module is not loaded
  # in this node (e.g. a test stub name) we get :error and skip.
  defp worker_module(worker) when is_binary(worker) do
    mod = Module.safe_concat([worker])
    (Code.ensure_loaded?(mod) && mod) || nil
  rescue
    ArgumentError -> nil
  end

  defp worker_module(worker) when is_atom(worker) do
    if Code.ensure_loaded?(worker), do: worker
  end

  defp worker_module(_), do: nil

  defp safely_invoke(mod, fun, job) do
    apply(mod, fun, [job])
    :ok
  rescue
    error ->
      Logger.warning(
        "ObanErrorReporter: #{inspect(mod)}.#{fun}/1 raised: " <> Exception.format(:error, error, __STACKTRACE__)
      )

      :ok
  catch
    kind, reason ->
      Logger.warning("ObanErrorReporter: #{inspect(mod)}.#{fun}/1 #{kind}: #{inspect(reason)}")

      :ok
  end
end