defmodule Microwaveprop.ObanErrorReporter do @moduledoc """ Structured logging + per-worker recovery tap for Oban job exceptions. `PromEx.Plugins.Oban` already counts job exceptions as a Prometheus counter, but a counter alone can't tell us *which* worker failed, why, or whether the job is about to be retried. This module closes that gap by attaching a `:telemetry` handler to `[:oban, :job, :exception]` that: * logs a single `Logger.error` line with structured metadata — `worker`, `queue`, `attempt`, `max_attempts`, `retry_exhausted?`, `kind`, `reason` class, `job_id`, and a safe-for-log args summary (keys only, no values that might contain secrets or bodies); and * dispatches to opt-in per-worker callbacks for recovery actions: def on_permanent_failure(%Oban.Job{} = job), do: ... def on_transient_failure(%Oban.Job{} = job), do: ... Callback failures are caught so a bug in one worker's recovery path cannot detach the telemetry handler or crash another job's logging. The reporter does not emit a second telemetry event — PromEx counts come from Oban's own `[:oban, :job, :exception]` event, so emitting anything here would double-count. """ require Logger @handler_id "microwaveprop-oban-error-reporter" @event [:oban, :job, :exception] @doc """ Attach the reporter to Oban's exception telemetry event. Safe to call more than once — a pre-existing attachment is treated as success. """ @spec attach() :: :ok def attach do case :telemetry.attach(@handler_id, @event, &__MODULE__.handle_event/4, %{}) do :ok -> :ok {:error, :already_exists} -> :ok end end @doc """ Detach the reporter. Primarily useful for tests. """ @spec detach() :: :ok def detach do _ = :telemetry.detach(@handler_id) :ok end @doc false @spec handle_event(:telemetry.event(), map(), map(), any()) :: :ok def handle_event(@event, measurements, metadata, _config) do do_handle(measurements, metadata) rescue error -> # A bug in the reporter must never detach the handler. Logger.error("ObanErrorReporter handler crashed: " <> Exception.format(:error, error, __STACKTRACE__)) end defp do_handle(_measurements, metadata) do ctx = context(metadata) log_exception(ctx, metadata) dispatch_callback(ctx, metadata) :ok end # ——— logging ———————————————————————————————————————————————————————— defp log_exception(ctx, metadata) do kind = Map.get(metadata, :kind, :error) reason = Map.get(metadata, :reason) Logger.error( "Oban job exception " <> "worker=#{ctx.worker} " <> "queue=#{ctx.queue} " <> "job_id=#{ctx.job_id} " <> "attempt=#{ctx.attempt} " <> "max_attempts=#{ctx.max_attempts} " <> "retry_exhausted?=#{ctx.retry_exhausted?} " <> "kind=#{kind} " <> "reason_class=#{reason_class(reason)} " <> "reason=#{format_reason(reason)} " <> "args_keys=#{inspect(ctx.args_summary)}" ) end defp format_reason(%{__exception__: true} = ex) do "#{inspect(ex.__struct__)}: #{Exception.message(ex)}" end defp format_reason(other), do: inspect(other) defp reason_class(%{__exception__: true} = ex), do: inspect(ex.__struct__) defp reason_class(other) when is_atom(other), do: inspect(other) defp reason_class(_), do: "term" # ——— context extraction —————————————————————————————————————————————— defp context(%{job: job} = metadata) when is_map(job) do attempt = Map.get(job, :attempt, 0) max_attempts = Map.get(job, :max_attempts, 0) worker = Map.get(job, :worker) || Map.get(metadata, :worker) || "unknown" queue = Map.get(job, :queue) || Map.get(metadata, :queue) || "unknown" %{ worker: worker, queue: queue, job_id: Map.get(job, :id, 0), attempt: attempt, max_attempts: max_attempts, retry_exhausted?: retry_exhausted?(attempt, max_attempts), args_summary: args_summary(Map.get(job, :args)) } end defp context(metadata) do %{ worker: Map.get(metadata, :worker, "unknown"), queue: Map.get(metadata, :queue, "unknown"), job_id: 0, attempt: 0, max_attempts: 0, retry_exhausted?: false, args_summary: [] } end defp retry_exhausted?(attempt, max) when is_integer(attempt) and is_integer(max) and max > 0, do: attempt >= max defp retry_exhausted?(_, _), do: false # Never log args *values* — they can contain bodies, tokens, or PII. # A sorted key list is enough to identify "which shape of payload". defp args_summary(args) when is_map(args) do args |> Map.keys() |> Enum.map(&to_string/1) |> Enum.sort() end defp args_summary(_), do: [] # ——— per-worker callbacks ——————————————————————————————————————————— defp dispatch_callback(%{retry_exhausted?: exhausted?} = ctx, metadata) do job = Map.get(metadata, :job) fun = if exhausted?, do: :on_permanent_failure, else: :on_transient_failure case worker_module(ctx.worker) do nil -> :ok mod -> if function_exported?(mod, fun, 1) do safely_invoke(mod, fun, job) else :ok end end end # `Module.safe_concat/1` avoids creating an atom for unknown workers, # and returns an existing atom otherwise. If the module is not loaded # in this node (e.g. a test stub name) we get :error and skip. defp worker_module(worker) when is_binary(worker) do mod = Module.safe_concat([worker]) (Code.ensure_loaded?(mod) && mod) || nil rescue ArgumentError -> nil end defp worker_module(worker) when is_atom(worker) do if Code.ensure_loaded?(worker), do: worker end defp worker_module(_), do: nil defp safely_invoke(mod, fun, job) do apply(mod, fun, [job]) :ok rescue error -> Logger.warning( "ObanErrorReporter: #{inspect(mod)}.#{fun}/1 raised: " <> Exception.format(:error, error, __STACKTRACE__) ) :ok catch kind, reason -> Logger.warning("ObanErrorReporter: #{inspect(mod)}.#{fun}/1 #{kind}: #{inspect(reason)}") :ok end end