refactor: filter harmless Oban shutdown messages during deployments

When Kubernetes performs rolling deployments, the old pod gracefully shuts down
and Oban's internal processes receive EXIT :shutdown signals. These are logged
as "unexpected" even though they're normal during shutdown.

Added custom logger filter to drop these messages in production:
- Oban.Queue.Watchman/Producer receiving EXIT :shutdown
- Oban plugin processes receiving EXIT :shutdown
- Reduces log noise during deployments without hiding real errors

The filter is production-only to preserve full debugging in development.

config/prod.exs

@@ -1,7 +1,18 @@
 import Config
 
+# Filter out harmless Oban shutdown messages during deployments
+config :logger, :default_handler,
+  filters: [
+    drop_oban_shutdown: {
+      &Towerops.LoggerFilters.drop_oban_shutdown/2,
+      []
+    }
+  ]
+
 # Do not print debug messages in production
-config :logger, level: :info
+config :logger,
+  level: :info,
+  metadata_filter: [oban_shutdown_filter: true]
 
 # Configure Swoosh API Client
 config :swoosh, api_client: Swoosh.ApiClient.Req

lib/towerops/logger_filters.ex

@@ -0,0 +1,45 @@
+defmodule Towerops.LoggerFilters do
+  @moduledoc """
+  Custom Logger filters to reduce noise in production logs.
+  """
+
+  @doc """
+  Drops harmless Oban shutdown messages that occur during Kubernetes rolling deployments.
+
+  These messages are expected when the old pod gracefully terminates during a deployment,
+  but Oban's internal processes log them as "unexpected" even though they're normal.
+
+  Example messages filtered:
+  - "Received unexpected message: {:EXIT, #PID<...>, :shutdown}"
+  - "Oban.Queue.Watchman #PID<...> received unexpected message in handle_info/2: {:EXIT, ...}"
+  """
+  def drop_oban_shutdown(log_event, _opts) do
+    cond do
+      oban_source_shutdown?(log_event) -> :stop
+      oban_queue_shutdown?(log_event) -> :stop
+      true -> :ignore
+    end
+  end
+
+  defp oban_source_shutdown?(%{meta: %{source: :oban}, msg: {:string, msg}}) when is_binary(msg) do
+    String.contains?(msg, "EXIT") and String.contains?(msg, ":shutdown")
+  end
+
+  defp oban_source_shutdown?(_), do: false
+
+  defp oban_queue_shutdown?(%{meta: %{error_logger: %{tag: :error_msg}}, msg: {:string, msg}}) when is_binary(msg) do
+    oban_component?(msg) and shutdown_message?(msg)
+  end
+
+  defp oban_queue_shutdown?(_), do: false
+
+  defp oban_component?(msg) do
+    String.contains?(msg, "Oban.Queue") or String.contains?(msg, "Oban.Plugins")
+  end
+
+  defp shutdown_message?(msg) do
+    String.contains?(msg, "received unexpected message") and
+      String.contains?(msg, "EXIT") and
+      String.contains?(msg, ":shutdown")
+  end
+end