diff --git a/config/prod.exs b/config/prod.exs index 52ab0ea1..a6990679 100644 --- a/config/prod.exs +++ b/config/prod.exs @@ -1,7 +1,18 @@ import Config +# Filter out harmless Oban shutdown messages during deployments +config :logger, :default_handler, + filters: [ + drop_oban_shutdown: { + &Towerops.LoggerFilters.drop_oban_shutdown/2, + [] + } + ] + # Do not print debug messages in production -config :logger, level: :info +config :logger, + level: :info, + metadata_filter: [oban_shutdown_filter: true] # Configure Swoosh API Client config :swoosh, api_client: Swoosh.ApiClient.Req diff --git a/lib/towerops/logger_filters.ex b/lib/towerops/logger_filters.ex new file mode 100644 index 00000000..47c0c015 --- /dev/null +++ b/lib/towerops/logger_filters.ex @@ -0,0 +1,45 @@ +defmodule Towerops.LoggerFilters do + @moduledoc """ + Custom Logger filters to reduce noise in production logs. + """ + + @doc """ + Drops harmless Oban shutdown messages that occur during Kubernetes rolling deployments. + + These messages are expected when the old pod gracefully terminates during a deployment, + but Oban's internal processes log them as "unexpected" even though they're normal. + + Example messages filtered: + - "Received unexpected message: {:EXIT, #PID<...>, :shutdown}" + - "Oban.Queue.Watchman #PID<...> received unexpected message in handle_info/2: {:EXIT, ...}" + """ + def drop_oban_shutdown(log_event, _opts) do + cond do + oban_source_shutdown?(log_event) -> :stop + oban_queue_shutdown?(log_event) -> :stop + true -> :ignore + end + end + + defp oban_source_shutdown?(%{meta: %{source: :oban}, msg: {:string, msg}}) when is_binary(msg) do + String.contains?(msg, "EXIT") and String.contains?(msg, ":shutdown") + end + + defp oban_source_shutdown?(_), do: false + + defp oban_queue_shutdown?(%{meta: %{error_logger: %{tag: :error_msg}}, msg: {:string, msg}}) when is_binary(msg) do + oban_component?(msg) and shutdown_message?(msg) + end + + defp oban_queue_shutdown?(_), do: false + + defp oban_component?(msg) do + String.contains?(msg, "Oban.Queue") or String.contains?(msg, "Oban.Plugins") + end + + defp shutdown_message?(msg) do + String.contains?(msg, "received unexpected message") and + String.contains?(msg, "EXIT") and + String.contains?(msg, ":shutdown") + end +end