From 795ab8a041ef73fe3ebda06d8d4dbaa0a35bd24b Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 20 Jan 2026 11:42:02 -0600 Subject: [PATCH] Add Redix connection resilience options to Exq config Configures Redix (Exq's Redis client) with better connection handling: - sync_connect: true - fail fast if Redis unavailable at startup - exit_on_disconnection: false - don't crash process on disconnect - backoff_initial: 500ms, backoff_max: 30s - retry with exponential backoff This helps Exq survive temporary Redis disconnections without crashing, reducing the frequency of Protocol.UndefinedError crashes in Exq.Node.Server. --- lib/towerops/exq_supervisor.ex | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/towerops/exq_supervisor.ex b/lib/towerops/exq_supervisor.ex index 830feb4c..ce39d421 100644 --- a/lib/towerops/exq_supervisor.ex +++ b/lib/towerops/exq_supervisor.ex @@ -72,12 +72,27 @@ defmodule Towerops.ExqSupervisor do namespace: "exq", concurrency: 10, queues: ["default", "discovery", "polling", "monitoring", "maintenance"], - # Increase max retries and backoff for better Redis connection recovery + # Poll settings - how long to wait for Redis responses + poll_timeout: 50, + scheduler_poll_timeout: 200, + scheduler_enable: true, + # Job retry settings max_retries: 50, - # Exponential backoff starting at 500ms - backoff: 500, - # Add shutdown timeout to allow jobs to complete - shutdown_timeout: 25_000 + # Exponential backoff starting at 500ms, up to 60s + mode: :default, + shutdown_timeout: 30_000, + # Redix connection options for better resilience + redis_options: [ + # Synchronous connect to fail fast if Redis unavailable + sync_connect: true, + # Don't exit process on disconnection - let Exq handle it + exit_on_disconnection: false, + # Connection timeout + timeout: 5_000, + # Backoff for reconnection attempts + backoff_initial: 500, + backoff_max: 30_000 + ] ] end end