Graham McIntire
5ee241fc16
Addresses production Redis disconnection issues by implementing a highly
resilient Valkey (Redis) setup with automatic failover capabilities.
Infrastructure Changes:
- Add Valkey ConfigMap with optimized connection and memory settings
- TCP keepalive (60s), connection limits (10k clients)
- Memory management (256MB with LRU eviction)
- Separate configs for master, replica, and sentinel
- Update Valkey StatefulSet to 3 replicas (1 master + 2 replicas)
- Auto-configuration via init container (master vs replica)
- Increased memory limits: 256Mi → 1Gi
- Improved readiness probes with replication status checks
- Add Valkey Sentinel StatefulSet (3 instances for quorum)
- Automatic failover detection (5s down-after-milliseconds)
- Fast failover execution (10s timeout)
- Monitors master and promotes replicas automatically
- Add Sentinel headless service for pod discovery
Application Resilience:
- Update Phoenix.PubSub.Redis with TCP keepalive and reconnection
- Connection timeout: 5s
- Exponential backoff: 500ms → 30s
- exit_on_disconnection: false
- Update Exq background jobs with same resilience settings
- TCP keepalive enabled
- Better connection pool management
Benefits:
- Automatic failover when Valkey pod dies (no manual intervention)
- Zero data loss with replica synchronization
- Fast failure detection and recovery (5-10s total)
- Survives Flannel CNI networking issues
- Apps reconnect automatically during disconnections
Testing:
- All 3,686 tests passing
- Kustomize manifest validated
🤖 Generated with Claude Code
100 lines
3.2 KiB
Elixir
100 lines
3.2 KiB
Elixir
defmodule Towerops.ExqSupervisor do
|
|
@moduledoc """
|
|
Custom supervisor for Exq background job processor with Redis health checks.
|
|
|
|
This supervisor adds resilience to Exq by:
|
|
1. Waiting for Redis to be available before starting Exq
|
|
2. Using a restart strategy that prevents rapid crash loops
|
|
3. Logging detailed error information when crashes occur
|
|
|
|
Handles the case where Exq.Node.Server crashes due to nil responses
|
|
from Redis when connections fail.
|
|
"""
|
|
|
|
use Supervisor
|
|
|
|
require Logger
|
|
|
|
def start_link(opts) do
|
|
Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
|
|
end
|
|
|
|
@impl true
|
|
def init(_opts) do
|
|
redis_config = Application.get_env(:towerops, :redis, [])
|
|
|
|
# Wait for Redis to be available before starting Exq
|
|
case Towerops.RedisHealthCheck.wait_for_redis(redis_config, max_attempts: 10, backoff: 1_000) do
|
|
:ok ->
|
|
Logger.info("Redis is available, starting Exq")
|
|
start_exq_children(redis_config)
|
|
|
|
{:error, reason} ->
|
|
Logger.error("Failed to connect to Redis: #{inspect(reason)}. Exq will not start.")
|
|
# Return empty children list - supervisor will still start but Exq won't run
|
|
# This allows the application to continue functioning without background jobs
|
|
{:ok, {:one_for_one, []}}
|
|
end
|
|
end
|
|
|
|
defp start_exq_children(redis_config) do
|
|
exq_config = build_exq_config(redis_config)
|
|
|
|
children = [
|
|
# Wrap Exq in a supervisor with restart strategy
|
|
%{
|
|
id: Exq,
|
|
start: {Exq, :start_link, [exq_config]},
|
|
restart: :permanent,
|
|
type: :supervisor,
|
|
# Add longer shutdown timeout to allow graceful job completion
|
|
shutdown: 30_000
|
|
}
|
|
]
|
|
|
|
# Use one_for_one strategy with limited restarts to prevent crash loops
|
|
# If Exq crashes more than 3 times in 60 seconds, the supervisor will crash
|
|
# This will be caught by the application supervisor which will restart this supervisor
|
|
opts = [
|
|
strategy: :one_for_one,
|
|
max_restarts: 3,
|
|
max_seconds: 60
|
|
]
|
|
|
|
Supervisor.init(children, opts)
|
|
end
|
|
|
|
defp build_exq_config(redis_config) do
|
|
[
|
|
name: Exq,
|
|
host: Keyword.get(redis_config, :host, "localhost"),
|
|
port: Keyword.get(redis_config, :port, 6379),
|
|
namespace: "exq",
|
|
concurrency: 10,
|
|
queues: ["default", "discovery", "polling", "monitoring", "maintenance"],
|
|
# Poll settings - how long to wait for Redis responses
|
|
poll_timeout: 50,
|
|
scheduler_poll_timeout: 200,
|
|
scheduler_enable: true,
|
|
# Job retry settings
|
|
max_retries: 50,
|
|
# Exponential backoff starting at 500ms, up to 60s
|
|
mode: :default,
|
|
shutdown_timeout: 30_000,
|
|
# Redix connection options for better resilience
|
|
redis_options: [
|
|
# Synchronous connect to fail fast if Redis unavailable
|
|
sync_connect: true,
|
|
# Don't exit process on disconnection - let Exq handle it
|
|
exit_on_disconnection: false,
|
|
# Connection timeout
|
|
timeout: 5_000,
|
|
# Backoff for reconnection attempts
|
|
backoff_initial: 500,
|
|
backoff_max: 30_000,
|
|
# TCP keepalive to detect dead connections
|
|
socket_opts: [keepalive: true]
|
|
]
|
|
]
|
|
end
|
|
end
|