From 071729f65c4c9e2318932efee40ca70685c8a2af Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 20 Mar 2026 10:39:52 -0500 Subject: [PATCH] Reduce pool size and deployment surge to prevent PG connection exhaustion (#94) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PG max_connections=100 (97 usable). With pool_size=15 and maxSurge=100%, rolling deploys briefly run 4 pods × 15 = 60 pool connections plus Oban notifiers, exceeding the limit and causing too_many_connections errors that cascade into ssl recv: closed timeouts. - pool_size: 15 → 10 (steady state: 2×10=20, deploy: 3×10=30) - maxSurge: 100% → 1 (max 3 pods during rollout instead of 4) Reviewed-on: https://git.mcintire.me/graham/towerops-web/pulls/94 --- config/runtime.exs | 2 +- k8s/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 9d560c4e..7dcdaa0e 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -322,7 +322,7 @@ if config_env() == :prod do config :towerops, Towerops.Repo, ssl: ssl_config, url: database_url, - pool_size: String.to_integer(System.get_env("POOL_SIZE") || "15"), + pool_size: String.to_integer(System.get_env("POOL_SIZE") || "10"), # Proactively ping idle connections every 1s to detect dead SSL connections # before they're handed to a real query. Without this, a dead connection # sits in the pool until a query checks it out and hangs for the full timeout. diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index ff7f2339..b5b56f16 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -8,7 +8,7 @@ spec: strategy: type: RollingUpdate rollingUpdate: - maxSurge: 100% # Start all new pods at once for fast rollouts + maxSurge: 1 # Limit to 3 pods during rollout to avoid exceeding PG max_connections maxUnavailable: 0 # Keep all pods running during rollout minReadySeconds: 10 # Wait 10s after pod is ready before considering it available selector: