From b77950aed6dc17b78258927d01b53c1e23f2a14b Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Fri, 20 Mar 2026 10:19:43 -0500
Subject: [PATCH] Fix persistent DB connection timeouts in k8s (#93)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- TCP keepalive more aggressive (14s detection vs 30s) so dead
  connections are found before the 15s checkout timeout fires
- Reduce pool size 30→15 per pod (2 pods = 30 total, avoids
  exhaustion during rolling deploys when 4 pods run briefly)
- Add idle_interval: 1s to proactively ping idle connections and
  detect dead SSL sockets before they're handed to real queries
- Simplify SSL config: remove redundant verify_fun and SNI disable
  when verify_none already skips certificate validation

Reviewed-on: https://git.mcintire.me/graham/towerops-web/pulls/93
---
 config/runtime.exs | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index 214ba2e9..9d560c4e 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -111,11 +111,7 @@ if config_env() == :prod do
       "true" ->
         case System.get_env("DATABASE_SSL_VERIFY") do
           "false" ->
-            [
-              verify: :verify_none,
-              server_name_indication: :disable,
-              verify_fun: {fn _, _, _ -> {:valid, :ok} end, :ok}
-            ]
+            [verify: :verify_none]
 
           _ ->
             [verify: :verify_peer]
@@ -152,16 +148,16 @@ if config_env() == :prod do
   # Default OS keepalive is ~2 hours on Linux, far too slow for k8s where
   # load balancers and proxies silently drop idle connections within minutes.
   # Linux raw socket options (SOL_TCP=6):
-  #   TCP_KEEPIDLE  (4) = 15s  - seconds idle before first keepalive probe
-  #   TCP_KEEPINTVL (5) = 5s   - seconds between subsequent probes
+  #   TCP_KEEPIDLE  (4) = 5s   - seconds idle before first keepalive probe
+  #   TCP_KEEPINTVL (5) = 3s   - seconds between subsequent probes
   #   TCP_KEEPCNT   (6) = 3    - failed probes before connection is dead
-  # Total dead-connection detection: 15 + (5 * 3) = 30 seconds
+  # Total dead-connection detection: 5 + (3 * 3) = 14 seconds
   tcp_keepalive_opts =
     if System.get_env("KUBERNETES_SERVICE_HOST") do
       [
         {:keepalive, true},
-        {:raw, 6, 4, <<15::native-32>>},
-        {:raw, 6, 5, <<5::native-32>>},
+        {:raw, 6, 4, <<5::native-32>>},
+        {:raw, 6, 5, <<3::native-32>>},
         {:raw, 6, 6, <<3::native-32>>}
       ]
     else
@@ -326,7 +322,11 @@ if config_env() == :prod do
   config :towerops, Towerops.Repo,
     ssl: ssl_config,
     url: database_url,
-    pool_size: String.to_integer(System.get_env("POOL_SIZE") || "30"),
+    pool_size: String.to_integer(System.get_env("POOL_SIZE") || "15"),
+    # Proactively ping idle connections every 1s to detect dead SSL connections
+    # before they're handed to a real query. Without this, a dead connection
+    # sits in the pool until a query checks it out and hangs for the full timeout.
+    idle_interval: 1_000,
     # Detect stale connections faster: if a query takes longer than queue_target (2s)
     # on average over queue_interval (5s), DBConnection will disconnect idle connections
     # to cycle in fresh ones. Helps recover from "ssl recv: closed" errors.