From ed8b080b13457986c03c78158a35e50f81d452e4 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Wed, 29 Apr 2026 14:17:40 -0500
Subject: [PATCH] ops(prop): bump preStop sleep to 25s for cloudflared
 connection drain
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The ingress is cloudflared (Cloudflare Tunnel), which keeps an HTTP/2
connection pool to upstream Service IPs and reuses connections
aggressively. After a pod is removed from Service endpoints,
cloudflared can still hold in-flight HTTP/2 streams to the dying pod
for a noticeable window — 10 s wasn't enough, deploys still bounced
502 on LiveView reconnect.

Bump preStop sleep 10s → 25s and terminationGracePeriodSeconds 40s →
60s so cloudflared has time to turn over its upstream connections
before Phoenix begins draining.
---
 k8s/deployment.yaml | 32 +++++++++++++++++---------------
 1 file changed, 17 insertions(+), 15 deletions(-)

diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
index a4d92e9c..bda931fc 100644
--- a/k8s/deployment.yaml
+++ b/k8s/deployment.yaml
@@ -38,17 +38,20 @@ spec:
       serviceAccountName: prop
       imagePullSecrets:
         - name: forgejo-registry
-      # The deploy was producing 502s on LiveView reconnect: kubelet sent
-      # SIGTERM to the old pod and Phoenix immediately stopped accepting
-      # WebSocket connections, but the Service endpoint controller and
-      # ingress hadn't yet propagated the pod's removal from endpoints.
-      # The browser's reconnect attempt would route to the dead pod and
-      # bounce 502.
+      # The ingress here is cloudflared (Cloudflare Tunnel) which keeps
+      # an HTTP/2 connection pool to upstream Service IPs and reuses
+      # connections aggressively. After a pod is removed from Service
+      # endpoints, cloudflared can still hold an in-flight HTTP/2 stream
+      # to the dying pod for a noticeable window — long enough that
+      # client LiveView reconnect attempts during a deploy land on the
+      # dead pod and bounce 502.
       #
-      # 30s grace + a 10s preStop sleep on the container gives the
-      # endpoint controller a comfortable window to drain this pod from
-      # the load balancer before Phoenix starts shutting down.
-      terminationGracePeriodSeconds: 40
+      # The 25 s preStop sleep keeps the pod listening (and accepting
+      # final requests) while Service endpoint propagation + cloudflared
+      # connection-pool turnover both complete. SIGTERM only fires after
+      # that window, and Phoenix gets up to 60 s total before SIGKILL to
+      # finish draining open LiveView WebSockets.
+      terminationGracePeriodSeconds: 60
       securityContext:
         runAsUser: 65534
         runAsNonRoot: true
@@ -111,14 +114,13 @@ spec:
             periodSeconds: 10
             failureThreshold: 3
             timeoutSeconds: 5
-          # Sleep before SIGTERM hits Phoenix so the Service endpoint
-          # controller has time to remove this pod from the LB pool —
-          # without it, in-flight LiveView reconnects raced the
-          # endpoint update and bounced 502 against the dying pod.
+          # See terminationGracePeriodSeconds comment above — this sleep
+          # bridges the cloudflared connection-pool turnover so deploys
+          # don't bounce 502 on LiveView reconnect.
           lifecycle:
             preStop:
               exec:
-                command: ["sh", "-c", "sleep 10"]
+                command: ["sh", "-c", "sleep 25"]
           resources:
             requests:
               cpu: 100m