From 45d84668081dc5ddb377702e7741c547adfa1aa3 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Wed, 29 Apr 2026 14:13:22 -0500
Subject: [PATCH] ops(prop): preStop sleep so Service drains before Phoenix
 shuts down
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Deploys were causing 502 Bad Gateway on LiveView reconnect. The kubelet
sent SIGTERM to the old pod and Phoenix immediately stopped accepting
connections, but the Service endpoint controller and ingress hadn't
yet propagated the pod's removal from endpoints — the browser's
reconnect raced the endpoint update and bounced 502 against the dead
pod, then gave up.

Add a 10s preStop sleep on the container and bump
terminationGracePeriodSeconds to 40 so the load balancer has time to
drain this pod before Phoenix begins shutting down.
---
 k8s/deployment.yaml | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
index ac510baf..a4d92e9c 100644
--- a/k8s/deployment.yaml
+++ b/k8s/deployment.yaml
@@ -38,6 +38,17 @@ spec:
       serviceAccountName: prop
       imagePullSecrets:
         - name: forgejo-registry
+      # The deploy was producing 502s on LiveView reconnect: kubelet sent
+      # SIGTERM to the old pod and Phoenix immediately stopped accepting
+      # WebSocket connections, but the Service endpoint controller and
+      # ingress hadn't yet propagated the pod's removal from endpoints.
+      # The browser's reconnect attempt would route to the dead pod and
+      # bounce 502.
+      #
+      # 30s grace + a 10s preStop sleep on the container gives the
+      # endpoint controller a comfortable window to drain this pod from
+      # the load balancer before Phoenix starts shutting down.
+      terminationGracePeriodSeconds: 40
       securityContext:
         runAsUser: 65534
         runAsNonRoot: true
@@ -100,6 +111,14 @@ spec:
             periodSeconds: 10
             failureThreshold: 3
             timeoutSeconds: 5
+          # Sleep before SIGTERM hits Phoenix so the Service endpoint
+          # controller has time to remove this pod from the LB pool —
+          # without it, in-flight LiveView reconnects raced the
+          # endpoint update and bounced 502 against the dying pod.
+          lifecycle:
+            preStop:
+              exec:
+                command: ["sh", "-c", "sleep 10"]
           resources:
             requests:
               cpu: 100m