From 9c2d95b55a9deed686255b363ea76144a3c0d50b Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sat, 17 Jan 2026 12:44:22 -0600
Subject: [PATCH] Add zero-downtime deployment configuration

Prevent WebSocket disconnections during deployments by:

1. Rolling Update Strategy:
   - maxSurge: 1 (allow 1 extra pod during rollout)
   - maxUnavailable: 0 (keep all pods running)
   - minReadySeconds: 10 (wait before continuing rollout)

2. Graceful Shutdown:
   - terminationGracePeriodSeconds: 30 (allow Phoenix to drain connections)

3. PodDisruptionBudget:
   - minAvailable: 1 (ensure at least 1 pod always available)
   - Prevents all pods from being terminated simultaneously

4. Improved Health Checks:
   - Faster readiness probe (5s initial, 3s period)
   - More aggressive success/failure thresholds
   - New pods marked ready faster

This ensures new pods are fully ready and serving traffic before old
pods are terminated, maintaining WebSocket connections and preventing
'something went wrong' errors during deployments.
---
 k8s/deployment.yaml          | 16 ++++++++++++++--
 k8s/kustomization.yaml       |  1 +
 k8s/poddisruptionbudget.yaml | 11 +++++++++++
 3 files changed, 26 insertions(+), 2 deletions(-)
 create mode 100644 k8s/poddisruptionbudget.yaml

diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
index 6effd4cb..54186ab2 100644
--- a/k8s/deployment.yaml
+++ b/k8s/deployment.yaml
@@ -6,6 +6,12 @@ metadata:
   namespace: towerops
 spec:
   replicas: 2
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1        # Allow 1 extra pod during rollout
+      maxUnavailable: 0  # Keep all pods running during rollout
+  minReadySeconds: 10    # Wait 10s after pod is ready before continuing
   selector:
     matchLabels:
       app: towerops
@@ -14,6 +20,7 @@ spec:
       labels:
         app: towerops
     spec:
+      terminationGracePeriodSeconds: 30  # Allow 30s for graceful shutdown
       imagePullSecrets:
         - name: gitlab-registry
       securityContext:
@@ -95,9 +102,14 @@ spec:
               port: 4000
             initialDelaySeconds: 30
             periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 3
           readinessProbe:
             httpGet:
               path: /health
               port: 4000
-            initialDelaySeconds: 10
-            periodSeconds: 5
+            initialDelaySeconds: 5
+            periodSeconds: 3
+            timeoutSeconds: 2
+            successThreshold: 1
+            failureThreshold: 2
diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml
index 37dff34c..1456b5b8 100644
--- a/k8s/kustomization.yaml
+++ b/k8s/kustomization.yaml
@@ -8,3 +8,4 @@ resources:
   - service-headless.yaml
   - certificate.yaml
   - ingressroute.yaml
+  - poddisruptionbudget.yaml
diff --git a/k8s/poddisruptionbudget.yaml b/k8s/poddisruptionbudget.yaml
new file mode 100644
index 00000000..c538e4bd
--- /dev/null
+++ b/k8s/poddisruptionbudget.yaml
@@ -0,0 +1,11 @@
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: towerops-pdb
+  namespace: towerops
+spec:
+  minAvailable: 1  # Keep at least 1 pod running during voluntary disruptions
+  selector:
+    matchLabels:
+      app: towerops