From 76854479c98fe89a4b0aa1ebc4748207fd899009 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sun, 25 Jan 2026 15:29:38 -0600
Subject: [PATCH] update deployment

---
 k8s/deployment.yaml | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
index 9ed6bf9a..773cc7b7 100644
--- a/k8s/deployment.yaml
+++ b/k8s/deployment.yaml
@@ -5,13 +5,13 @@ metadata:
   name: towerops
   namespace: towerops
 spec:
-  replicas: 1
+  replicas: 3 # Run 3 replicas for high availability
   strategy:
     type: RollingUpdate
     rollingUpdate:
       maxSurge: 1 # Allow 1 extra pod during rollout
       maxUnavailable: 0 # Keep all pods running during rollout
-  minReadySeconds: 20 # Wait 20s after pod is ready for cluster sync before continuing
+  minReadySeconds: 30 # Wait 30s after pod is ready before considering it available
   selector:
     matchLabels:
       app: towerops
@@ -22,7 +22,7 @@ spec:
     spec:
       # Use system-cluster-critical priority to ensure towerops starts after CNI is ready
       priorityClassName: system-cluster-critical
-      terminationGracePeriodSeconds: 60 # Allow 60s for Horde graceful shutdown (30s) + drain time
+      terminationGracePeriodSeconds: 90 # Allow 90s for graceful shutdown and connection draining
       imagePullSecrets:
         - name: gitlab-registry
       securityContext:
@@ -58,10 +58,10 @@ spec:
                   - /bin/sh
                   - -c
                   - |
-                    # Stop accepting new connections
-                    # Give etcd coordinator time to release locks (watches trigger instant failover)
-                    # This sleep allows graceful shutdown before SIGTERM is sent
-                    sleep 10
+                    # Stop accepting new connections by failing readiness probe
+                    # Give load balancer time to remove pod from rotation
+                    # Allow existing connections to complete
+                    sleep 20
           env:
             - name: POD_IP
               valueFrom:
@@ -133,7 +133,8 @@ spec:
             httpGet:
               path: /health
               port: 4000
-            periodSeconds: 3
+            initialDelaySeconds: 5
+            periodSeconds: 5
             timeoutSeconds: 2
-            successThreshold: 1
-            failureThreshold: 1 # Fast deregistration from service during shutdown
+            successThreshold: 2 # Require 2 consecutive successes before marking ready
+            failureThreshold: 2 # Allow 1 failure before marking not ready