From 04146af1fa7315a9500aab50d20e15f00efcff58 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Wed, 22 Apr 2026 13:56:10 -0500
Subject: [PATCH] fix(k8s): raise prop-backfill probe timeout to 15s

The exec probe runs `/app/bin/microwaveprop rpc ':ok'` which spawns a
release CLI and round-trips through distributed Erlang. Under an IEM
429 retry storm (WeatherFetchWorker sleeps 1-16 s between attempts),
the BEAM scheduler stays responsive but probe latency spikes past 5 s
and the kubelet SIGKILLs the pod every ~10 min. 15 s is still well
below the 30 s period so the probe still catches a genuinely wedged
VM inside one cycle.
---
 k8s/deployment-backfill.yaml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/k8s/deployment-backfill.yaml b/k8s/deployment-backfill.yaml
index 21364dbf..6e218212 100644
--- a/k8s/deployment-backfill.yaml
+++ b/k8s/deployment-backfill.yaml
@@ -73,20 +73,25 @@ spec:
             - secretRef:
                 name: prop-secrets
           # No HTTP probes (no web server). Use exec probe against the
-          # running BEAM to confirm the VM is responsive.
+          # running BEAM to confirm the VM is responsive. timeoutSeconds
+          # is generous because `rpc ':ok'` has to spawn a release CLI
+          # and round-trip through distributed Erlang; during an IEM
+          # 429 retry storm (WeatherFetchWorker sleeping 1-16 s between
+          # attempts) the scheduler is responsive but slow, and a 5 s
+          # probe timeout was tripping failureThreshold every ~10 min.
           startupProbe:
             exec:
               command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
             initialDelaySeconds: 10
             periodSeconds: 5
             failureThreshold: 24
-            timeoutSeconds: 5
+            timeoutSeconds: 15
           livenessProbe:
             exec:
               command: ["/bin/sh", "-c", "/app/bin/microwaveprop rpc ':ok'"]
             periodSeconds: 30
             failureThreshold: 3
-            timeoutSeconds: 5
+            timeoutSeconds: 15
           resources:
             # 8GB T620 box — leave ~3GB for kubelet/containerd/OS.
             requests: