From 0bc8af22dd5795f5c921f824108034d81cea5bce Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Fri, 24 Apr 2026 10:52:57 -0500
Subject: [PATCH] fix(k8s): narrow backfill anti-affinity to tier=hot to
 unblock rollouts

Root cause of the deadlock prod hit today:

- prop-backfill's podAntiAffinity matched `app: prop`, which applies to
  backfill pods themselves as well as hot pods.
- The hot Deployment runs maxSurge=1/maxUnavailable=1 with HPA min 2 /
  max 5; when HPA was pegged at max and a rollout began, the old
  ReplicaSet's requiredDuringScheduling/IgnoredDuringExecution rule
  stayed pinned on every already-running hot pod.
- backfill's rule then excluded new hot pods from every physical host
  that had an app=prop pod on it, which was all of them, and the
  rollout couldn't progress because no old hot pod could come down
  until a new one went Ready first.

Narrowing the selector to `{app: prop, tier: hot}` means:
- backfill never constrains itself (it carries tier=backfill).
- Hot pods carry no anti-affinity of their own, so HPA can continue
  past the 4-physical-host count by co-locating when it has to.
- Backfill still lands on whichever physical host hot pods haven't
  filled up, which is the original intent.
---
 k8s/deployment-backfill.yaml | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/k8s/deployment-backfill.yaml b/k8s/deployment-backfill.yaml
index aef28a46..935b7f58 100644
--- a/k8s/deployment-backfill.yaml
+++ b/k8s/deployment-backfill.yaml
@@ -4,11 +4,18 @@ metadata:
   name: prop-backfill
   namespace: prop
 spec:
-  # Single replica. podAntiAffinity on app=prop / physical-host means
-  # this pod lands on whichever physical host the hot replicas aren't on
-  # (currently talos4 / node4); no nodeSelector or taint is needed to
-  # pin it — the antiaffinity is sufficient and keeps talos4 available
-  # as a normal scheduling target for the rest of the cluster.
+  # Single replica. podAntiAffinity targets `tier: hot` (not `app: prop`)
+  # so backfill only repels the web pods, not itself. An `app: prop`
+  # selector would also match backfill pods — fine at steady state, but
+  # during a rolling hot-pod rollout with HPA at max replicas the old
+  # RS's requiredDuringScheduling/IgnoredDuringExecution stays pinned on
+  # already-running hot pods, backfill's rule excludes the new hot pod
+  # from every physical host backfill hasn't touched, and the rollout
+  # deadlocks because maxUnavailable=1 can't bring an old hot pod down
+  # until a new one is Ready. Matching only `tier: hot` avoids that:
+  # backfill never constrains itself, and hot pods have no anti-affinity
+  # of their own so they can co-locate freely if HPA scales past the
+  # physical-host count.
   replicas: 1
   minReadySeconds: 5
   strategy:
@@ -35,6 +42,7 @@ spec:
             - labelSelector:
                 matchLabels:
                   app: prop
+                  tier: hot
               topologyKey: topology.kubernetes.io/physical-host
       serviceAccountName: prop
       imagePullSecrets: