From 6c73012d09cbe4518157fe86ff98d9ebe314b17a Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sun, 19 Apr 2026 16:58:12 -0500
Subject: [PATCH] fix(k8s): tier label isolates hot pods from backfill
 endpoints
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Both the hot (`prop`) Deployment and the `prop-backfill` Deployment
labelled their pods with `app: prop` so both joined the same libcluster
Erlang cluster. The `prop` Service selector was `app: prop` too — which
meant the backfill pod (PHX_SERVER=false, no listener on :5000) was in
the Service endpoint pool.

With sessionAffinity: ClientIP, Cloudflared's source IP got hashed to
the backfill endpoint after a backfill pod roll and every request
from that client returned 502 (connection refused on :5000).

Fix: add `tier: hot` to hot-pod template labels; narrow the Service
selector to `{app: prop, tier: hot}`. Backfill keeps `tier: backfill`
and is cleanly excluded. libcluster selector (still `app: prop`) is
unchanged so backfill stays in the Erlang cluster for Oban queue
leader election and Oban.Pro's Smart engine.
---
 k8s/deployment.yaml | 5 +++++
 k8s/service.yaml    | 5 +++++
 2 files changed, 10 insertions(+)

diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
index 81296e89..61b0c147 100644
--- a/k8s/deployment.yaml
+++ b/k8s/deployment.yaml
@@ -17,7 +17,12 @@ spec:
   template:
     metadata:
       labels:
+        # `tier: hot` distinguishes these pods from prop-backfill (same
+        # app: prop label) so the Service selector can exclude backfill
+        # from its endpoint pool — backfill runs PHX_SERVER=false and
+        # doesn't listen on :5000.
         app: prop
+        tier: hot
     spec:
       tolerations:
         - key: node-role.kubernetes.io/control-plane
diff --git a/k8s/service.yaml b/k8s/service.yaml
index 8df5abc4..3fc249a0 100644
--- a/k8s/service.yaml
+++ b/k8s/service.yaml
@@ -11,6 +11,11 @@ spec:
       timeoutSeconds: 3600
   selector:
     app: prop
+    # Exclude prop-backfill (tier: backfill) — it runs PHX_SERVER=false
+    # and doesn't listen on :5000. A pre-tier-label deploy caused a
+    # Cloudflare 502 when ClientIP affinity pinned a source to the
+    # backfill endpoint.
+    tier: hot
   ports:
     - name: http
       port: 5000