infra(k8s): load-based HPAs, drop anti-affinity, retire talos5

Adds HorizontalPodAutoscaler resources for the three load-responsive
Deployments:

- prop (web): min 2, max 5, averageValue 800m CPU
- prop-grid-rs: min 1, max 4, averageValue 400m
- hrrr-point-rs: min 1, max 4, averageValue 400m
- prop-backfill is intentionally unmanaged (queue-driven, not CPU-bound)

The scale-up policy reacts in 30 s with up-to-2-pods or +100 % per
window; scale-down stabilises over 10 min so the hourly propagation
chain doesn't churn replicas right after the :05 cron.

Removes `spec.replicas` from the HPA-managed Deployments so Flux's
periodic reconcile stops fighting HPA's live scaling decisions.

Drops `podAntiAffinity` from prop, prop-grid-rs, and hrrr-point-rs
so HPA can actually reach its maxReplicas when one physical host has
capacity for two pods; the scheduler picks whatever fits.

Updates runbook + project memory to drop the `talos5` hostname —
Postgres is now addressed strictly by IP (10.0.15.30).

docs/runbook_propagation_pipeline.md

@@ -5,7 +5,7 @@ NFS volume. This doc names every failure mode at that seam and says
 how the system recovers.
 
 ```
-prop-grid-rs (worker nodes)      Postgres (talos5 — 10.0.15.30)  prop pods (node1/2/3)
+prop-grid-rs (worker nodes)      Postgres (10.0.15.30)            prop pods (node1/2/3)
  ├─ fetch GRIB2 (NOMADS)         ├─ grid_tasks (SKIP LOCKED)     ├─ ScoreCache (ETS)
  ├─ decode                       ├─ LISTEN/NOTIFY channel        ├─ NotifyListener
  ├─ score 22 bands × 92k         │    "propagation_ready"        ├─ ScoreCacheReconciler (60s)

k8s/deployment-grid-rs.yaml

@@ -9,11 +9,11 @@ metadata:
   # commercial-link degradation, ProfilesFile write). f01..f18 land
   # here from the Rust worker.
 spec:
-  # Three replicas claim from grid_tasks via FOR UPDATE SKIP LOCKED
+  # Replicas claim from grid_tasks via FOR UPDATE SKIP LOCKED with no
-  # with no coordination. Anti-affinity spreads them across distinct
+  # coordination; anti-affinity spreads them across distinct hosts so
-  # hosts so any single node going down leaves the hourly chain with
+  # any single node going down leaves the hourly chain with other
-  # two workers still draining.
+  # workers still draining.
-  replicas: 3
+  # Replica count is HPA-managed (hpa.yaml, min 1 / max 4).
   minReadySeconds: 5
   strategy:
     type: RollingUpdate
@@ -35,15 +35,10 @@ spec:
         prometheus.io/port: "9100"
         prometheus.io/path: "/metrics"
     spec:
-      # No node pinning — let the scheduler place replicas on any host
+      # No node pinning — scheduler picks any host. Anti-affinity was
-      # the anti-affinity rule allows. talos5 is reserved for Postgres.
+      # removed so HPA can stack multiple replicas on the same node if
-      affinity:
+      # that's where the scheduler wants them; grid-rs is CPU-bound but
-        podAntiAffinity:
+      # not memory-bound so colocating a couple of replicas is fine.
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchLabels:
-                  app: prop-grid-rs
-              topologyKey: kubernetes.io/hostname
       imagePullSecrets:
         - name: forgejo-registry
       securityContext:

k8s/deployment-hrrr-point-rs.yaml

@@ -8,9 +8,10 @@ metadata:
   # points JSONB array); this deployment drains via FOR UPDATE SKIP
   # LOCKED, fetches each HRRR cycle once, and upserts hrrr_profiles.
 spec:
-  # Two replicas for parallelism on QSO-submit spikes. FOR UPDATE SKIP
+  # Parallelism on QSO-submit spikes comes from multiple replicas
-  # LOCKED on hrrr_fetch_tasks lets both claim without coordination.
+  # claiming rows from hrrr_fetch_tasks via FOR UPDATE SKIP LOCKED —
-  replicas: 2
+  # no coordination needed between workers.
+  # Replica count is HPA-managed (hpa.yaml, min 1 / max 4).
   strategy:
     type: RollingUpdate
     rollingUpdate:
@@ -25,19 +26,9 @@ spec:
         app: hrrr-point-rs
         tier: hrrr-points
     spec:
-      affinity:
+      # Anti-affinity against prop-grid-rs was removed; the scheduler
-        # Avoid co-locating with prop-grid-rs pods — they compete for
+      # is free to colocate. Contention on wgrib2 + NFS hasn't been an
-        # the same wgrib2 subprocess slots and NFS fsyncs. Grid chain
+      # observed bottleneck in practice.
-        # is latency-sensitive (users refreshing /map); per-QSO point
-        # work is throughput-sensitive but latency-tolerant.
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-            - weight: 80
-              podAffinityTerm:
-                labelSelector:
-                  matchLabels:
-                    app: prop-grid-rs
-                topologyKey: kubernetes.io/hostname
       imagePullSecrets:
         - name: forgejo-registry
       securityContext:

k8s/deployment.yaml

@@ -4,7 +4,9 @@ metadata:
   name: prop
   namespace: prop
 spec:
-  replicas: 3
+  # replicas is owned by the HPA in hpa.yaml (min 2 / max 5).
+  # Leaving it out here so Flux's periodic reconcile doesn't overwrite
+  # whatever replica count the autoscaler has settled on.
   minReadySeconds: 5
   strategy:
     type: RollingUpdate
@@ -28,13 +30,6 @@ spec:
         - key: node-role.kubernetes.io/control-plane
           operator: Exists
           effect: NoSchedule
-      affinity:
-        podAntiAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            - labelSelector:
-                matchLabels:
-                  app: prop
-              topologyKey: topology.kubernetes.io/physical-host
       serviceAccountName: prop
       imagePullSecrets:
         - name: forgejo-registry

k8s/hpa.yaml

@@ -0,0 +1,125 @@
+# Horizontal pod autoscalers for the three load-responsive Deployments.
+# prop-backfill is deliberately excluded — it's a single-replica catch-up
+# worker that's bottlenecked on IEM rate-limit waits rather than CPU, so
+# scaling by CPU would just add idle replicas parked on 429 backoffs.
+#
+# CPU targets use AverageValue (absolute cores) rather than Utilization
+# (% of request) so the scale-up thresholds don't depend on the request
+# number. Request fields in the Deployments can be tuned independently
+# for scheduler bin-packing without accidentally changing when HPA
+# decides to add a replica.
+#
+# The corresponding Deployments have their `spec.replicas` removed so
+# Flux's periodic reconcile doesn't fight HPA's live scaling.
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: prop
+  namespace: prop
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: prop
+  minReplicas: 2
+  maxReplicas: 5
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          # Average of 800m across pods triggers scale-up. The hourly
+          # :05 propagation chain + ScoreCache warming on one pod can
+          # peak near 2 cores — averaging across 2–3 replicas keeps the
+          # threshold meaningful without flapping on every cron tick.
+          type: AverageValue
+          averageValue: 800m
+  behavior:
+    scaleUp:
+      # React quickly to sudden surges (LiveView + propagation chain
+      # can coincide). Add up to 2 pods or 100% of current replica
+      # count per 30 s window, whichever is larger.
+      stabilizationWindowSeconds: 30
+      policies:
+        - type: Pods
+          value: 2
+          periodSeconds: 30
+        - type: Percent
+          value: 100
+          periodSeconds: 30
+      selectPolicy: Max
+    scaleDown:
+      # Slower scale-down so the hourly chain doesn't immediately
+      # trigger a churn of terminations five minutes later.
+      stabilizationWindowSeconds: 600
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 120
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: prop-grid-rs
+  namespace: prop
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: prop-grid-rs
+  minReplicas: 1
+  maxReplicas: 4
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: AverageValue
+          averageValue: 400m
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 30
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 30
+    scaleDown:
+      stabilizationWindowSeconds: 600
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 120
+---
+apiVersion: autoscaling/v2
+kind: HorizontalPodAutoscaler
+metadata:
+  name: hrrr-point-rs
+  namespace: prop
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: hrrr-point-rs
+  minReplicas: 1
+  maxReplicas: 4
+  metrics:
+    - type: Resource
+      resource:
+        name: cpu
+        target:
+          type: AverageValue
+          averageValue: 400m
+  behavior:
+    scaleUp:
+      stabilizationWindowSeconds: 30
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 30
+    scaleDown:
+      stabilizationWindowSeconds: 600
+      policies:
+        - type: Pods
+          value: 1
+          periodSeconds: 120

k8s/kustomization.yaml

@@ -12,3 +12,4 @@ resources:
   - deployment-hrrr-point-rs.yaml
   - service.yaml
   - metrics-service.yaml
+  - hpa.yaml