From 86b27189160fcb1d7039d29729f6e6c07139993f Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 23 Apr 2026 18:42:39 -0500 Subject: [PATCH] infra(k8s): load-based HPAs, drop anti-affinity, retire talos5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds HorizontalPodAutoscaler resources for the three load-responsive Deployments: - prop (web): min 2, max 5, averageValue 800m CPU - prop-grid-rs: min 1, max 4, averageValue 400m - hrrr-point-rs: min 1, max 4, averageValue 400m - prop-backfill is intentionally unmanaged (queue-driven, not CPU-bound) The scale-up policy reacts in 30 s with up-to-2-pods or +100 % per window; scale-down stabilises over 10 min so the hourly propagation chain doesn't churn replicas right after the :05 cron. Removes `spec.replicas` from the HPA-managed Deployments so Flux's periodic reconcile stops fighting HPA's live scaling decisions. Drops `podAntiAffinity` from prop, prop-grid-rs, and hrrr-point-rs so HPA can actually reach its maxReplicas when one physical host has capacity for two pods; the scheduler picks whatever fits. Updates runbook + project memory to drop the `talos5` hostname — Postgres is now addressed strictly by IP (10.0.15.30). --- docs/runbook_propagation_pipeline.md | 2 +- k8s/deployment-grid-rs.yaml | 23 ++--- k8s/deployment-hrrr-point-rs.yaml | 23 ++--- k8s/deployment.yaml | 11 +-- k8s/hpa.yaml | 125 +++++++++++++++++++++++++++ k8s/kustomization.yaml | 1 + 6 files changed, 146 insertions(+), 39 deletions(-) create mode 100644 k8s/hpa.yaml diff --git a/docs/runbook_propagation_pipeline.md b/docs/runbook_propagation_pipeline.md index b09cb67c..0e19e0ec 100644 --- a/docs/runbook_propagation_pipeline.md +++ b/docs/runbook_propagation_pipeline.md @@ -5,7 +5,7 @@ NFS volume. This doc names every failure mode at that seam and says how the system recovers. ``` -prop-grid-rs (worker nodes) Postgres (talos5 — 10.0.15.30) prop pods (node1/2/3) +prop-grid-rs (worker nodes) Postgres (10.0.15.30) prop pods (node1/2/3) ├─ fetch GRIB2 (NOMADS) ├─ grid_tasks (SKIP LOCKED) ├─ ScoreCache (ETS) ├─ decode ├─ LISTEN/NOTIFY channel ├─ NotifyListener ├─ score 22 bands × 92k │ "propagation_ready" ├─ ScoreCacheReconciler (60s) diff --git a/k8s/deployment-grid-rs.yaml b/k8s/deployment-grid-rs.yaml index ce499d76..8a140cb8 100644 --- a/k8s/deployment-grid-rs.yaml +++ b/k8s/deployment-grid-rs.yaml @@ -9,11 +9,11 @@ metadata: # commercial-link degradation, ProfilesFile write). f01..f18 land # here from the Rust worker. spec: - # Three replicas claim from grid_tasks via FOR UPDATE SKIP LOCKED - # with no coordination. Anti-affinity spreads them across distinct - # hosts so any single node going down leaves the hourly chain with - # two workers still draining. - replicas: 3 + # Replicas claim from grid_tasks via FOR UPDATE SKIP LOCKED with no + # coordination; anti-affinity spreads them across distinct hosts so + # any single node going down leaves the hourly chain with other + # workers still draining. + # Replica count is HPA-managed (hpa.yaml, min 1 / max 4). minReadySeconds: 5 strategy: type: RollingUpdate @@ -35,15 +35,10 @@ spec: prometheus.io/port: "9100" prometheus.io/path: "/metrics" spec: - # No node pinning — let the scheduler place replicas on any host - # the anti-affinity rule allows. talos5 is reserved for Postgres. - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: prop-grid-rs - topologyKey: kubernetes.io/hostname + # No node pinning — scheduler picks any host. Anti-affinity was + # removed so HPA can stack multiple replicas on the same node if + # that's where the scheduler wants them; grid-rs is CPU-bound but + # not memory-bound so colocating a couple of replicas is fine. imagePullSecrets: - name: forgejo-registry securityContext: diff --git a/k8s/deployment-hrrr-point-rs.yaml b/k8s/deployment-hrrr-point-rs.yaml index a8fc7ad1..5ae72812 100644 --- a/k8s/deployment-hrrr-point-rs.yaml +++ b/k8s/deployment-hrrr-point-rs.yaml @@ -8,9 +8,10 @@ metadata: # points JSONB array); this deployment drains via FOR UPDATE SKIP # LOCKED, fetches each HRRR cycle once, and upserts hrrr_profiles. spec: - # Two replicas for parallelism on QSO-submit spikes. FOR UPDATE SKIP - # LOCKED on hrrr_fetch_tasks lets both claim without coordination. - replicas: 2 + # Parallelism on QSO-submit spikes comes from multiple replicas + # claiming rows from hrrr_fetch_tasks via FOR UPDATE SKIP LOCKED — + # no coordination needed between workers. + # Replica count is HPA-managed (hpa.yaml, min 1 / max 4). strategy: type: RollingUpdate rollingUpdate: @@ -25,19 +26,9 @@ spec: app: hrrr-point-rs tier: hrrr-points spec: - affinity: - # Avoid co-locating with prop-grid-rs pods — they compete for - # the same wgrib2 subprocess slots and NFS fsyncs. Grid chain - # is latency-sensitive (users refreshing /map); per-QSO point - # work is throughput-sensitive but latency-tolerant. - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 80 - podAffinityTerm: - labelSelector: - matchLabels: - app: prop-grid-rs - topologyKey: kubernetes.io/hostname + # Anti-affinity against prop-grid-rs was removed; the scheduler + # is free to colocate. Contention on wgrib2 + NFS hasn't been an + # observed bottleneck in practice. imagePullSecrets: - name: forgejo-registry securityContext: diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml index cd346745..f8ae99b7 100644 --- a/k8s/deployment.yaml +++ b/k8s/deployment.yaml @@ -4,7 +4,9 @@ metadata: name: prop namespace: prop spec: - replicas: 3 + # replicas is owned by the HPA in hpa.yaml (min 2 / max 5). + # Leaving it out here so Flux's periodic reconcile doesn't overwrite + # whatever replica count the autoscaler has settled on. minReadySeconds: 5 strategy: type: RollingUpdate @@ -28,13 +30,6 @@ spec: - key: node-role.kubernetes.io/control-plane operator: Exists effect: NoSchedule - affinity: - podAntiAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - - labelSelector: - matchLabels: - app: prop - topologyKey: topology.kubernetes.io/physical-host serviceAccountName: prop imagePullSecrets: - name: forgejo-registry diff --git a/k8s/hpa.yaml b/k8s/hpa.yaml new file mode 100644 index 00000000..731d60c7 --- /dev/null +++ b/k8s/hpa.yaml @@ -0,0 +1,125 @@ +# Horizontal pod autoscalers for the three load-responsive Deployments. +# prop-backfill is deliberately excluded — it's a single-replica catch-up +# worker that's bottlenecked on IEM rate-limit waits rather than CPU, so +# scaling by CPU would just add idle replicas parked on 429 backoffs. +# +# CPU targets use AverageValue (absolute cores) rather than Utilization +# (% of request) so the scale-up thresholds don't depend on the request +# number. Request fields in the Deployments can be tuned independently +# for scheduler bin-packing without accidentally changing when HPA +# decides to add a replica. +# +# The corresponding Deployments have their `spec.replicas` removed so +# Flux's periodic reconcile doesn't fight HPA's live scaling. +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: prop + namespace: prop +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: prop + minReplicas: 2 + maxReplicas: 5 + metrics: + - type: Resource + resource: + name: cpu + target: + # Average of 800m across pods triggers scale-up. The hourly + # :05 propagation chain + ScoreCache warming on one pod can + # peak near 2 cores — averaging across 2–3 replicas keeps the + # threshold meaningful without flapping on every cron tick. + type: AverageValue + averageValue: 800m + behavior: + scaleUp: + # React quickly to sudden surges (LiveView + propagation chain + # can coincide). Add up to 2 pods or 100% of current replica + # count per 30 s window, whichever is larger. + stabilizationWindowSeconds: 30 + policies: + - type: Pods + value: 2 + periodSeconds: 30 + - type: Percent + value: 100 + periodSeconds: 30 + selectPolicy: Max + scaleDown: + # Slower scale-down so the hourly chain doesn't immediately + # trigger a churn of terminations five minutes later. + stabilizationWindowSeconds: 600 + policies: + - type: Pods + value: 1 + periodSeconds: 120 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: prop-grid-rs + namespace: prop +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: prop-grid-rs + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: AverageValue + averageValue: 400m + behavior: + scaleUp: + stabilizationWindowSeconds: 30 + policies: + - type: Pods + value: 1 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 600 + policies: + - type: Pods + value: 1 + periodSeconds: 120 +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: hrrr-point-rs + namespace: prop +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: hrrr-point-rs + minReplicas: 1 + maxReplicas: 4 + metrics: + - type: Resource + resource: + name: cpu + target: + type: AverageValue + averageValue: 400m + behavior: + scaleUp: + stabilizationWindowSeconds: 30 + policies: + - type: Pods + value: 1 + periodSeconds: 30 + scaleDown: + stabilizationWindowSeconds: 600 + policies: + - type: Pods + value: 1 + periodSeconds: 120 diff --git a/k8s/kustomization.yaml b/k8s/kustomization.yaml index ea616ff3..da5e25df 100644 --- a/k8s/kustomization.yaml +++ b/k8s/kustomization.yaml @@ -12,3 +12,4 @@ resources: - deployment-hrrr-point-rs.yaml - service.yaml - metrics-service.yaml + - hpa.yaml