Graham McIntire
86b2718916
Adds HorizontalPodAutoscaler resources for the three load-responsive Deployments: - prop (web): min 2, max 5, averageValue 800m CPU - prop-grid-rs: min 1, max 4, averageValue 400m - hrrr-point-rs: min 1, max 4, averageValue 400m - prop-backfill is intentionally unmanaged (queue-driven, not CPU-bound) The scale-up policy reacts in 30 s with up-to-2-pods or +100 % per window; scale-down stabilises over 10 min so the hourly propagation chain doesn't churn replicas right after the :05 cron. Removes `spec.replicas` from the HPA-managed Deployments so Flux's periodic reconcile stops fighting HPA's live scaling decisions. Drops `podAntiAffinity` from prop, prop-grid-rs, and hrrr-point-rs so HPA can actually reach its maxReplicas when one physical host has capacity for two pods; the scheduler picks whatever fits. Updates runbook + project memory to drop the `talos5` hostname — Postgres is now addressed strictly by IP (10.0.15.30).
96 lines
3.5 KiB
YAML
96 lines
3.5 KiB
YAML
apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: hrrr-point-rs
|
|
namespace: prop
|
|
# Phase 3 Stream C: per-QSO HRRR fetches move off the BEAM pods.
|
|
# Elixir writes to hrrr_fetch_tasks (one row per valid_time with a
|
|
# points JSONB array); this deployment drains via FOR UPDATE SKIP
|
|
# LOCKED, fetches each HRRR cycle once, and upserts hrrr_profiles.
|
|
spec:
|
|
# Parallelism on QSO-submit spikes comes from multiple replicas
|
|
# claiming rows from hrrr_fetch_tasks via FOR UPDATE SKIP LOCKED —
|
|
# no coordination needed between workers.
|
|
# Replica count is HPA-managed (hpa.yaml, min 1 / max 4).
|
|
strategy:
|
|
type: RollingUpdate
|
|
rollingUpdate:
|
|
maxSurge: 0
|
|
maxUnavailable: 1
|
|
selector:
|
|
matchLabels:
|
|
app: hrrr-point-rs
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: hrrr-point-rs
|
|
tier: hrrr-points
|
|
spec:
|
|
# Anti-affinity against prop-grid-rs was removed; the scheduler
|
|
# is free to colocate. Contention on wgrib2 + NFS hasn't been an
|
|
# observed bottleneck in practice.
|
|
imagePullSecrets:
|
|
- name: forgejo-registry
|
|
securityContext:
|
|
runAsUser: 65534
|
|
runAsNonRoot: true
|
|
fsGroup: 65534
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
containers:
|
|
- name: hrrr-point-rs
|
|
# Shares the prop-grid-rs image so one CI pipeline produces
|
|
# both binaries. k8s `command:` overrides the default
|
|
# ENTRYPOINT to launch the per-QSO point worker.
|
|
image: git.mcintire.me/graham/prop-grid-rs:main-1776894678-93b526c # {"$imagepolicy": "flux-system:prop-grid-rs"}
|
|
imagePullPolicy: IfNotPresent
|
|
command: ["/usr/bin/tini", "--", "/usr/local/bin/hrrr-point-worker"]
|
|
env:
|
|
- name: HRRR_BASE_URL
|
|
value: "http://skippy.w5isp.com:8080"
|
|
- name: HRRR_FALLBACK_BASE_URL
|
|
value: "https://noaa-hrrr-bdp-pds.s3.amazonaws.com"
|
|
- name: RUST_LOG
|
|
value: "info"
|
|
- name: PROP_GRID_RS_PG_CONNS
|
|
value: "4"
|
|
# Shared NFS idx cache — same directory as prop-grid-rs so a
|
|
# chain-step idx fetch warms both workers.
|
|
- name: HRRR_IDX_CACHE_DIR
|
|
value: "/data/hrrr_idx"
|
|
- name: OTEL_EXPORTER_OTLP_ENDPOINT
|
|
value: "http://otel-collector.observability.svc.cluster.local:4317"
|
|
envFrom:
|
|
- secretRef:
|
|
name: prop-secrets
|
|
resources:
|
|
requests:
|
|
cpu: 100m
|
|
memory: 256Mi
|
|
limits:
|
|
# Per-batch peak is a single CONUS surface + pressure
|
|
# decode (~40 MB blob, ~200 MB wgrib2 working set plus
|
|
# the full 92k-cell merged CellValues map kept in memory
|
|
# until all requested points are drained). 1 Gi OOMKilled
|
|
# repeatedly — four restarts in ten minutes — so bumped
|
|
# to 2 Gi. The scheduler-on-worker2 placement has plenty
|
|
# of headroom.
|
|
cpu: "2"
|
|
memory: 2Gi
|
|
securityContext:
|
|
allowPrivilegeEscalation: false
|
|
runAsNonRoot: true
|
|
runAsUser: 65534
|
|
capabilities:
|
|
drop:
|
|
- ALL
|
|
seccompProfile:
|
|
type: RuntimeDefault
|
|
volumeMounts:
|
|
- name: data
|
|
mountPath: /data
|
|
volumes:
|
|
- name: data
|
|
nfs:
|
|
server: 10.0.15.103
|
|
path: /data
|