Graham McIntire
ca33b8331b
1 Gi OOM'd during a real chain step. Each forecast hour loads both a surface and pressure-level HRRR GRIB2 (~40 MB combined), decodes via wgrib2 subprocess (which allocates its own buffers), scores 92k grid points × 23 bands (~2.1M intermediates), and writes 23 score files. 2 Gi absorbs the spike; still ~3× smaller than the Elixir-per-pod budget we're replacing.
98 lines
3.3 KiB
YAML
98 lines
3.3 KiB
YAML
apiVersion: apps/v1
|
||
kind: Deployment
|
||
metadata:
|
||
name: prop-grid-rs
|
||
namespace: prop
|
||
# Phase 1 (shadow) default: write to /data/scores_shadow so Elixir keeps
|
||
# owning the live /data/scores tree. Flip PROP_SCORES_DIR to /data/scores
|
||
# at cutover after ShadowComparator shows < 0.5 mean delta for 72h.
|
||
spec:
|
||
replicas: 1
|
||
minReadySeconds: 5
|
||
strategy:
|
||
type: RollingUpdate
|
||
rollingUpdate:
|
||
maxSurge: 0
|
||
maxUnavailable: 1
|
||
selector:
|
||
matchLabels:
|
||
app: prop-grid-rs
|
||
template:
|
||
metadata:
|
||
labels:
|
||
app: prop-grid-rs
|
||
tier: grid-rs
|
||
spec:
|
||
# Pin to talos5 — the 32 GB NUC with NVMe. The worker peaks at ~500 Mi
|
||
# steady-state but benefits from local NVMe for the NFS scores-dir
|
||
# writes (faster fsync) and from never sharing the node with a hot
|
||
# Elixir pod. Label talos5 with `prop-grid-rs=primary` and taint it
|
||
# `workload=grid-rs:NoSchedule` to enforce this.
|
||
nodeSelector:
|
||
prop-grid-rs: "primary"
|
||
tolerations:
|
||
- key: workload
|
||
operator: Equal
|
||
value: grid-rs
|
||
effect: NoSchedule
|
||
imagePullSecrets:
|
||
- name: forgejo-registry
|
||
securityContext:
|
||
runAsUser: 65534
|
||
runAsNonRoot: true
|
||
fsGroup: 65534
|
||
seccompProfile:
|
||
type: RuntimeDefault
|
||
containers:
|
||
- name: prop-grid-rs
|
||
# Image built by a new CI pipeline from rust/prop_grid_rs/.
|
||
# Multi-arch (amd64 + arm64) — reuses the repo's existing
|
||
# buildx wiring.
|
||
image: git.mcintire.me/graham/prop-grid-rs:main-1776634333-2d67cf7 # {"$imagepolicy": "flux-system:prop-grid-rs"}
|
||
imagePullPolicy: IfNotPresent
|
||
env:
|
||
# Elixir secrets bundle carries DATABASE_URL already. Rust
|
||
# reads it directly (sqlx connects with the same URL).
|
||
- name: HRRR_BASE_URL
|
||
value: "http://skippy.w5isp.com:8080"
|
||
- name: PROP_SCORES_DIR
|
||
value: "/data/scores_shadow"
|
||
- name: RUST_LOG
|
||
value: "info"
|
||
- name: PROP_GRID_RS_PG_CONNS
|
||
value: "4"
|
||
envFrom:
|
||
- secretRef:
|
||
name: prop-secrets
|
||
# No HTTP surface. Liveness is "the process is still running";
|
||
# let the kubelet restart us if we crash. Claiming work from
|
||
# grid_tasks is the implicit readiness signal — an idle pod is
|
||
# still healthy (the queue is simply empty).
|
||
resources:
|
||
requests:
|
||
cpu: 100m
|
||
memory: 512Mi
|
||
limits:
|
||
cpu: "4"
|
||
# OOM at 1 Gi during active chain work (92k points × 23
|
||
# bands ≈ 2.1M scores, plus decoded GRIB2 buffers, plus
|
||
# wgrib2 subprocess memory). 2 Gi leaves headroom; we're
|
||
# still an order of magnitude below the 6 Gi Elixir budget.
|
||
memory: 2Gi
|
||
securityContext:
|
||
allowPrivilegeEscalation: false
|
||
runAsNonRoot: true
|
||
runAsUser: 65534
|
||
capabilities:
|
||
drop:
|
||
- ALL
|
||
seccompProfile:
|
||
type: RuntimeDefault
|
||
volumeMounts:
|
||
- name: data
|
||
mountPath: /data
|
||
volumes:
|
||
- name: data
|
||
nfs:
|
||
server: 10.0.15.103
|
||
path: /data
|