From a3cfff304858057a7d08474418cf7d6ddf0ac4ab Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 20 Apr 2026 08:20:43 -0500 Subject: [PATCH] =?UTF-8?q?fix(grid-rs):=20bump=20memory=202=E2=86=923=20G?= =?UTF-8?q?i,=20parallelism=203=E2=86=922,=20re-trigger=20Rust=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs surfacing 13 h after Stream A cutover: 1. prop-grid-rs pods OOMKilled 16–18× overnight on the analysis step. f00's native-level GRIB2 (~530 MB + wgrib2 working set) runs on top of 2 concurrent forecast tasks, briefly reaching ~2 Gi. 2 Gi limit was too tight — 3 Gi plus parallelism 3→2 gives the analysis step room without eliminating forecast-lane headroom. 2. hrrr-point-rs pod is CrashLoopBackOff because its container is running image main-1776640915-65f7963 (pre-Stream-C) which doesn't contain the hrrr_point_worker binary. The grid-rs CI run for commit 4fefb81 failed and no newer image got published. Touch the Dockerfile to re-trigger the workflow so flux picks up a new tag with both binaries. No functional Dockerfile change, just a docstring update so the path-filter kicks. Forecast + analysis both claim per FOR UPDATE SKIP LOCKED, so reducing per-pod parallelism doesn't break the chain — the two replicas cover 4 slots cluster-wide, which still drains f01..f18 in ~5 min. --- k8s/deployment-grid-rs.yaml | 30 +++++++++++++++--------------- rust/prop_grid_rs/Dockerfile | 4 ++++ 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/k8s/deployment-grid-rs.yaml b/k8s/deployment-grid-rs.yaml index 7bc871c2..ca6aa09b 100644 --- a/k8s/deployment-grid-rs.yaml +++ b/k8s/deployment-grid-rs.yaml @@ -82,19 +82,17 @@ spec: value: "/data/scores" - name: RUST_LOG value: "info" - # talos5 is 4c/4t (i5-7260U). Each replica claims up to this - # many forecast hours in parallel. With 2 replicas × 3 → 6 - # concurrent tasks, the 18-hour chain completes in ~3 min. - # Per-task peak RAM is ~250 Mi post-streaming refactor, so - # 3×250 ≈ 750 Mi per pod, well under the 2 Gi limit. The - # secondary replica may land on a smaller node — 3 keeps it - # honest. + # talos5 is 4c/4t (i5-7260U). 2 concurrent tasks per pod × + # 2 replicas = 4 cluster-wide, enough to drain f01..f18 in + # ~5 min while keeping analysis (f00) room for its native- + # duct wgrib2 peak without OOM. Raising to 3 reintroduced + # the Stream-A-cutover OOM on the analysis step. - name: PROP_GRID_RS_PARALLELISM - value: "3" + value: "2" # Pool size = parallelism + 2 (NOTIFY + retry slack). Kept in # sync with PROP_GRID_RS_PARALLELISM when either changes. - name: PROP_GRID_RS_PG_CONNS - value: "5" + value: "4" - name: METRICS_ADDR value: "0.0.0.0:9100" envFrom: @@ -126,14 +124,16 @@ spec: resources: requests: cpu: 100m - memory: 512Mi + memory: 768Mi limits: cpu: "4" - # OOM at 1 Gi during active chain work (92k points × 23 - # bands ≈ 2.1M scores, plus decoded GRIB2 buffers, plus - # wgrib2 subprocess memory). 2 Gi leaves headroom; we're - # still an order of magnitude below the 6 Gi Elixir budget. - memory: 2Gi + # Analysis tasks (f00) run the native-level HRRR decode + # (~530 MB GRIB2 + wgrib2 working set) on top of 2 concurrent + # forecast tasks. 2 Gi OOM'd reliably in the 13 h after + # Stream A cutover. 3 Gi leaves room for the analysis + # wgrib2 peak without eliminating the forecast-lane + # parallelism headroom. + memory: 3Gi securityContext: allowPrivilegeEscalation: false runAsNonRoot: true diff --git a/rust/prop_grid_rs/Dockerfile b/rust/prop_grid_rs/Dockerfile index cf5c6c01..ca349dfa 100644 --- a/rust/prop_grid_rs/Dockerfile +++ b/rust/prop_grid_rs/Dockerfile @@ -6,6 +6,10 @@ # the already-published Elixir image. # # Runtime uid 65534 matches the fsGroup on the NFS scores mount. +# +# This image ships two binaries: `prop-grid-rs` (default ENTRYPOINT, +# drains grid_tasks) and `hrrr-point-worker` (k8s `command:` override, +# drains hrrr_fetch_tasks). ARG WGRIB2_IMAGE=git.mcintire.me/graham/prop:latest FROM ${WGRIB2_IMAGE} AS wgrib2-src