diff --git a/k8s/deployment-grid-rs.yaml b/k8s/deployment-grid-rs.yaml index 7bc871c2..ca6aa09b 100644 --- a/k8s/deployment-grid-rs.yaml +++ b/k8s/deployment-grid-rs.yaml @@ -82,19 +82,17 @@ spec: value: "/data/scores" - name: RUST_LOG value: "info" - # talos5 is 4c/4t (i5-7260U). Each replica claims up to this - # many forecast hours in parallel. With 2 replicas × 3 → 6 - # concurrent tasks, the 18-hour chain completes in ~3 min. - # Per-task peak RAM is ~250 Mi post-streaming refactor, so - # 3×250 ≈ 750 Mi per pod, well under the 2 Gi limit. The - # secondary replica may land on a smaller node — 3 keeps it - # honest. + # talos5 is 4c/4t (i5-7260U). 2 concurrent tasks per pod × + # 2 replicas = 4 cluster-wide, enough to drain f01..f18 in + # ~5 min while keeping analysis (f00) room for its native- + # duct wgrib2 peak without OOM. Raising to 3 reintroduced + # the Stream-A-cutover OOM on the analysis step. - name: PROP_GRID_RS_PARALLELISM - value: "3" + value: "2" # Pool size = parallelism + 2 (NOTIFY + retry slack). Kept in # sync with PROP_GRID_RS_PARALLELISM when either changes. - name: PROP_GRID_RS_PG_CONNS - value: "5" + value: "4" - name: METRICS_ADDR value: "0.0.0.0:9100" envFrom: @@ -126,14 +124,16 @@ spec: resources: requests: cpu: 100m - memory: 512Mi + memory: 768Mi limits: cpu: "4" - # OOM at 1 Gi during active chain work (92k points × 23 - # bands ≈ 2.1M scores, plus decoded GRIB2 buffers, plus - # wgrib2 subprocess memory). 2 Gi leaves headroom; we're - # still an order of magnitude below the 6 Gi Elixir budget. - memory: 2Gi + # Analysis tasks (f00) run the native-level HRRR decode + # (~530 MB GRIB2 + wgrib2 working set) on top of 2 concurrent + # forecast tasks. 2 Gi OOM'd reliably in the 13 h after + # Stream A cutover. 3 Gi leaves room for the analysis + # wgrib2 peak without eliminating the forecast-lane + # parallelism headroom. + memory: 3Gi securityContext: allowPrivilegeEscalation: false runAsNonRoot: true diff --git a/rust/prop_grid_rs/Dockerfile b/rust/prop_grid_rs/Dockerfile index cf5c6c01..ca349dfa 100644 --- a/rust/prop_grid_rs/Dockerfile +++ b/rust/prop_grid_rs/Dockerfile @@ -6,6 +6,10 @@ # the already-published Elixir image. # # Runtime uid 65534 matches the fsGroup on the NFS scores mount. +# +# This image ships two binaries: `prop-grid-rs` (default ENTRYPOINT, +# drains grid_tasks) and `hrrr-point-worker` (k8s `command:` override, +# drains hrrr_fetch_tasks). ARG WGRIB2_IMAGE=git.mcintire.me/graham/prop:latest FROM ${WGRIB2_IMAGE} AS wgrib2-src