From 68910f4d72c9c48615063151b3e1ed26811fc476 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 20 Apr 2026 09:06:10 -0500 Subject: [PATCH] k8s(grid-rs): drop parallelism to 1 after 3 Gi still OOMs at 2 Two concurrent forecast tasks plus the NFS score-file write cache (23 files x ~2 MB) plus the analysis step's wgrib2 peak repeatedly crossed the 3 Gi cgroup limit. Single-lane parallelism per pod keeps steady RSS under 2 Gi; with 2 replicas that still gives 2 concurrent tasks cluster-wide. PROP_GRID_RS_PG_CONNS follows the parallelism+2 formula, dropping from 4 to 3. --- k8s/deployment-grid-rs.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/k8s/deployment-grid-rs.yaml b/k8s/deployment-grid-rs.yaml index a92e24cf..51e87f7b 100644 --- a/k8s/deployment-grid-rs.yaml +++ b/k8s/deployment-grid-rs.yaml @@ -82,17 +82,21 @@ spec: value: "/data/scores" - name: RUST_LOG value: "info" - # talos5 is 4c/4t (i5-7260U). 2 concurrent tasks per pod × - # 2 replicas = 4 cluster-wide, enough to drain f01..f18 in - # ~5 min while keeping analysis (f00) room for its native- - # duct wgrib2 peak without OOM. Raising to 3 reintroduced - # the Stream-A-cutover OOM on the analysis step. + # Single concurrent task per pod. 2 replicas → 2 concurrent + # cluster-wide. Raised from 2 after the new 3 Gi limit still + # OOMed under 2-parallel forecast load — the NFS-backed + # score-file writes (23 files × ~2 MB each) keep cgroup page + # cache high enough that two concurrent tasks plus the + # wgrib2 working set regularly crosses 3 Gi. Single-lane + # parallelism stays comfortably under 2 Gi and lets the + # analysis step (native duct + NEXRAD + commercial merges) + # run safely within the same budget. - name: PROP_GRID_RS_PARALLELISM - value: "2" + value: "1" # Pool size = parallelism + 2 (NOTIFY + retry slack). Kept in # sync with PROP_GRID_RS_PARALLELISM when either changes. - name: PROP_GRID_RS_PG_CONNS - value: "4" + value: "3" - name: METRICS_ADDR value: "0.0.0.0:9100" envFrom: