From ca33b8331b29c588c15a45e8a28f618d13c54927 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 19 Apr 2026 16:37:05 -0500 Subject: [PATCH] fix(grid-rs): bump memory limit to 2 Gi MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 Gi OOM'd during a real chain step. Each forecast hour loads both a surface and pressure-level HRRR GRIB2 (~40 MB combined), decodes via wgrib2 subprocess (which allocates its own buffers), scores 92k grid points × 23 bands (~2.1M intermediates), and writes 23 score files. 2 Gi absorbs the spike; still ~3× smaller than the Elixir-per-pod budget we're replacing. --- k8s/deployment-grid-rs.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/k8s/deployment-grid-rs.yaml b/k8s/deployment-grid-rs.yaml index d27532a2..30424276 100644 --- a/k8s/deployment-grid-rs.yaml +++ b/k8s/deployment-grid-rs.yaml @@ -71,14 +71,14 @@ spec: resources: requests: cpu: 100m - memory: 256Mi + memory: 512Mi limits: cpu: "4" - # 512 Mi OOM-killed the real binary at startup. 1 Gi gives - # room for sqlx pool + reqwest pool + tokio runtime + - # tracing buffers, with headroom vs the steady-state - # target (~500 Mi during a chain step). - memory: 1Gi + # OOM at 1 Gi during active chain work (92k points × 23 + # bands ≈ 2.1M scores, plus decoded GRIB2 buffers, plus + # wgrib2 subprocess memory). 2 Gi leaves headroom; we're + # still an order of magnitude below the 6 Gi Elixir budget. + memory: 2Gi securityContext: allowPrivilegeEscalation: false runAsNonRoot: true