From 89eccb7d50d139572d930ae6cb0fcfe0f53816bb Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 30 Apr 2026 08:32:37 -0500 Subject: [PATCH] =?UTF-8?q?fix(hrdps):=20bump=20wgrib2=203.6.0=20=E2=86=92?= =?UTF-8?q?=203.8.0=20to=20fix=20HRDPS=20extraction=20crash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reproduced in production pod 2026-04-30: wgrib2 3.6.0 crashes with `free(): invalid size` when -lon point extraction is run against any HRDPS rotated lat/lon GRIB2 file, regardless of batch size (failed at N=200, N=1000, N=2000). Output before the abort is denormal f32 garbage (val=4.93944e-41) — wgrib2's internal record buffer is being corrupted. Same files extract cleanly with wgrib2 3.8.0 locally. Bumping the build arg in both Dockerfile (Elixir image, used by hrrr-point-rs/per-QSO HRDPS lookups) and rust/prop_grid_rs/Dockerfile (Rust grid worker) so both consumers get the fix. Also bumped POINT_BATCH 200 → 1000 in decoder.rs. With wgrib2 3.8.0 stable at 1000+ point batches, this drops the chain-step invocation count from 286 → 57 per step. Comment captures the reproduction context so a future bisection finds it quickly. --- Dockerfile | 15 ++++++++++----- rust/prop_grid_rs/Dockerfile | 10 +++++++++- rust/prop_grid_rs/src/decoder.rs | 31 +++++++++++++++++++++++++------ 3 files changed, 44 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index b2c4938e..742231e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,12 +22,17 @@ ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}" # ---- wgrib2 build stage (cached independently) ---- FROM ${RUNNER_IMAGE} AS wgrib2-builder -ARG WGRIB2_VERSION=3.6.0 +ARG WGRIB2_VERSION=3.8.0 +# wgrib2 3.6.0 has a memory-corruption bug that surfaces on HRDPS +# rotated lat/lon files via -lon point extraction: emits denormal +# garbage values, then aborts with `free(): invalid size`. 3.8.0 +# fixes it. Verified by reproducing the failure with wgrib2 3.6.0 +# in the production pod against an HRDPS file on 2026-04-30 and +# confirming 3.8.0 handles the same file correctly locally. Keep +# this aligned with rust/prop_grid_rs/Dockerfile. # g2c v2.1.0 is the first release where BUILD_G2C defaults ON, which -# is what wgrib2 v3.6.0 needs at link time (g2c_enc_jpeg2000 / -# g2c_dec_jpeg2000 come from the "new" file-based API). v1.9.0's -# same flag was OFF by default with a note about the API being -# experimental until 2.0.0. +# wgrib2 needs at link time (g2c_enc_jpeg2000 / g2c_dec_jpeg2000 +# come from the "new" file-based API). ARG G2C_VERSION=2.1.0 # BuildKit cache mounts keep the .deb archive + apt-lists out of the diff --git a/rust/prop_grid_rs/Dockerfile b/rust/prop_grid_rs/Dockerfile index 589bc848..a87ae4de 100644 --- a/rust/prop_grid_rs/Dockerfile +++ b/rust/prop_grid_rs/Dockerfile @@ -19,8 +19,16 @@ ARG RUNNER_IMAGE="docker.io/debian:${DEBIAN_VERSION}" # ---- wgrib2 builder (mirrors the same stage in the Elixir Dockerfile) ---- FROM ${RUNNER_IMAGE} AS wgrib2-builder -ARG WGRIB2_VERSION=3.6.0 +ARG WGRIB2_VERSION=3.8.0 ARG G2C_VERSION=2.1.0 +# wgrib2 3.6.0 has a memory-corruption bug exposed by HRDPS rotated +# lat/lon GRIB2 files plus -lon point extraction: it emits denormal +# garbage values then aborts with `free(): invalid size`. Reproduced +# directly in the production pod against an HRDPS sample on +# 2026-04-30. 3.8.0 (verified locally against the same file) doesn't +# have the bug. Keep both Dockerfiles aligned — bumping only one +# means the Elixir-side per-QSO HRDPS lookups would still hit the +# bad version. RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ diff --git a/rust/prop_grid_rs/src/decoder.rs b/rust/prop_grid_rs/src/decoder.rs index f8f1cbf1..888d4045 100644 --- a/rust/prop_grid_rs/src/decoder.rs +++ b/rust/prop_grid_rs/src/decoder.rs @@ -120,7 +120,17 @@ pub fn extract_points( result } -const POINT_BATCH: usize = 2_000; +// 1000 points per wgrib2 invocation. With ~57k Canadian cells that's +// 57 invocations per chain step — enough to amortize process-spawn +// overhead, small enough that any future wgrib2 regression around +// large -lon arg lists trips on a manageable failure mode. +// +// Note: wgrib2 3.6.0 had a memory-corruption bug surfaced by HRDPS +// rotated lat/lon files at *any* batch size (verified at N=200 in +// production on 2026-04-30 — `free(): invalid size`). The fix was +// bumping the Dockerfile to wgrib2 3.8.0; this batch size is just +// a normal performance tuning choice, not a workaround. +const POINT_BATCH: usize = 1_000; fn run_wgrib2_lon_batched( wgrib2: &Path, @@ -158,13 +168,22 @@ fn run_wgrib2_lon_one( } = cmd.output()?; if !status.success() { - let mut combined = stdout; - combined.extend_from_slice(&stderr); - let text = String::from_utf8_lossy(&combined).to_string(); - let snippet: String = text.chars().take(200).collect(); + // Surface the FIRST chars of stderr (the actual diagnostic) + // rather than stdout (which is mostly per-cell numeric output + // that drowned the interesting failure context). status.code() + // returns None on signal-kill — flag that with -1 so the + // failure mode is distinguishable from a real exit code. + let stderr_text = String::from_utf8_lossy(&stderr).to_string(); + let stderr_snippet: String = stderr_text.chars().take(400).collect(); + let signaled = status.code().is_none(); + let suffix = if signaled { + format!(" (killed by signal, batch={})", batch.len()) + } else { + String::new() + }; return Err(DecodeError::Wgrib2Failed { code: status.code().unwrap_or(-1), - stderr: snippet, + stderr: format!("{stderr_snippet}{suffix}"), }); }