From 8f0548b31ee260a3ed7e36a274655a04d91e9ef6 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 16 Apr 2026 15:14:40 -0500 Subject: [PATCH] fix(narr): handle cdo 2.5.1 mixed name/code output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse_cdo_row/1 assumed cdo -outputtab,code,lev,value would always emit numeric codes in the first column, but Debian's cdo 2.5.1 (the version in the prod container) emits mixed output: - 'codeNN' with a literal 'code' prefix for most GRIB1 parameters - Short GRIB1 names for a few: 2tag2 (PRES sfc), saip (DPT 2m), tpag10 (HGT) Result: every row fell through to :error, yielding a 'no parseable values' error and 986 discarded NarrFetchWorker jobs. Fix: normalize the code token by stripping 'code' prefix when present and mapping known 2.5.1 short names. UGRD/VGRD intentionally omitted (no discard evidence yet) — comment documents the add-when-they-surface plan. Also exposed parse_cdo_outputtab/1 as @doc false for testability. --- lib/microwaveprop/weather/narr_client.ex | 50 +++++++++++++++++-- .../weather/narr_client_test.exs | 47 +++++++++++++++++ 2 files changed, 93 insertions(+), 4 deletions(-) diff --git a/lib/microwaveprop/weather/narr_client.ex b/lib/microwaveprop/weather/narr_client.ex index 5622aab8..38e3db36 100644 --- a/lib/microwaveprop/weather/narr_client.ex +++ b/lib/microwaveprop/weather/narr_client.ex @@ -55,6 +55,19 @@ defmodule Microwaveprop.Weather.NarrClient do 221 => "HPBL" } + # cdo 2.5.1 on Debian ignores `-outputtab,code` for records whose GRIB1 + # parameter table entry has a shortname, emitting the shortname string + # instead of the numeric code. Mapping verified against prod discard + # output: 2tag2=PRES (985 mb surface pressure), saip=DPT (292 K 2-m + # dewpoint), tpag10=HGT (983 m at 900 hPa geopotential height). UGRD + # and VGRD aren't yet observed in that output — add them when they + # surface in a real discard. + @cdo_shortname_to_code %{ + "2tag2" => 1, + "saip" => 17, + "tpag10" => 7 + } + @doc """ Builds the NCEI HTTPS URL for the NARR analysis GRIB1 file at `valid_time`. @@ -305,7 +318,8 @@ defmodule Microwaveprop.Weather.NarrClient do end end - defp parse_cdo_outputtab(stdout) do + @doc false + def parse_cdo_outputtab(stdout) do raw = stdout |> String.split("\n", trim: true) @@ -329,12 +343,17 @@ defmodule Microwaveprop.Weather.NarrClient do end end - # Rows are three whitespace-separated numeric columns: . - # Header, cdi warnings, and cdo progress lines lack the three-number shape + # Rows are three whitespace-separated columns: . The + # token is one of: + # * a bare integer (cdo 2.6 on Mac / dev) + # * `codeNN` with "code" prefix (cdo 2.5.1 on Debian / prod) + # * a GRIB1 shortname like `2tag2` / `saip` / `tpag10` (cdo 2.5.1 again, + # for params whose GRIB1 parameter-table entry has a shortname) + # Header, cdi warnings, and cdo progress lines lack the three-column shape # and fall through to :error. defp parse_cdo_row(line) do with [code_str, lev_str, value_str] <- String.split(line, ~r/\s+/, trim: true), - {code_int, ""} <- Integer.parse(code_str), + {:ok, code_int} <- normalize_cdo_code_token(code_str), {:ok, name} <- Map.fetch(@cdo_code_to_name, code_int), {lev_int, ""} <- Integer.parse(lev_str), {value_float, ""} <- Float.parse(value_str) do @@ -344,6 +363,29 @@ defmodule Microwaveprop.Weather.NarrClient do end end + # Resolves the first column of a cdo `-outputtab,code,...` row to its + # numeric GRIB1 parameter code. Handles all three cdo output dialects + # we've seen in the wild (see `parse_cdo_row/1` for context). + defp normalize_cdo_code_token("code" <> rest) do + case Integer.parse(rest) do + {code_int, ""} -> {:ok, code_int} + _ -> :error + end + end + + defp normalize_cdo_code_token(token) do + case Integer.parse(token) do + {code_int, ""} -> + {:ok, code_int} + + _ -> + case Map.fetch(@cdo_shortname_to_code, token) do + {:ok, code_int} -> {:ok, code_int} + :error -> :error + end + end + end + defp build_profile_attrs(raw) do profile = build_pressure_profile(raw) diff --git a/test/microwaveprop/weather/narr_client_test.exs b/test/microwaveprop/weather/narr_client_test.exs index b822d475..e0daca87 100644 --- a/test/microwaveprop/weather/narr_client_test.exs +++ b/test/microwaveprop/weather/narr_client_test.exs @@ -248,6 +248,53 @@ defmodule Microwaveprop.Weather.NarrClientTest do end end + describe "parse_cdo_outputtab/1" do + # cdo 2.5.1 on Debian (prod) emits mixed output despite `-outputtab,code`: + # rows whose first token is either `codeNN` (numeric code with "code" prefix) + # or a GRIB1 shortname like `2tag2` / `saip` / `tpag10`. cdo 2.6 on Mac + # (dev) emits bare integers. The parser must handle both. + test "parses a cdo 2.5.1 output fixture with codeNN + shortname tokens" do + # Trimmed multi-line fixture modeled on real prod discard output. + # Codes: 221=HPBL, 1=PRES (2tag2), 11=TMP, 17=DPT (saip), 54=PWAT, + # 7=HGT (tpag10), 51=SPFH. + stdout = """ + # name lev value + cdo(1) remapnn: some warning line + code221 0 1387.958 + 2tag2 0 98509.35 + code11 2 299.4731 + saip 2 292.6315 + code54 0 37.28385 + tpag10 90000 983.7363 + code11 10000 209.1171 + tpag10 85000 1547.8 + code11 85000 291.52 + code51 85000 0.0104 + """ + + assert {:ok, attrs} = NarrClient.parse_cdo_outputtab(stdout) + + # HPBL at sfc (code 221) = 1387.958 m + assert_in_delta attrs.hpbl_m, 1387.958, 0.01 + # PRES at sfc (2tag2 → code 1) = 98509.35 Pa → 985.09 mb + assert_in_delta attrs.surface_pressure_mb, 985.09, 0.1 + # TMP at 2m (code 11, lev 2) = 299.47 K → 26.32 C + assert_in_delta attrs.surface_temp_c, 26.32, 0.1 + # DPT at 2m (saip → code 17, lev 2) = 292.63 K → 19.48 C + assert_in_delta attrs.surface_dewpoint_c, 19.48, 0.1 + # PWAT (code 54) = 37.28 mm + assert_in_delta attrs.pwat_mm, 37.28, 0.01 + + # Profile — 900 hPa has HGT+TMP+SPFH? No, only TMP is there at 900. Only + # 850 hPa has all three (TMP, HGT, SPFH) → one profile entry. + assert length(attrs.profile) == 1 + [level_850] = attrs.profile + assert_in_delta level_850["pres"], 850.0, 0.01 + assert_in_delta level_850["tmpc"], 18.37, 0.5 + assert_in_delta level_850["hght"], 1547.8, 0.5 + end + end + describe "fetch_profile_at/2" do @inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv" @grb_fixture "test/fixtures/narr/narr_dfw_2010-06-15_12z.grb"