fix(narr): handle cdo 2.5.1 mixed name/code output

parse_cdo_row/1 assumed cdo -outputtab,code,lev,value would always emit
numeric codes in the first column, but Debian's cdo 2.5.1 (the version
in the prod container) emits mixed output:

- 'codeNN' with a literal 'code' prefix for most GRIB1 parameters
- Short GRIB1 names for a few: 2tag2 (PRES sfc), saip (DPT 2m),
  tpag10 (HGT)

Result: every row fell through to :error, yielding a 'no parseable
values' error and 986 discarded NarrFetchWorker jobs.

Fix: normalize the code token by stripping 'code' prefix when present
and mapping known 2.5.1 short names. UGRD/VGRD intentionally omitted
(no discard evidence yet) — comment documents the add-when-they-surface
plan.

Also exposed parse_cdo_outputtab/1 as @doc false for testability.
This commit is contained in:
Graham McIntire 2026-04-16 15:14:40 -05:00
parent fbbfe037bc
commit 8f0548b31e
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 93 additions and 4 deletions

View file

@ -55,6 +55,19 @@ defmodule Microwaveprop.Weather.NarrClient do
221 => "HPBL"
}
# cdo 2.5.1 on Debian ignores `-outputtab,code` for records whose GRIB1
# parameter table entry has a shortname, emitting the shortname string
# instead of the numeric code. Mapping verified against prod discard
# output: 2tag2=PRES (985 mb surface pressure), saip=DPT (292 K 2-m
# dewpoint), tpag10=HGT (983 m at 900 hPa geopotential height). UGRD
# and VGRD aren't yet observed in that output — add them when they
# surface in a real discard.
@cdo_shortname_to_code %{
"2tag2" => 1,
"saip" => 17,
"tpag10" => 7
}
@doc """
Builds the NCEI HTTPS URL for the NARR analysis GRIB1 file at `valid_time`.
@ -305,7 +318,8 @@ defmodule Microwaveprop.Weather.NarrClient do
end
end
defp parse_cdo_outputtab(stdout) do
@doc false
def parse_cdo_outputtab(stdout) do
raw =
stdout
|> String.split("\n", trim: true)
@ -329,12 +343,17 @@ defmodule Microwaveprop.Weather.NarrClient do
end
end
# Rows are three whitespace-separated numeric columns: <code> <lev> <value>.
# Header, cdi warnings, and cdo progress lines lack the three-number shape
# Rows are three whitespace-separated columns: <code> <lev> <value>. The
# <code> token is one of:
# * a bare integer (cdo 2.6 on Mac / dev)
# * `codeNN` with "code" prefix (cdo 2.5.1 on Debian / prod)
# * a GRIB1 shortname like `2tag2` / `saip` / `tpag10` (cdo 2.5.1 again,
# for params whose GRIB1 parameter-table entry has a shortname)
# Header, cdi warnings, and cdo progress lines lack the three-column shape
# and fall through to :error.
defp parse_cdo_row(line) do
with [code_str, lev_str, value_str] <- String.split(line, ~r/\s+/, trim: true),
{code_int, ""} <- Integer.parse(code_str),
{:ok, code_int} <- normalize_cdo_code_token(code_str),
{:ok, name} <- Map.fetch(@cdo_code_to_name, code_int),
{lev_int, ""} <- Integer.parse(lev_str),
{value_float, ""} <- Float.parse(value_str) do
@ -344,6 +363,29 @@ defmodule Microwaveprop.Weather.NarrClient do
end
end
# Resolves the first column of a cdo `-outputtab,code,...` row to its
# numeric GRIB1 parameter code. Handles all three cdo output dialects
# we've seen in the wild (see `parse_cdo_row/1` for context).
defp normalize_cdo_code_token("code" <> rest) do
case Integer.parse(rest) do
{code_int, ""} -> {:ok, code_int}
_ -> :error
end
end
defp normalize_cdo_code_token(token) do
case Integer.parse(token) do
{code_int, ""} ->
{:ok, code_int}
_ ->
case Map.fetch(@cdo_shortname_to_code, token) do
{:ok, code_int} -> {:ok, code_int}
:error -> :error
end
end
end
defp build_profile_attrs(raw) do
profile = build_pressure_profile(raw)

View file

@ -248,6 +248,53 @@ defmodule Microwaveprop.Weather.NarrClientTest do
end
end
describe "parse_cdo_outputtab/1" do
# cdo 2.5.1 on Debian (prod) emits mixed output despite `-outputtab,code`:
# rows whose first token is either `codeNN` (numeric code with "code" prefix)
# or a GRIB1 shortname like `2tag2` / `saip` / `tpag10`. cdo 2.6 on Mac
# (dev) emits bare integers. The parser must handle both.
test "parses a cdo 2.5.1 output fixture with codeNN + shortname tokens" do
# Trimmed multi-line fixture modeled on real prod discard output.
# Codes: 221=HPBL, 1=PRES (2tag2), 11=TMP, 17=DPT (saip), 54=PWAT,
# 7=HGT (tpag10), 51=SPFH.
stdout = """
# name lev value
cdo(1) remapnn: some warning line
code221 0 1387.958
2tag2 0 98509.35
code11 2 299.4731
saip 2 292.6315
code54 0 37.28385
tpag10 90000 983.7363
code11 10000 209.1171
tpag10 85000 1547.8
code11 85000 291.52
code51 85000 0.0104
"""
assert {:ok, attrs} = NarrClient.parse_cdo_outputtab(stdout)
# HPBL at sfc (code 221) = 1387.958 m
assert_in_delta attrs.hpbl_m, 1387.958, 0.01
# PRES at sfc (2tag2 → code 1) = 98509.35 Pa → 985.09 mb
assert_in_delta attrs.surface_pressure_mb, 985.09, 0.1
# TMP at 2m (code 11, lev 2) = 299.47 K → 26.32 C
assert_in_delta attrs.surface_temp_c, 26.32, 0.1
# DPT at 2m (saip → code 17, lev 2) = 292.63 K → 19.48 C
assert_in_delta attrs.surface_dewpoint_c, 19.48, 0.1
# PWAT (code 54) = 37.28 mm
assert_in_delta attrs.pwat_mm, 37.28, 0.01
# Profile — 900 hPa has HGT+TMP+SPFH? No, only TMP is there at 900. Only
# 850 hPa has all three (TMP, HGT, SPFH) → one profile entry.
assert length(attrs.profile) == 1
[level_850] = attrs.profile
assert_in_delta level_850["pres"], 850.0, 0.01
assert_in_delta level_850["tmpc"], 18.37, 0.5
assert_in_delta level_850["hght"], 1547.8, 0.5
end
end
describe "fetch_profile_at/2" do
@inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
@grb_fixture "test/fixtures/narr/narr_dfw_2010-06-15_12z.grb"