NarrClient: parse cdo numeric codes for cdo-version independence

Prod cdo 2.5.1 (Debian) emits named shortnames (tpag10, 2tag2, saip, code11…)
where cdo 2.6.0 (Mac) emits var11/var33/etc. The parser only recognized
varNN, so every prod NARR fetch failed with "no parseable values in cdo
output" and all 350 in-flight jobs discarded.

Switch cdo args from `-outputtab,name,lev,value` to `-outputtab,code,...`.
cdo then outputs the raw numeric NCEP GRIB1 parameter code regardless of
version, and @cdo_code_to_name maps that integer to the semantic name we
already use downstream (TMP/HGT/SPFH/…).

Verified by reproducing on both a 2010 NARR file (fixture) and a 2007 file
fetched live — both now emit identical numeric code rows.
This commit is contained in:
Graham McIntire 2026-04-16 09:45:21 -05:00
parent c1cac4f394
commit c7f9aac3f9
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59

View file

@ -38,19 +38,21 @@ defmodule Microwaveprop.Weather.NarrClient do
{"PWAT", "atmos col"} {"PWAT", "atmos col"}
] ]
# cdo emits NCEP GRIB1 parameters as `varNNN`. This table maps the cdo # Map NCEP GRIB1 parameter codes to our semantic short names.
# parameter token to the semantic NCEP short name, verified empirically # Using `-outputtab,code,...` (instead of `name`) makes cdo emit the raw
# in the spike (see plan doc cdo lookup table). # numeric param code, so the same parser works across cdo versions that
@cdo_var_to_name %{ # would otherwise produce either `varNN` (Mac / cdo 2.6) or `codeNN` +
"var1" => "PRES", # named shortnames like `tpag10` / `2tag2` / `saip` (Debian / cdo 2.5.1).
"var7" => "HGT", @cdo_code_to_name %{
"var11" => "TMP", 1 => "PRES",
"var17" => "DPT", 7 => "HGT",
"var33" => "UGRD", 11 => "TMP",
"var34" => "VGRD", 17 => "DPT",
"var51" => "SPFH", 33 => "UGRD",
"var54" => "PWAT", 34 => "VGRD",
"var221" => "HPBL" 51 => "SPFH",
54 => "PWAT",
221 => "HPBL"
} }
@doc """ @doc """
@ -289,7 +291,7 @@ defmodule Microwaveprop.Weather.NarrClient do
defp run_cdo_outputtab(grb_path, lat, lon) do defp run_cdo_outputtab(grb_path, lat, lon) do
args = [ args = [
"-outputtab,name,lev,value", "-outputtab,code,lev,value",
"-remapnn,lon=#{lon}_lat=#{lat}", "-remapnn,lon=#{lon}_lat=#{lat}",
grb_path grb_path
] ]
@ -307,8 +309,6 @@ defmodule Microwaveprop.Weather.NarrClient do
raw = raw =
stdout stdout
|> String.split("\n", trim: true) |> String.split("\n", trim: true)
|> Enum.map(&String.trim/1)
|> Enum.filter(&String.starts_with?(&1, "var"))
|> Enum.reduce(%{}, &accumulate_cdo_row/2) |> Enum.reduce(%{}, &accumulate_cdo_row/2)
if raw == %{} do if raw == %{} do
@ -329,9 +329,13 @@ defmodule Microwaveprop.Weather.NarrClient do
end end
end end
# Rows are three whitespace-separated numeric columns: <code> <lev> <value>.
# Header, cdi warnings, and cdo progress lines lack the three-number shape
# and fall through to :error.
defp parse_cdo_row(line) do defp parse_cdo_row(line) do
with [var_token, lev_str, value_str] <- String.split(line, ~r/\s+/, trim: true), with [code_str, lev_str, value_str] <- String.split(line, ~r/\s+/, trim: true),
{:ok, name} <- Map.fetch(@cdo_var_to_name, var_token), {code_int, ""} <- Integer.parse(code_str),
{:ok, name} <- Map.fetch(@cdo_code_to_name, code_int),
{lev_int, ""} <- Integer.parse(lev_str), {lev_int, ""} <- Integer.parse(lev_str),
{value_float, ""} <- Float.parse(value_str) do {value_float, ""} <- Float.parse(value_str) do
{{name, lev_int}, value_float} {{name, lev_int}, value_float}