fix(narr): handle cdo 2.5.1 mixed name/code output
parse_cdo_row/1 assumed cdo -outputtab,code,lev,value would always emit numeric codes in the first column, but Debian's cdo 2.5.1 (the version in the prod container) emits mixed output: - 'codeNN' with a literal 'code' prefix for most GRIB1 parameters - Short GRIB1 names for a few: 2tag2 (PRES sfc), saip (DPT 2m), tpag10 (HGT) Result: every row fell through to :error, yielding a 'no parseable values' error and 986 discarded NarrFetchWorker jobs. Fix: normalize the code token by stripping 'code' prefix when present and mapping known 2.5.1 short names. UGRD/VGRD intentionally omitted (no discard evidence yet) — comment documents the add-when-they-surface plan. Also exposed parse_cdo_outputtab/1 as @doc false for testability.
This commit is contained in:
parent
fbbfe037bc
commit
8f0548b31e
2 changed files with 93 additions and 4 deletions
|
|
@ -55,6 +55,19 @@ defmodule Microwaveprop.Weather.NarrClient do
|
|||
221 => "HPBL"
|
||||
}
|
||||
|
||||
# cdo 2.5.1 on Debian ignores `-outputtab,code` for records whose GRIB1
|
||||
# parameter table entry has a shortname, emitting the shortname string
|
||||
# instead of the numeric code. Mapping verified against prod discard
|
||||
# output: 2tag2=PRES (985 mb surface pressure), saip=DPT (292 K 2-m
|
||||
# dewpoint), tpag10=HGT (983 m at 900 hPa geopotential height). UGRD
|
||||
# and VGRD aren't yet observed in that output — add them when they
|
||||
# surface in a real discard.
|
||||
@cdo_shortname_to_code %{
|
||||
"2tag2" => 1,
|
||||
"saip" => 17,
|
||||
"tpag10" => 7
|
||||
}
|
||||
|
||||
@doc """
|
||||
Builds the NCEI HTTPS URL for the NARR analysis GRIB1 file at `valid_time`.
|
||||
|
||||
|
|
@ -305,7 +318,8 @@ defmodule Microwaveprop.Weather.NarrClient do
|
|||
end
|
||||
end
|
||||
|
||||
defp parse_cdo_outputtab(stdout) do
|
||||
@doc false
|
||||
def parse_cdo_outputtab(stdout) do
|
||||
raw =
|
||||
stdout
|
||||
|> String.split("\n", trim: true)
|
||||
|
|
@ -329,12 +343,17 @@ defmodule Microwaveprop.Weather.NarrClient do
|
|||
end
|
||||
end
|
||||
|
||||
# Rows are three whitespace-separated numeric columns: <code> <lev> <value>.
|
||||
# Header, cdi warnings, and cdo progress lines lack the three-number shape
|
||||
# Rows are three whitespace-separated columns: <code> <lev> <value>. The
|
||||
# <code> token is one of:
|
||||
# * a bare integer (cdo 2.6 on Mac / dev)
|
||||
# * `codeNN` with "code" prefix (cdo 2.5.1 on Debian / prod)
|
||||
# * a GRIB1 shortname like `2tag2` / `saip` / `tpag10` (cdo 2.5.1 again,
|
||||
# for params whose GRIB1 parameter-table entry has a shortname)
|
||||
# Header, cdi warnings, and cdo progress lines lack the three-column shape
|
||||
# and fall through to :error.
|
||||
defp parse_cdo_row(line) do
|
||||
with [code_str, lev_str, value_str] <- String.split(line, ~r/\s+/, trim: true),
|
||||
{code_int, ""} <- Integer.parse(code_str),
|
||||
{:ok, code_int} <- normalize_cdo_code_token(code_str),
|
||||
{:ok, name} <- Map.fetch(@cdo_code_to_name, code_int),
|
||||
{lev_int, ""} <- Integer.parse(lev_str),
|
||||
{value_float, ""} <- Float.parse(value_str) do
|
||||
|
|
@ -344,6 +363,29 @@ defmodule Microwaveprop.Weather.NarrClient do
|
|||
end
|
||||
end
|
||||
|
||||
# Resolves the first column of a cdo `-outputtab,code,...` row to its
|
||||
# numeric GRIB1 parameter code. Handles all three cdo output dialects
|
||||
# we've seen in the wild (see `parse_cdo_row/1` for context).
|
||||
defp normalize_cdo_code_token("code" <> rest) do
|
||||
case Integer.parse(rest) do
|
||||
{code_int, ""} -> {:ok, code_int}
|
||||
_ -> :error
|
||||
end
|
||||
end
|
||||
|
||||
defp normalize_cdo_code_token(token) do
|
||||
case Integer.parse(token) do
|
||||
{code_int, ""} ->
|
||||
{:ok, code_int}
|
||||
|
||||
_ ->
|
||||
case Map.fetch(@cdo_shortname_to_code, token) do
|
||||
{:ok, code_int} -> {:ok, code_int}
|
||||
:error -> :error
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp build_profile_attrs(raw) do
|
||||
profile = build_pressure_profile(raw)
|
||||
|
||||
|
|
|
|||
|
|
@ -248,6 +248,53 @@ defmodule Microwaveprop.Weather.NarrClientTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "parse_cdo_outputtab/1" do
|
||||
# cdo 2.5.1 on Debian (prod) emits mixed output despite `-outputtab,code`:
|
||||
# rows whose first token is either `codeNN` (numeric code with "code" prefix)
|
||||
# or a GRIB1 shortname like `2tag2` / `saip` / `tpag10`. cdo 2.6 on Mac
|
||||
# (dev) emits bare integers. The parser must handle both.
|
||||
test "parses a cdo 2.5.1 output fixture with codeNN + shortname tokens" do
|
||||
# Trimmed multi-line fixture modeled on real prod discard output.
|
||||
# Codes: 221=HPBL, 1=PRES (2tag2), 11=TMP, 17=DPT (saip), 54=PWAT,
|
||||
# 7=HGT (tpag10), 51=SPFH.
|
||||
stdout = """
|
||||
# name lev value
|
||||
cdo(1) remapnn: some warning line
|
||||
code221 0 1387.958
|
||||
2tag2 0 98509.35
|
||||
code11 2 299.4731
|
||||
saip 2 292.6315
|
||||
code54 0 37.28385
|
||||
tpag10 90000 983.7363
|
||||
code11 10000 209.1171
|
||||
tpag10 85000 1547.8
|
||||
code11 85000 291.52
|
||||
code51 85000 0.0104
|
||||
"""
|
||||
|
||||
assert {:ok, attrs} = NarrClient.parse_cdo_outputtab(stdout)
|
||||
|
||||
# HPBL at sfc (code 221) = 1387.958 m
|
||||
assert_in_delta attrs.hpbl_m, 1387.958, 0.01
|
||||
# PRES at sfc (2tag2 → code 1) = 98509.35 Pa → 985.09 mb
|
||||
assert_in_delta attrs.surface_pressure_mb, 985.09, 0.1
|
||||
# TMP at 2m (code 11, lev 2) = 299.47 K → 26.32 C
|
||||
assert_in_delta attrs.surface_temp_c, 26.32, 0.1
|
||||
# DPT at 2m (saip → code 17, lev 2) = 292.63 K → 19.48 C
|
||||
assert_in_delta attrs.surface_dewpoint_c, 19.48, 0.1
|
||||
# PWAT (code 54) = 37.28 mm
|
||||
assert_in_delta attrs.pwat_mm, 37.28, 0.01
|
||||
|
||||
# Profile — 900 hPa has HGT+TMP+SPFH? No, only TMP is there at 900. Only
|
||||
# 850 hPa has all three (TMP, HGT, SPFH) → one profile entry.
|
||||
assert length(attrs.profile) == 1
|
||||
[level_850] = attrs.profile
|
||||
assert_in_delta level_850["pres"], 850.0, 0.01
|
||||
assert_in_delta level_850["tmpc"], 18.37, 0.5
|
||||
assert_in_delta level_850["hght"], 1547.8, 0.5
|
||||
end
|
||||
end
|
||||
|
||||
describe "fetch_profile_at/2" do
|
||||
@inv_fixture "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
|
||||
@grb_fixture "test/fixtures/narr/narr_dfw_2010-06-15_12z.grb"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue