From 081cd47bb5d26209910493fce4d48f4765707b1f Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Wed, 15 Apr 2026 17:22:52 -0500 Subject: [PATCH] NarrClient.parse_inventory/2 + fetch_inventory/1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse_inventory walks the wgrib1-format .inv text and returns a {var, level} -> {byte_offset, length} index, computing each record's length from the next record's offset (or file_size for the last one). fetch_inventory does a GET on the .inv URL plus a HEAD on the .grb URL to learn its size, then delegates to parse_inventory. Both stubbable via Req.Test (config/test.exs adds narr_req_options matching the existing era5_req_options / giro_req_options pattern). Spike fixture at test/fixtures/narr/narr-a_221_20100615_1200_000.inv backs the parser test — verifies the ("TMP", "2 m above gnd") and ("WVVFLX", "atmos col") records line up with the real bytes. --- config/test.exs | 1 + lib/microwaveprop/weather/narr_client.ex | 118 +++++++++++++++++- .../weather/narr_client_test.exs | 109 ++++++++++++++++ 3 files changed, 225 insertions(+), 3 deletions(-) diff --git a/config/test.exs b/config/test.exs index c5187a0f..90aee311 100644 --- a/config/test.exs +++ b/config/test.exs @@ -62,6 +62,7 @@ config :microwaveprop, era5_req_options: [plug: {Req.Test, Microwaveprop.Weather config :microwaveprop, giro_req_options: [plug: {Req.Test, Microwaveprop.Ionosphere.GiroClient}, retry: false] config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false] config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false] +config :microwaveprop, narr_req_options: [plug: {Req.Test, Microwaveprop.Weather.NarrClient}, retry: false] # Route HTTP requests through Req.Test stubs config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weather.NexradClient}, retry: false] diff --git a/lib/microwaveprop/weather/narr_client.ex b/lib/microwaveprop/weather/narr_client.ex index 7960dec4..a18c9be8 100644 --- a/lib/microwaveprop/weather/narr_client.ex +++ b/lib/microwaveprop/weather/narr_client.ex @@ -9,9 +9,9 @@ defmodule Microwaveprop.Weather.NarrClient do architecture — the filename is a historical artifact from an earlier MERRA-2 plan. - This module currently provides only the pure URL/time helpers. The - byte-range `fetch_inventory/1` and `fetch_profile_at/2` pipeline lands - in follow-up tasks of the same plan. + Currently provides URL/time helpers, the inventory parser, and the + inventory HTTP fetch. The byte-range `fetch_profile_at/2` pipeline + lands in a follow-up task of the same plan. """ @base_url "https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/access/3-hourly" @@ -59,6 +59,118 @@ defmodule Microwaveprop.Weather.NarrClient do %{datetime | hour: snapped_hour, minute: 0, second: 0, microsecond: {0, 0}} end + @doc """ + Parses the raw text of a NARR `.inv` file into a map keyed by + `{var, level}` with values of `{byte_offset, length}`. + + Each record's length is computed as `next_offset - this_offset`. The + final record's length uses `file_size` as its "next offset". + + Blank lines and trailing whitespace are ignored. `var` and `level` are + returned as-is (no trimming) so levels like `"2 m above gnd"` round-trip + unchanged. + """ + @spec parse_inventory(String.t(), non_neg_integer()) :: + {:ok, %{{String.t(), String.t()} => {non_neg_integer(), non_neg_integer()}}} + def parse_inventory(inv_text, file_size) when is_binary(inv_text) and is_integer(file_size) do + records = + inv_text + |> String.split("\n", trim: true) + |> Enum.map(&parse_inventory_line/1) + |> Enum.reject(&is_nil/1) + + index = + records + |> Enum.zip(Enum.drop(records, 1) ++ [{nil, file_size, nil, nil}]) + |> Map.new(fn {{_num, offset, var, level}, {_next_num, next_offset, _next_var, _next_level}} -> + {{var, level}, {offset, next_offset - offset}} + end) + + {:ok, index} + end + + @doc """ + Fetches the `.inv` text and the `.grb` `Content-Length` for `valid_time`, + then delegates to `parse_inventory/2`. + + Does one `GET` against the inventory URL and one `HEAD` against the GRIB + URL. Returns `{:ok, index}` on success; `{:error, reason}` if either HTTP + call is non-200, transport-errored, or missing a `content-length` header. + """ + @spec fetch_inventory(DateTime.t()) :: + {:ok, %{{String.t(), String.t()} => {non_neg_integer(), non_neg_integer()}}} + | {:error, term()} + def fetch_inventory(%DateTime{} = valid_time) do + inv_url = url_for_inventory(valid_time) + grb_url = url_for(valid_time) + + with {:ok, inv_body} <- do_get_inventory(inv_url), + {:ok, file_size} <- do_head_grib_size(grb_url) do + parse_inventory(inv_body, file_size) + end + end + + defp do_get_inventory(url) do + case Req.get(url, [receive_timeout: 30_000] ++ req_options()) do + {:ok, %{status: 200, body: body}} when is_binary(body) -> + {:ok, body} + + {:ok, %{status: status, body: body}} -> + {:error, "NARR inventory HTTP #{status}: #{inspect(body)}"} + + {:error, reason} -> + {:error, "NARR inventory request failed: #{inspect(reason)}"} + end + end + + defp do_head_grib_size(url) do + case Req.head(url, [receive_timeout: 30_000] ++ req_options()) do + {:ok, %{status: 200} = response} -> + case content_length(response) do + {:ok, size} -> {:ok, size} + :error -> {:error, "NARR grib HEAD missing content-length"} + end + + {:ok, %{status: status}} -> + {:error, "NARR grib HEAD HTTP #{status}"} + + {:error, reason} -> + {:error, "NARR grib HEAD failed: #{inspect(reason)}"} + end + end + + defp content_length(%Req.Response{} = response) do + case response |> Req.Response.get_header("content-length") |> List.first() do + nil -> + :error + + value when is_binary(value) -> + case Integer.parse(value) do + {size, ""} when size >= 0 -> {:ok, size} + _ -> :error + end + end + end + + defp parse_inventory_line(line) do + case String.split(line, ":") do + [num, offset, _date, var, level | _rest] -> + with {num_int, ""} <- Integer.parse(num), + {offset_int, ""} <- Integer.parse(offset) do + {num_int, offset_int, var, level} + else + _ -> nil + end + + _ -> + nil + end + end + + defp req_options do + Application.get_env(:microwaveprop, :narr_req_options, []) + end + defp validate_analysis_time!(%DateTime{hour: hour, minute: 0, second: 0} = _vt) when hour in @valid_hours, do: :ok defp validate_analysis_time!(vt) do diff --git a/test/microwaveprop/weather/narr_client_test.exs b/test/microwaveprop/weather/narr_client_test.exs index 7a09b775..832eb59a 100644 --- a/test/microwaveprop/weather/narr_client_test.exs +++ b/test/microwaveprop/weather/narr_client_test.exs @@ -73,4 +73,113 @@ defmodule Microwaveprop.Weather.NarrClientTest do assert snapped.microsecond == {0, 0} end end + + describe "parse_inventory/2" do + @fixture_path "test/fixtures/narr/narr-a_221_20100615_1200_000.inv" + @fixture_file_size 56_221_430 + + test "parses the spike fixture inventory into an offset/length index" do + inv = File.read!(@fixture_path) + + {:ok, index} = NarrClient.parse_inventory(inv, @fixture_file_size) + + # Record 288: TMP @ 2 m above gnd, offset 37836396, next record at 37967364. + assert {37_836_396, 130_968} = index[{"TMP", "2 m above gnd"}] + + # Record 15: HPBL at surface — length is positive and computed from next record. + assert {1_940_538, len_hpbl} = index[{"HPBL", "sfc"}] + assert len_hpbl > 0 + + # Record 202: TMP at 850 mb — pressure-level record present in the index. + assert {_off_850, _len_850} = index[{"TMP", "850 mb"}] + + # Record 317: PWAT at atmos col. + assert {_off_pwat, _len_pwat} = index[{"PWAT", "atmos col"}] + + # Last record in the file (423): length is computed against file_size. + assert {56_042_950, last_len} = index[{"WVVFLX", "atmos col"}] + assert last_len == @fixture_file_size - 56_042_950 + assert last_len > 0 + end + + test "returns an empty index for empty input" do + assert {:ok, %{}} = NarrClient.parse_inventory("", 0) + end + + test "ignores blank lines and trailing whitespace" do + inv = """ + 1:0:D=2010061512:MSLET:MSL:kpds=130,102,0:anl:winds in grid direction:"Mean sea level pressure [Pa] + + 2:166602:D=2010061512:PRMSL:MSL:kpds=2,102,0:anl:winds in grid direction:"Pressure reduced to MSL [Pa] + + """ + + {:ok, index} = NarrClient.parse_inventory(inv, 333_204) + + assert {0, 166_602} = index[{"MSLET", "MSL"}] + assert {166_602, 166_602} = index[{"PRMSL", "MSL"}] + end + end + + describe "fetch_inventory/1" do + @fixture_path "test/fixtures/narr/narr-a_221_20100615_1200_000.inv" + @fixture_file_size 56_221_430 + + test "fetches the .inv body and the .grb HEAD content-length, then parses" do + inv_body = File.read!(@fixture_path) + + Req.Test.stub(NarrClient, fn conn -> + cond do + String.ends_with?(conn.request_path, ".inv") and conn.method == "GET" -> + Plug.Conn.send_resp(conn, 200, inv_body) + + String.ends_with?(conn.request_path, ".grb") and conn.method == "HEAD" -> + conn + |> Plug.Conn.put_resp_header("content-length", Integer.to_string(@fixture_file_size)) + |> Plug.Conn.send_resp(200, "") + + true -> + Plug.Conn.send_resp(conn, 500, "unexpected #{conn.method} #{conn.request_path}") + end + end) + + assert {:ok, index} = NarrClient.fetch_inventory(~U[2010-06-15 12:00:00Z]) + + # Matches the parse_inventory spot-check for the same fixture. + assert {37_836_396, 130_968} = index[{"TMP", "2 m above gnd"}] + assert {56_042_950, last_len} = index[{"WVVFLX", "atmos col"}] + assert last_len == @fixture_file_size - 56_042_950 + end + + test "returns {:error, _} when the inv fetch fails" do + Req.Test.stub(NarrClient, fn conn -> + if String.ends_with?(conn.request_path, ".inv") do + Plug.Conn.send_resp(conn, 404, "not found") + else + Plug.Conn.send_resp(conn, 200, "") + end + end) + + assert {:error, _reason} = NarrClient.fetch_inventory(~U[2010-06-15 12:00:00Z]) + end + + test "returns {:error, _} when the grb HEAD fails" do + inv_body = File.read!(@fixture_path) + + Req.Test.stub(NarrClient, fn conn -> + cond do + String.ends_with?(conn.request_path, ".inv") -> + Plug.Conn.send_resp(conn, 200, inv_body) + + String.ends_with?(conn.request_path, ".grb") -> + Plug.Conn.send_resp(conn, 500, "server error") + + true -> + Plug.Conn.send_resp(conn, 500, "unexpected") + end + end) + + assert {:error, _reason} = NarrClient.fetch_inventory(~U[2010-06-15 12:00:00Z]) + end + end end