Graham McIntire
081cd47bb5
parse_inventory walks the wgrib1-format .inv text and returns a
{var, level} -> {byte_offset, length} index, computing each record's
length from the next record's offset (or file_size for the last one).
fetch_inventory does a GET on the .inv URL plus a HEAD on the .grb URL
to learn its size, then delegates to parse_inventory.
Both stubbable via Req.Test (config/test.exs adds narr_req_options
matching the existing era5_req_options / giro_req_options pattern).
Spike fixture at test/fixtures/narr/narr-a_221_20100615_1200_000.inv
backs the parser test — verifies the ("TMP", "2 m above gnd") and
("WVVFLX", "atmos col") records line up with the real bytes.
185 lines
6.4 KiB
Elixir
185 lines
6.4 KiB
Elixir
defmodule Microwaveprop.Weather.NarrClientTest do
|
|
use ExUnit.Case, async: true
|
|
|
|
alias Microwaveprop.Weather.NarrClient
|
|
|
|
describe "url_for/1" do
|
|
test "builds the NCEI NARR HTTPS URL for a 3-hourly analysis time" do
|
|
vt = ~U[2010-06-15 12:00:00Z]
|
|
|
|
assert NarrClient.url_for(vt) ==
|
|
"https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/access/3-hourly/201006/20100615/narr-a_221_20100615_1200_000.grb"
|
|
end
|
|
|
|
test "uses zero-padded HHMM for off-hour analyses" do
|
|
vt = ~U[2010-06-15 03:00:00Z]
|
|
|
|
assert NarrClient.url_for(vt) ==
|
|
"https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/access/3-hourly/201006/20100615/narr-a_221_20100615_0300_000.grb"
|
|
end
|
|
|
|
test "raises ArgumentError on non-3-hourly hours" do
|
|
assert_raise ArgumentError, fn ->
|
|
NarrClient.url_for(~U[2010-06-15 13:00:00Z])
|
|
end
|
|
end
|
|
|
|
test "raises ArgumentError when minutes are non-zero" do
|
|
assert_raise ArgumentError, fn ->
|
|
NarrClient.url_for(~U[2010-06-15 12:30:00Z])
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "url_for_inventory/1" do
|
|
test "swaps .grb for .inv" do
|
|
vt = ~U[2010-06-15 12:00:00Z]
|
|
|
|
assert NarrClient.url_for_inventory(vt) ==
|
|
"https://www.ncei.noaa.gov/data/north-american-regional-reanalysis/access/3-hourly/201006/20100615/narr-a_221_20100615_1200_000.inv"
|
|
end
|
|
end
|
|
|
|
describe "snap_to_analysis_hour/1" do
|
|
test "rounds down to the nearest 3-hourly NARR slot" do
|
|
assert NarrClient.snap_to_analysis_hour(~U[2010-06-15 13:42:00Z]) ==
|
|
~U[2010-06-15 12:00:00Z]
|
|
|
|
assert NarrClient.snap_to_analysis_hour(~U[2010-06-15 14:30:00Z]) ==
|
|
~U[2010-06-15 12:00:00Z]
|
|
|
|
assert NarrClient.snap_to_analysis_hour(~U[2010-06-15 15:00:00Z]) ==
|
|
~U[2010-06-15 15:00:00Z]
|
|
|
|
assert NarrClient.snap_to_analysis_hour(~U[2010-06-15 23:59:59Z]) ==
|
|
~U[2010-06-15 21:00:00Z]
|
|
|
|
assert NarrClient.snap_to_analysis_hour(~U[2010-06-15 00:00:00Z]) ==
|
|
~U[2010-06-15 00:00:00Z]
|
|
|
|
assert NarrClient.snap_to_analysis_hour(~U[2010-06-15 02:00:00Z]) ==
|
|
~U[2010-06-15 00:00:00Z]
|
|
end
|
|
|
|
test "preserves the date when snapping doesn't cross midnight" do
|
|
assert NarrClient.snap_to_analysis_hour(~U[2010-06-15 05:17:42Z]) ==
|
|
~U[2010-06-15 03:00:00Z]
|
|
end
|
|
|
|
test "snaps minutes and seconds to zero" do
|
|
snapped = NarrClient.snap_to_analysis_hour(~U[2010-06-15 13:42:37Z])
|
|
assert snapped.minute == 0
|
|
assert snapped.second == 0
|
|
assert snapped.microsecond == {0, 0}
|
|
end
|
|
end
|
|
|
|
describe "parse_inventory/2" do
|
|
@fixture_path "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
|
|
@fixture_file_size 56_221_430
|
|
|
|
test "parses the spike fixture inventory into an offset/length index" do
|
|
inv = File.read!(@fixture_path)
|
|
|
|
{:ok, index} = NarrClient.parse_inventory(inv, @fixture_file_size)
|
|
|
|
# Record 288: TMP @ 2 m above gnd, offset 37836396, next record at 37967364.
|
|
assert {37_836_396, 130_968} = index[{"TMP", "2 m above gnd"}]
|
|
|
|
# Record 15: HPBL at surface — length is positive and computed from next record.
|
|
assert {1_940_538, len_hpbl} = index[{"HPBL", "sfc"}]
|
|
assert len_hpbl > 0
|
|
|
|
# Record 202: TMP at 850 mb — pressure-level record present in the index.
|
|
assert {_off_850, _len_850} = index[{"TMP", "850 mb"}]
|
|
|
|
# Record 317: PWAT at atmos col.
|
|
assert {_off_pwat, _len_pwat} = index[{"PWAT", "atmos col"}]
|
|
|
|
# Last record in the file (423): length is computed against file_size.
|
|
assert {56_042_950, last_len} = index[{"WVVFLX", "atmos col"}]
|
|
assert last_len == @fixture_file_size - 56_042_950
|
|
assert last_len > 0
|
|
end
|
|
|
|
test "returns an empty index for empty input" do
|
|
assert {:ok, %{}} = NarrClient.parse_inventory("", 0)
|
|
end
|
|
|
|
test "ignores blank lines and trailing whitespace" do
|
|
inv = """
|
|
1:0:D=2010061512:MSLET:MSL:kpds=130,102,0:anl:winds in grid direction:"Mean sea level pressure [Pa]
|
|
|
|
2:166602:D=2010061512:PRMSL:MSL:kpds=2,102,0:anl:winds in grid direction:"Pressure reduced to MSL [Pa]
|
|
|
|
"""
|
|
|
|
{:ok, index} = NarrClient.parse_inventory(inv, 333_204)
|
|
|
|
assert {0, 166_602} = index[{"MSLET", "MSL"}]
|
|
assert {166_602, 166_602} = index[{"PRMSL", "MSL"}]
|
|
end
|
|
end
|
|
|
|
describe "fetch_inventory/1" do
|
|
@fixture_path "test/fixtures/narr/narr-a_221_20100615_1200_000.inv"
|
|
@fixture_file_size 56_221_430
|
|
|
|
test "fetches the .inv body and the .grb HEAD content-length, then parses" do
|
|
inv_body = File.read!(@fixture_path)
|
|
|
|
Req.Test.stub(NarrClient, fn conn ->
|
|
cond do
|
|
String.ends_with?(conn.request_path, ".inv") and conn.method == "GET" ->
|
|
Plug.Conn.send_resp(conn, 200, inv_body)
|
|
|
|
String.ends_with?(conn.request_path, ".grb") and conn.method == "HEAD" ->
|
|
conn
|
|
|> Plug.Conn.put_resp_header("content-length", Integer.to_string(@fixture_file_size))
|
|
|> Plug.Conn.send_resp(200, "")
|
|
|
|
true ->
|
|
Plug.Conn.send_resp(conn, 500, "unexpected #{conn.method} #{conn.request_path}")
|
|
end
|
|
end)
|
|
|
|
assert {:ok, index} = NarrClient.fetch_inventory(~U[2010-06-15 12:00:00Z])
|
|
|
|
# Matches the parse_inventory spot-check for the same fixture.
|
|
assert {37_836_396, 130_968} = index[{"TMP", "2 m above gnd"}]
|
|
assert {56_042_950, last_len} = index[{"WVVFLX", "atmos col"}]
|
|
assert last_len == @fixture_file_size - 56_042_950
|
|
end
|
|
|
|
test "returns {:error, _} when the inv fetch fails" do
|
|
Req.Test.stub(NarrClient, fn conn ->
|
|
if String.ends_with?(conn.request_path, ".inv") do
|
|
Plug.Conn.send_resp(conn, 404, "not found")
|
|
else
|
|
Plug.Conn.send_resp(conn, 200, "")
|
|
end
|
|
end)
|
|
|
|
assert {:error, _reason} = NarrClient.fetch_inventory(~U[2010-06-15 12:00:00Z])
|
|
end
|
|
|
|
test "returns {:error, _} when the grb HEAD fails" do
|
|
inv_body = File.read!(@fixture_path)
|
|
|
|
Req.Test.stub(NarrClient, fn conn ->
|
|
cond do
|
|
String.ends_with?(conn.request_path, ".inv") ->
|
|
Plug.Conn.send_resp(conn, 200, inv_body)
|
|
|
|
String.ends_with?(conn.request_path, ".grb") ->
|
|
Plug.Conn.send_resp(conn, 500, "server error")
|
|
|
|
true ->
|
|
Plug.Conn.send_resp(conn, 500, "unexpected")
|
|
end
|
|
end)
|
|
|
|
assert {:error, _reason} = NarrClient.fetch_inventory(~U[2010-06-15 12:00:00Z])
|
|
end
|
|
end
|
|
end
|