prop/test/microwaveprop/weather/hrrr_client_test.exs
Graham McIntire 4ef755f0cf
Download HRRR GRIB ranges individually instead of multi-range
AWS S3 doesn't support multi-range HTTP requests. When given
Range: bytes=0-999, 2000-2999 it ignores the header and returns
the full file (200 instead of 206). Download each range separately.
2026-03-30 09:39:24 -05:00

209 lines
6.6 KiB
Elixir

defmodule Microwaveprop.Weather.HrrrClientTest do
use ExUnit.Case, async: true
alias Microwaveprop.Weather.HrrrClient
describe "nearest_hrrr_hour/1" do
test "rounds down when within first 30 minutes" do
assert HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:15:00Z]) ==
~U[2026-03-28 18:00:00Z]
end
test "rounds up when past 30 minutes" do
assert HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:45:00Z]) ==
~U[2026-03-28 19:00:00Z]
end
test "stays same when exactly on the hour" do
assert HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:00:00Z]) ==
~U[2026-03-28 18:00:00Z]
end
test "rounds at exactly 30 minutes" do
result = HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:30:00Z])
assert result in [~U[2026-03-28 18:00:00Z], ~U[2026-03-28 19:00:00Z]]
end
end
describe "hrrr_url/3" do
test "builds surface product URL" do
url = HrrrClient.hrrr_url(~D[2026-03-28], 18, :surface)
assert url ==
"https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t18z.wrfsfcf00.grib2"
end
test "builds pressure product URL" do
url = HrrrClient.hrrr_url(~D[2026-03-28], 6, :pressure)
assert url ==
"https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t06z.wrfprsf00.grib2"
end
test "pads single-digit hours" do
url = HrrrClient.hrrr_url(~D[2026-03-28], 3, :surface)
assert String.contains?(url, "hrrr.t03z")
end
end
describe "parse_idx/1" do
@sample_idx """
1:0:d=2026032818:TMP:2 m above ground:anl:
2:1234567:d=2026032818:DPT:2 m above ground:anl:
3:2345678:d=2026032818:PRES:surface:anl:
4:3456789:d=2026032818:HPBL:surface:anl:
5:4567890:d=2026032818:PWAT:entire atmosphere (considered as a single layer):anl:
6:5678901:d=2026032818:TMP:1000 mb:anl:
"""
test "parses idx into structured list" do
entries = HrrrClient.parse_idx(@sample_idx)
assert length(entries) == 6
first = hd(entries)
assert first.msg == 1
assert first.offset == 0
assert first.var == "TMP"
assert first.level == "2 m above ground"
end
test "correctly extracts offsets" do
entries = HrrrClient.parse_idx(@sample_idx)
offsets = Enum.map(entries, & &1.offset)
assert offsets == [0, 1_234_567, 2_345_678, 3_456_789, 4_567_890, 5_678_901]
end
test "handles empty input" do
assert HrrrClient.parse_idx("") == []
end
end
describe "byte_ranges_for_messages/2" do
test "computes byte ranges from idx entries" do
entries = [
%{msg: 1, offset: 0, var: "TMP", level: "2 m above ground"},
%{msg: 2, offset: 1000, var: "DPT", level: "2 m above ground"},
%{msg: 3, offset: 2000, var: "PRES", level: "surface"},
%{msg: 4, offset: 3000, var: "HPBL", level: "surface"}
]
wanted = [
%{var: "TMP", level: "2 m above ground"},
%{var: "PRES", level: "surface"}
]
ranges = HrrrClient.byte_ranges_for_messages(entries, wanted)
assert length(ranges) == 2
assert {0, 999} in ranges
assert {2000, 2999} in ranges
end
test "handles last message in file" do
entries = [
%{msg: 1, offset: 0, var: "TMP", level: "2 m above ground"},
%{msg: 2, offset: 1000, var: "DPT", level: "2 m above ground"}
]
wanted = [%{var: "DPT", level: "2 m above ground"}]
ranges = HrrrClient.byte_ranges_for_messages(entries, wanted)
# Last message - uses a large end offset
assert length(ranges) == 1
[{start, _end}] = ranges
assert start == 1000
end
end
describe "fetch_profile/3" do
test "succeeds when ranges are downloaded individually" do
grib_data = File.read!("test/fixtures/grib2/hrrr_tmp_2m.grib2")
# Idx with multiple surface variables to trigger multi-range download
grib_size = byte_size(grib_data)
idx_text = """
1:0:d=2026032818:TMP:2 m above ground:anl:
2:#{grib_size}:d=2026032818:DPT:2 m above ground:anl:
3:#{grib_size * 2}:d=2026032818:PRES:surface:anl:
"""
Req.Test.stub(HrrrClient, fn conn ->
if String.ends_with?(conn.request_path, ".idx") do
Plug.Conn.send_resp(conn, 200, idx_text)
else
[range] = Plug.Conn.get_req_header(conn, "range")
if String.contains?(range, ",") do
# Multi-range: S3 ignores Range header, returns full file
Plug.Conn.send_resp(conn, 200, "full file")
else
# Single range: S3 supports this fine
Plug.Conn.send_resp(conn, 206, grib_data)
end
end
end)
assert {:ok, profile} = HrrrClient.fetch_profile(32.90, -97.04, ~U[2026-03-28 18:00:00Z])
assert is_float(profile.surface_temp_c)
assert is_struct(profile.run_time, DateTime)
end
end
describe "build_profile/1" do
test "assembles profile from parsed data" do
parsed = %{
"TMP:1000 mb" => 298.0,
"DPT:1000 mb" => 291.0,
"HGT:1000 mb" => 110.0,
"TMP:975 mb" => 296.0,
"DPT:975 mb" => 289.0,
"HGT:975 mb" => 350.0,
"TMP:950 mb" => 294.0,
"DPT:950 mb" => 287.0,
"HGT:950 mb" => 590.0,
"TMP:2 m above ground" => 299.0,
"DPT:2 m above ground" => 292.0,
"PRES:surface" => 101_350.0,
"HPBL:surface" => 1500.0,
"PWAT:entire atmosphere (considered as a single layer)" => 25.0
}
result = HrrrClient.build_profile(parsed)
assert result.surface_temp_c == 299.0 - 273.15
assert result.surface_dewpoint_c == 292.0 - 273.15
assert_in_delta result.surface_pressure_mb, 1013.5, 0.1
assert result.hpbl_m == 1500.0
assert result.pwat_mm == 25.0
assert length(result.profile) == 3
first_level = hd(result.profile)
assert first_level["pres"] == 1000.0
assert first_level["tmpc"] == 298.0 - 273.15
assert first_level["dwpc"] == 291.0 - 273.15
assert first_level["hght"] == 110.0
end
test "skips pressure levels with missing data" do
parsed = %{
"TMP:1000 mb" => 298.0,
"DPT:1000 mb" => 291.0,
"HGT:1000 mb" => 110.0,
# 975 mb missing HGT
"TMP:975 mb" => 296.0,
"DPT:975 mb" => 289.0,
"TMP:2 m above ground" => 299.0,
"DPT:2 m above ground" => 292.0,
"PRES:surface" => 101_350.0,
"HPBL:surface" => 1500.0,
"PWAT:entire atmosphere (considered as a single layer)" => 25.0
}
result = HrrrClient.build_profile(parsed)
assert length(result.profile) == 1
end
end
end