AWS S3 doesn't support multi-range HTTP requests. When given Range: bytes=0-999, 2000-2999 it ignores the header and returns the full file (200 instead of 206). Download each range separately.
209 lines
6.6 KiB
Elixir
209 lines
6.6 KiB
Elixir
defmodule Microwaveprop.Weather.HrrrClientTest do
|
|
use ExUnit.Case, async: true
|
|
|
|
alias Microwaveprop.Weather.HrrrClient
|
|
|
|
describe "nearest_hrrr_hour/1" do
|
|
test "rounds down when within first 30 minutes" do
|
|
assert HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:15:00Z]) ==
|
|
~U[2026-03-28 18:00:00Z]
|
|
end
|
|
|
|
test "rounds up when past 30 minutes" do
|
|
assert HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:45:00Z]) ==
|
|
~U[2026-03-28 19:00:00Z]
|
|
end
|
|
|
|
test "stays same when exactly on the hour" do
|
|
assert HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:00:00Z]) ==
|
|
~U[2026-03-28 18:00:00Z]
|
|
end
|
|
|
|
test "rounds at exactly 30 minutes" do
|
|
result = HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:30:00Z])
|
|
assert result in [~U[2026-03-28 18:00:00Z], ~U[2026-03-28 19:00:00Z]]
|
|
end
|
|
end
|
|
|
|
describe "hrrr_url/3" do
|
|
test "builds surface product URL" do
|
|
url = HrrrClient.hrrr_url(~D[2026-03-28], 18, :surface)
|
|
|
|
assert url ==
|
|
"https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t18z.wrfsfcf00.grib2"
|
|
end
|
|
|
|
test "builds pressure product URL" do
|
|
url = HrrrClient.hrrr_url(~D[2026-03-28], 6, :pressure)
|
|
|
|
assert url ==
|
|
"https://noaa-hrrr-bdp-pds.s3.amazonaws.com/hrrr.20260328/conus/hrrr.t06z.wrfprsf00.grib2"
|
|
end
|
|
|
|
test "pads single-digit hours" do
|
|
url = HrrrClient.hrrr_url(~D[2026-03-28], 3, :surface)
|
|
assert String.contains?(url, "hrrr.t03z")
|
|
end
|
|
end
|
|
|
|
describe "parse_idx/1" do
|
|
@sample_idx """
|
|
1:0:d=2026032818:TMP:2 m above ground:anl:
|
|
2:1234567:d=2026032818:DPT:2 m above ground:anl:
|
|
3:2345678:d=2026032818:PRES:surface:anl:
|
|
4:3456789:d=2026032818:HPBL:surface:anl:
|
|
5:4567890:d=2026032818:PWAT:entire atmosphere (considered as a single layer):anl:
|
|
6:5678901:d=2026032818:TMP:1000 mb:anl:
|
|
"""
|
|
|
|
test "parses idx into structured list" do
|
|
entries = HrrrClient.parse_idx(@sample_idx)
|
|
|
|
assert length(entries) == 6
|
|
|
|
first = hd(entries)
|
|
assert first.msg == 1
|
|
assert first.offset == 0
|
|
assert first.var == "TMP"
|
|
assert first.level == "2 m above ground"
|
|
end
|
|
|
|
test "correctly extracts offsets" do
|
|
entries = HrrrClient.parse_idx(@sample_idx)
|
|
offsets = Enum.map(entries, & &1.offset)
|
|
assert offsets == [0, 1_234_567, 2_345_678, 3_456_789, 4_567_890, 5_678_901]
|
|
end
|
|
|
|
test "handles empty input" do
|
|
assert HrrrClient.parse_idx("") == []
|
|
end
|
|
end
|
|
|
|
describe "byte_ranges_for_messages/2" do
|
|
test "computes byte ranges from idx entries" do
|
|
entries = [
|
|
%{msg: 1, offset: 0, var: "TMP", level: "2 m above ground"},
|
|
%{msg: 2, offset: 1000, var: "DPT", level: "2 m above ground"},
|
|
%{msg: 3, offset: 2000, var: "PRES", level: "surface"},
|
|
%{msg: 4, offset: 3000, var: "HPBL", level: "surface"}
|
|
]
|
|
|
|
wanted = [
|
|
%{var: "TMP", level: "2 m above ground"},
|
|
%{var: "PRES", level: "surface"}
|
|
]
|
|
|
|
ranges = HrrrClient.byte_ranges_for_messages(entries, wanted)
|
|
|
|
assert length(ranges) == 2
|
|
assert {0, 999} in ranges
|
|
assert {2000, 2999} in ranges
|
|
end
|
|
|
|
test "handles last message in file" do
|
|
entries = [
|
|
%{msg: 1, offset: 0, var: "TMP", level: "2 m above ground"},
|
|
%{msg: 2, offset: 1000, var: "DPT", level: "2 m above ground"}
|
|
]
|
|
|
|
wanted = [%{var: "DPT", level: "2 m above ground"}]
|
|
|
|
ranges = HrrrClient.byte_ranges_for_messages(entries, wanted)
|
|
|
|
# Last message - uses a large end offset
|
|
assert length(ranges) == 1
|
|
[{start, _end}] = ranges
|
|
assert start == 1000
|
|
end
|
|
end
|
|
|
|
describe "fetch_profile/3" do
|
|
test "succeeds when ranges are downloaded individually" do
|
|
grib_data = File.read!("test/fixtures/grib2/hrrr_tmp_2m.grib2")
|
|
|
|
# Idx with multiple surface variables to trigger multi-range download
|
|
grib_size = byte_size(grib_data)
|
|
|
|
idx_text = """
|
|
1:0:d=2026032818:TMP:2 m above ground:anl:
|
|
2:#{grib_size}:d=2026032818:DPT:2 m above ground:anl:
|
|
3:#{grib_size * 2}:d=2026032818:PRES:surface:anl:
|
|
"""
|
|
|
|
Req.Test.stub(HrrrClient, fn conn ->
|
|
if String.ends_with?(conn.request_path, ".idx") do
|
|
Plug.Conn.send_resp(conn, 200, idx_text)
|
|
else
|
|
[range] = Plug.Conn.get_req_header(conn, "range")
|
|
|
|
if String.contains?(range, ",") do
|
|
# Multi-range: S3 ignores Range header, returns full file
|
|
Plug.Conn.send_resp(conn, 200, "full file")
|
|
else
|
|
# Single range: S3 supports this fine
|
|
Plug.Conn.send_resp(conn, 206, grib_data)
|
|
end
|
|
end
|
|
end)
|
|
|
|
assert {:ok, profile} = HrrrClient.fetch_profile(32.90, -97.04, ~U[2026-03-28 18:00:00Z])
|
|
assert is_float(profile.surface_temp_c)
|
|
assert is_struct(profile.run_time, DateTime)
|
|
end
|
|
end
|
|
|
|
describe "build_profile/1" do
|
|
test "assembles profile from parsed data" do
|
|
parsed = %{
|
|
"TMP:1000 mb" => 298.0,
|
|
"DPT:1000 mb" => 291.0,
|
|
"HGT:1000 mb" => 110.0,
|
|
"TMP:975 mb" => 296.0,
|
|
"DPT:975 mb" => 289.0,
|
|
"HGT:975 mb" => 350.0,
|
|
"TMP:950 mb" => 294.0,
|
|
"DPT:950 mb" => 287.0,
|
|
"HGT:950 mb" => 590.0,
|
|
"TMP:2 m above ground" => 299.0,
|
|
"DPT:2 m above ground" => 292.0,
|
|
"PRES:surface" => 101_350.0,
|
|
"HPBL:surface" => 1500.0,
|
|
"PWAT:entire atmosphere (considered as a single layer)" => 25.0
|
|
}
|
|
|
|
result = HrrrClient.build_profile(parsed)
|
|
|
|
assert result.surface_temp_c == 299.0 - 273.15
|
|
assert result.surface_dewpoint_c == 292.0 - 273.15
|
|
assert_in_delta result.surface_pressure_mb, 1013.5, 0.1
|
|
assert result.hpbl_m == 1500.0
|
|
assert result.pwat_mm == 25.0
|
|
assert length(result.profile) == 3
|
|
|
|
first_level = hd(result.profile)
|
|
assert first_level["pres"] == 1000.0
|
|
assert first_level["tmpc"] == 298.0 - 273.15
|
|
assert first_level["dwpc"] == 291.0 - 273.15
|
|
assert first_level["hght"] == 110.0
|
|
end
|
|
|
|
test "skips pressure levels with missing data" do
|
|
parsed = %{
|
|
"TMP:1000 mb" => 298.0,
|
|
"DPT:1000 mb" => 291.0,
|
|
"HGT:1000 mb" => 110.0,
|
|
# 975 mb missing HGT
|
|
"TMP:975 mb" => 296.0,
|
|
"DPT:975 mb" => 289.0,
|
|
"TMP:2 m above ground" => 299.0,
|
|
"DPT:2 m above ground" => 292.0,
|
|
"PRES:surface" => 101_350.0,
|
|
"HPBL:surface" => 1500.0,
|
|
"PWAT:entire atmosphere (considered as a single layer)" => 25.0
|
|
}
|
|
|
|
result = HrrrClient.build_profile(parsed)
|
|
assert length(result.profile) == 1
|
|
end
|
|
end
|
|
end
|