Download HRRR GRIB ranges individually instead of multi-range

AWS S3 doesn't support multi-range HTTP requests. When given
Range: bytes=0-999, 2000-2999 it ignores the header and returns
the full file (200 instead of 206). Download each range separately.
This commit is contained in:
Graham McIntire 2026-03-30 09:39:24 -05:00
parent bb590a3b64
commit 4ef755f0cf
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 45 additions and 15 deletions

View file

@ -174,23 +174,18 @@ defmodule Microwaveprop.Weather.HrrrClient do
defp download_grib_ranges(_url, []), do: {:ok, <<>>}
defp download_grib_ranges(url, ranges) do
range_header =
Enum.map_join(ranges, ", ", fn {start, stop} -> "#{start}-#{stop}" end)
Enum.reduce_while(ranges, {:ok, <<>>}, fn {start, stop}, {:ok, acc} ->
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
{:ok, %{status: 206, body: body}} ->
{:cont, {:ok, acc <> body}}
case Req.get(url, [{:headers, [{"Range", "bytes=#{range_header}"}]} | req_options()]) do
{:ok, %{status: 206, body: body}} ->
{:ok, body}
{:ok, %{status: status}} ->
{:halt, {:error, "HRRR grib HTTP #{status}"}}
{:ok, %{status: 200}} ->
Logger.warning("HRRR server ignored Range header, returned full file")
{:error, "HRRR range request not supported"}
{:ok, %{status: status}} ->
{:error, "HRRR grib HTTP #{status}"}
{:error, reason} ->
{:error, reason}
end
{:error, reason} ->
{:halt, {:error, reason}}
end
end)
end
defp req_options do

View file

@ -117,6 +117,41 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
end
end
describe "fetch_profile/3" do
test "succeeds when ranges are downloaded individually" do
grib_data = File.read!("test/fixtures/grib2/hrrr_tmp_2m.grib2")
# Idx with multiple surface variables to trigger multi-range download
grib_size = byte_size(grib_data)
idx_text = """
1:0:d=2026032818:TMP:2 m above ground:anl:
2:#{grib_size}:d=2026032818:DPT:2 m above ground:anl:
3:#{grib_size * 2}:d=2026032818:PRES:surface:anl:
"""
Req.Test.stub(HrrrClient, fn conn ->
if String.ends_with?(conn.request_path, ".idx") do
Plug.Conn.send_resp(conn, 200, idx_text)
else
[range] = Plug.Conn.get_req_header(conn, "range")
if String.contains?(range, ",") do
# Multi-range: S3 ignores Range header, returns full file
Plug.Conn.send_resp(conn, 200, "full file")
else
# Single range: S3 supports this fine
Plug.Conn.send_resp(conn, 206, grib_data)
end
end
end)
assert {:ok, profile} = HrrrClient.fetch_profile(32.90, -97.04, ~U[2026-03-28 18:00:00Z])
assert is_float(profile.surface_temp_c)
assert is_struct(profile.run_time, DateTime)
end
end
describe "build_profile/1" do
test "assembles profile from parsed data" do
parsed = %{