Download HRRR GRIB ranges individually instead of multi-range
AWS S3 doesn't support multi-range HTTP requests. When given Range: bytes=0-999, 2000-2999 it ignores the header and returns the full file (200 instead of 206). Download each range separately.
This commit is contained in:
parent
bb590a3b64
commit
4ef755f0cf
2 changed files with 45 additions and 15 deletions
|
|
@ -174,23 +174,18 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
defp download_grib_ranges(_url, []), do: {:ok, <<>>}
|
||||
|
||||
defp download_grib_ranges(url, ranges) do
|
||||
range_header =
|
||||
Enum.map_join(ranges, ", ", fn {start, stop} -> "#{start}-#{stop}" end)
|
||||
Enum.reduce_while(ranges, {:ok, <<>>}, fn {start, stop}, {:ok, acc} ->
|
||||
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
|
||||
{:ok, %{status: 206, body: body}} ->
|
||||
{:cont, {:ok, acc <> body}}
|
||||
|
||||
case Req.get(url, [{:headers, [{"Range", "bytes=#{range_header}"}]} | req_options()]) do
|
||||
{:ok, %{status: 206, body: body}} ->
|
||||
{:ok, body}
|
||||
{:ok, %{status: status}} ->
|
||||
{:halt, {:error, "HRRR grib HTTP #{status}"}}
|
||||
|
||||
{:ok, %{status: 200}} ->
|
||||
Logger.warning("HRRR server ignored Range header, returned full file")
|
||||
{:error, "HRRR range request not supported"}
|
||||
|
||||
{:ok, %{status: status}} ->
|
||||
{:error, "HRRR grib HTTP #{status}"}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
{:error, reason} ->
|
||||
{:halt, {:error, reason}}
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp req_options do
|
||||
|
|
|
|||
|
|
@ -117,6 +117,41 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "fetch_profile/3" do
|
||||
test "succeeds when ranges are downloaded individually" do
|
||||
grib_data = File.read!("test/fixtures/grib2/hrrr_tmp_2m.grib2")
|
||||
|
||||
# Idx with multiple surface variables to trigger multi-range download
|
||||
grib_size = byte_size(grib_data)
|
||||
|
||||
idx_text = """
|
||||
1:0:d=2026032818:TMP:2 m above ground:anl:
|
||||
2:#{grib_size}:d=2026032818:DPT:2 m above ground:anl:
|
||||
3:#{grib_size * 2}:d=2026032818:PRES:surface:anl:
|
||||
"""
|
||||
|
||||
Req.Test.stub(HrrrClient, fn conn ->
|
||||
if String.ends_with?(conn.request_path, ".idx") do
|
||||
Plug.Conn.send_resp(conn, 200, idx_text)
|
||||
else
|
||||
[range] = Plug.Conn.get_req_header(conn, "range")
|
||||
|
||||
if String.contains?(range, ",") do
|
||||
# Multi-range: S3 ignores Range header, returns full file
|
||||
Plug.Conn.send_resp(conn, 200, "full file")
|
||||
else
|
||||
# Single range: S3 supports this fine
|
||||
Plug.Conn.send_resp(conn, 206, grib_data)
|
||||
end
|
||||
end
|
||||
end)
|
||||
|
||||
assert {:ok, profile} = HrrrClient.fetch_profile(32.90, -97.04, ~U[2026-03-28 18:00:00Z])
|
||||
assert is_float(profile.surface_temp_c)
|
||||
assert is_struct(profile.run_time, DateTime)
|
||||
end
|
||||
end
|
||||
|
||||
describe "build_profile/1" do
|
||||
test "assembles profile from parsed data" do
|
||||
parsed = %{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue