Download HRRR GRIB ranges individually instead of multi-range
AWS S3 doesn't support multi-range HTTP requests. When given Range: bytes=0-999, 2000-2999 it ignores the header and returns the full file (200 instead of 206). Download each range separately.
This commit is contained in:
parent
bb590a3b64
commit
4ef755f0cf
2 changed files with 45 additions and 15 deletions
|
|
@ -174,23 +174,18 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
defp download_grib_ranges(_url, []), do: {:ok, <<>>}
|
defp download_grib_ranges(_url, []), do: {:ok, <<>>}
|
||||||
|
|
||||||
defp download_grib_ranges(url, ranges) do
|
defp download_grib_ranges(url, ranges) do
|
||||||
range_header =
|
Enum.reduce_while(ranges, {:ok, <<>>}, fn {start, stop}, {:ok, acc} ->
|
||||||
Enum.map_join(ranges, ", ", fn {start, stop} -> "#{start}-#{stop}" end)
|
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
|
||||||
|
{:ok, %{status: 206, body: body}} ->
|
||||||
|
{:cont, {:ok, acc <> body}}
|
||||||
|
|
||||||
case Req.get(url, [{:headers, [{"Range", "bytes=#{range_header}"}]} | req_options()]) do
|
{:ok, %{status: status}} ->
|
||||||
{:ok, %{status: 206, body: body}} ->
|
{:halt, {:error, "HRRR grib HTTP #{status}"}}
|
||||||
{:ok, body}
|
|
||||||
|
|
||||||
{:ok, %{status: 200}} ->
|
{:error, reason} ->
|
||||||
Logger.warning("HRRR server ignored Range header, returned full file")
|
{:halt, {:error, reason}}
|
||||||
{:error, "HRRR range request not supported"}
|
end
|
||||||
|
end)
|
||||||
{:ok, %{status: status}} ->
|
|
||||||
{:error, "HRRR grib HTTP #{status}"}
|
|
||||||
|
|
||||||
{:error, reason} ->
|
|
||||||
{:error, reason}
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
|
|
||||||
defp req_options do
|
defp req_options do
|
||||||
|
|
|
||||||
|
|
@ -117,6 +117,41 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe "fetch_profile/3" do
|
||||||
|
test "succeeds when ranges are downloaded individually" do
|
||||||
|
grib_data = File.read!("test/fixtures/grib2/hrrr_tmp_2m.grib2")
|
||||||
|
|
||||||
|
# Idx with multiple surface variables to trigger multi-range download
|
||||||
|
grib_size = byte_size(grib_data)
|
||||||
|
|
||||||
|
idx_text = """
|
||||||
|
1:0:d=2026032818:TMP:2 m above ground:anl:
|
||||||
|
2:#{grib_size}:d=2026032818:DPT:2 m above ground:anl:
|
||||||
|
3:#{grib_size * 2}:d=2026032818:PRES:surface:anl:
|
||||||
|
"""
|
||||||
|
|
||||||
|
Req.Test.stub(HrrrClient, fn conn ->
|
||||||
|
if String.ends_with?(conn.request_path, ".idx") do
|
||||||
|
Plug.Conn.send_resp(conn, 200, idx_text)
|
||||||
|
else
|
||||||
|
[range] = Plug.Conn.get_req_header(conn, "range")
|
||||||
|
|
||||||
|
if String.contains?(range, ",") do
|
||||||
|
# Multi-range: S3 ignores Range header, returns full file
|
||||||
|
Plug.Conn.send_resp(conn, 200, "full file")
|
||||||
|
else
|
||||||
|
# Single range: S3 supports this fine
|
||||||
|
Plug.Conn.send_resp(conn, 206, grib_data)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
|
||||||
|
assert {:ok, profile} = HrrrClient.fetch_profile(32.90, -97.04, ~U[2026-03-28 18:00:00Z])
|
||||||
|
assert is_float(profile.surface_temp_c)
|
||||||
|
assert is_struct(profile.run_time, DateTime)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
describe "build_profile/1" do
|
describe "build_profile/1" do
|
||||||
test "assembles profile from parsed data" do
|
test "assembles profile from parsed data" do
|
||||||
parsed = %{
|
parsed = %{
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue