Download HRRR ranges sequentially to prevent memory accumulation

The parallel download was holding all ~530MB of range responses in
memory before writing to disk. Now each range is fetched and written
one at a time, so only one chunk is in memory at a time.
This commit is contained in:
Graham McIntire 2026-04-10 15:58:38 -05:00
parent 525a63d246
commit 2973fe978b

View file

@ -347,46 +347,29 @@ defmodule Microwaveprop.Weather.HrrrClient do
Returns `:ok` or `{:error, reason}`.
"""
def download_grib_ranges_to_file(_url, [], _dest_path), do: {:ok, <<>>}
def download_grib_ranges_to_file(_url, [], _dest_path), do: :ok
def download_grib_ranges_to_file(url, ranges, dest_path) do
merged = merge_ranges(ranges)
merged = merge_ranges(ranges) |> Enum.sort_by(&elem(&1, 0))
results =
merged
|> Task.async_stream(
fn {start, stop} ->
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
{:ok, %{status: 206, body: body}} -> {:ok, start, body}
{:ok, %{status: status}} -> {:error, "HRRR grib HTTP #{status}"}
{:error, reason} -> {:error, reason}
end
end,
max_concurrency: 4,
timeout: 120_000
)
|> Enum.map(fn
{:ok, result} -> result
{:exit, reason} -> {:error, reason}
end)
file = File.open!(dest_path, [:write, :binary])
case Enum.find(results, &match?({:error, _}, &1)) do
{:error, reason} ->
{:error, reason}
nil ->
sorted = Enum.sort_by(results, fn {:ok, offset, _} -> offset end)
file = File.open!(dest_path, [:write, :binary])
try do
Enum.each(sorted, fn {:ok, _offset, body} ->
try do
Enum.reduce_while(merged, :ok, fn {start, stop}, :ok ->
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
{:ok, %{status: 206, body: body}} ->
IO.binwrite(file, body)
end)
after
File.close(file)
end
{:cont, :ok}
:ok
{:ok, %{status: status}} ->
{:halt, {:error, "HRRR grib HTTP #{status}"}}
{:error, reason} ->
{:halt, {:error, reason}}
end
end)
after
File.close(file)
end
end