Download HRRR ranges sequentially to prevent memory accumulation
The parallel download was holding all ~530MB of range responses in memory before writing to disk. Now each range is fetched and written one at a time, so only one chunk is in memory at a time.
This commit is contained in:
parent
525a63d246
commit
2973fe978b
1 changed files with 17 additions and 34 deletions
|
|
@ -347,46 +347,29 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
|
||||
Returns `:ok` or `{:error, reason}`.
|
||||
"""
|
||||
def download_grib_ranges_to_file(_url, [], _dest_path), do: {:ok, <<>>}
|
||||
def download_grib_ranges_to_file(_url, [], _dest_path), do: :ok
|
||||
|
||||
def download_grib_ranges_to_file(url, ranges, dest_path) do
|
||||
merged = merge_ranges(ranges)
|
||||
merged = merge_ranges(ranges) |> Enum.sort_by(&elem(&1, 0))
|
||||
|
||||
results =
|
||||
merged
|
||||
|> Task.async_stream(
|
||||
fn {start, stop} ->
|
||||
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
|
||||
{:ok, %{status: 206, body: body}} -> {:ok, start, body}
|
||||
{:ok, %{status: status}} -> {:error, "HRRR grib HTTP #{status}"}
|
||||
{:error, reason} -> {:error, reason}
|
||||
end
|
||||
end,
|
||||
max_concurrency: 4,
|
||||
timeout: 120_000
|
||||
)
|
||||
|> Enum.map(fn
|
||||
{:ok, result} -> result
|
||||
{:exit, reason} -> {:error, reason}
|
||||
end)
|
||||
file = File.open!(dest_path, [:write, :binary])
|
||||
|
||||
case Enum.find(results, &match?({:error, _}, &1)) do
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
|
||||
nil ->
|
||||
sorted = Enum.sort_by(results, fn {:ok, offset, _} -> offset end)
|
||||
file = File.open!(dest_path, [:write, :binary])
|
||||
|
||||
try do
|
||||
Enum.each(sorted, fn {:ok, _offset, body} ->
|
||||
try do
|
||||
Enum.reduce_while(merged, :ok, fn {start, stop}, :ok ->
|
||||
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
|
||||
{:ok, %{status: 206, body: body}} ->
|
||||
IO.binwrite(file, body)
|
||||
end)
|
||||
after
|
||||
File.close(file)
|
||||
end
|
||||
{:cont, :ok}
|
||||
|
||||
:ok
|
||||
{:ok, %{status: status}} ->
|
||||
{:halt, {:error, "HRRR grib HTTP #{status}"}}
|
||||
|
||||
{:error, reason} ->
|
||||
{:halt, {:error, reason}}
|
||||
end
|
||||
end)
|
||||
after
|
||||
File.close(file)
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue