Download HRRR ranges sequentially to prevent memory accumulation
The parallel download was holding all ~530MB of range responses in memory before writing to disk. Now each range is fetched and written one at a time, so only one chunk is in memory at a time.
This commit is contained in:
parent
525a63d246
commit
2973fe978b
1 changed files with 17 additions and 34 deletions
|
|
@ -347,46 +347,29 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
|
|
||||||
Returns `:ok` or `{:error, reason}`.
|
Returns `:ok` or `{:error, reason}`.
|
||||||
"""
|
"""
|
||||||
def download_grib_ranges_to_file(_url, [], _dest_path), do: {:ok, <<>>}
|
def download_grib_ranges_to_file(_url, [], _dest_path), do: :ok
|
||||||
|
|
||||||
def download_grib_ranges_to_file(url, ranges, dest_path) do
|
def download_grib_ranges_to_file(url, ranges, dest_path) do
|
||||||
merged = merge_ranges(ranges)
|
merged = merge_ranges(ranges) |> Enum.sort_by(&elem(&1, 0))
|
||||||
|
|
||||||
results =
|
file = File.open!(dest_path, [:write, :binary])
|
||||||
merged
|
|
||||||
|> Task.async_stream(
|
|
||||||
fn {start, stop} ->
|
|
||||||
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
|
|
||||||
{:ok, %{status: 206, body: body}} -> {:ok, start, body}
|
|
||||||
{:ok, %{status: status}} -> {:error, "HRRR grib HTTP #{status}"}
|
|
||||||
{:error, reason} -> {:error, reason}
|
|
||||||
end
|
|
||||||
end,
|
|
||||||
max_concurrency: 4,
|
|
||||||
timeout: 120_000
|
|
||||||
)
|
|
||||||
|> Enum.map(fn
|
|
||||||
{:ok, result} -> result
|
|
||||||
{:exit, reason} -> {:error, reason}
|
|
||||||
end)
|
|
||||||
|
|
||||||
case Enum.find(results, &match?({:error, _}, &1)) do
|
try do
|
||||||
{:error, reason} ->
|
Enum.reduce_while(merged, :ok, fn {start, stop}, :ok ->
|
||||||
{:error, reason}
|
case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do
|
||||||
|
{:ok, %{status: 206, body: body}} ->
|
||||||
nil ->
|
|
||||||
sorted = Enum.sort_by(results, fn {:ok, offset, _} -> offset end)
|
|
||||||
file = File.open!(dest_path, [:write, :binary])
|
|
||||||
|
|
||||||
try do
|
|
||||||
Enum.each(sorted, fn {:ok, _offset, body} ->
|
|
||||||
IO.binwrite(file, body)
|
IO.binwrite(file, body)
|
||||||
end)
|
{:cont, :ok}
|
||||||
after
|
|
||||||
File.close(file)
|
|
||||||
end
|
|
||||||
|
|
||||||
:ok
|
{:ok, %{status: status}} ->
|
||||||
|
{:halt, {:error, "HRRR grib HTTP #{status}"}}
|
||||||
|
|
||||||
|
{:error, reason} ->
|
||||||
|
{:halt, {:error, reason}}
|
||||||
|
end
|
||||||
|
end)
|
||||||
|
after
|
||||||
|
File.close(file)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue