fix: log swallowed async task exits in GEFS scoring and HRRR range fetch
GefsFetch silently dropped crashed score-computation tasks via
`{:exit, _reason} -> []`, hiding partial-grid outages from k8s logs.
HRRR range downloads mapped exits to `{:error, reason}` without logging
the URL, so the failing range was lost by the time the caller surfaced
the error.
All three sites now Logger.error with enough context (fh, valid_time,
url) to debug from production logs.
This commit is contained in:
parent
388129865e
commit
301935f8c1
2 changed files with 23 additions and 6 deletions
|
|
@ -566,8 +566,13 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
timeout: 120_000
|
timeout: 120_000
|
||||||
)
|
)
|
||||||
|> Enum.map(fn
|
|> Enum.map(fn
|
||||||
{:ok, result} -> result
|
{:ok, result} ->
|
||||||
{:exit, reason} -> {:error, reason}
|
result
|
||||||
|
|
||||||
|
{:exit, reason} ->
|
||||||
|
Logger.error("HRRR grib range download crashed (to_file): url=#{url} reason=#{inspect(reason)}")
|
||||||
|
|
||||||
|
{:error, reason}
|
||||||
end)
|
end)
|
||||||
|
|
||||||
case Enum.find(results, &match?({:error, _}, &1)) do
|
case Enum.find(results, &match?({:error, _}, &1)) do
|
||||||
|
|
@ -612,8 +617,13 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
timeout: 120_000
|
timeout: 120_000
|
||||||
)
|
)
|
||||||
|> Enum.map(fn
|
|> Enum.map(fn
|
||||||
{:ok, result} -> result
|
{:ok, result} ->
|
||||||
{:exit, reason} -> {:error, reason}
|
result
|
||||||
|
|
||||||
|
{:exit, reason} ->
|
||||||
|
Logger.error("HRRR grib range download crashed: url=#{url} reason=#{inspect(reason)}")
|
||||||
|
|
||||||
|
{:error, reason}
|
||||||
end)
|
end)
|
||||||
|
|
||||||
# Sort by offset and concatenate
|
# Sort by offset and concatenate
|
||||||
|
|
|
||||||
|
|
@ -172,8 +172,15 @@ defmodule Microwaveprop.Workers.GefsFetchWorker do
|
||||||
timeout: 30_000
|
timeout: 30_000
|
||||||
)
|
)
|
||||||
|> Stream.flat_map(fn
|
|> Stream.flat_map(fn
|
||||||
{:ok, results} -> results
|
{:ok, results} ->
|
||||||
{:exit, _reason} -> []
|
results
|
||||||
|
|
||||||
|
{:exit, reason} ->
|
||||||
|
Logger.error(
|
||||||
|
"GefsFetch scoring task crashed: fh=#{fh} valid_time=#{inspect(valid_time)} reason=#{inspect(reason)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
[]
|
||||||
end)
|
end)
|
||||||
|> Enum.to_list()
|
|> Enum.to_list()
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue