From 301935f8c1f2ad48c0b5fb1667727a0bae747926 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 27 Apr 2026 14:22:10 -0500 Subject: [PATCH] fix: log swallowed async task exits in GEFS scoring and HRRR range fetch GefsFetch silently dropped crashed score-computation tasks via `{:exit, _reason} -> []`, hiding partial-grid outages from k8s logs. HRRR range downloads mapped exits to `{:error, reason}` without logging the URL, so the failing range was lost by the time the caller surfaced the error. All three sites now Logger.error with enough context (fh, valid_time, url) to debug from production logs. --- lib/microwaveprop/weather/hrrr_client.ex | 18 ++++++++++++++---- lib/microwaveprop/workers/gefs_fetch_worker.ex | 11 +++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/microwaveprop/weather/hrrr_client.ex b/lib/microwaveprop/weather/hrrr_client.ex index 20277302..0e0190b6 100644 --- a/lib/microwaveprop/weather/hrrr_client.ex +++ b/lib/microwaveprop/weather/hrrr_client.ex @@ -566,8 +566,13 @@ defmodule Microwaveprop.Weather.HrrrClient do timeout: 120_000 ) |> Enum.map(fn - {:ok, result} -> result - {:exit, reason} -> {:error, reason} + {:ok, result} -> + result + + {:exit, reason} -> + Logger.error("HRRR grib range download crashed (to_file): url=#{url} reason=#{inspect(reason)}") + + {:error, reason} end) case Enum.find(results, &match?({:error, _}, &1)) do @@ -612,8 +617,13 @@ defmodule Microwaveprop.Weather.HrrrClient do timeout: 120_000 ) |> Enum.map(fn - {:ok, result} -> result - {:exit, reason} -> {:error, reason} + {:ok, result} -> + result + + {:exit, reason} -> + Logger.error("HRRR grib range download crashed: url=#{url} reason=#{inspect(reason)}") + + {:error, reason} end) # Sort by offset and concatenate diff --git a/lib/microwaveprop/workers/gefs_fetch_worker.ex b/lib/microwaveprop/workers/gefs_fetch_worker.ex index a998426d..0cc13c4b 100644 --- a/lib/microwaveprop/workers/gefs_fetch_worker.ex +++ b/lib/microwaveprop/workers/gefs_fetch_worker.ex @@ -172,8 +172,15 @@ defmodule Microwaveprop.Workers.GefsFetchWorker do timeout: 30_000 ) |> Stream.flat_map(fn - {:ok, results} -> results - {:exit, _reason} -> [] + {:ok, results} -> + results + + {:exit, reason} -> + Logger.error( + "GefsFetch scoring task crashed: fh=#{fh} valid_time=#{inspect(valid_time)} reason=#{inspect(reason)}" + ) + + [] end) |> Enum.to_list()