From 418f6426e6d894d1e739015db7322b2c8ad4da78 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sat, 25 Apr 2026 15:56:59 -0500 Subject: [PATCH] fix(path): handle ProfilesFile cell shape correctly + log async exits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /path calculator showed "0 / 9 HRRR points" in production despite the on-disk profile store being current. Root cause: profile_from_cell/2 treated the cell's :profile key as a wrapper sub-map and called Map.put_new on it — but the :profile key actually holds the vertical pressure-level LIST. Every point sample crashed with BadMapError, the crash propagated as {:exit, _} through Task.async_stream, and the consumer silently dropped all 9 results. Fix: stop wrapping. Cells are already flat HrrrProfile-shaped maps; just stamp lat/lon (from the caller, since cells don't carry their own coords — those are the map key) and valid_time onto the cell. Audit + log every other async error path so the next silent failure isn't invisible: - PathLive HRRR point lookup - Propagation.point_forecast per-hour reads - Viewshed ray crashes - IemClient ASOS network fetches - RtmaClient range-download tasks - Recalibrator factor-vector batches (positive + negative samples) - MapLive forecast preload tasks - RoverLive station resolution LiveViews already had handle_async/3 exit clauses with logging. The gap was always in Task.async_stream consumers that wrote {:exit, _} -> [] without surfacing the reason. Add the rule to CLAUDE.md and project memory so this never repeats. Also fix a pre-existing skewt_svg.ex compiler warning where @critical_label_min_dy was used before being defined. --- CLAUDE.md | 1 + lib/microwaveprop/propagation.ex | 15 +++++-- lib/microwaveprop/propagation/recalibrator.ex | 24 +++++++--- lib/microwaveprop/terrain/viewshed.ex | 14 +++++- lib/microwaveprop/weather/iem_client.ex | 15 ++++++- lib/microwaveprop/weather/rtma_client.ex | 7 ++- lib/microwaveprop_web/live/map_live.ex | 12 ++++- lib/microwaveprop_web/live/path_live.ex | 44 ++++++++++++------- lib/microwaveprop_web/live/rover_live.ex | 15 ++++++- lib/microwaveprop_web/live/skewt_svg.ex | 13 +++--- 10 files changed, 120 insertions(+), 40 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1ef99346..9503fee5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -190,6 +190,7 @@ User submits QSO → enqueue_for_qso() → weather/hrrr/terrain/iemre workers - Use `struct.field` not `struct[:field]` (structs don't implement Access) - Predicate functions: `thing?` not `is_thing` (reserve `is_` for guards) - Use `Req` for HTTP requests, never httpoison/tesla/httpc +- **Always log anything that would normally be swallowed.** Any place that drops `{:exit, _}` from `Task.async_stream` / `Task.Supervisor.async_stream_nolink`, ignores `{:error, _}` clauses, or otherwise discards a failure path MUST `Logger.error`/`Logger.warning` with enough context (input, key, reason) to debug from k8s logs. Silently dropping a tuple in a `flat_map`/`reduce` clause is a bug — production crashes inside an async task otherwise vanish (the path-calculator "0 / 9 HRRR points" outage was exactly this). LiveView `start_async` must have a matching `handle_async(slot, {:exit, reason}, socket)` clause that logs. ### Ecto - All schemas use `@primary_key {:id, :binary_id, autogenerate: true}` diff --git a/lib/microwaveprop/propagation.ex b/lib/microwaveprop/propagation.ex index 7d628cab..32710eae 100644 --- a/lib/microwaveprop/propagation.ex +++ b/lib/microwaveprop/propagation.ex @@ -434,9 +434,18 @@ defmodule Microwaveprop.Propagation do timeout: 5_000 ) |> Enum.flat_map(fn - {:ok, nil} -> [] - {:ok, entry} -> [entry] - {:exit, _} -> [] + {:ok, nil} -> + [] + + {:ok, entry} -> + [entry] + + {:exit, reason} -> + Logger.error( + "Propagation.point_forecast async lookup failed: band_mhz=#{band_mhz} lat=#{snapped_lat} lon=#{snapped_lon} reason=#{inspect(reason)}" + ) + + [] end) end) end diff --git a/lib/microwaveprop/propagation/recalibrator.ex b/lib/microwaveprop/propagation/recalibrator.ex index 036557f6..c7705892 100644 --- a/lib/microwaveprop/propagation/recalibrator.ex +++ b/lib/microwaveprop/propagation/recalibrator.ex @@ -190,9 +190,15 @@ defmodule Microwaveprop.Propagation.Recalibrator do on_timeout: :kill_task ) |> Enum.flat_map(fn - {:ok, nil} -> [] - {:ok, vec} -> [vec] - {:exit, _} -> [] + {:ok, nil} -> + [] + + {:ok, vec} -> + [vec] + + {:exit, reason} -> + Logger.error("Recalibrator contact_to_factors crashed: reason=#{inspect(reason)}") + [] end) {factor_vectors, contacts} @@ -231,9 +237,15 @@ defmodule Microwaveprop.Propagation.Recalibrator do on_timeout: :kill_task ) |> Enum.flat_map(fn - {:ok, nil} -> [] - {:ok, vec} -> [vec] - {:exit, _} -> [] + {:ok, nil} -> + [] + + {:ok, vec} -> + [vec] + + {:exit, reason} -> + Logger.error("Recalibrator negative-sample factor compute crashed: reason=#{inspect(reason)}") + [] end) end diff --git a/lib/microwaveprop/terrain/viewshed.ex b/lib/microwaveprop/terrain/viewshed.ex index 492342d0..86f573fe 100644 --- a/lib/microwaveprop/terrain/viewshed.ex +++ b/lib/microwaveprop/terrain/viewshed.ex @@ -4,6 +4,8 @@ defmodule Microwaveprop.Terrain.Viewshed do alias Microwaveprop.Terrain.Srtm alias Microwaveprop.Terrain.TerrainAnalysis + require Logger + @type boundary_point :: %{bearing: non_neg_integer(), reach_km: float(), lat: float(), lon: float()} @type viewshed_result :: %{ @@ -135,9 +137,17 @@ defmodule Microwaveprop.Terrain.Viewshed do timeout: 30_000, on_timeout: :kill_task ) + |> Enum.zip(bearings) |> Enum.map(fn - {:ok, result} -> result - {:exit, _} -> nil + {{:ok, result}, _bearing} -> + result + + {{:exit, reason}, bearing} -> + Logger.error( + "Viewshed ray crash: lat=#{lat} lon=#{lon} bearing=#{bearing} freq=#{freq_ghz} reason=#{inspect(reason)}" + ) + + nil end) |> Enum.reject(&is_nil/1) |> smooth_boundary(lat, lon) diff --git a/lib/microwaveprop/weather/iem_client.ex b/lib/microwaveprop/weather/iem_client.ex index 44846de7..2e439016 100644 --- a/lib/microwaveprop/weather/iem_client.ex +++ b/lib/microwaveprop/weather/iem_client.ex @@ -3,6 +3,8 @@ defmodule Microwaveprop.Weather.IemClient do alias Microwaveprop.Weather.IemRateLimiter + require Logger + @iem_base "https://mesonet.agron.iastate.edu" # --- URL builders --- @@ -183,9 +185,18 @@ defmodule Microwaveprop.Weather.IemClient do max_concurrency: 10, timeout: 30_000 ) + |> Enum.zip(state_networks) |> Enum.flat_map(fn - {:ok, {:ok, data}} -> data - _ -> [] + {{:ok, {:ok, data}}, _network} -> + data + + {{:ok, {:error, reason}}, network} -> + Logger.warning("IemClient.fetch_current_asos #{network}: #{inspect(reason)}") + [] + + {{:exit, reason}, network} -> + Logger.error("IemClient.fetch_current_asos #{network} crashed: #{inspect(reason)}") + [] end) observations = diff --git a/lib/microwaveprop/weather/rtma_client.ex b/lib/microwaveprop/weather/rtma_client.ex index 495c2af2..6070c71b 100644 --- a/lib/microwaveprop/weather/rtma_client.ex +++ b/lib/microwaveprop/weather/rtma_client.ex @@ -147,7 +147,12 @@ defmodule Microwaveprop.Weather.RtmaClient do {:ok, {:error, reason}}, _acc -> {:error, "RTMA download failed: #{inspect(reason)}"} - _, acc -> + {:exit, reason}, _acc -> + Logger.error("RtmaClient.download_ranges task crashed: url=#{url} reason=#{inspect(reason)}") + {:error, "RTMA download crashed: #{inspect(reason)}"} + + other, acc -> + Logger.warning("RtmaClient.download_ranges unexpected stream entry: #{inspect(other)}") acc end) diff --git a/lib/microwaveprop_web/live/map_live.ex b/lib/microwaveprop_web/live/map_live.ex index 278a7196..579bbd49 100644 --- a/lib/microwaveprop_web/live/map_live.ex +++ b/lib/microwaveprop_web/live/map_live.ex @@ -430,9 +430,17 @@ defmodule MicrowavepropWeb.MapLive do timeout: 10_000, on_timeout: :kill_task ) + |> Enum.zip(forecast_times) |> Enum.flat_map(fn - {:ok, h} -> [h] - {:exit, _} -> [] + {{:ok, h}, _t} -> + [h] + + {{:exit, reason}, t} -> + Logger.error( + "MapLive forecast preload task crashed: band=#{band} valid_time=#{inspect(t)} reason=#{inspect(reason)}" + ) + + [] end) {:noreply, diff --git a/lib/microwaveprop_web/live/path_live.ex b/lib/microwaveprop_web/live/path_live.ex index 9d442a0b..c01666ee 100644 --- a/lib/microwaveprop_web/live/path_live.ex +++ b/lib/microwaveprop_web/live/path_live.ex @@ -18,6 +18,8 @@ defmodule MicrowavepropWeb.PathLive do alias Microwaveprop.Weather alias Microwaveprop.Weather.Station + require Logger + @band_options BandConfig.band_options() @url_params ~w(source destination band src_height_ft dst_height_ft tx_power_dbm src_gain_dbi dst_gain_dbi) @@ -295,18 +297,29 @@ defmodule MicrowavepropWeb.PathLive do profile_valid_time = latest_profile_valid_time(now) + sample_points = path_sample_points(src, dst, 9) + hrrr_points = - src - |> path_sample_points(dst, 9) + sample_points |> Task.async_stream(&label_hrrr_point(&1, now, profile_valid_time), max_concurrency: 9, timeout: 10_000, on_timeout: :kill_task ) + |> Enum.zip(sample_points) |> Enum.flat_map(fn - {:ok, nil} -> [] - {:ok, point} -> [point] - {:exit, _} -> [] + {{:ok, nil}, _} -> + [] + + {{:ok, point}, _} -> + [point] + + {{:exit, reason}, {label, lat, lon}} -> + Logger.error( + "PathLive HRRR point lookup failed: label=#{inspect(label)} lat=#{lat} lon=#{lon} valid_time=#{inspect(profile_valid_time)} reason=#{inspect(reason)}" + ) + + [] end) hrrr_profiles = Enum.map(hrrr_points, & &1.profile) @@ -445,7 +458,7 @@ defmodule MicrowavepropWeb.PathLive do defp label_hrrr_point({label, lat, lon}, now, profile_valid_time) do case profile_valid_time && ProfilesFile.read_point(profile_valid_time, lat, lon) do %{} = cell -> - %{label: label, profile: profile_from_cell(cell, profile_valid_time)} + %{label: label, profile: profile_from_cell(cell, lat, lon, profile_valid_time)} _ -> case Weather.find_nearest_hrrr(lat, lon, now) do @@ -473,16 +486,15 @@ defmodule MicrowavepropWeb.PathLive do end end - # ProfilesFile cells nest the per-cell weather under `:profile`; the - # template + scoring code expect a flat HrrrProfile-shaped map. Lift - # the inner fields up and stamp `lat`/`lon`/`valid_time` so downstream - # code is shape-agnostic. - defp profile_from_cell(cell, valid_time) do - inner = Map.get(cell, :profile, %{}) || %{} - - inner - |> Map.put_new(:lat, Map.get(cell, :lat)) - |> Map.put_new(:lon, Map.get(cell, :lon)) + # ProfilesFile cells are already flat HrrrProfile-shaped maps — the + # `:profile` key holds the vertical pressure-level list, NOT a wrapper + # around the surface fields. Stamp `lat`/`lon` (from the sample point, + # since cells don't carry their own coordinates — they're the map key) + # and `valid_time` so downstream template + scoring code is shape-agnostic. + defp profile_from_cell(cell, lat, lon, valid_time) do + cell + |> Map.put_new(:lat, lat) + |> Map.put_new(:lon, lon) |> Map.put_new(:valid_time, valid_time) end diff --git a/lib/microwaveprop_web/live/rover_live.ex b/lib/microwaveprop_web/live/rover_live.ex index 1744007e..d957a255 100644 --- a/lib/microwaveprop_web/live/rover_live.ex +++ b/lib/microwaveprop_web/live/rover_live.ex @@ -10,6 +10,8 @@ defmodule MicrowavepropWeb.RoverLive do alias Microwaveprop.Radio.CallsignClient alias Microwaveprop.Radio.Maidenhead + require Logger + @band_options BandConfig.band_options() @default_band 10_000 @@ -116,9 +118,18 @@ defmodule MicrowavepropWeb.RoverLive do stations = calls |> Task.async_stream(&resolve_station/1, max_concurrency: 4, timeout: 10_000) + |> Enum.zip(calls) |> Enum.flat_map(fn - {:ok, {:ok, s}} -> [s] - _ -> [] + {{:ok, {:ok, s}}, _call} -> + [s] + + {{:ok, {:error, reason}}, call} -> + Logger.warning("RoverLive station resolve failed: call=#{inspect(call)} reason=#{inspect(reason)}") + [] + + {{:exit, reason}, call} -> + Logger.error("RoverLive station resolve crashed: call=#{inspect(call)} reason=#{inspect(reason)}") + [] end) socket = diff --git a/lib/microwaveprop_web/live/skewt_svg.ex b/lib/microwaveprop_web/live/skewt_svg.ex index 8aca5362..a8e61beb 100644 --- a/lib/microwaveprop_web/live/skewt_svg.ex +++ b/lib/microwaveprop_web/live/skewt_svg.ex @@ -27,6 +27,12 @@ defmodule MicrowavepropWeb.SkewtSvg do @t_min -40.0 @t_max 40.0 + # Minimum vertical gap between two stacked critical-level label + # baselines (px in SVG user units). 14 leaves the 10 px text plus a + # hair of breathing room — anything tighter starts overlapping + # descenders. + @critical_label_min_dy 14 + # Skew factor — pixels of horizontal offset added per pixel of # height. 1.0 puts isotherms at 45°. @skew 1.0 @@ -159,7 +165,7 @@ defmodule MicrowavepropWeb.SkewtSvg do [] end) |> Enum.sort_by(fn {_lbl, _p, y} -> y end) - |> Enum.map_reduce(@top, fn {label, p, y}, last_text_y -> + |> Enum.map_reduce(@top * 1.0, fn {label, p, y}, last_text_y -> text_y = max(y, last_text_y + @critical_label_min_dy) iodata = render_critical_marker(label, p, y, text_y) {iodata, text_y} @@ -167,11 +173,6 @@ defmodule MicrowavepropWeb.SkewtSvg do |> elem(0) end - # Minimum vertical gap between two stacked label baselines (px in - # SVG user units). 14 leaves the 10 px text plus a hair of breathing - # room — anything tighter starts overlapping descenders. - @critical_label_min_dy 14 - defp render_critical_marker(label, pressure_mb, y, text_y) do leader = if abs(text_y - y) > 0.5 do