fix(path): handle ProfilesFile cell shape correctly + log async exits

The /path calculator showed "0 / 9 HRRR points" in production despite
the on-disk profile store being current. Root cause: profile_from_cell/2
treated the cell's :profile key as a wrapper sub-map and called
Map.put_new on it — but the :profile key actually holds the vertical
pressure-level LIST. Every point sample crashed with BadMapError, the
crash propagated as {:exit, _} through Task.async_stream, and the
consumer silently dropped all 9 results.

Fix: stop wrapping. Cells are already flat HrrrProfile-shaped maps;
just stamp lat/lon (from the caller, since cells don't carry their
own coords — those are the map key) and valid_time onto the cell.

Audit + log every other async error path so the next silent failure
isn't invisible:
- PathLive HRRR point lookup
- Propagation.point_forecast per-hour reads
- Viewshed ray crashes
- IemClient ASOS network fetches
- RtmaClient range-download tasks
- Recalibrator factor-vector batches (positive + negative samples)
- MapLive forecast preload tasks
- RoverLive station resolution

LiveViews already had handle_async/3 exit clauses with logging. The
gap was always in Task.async_stream consumers that wrote {:exit, _} -> []
without surfacing the reason.

Add the rule to CLAUDE.md and project memory so this never repeats.

Also fix a pre-existing skewt_svg.ex compiler warning where
@critical_label_min_dy was used before being defined.
This commit is contained in:
Graham McIntire 2026-04-25 15:56:59 -05:00
parent ba6c4100f9
commit 418f6426e6
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
10 changed files with 120 additions and 40 deletions

View file

@ -190,6 +190,7 @@ User submits QSO → enqueue_for_qso() → weather/hrrr/terrain/iemre workers
- Use `struct.field` not `struct[:field]` (structs don't implement Access) - Use `struct.field` not `struct[:field]` (structs don't implement Access)
- Predicate functions: `thing?` not `is_thing` (reserve `is_` for guards) - Predicate functions: `thing?` not `is_thing` (reserve `is_` for guards)
- Use `Req` for HTTP requests, never httpoison/tesla/httpc - Use `Req` for HTTP requests, never httpoison/tesla/httpc
- **Always log anything that would normally be swallowed.** Any place that drops `{:exit, _}` from `Task.async_stream` / `Task.Supervisor.async_stream_nolink`, ignores `{:error, _}` clauses, or otherwise discards a failure path MUST `Logger.error`/`Logger.warning` with enough context (input, key, reason) to debug from k8s logs. Silently dropping a tuple in a `flat_map`/`reduce` clause is a bug — production crashes inside an async task otherwise vanish (the path-calculator "0 / 9 HRRR points" outage was exactly this). LiveView `start_async` must have a matching `handle_async(slot, {:exit, reason}, socket)` clause that logs.
### Ecto ### Ecto
- All schemas use `@primary_key {:id, :binary_id, autogenerate: true}` - All schemas use `@primary_key {:id, :binary_id, autogenerate: true}`

View file

@ -434,9 +434,18 @@ defmodule Microwaveprop.Propagation do
timeout: 5_000 timeout: 5_000
) )
|> Enum.flat_map(fn |> Enum.flat_map(fn
{:ok, nil} -> [] {:ok, nil} ->
{:ok, entry} -> [entry] []
{:exit, _} -> []
{:ok, entry} ->
[entry]
{:exit, reason} ->
Logger.error(
"Propagation.point_forecast async lookup failed: band_mhz=#{band_mhz} lat=#{snapped_lat} lon=#{snapped_lon} reason=#{inspect(reason)}"
)
[]
end) end)
end) end)
end end

View file

@ -190,9 +190,15 @@ defmodule Microwaveprop.Propagation.Recalibrator do
on_timeout: :kill_task on_timeout: :kill_task
) )
|> Enum.flat_map(fn |> Enum.flat_map(fn
{:ok, nil} -> [] {:ok, nil} ->
{:ok, vec} -> [vec] []
{:exit, _} -> []
{:ok, vec} ->
[vec]
{:exit, reason} ->
Logger.error("Recalibrator contact_to_factors crashed: reason=#{inspect(reason)}")
[]
end) end)
{factor_vectors, contacts} {factor_vectors, contacts}
@ -231,9 +237,15 @@ defmodule Microwaveprop.Propagation.Recalibrator do
on_timeout: :kill_task on_timeout: :kill_task
) )
|> Enum.flat_map(fn |> Enum.flat_map(fn
{:ok, nil} -> [] {:ok, nil} ->
{:ok, vec} -> [vec] []
{:exit, _} -> []
{:ok, vec} ->
[vec]
{:exit, reason} ->
Logger.error("Recalibrator negative-sample factor compute crashed: reason=#{inspect(reason)}")
[]
end) end)
end end

View file

@ -4,6 +4,8 @@ defmodule Microwaveprop.Terrain.Viewshed do
alias Microwaveprop.Terrain.Srtm alias Microwaveprop.Terrain.Srtm
alias Microwaveprop.Terrain.TerrainAnalysis alias Microwaveprop.Terrain.TerrainAnalysis
require Logger
@type boundary_point :: %{bearing: non_neg_integer(), reach_km: float(), lat: float(), lon: float()} @type boundary_point :: %{bearing: non_neg_integer(), reach_km: float(), lat: float(), lon: float()}
@type viewshed_result :: %{ @type viewshed_result :: %{
@ -135,9 +137,17 @@ defmodule Microwaveprop.Terrain.Viewshed do
timeout: 30_000, timeout: 30_000,
on_timeout: :kill_task on_timeout: :kill_task
) )
|> Enum.zip(bearings)
|> Enum.map(fn |> Enum.map(fn
{:ok, result} -> result {{:ok, result}, _bearing} ->
{:exit, _} -> nil result
{{:exit, reason}, bearing} ->
Logger.error(
"Viewshed ray crash: lat=#{lat} lon=#{lon} bearing=#{bearing} freq=#{freq_ghz} reason=#{inspect(reason)}"
)
nil
end) end)
|> Enum.reject(&is_nil/1) |> Enum.reject(&is_nil/1)
|> smooth_boundary(lat, lon) |> smooth_boundary(lat, lon)

View file

@ -3,6 +3,8 @@ defmodule Microwaveprop.Weather.IemClient do
alias Microwaveprop.Weather.IemRateLimiter alias Microwaveprop.Weather.IemRateLimiter
require Logger
@iem_base "https://mesonet.agron.iastate.edu" @iem_base "https://mesonet.agron.iastate.edu"
# --- URL builders --- # --- URL builders ---
@ -183,9 +185,18 @@ defmodule Microwaveprop.Weather.IemClient do
max_concurrency: 10, max_concurrency: 10,
timeout: 30_000 timeout: 30_000
) )
|> Enum.zip(state_networks)
|> Enum.flat_map(fn |> Enum.flat_map(fn
{:ok, {:ok, data}} -> data {{:ok, {:ok, data}}, _network} ->
_ -> [] data
{{:ok, {:error, reason}}, network} ->
Logger.warning("IemClient.fetch_current_asos #{network}: #{inspect(reason)}")
[]
{{:exit, reason}, network} ->
Logger.error("IemClient.fetch_current_asos #{network} crashed: #{inspect(reason)}")
[]
end) end)
observations = observations =

View file

@ -147,7 +147,12 @@ defmodule Microwaveprop.Weather.RtmaClient do
{:ok, {:error, reason}}, _acc -> {:ok, {:error, reason}}, _acc ->
{:error, "RTMA download failed: #{inspect(reason)}"} {:error, "RTMA download failed: #{inspect(reason)}"}
_, acc -> {:exit, reason}, _acc ->
Logger.error("RtmaClient.download_ranges task crashed: url=#{url} reason=#{inspect(reason)}")
{:error, "RTMA download crashed: #{inspect(reason)}"}
other, acc ->
Logger.warning("RtmaClient.download_ranges unexpected stream entry: #{inspect(other)}")
acc acc
end) end)

View file

@ -430,9 +430,17 @@ defmodule MicrowavepropWeb.MapLive do
timeout: 10_000, timeout: 10_000,
on_timeout: :kill_task on_timeout: :kill_task
) )
|> Enum.zip(forecast_times)
|> Enum.flat_map(fn |> Enum.flat_map(fn
{:ok, h} -> [h] {{:ok, h}, _t} ->
{:exit, _} -> [] [h]
{{:exit, reason}, t} ->
Logger.error(
"MapLive forecast preload task crashed: band=#{band} valid_time=#{inspect(t)} reason=#{inspect(reason)}"
)
[]
end) end)
{:noreply, {:noreply,

View file

@ -18,6 +18,8 @@ defmodule MicrowavepropWeb.PathLive do
alias Microwaveprop.Weather alias Microwaveprop.Weather
alias Microwaveprop.Weather.Station alias Microwaveprop.Weather.Station
require Logger
@band_options BandConfig.band_options() @band_options BandConfig.band_options()
@url_params ~w(source destination band src_height_ft dst_height_ft tx_power_dbm src_gain_dbi dst_gain_dbi) @url_params ~w(source destination band src_height_ft dst_height_ft tx_power_dbm src_gain_dbi dst_gain_dbi)
@ -295,18 +297,29 @@ defmodule MicrowavepropWeb.PathLive do
profile_valid_time = latest_profile_valid_time(now) profile_valid_time = latest_profile_valid_time(now)
sample_points = path_sample_points(src, dst, 9)
hrrr_points = hrrr_points =
src sample_points
|> path_sample_points(dst, 9)
|> Task.async_stream(&label_hrrr_point(&1, now, profile_valid_time), |> Task.async_stream(&label_hrrr_point(&1, now, profile_valid_time),
max_concurrency: 9, max_concurrency: 9,
timeout: 10_000, timeout: 10_000,
on_timeout: :kill_task on_timeout: :kill_task
) )
|> Enum.zip(sample_points)
|> Enum.flat_map(fn |> Enum.flat_map(fn
{:ok, nil} -> [] {{:ok, nil}, _} ->
{:ok, point} -> [point] []
{:exit, _} -> []
{{:ok, point}, _} ->
[point]
{{:exit, reason}, {label, lat, lon}} ->
Logger.error(
"PathLive HRRR point lookup failed: label=#{inspect(label)} lat=#{lat} lon=#{lon} valid_time=#{inspect(profile_valid_time)} reason=#{inspect(reason)}"
)
[]
end) end)
hrrr_profiles = Enum.map(hrrr_points, & &1.profile) hrrr_profiles = Enum.map(hrrr_points, & &1.profile)
@ -445,7 +458,7 @@ defmodule MicrowavepropWeb.PathLive do
defp label_hrrr_point({label, lat, lon}, now, profile_valid_time) do defp label_hrrr_point({label, lat, lon}, now, profile_valid_time) do
case profile_valid_time && ProfilesFile.read_point(profile_valid_time, lat, lon) do case profile_valid_time && ProfilesFile.read_point(profile_valid_time, lat, lon) do
%{} = cell -> %{} = cell ->
%{label: label, profile: profile_from_cell(cell, profile_valid_time)} %{label: label, profile: profile_from_cell(cell, lat, lon, profile_valid_time)}
_ -> _ ->
case Weather.find_nearest_hrrr(lat, lon, now) do case Weather.find_nearest_hrrr(lat, lon, now) do
@ -473,16 +486,15 @@ defmodule MicrowavepropWeb.PathLive do
end end
end end
# ProfilesFile cells nest the per-cell weather under `:profile`; the # ProfilesFile cells are already flat HrrrProfile-shaped maps — the
# template + scoring code expect a flat HrrrProfile-shaped map. Lift # `:profile` key holds the vertical pressure-level list, NOT a wrapper
# the inner fields up and stamp `lat`/`lon`/`valid_time` so downstream # around the surface fields. Stamp `lat`/`lon` (from the sample point,
# code is shape-agnostic. # since cells don't carry their own coordinates — they're the map key)
defp profile_from_cell(cell, valid_time) do # and `valid_time` so downstream template + scoring code is shape-agnostic.
inner = Map.get(cell, :profile, %{}) || %{} defp profile_from_cell(cell, lat, lon, valid_time) do
cell
inner |> Map.put_new(:lat, lat)
|> Map.put_new(:lat, Map.get(cell, :lat)) |> Map.put_new(:lon, lon)
|> Map.put_new(:lon, Map.get(cell, :lon))
|> Map.put_new(:valid_time, valid_time) |> Map.put_new(:valid_time, valid_time)
end end

View file

@ -10,6 +10,8 @@ defmodule MicrowavepropWeb.RoverLive do
alias Microwaveprop.Radio.CallsignClient alias Microwaveprop.Radio.CallsignClient
alias Microwaveprop.Radio.Maidenhead alias Microwaveprop.Radio.Maidenhead
require Logger
@band_options BandConfig.band_options() @band_options BandConfig.band_options()
@default_band 10_000 @default_band 10_000
@ -116,9 +118,18 @@ defmodule MicrowavepropWeb.RoverLive do
stations = stations =
calls calls
|> Task.async_stream(&resolve_station/1, max_concurrency: 4, timeout: 10_000) |> Task.async_stream(&resolve_station/1, max_concurrency: 4, timeout: 10_000)
|> Enum.zip(calls)
|> Enum.flat_map(fn |> Enum.flat_map(fn
{:ok, {:ok, s}} -> [s] {{:ok, {:ok, s}}, _call} ->
_ -> [] [s]
{{:ok, {:error, reason}}, call} ->
Logger.warning("RoverLive station resolve failed: call=#{inspect(call)} reason=#{inspect(reason)}")
[]
{{:exit, reason}, call} ->
Logger.error("RoverLive station resolve crashed: call=#{inspect(call)} reason=#{inspect(reason)}")
[]
end) end)
socket = socket =

View file

@ -27,6 +27,12 @@ defmodule MicrowavepropWeb.SkewtSvg do
@t_min -40.0 @t_min -40.0
@t_max 40.0 @t_max 40.0
# Minimum vertical gap between two stacked critical-level label
# baselines (px in SVG user units). 14 leaves the 10 px text plus a
# hair of breathing room — anything tighter starts overlapping
# descenders.
@critical_label_min_dy 14
# Skew factor — pixels of horizontal offset added per pixel of # Skew factor — pixels of horizontal offset added per pixel of
# height. 1.0 puts isotherms at 45°. # height. 1.0 puts isotherms at 45°.
@skew 1.0 @skew 1.0
@ -159,7 +165,7 @@ defmodule MicrowavepropWeb.SkewtSvg do
[] []
end) end)
|> Enum.sort_by(fn {_lbl, _p, y} -> y end) |> Enum.sort_by(fn {_lbl, _p, y} -> y end)
|> Enum.map_reduce(@top, fn {label, p, y}, last_text_y -> |> Enum.map_reduce(@top * 1.0, fn {label, p, y}, last_text_y ->
text_y = max(y, last_text_y + @critical_label_min_dy) text_y = max(y, last_text_y + @critical_label_min_dy)
iodata = render_critical_marker(label, p, y, text_y) iodata = render_critical_marker(label, p, y, text_y)
{iodata, text_y} {iodata, text_y}
@ -167,11 +173,6 @@ defmodule MicrowavepropWeb.SkewtSvg do
|> elem(0) |> elem(0)
end end
# Minimum vertical gap between two stacked label baselines (px in
# SVG user units). 14 leaves the 10 px text plus a hair of breathing
# room — anything tighter starts overlapping descenders.
@critical_label_min_dy 14
defp render_critical_marker(label, pressure_mb, y, text_y) do defp render_critical_marker(label, pressure_mb, y, text_y) do
leader = leader =
if abs(text_y - y) > 0.5 do if abs(text_y - y) > 0.5 do