fix(path): handle ProfilesFile cell shape correctly + log async exits
The /path calculator showed "0 / 9 HRRR points" in production despite
the on-disk profile store being current. Root cause: profile_from_cell/2
treated the cell's :profile key as a wrapper sub-map and called
Map.put_new on it — but the :profile key actually holds the vertical
pressure-level LIST. Every point sample crashed with BadMapError, the
crash propagated as {:exit, _} through Task.async_stream, and the
consumer silently dropped all 9 results.
Fix: stop wrapping. Cells are already flat HrrrProfile-shaped maps;
just stamp lat/lon (from the caller, since cells don't carry their
own coords — those are the map key) and valid_time onto the cell.
Audit + log every other async error path so the next silent failure
isn't invisible:
- PathLive HRRR point lookup
- Propagation.point_forecast per-hour reads
- Viewshed ray crashes
- IemClient ASOS network fetches
- RtmaClient range-download tasks
- Recalibrator factor-vector batches (positive + negative samples)
- MapLive forecast preload tasks
- RoverLive station resolution
LiveViews already had handle_async/3 exit clauses with logging. The
gap was always in Task.async_stream consumers that wrote {:exit, _} -> []
without surfacing the reason.
Add the rule to CLAUDE.md and project memory so this never repeats.
Also fix a pre-existing skewt_svg.ex compiler warning where
@critical_label_min_dy was used before being defined.
This commit is contained in:
parent
ba6c4100f9
commit
418f6426e6
10 changed files with 120 additions and 40 deletions
|
|
@ -190,6 +190,7 @@ User submits QSO → enqueue_for_qso() → weather/hrrr/terrain/iemre workers
|
||||||
- Use `struct.field` not `struct[:field]` (structs don't implement Access)
|
- Use `struct.field` not `struct[:field]` (structs don't implement Access)
|
||||||
- Predicate functions: `thing?` not `is_thing` (reserve `is_` for guards)
|
- Predicate functions: `thing?` not `is_thing` (reserve `is_` for guards)
|
||||||
- Use `Req` for HTTP requests, never httpoison/tesla/httpc
|
- Use `Req` for HTTP requests, never httpoison/tesla/httpc
|
||||||
|
- **Always log anything that would normally be swallowed.** Any place that drops `{:exit, _}` from `Task.async_stream` / `Task.Supervisor.async_stream_nolink`, ignores `{:error, _}` clauses, or otherwise discards a failure path MUST `Logger.error`/`Logger.warning` with enough context (input, key, reason) to debug from k8s logs. Silently dropping a tuple in a `flat_map`/`reduce` clause is a bug — production crashes inside an async task otherwise vanish (the path-calculator "0 / 9 HRRR points" outage was exactly this). LiveView `start_async` must have a matching `handle_async(slot, {:exit, reason}, socket)` clause that logs.
|
||||||
|
|
||||||
### Ecto
|
### Ecto
|
||||||
- All schemas use `@primary_key {:id, :binary_id, autogenerate: true}`
|
- All schemas use `@primary_key {:id, :binary_id, autogenerate: true}`
|
||||||
|
|
|
||||||
|
|
@ -434,9 +434,18 @@ defmodule Microwaveprop.Propagation do
|
||||||
timeout: 5_000
|
timeout: 5_000
|
||||||
)
|
)
|
||||||
|> Enum.flat_map(fn
|
|> Enum.flat_map(fn
|
||||||
{:ok, nil} -> []
|
{:ok, nil} ->
|
||||||
{:ok, entry} -> [entry]
|
[]
|
||||||
{:exit, _} -> []
|
|
||||||
|
{:ok, entry} ->
|
||||||
|
[entry]
|
||||||
|
|
||||||
|
{:exit, reason} ->
|
||||||
|
Logger.error(
|
||||||
|
"Propagation.point_forecast async lookup failed: band_mhz=#{band_mhz} lat=#{snapped_lat} lon=#{snapped_lon} reason=#{inspect(reason)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
[]
|
||||||
end)
|
end)
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -190,9 +190,15 @@ defmodule Microwaveprop.Propagation.Recalibrator do
|
||||||
on_timeout: :kill_task
|
on_timeout: :kill_task
|
||||||
)
|
)
|
||||||
|> Enum.flat_map(fn
|
|> Enum.flat_map(fn
|
||||||
{:ok, nil} -> []
|
{:ok, nil} ->
|
||||||
{:ok, vec} -> [vec]
|
[]
|
||||||
{:exit, _} -> []
|
|
||||||
|
{:ok, vec} ->
|
||||||
|
[vec]
|
||||||
|
|
||||||
|
{:exit, reason} ->
|
||||||
|
Logger.error("Recalibrator contact_to_factors crashed: reason=#{inspect(reason)}")
|
||||||
|
[]
|
||||||
end)
|
end)
|
||||||
|
|
||||||
{factor_vectors, contacts}
|
{factor_vectors, contacts}
|
||||||
|
|
@ -231,9 +237,15 @@ defmodule Microwaveprop.Propagation.Recalibrator do
|
||||||
on_timeout: :kill_task
|
on_timeout: :kill_task
|
||||||
)
|
)
|
||||||
|> Enum.flat_map(fn
|
|> Enum.flat_map(fn
|
||||||
{:ok, nil} -> []
|
{:ok, nil} ->
|
||||||
{:ok, vec} -> [vec]
|
[]
|
||||||
{:exit, _} -> []
|
|
||||||
|
{:ok, vec} ->
|
||||||
|
[vec]
|
||||||
|
|
||||||
|
{:exit, reason} ->
|
||||||
|
Logger.error("Recalibrator negative-sample factor compute crashed: reason=#{inspect(reason)}")
|
||||||
|
[]
|
||||||
end)
|
end)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,8 @@ defmodule Microwaveprop.Terrain.Viewshed do
|
||||||
alias Microwaveprop.Terrain.Srtm
|
alias Microwaveprop.Terrain.Srtm
|
||||||
alias Microwaveprop.Terrain.TerrainAnalysis
|
alias Microwaveprop.Terrain.TerrainAnalysis
|
||||||
|
|
||||||
|
require Logger
|
||||||
|
|
||||||
@type boundary_point :: %{bearing: non_neg_integer(), reach_km: float(), lat: float(), lon: float()}
|
@type boundary_point :: %{bearing: non_neg_integer(), reach_km: float(), lat: float(), lon: float()}
|
||||||
|
|
||||||
@type viewshed_result :: %{
|
@type viewshed_result :: %{
|
||||||
|
|
@ -135,9 +137,17 @@ defmodule Microwaveprop.Terrain.Viewshed do
|
||||||
timeout: 30_000,
|
timeout: 30_000,
|
||||||
on_timeout: :kill_task
|
on_timeout: :kill_task
|
||||||
)
|
)
|
||||||
|
|> Enum.zip(bearings)
|
||||||
|> Enum.map(fn
|
|> Enum.map(fn
|
||||||
{:ok, result} -> result
|
{{:ok, result}, _bearing} ->
|
||||||
{:exit, _} -> nil
|
result
|
||||||
|
|
||||||
|
{{:exit, reason}, bearing} ->
|
||||||
|
Logger.error(
|
||||||
|
"Viewshed ray crash: lat=#{lat} lon=#{lon} bearing=#{bearing} freq=#{freq_ghz} reason=#{inspect(reason)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
nil
|
||||||
end)
|
end)
|
||||||
|> Enum.reject(&is_nil/1)
|
|> Enum.reject(&is_nil/1)
|
||||||
|> smooth_boundary(lat, lon)
|
|> smooth_boundary(lat, lon)
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,8 @@ defmodule Microwaveprop.Weather.IemClient do
|
||||||
|
|
||||||
alias Microwaveprop.Weather.IemRateLimiter
|
alias Microwaveprop.Weather.IemRateLimiter
|
||||||
|
|
||||||
|
require Logger
|
||||||
|
|
||||||
@iem_base "https://mesonet.agron.iastate.edu"
|
@iem_base "https://mesonet.agron.iastate.edu"
|
||||||
|
|
||||||
# --- URL builders ---
|
# --- URL builders ---
|
||||||
|
|
@ -183,9 +185,18 @@ defmodule Microwaveprop.Weather.IemClient do
|
||||||
max_concurrency: 10,
|
max_concurrency: 10,
|
||||||
timeout: 30_000
|
timeout: 30_000
|
||||||
)
|
)
|
||||||
|
|> Enum.zip(state_networks)
|
||||||
|> Enum.flat_map(fn
|
|> Enum.flat_map(fn
|
||||||
{:ok, {:ok, data}} -> data
|
{{:ok, {:ok, data}}, _network} ->
|
||||||
_ -> []
|
data
|
||||||
|
|
||||||
|
{{:ok, {:error, reason}}, network} ->
|
||||||
|
Logger.warning("IemClient.fetch_current_asos #{network}: #{inspect(reason)}")
|
||||||
|
[]
|
||||||
|
|
||||||
|
{{:exit, reason}, network} ->
|
||||||
|
Logger.error("IemClient.fetch_current_asos #{network} crashed: #{inspect(reason)}")
|
||||||
|
[]
|
||||||
end)
|
end)
|
||||||
|
|
||||||
observations =
|
observations =
|
||||||
|
|
|
||||||
|
|
@ -147,7 +147,12 @@ defmodule Microwaveprop.Weather.RtmaClient do
|
||||||
{:ok, {:error, reason}}, _acc ->
|
{:ok, {:error, reason}}, _acc ->
|
||||||
{:error, "RTMA download failed: #{inspect(reason)}"}
|
{:error, "RTMA download failed: #{inspect(reason)}"}
|
||||||
|
|
||||||
_, acc ->
|
{:exit, reason}, _acc ->
|
||||||
|
Logger.error("RtmaClient.download_ranges task crashed: url=#{url} reason=#{inspect(reason)}")
|
||||||
|
{:error, "RTMA download crashed: #{inspect(reason)}"}
|
||||||
|
|
||||||
|
other, acc ->
|
||||||
|
Logger.warning("RtmaClient.download_ranges unexpected stream entry: #{inspect(other)}")
|
||||||
acc
|
acc
|
||||||
end)
|
end)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -430,9 +430,17 @@ defmodule MicrowavepropWeb.MapLive do
|
||||||
timeout: 10_000,
|
timeout: 10_000,
|
||||||
on_timeout: :kill_task
|
on_timeout: :kill_task
|
||||||
)
|
)
|
||||||
|
|> Enum.zip(forecast_times)
|
||||||
|> Enum.flat_map(fn
|
|> Enum.flat_map(fn
|
||||||
{:ok, h} -> [h]
|
{{:ok, h}, _t} ->
|
||||||
{:exit, _} -> []
|
[h]
|
||||||
|
|
||||||
|
{{:exit, reason}, t} ->
|
||||||
|
Logger.error(
|
||||||
|
"MapLive forecast preload task crashed: band=#{band} valid_time=#{inspect(t)} reason=#{inspect(reason)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
[]
|
||||||
end)
|
end)
|
||||||
|
|
||||||
{:noreply,
|
{:noreply,
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,8 @@ defmodule MicrowavepropWeb.PathLive do
|
||||||
alias Microwaveprop.Weather
|
alias Microwaveprop.Weather
|
||||||
alias Microwaveprop.Weather.Station
|
alias Microwaveprop.Weather.Station
|
||||||
|
|
||||||
|
require Logger
|
||||||
|
|
||||||
@band_options BandConfig.band_options()
|
@band_options BandConfig.band_options()
|
||||||
|
|
||||||
@url_params ~w(source destination band src_height_ft dst_height_ft tx_power_dbm src_gain_dbi dst_gain_dbi)
|
@url_params ~w(source destination band src_height_ft dst_height_ft tx_power_dbm src_gain_dbi dst_gain_dbi)
|
||||||
|
|
@ -295,18 +297,29 @@ defmodule MicrowavepropWeb.PathLive do
|
||||||
|
|
||||||
profile_valid_time = latest_profile_valid_time(now)
|
profile_valid_time = latest_profile_valid_time(now)
|
||||||
|
|
||||||
|
sample_points = path_sample_points(src, dst, 9)
|
||||||
|
|
||||||
hrrr_points =
|
hrrr_points =
|
||||||
src
|
sample_points
|
||||||
|> path_sample_points(dst, 9)
|
|
||||||
|> Task.async_stream(&label_hrrr_point(&1, now, profile_valid_time),
|
|> Task.async_stream(&label_hrrr_point(&1, now, profile_valid_time),
|
||||||
max_concurrency: 9,
|
max_concurrency: 9,
|
||||||
timeout: 10_000,
|
timeout: 10_000,
|
||||||
on_timeout: :kill_task
|
on_timeout: :kill_task
|
||||||
)
|
)
|
||||||
|
|> Enum.zip(sample_points)
|
||||||
|> Enum.flat_map(fn
|
|> Enum.flat_map(fn
|
||||||
{:ok, nil} -> []
|
{{:ok, nil}, _} ->
|
||||||
{:ok, point} -> [point]
|
[]
|
||||||
{:exit, _} -> []
|
|
||||||
|
{{:ok, point}, _} ->
|
||||||
|
[point]
|
||||||
|
|
||||||
|
{{:exit, reason}, {label, lat, lon}} ->
|
||||||
|
Logger.error(
|
||||||
|
"PathLive HRRR point lookup failed: label=#{inspect(label)} lat=#{lat} lon=#{lon} valid_time=#{inspect(profile_valid_time)} reason=#{inspect(reason)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
[]
|
||||||
end)
|
end)
|
||||||
|
|
||||||
hrrr_profiles = Enum.map(hrrr_points, & &1.profile)
|
hrrr_profiles = Enum.map(hrrr_points, & &1.profile)
|
||||||
|
|
@ -445,7 +458,7 @@ defmodule MicrowavepropWeb.PathLive do
|
||||||
defp label_hrrr_point({label, lat, lon}, now, profile_valid_time) do
|
defp label_hrrr_point({label, lat, lon}, now, profile_valid_time) do
|
||||||
case profile_valid_time && ProfilesFile.read_point(profile_valid_time, lat, lon) do
|
case profile_valid_time && ProfilesFile.read_point(profile_valid_time, lat, lon) do
|
||||||
%{} = cell ->
|
%{} = cell ->
|
||||||
%{label: label, profile: profile_from_cell(cell, profile_valid_time)}
|
%{label: label, profile: profile_from_cell(cell, lat, lon, profile_valid_time)}
|
||||||
|
|
||||||
_ ->
|
_ ->
|
||||||
case Weather.find_nearest_hrrr(lat, lon, now) do
|
case Weather.find_nearest_hrrr(lat, lon, now) do
|
||||||
|
|
@ -473,16 +486,15 @@ defmodule MicrowavepropWeb.PathLive do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
# ProfilesFile cells nest the per-cell weather under `:profile`; the
|
# ProfilesFile cells are already flat HrrrProfile-shaped maps — the
|
||||||
# template + scoring code expect a flat HrrrProfile-shaped map. Lift
|
# `:profile` key holds the vertical pressure-level list, NOT a wrapper
|
||||||
# the inner fields up and stamp `lat`/`lon`/`valid_time` so downstream
|
# around the surface fields. Stamp `lat`/`lon` (from the sample point,
|
||||||
# code is shape-agnostic.
|
# since cells don't carry their own coordinates — they're the map key)
|
||||||
defp profile_from_cell(cell, valid_time) do
|
# and `valid_time` so downstream template + scoring code is shape-agnostic.
|
||||||
inner = Map.get(cell, :profile, %{}) || %{}
|
defp profile_from_cell(cell, lat, lon, valid_time) do
|
||||||
|
cell
|
||||||
inner
|
|> Map.put_new(:lat, lat)
|
||||||
|> Map.put_new(:lat, Map.get(cell, :lat))
|
|> Map.put_new(:lon, lon)
|
||||||
|> Map.put_new(:lon, Map.get(cell, :lon))
|
|
||||||
|> Map.put_new(:valid_time, valid_time)
|
|> Map.put_new(:valid_time, valid_time)
|
||||||
end
|
end
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -10,6 +10,8 @@ defmodule MicrowavepropWeb.RoverLive do
|
||||||
alias Microwaveprop.Radio.CallsignClient
|
alias Microwaveprop.Radio.CallsignClient
|
||||||
alias Microwaveprop.Radio.Maidenhead
|
alias Microwaveprop.Radio.Maidenhead
|
||||||
|
|
||||||
|
require Logger
|
||||||
|
|
||||||
@band_options BandConfig.band_options()
|
@band_options BandConfig.band_options()
|
||||||
|
|
||||||
@default_band 10_000
|
@default_band 10_000
|
||||||
|
|
@ -116,9 +118,18 @@ defmodule MicrowavepropWeb.RoverLive do
|
||||||
stations =
|
stations =
|
||||||
calls
|
calls
|
||||||
|> Task.async_stream(&resolve_station/1, max_concurrency: 4, timeout: 10_000)
|
|> Task.async_stream(&resolve_station/1, max_concurrency: 4, timeout: 10_000)
|
||||||
|
|> Enum.zip(calls)
|
||||||
|> Enum.flat_map(fn
|
|> Enum.flat_map(fn
|
||||||
{:ok, {:ok, s}} -> [s]
|
{{:ok, {:ok, s}}, _call} ->
|
||||||
_ -> []
|
[s]
|
||||||
|
|
||||||
|
{{:ok, {:error, reason}}, call} ->
|
||||||
|
Logger.warning("RoverLive station resolve failed: call=#{inspect(call)} reason=#{inspect(reason)}")
|
||||||
|
[]
|
||||||
|
|
||||||
|
{{:exit, reason}, call} ->
|
||||||
|
Logger.error("RoverLive station resolve crashed: call=#{inspect(call)} reason=#{inspect(reason)}")
|
||||||
|
[]
|
||||||
end)
|
end)
|
||||||
|
|
||||||
socket =
|
socket =
|
||||||
|
|
|
||||||
|
|
@ -27,6 +27,12 @@ defmodule MicrowavepropWeb.SkewtSvg do
|
||||||
@t_min -40.0
|
@t_min -40.0
|
||||||
@t_max 40.0
|
@t_max 40.0
|
||||||
|
|
||||||
|
# Minimum vertical gap between two stacked critical-level label
|
||||||
|
# baselines (px in SVG user units). 14 leaves the 10 px text plus a
|
||||||
|
# hair of breathing room — anything tighter starts overlapping
|
||||||
|
# descenders.
|
||||||
|
@critical_label_min_dy 14
|
||||||
|
|
||||||
# Skew factor — pixels of horizontal offset added per pixel of
|
# Skew factor — pixels of horizontal offset added per pixel of
|
||||||
# height. 1.0 puts isotherms at 45°.
|
# height. 1.0 puts isotherms at 45°.
|
||||||
@skew 1.0
|
@skew 1.0
|
||||||
|
|
@ -159,7 +165,7 @@ defmodule MicrowavepropWeb.SkewtSvg do
|
||||||
[]
|
[]
|
||||||
end)
|
end)
|
||||||
|> Enum.sort_by(fn {_lbl, _p, y} -> y end)
|
|> Enum.sort_by(fn {_lbl, _p, y} -> y end)
|
||||||
|> Enum.map_reduce(@top, fn {label, p, y}, last_text_y ->
|
|> Enum.map_reduce(@top * 1.0, fn {label, p, y}, last_text_y ->
|
||||||
text_y = max(y, last_text_y + @critical_label_min_dy)
|
text_y = max(y, last_text_y + @critical_label_min_dy)
|
||||||
iodata = render_critical_marker(label, p, y, text_y)
|
iodata = render_critical_marker(label, p, y, text_y)
|
||||||
{iodata, text_y}
|
{iodata, text_y}
|
||||||
|
|
@ -167,11 +173,6 @@ defmodule MicrowavepropWeb.SkewtSvg do
|
||||||
|> elem(0)
|
|> elem(0)
|
||||||
end
|
end
|
||||||
|
|
||||||
# Minimum vertical gap between two stacked label baselines (px in
|
|
||||||
# SVG user units). 14 leaves the 10 px text plus a hair of breathing
|
|
||||||
# room — anything tighter starts overlapping descenders.
|
|
||||||
@critical_label_min_dy 14
|
|
||||||
|
|
||||||
defp render_critical_marker(label, pressure_mb, y, text_y) do
|
defp render_critical_marker(label, pressure_mb, y, text_y) do
|
||||||
leader =
|
leader =
|
||||||
if abs(text_y - y) > 0.5 do
|
if abs(text_y - y) > 0.5 do
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue