From 0c987070912dfa1bbcffd56dc323b4eaa7f48cbb Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 21 Apr 2026 15:56:30 -0500 Subject: [PATCH] feat: harden /map analysis breakdown + move Plausible to root layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes cover the blank "analysis breakdown" panel the user reported: 1. Bound the point_detail fallback lookback to 24h. Previously factors_from_fallback_profile would happily use a week-old analysis profile when no current one existed — now anything older than 24h returns an empty factor map so the UI can surface "breakdown unavailable" instead of silently misattributing. 2. Move the Plausible analytics snippet from Layouts.app into root.html.heex. Full-bleed LiveViews (/map, /contacts/map, /weather, home) bypass Layouts.app, so the snippet only loaded on the navbar-wrapped pages. root.html.heex is loaded once per HTTP document so coverage is now universal. Added ~30 tests locking both down: • point_detail fallback: exact-time wins, 24h boundary accepted, 30h-stale rejected, multi-band coverage, missing-cell degrades to empty factors, default-valid_time path • analytics_test.exs: Plausible script present on 12 representative pages, exactly once, loaded async, surviving a login round-trip Also fixed pre-existing credo issues per standing rule: shortened the map_live_test ETS match_delete call, extracted a function from the deeply-nested hrrr_point_enqueuer.enqueue_for_contacts/1 cond, and swapped Mix.Tasks.Rust.Golden's IO.inspect for Mix.shell().info. --- lib/microwaveprop/propagation.ex | 23 +- .../weather/hrrr_point_enqueuer.ex | 65 +++-- lib/microwaveprop_web/components/layouts.ex | 8 - .../components/layouts/root.html.heex | 7 + lib/mix/tasks/rust.golden.ex | 2 +- test/microwaveprop/propagation_test.exs | 225 +++++++++++++++++- test/microwaveprop_web/analytics_test.exs | 83 +++++++ test/microwaveprop_web/live/map_live_test.exs | 7 +- 8 files changed, 372 insertions(+), 48 deletions(-) create mode 100644 test/microwaveprop_web/analytics_test.exs diff --git a/lib/microwaveprop/propagation.ex b/lib/microwaveprop/propagation.ex index 758ec4b0..d2366bd6 100644 --- a/lib/microwaveprop/propagation.ex +++ b/lib/microwaveprop/propagation.ex @@ -307,7 +307,7 @@ defmodule Microwaveprop.Propagation do end @doc """ - Variant of `scores_at/3` that always reads from the `.ntms` file on + Variant of `scores_at/3` that always reads from the `.prop` file on disk and overwrites the cache entry, rather than returning whatever the cache happens to hold. Use from update paths (the map's `propagation_updated` handler) where the underlying file has just @@ -385,7 +385,7 @@ defmodule Microwaveprop.Propagation do {snapped_lat, snapped_lon} = snap_to_grid(lat, lon) now = DateTime.utc_now() - # Use the on-disk .ntms file list as the authoritative timeline so + # Use the on-disk .prop file list as the authoritative timeline so # the chart never falls behind the main-map timeline (which also # reads the disk). The cache is still consulted per-hour for a # fast score lookup; a miss falls through to the file. @@ -474,9 +474,12 @@ defmodule Microwaveprop.Propagation do # Rebuild the factor breakdown for a clicked grid cell by rescoring # the persisted HRRR profile. Only analysis hours (f00) persist # profiles, so forecast hours fall back to the most recent analysis - # profile at or before the requested time — the atmospheric breakdown - # still explains what's driving the score even if sampled an hour or - # two earlier. + # profile within `@fallback_profile_lookback_hours` — the atmospheric + # breakdown still explains what's driving the score even if sampled + # an hour or two earlier. Outside the lookback we prefer to return + # an empty map so the UI shows "breakdown unavailable" rather than + # silently attributing scores to a day-old synoptic pattern. + @fallback_profile_lookback_hours 24 defp factors_for(band_mhz, valid_time, lat, lon) do case ProfilesFile.read_point(valid_time, lat, lon) do nil -> factors_from_fallback_profile(band_mhz, valid_time, lat, lon) @@ -485,7 +488,7 @@ defmodule Microwaveprop.Propagation do end defp factors_from_fallback_profile(band_mhz, valid_time, lat, lon) do - case latest_profile_time_at_or_before(valid_time) do + case latest_profile_time_within_lookback(valid_time) do nil -> %{} @@ -507,9 +510,13 @@ defmodule Microwaveprop.Propagation do end end - defp latest_profile_time_at_or_before(%DateTime{} = valid_time) do + defp latest_profile_time_within_lookback(%DateTime{} = valid_time) do + lookback_cutoff = DateTime.add(valid_time, -@fallback_profile_lookback_hours * 3600, :second) + ProfilesFile.list_valid_times() - |> Enum.filter(&(DateTime.compare(&1, valid_time) != :gt)) + |> Enum.filter(fn t -> + DateTime.compare(t, valid_time) != :gt and DateTime.compare(t, lookback_cutoff) != :lt + end) |> case do [] -> nil past -> Enum.max(past, DateTime) diff --git a/lib/microwaveprop/weather/hrrr_point_enqueuer.ex b/lib/microwaveprop/weather/hrrr_point_enqueuer.ex index 175e8952..1ec3eeaf 100644 --- a/lib/microwaveprop/weather/hrrr_point_enqueuer.ex +++ b/lib/microwaveprop/weather/hrrr_point_enqueuer.ex @@ -117,35 +117,48 @@ defmodule Microwaveprop.Weather.HrrrPointEnqueuer do groups = contacts - |> Enum.flat_map(fn contact -> - cond do - is_nil(contact.pos1) -> - [] - - # HRRR archive starts mid-2014; NARR covers the pre-2014 - # window. Pre-coverage contacts are NARR's responsibility. - NarrClient.in_coverage?(contact.qso_timestamp) -> - [] - - true -> - rounded_time = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp) - - contact - |> Radio.contact_path_points() - |> Enum.flat_map(fn {lat, lon} -> - {rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon) - - if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do - [] - else - [{rounded_time, {rlat, rlon}}] - end - end) - end - end) + |> Enum.flat_map(&contact_points/1) |> Enum.group_by(fn {time, _pt} -> time end, fn {_time, pt} -> pt end) |> Map.new(fn {time, pts} -> {time, Enum.uniq(pts)} end) enqueue(groups) end + + # Per-contact expansion shared by `enqueue_for_contacts/1`. Returns a + # flat list of `{valid_time, {lat, lon}}` tuples ready for grouping. + # HRRR archive starts mid-2014; contacts older than that belong to + # NARR, so they're dropped here. + defp contact_points(contact) do + alias Microwaveprop.Radio + alias Microwaveprop.Weather + alias Microwaveprop.Weather.HrrrClient + alias Microwaveprop.Weather.NarrClient + + cond do + is_nil(contact.pos1) -> + [] + + NarrClient.in_coverage?(contact.qso_timestamp) -> + [] + + true -> + rounded_time = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp) + + contact + |> Radio.contact_path_points() + |> Enum.flat_map(&point_for_rounded_time(&1, rounded_time)) + end + end + + defp point_for_rounded_time({lat, lon}, rounded_time) do + alias Microwaveprop.Weather + + {rlat, rlon} = Weather.round_to_hrrr_grid(lat, lon) + + if Weather.has_hrrr_profile?(rlat, rlon, rounded_time) do + [] + else + [{rounded_time, {rlat, rlon}}] + end + end end diff --git a/lib/microwaveprop_web/components/layouts.ex b/lib/microwaveprop_web/components/layouts.ex index e48e8d28..87e476d9 100644 --- a/lib/microwaveprop_web/components/layouts.ex +++ b/lib/microwaveprop_web/components/layouts.ex @@ -106,14 +106,6 @@ defmodule MicrowavepropWeb.Layouts do - - - - <.flash_group flash={@flash} /> """ end diff --git a/lib/microwaveprop_web/components/layouts/root.html.heex b/lib/microwaveprop_web/components/layouts/root.html.heex index ecf43eee..fd323ec6 100644 --- a/lib/microwaveprop_web/components/layouts/root.html.heex +++ b/lib/microwaveprop_web/components/layouts/root.html.heex @@ -46,5 +46,12 @@ {@inner_content} + + diff --git a/lib/mix/tasks/rust.golden.ex b/lib/mix/tasks/rust.golden.ex index 05c59b57..3680b98c 100644 --- a/lib/mix/tasks/rust.golden.ex +++ b/lib/mix/tasks/rust.golden.ex @@ -55,7 +55,7 @@ defmodule Mix.Tasks.Rust.Golden do Mix.shell().info("wrote #{length(samples)} samples to #{path}") if n = opts[:print] do - samples |> Enum.take(n) |> Enum.each(&IO.inspect/1) + samples |> Enum.take(n) |> Enum.each(&Mix.shell().info(inspect(&1))) end end diff --git a/test/microwaveprop/propagation_test.exs b/test/microwaveprop/propagation_test.exs index 3589a1fc..dddd141a 100644 --- a/test/microwaveprop/propagation_test.exs +++ b/test/microwaveprop/propagation_test.exs @@ -8,6 +8,26 @@ defmodule Microwaveprop.PropagationTest do setup do ScoreCache.clear() + + # Tests share a single `propagation_scores_dir` across the run, so + # profiles left behind by one test can surface inside another — most + # visibly in the point_detail fallback tests where "no profile for + # this point in the last 24h" is the scenario under assertion. + # Wipe the profiles tree before each test. + base = Application.get_env(:microwaveprop, :propagation_scores_dir, "/data/scores") + profiles_dir = Path.join(base, "profiles") + File.rm_rf(profiles_dir) + + :ets.match_delete( + :microwaveprop_cache, + {{Microwaveprop.Propagation.ProfilesFile, :_, :_, :_}, :_, :_} + ) + + :ets.match_delete( + :microwaveprop_cache, + {{Microwaveprop.Propagation.ProfilesFile, :_, :_}, :_, :_} + ) + :ok end @@ -283,6 +303,203 @@ defmodule Microwaveprop.PropagationTest do assert is_map(detail.factors) assert Map.has_key?(detail.factors, :humidity) end + + test "uses the most recent profile at-or-before the requested time when several analysis hours exist" do + older = ~U[2026-07-15 10:00:00Z] + newer = ~U[2026-07-15 12:00:00Z] + forecast_time = ~U[2026-07-15 14:00:00Z] + lat = 32.75 + lon = -97.125 + + older_profile = build_profile(temp_c: 30.0, dewpoint_c: 5.0) + newer_profile = build_profile(temp_c: 22.0, dewpoint_c: 18.0) + + ProfilesFile.write!(older, %{{lat, lon} => older_profile}) + ProfilesFile.write!(newer, %{{lat, lon} => newer_profile}) + + Propagation.replace_scores( + [%{lat: lat, lon: lon, valid_time: forecast_time, band_mhz: 10_000, score: 55, factors: nil}], + forecast_time + ) + + detail = Propagation.point_detail(10_000, lat, lon, forecast_time) + # The newer profile has a much higher dewpoint, which drives a + # distinctly different humidity factor than the older one. + # Asserting the humidity reflects `newer_profile` proves the + # fallback picked the latest past analysis, not the oldest. + newer_humidity = humidity_factor_for(lat, lon, newer_profile, newer, 10_000) + assert detail.factors.humidity == newer_humidity + end + + test "returns empty factors when the only persisted profile is older than the 24h lookback" do + ancient = DateTime.add(~U[2026-07-15 13:00:00Z], -30 * 3600, :second) + forecast_time = ~U[2026-07-15 13:00:00Z] + lat = 32.75 + lon = -97.125 + + ProfilesFile.write!(ancient, %{{lat, lon} => build_profile()}) + + Propagation.replace_scores( + [%{lat: lat, lon: lon, valid_time: forecast_time, band_mhz: 10_000, score: 40, factors: nil}], + forecast_time + ) + + detail = Propagation.point_detail(10_000, lat, lon, forecast_time) + assert detail.score == 40 + # 30h > 24h lookback — the UI needs to show "breakdown unavailable" + # rather than attribute the current score to a day-old pattern. + assert detail.factors == %{} + end + + test "accepts a profile exactly at the 24h lookback boundary" do + fallback_time = DateTime.add(~U[2026-07-15 13:00:00Z], -24 * 3600, :second) + forecast_time = ~U[2026-07-15 13:00:00Z] + lat = 32.75 + lon = -97.125 + + ProfilesFile.write!(fallback_time, %{{lat, lon} => build_profile()}) + + Propagation.replace_scores( + [%{lat: lat, lon: lon, valid_time: forecast_time, band_mhz: 10_000, score: 50, factors: nil}], + forecast_time + ) + + detail = Propagation.point_detail(10_000, lat, lon, forecast_time) + assert Map.has_key?(detail.factors, :humidity) + end + + test "prefers an exact-time profile over any older fallback" do + exact = ~U[2026-07-15 14:00:00Z] + older = ~U[2026-07-15 10:00:00Z] + lat = 32.75 + lon = -97.125 + + wet = build_profile(dewpoint_c: 20.0) + dry = build_profile(dewpoint_c: -5.0) + + # Older (dry) profile also present — must be ignored once the + # exact-time (wet) profile exists. + ProfilesFile.write!(older, %{{lat, lon} => dry}) + ProfilesFile.write!(exact, %{{lat, lon} => wet}) + + Propagation.replace_scores( + [%{lat: lat, lon: lon, valid_time: exact, band_mhz: 10_000, score: 70, factors: nil}], + exact + ) + + detail = Propagation.point_detail(10_000, lat, lon, exact) + wet_humidity = humidity_factor_for(lat, lon, wet, exact, 10_000) + assert detail.factors.humidity == wet_humidity + end + + test "returns empty factors when the fallback profile has no cell for this lat/lon" do + # Analysis profile exists for a different cell. The fallback + # timestamp resolves, but read_point at (lat, lon) comes back nil + # on the fallback too — must degrade to `factors: %{}` rather + # than crashing. + analysis_time = ~U[2026-07-15 13:00:00Z] + forecast_time = ~U[2026-07-15 15:00:00Z] + click_lat = 32.75 + click_lon = -97.125 + far_away_lat = 40.0 + far_away_lon = -75.0 + + ProfilesFile.write!(analysis_time, %{{far_away_lat, far_away_lon} => build_profile()}) + + Propagation.replace_scores( + [%{lat: click_lat, lon: click_lon, valid_time: forecast_time, band_mhz: 10_000, score: 44, factors: nil}], + forecast_time + ) + + detail = Propagation.point_detail(10_000, click_lat, click_lon, forecast_time) + assert detail.score == 44 + assert detail.factors == %{} + end + + test "produces a full factor breakdown for every band when the profile is present" do + valid_time = ~U[2026-07-15 13:00:00Z] + lat = 32.75 + lon = -97.125 + ProfilesFile.write!(valid_time, %{{lat, lon} => build_profile()}) + + for band_mhz <- [10_000, 24_000, 47_000, 122_000, 241_000] do + Propagation.replace_scores( + [%{lat: lat, lon: lon, valid_time: valid_time, band_mhz: band_mhz, score: 60, factors: nil}], + valid_time + ) + + detail = Propagation.point_detail(band_mhz, lat, lon, valid_time) + assert detail, "no point_detail for band #{band_mhz}" + assert is_map(detail.factors) + + # Every non-empty factor map carries the same set of weighted + # keys across bands — proves we're not silently returning %{} for + # higher bands during the /map breakdown flow. + for required <- [:humidity, :rain, :pressure, :season, :sky, :refractivity, :wind, :td_depression] do + assert Map.has_key?(detail.factors, required), + "band #{band_mhz} missing factor #{required}" + end + end + end + + test "defaults to the latest valid_time when the caller omits one" do + older = ~U[2026-07-15 13:00:00Z] + latest = ~U[2026-07-15 14:00:00Z] + lat = 32.75 + lon = -97.125 + + Propagation.replace_scores( + [%{lat: lat, lon: lon, valid_time: older, band_mhz: 10_000, score: 10, factors: nil}], + older + ) + + Propagation.replace_scores( + [%{lat: lat, lon: lon, valid_time: latest, band_mhz: 10_000, score: 99, factors: nil}], + latest + ) + + ProfilesFile.write!(latest, %{{lat, lon} => build_profile()}) + + detail = Propagation.point_detail(10_000, lat, lon) + assert detail.valid_time == latest + assert detail.score == 99 + assert Map.has_key?(detail.factors, :humidity) + end + end + + # Build a canonical grid profile for point_detail fallback tests. + # Keyword overrides: :temp_c, :dewpoint_c, :hpbl_m map to surface_* keys. + defp build_profile(overrides \\ []) do + base = %{ + surface_temp_c: 25.0, + surface_dewpoint_c: 18.0, + surface_pressure_mb: 1013.0, + hpbl_m: 500.0, + wind_u: 3.0, + wind_v: 2.0, + cloud_cover_pct: 15.0, + precip_mm: 0.0, + profile: [ + %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, + %{"pres" => 975.0, "tmpc" => 22.0, "dwpc" => 15.0, "hght" => 350.0}, + %{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0} + ] + } + + Enum.reduce(overrides, base, fn + {:temp_c, v}, acc -> %{acc | surface_temp_c: v} + {:dewpoint_c, v}, acc -> %{acc | surface_dewpoint_c: v} + {:hpbl_m, v}, acc -> %{acc | hpbl_m: v} + {k, v}, acc -> Map.put(acc, k, v) + end) + end + + defp humidity_factor_for(lat, lon, profile, valid_time, band_mhz) do + profile + |> Propagation.score_grid_point(valid_time, lat, lon) + |> Enum.find(fn r -> r.band_mhz == band_mhz end) + |> Map.fetch!(:factors) + |> Map.fetch!(:humidity) end describe "latest_scores/1" do @@ -509,7 +726,7 @@ defmodule Microwaveprop.PropagationTest do describe "point_forecast/3 from store" do test "includes recent past valid_times and falls back to the latest when stale" do - # Three .ntms files on disk: one 2h old (dropped by cutoff), + # Three .prop files on disk: one 2h old (dropped by cutoff), # one 30min old (kept as "now"), one 1h future (kept). stale = DateTime.utc_now() |> DateTime.add(-2 * 3600, :second) |> DateTime.truncate(:second) recent = DateTime.utc_now() |> DateTime.add(-1800, :second) |> DateTime.truncate(:second) @@ -552,7 +769,7 @@ defmodule Microwaveprop.PropagationTest do end describe "available_valid_times/1" do - test "reads from the on-disk .ntms store" do + test "reads from the on-disk .prop store" do valid_time = DateTime.truncate(DateTime.utc_now(), :second) Propagation.replace_scores( @@ -600,7 +817,7 @@ defmodule Microwaveprop.PropagationTest do end test "sees a newly written forecast hour even when the cache is warm with earlier times" do - # Reproduces the race where PropagationGridWorker writes a new .ntms + # Reproduces the race where PropagationGridWorker writes a new .prop # file but MapLive's propagation_updated handler runs before the # ScoreCache has absorbed the cache_refresh broadcast. The cached # view alone is not enough — the timeline must pick up the new file. @@ -610,7 +827,7 @@ defmodule Microwaveprop.PropagationTest do # Warm the cache with the earlier valid_time only. ScoreCache.put(10_000, t_earlier, [%{lat: 25.0, lon: -125.0, score: 60}]) - # New .ntms file lands on disk for the next hour — but ScoreCache + # New .prop file lands on disk for the next hour — but ScoreCache # has not yet received the cache_refresh for `t_new`. Propagation.replace_scores( [%{lat: 25.0, lon: -125.0, valid_time: t_new, band_mhz: 10_000, score: 85, factors: nil}], diff --git a/test/microwaveprop_web/analytics_test.exs b/test/microwaveprop_web/analytics_test.exs new file mode 100644 index 00000000..cf978a91 --- /dev/null +++ b/test/microwaveprop_web/analytics_test.exs @@ -0,0 +1,83 @@ +defmodule MicrowavepropWeb.AnalyticsTest do + @moduledoc """ + Ensures the Plausible analytics snippet is present on every page the + app serves. Plausible lives in `root.html.heex` (loaded by every + request through the root layout) precisely so /map, /weather, and the + other full-bleed live views that bypass `` still load + the tracker. These tests lock that behaviour in so a future refactor + that moves the snippet back into `Layouts.app` can't silently drop + analytics on the three highest-traffic pages. + """ + use MicrowavepropWeb.ConnCase, async: true + + @plausible_script_url "https://a.w5isp.com/js/pa-3eKACFxtbw6MHrTDA28wG.js" + + # Pages in this list are fetched through the router and their rendered + # HTML asserted to contain the Plausible snippet. Any HTML route that + # a human-visible page lives behind belongs here. + @pages_to_check [ + {"/map", "propagation map"}, + {"/contacts/map", "contact map"}, + {"/weather", "weather map"}, + {"/about", "about page"}, + {"/algo", "algorithm docs"}, + {"/privacy", "privacy policy"}, + {"/submit", "QSO submission form"}, + {"/contacts", "contact list"}, + {"/path", "path analysis"}, + {"/beacons", "beacon list"}, + {"/users/register", "registration form"}, + {"/users/log-in", "login form"} + ] + + for {path, label} <- @pages_to_check do + test "#{label} (#{path}) includes the Plausible script tag", %{conn: conn} do + html = conn |> get(unquote(path)) |> html_response(200) + + assert html =~ unquote(@plausible_script_url), + "Plausible script missing on #{unquote(path)} — check root.html.heex" + end + + test "#{label} (#{path}) wires up window.plausible() via plausible.init", %{conn: conn} do + html = conn |> get(unquote(path)) |> html_response(200) + + assert html =~ "window.plausible", + "Plausible init stub missing on #{unquote(path)}" + + assert html =~ "plausible.init", + "plausible.init() call missing on #{unquote(path)}" + end + end + + test "the script tag is emitted exactly once per page (no duplicates)", %{conn: conn} do + # If the snippet accidentally returns to both `root.html.heex` AND + # `Layouts.app`, pages that use `` would load the tag + # twice and double-count pageviews. Assert a single occurrence on a + # layout-wrapped page. + html = conn |> get(~p"/about") |> html_response(200) + occurrences = html |> String.split(@plausible_script_url) |> length() + # String.split + length: N occurrences → N+1 pieces. One occurrence + # is the expected state. + assert occurrences == 2, + "Plausible script appears #{occurrences - 1}×; expected 1 on /about" + end + + test "the script loads asynchronously so it can't block the first paint", %{conn: conn} do + html = conn |> get(~p"/about") |> html_response(200) + + assert html =~ ~r/ get(~p"/users/settings") |> html_response(200) + assert html =~ @plausible_script_url + end +end diff --git a/test/microwaveprop_web/live/map_live_test.exs b/test/microwaveprop_web/live/map_live_test.exs index 4e4c06d7..40629051 100644 --- a/test/microwaveprop_web/live/map_live_test.exs +++ b/test/microwaveprop_web/live/map_live_test.exs @@ -186,7 +186,12 @@ defmodule MicrowavepropWeb.MapLiveTest do # "no data" state its assertion describes. scores_dir = Application.fetch_env!(:microwaveprop, :propagation_scores_dir) _ = File.rm_rf(scores_dir) - :ets.match_delete(:microwaveprop_cache, {{Microwaveprop.Propagation.ScoresFile, :list_valid_times, :_, :_}, :_, :_}) + + :ets.match_delete( + :microwaveprop_cache, + {{Microwaveprop.Propagation.ScoresFile, :list_valid_times, :_, :_}, :_, :_} + ) + :ok end