prop/test/microwaveprop/propagation_test.exs
Graham McIntire e227978f2a
Let the /path forecast chart start at the current analysis hour
point_forecast used a strict \`>= now\` cutoff, so the leftmost
"now" sample was always dropped — the HRRR publishing lag means the
newest analysis file is typically 30–60 min behind the wall clock,
which made the filter evict the very hour the chart most wanted to
anchor on. If the chain ever fell behind by even one step the whole
forecast went empty and the chart disappeared.

Use the same window as available_valid_times: keep everything from
one hour before now onward, and fall back to the single newest file
when nothing is fresh enough. Both the cache-path and store-path
flows go through a shared forecast_window helper so the behavior
stays in lockstep with the map timeline.
2026-04-15 10:46:42 -05:00

582 lines
21 KiB
Elixir

defmodule Microwaveprop.PropagationTest do
use Microwaveprop.DataCase, async: false
alias Microwaveprop.Propagation
alias Microwaveprop.Propagation.ProfilesFile
alias Microwaveprop.Propagation.ScoreCache
alias Microwaveprop.Propagation.ScoresFile
setup do
ScoreCache.clear()
:ok
end
describe "score_grid_point/4" do
test "scores a single point for all bands" do
hrrr_profile = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
precip_mm: 0.0,
profile: [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 975.0, "tmpc" => 22.0, "dwpc" => 15.0, "hght" => 350.0},
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
]
}
valid_time = ~U[2026-07-15 13:00:00Z]
results = Propagation.score_grid_point(hrrr_profile, valid_time, 33.0, -97.0)
assert length(results) == 19
Enum.each(results, fn result ->
assert result.score >= 0 and result.score <= 100
assert is_map(result.factors)
assert is_integer(result.band_mhz)
end)
end
test "NEXRAD reflectivity drops the 24 GHz rain score when HRRR precip_mm is 0" do
# Same profile, once without NEXRAD and once with a heavy-rain dBZ. The
# 24 GHz rain-score factor must drop because NEXRAD can report rain that
# HRRR's hourly precip accumulation hasn't caught yet.
base_profile = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
precip_mm: 0.0,
profile: [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
]
}
valid_time = ~U[2026-07-15 13:00:00Z]
without = Propagation.score_grid_point(base_profile, valid_time, 33.0, -97.0)
with_nexrad =
base_profile
|> Map.put(:nexrad_max_reflectivity_dbz, 45.0)
|> Propagation.score_grid_point(valid_time, 33.0, -97.0)
rain_dry = Enum.find(without, &(&1.band_mhz == 24_000)).factors.rain
rain_wet = Enum.find(with_nexrad, &(&1.band_mhz == 24_000)).factors.rain
assert rain_wet < rain_dry
end
test "uses a persisted min_refractivity_gradient scalar without decoding the profile array" do
# AsosAdjustmentWorker loads 92k HRRR profile rows per 10-min tick and
# can't afford to pull the ~1 KB JSONB `profile` column for each one —
# Postgrex's Jason.decode! per row blew past the 15s pool checkout.
# hrrr_profiles persists min_refractivity_gradient as a scalar at
# ingestion time, so score_grid_point must honour it when the profile
# array is absent instead of trying to re-derive from a missing list.
hrrr_profile_without_array = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
pwat_mm: 28.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
precip_mm: 0.0,
min_refractivity_gradient: -400.0
}
valid_time = ~U[2026-07-15 13:00:00Z]
results = Propagation.score_grid_point(hrrr_profile_without_array, valid_time, 33.0, -97.0)
assert length(results) == 19
Enum.each(results, fn result ->
assert result.score >= 0 and result.score <= 100
end)
result_10g = Enum.find(results, &(&1.band_mhz == 10_000))
# A strong negative refractivity gradient (< -300 N/km) should leave
# the refractivity factor visibly positive; if score_grid_point had
# silently used 0.0 we'd see the neutral baseline instead.
assert result_10g.factors.refractivity > 55
end
test "NEXRAD is ignored when HRRR precip_mm already reports heavier rain" do
base_profile = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
# 30 mm/hr → far above the ~3 mm/hr NEXRAD gives for 30 dBZ, so HRRR wins.
precip_mm: 30.0,
profile: [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
]
}
valid_time = ~U[2026-07-15 13:00:00Z]
with_light_nexrad =
base_profile
|> Map.put(:nexrad_max_reflectivity_dbz, 30.0)
|> Propagation.score_grid_point(valid_time, 33.0, -97.0)
without = Propagation.score_grid_point(base_profile, valid_time, 33.0, -97.0)
rain_no_nx = Enum.find(without, &(&1.band_mhz == 24_000)).factors.rain
rain_with_nx = Enum.find(with_light_nexrad, &(&1.band_mhz == 24_000)).factors.rain
assert rain_with_nx == rain_no_nx
end
end
describe "replace_scores/2" do
test "writes one ScoresFile per band for a fresh valid_time" do
valid_time = ~U[2026-07-15 13:00:00Z]
scores = [
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil},
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 24_000, score: 60, factors: nil}
]
assert {:ok, 2} = Propagation.replace_scores(scores, valid_time)
assert {:ok, %{band_mhz: 10_000}} = ScoresFile.read(10_000, valid_time)
assert {:ok, %{band_mhz: 24_000}} = ScoresFile.read(24_000, valid_time)
end
test "overwrites the file when called again for the same valid_time" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 10, factors: nil}],
valid_time
)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil}],
valid_time
)
assert ScoresFile.read_point(10_000, valid_time, 25.0, -125.0) == 75
end
test "leaves files for other valid_times untouched" do
other_time = ~U[2026-07-15 12:00:00Z]
replaced_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: other_time, band_mhz: 10_000, score: 40, factors: nil}],
other_time
)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: replaced_time, band_mhz: 10_000, score: 90, factors: nil}],
replaced_time
)
assert ScoresFile.read_point(10_000, other_time, 25.0, -125.0) == 40
assert ScoresFile.read_point(10_000, replaced_time, 25.0, -125.0) == 90
end
end
describe "point_detail/4" do
test "returns the score from the on-disk file with an empty factors map when no profile is persisted" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 42, factors: nil}],
valid_time
)
assert %{score: 42, factors: %{}} =
Propagation.point_detail(10_000, 25.0, -125.0, valid_time)
end
test "rebuilds the factor breakdown from a persisted f00 profile" do
valid_time = ~U[2026-07-15 13:00:00Z]
lat = 32.75
lon = -97.125
profile = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
precip_mm: 0.0,
profile: [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 975.0, "tmpc" => 22.0, "dwpc" => 15.0, "hght" => 350.0},
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
]
}
ProfilesFile.write!(valid_time, %{{lat, lon} => profile})
Propagation.replace_scores(
[%{lat: lat, lon: lon, valid_time: valid_time, band_mhz: 10_000, score: 71, factors: nil}],
valid_time
)
detail = Propagation.point_detail(10_000, lat, lon, valid_time)
assert detail.score == 71
assert is_map(detail.factors)
# The algorithm populates every weighted factor; humidity is the
# dominant input for 10 GHz so it must be present.
assert Map.has_key?(detail.factors, :humidity)
end
test "returns nil when the file doesn't exist for the requested point" do
assert Propagation.point_detail(10_000, 25.0, -125.0, ~U[2026-07-15 13:00:00Z]) == nil
end
end
describe "latest_scores/1" do
test "returns latest scores for a band" do
valid_time = ~U[2026-07-15 13:00:00Z]
scores = [
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil},
%{lat: 25.125, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 80, factors: nil}
]
Propagation.replace_scores(scores, valid_time)
results = Propagation.latest_scores(10_000)
assert length(results) == 2
end
test "returns empty list when no data" do
assert Propagation.latest_scores(10_000) == []
end
end
describe "scores_at/3 with cache" do
test "populates the cache on a cold read" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil}],
valid_time
)
assert ScoreCache.fetch(10_000, valid_time) == :miss
_ = Propagation.scores_at(10_000, valid_time)
assert {:ok, [%{lat: 25.0, lon: -125.0, score: 75}]} = ScoreCache.fetch(10_000, valid_time)
end
test "returns cached bounds-filtered results without touching the filesystem" do
valid_time = ~U[2026-07-15 13:00:00Z]
# Seed the cache directly — no ScoresFile exists, so if the
# result matches the cached payload we know the disk was not
# consulted.
ScoreCache.put(10_000, valid_time, [
%{lat: 32.0, lon: -97.0, score: 75},
%{lat: 40.0, lon: -74.0, score: 50}
])
bounds = %{"south" => 30.0, "north" => 35.0, "west" => -100.0, "east" => -95.0}
result = Propagation.scores_at(10_000, valid_time, bounds)
assert [%{lat: 32.0, lon: -97.0, score: 75}] = result
end
test "returns empty list when no data anywhere" do
assert Propagation.scores_at(10_000, ~U[2026-07-15 13:00:00Z]) == []
end
end
describe "point_forecast/3 with cache" do
test "returns forecast timeline from cached scores when warm" do
t1 = DateTime.truncate(DateTime.add(DateTime.utc_now(), 3600, :second), :second)
t2 = DateTime.add(t1, 3600, :second)
ScoreCache.put(10_000, t1, [%{lat: 32.875, lon: -97.0, score: 70}])
ScoreCache.put(10_000, t2, [%{lat: 32.875, lon: -97.0, score: 75}])
result = Propagation.point_forecast(10_000, 32.875, -97.0)
assert [
%{valid_time: ^t1, score: 70},
%{valid_time: ^t2, score: 75}
] = result
end
test "filters valid_times older than one hour but keeps recent past as the 'now' anchor" do
# Two hours old: dropped. Thirty minutes old: kept as the
# leftmost "now" point. One hour future: kept.
stale = DateTime.utc_now() |> DateTime.add(-2 * 3600, :second) |> DateTime.truncate(:second)
recent = DateTime.utc_now() |> DateTime.add(-1800, :second) |> DateTime.truncate(:second)
future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.truncate(:second)
ScoreCache.put(10_000, stale, [%{lat: 32.875, lon: -97.0, score: 40}])
ScoreCache.put(10_000, recent, [%{lat: 32.875, lon: -97.0, score: 60}])
ScoreCache.put(10_000, future, [%{lat: 32.875, lon: -97.0, score: 80}])
result = Propagation.point_forecast(10_000, 32.875, -97.0)
assert [
%{valid_time: ^recent, score: 60},
%{valid_time: ^future, score: 80}
] = result
end
test "falls back to the single latest valid_time when everything is older than one hour" do
# HRRR just cut out (nothing fresh); we still want the chart to
# render one point at the newest time we have, matching
# available_valid_times semantics.
older = DateTime.utc_now() |> DateTime.add(-6 * 3600, :second) |> DateTime.truncate(:second)
newer = DateTime.utc_now() |> DateTime.add(-3 * 3600, :second) |> DateTime.truncate(:second)
ScoreCache.put(10_000, older, [%{lat: 32.875, lon: -97.0, score: 30}])
ScoreCache.put(10_000, newer, [%{lat: 32.875, lon: -97.0, score: 45}])
result = Propagation.point_forecast(10_000, 32.875, -97.0)
assert [%{valid_time: ^newer, score: 45}] = result
end
test "snaps coordinates to the nearest grid point" do
valid_time = DateTime.truncate(DateTime.add(DateTime.utc_now(), 3600, :second), :second)
ScoreCache.put(10_000, valid_time, [%{lat: 32.875, lon: -97.0, score: 65}])
# Off-grid query snaps to 32.875, -97.0
result = Propagation.point_forecast(10_000, 32.88, -96.98)
assert [%{valid_time: ^valid_time, score: 65}] = result
end
test "returns empty list when no data for the point" do
assert Propagation.point_forecast(10_000, 32.875, -97.0) == []
end
end
describe "point_forecast/3 from store" do
test "includes recent past valid_times and falls back to the latest when stale" do
# Three .ntms files on disk: one 2h old (dropped by cutoff),
# one 30min old (kept as "now"), one 1h future (kept).
stale = DateTime.utc_now() |> DateTime.add(-2 * 3600, :second) |> DateTime.truncate(:second)
recent = DateTime.utc_now() |> DateTime.add(-1800, :second) |> DateTime.truncate(:second)
future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.truncate(:second)
Enum.each([{stale, 40}, {recent, 60}, {future, 80}], fn {t, score} ->
Propagation.replace_scores(
[%{lat: 32.875, lon: -97.0, valid_time: t, band_mhz: 10_000, score: score, factors: nil}],
t
)
end)
# Bypass the cache so we go through the store-backed path.
ScoreCache.clear()
result = Propagation.point_forecast(10_000, 32.875, -97.0)
assert [
%{valid_time: ^recent, score: 60},
%{valid_time: ^future, score: 80}
] = result
end
test "returns just the latest store entry when every hour is stale" do
older = DateTime.utc_now() |> DateTime.add(-6 * 3600, :second) |> DateTime.truncate(:second)
newer = DateTime.utc_now() |> DateTime.add(-3 * 3600, :second) |> DateTime.truncate(:second)
Enum.each([{older, 30}, {newer, 45}], fn {t, score} ->
Propagation.replace_scores(
[%{lat: 32.875, lon: -97.0, valid_time: t, band_mhz: 10_000, score: score, factors: nil}],
t
)
end)
ScoreCache.clear()
result = Propagation.point_forecast(10_000, 32.875, -97.0)
assert [%{valid_time: ^newer, score: 45}] = result
end
end
describe "available_valid_times/1" do
test "reads from the on-disk .ntms store" do
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil}],
valid_time
)
assert ScoreCache.valid_times(10_000) == []
assert Propagation.available_valid_times(10_000) == [valid_time]
end
test "filters valid_times older than the 1-hour cutoff" do
now = DateTime.truncate(DateTime.utc_now(), :second)
stale = DateTime.add(now, -7200, :second)
fresh = DateTime.add(now, 1800, :second)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: stale, band_mhz: 10_000, score: 50, factors: nil}],
stale
)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: fresh, band_mhz: 10_000, score: 80, factors: nil}],
fresh
)
assert Propagation.available_valid_times(10_000) == [fresh]
end
test "sees a newly written forecast hour even when the cache is warm with earlier times" do
# Reproduces the race where PropagationGridWorker writes a new .ntms
# file but MapLive's propagation_updated handler runs before the
# ScoreCache has absorbed the cache_refresh broadcast. The cached
# view alone is not enough — the timeline must pick up the new file.
t_earlier = ~U[2026-07-15 13:00:00Z]
t_new = ~U[2026-07-15 14:00:00Z]
# Warm the cache with the earlier valid_time only.
ScoreCache.put(10_000, t_earlier, [%{lat: 25.0, lon: -125.0, score: 60}])
# New .ntms file lands on disk for the next hour — but ScoreCache
# has not yet received the cache_refresh for `t_new`.
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: t_new, band_mhz: 10_000, score: 85, factors: nil}],
t_new
)
assert t_new in Propagation.available_valid_times(10_000)
end
end
describe "scores_at_fresh/3" do
test "bypasses a stale cache entry and returns the on-disk scores" do
# Reproduces the race where MapLive handles a propagation_updated
# message while the ScoreCache still holds the previous chain's
# scores for the same (band, valid_time). The fresh path must
# always return what's on disk and rewrite the cache accordingly.
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
stale_scores = [%{lat: 25.0, lon: -125.0, score: 10}]
ScoreCache.put(10_000, valid_time, stale_scores)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 95, factors: nil}],
valid_time
)
fresh = Propagation.scores_at_fresh(10_000, valid_time, nil)
assert [%{lat: 25.0, lon: -125.0, score: 95, valid_time: ^valid_time}] = fresh
end
test "refreshes the cache so subsequent scores_at calls see the new data" do
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
ScoreCache.put(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 10}])
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 95, factors: nil}],
valid_time
)
_ = Propagation.scores_at_fresh(10_000, valid_time, nil)
assert {:ok, [%{score: 95}]} = ScoreCache.fetch(10_000, valid_time)
end
test "applies bounding-box filtering just like scores_at/3" do
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
Propagation.replace_scores(
[
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 80, factors: nil},
%{lat: 40.0, lon: -80.0, valid_time: valid_time, band_mhz: 10_000, score: 70, factors: nil}
],
valid_time
)
bounds = %{"south" => 20.0, "north" => 30.0, "west" => -130.0, "east" => -120.0}
fresh = Propagation.scores_at_fresh(10_000, valid_time, bounds)
assert length(fresh) == 1
assert hd(fresh).lat == 25.0
end
end
describe "warm_cache_and_broadcast/2" do
test "loads scores from the on-disk store into the cache" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil},
%{lat: 25.125, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 80, factors: nil}
],
valid_time
)
assert ScoreCache.fetch(10_000, valid_time) == :miss
Propagation.warm_cache_and_broadcast(10_000, valid_time)
ScoreCache.sync()
assert {:ok, scores} = ScoreCache.fetch(10_000, valid_time)
assert length(scores) == 2
end
test "only warms the requested band" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil},
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 24_000, score: 30, factors: nil}
],
valid_time
)
Propagation.warm_cache_and_broadcast(10_000, valid_time)
ScoreCache.sync()
assert {:ok, [_]} = ScoreCache.fetch(10_000, valid_time)
assert ScoreCache.fetch(24_000, valid_time) == :miss
end
end
describe "latest_valid_time/0" do
test "returns nil when empty" do
assert Propagation.latest_valid_time() == nil
end
test "returns the most recent valid_time across bands" do
t1 = ~U[2026-07-15 12:00:00Z]
t2 = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: t1, band_mhz: 10_000, score: 50, factors: nil}],
t1
)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: t2, band_mhz: 10_000, score: 60, factors: nil}],
t2
)
assert Propagation.latest_valid_time() == t2
end
end
end