prop/test/microwaveprop/propagation_test.exs
Graham McIntire 2d9e5a2d1f
Make the map timeline read .ntms files authoritatively
Two related fixes so the main map reliably picks up new binary
propagation score files as soon as PropagationGridWorker writes them.

1. Propagation.available_valid_times/1 previously preferred ScoreCache
   over ScoresFile, using the cache as an index of what was available.
   The cache is a lazy ETS of whatever hours have been fetched or
   broadcast, which is a strict subset of what's on disk. A new
   forecast hour landing on disk while the cache was warm with older
   entries was invisible to the timeline until the cache happened to
   catch up. Read directly from ScoresFile so the disk store is the
   source of truth.

2. Add Propagation.scores_at_fresh/3 that always reads the .ntms file
   and overwrites the cache entry, and use it from MapLive's
   propagation_updated handler. PropagationGridWorker publishes the
   cache_refresh on `propagation:cache` and the timeline ping on
   `propagation:updated` as separate PubSub broadcasts, so by the time
   MapLive runs through scores_at the ScoreCache GenServer may not
   have processed the refresh yet — fetch_bounds then returns the
   previous chain's bytes. scores_at_fresh takes disk as the source
   of truth for the refresh path and warms the cache as a side effect
   so subsequent readers see the new data.
2026-04-14 17:42:30 -05:00

516 lines
18 KiB
Elixir

defmodule Microwaveprop.PropagationTest do
use Microwaveprop.DataCase, async: false
alias Microwaveprop.Propagation
alias Microwaveprop.Propagation.ProfilesFile
alias Microwaveprop.Propagation.ScoreCache
alias Microwaveprop.Propagation.ScoresFile
setup do
ScoreCache.clear()
:ok
end
describe "score_grid_point/4" do
test "scores a single point for all bands" do
hrrr_profile = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
precip_mm: 0.0,
profile: [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 975.0, "tmpc" => 22.0, "dwpc" => 15.0, "hght" => 350.0},
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
]
}
valid_time = ~U[2026-07-15 13:00:00Z]
results = Propagation.score_grid_point(hrrr_profile, valid_time, 33.0, -97.0)
assert length(results) == 19
Enum.each(results, fn result ->
assert result.score >= 0 and result.score <= 100
assert is_map(result.factors)
assert is_integer(result.band_mhz)
end)
end
test "NEXRAD reflectivity drops the 24 GHz rain score when HRRR precip_mm is 0" do
# Same profile, once without NEXRAD and once with a heavy-rain dBZ. The
# 24 GHz rain-score factor must drop because NEXRAD can report rain that
# HRRR's hourly precip accumulation hasn't caught yet.
base_profile = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
precip_mm: 0.0,
profile: [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
]
}
valid_time = ~U[2026-07-15 13:00:00Z]
without = Propagation.score_grid_point(base_profile, valid_time, 33.0, -97.0)
with_nexrad =
base_profile
|> Map.put(:nexrad_max_reflectivity_dbz, 45.0)
|> Propagation.score_grid_point(valid_time, 33.0, -97.0)
rain_dry = Enum.find(without, &(&1.band_mhz == 24_000)).factors.rain
rain_wet = Enum.find(with_nexrad, &(&1.band_mhz == 24_000)).factors.rain
assert rain_wet < rain_dry
end
test "uses a persisted min_refractivity_gradient scalar without decoding the profile array" do
# AsosAdjustmentWorker loads 92k HRRR profile rows per 10-min tick and
# can't afford to pull the ~1 KB JSONB `profile` column for each one —
# Postgrex's Jason.decode! per row blew past the 15s pool checkout.
# hrrr_profiles persists min_refractivity_gradient as a scalar at
# ingestion time, so score_grid_point must honour it when the profile
# array is absent instead of trying to re-derive from a missing list.
hrrr_profile_without_array = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
pwat_mm: 28.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
precip_mm: 0.0,
min_refractivity_gradient: -400.0
}
valid_time = ~U[2026-07-15 13:00:00Z]
results = Propagation.score_grid_point(hrrr_profile_without_array, valid_time, 33.0, -97.0)
assert length(results) == 19
Enum.each(results, fn result ->
assert result.score >= 0 and result.score <= 100
end)
result_10g = Enum.find(results, &(&1.band_mhz == 10_000))
# A strong negative refractivity gradient (< -300 N/km) should leave
# the refractivity factor visibly positive; if score_grid_point had
# silently used 0.0 we'd see the neutral baseline instead.
assert result_10g.factors.refractivity > 55
end
test "NEXRAD is ignored when HRRR precip_mm already reports heavier rain" do
base_profile = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
# 30 mm/hr → far above the ~3 mm/hr NEXRAD gives for 30 dBZ, so HRRR wins.
precip_mm: 30.0,
profile: [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
]
}
valid_time = ~U[2026-07-15 13:00:00Z]
with_light_nexrad =
base_profile
|> Map.put(:nexrad_max_reflectivity_dbz, 30.0)
|> Propagation.score_grid_point(valid_time, 33.0, -97.0)
without = Propagation.score_grid_point(base_profile, valid_time, 33.0, -97.0)
rain_no_nx = Enum.find(without, &(&1.band_mhz == 24_000)).factors.rain
rain_with_nx = Enum.find(with_light_nexrad, &(&1.band_mhz == 24_000)).factors.rain
assert rain_with_nx == rain_no_nx
end
end
describe "replace_scores/2" do
test "writes one ScoresFile per band for a fresh valid_time" do
valid_time = ~U[2026-07-15 13:00:00Z]
scores = [
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil},
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 24_000, score: 60, factors: nil}
]
assert {:ok, 2} = Propagation.replace_scores(scores, valid_time)
assert {:ok, %{band_mhz: 10_000}} = ScoresFile.read(10_000, valid_time)
assert {:ok, %{band_mhz: 24_000}} = ScoresFile.read(24_000, valid_time)
end
test "overwrites the file when called again for the same valid_time" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 10, factors: nil}],
valid_time
)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil}],
valid_time
)
assert ScoresFile.read_point(10_000, valid_time, 25.0, -125.0) == 75
end
test "leaves files for other valid_times untouched" do
other_time = ~U[2026-07-15 12:00:00Z]
replaced_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: other_time, band_mhz: 10_000, score: 40, factors: nil}],
other_time
)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: replaced_time, band_mhz: 10_000, score: 90, factors: nil}],
replaced_time
)
assert ScoresFile.read_point(10_000, other_time, 25.0, -125.0) == 40
assert ScoresFile.read_point(10_000, replaced_time, 25.0, -125.0) == 90
end
end
describe "point_detail/4" do
test "returns the score from the on-disk file with an empty factors map when no profile is persisted" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 42, factors: nil}],
valid_time
)
assert %{score: 42, factors: %{}} =
Propagation.point_detail(10_000, 25.0, -125.0, valid_time)
end
test "rebuilds the factor breakdown from a persisted f00 profile" do
valid_time = ~U[2026-07-15 13:00:00Z]
lat = 32.75
lon = -97.125
profile = %{
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1013.0,
hpbl_m: 500.0,
wind_u: 3.0,
wind_v: 2.0,
cloud_cover_pct: 15.0,
precip_mm: 0.0,
profile: [
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
%{"pres" => 975.0, "tmpc" => 22.0, "dwpc" => 15.0, "hght" => 350.0},
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
]
}
ProfilesFile.write!(valid_time, %{{lat, lon} => profile})
Propagation.replace_scores(
[%{lat: lat, lon: lon, valid_time: valid_time, band_mhz: 10_000, score: 71, factors: nil}],
valid_time
)
detail = Propagation.point_detail(10_000, lat, lon, valid_time)
assert detail.score == 71
assert is_map(detail.factors)
# The algorithm populates every weighted factor; humidity is the
# dominant input for 10 GHz so it must be present.
assert Map.has_key?(detail.factors, :humidity)
end
test "returns nil when the file doesn't exist for the requested point" do
assert Propagation.point_detail(10_000, 25.0, -125.0, ~U[2026-07-15 13:00:00Z]) == nil
end
end
describe "latest_scores/1" do
test "returns latest scores for a band" do
valid_time = ~U[2026-07-15 13:00:00Z]
scores = [
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil},
%{lat: 25.125, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 80, factors: nil}
]
Propagation.replace_scores(scores, valid_time)
results = Propagation.latest_scores(10_000)
assert length(results) == 2
end
test "returns empty list when no data" do
assert Propagation.latest_scores(10_000) == []
end
end
describe "scores_at/3 with cache" do
test "populates the cache on a cold read" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil}],
valid_time
)
assert ScoreCache.fetch(10_000, valid_time) == :miss
_ = Propagation.scores_at(10_000, valid_time)
assert {:ok, [%{lat: 25.0, lon: -125.0, score: 75}]} = ScoreCache.fetch(10_000, valid_time)
end
test "returns cached bounds-filtered results without touching the filesystem" do
valid_time = ~U[2026-07-15 13:00:00Z]
# Seed the cache directly — no ScoresFile exists, so if the
# result matches the cached payload we know the disk was not
# consulted.
ScoreCache.put(10_000, valid_time, [
%{lat: 32.0, lon: -97.0, score: 75},
%{lat: 40.0, lon: -74.0, score: 50}
])
bounds = %{"south" => 30.0, "north" => 35.0, "west" => -100.0, "east" => -95.0}
result = Propagation.scores_at(10_000, valid_time, bounds)
assert [%{lat: 32.0, lon: -97.0, score: 75}] = result
end
test "returns empty list when no data anywhere" do
assert Propagation.scores_at(10_000, ~U[2026-07-15 13:00:00Z]) == []
end
end
describe "point_forecast/3 with cache" do
test "returns forecast timeline from cached scores when warm" do
t1 = DateTime.truncate(DateTime.add(DateTime.utc_now(), 3600, :second), :second)
t2 = DateTime.add(t1, 3600, :second)
ScoreCache.put(10_000, t1, [%{lat: 32.875, lon: -97.0, score: 70}])
ScoreCache.put(10_000, t2, [%{lat: 32.875, lon: -97.0, score: 75}])
result = Propagation.point_forecast(10_000, 32.875, -97.0)
assert [
%{valid_time: ^t1, score: 70},
%{valid_time: ^t2, score: 75}
] = result
end
test "filters past valid_times out" do
past = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.truncate(:second)
ScoreCache.put(10_000, past, [%{lat: 32.875, lon: -97.0, score: 60}])
ScoreCache.put(10_000, future, [%{lat: 32.875, lon: -97.0, score: 80}])
result = Propagation.point_forecast(10_000, 32.875, -97.0)
assert [%{valid_time: ^future, score: 80}] = result
end
test "snaps coordinates to the nearest grid point" do
valid_time = DateTime.truncate(DateTime.add(DateTime.utc_now(), 3600, :second), :second)
ScoreCache.put(10_000, valid_time, [%{lat: 32.875, lon: -97.0, score: 65}])
# Off-grid query snaps to 32.875, -97.0
result = Propagation.point_forecast(10_000, 32.88, -96.98)
assert [%{valid_time: ^valid_time, score: 65}] = result
end
test "returns empty list when no data for the point" do
assert Propagation.point_forecast(10_000, 32.875, -97.0) == []
end
end
describe "available_valid_times/1" do
test "reads from the on-disk .ntms store" do
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil}],
valid_time
)
assert ScoreCache.valid_times(10_000) == []
assert Propagation.available_valid_times(10_000) == [valid_time]
end
test "filters valid_times older than the 1-hour cutoff" do
now = DateTime.truncate(DateTime.utc_now(), :second)
stale = DateTime.add(now, -7200, :second)
fresh = DateTime.add(now, 1800, :second)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: stale, band_mhz: 10_000, score: 50, factors: nil}],
stale
)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: fresh, band_mhz: 10_000, score: 80, factors: nil}],
fresh
)
assert Propagation.available_valid_times(10_000) == [fresh]
end
test "sees a newly written forecast hour even when the cache is warm with earlier times" do
# Reproduces the race where PropagationGridWorker writes a new .ntms
# file but MapLive's propagation_updated handler runs before the
# ScoreCache has absorbed the cache_refresh broadcast. The cached
# view alone is not enough — the timeline must pick up the new file.
t_earlier = ~U[2026-07-15 13:00:00Z]
t_new = ~U[2026-07-15 14:00:00Z]
# Warm the cache with the earlier valid_time only.
ScoreCache.put(10_000, t_earlier, [%{lat: 25.0, lon: -125.0, score: 60}])
# New .ntms file lands on disk for the next hour — but ScoreCache
# has not yet received the cache_refresh for `t_new`.
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: t_new, band_mhz: 10_000, score: 85, factors: nil}],
t_new
)
assert t_new in Propagation.available_valid_times(10_000)
end
end
describe "scores_at_fresh/3" do
test "bypasses a stale cache entry and returns the on-disk scores" do
# Reproduces the race where MapLive handles a propagation_updated
# message while the ScoreCache still holds the previous chain's
# scores for the same (band, valid_time). The fresh path must
# always return what's on disk and rewrite the cache accordingly.
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
stale_scores = [%{lat: 25.0, lon: -125.0, score: 10}]
ScoreCache.put(10_000, valid_time, stale_scores)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 95, factors: nil}],
valid_time
)
fresh = Propagation.scores_at_fresh(10_000, valid_time, nil)
assert [%{lat: 25.0, lon: -125.0, score: 95, valid_time: ^valid_time}] = fresh
end
test "refreshes the cache so subsequent scores_at calls see the new data" do
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
ScoreCache.put(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 10}])
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 95, factors: nil}],
valid_time
)
_ = Propagation.scores_at_fresh(10_000, valid_time, nil)
assert {:ok, [%{score: 95}]} = ScoreCache.fetch(10_000, valid_time)
end
test "applies bounding-box filtering just like scores_at/3" do
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
Propagation.replace_scores(
[
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 80, factors: nil},
%{lat: 40.0, lon: -80.0, valid_time: valid_time, band_mhz: 10_000, score: 70, factors: nil}
],
valid_time
)
bounds = %{"south" => 20.0, "north" => 30.0, "west" => -130.0, "east" => -120.0}
fresh = Propagation.scores_at_fresh(10_000, valid_time, bounds)
assert length(fresh) == 1
assert hd(fresh).lat == 25.0
end
end
describe "warm_cache_and_broadcast/2" do
test "loads scores from the on-disk store into the cache" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil},
%{lat: 25.125, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 80, factors: nil}
],
valid_time
)
assert ScoreCache.fetch(10_000, valid_time) == :miss
Propagation.warm_cache_and_broadcast(10_000, valid_time)
ScoreCache.sync()
assert {:ok, scores} = ScoreCache.fetch(10_000, valid_time)
assert length(scores) == 2
end
test "only warms the requested band" do
valid_time = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: nil},
%{lat: 25.0, lon: -125.0, valid_time: valid_time, band_mhz: 24_000, score: 30, factors: nil}
],
valid_time
)
Propagation.warm_cache_and_broadcast(10_000, valid_time)
ScoreCache.sync()
assert {:ok, [_]} = ScoreCache.fetch(10_000, valid_time)
assert ScoreCache.fetch(24_000, valid_time) == :miss
end
end
describe "latest_valid_time/0" do
test "returns nil when empty" do
assert Propagation.latest_valid_time() == nil
end
test "returns the most recent valid_time across bands" do
t1 = ~U[2026-07-15 12:00:00Z]
t2 = ~U[2026-07-15 13:00:00Z]
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: t1, band_mhz: 10_000, score: 50, factors: nil}],
t1
)
Propagation.replace_scores(
[%{lat: 25.0, lon: -125.0, valid_time: t2, band_mhz: 10_000, score: 60, factors: nil}],
t2
)
assert Propagation.latest_valid_time() == t2
end
end
end