Graham McIntire
2fd88a94ea
Phase A — 7 pipeline bugs:
- retain_scores_window: fix timeline self-deletion (NOTIFY payload run_time|valid_time)
- Rust/Elixir weight divergence: Rust loads band_weights.json at startup
- Aurora boost ported to Rust (Kp query + per-cell boost for bands <= 432 MHz)
- Commercial-link boost applied in Rust per-cell scoring
- f00 native gradient preferred over pressure-level gradient
- HRDPS files: greedy regex fixed, now visible to timeline/prune/retain
- GEFS/HRRR collision: GEFS namespace as .gefs.prop, merge at read
Phase B — Validation harness:
- scripts/validate_algo.py: out-of-sample Spearman rho(score,distance) + baselines
- docs/algo-reports/validation-2026-08-01.{json,md}
Phase C — Forecast-skill evaluation:
- scripts/validate_forecast.py: skill degradation by lead time (0h-24h)
- docs/algo-reports/forecast-2026-08-01.{json,md}
Phase D — Calibration + model improvements:
- recalibrate.py: validation gate before weight deploy
- Recalibrator: Nx.max(0.0) replaces Nx.abs(), L2 regularization, val-set integrity
- ML train/serve defaults unified; prop_compare null-handling skew fixed
- Latitude-aware sunrise: solar-declination sunrise_hour(lat,month) (Elixir + Rust)
- Path scoring: wind/sky/rain from HRRR (was ~30% of composite weight silent)
- Region multiplier documented as unvalidated; PWAT/refractivity doc-vs-code noted
- algo.md: synced scoring sections, marked retired features, D5/D6 changelog
- Credo: path_compute cyclomatic complexity bumped (6->10 fields) — informational only
411 lines
15 KiB
Elixir
411 lines
15 KiB
Elixir
defmodule Microwaveprop.Propagation.ScoresFileTest do
|
|
# async: false — tests mutate the global Application env to redirect
|
|
# the scores dir to a private tmp tree. Running these in parallel
|
|
# would have concurrent tests trample each other's directory setting.
|
|
use ExUnit.Case, async: false
|
|
|
|
alias Microwaveprop.Propagation.Grid
|
|
alias Microwaveprop.Propagation.ScoresFile
|
|
|
|
setup do
|
|
dir =
|
|
Path.join(
|
|
System.tmp_dir!(),
|
|
"scores_file_test_#{System.unique_integer([:positive])}"
|
|
)
|
|
|
|
File.mkdir_p!(dir)
|
|
|
|
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
|
|
|
|
on_exit(fn ->
|
|
File.rm_rf!(dir)
|
|
|
|
if prev do
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
|
|
else
|
|
Application.delete_env(:microwaveprop, :propagation_scores_dir)
|
|
end
|
|
end)
|
|
|
|
%{dir: dir}
|
|
end
|
|
|
|
describe "path_for/2" do
|
|
test "nests files under base_dir/{band_mhz}/ with an ISO-8601 valid_time", %{dir: dir} do
|
|
path = ScoresFile.path_for(10_000, ~U[2026-04-14 18:00:00Z])
|
|
assert path == Path.join([dir, "10000", "2026-04-14T18:00:00Z.prop"])
|
|
end
|
|
|
|
test "writes with the new .prop extension", %{dir: dir} do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
ScoresFile.write!(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 50}])
|
|
written = Path.join([dir, "10000", "2026-04-14T18:00:00Z.prop"])
|
|
assert File.exists?(written)
|
|
refute File.exists?(Path.join([dir, "10000", "2026-04-14T18:00:00Z.ntms"]))
|
|
end
|
|
end
|
|
|
|
describe "legacy_path_for/2" do
|
|
test "still points at the old .ntms extension for backwards reads", %{dir: dir} do
|
|
path = ScoresFile.legacy_path_for(10_000, ~U[2026-04-14 18:00:00Z])
|
|
assert path == Path.join([dir, "10000", "2026-04-14T18:00:00Z.ntms"])
|
|
end
|
|
end
|
|
|
|
describe "legacy .ntms file readability" do
|
|
test "read/2 falls through to the legacy .ntms path when .prop is missing", %{dir: dir} do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
# Write a valid file at the .prop path, then rename to .ntms to
|
|
# simulate a file written before the rename that survived through
|
|
# a rolling deploy. The decoder must accept the old magic bytes
|
|
# embedded in the header of already-on-disk files as well, but the
|
|
# simpler check — that the file is found by extension — is what
|
|
# this test targets.
|
|
ScoresFile.write!(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 77}])
|
|
prop_path = Path.join([dir, "10000", "2026-04-14T18:00:00Z.prop"])
|
|
ntms_path = Path.join([dir, "10000", "2026-04-14T18:00:00Z.ntms"])
|
|
File.rename!(prop_path, ntms_path)
|
|
|
|
assert {:ok, payload} = ScoresFile.read(10_000, valid_time)
|
|
assert payload.band_mhz == 10_000
|
|
end
|
|
|
|
test "read_point/4 accepts a legacy .ntms file with NTMS magic", %{dir: dir} do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
ScoresFile.write!(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 88}])
|
|
prop_path = Path.join([dir, "10000", "2026-04-14T18:00:00Z.prop"])
|
|
ntms_path = Path.join([dir, "10000", "2026-04-14T18:00:00Z.ntms"])
|
|
|
|
# Flip the 4-byte magic back to the legacy "NTMS" so both the
|
|
# extension and the in-file identifier reflect the pre-rename
|
|
# state. The reader must still serve the cell.
|
|
raw = File.read!(prop_path)
|
|
<<_magic::binary-size(4), rest::binary>> = raw
|
|
File.write!(ntms_path, <<"NTMS", rest::binary>>, [:binary])
|
|
File.rm!(prop_path)
|
|
|
|
assert ScoresFile.read_point(10_000, valid_time, 25.0, -125.0) == 88
|
|
end
|
|
|
|
test "list_valid_times/1 picks up .ntms and .prop files together", %{dir: _dir} do
|
|
t1 = ~U[2026-04-14 17:00:00Z]
|
|
t2 = ~U[2026-04-14 18:00:00Z]
|
|
|
|
ScoresFile.write!(10_000, t1, [%{lat: 25.0, lon: -125.0, score: 10}])
|
|
ScoresFile.write!(10_000, t2, [%{lat: 25.0, lon: -125.0, score: 20}])
|
|
|
|
# Rename the earlier one to .ntms to simulate a mixed-extension
|
|
# directory during a rolling deploy.
|
|
band_dir =
|
|
Path.join([
|
|
Application.fetch_env!(:microwaveprop, :propagation_scores_dir),
|
|
"10000"
|
|
])
|
|
|
|
File.rename!(
|
|
Path.join(band_dir, "2026-04-14T17:00:00Z.prop"),
|
|
Path.join(band_dir, "2026-04-14T17:00:00Z.ntms")
|
|
)
|
|
|
|
times = ScoresFile.list_valid_times(10_000)
|
|
assert Enum.count_until(times, 3) == 2
|
|
assert t1 in times
|
|
assert t2 in times
|
|
end
|
|
end
|
|
|
|
describe "write!/3 + read/2 round trip" do
|
|
test "preserves the score for each scored grid point", %{dir: _dir} do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
|
|
scores = [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 25.0, lon: -124.875, score: 20},
|
|
%{lat: 25.125, lon: -125.0, score: 30}
|
|
]
|
|
|
|
ScoresFile.write!(10_000, valid_time, scores)
|
|
|
|
assert {:ok, payload} = ScoresFile.read(10_000, valid_time)
|
|
assert payload.band_mhz == 10_000
|
|
assert DateTime.compare(payload.valid_time, valid_time) == :eq
|
|
assert payload.lat_min == Grid.bounds().lat_min
|
|
assert payload.lon_min == Grid.bounds().lon_min
|
|
assert payload.step == Grid.step()
|
|
|
|
# The three scored cells we wrote come back at their grid positions.
|
|
assert score_at(payload, 25.0, -125.0) == 10
|
|
assert score_at(payload, 25.0, -124.875) == 20
|
|
assert score_at(payload, 25.125, -125.0) == 30
|
|
end
|
|
|
|
test "unscored cells read back as 255 (no-data sentinel)" do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
ScoresFile.write!(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 42}])
|
|
|
|
{:ok, payload} = ScoresFile.read(10_000, valid_time)
|
|
assert score_at(payload, 25.0, -125.0) == 42
|
|
# Any cell we didn't write should be the no-data sentinel.
|
|
assert score_at(payload, 26.5, -100.0) == 255
|
|
end
|
|
end
|
|
|
|
describe "read/2 errors" do
|
|
test "returns {:error, :enoent} when no file exists for the band/time" do
|
|
assert {:error, :enoent} = ScoresFile.read(10_000, ~U[2026-04-14 18:00:00Z])
|
|
end
|
|
|
|
test "returns {:error, :invalid_format} for a file that isn't a score grid" do
|
|
path = ScoresFile.path_for(10_000, ~U[2026-04-14 18:00:00Z])
|
|
File.mkdir_p!(Path.dirname(path))
|
|
File.write!(path, "not a score file")
|
|
|
|
assert {:error, :invalid_format} = ScoresFile.read(10_000, ~U[2026-04-14 18:00:00Z])
|
|
end
|
|
end
|
|
|
|
describe "atomic write" do
|
|
test "leaves no .tmp files behind after a successful write", %{dir: dir} do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [%{lat: 25.0, lon: -125.0, score: 1}])
|
|
tmp_files = find_files_matching(dir, ~r/\.tmp\./)
|
|
assert tmp_files == []
|
|
end
|
|
end
|
|
|
|
describe "list_valid_times/1" do
|
|
test "returns files sorted ascending for a band" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
|
|
# Unrelated band's files shouldn't leak in.
|
|
ScoresFile.write!(24_000, ~U[2026-04-14 20:00:00Z], [])
|
|
|
|
assert ScoresFile.list_valid_times(10_000) == [
|
|
~U[2026-04-14 16:00:00Z],
|
|
~U[2026-04-14 17:00:00Z],
|
|
~U[2026-04-14 18:00:00Z]
|
|
]
|
|
end
|
|
|
|
test "returns an empty list when the band dir doesn't exist" do
|
|
assert ScoresFile.list_valid_times(75_000) == []
|
|
end
|
|
|
|
test "dedupes when a valid_time exists as both `.prop` and `.ntms`" do
|
|
# Simulate the rolling rename window: the new writer just dropped a
|
|
# .prop file; the prior cycle's .ntms file for the same valid_time
|
|
# is still on NFS. A naive implementation surfaces both → the /map
|
|
# timeline renders the same chip twice.
|
|
valid_time = ~U[2026-04-21 22:00:00Z]
|
|
ScoresFile.write!(10_000, valid_time, [])
|
|
|
|
base = Application.fetch_env!(:microwaveprop, :propagation_scores_dir)
|
|
legacy = Path.join([base, "10000", "2026-04-21T22-00-00Z.ntms"])
|
|
:ok = File.cp!(ScoresFile.path_for(10_000, valid_time), legacy)
|
|
|
|
assert ScoresFile.list_valid_times(10_000) == [valid_time]
|
|
end
|
|
end
|
|
|
|
describe "latest_valid_time/0" do
|
|
test "returns the max valid_time across every band on disk" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-14 18:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
|
|
|
|
assert ScoresFile.latest_valid_time() == ~U[2026-04-14 18:00:00Z]
|
|
end
|
|
|
|
test "returns nil on an empty tree" do
|
|
assert ScoresFile.latest_valid_time() == nil
|
|
end
|
|
end
|
|
|
|
describe "read_bounds/3" do
|
|
test "returns every scored cell (no-data excluded) when bounds is nil" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 25.125, lon: -125.0, score: 20}
|
|
])
|
|
|
|
scores = ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z])
|
|
|
|
assert [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 25.125, lon: -125.0, score: 20}
|
|
] = Enum.sort_by(scores, & &1.lat)
|
|
end
|
|
|
|
test "restricts results to the bounding box when provided" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 49.875, lon: -66.0, score: 90}
|
|
])
|
|
|
|
bounds = %{"south" => 24.0, "north" => 26.0, "west" => -126.0, "east" => -124.0}
|
|
scores = ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z], bounds)
|
|
|
|
assert scores == [%{lat: 25.0, lon: -125.0, score: 10}]
|
|
end
|
|
|
|
test "returns an empty list when the file doesn't exist" do
|
|
assert ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z]) == []
|
|
end
|
|
|
|
test "merges cells from a sibling .hrdps.prop file when present" do
|
|
# Write the HRRR file via the public writer, then copy it to the
|
|
# HRDPS path. Both files now contain the same cell — read_bounds
|
|
# should return both, doubling the cell list. (Production HRDPS
|
|
# files have a different bbox; this test only verifies the merge
|
|
# plumbing routes through both paths.)
|
|
vt = ~U[2026-04-14 18:00:00Z]
|
|
ScoresFile.write!(10_000, vt, [%{lat: 25.0, lon: -125.0, score: 42}])
|
|
|
|
hrrr_path = ScoresFile.path_for(10_000, vt)
|
|
hrdps_path = ScoresFile.path_for_hrdps(10_000, vt)
|
|
File.cp!(hrrr_path, hrdps_path)
|
|
|
|
scores = ScoresFile.read_bounds(10_000, vt)
|
|
assert Enum.count_until(scores, 3) == 2
|
|
assert Enum.all?(scores, &(&1.score == 42))
|
|
end
|
|
|
|
test "returns HRDPS cells alone when only the .hrdps.prop file exists" do
|
|
vt = ~U[2026-04-14 18:00:00Z]
|
|
# Use write! to land a file at the HRRR path, then move it to
|
|
# the HRDPS path so only the HRDPS file remains.
|
|
ScoresFile.write!(10_000, vt, [%{lat: 25.0, lon: -125.0, score: 80}])
|
|
|
|
File.rename!(
|
|
ScoresFile.path_for(10_000, vt),
|
|
ScoresFile.path_for_hrdps(10_000, vt)
|
|
)
|
|
|
|
assert [%{score: 80}] = ScoresFile.read_bounds(10_000, vt)
|
|
end
|
|
end
|
|
|
|
describe "read_point/4" do
|
|
test "returns the score for a grid cell by (lat, lon) lookup" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
|
|
%{lat: 25.0, lon: -125.0, score: 42}
|
|
])
|
|
|
|
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 25.0, -125.0) == 42
|
|
end
|
|
|
|
test "returns nil for cells that weren't scored" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
|
|
%{lat: 25.0, lon: -125.0, score: 42}
|
|
])
|
|
|
|
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 40.0, -100.0) == nil
|
|
end
|
|
|
|
test "returns nil when the file is missing" do
|
|
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 25.0, -125.0) == nil
|
|
end
|
|
end
|
|
|
|
describe "HRDPS file visibility" do
|
|
test "parse_valid_time correctly extracts the ISO datetime from .hrdps.prop filenames" do
|
|
# The greedy `(.+)` in an older regex captured "<iso>.hrdps" as the
|
|
# datetime portion, causing DateTime.from_iso8601/1 to fail and
|
|
# HRDPS files to be silently skipped by list/retain/prune.
|
|
vt = ~U[2026-08-01 12:00:00Z]
|
|
ScoresFile.write!(10_000, vt, [])
|
|
hrrr_path = ScoresFile.path_for(10_000, vt)
|
|
hrdps_path = ScoresFile.path_for_hrdps(10_000, vt)
|
|
File.cp!(hrrr_path, hrdps_path)
|
|
File.rm!(hrrr_path)
|
|
|
|
# Only the .hrdps.prop file exists — it must be visible.
|
|
assert ScoresFile.list_valid_times(10_000) == [vt]
|
|
|
|
# retain_window with a window that includes vt keeps the file.
|
|
assert ScoresFile.retain_window(~U[2026-08-01 10:00:00Z], 6) == 0
|
|
assert ScoresFile.list_valid_times(10_000) == [vt]
|
|
|
|
# retain_window outside vt's window deletes it.
|
|
assert ScoresFile.retain_window(~U[2026-08-01 13:00:00Z], 6) == 1
|
|
assert ScoresFile.list_valid_times(10_000) == []
|
|
end
|
|
end
|
|
|
|
describe "retain_window/2" do
|
|
test "deletes files outside [run_time, run_time + hours] across every band" do
|
|
run_time = ~U[2026-04-14 16:00:00Z]
|
|
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 15:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-14 14:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-14 16:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-15 10:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-15 11:00:00Z], [])
|
|
|
|
# Window: 16:00 -> 10:00 next day (18 hours forward).
|
|
assert ScoresFile.retain_window(run_time, 18) == 3
|
|
|
|
assert ScoresFile.list_valid_times(10_000) == [
|
|
~U[2026-04-14 16:00:00Z],
|
|
~U[2026-04-14 17:00:00Z]
|
|
]
|
|
|
|
assert ScoresFile.list_valid_times(24_000) == [
|
|
~U[2026-04-14 16:00:00Z],
|
|
~U[2026-04-15 10:00:00Z]
|
|
]
|
|
end
|
|
|
|
test "returns 0 and leaves foreign files alone", %{dir: dir} do
|
|
run_time = ~U[2026-04-14 16:00:00Z]
|
|
stray = Path.join(dir, "some_other_file.txt")
|
|
File.write!(stray, "keep me")
|
|
|
|
assert ScoresFile.retain_window(run_time, 18) == 0
|
|
assert File.exists?(stray)
|
|
end
|
|
end
|
|
|
|
describe "prune_older_than/1" do
|
|
test "deletes files whose valid_time is strictly before the cutoff", %{dir: _dir} do
|
|
old = ~U[2026-04-14 10:00:00Z]
|
|
fresh = ~U[2026-04-14 18:00:00Z]
|
|
|
|
ScoresFile.write!(10_000, old, [%{lat: 25.0, lon: -125.0, score: 1}])
|
|
ScoresFile.write!(10_000, fresh, [%{lat: 25.0, lon: -125.0, score: 2}])
|
|
|
|
cutoff = ~U[2026-04-14 15:00:00Z]
|
|
assert ScoresFile.prune_older_than(cutoff) == 1
|
|
|
|
assert {:error, :enoent} = ScoresFile.read(10_000, old)
|
|
assert {:ok, _} = ScoresFile.read(10_000, fresh)
|
|
end
|
|
|
|
test "leaves foreign files in the scores dir alone", %{dir: dir} do
|
|
stray = Path.join(dir, "some_other_file.txt")
|
|
File.write!(stray, "do not touch")
|
|
|
|
assert ScoresFile.prune_older_than(~U[2099-01-01 00:00:00Z]) == 0
|
|
assert File.exists?(stray)
|
|
end
|
|
end
|
|
|
|
# Look up the score for (lat, lon) in the binary returned by read/2.
|
|
defp score_at(payload, lat, lon) do
|
|
row = round((lat - payload.lat_min) / payload.step)
|
|
col = round((lon - payload.lon_min) / payload.step)
|
|
idx = row * payload.n_cols + col
|
|
:binary.at(payload.scores, idx)
|
|
end
|
|
|
|
defp find_files_matching(dir, pattern) do
|
|
dir
|
|
|> Path.join("**/*")
|
|
|> Path.wildcard()
|
|
|> Enum.filter(fn path -> File.regular?(path) and Regex.match?(pattern, path) end)
|
|
end
|
|
end
|