prop/test/microwaveprop/propagation/scores_file_test.exs
Graham McIntire 07ffcf52d7
Flip map read path to ScoresFile, stop writing grid HRRR profiles
Read-side cutover for the binary scores store and a companion
cleanup that removes the biggest remaining DB write from the hot
path.

Propagation.scores_at/3, available_valid_times/1, latest_valid_time/0,
latest_valid_time/1, earliest_valid_time/1, point_detail/4, and
point_forecast/3 all now prefer ScoresFile and fall back to the
propagation_scores table when a file is missing. The map render
path reads from /data/scores first; Postgres stays as a safety
net while dual-write is on. point_detail still pulls factors from
Postgres (analysis-hour rows only) and coalesces nil to an empty
map so the JS popup iterates cleanly.

replace_scores/2 is now gated by a postgres_writes_enabled? flag
(runtime env MICROWAVEPROP_SCORES_POSTGRES=false, or the
:propagation_scores_postgres app env key) so the binary-only path
can be benchmarked locally without the DB insert. Default stays
true.

PropagationGridWorker no longer calls store_hrrr_profiles —
persisting 92k grid rows × 19 forecast hours of JSONB profiles
was ~12 min of wall time per chain for a table only
AsosAdjustmentWorker read from. Per-contact HRRR enrichment
through HrrrFetchWorker still writes its own (is_grid_point:
false) rows. AsosAdjustmentWorker is disabled in all three cron
configs since its data source is gone.

DataCase resets the scores tree between tests so per-test
ScoresFile writes don't leak across cases, and ScoresFileTest
switches to async: false because it mutates the global
:propagation_scores_dir env.
2026-04-14 14:50:43 -05:00

226 lines
7.7 KiB
Elixir

defmodule Microwaveprop.Propagation.ScoresFileTest do
# async: false — tests mutate the global Application env to redirect
# the scores dir to a private tmp tree. Running these in parallel
# would have concurrent tests trample each other's directory setting.
use ExUnit.Case, async: false
alias Microwaveprop.Propagation.Grid
alias Microwaveprop.Propagation.ScoresFile
setup do
dir =
Path.join(
System.tmp_dir!(),
"scores_file_test_#{System.unique_integer([:positive])}"
)
File.mkdir_p!(dir)
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
on_exit(fn ->
File.rm_rf!(dir)
if prev do
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
else
Application.delete_env(:microwaveprop, :propagation_scores_dir)
end
end)
%{dir: dir}
end
describe "path_for/2" do
test "nests files under base_dir/{band_mhz}/ with an ISO-8601 valid_time", %{dir: dir} do
path = ScoresFile.path_for(10_000, ~U[2026-04-14 18:00:00Z])
assert path == Path.join([dir, "10000", "2026-04-14T18:00:00Z.ntms"])
end
end
describe "write!/3 + read/2 round trip" do
test "preserves the score for each scored grid point", %{dir: _dir} do
valid_time = ~U[2026-04-14 18:00:00Z]
scores = [
%{lat: 25.0, lon: -125.0, score: 10},
%{lat: 25.0, lon: -124.875, score: 20},
%{lat: 25.125, lon: -125.0, score: 30}
]
ScoresFile.write!(10_000, valid_time, scores)
assert {:ok, payload} = ScoresFile.read(10_000, valid_time)
assert payload.band_mhz == 10_000
assert DateTime.compare(payload.valid_time, valid_time) == :eq
assert payload.lat_min == Grid.bounds().lat_min
assert payload.lon_min == Grid.bounds().lon_min
assert payload.step == Grid.step()
# The three scored cells we wrote come back at their grid positions.
assert score_at(payload, 25.0, -125.0) == 10
assert score_at(payload, 25.0, -124.875) == 20
assert score_at(payload, 25.125, -125.0) == 30
end
test "unscored cells read back as 255 (no-data sentinel)" do
valid_time = ~U[2026-04-14 18:00:00Z]
ScoresFile.write!(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 42}])
{:ok, payload} = ScoresFile.read(10_000, valid_time)
assert score_at(payload, 25.0, -125.0) == 42
# Any cell we didn't write should be the no-data sentinel.
assert score_at(payload, 26.5, -100.0) == 255
end
end
describe "read/2 errors" do
test "returns {:error, :enoent} when no file exists for the band/time" do
assert {:error, :enoent} = ScoresFile.read(10_000, ~U[2026-04-14 18:00:00Z])
end
test "returns {:error, :invalid_format} for a file that isn't a score grid" do
path = ScoresFile.path_for(10_000, ~U[2026-04-14 18:00:00Z])
File.mkdir_p!(Path.dirname(path))
File.write!(path, "not a score file")
assert {:error, :invalid_format} = ScoresFile.read(10_000, ~U[2026-04-14 18:00:00Z])
end
end
describe "atomic write" do
test "leaves no .tmp files behind after a successful write", %{dir: dir} do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [%{lat: 25.0, lon: -125.0, score: 1}])
tmp_files = find_files_matching(dir, ~r/\.tmp\./)
assert tmp_files == []
end
end
describe "list_valid_times/1" do
test "returns files sorted ascending for a band" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [])
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
# Unrelated band's files shouldn't leak in.
ScoresFile.write!(24_000, ~U[2026-04-14 20:00:00Z], [])
assert ScoresFile.list_valid_times(10_000) == [
~U[2026-04-14 16:00:00Z],
~U[2026-04-14 17:00:00Z],
~U[2026-04-14 18:00:00Z]
]
end
test "returns an empty list when the band dir doesn't exist" do
assert ScoresFile.list_valid_times(75_000) == []
end
end
describe "latest_valid_time/0" do
test "returns the max valid_time across every band on disk" do
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
ScoresFile.write!(24_000, ~U[2026-04-14 18:00:00Z], [])
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
assert ScoresFile.latest_valid_time() == ~U[2026-04-14 18:00:00Z]
end
test "returns nil on an empty tree" do
assert ScoresFile.latest_valid_time() == nil
end
end
describe "read_bounds/3" do
test "returns every scored cell (no-data excluded) when bounds is nil" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
%{lat: 25.0, lon: -125.0, score: 10},
%{lat: 25.125, lon: -125.0, score: 20}
])
scores = ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z])
assert [
%{lat: 25.0, lon: -125.0, score: 10},
%{lat: 25.125, lon: -125.0, score: 20}
] = Enum.sort_by(scores, & &1.lat)
end
test "restricts results to the bounding box when provided" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
%{lat: 25.0, lon: -125.0, score: 10},
%{lat: 49.875, lon: -66.0, score: 90}
])
bounds = %{"south" => 24.0, "north" => 26.0, "west" => -126.0, "east" => -124.0}
scores = ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z], bounds)
assert scores == [%{lat: 25.0, lon: -125.0, score: 10}]
end
test "returns an empty list when the file doesn't exist" do
assert ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z]) == []
end
end
describe "read_point/4" do
test "returns the score for a grid cell by (lat, lon) lookup" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
%{lat: 25.0, lon: -125.0, score: 42}
])
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 25.0, -125.0) == 42
end
test "returns nil for cells that weren't scored" do
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
%{lat: 25.0, lon: -125.0, score: 42}
])
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 40.0, -100.0) == nil
end
test "returns nil when the file is missing" do
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 25.0, -125.0) == nil
end
end
describe "prune_older_than/1" do
test "deletes files whose valid_time is strictly before the cutoff", %{dir: _dir} do
old = ~U[2026-04-14 10:00:00Z]
fresh = ~U[2026-04-14 18:00:00Z]
ScoresFile.write!(10_000, old, [%{lat: 25.0, lon: -125.0, score: 1}])
ScoresFile.write!(10_000, fresh, [%{lat: 25.0, lon: -125.0, score: 2}])
cutoff = ~U[2026-04-14 15:00:00Z]
assert ScoresFile.prune_older_than(cutoff) == 1
assert {:error, :enoent} = ScoresFile.read(10_000, old)
assert {:ok, _} = ScoresFile.read(10_000, fresh)
end
test "leaves foreign files in the scores dir alone", %{dir: dir} do
stray = Path.join(dir, "some_other_file.txt")
File.write!(stray, "do not touch")
assert ScoresFile.prune_older_than(~U[2099-01-01 00:00:00Z]) == 0
assert File.exists?(stray)
end
end
# Look up the score for (lat, lon) in the binary returned by read/2.
defp score_at(payload, lat, lon) do
row = round((lat - payload.lat_min) / payload.step)
col = round((lon - payload.lon_min) / payload.step)
idx = row * payload.n_cols + col
:binary.at(payload.scores, idx)
end
defp find_files_matching(dir, pattern) do
dir
|> Path.join("**/*")
|> Path.wildcard()
|> Enum.filter(fn path -> File.regular?(path) and Regex.match?(pattern, path) end)
end
end