Graham McIntire
e6952a42c8
Flip the cron from every-3-hours to every-hour now that a full f00-f18 chain runs in ~45-60 min instead of ~170 min. With the :propagation queue's concurrency-of-2, a slow chain won't block the next one — they interleave, and the file store's last-writer -wins semantics give the newer run_time's analysis data naturally. Add ScoresFile.retain_window/2 that deletes any file whose valid_time falls outside [run_time, run_time + 18h]. Call it at fh=18 of the chain so leftover files from the previous chain (specifically the old f00 whose valid_time the new chain doesn't overwrite because it advanced by one hour) are discarded immediately instead of waiting for the 2h prune cron. The step transition path is extracted to handle_step_transition/2 to keep the nesting under credo's limit and make the final-step logic easier to read.
262 lines
9 KiB
Elixir
262 lines
9 KiB
Elixir
defmodule Microwaveprop.Propagation.ScoresFileTest do
|
|
# async: false — tests mutate the global Application env to redirect
|
|
# the scores dir to a private tmp tree. Running these in parallel
|
|
# would have concurrent tests trample each other's directory setting.
|
|
use ExUnit.Case, async: false
|
|
|
|
alias Microwaveprop.Propagation.Grid
|
|
alias Microwaveprop.Propagation.ScoresFile
|
|
|
|
setup do
|
|
dir =
|
|
Path.join(
|
|
System.tmp_dir!(),
|
|
"scores_file_test_#{System.unique_integer([:positive])}"
|
|
)
|
|
|
|
File.mkdir_p!(dir)
|
|
|
|
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
|
|
|
|
on_exit(fn ->
|
|
File.rm_rf!(dir)
|
|
|
|
if prev do
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
|
|
else
|
|
Application.delete_env(:microwaveprop, :propagation_scores_dir)
|
|
end
|
|
end)
|
|
|
|
%{dir: dir}
|
|
end
|
|
|
|
describe "path_for/2" do
|
|
test "nests files under base_dir/{band_mhz}/ with an ISO-8601 valid_time", %{dir: dir} do
|
|
path = ScoresFile.path_for(10_000, ~U[2026-04-14 18:00:00Z])
|
|
assert path == Path.join([dir, "10000", "2026-04-14T18:00:00Z.ntms"])
|
|
end
|
|
end
|
|
|
|
describe "write!/3 + read/2 round trip" do
|
|
test "preserves the score for each scored grid point", %{dir: _dir} do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
|
|
scores = [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 25.0, lon: -124.875, score: 20},
|
|
%{lat: 25.125, lon: -125.0, score: 30}
|
|
]
|
|
|
|
ScoresFile.write!(10_000, valid_time, scores)
|
|
|
|
assert {:ok, payload} = ScoresFile.read(10_000, valid_time)
|
|
assert payload.band_mhz == 10_000
|
|
assert DateTime.compare(payload.valid_time, valid_time) == :eq
|
|
assert payload.lat_min == Grid.bounds().lat_min
|
|
assert payload.lon_min == Grid.bounds().lon_min
|
|
assert payload.step == Grid.step()
|
|
|
|
# The three scored cells we wrote come back at their grid positions.
|
|
assert score_at(payload, 25.0, -125.0) == 10
|
|
assert score_at(payload, 25.0, -124.875) == 20
|
|
assert score_at(payload, 25.125, -125.0) == 30
|
|
end
|
|
|
|
test "unscored cells read back as 255 (no-data sentinel)" do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
ScoresFile.write!(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 42}])
|
|
|
|
{:ok, payload} = ScoresFile.read(10_000, valid_time)
|
|
assert score_at(payload, 25.0, -125.0) == 42
|
|
# Any cell we didn't write should be the no-data sentinel.
|
|
assert score_at(payload, 26.5, -100.0) == 255
|
|
end
|
|
end
|
|
|
|
describe "read/2 errors" do
|
|
test "returns {:error, :enoent} when no file exists for the band/time" do
|
|
assert {:error, :enoent} = ScoresFile.read(10_000, ~U[2026-04-14 18:00:00Z])
|
|
end
|
|
|
|
test "returns {:error, :invalid_format} for a file that isn't a score grid" do
|
|
path = ScoresFile.path_for(10_000, ~U[2026-04-14 18:00:00Z])
|
|
File.mkdir_p!(Path.dirname(path))
|
|
File.write!(path, "not a score file")
|
|
|
|
assert {:error, :invalid_format} = ScoresFile.read(10_000, ~U[2026-04-14 18:00:00Z])
|
|
end
|
|
end
|
|
|
|
describe "atomic write" do
|
|
test "leaves no .tmp files behind after a successful write", %{dir: dir} do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [%{lat: 25.0, lon: -125.0, score: 1}])
|
|
tmp_files = find_files_matching(dir, ~r/\.tmp\./)
|
|
assert tmp_files == []
|
|
end
|
|
end
|
|
|
|
describe "list_valid_times/1" do
|
|
test "returns files sorted ascending for a band" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
|
|
# Unrelated band's files shouldn't leak in.
|
|
ScoresFile.write!(24_000, ~U[2026-04-14 20:00:00Z], [])
|
|
|
|
assert ScoresFile.list_valid_times(10_000) == [
|
|
~U[2026-04-14 16:00:00Z],
|
|
~U[2026-04-14 17:00:00Z],
|
|
~U[2026-04-14 18:00:00Z]
|
|
]
|
|
end
|
|
|
|
test "returns an empty list when the band dir doesn't exist" do
|
|
assert ScoresFile.list_valid_times(75_000) == []
|
|
end
|
|
end
|
|
|
|
describe "latest_valid_time/0" do
|
|
test "returns the max valid_time across every band on disk" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-14 18:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
|
|
|
|
assert ScoresFile.latest_valid_time() == ~U[2026-04-14 18:00:00Z]
|
|
end
|
|
|
|
test "returns nil on an empty tree" do
|
|
assert ScoresFile.latest_valid_time() == nil
|
|
end
|
|
end
|
|
|
|
describe "read_bounds/3" do
|
|
test "returns every scored cell (no-data excluded) when bounds is nil" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 25.125, lon: -125.0, score: 20}
|
|
])
|
|
|
|
scores = ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z])
|
|
|
|
assert [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 25.125, lon: -125.0, score: 20}
|
|
] = Enum.sort_by(scores, & &1.lat)
|
|
end
|
|
|
|
test "restricts results to the bounding box when provided" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 49.875, lon: -66.0, score: 90}
|
|
])
|
|
|
|
bounds = %{"south" => 24.0, "north" => 26.0, "west" => -126.0, "east" => -124.0}
|
|
scores = ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z], bounds)
|
|
|
|
assert scores == [%{lat: 25.0, lon: -125.0, score: 10}]
|
|
end
|
|
|
|
test "returns an empty list when the file doesn't exist" do
|
|
assert ScoresFile.read_bounds(10_000, ~U[2026-04-14 18:00:00Z]) == []
|
|
end
|
|
end
|
|
|
|
describe "read_point/4" do
|
|
test "returns the score for a grid cell by (lat, lon) lookup" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
|
|
%{lat: 25.0, lon: -125.0, score: 42}
|
|
])
|
|
|
|
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 25.0, -125.0) == 42
|
|
end
|
|
|
|
test "returns nil for cells that weren't scored" do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [
|
|
%{lat: 25.0, lon: -125.0, score: 42}
|
|
])
|
|
|
|
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 40.0, -100.0) == nil
|
|
end
|
|
|
|
test "returns nil when the file is missing" do
|
|
assert ScoresFile.read_point(10_000, ~U[2026-04-14 18:00:00Z], 25.0, -125.0) == nil
|
|
end
|
|
end
|
|
|
|
describe "retain_window/2" do
|
|
test "deletes files outside [run_time, run_time + hours] across every band" do
|
|
run_time = ~U[2026-04-14 16:00:00Z]
|
|
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 15:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 16:00:00Z], [])
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 17:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-14 14:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-14 16:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-15 10:00:00Z], [])
|
|
ScoresFile.write!(24_000, ~U[2026-04-15 11:00:00Z], [])
|
|
|
|
# Window: 16:00 -> 10:00 next day (18 hours forward).
|
|
assert ScoresFile.retain_window(run_time, 18) == 3
|
|
|
|
assert ScoresFile.list_valid_times(10_000) == [
|
|
~U[2026-04-14 16:00:00Z],
|
|
~U[2026-04-14 17:00:00Z]
|
|
]
|
|
|
|
assert ScoresFile.list_valid_times(24_000) == [
|
|
~U[2026-04-14 16:00:00Z],
|
|
~U[2026-04-15 10:00:00Z]
|
|
]
|
|
end
|
|
|
|
test "returns 0 and leaves foreign files alone", %{dir: dir} do
|
|
run_time = ~U[2026-04-14 16:00:00Z]
|
|
stray = Path.join(dir, "some_other_file.txt")
|
|
File.write!(stray, "keep me")
|
|
|
|
assert ScoresFile.retain_window(run_time, 18) == 0
|
|
assert File.exists?(stray)
|
|
end
|
|
end
|
|
|
|
describe "prune_older_than/1" do
|
|
test "deletes files whose valid_time is strictly before the cutoff", %{dir: _dir} do
|
|
old = ~U[2026-04-14 10:00:00Z]
|
|
fresh = ~U[2026-04-14 18:00:00Z]
|
|
|
|
ScoresFile.write!(10_000, old, [%{lat: 25.0, lon: -125.0, score: 1}])
|
|
ScoresFile.write!(10_000, fresh, [%{lat: 25.0, lon: -125.0, score: 2}])
|
|
|
|
cutoff = ~U[2026-04-14 15:00:00Z]
|
|
assert ScoresFile.prune_older_than(cutoff) == 1
|
|
|
|
assert {:error, :enoent} = ScoresFile.read(10_000, old)
|
|
assert {:ok, _} = ScoresFile.read(10_000, fresh)
|
|
end
|
|
|
|
test "leaves foreign files in the scores dir alone", %{dir: dir} do
|
|
stray = Path.join(dir, "some_other_file.txt")
|
|
File.write!(stray, "do not touch")
|
|
|
|
assert ScoresFile.prune_older_than(~U[2099-01-01 00:00:00Z]) == 0
|
|
assert File.exists?(stray)
|
|
end
|
|
end
|
|
|
|
# Look up the score for (lat, lon) in the binary returned by read/2.
|
|
defp score_at(payload, lat, lon) do
|
|
row = round((lat - payload.lat_min) / payload.step)
|
|
col = round((lon - payload.lon_min) / payload.step)
|
|
idx = row * payload.n_cols + col
|
|
:binary.at(payload.scores, idx)
|
|
end
|
|
|
|
defp find_files_matching(dir, pattern) do
|
|
dir
|
|
|> Path.join("**/*")
|
|
|> Path.wildcard()
|
|
|> Enum.filter(fn path -> File.regular?(path) and Regex.match?(pattern, path) end)
|
|
end
|
|
end
|