First step of the disk-backed scores migration from the DuckDB plan doc. Ended up shipping the raw-binary variant instead of Parquet because the data is disposable after ~2h — the ecosystem benefits of Parquet only pay off for long-lived datasets, and the binary path has zero new dependencies. Microwaveprop.Propagation.ScoresFile writes one file per (band_mhz, valid_time) tuple under the configured scores dir, default /data/scores in prod and priv/dev_scores in dev. Layout is a 33-byte header plus a dense n_rows × n_cols uint8 array (255 is the no-data sentinel). The whole CONUS grid serializes to ~93 KB per band, and writes use the temp-then-rename pattern so NFSv4 concurrent readers never see a partial file. Propagation.replace_scores/2 now materializes the score stream once and dual-writes: Postgres on the primary path, then one ScoresFile per band as a best-effort follow-up (any file error is logged but doesn't fail the DB write, so we can verify the file path in prod before cutting readers over). Propagation.prune_old_scores/0 also clears expired score files so the existing 15-minute prune cron covers both storage layers. Dev configuration points at priv/dev_scores/, added to .gitignore. Test configuration points at a per-run tmp directory.
135 lines
4.5 KiB
Elixir
135 lines
4.5 KiB
Elixir
defmodule Microwaveprop.Propagation.ScoresFileTest do
|
|
use ExUnit.Case, async: true
|
|
|
|
alias Microwaveprop.Propagation.Grid
|
|
alias Microwaveprop.Propagation.ScoresFile
|
|
|
|
setup do
|
|
dir =
|
|
Path.join(
|
|
System.tmp_dir!(),
|
|
"scores_file_test_#{System.unique_integer([:positive])}"
|
|
)
|
|
|
|
File.mkdir_p!(dir)
|
|
|
|
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
|
|
|
|
on_exit(fn ->
|
|
File.rm_rf!(dir)
|
|
|
|
if prev do
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
|
|
else
|
|
Application.delete_env(:microwaveprop, :propagation_scores_dir)
|
|
end
|
|
end)
|
|
|
|
%{dir: dir}
|
|
end
|
|
|
|
describe "path_for/2" do
|
|
test "nests files under base_dir/{band_mhz}/ with an ISO-8601 valid_time", %{dir: dir} do
|
|
path = ScoresFile.path_for(10_000, ~U[2026-04-14 18:00:00Z])
|
|
assert path == Path.join([dir, "10000", "2026-04-14T18:00:00Z.ntms"])
|
|
end
|
|
end
|
|
|
|
describe "write!/3 + read/2 round trip" do
|
|
test "preserves the score for each scored grid point", %{dir: _dir} do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
|
|
scores = [
|
|
%{lat: 25.0, lon: -125.0, score: 10},
|
|
%{lat: 25.0, lon: -124.875, score: 20},
|
|
%{lat: 25.125, lon: -125.0, score: 30}
|
|
]
|
|
|
|
ScoresFile.write!(10_000, valid_time, scores)
|
|
|
|
assert {:ok, payload} = ScoresFile.read(10_000, valid_time)
|
|
assert payload.band_mhz == 10_000
|
|
assert DateTime.compare(payload.valid_time, valid_time) == :eq
|
|
assert payload.lat_min == Grid.bounds().lat_min
|
|
assert payload.lon_min == Grid.bounds().lon_min
|
|
assert payload.step == Grid.step()
|
|
|
|
# The three scored cells we wrote come back at their grid positions.
|
|
assert score_at(payload, 25.0, -125.0) == 10
|
|
assert score_at(payload, 25.0, -124.875) == 20
|
|
assert score_at(payload, 25.125, -125.0) == 30
|
|
end
|
|
|
|
test "unscored cells read back as 255 (no-data sentinel)" do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
ScoresFile.write!(10_000, valid_time, [%{lat: 25.0, lon: -125.0, score: 42}])
|
|
|
|
{:ok, payload} = ScoresFile.read(10_000, valid_time)
|
|
assert score_at(payload, 25.0, -125.0) == 42
|
|
# Any cell we didn't write should be the no-data sentinel.
|
|
assert score_at(payload, 26.5, -100.0) == 255
|
|
end
|
|
end
|
|
|
|
describe "read/2 errors" do
|
|
test "returns {:error, :enoent} when no file exists for the band/time" do
|
|
assert {:error, :enoent} = ScoresFile.read(10_000, ~U[2026-04-14 18:00:00Z])
|
|
end
|
|
|
|
test "returns {:error, :invalid_format} for a file that isn't a score grid" do
|
|
path = ScoresFile.path_for(10_000, ~U[2026-04-14 18:00:00Z])
|
|
File.mkdir_p!(Path.dirname(path))
|
|
File.write!(path, "not a score file")
|
|
|
|
assert {:error, :invalid_format} = ScoresFile.read(10_000, ~U[2026-04-14 18:00:00Z])
|
|
end
|
|
end
|
|
|
|
describe "atomic write" do
|
|
test "leaves no .tmp files behind after a successful write", %{dir: dir} do
|
|
ScoresFile.write!(10_000, ~U[2026-04-14 18:00:00Z], [%{lat: 25.0, lon: -125.0, score: 1}])
|
|
tmp_files = find_files_matching(dir, ~r/\.tmp\./)
|
|
assert tmp_files == []
|
|
end
|
|
end
|
|
|
|
describe "prune_older_than/1" do
|
|
test "deletes files whose valid_time is strictly before the cutoff", %{dir: _dir} do
|
|
old = ~U[2026-04-14 10:00:00Z]
|
|
fresh = ~U[2026-04-14 18:00:00Z]
|
|
|
|
ScoresFile.write!(10_000, old, [%{lat: 25.0, lon: -125.0, score: 1}])
|
|
ScoresFile.write!(10_000, fresh, [%{lat: 25.0, lon: -125.0, score: 2}])
|
|
|
|
cutoff = ~U[2026-04-14 15:00:00Z]
|
|
assert ScoresFile.prune_older_than(cutoff) == 1
|
|
|
|
assert {:error, :enoent} = ScoresFile.read(10_000, old)
|
|
assert {:ok, _} = ScoresFile.read(10_000, fresh)
|
|
end
|
|
|
|
test "leaves foreign files in the scores dir alone", %{dir: dir} do
|
|
stray = Path.join(dir, "some_other_file.txt")
|
|
File.write!(stray, "do not touch")
|
|
|
|
assert ScoresFile.prune_older_than(~U[2099-01-01 00:00:00Z]) == 0
|
|
assert File.exists?(stray)
|
|
end
|
|
end
|
|
|
|
# Look up the score for (lat, lon) in the binary returned by read/2.
|
|
defp score_at(payload, lat, lon) do
|
|
row = round((lat - payload.lat_min) / payload.step)
|
|
col = round((lon - payload.lon_min) / payload.step)
|
|
idx = row * payload.n_cols + col
|
|
:binary.at(payload.scores, idx)
|
|
end
|
|
|
|
defp find_files_matching(dir, pattern) do
|
|
dir
|
|
|> Path.join("**/*")
|
|
|> Path.wildcard()
|
|
|> Enum.filter(fn path -> File.regular?(path) and Regex.match?(pattern, path) end)
|
|
end
|
|
end
|