Graham McIntire
d41ced5850
Phase 3 Stream A: Rust's prop-grid-rs writes f00 profile files as
MessagePack (c4f309c). Elixir needs to read both formats during the
cutover window and indefinitely afterward — .mp.gz wins when both
exist.
Reader:
- path_for/1 stays on .etf.gz (Elixir legacy writer)
- mp_path_for/1 returns the .mp.gz sibling Rust lands
- read/1 prefers .mp.gz, falls back to .etf.gz, :enoent if neither
- decode_mp_body handles the top-level {v, valid_time, cells} shape
- normalize_profile atomizes a whitelist of known keys (surface_*,
native_min_gradient, ducts[], profile[] levels, ...) and leaves
everything else as strings so no String.to_atom bomb is possible
- Directory scan picks up both extensions so retain_window and
prune_older_than also work uniformly; list_valid_times dedupes
by timestamp
Adds msgpax ~> 2.4 dependency. 4 new tests cover format preference,
key atomization, :enoent path, and dedupe.
274 lines
9.1 KiB
Elixir
274 lines
9.1 KiB
Elixir
defmodule Microwaveprop.Propagation.ProfilesFileTest do
|
|
# async: false — tests mutate the global Application env to redirect
|
|
# the scores dir to a private tmp tree.
|
|
use ExUnit.Case, async: false
|
|
|
|
alias Microwaveprop.Propagation.ProfilesFile
|
|
|
|
setup do
|
|
dir =
|
|
Path.join(
|
|
System.tmp_dir!(),
|
|
"profiles_file_test_#{System.unique_integer([:positive])}"
|
|
)
|
|
|
|
File.mkdir_p!(dir)
|
|
|
|
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
|
|
|
|
on_exit(fn ->
|
|
File.rm_rf!(dir)
|
|
|
|
if prev do
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
|
|
else
|
|
Application.delete_env(:microwaveprop, :propagation_scores_dir)
|
|
end
|
|
end)
|
|
|
|
%{dir: dir}
|
|
end
|
|
|
|
describe "path_for/1" do
|
|
test "nests under base_dir/profiles/<iso>.etf.gz", %{dir: dir} do
|
|
assert ProfilesFile.path_for(~U[2026-04-14 18:00:00Z]) ==
|
|
Path.join([dir, "profiles", "2026-04-14T18:00:00Z.etf.gz"])
|
|
end
|
|
end
|
|
|
|
describe "read/1" do
|
|
test "returns the full grid_data map round-tripped through ETF" do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
|
|
grid_data = %{
|
|
{32.75, -97.125} => %{surface_temp_c: 25.0, surface_dewpoint_c: 15.0},
|
|
{33.0, -97.0} => %{surface_temp_c: 26.0}
|
|
}
|
|
|
|
ProfilesFile.write!(valid_time, grid_data)
|
|
|
|
assert {:ok, round_tripped} = ProfilesFile.read(valid_time)
|
|
assert round_tripped[{32.75, -97.125}][:surface_temp_c] == 25.0
|
|
assert round_tripped[{33.0, -97.0}][:surface_temp_c] == 26.0
|
|
end
|
|
|
|
test "returns :enoent when no file exists" do
|
|
assert ProfilesFile.read(~U[2026-04-14 18:00:00Z]) == {:error, :enoent}
|
|
end
|
|
|
|
test "round-trips the full production data shape including duct fields" do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
|
|
grid_data = %{
|
|
{32.75, -97.125} => %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 15.0,
|
|
surface_pressure_mb: 1013.25,
|
|
hpbl_m: 750.0,
|
|
pwat_mm: 32.5,
|
|
run_time: ~U[2026-04-14 17:00:00Z],
|
|
precip_mm: 0.0,
|
|
wind_u: 3.2,
|
|
wind_v: -1.7,
|
|
cloud_cover_pct: 25.0,
|
|
native_min_gradient: -78.4,
|
|
best_duct_freq_ghz: 24.0,
|
|
duct_count: 1,
|
|
max_duct_thickness_m: 200.0,
|
|
ducts: [
|
|
%{base_m: 200.0, top_m: 400.0, thickness_m: 200.0, min_freq_ghz: 10.0}
|
|
],
|
|
profile: [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 15.0, "hght" => 50.0}
|
|
]
|
|
}
|
|
}
|
|
|
|
ProfilesFile.write!(valid_time, grid_data)
|
|
|
|
assert {:ok, round_tripped} = ProfilesFile.read(valid_time)
|
|
|
|
point = round_tripped[{32.75, -97.125}]
|
|
assert point[:surface_temp_c] == 25.0
|
|
assert point[:run_time] == ~U[2026-04-14 17:00:00Z]
|
|
assert [duct] = point[:ducts]
|
|
assert duct[:base_m] == 200.0
|
|
assert duct[:thickness_m] == 200.0
|
|
assert duct[:min_freq_ghz] == 10.0
|
|
end
|
|
end
|
|
|
|
describe "list_valid_times/0" do
|
|
test "returns all persisted valid_times sorted ascending" do
|
|
assert ProfilesFile.list_valid_times() == []
|
|
|
|
for vt <- [~U[2026-04-14 18:00:00Z], ~U[2026-04-14 10:00:00Z], ~U[2026-04-14 14:00:00Z]] do
|
|
ProfilesFile.write!(vt, %{{25.0, -125.0} => %{surface_temp_c: 1.0}})
|
|
end
|
|
|
|
assert ProfilesFile.list_valid_times() == [
|
|
~U[2026-04-14 10:00:00Z],
|
|
~U[2026-04-14 14:00:00Z],
|
|
~U[2026-04-14 18:00:00Z]
|
|
]
|
|
end
|
|
end
|
|
|
|
describe "latest_valid_time/0" do
|
|
test "returns the most recent persisted valid_time" do
|
|
assert ProfilesFile.latest_valid_time() == nil
|
|
|
|
for vt <- [~U[2026-04-14 10:00:00Z], ~U[2026-04-14 18:00:00Z], ~U[2026-04-14 14:00:00Z]] do
|
|
ProfilesFile.write!(vt, %{{25.0, -125.0} => %{surface_temp_c: 1.0}})
|
|
end
|
|
|
|
assert ProfilesFile.latest_valid_time() == ~U[2026-04-14 18:00:00Z]
|
|
end
|
|
end
|
|
|
|
describe "write!/2 + read_point/3" do
|
|
test "round-trips a point and snaps lat/lon to the 0.125° grid" do
|
|
valid_time = ~U[2026-04-14 18:00:00Z]
|
|
|
|
grid_data = %{
|
|
{32.75, -97.125} => %{surface_temp_c: 25.0, surface_dewpoint_c: 15.0, wind_u: 3.0, wind_v: -1.0},
|
|
{25.0, -125.0} => %{surface_temp_c: 10.0}
|
|
}
|
|
|
|
ProfilesFile.write!(valid_time, grid_data)
|
|
|
|
# Exact match
|
|
assert %{surface_temp_c: 25.0, wind_u: 3.0} =
|
|
ProfilesFile.read_point(valid_time, 32.75, -97.125)
|
|
|
|
# Nudged within snap tolerance still resolves to the same cell
|
|
assert %{surface_temp_c: 25.0} =
|
|
ProfilesFile.read_point(valid_time, 32.7, -97.1)
|
|
|
|
# A point with no stored profile returns nil
|
|
assert ProfilesFile.read_point(valid_time, 40.0, -80.0) == nil
|
|
end
|
|
|
|
test "returns nil for a missing valid_time" do
|
|
assert ProfilesFile.read_point(~U[2026-04-14 18:00:00Z], 32.75, -97.125) == nil
|
|
end
|
|
end
|
|
|
|
describe "prune_older_than/1" do
|
|
test "deletes files with valid_time strictly before the cutoff" do
|
|
old = ~U[2026-04-14 10:00:00Z]
|
|
keep = ~U[2026-04-14 18:00:00Z]
|
|
|
|
ProfilesFile.write!(old, %{{25.0, -125.0} => %{surface_temp_c: 1.0}})
|
|
ProfilesFile.write!(keep, %{{25.0, -125.0} => %{surface_temp_c: 2.0}})
|
|
|
|
assert 1 == ProfilesFile.prune_older_than(~U[2026-04-14 12:00:00Z])
|
|
assert ProfilesFile.read_point(old, 25.0, -125.0) == nil
|
|
assert %{surface_temp_c: 2.0} = ProfilesFile.read_point(keep, 25.0, -125.0)
|
|
end
|
|
end
|
|
|
|
describe "retain_window/2" do
|
|
test "keeps only files whose valid_time is inside [run_time, run_time + max_fh * 3600]" do
|
|
run_time = ~U[2026-04-14 18:00:00Z]
|
|
|
|
inside_f00 = ~U[2026-04-14 18:00:00Z]
|
|
inside_f10 = ~U[2026-04-15 04:00:00Z]
|
|
after_window = ~U[2026-04-15 13:00:00Z]
|
|
before_window = ~U[2026-04-14 10:00:00Z]
|
|
|
|
for vt <- [inside_f00, inside_f10, after_window, before_window] do
|
|
ProfilesFile.write!(vt, %{{25.0, -125.0} => %{surface_temp_c: 1.0}})
|
|
end
|
|
|
|
assert 2 == ProfilesFile.retain_window(run_time, 18)
|
|
assert %{surface_temp_c: 1.0} = ProfilesFile.read_point(inside_f00, 25.0, -125.0)
|
|
assert %{surface_temp_c: 1.0} = ProfilesFile.read_point(inside_f10, 25.0, -125.0)
|
|
assert ProfilesFile.read_point(after_window, 25.0, -125.0) == nil
|
|
assert ProfilesFile.read_point(before_window, 25.0, -125.0) == nil
|
|
end
|
|
end
|
|
|
|
describe "MessagePack dual-format reader" do
|
|
test "read/1 prefers .mp.gz over .etf.gz when both exist", %{dir: dir} do
|
|
vt = ~U[2026-04-19 14:00:00Z]
|
|
|
|
# Elixir writes the legacy ETF path.
|
|
ProfilesFile.write!(vt, %{{32.9, -97.0} => %{surface_temp_c: 1.0}})
|
|
|
|
# Rust would land a .mp.gz sibling; simulate it.
|
|
mp_body = %{
|
|
"v" => 1,
|
|
"valid_time" => "2026-04-19T14:00:00Z",
|
|
"cells" => [
|
|
%{"lat" => 32.9, "lon" => -97.0, "profile" => %{"surface_temp_c" => 2.0}}
|
|
]
|
|
}
|
|
|
|
mp_path = ProfilesFile.mp_path_for(vt)
|
|
File.mkdir_p!(Path.dirname(mp_path))
|
|
File.write!(mp_path, :zlib.gzip(Msgpax.pack!(mp_body, iodata: false)))
|
|
|
|
# mp.gz wins.
|
|
assert {:ok, grid} = ProfilesFile.read(vt)
|
|
assert grid[{32.9, -97.0}] == %{surface_temp_c: 2.0}
|
|
end
|
|
|
|
test "read/1 atomizes whitelisted keys, leaves others as strings", %{dir: _dir} do
|
|
vt = ~U[2026-04-19 14:00:00Z]
|
|
|
|
mp_body = %{
|
|
"v" => 1,
|
|
"valid_time" => "2026-04-19T14:00:00Z",
|
|
"cells" => [
|
|
%{
|
|
"lat" => 32.9,
|
|
"lon" => -97.0,
|
|
"profile" => %{
|
|
"surface_temp_c" => 22.5,
|
|
"duct_count" => 1,
|
|
"ducts" => [
|
|
%{"base_m" => 100, "top_m" => 200, "thickness_m" => 100, "m_deficit" => 12.3, "min_freq_ghz" => 15.2}
|
|
],
|
|
"unknown_field" => "keep-as-string"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
|
|
mp_path = ProfilesFile.mp_path_for(vt)
|
|
File.mkdir_p!(Path.dirname(mp_path))
|
|
File.write!(mp_path, :zlib.gzip(Msgpax.pack!(mp_body, iodata: false)))
|
|
|
|
{:ok, grid} = ProfilesFile.read(vt)
|
|
profile = grid[{32.9, -97.0}]
|
|
|
|
assert profile[:surface_temp_c] == 22.5
|
|
assert profile[:duct_count] == 1
|
|
assert [duct] = profile[:ducts]
|
|
assert duct[:base_m] == 100
|
|
assert duct[:min_freq_ghz] == 15.2
|
|
# Keys outside the whitelist stay strings so they can't crash a
|
|
# later String.to_atom/1 bomb.
|
|
assert profile["unknown_field"] == "keep-as-string"
|
|
end
|
|
|
|
test "read/1 returns :enoent when neither format exists", %{dir: _dir} do
|
|
vt = ~U[2030-01-01 00:00:00Z]
|
|
assert ProfilesFile.read(vt) == {:error, :enoent}
|
|
end
|
|
|
|
test "list_valid_times/0 dedupes when both formats share a timestamp", %{dir: _dir} do
|
|
vt = ~U[2026-04-19 14:00:00Z]
|
|
ProfilesFile.write!(vt, %{{32.9, -97.0} => %{surface_temp_c: 1.0}})
|
|
|
|
mp_body = %{"v" => 1, "valid_time" => "2026-04-19T14:00:00Z", "cells" => []}
|
|
mp_path = ProfilesFile.mp_path_for(vt)
|
|
File.mkdir_p!(Path.dirname(mp_path))
|
|
File.write!(mp_path, :zlib.gzip(Msgpax.pack!(mp_body, iodata: false)))
|
|
|
|
assert ProfilesFile.list_valid_times() == [vt]
|
|
end
|
|
end
|
|
end
|