Graham McIntire
ac3a2517d9
New LiveView at /weather-ca duplicates the /weather UI but defaults the viewport to central Canada (55N, -100W, z=4) and renders the map div with data-source="hrdps". The JS hook reads that attribute and appends &source=hrdps to every cell fetch; the WeatherTileController routes those reads through Weather.weather_grid_hrdps_at/2, which only reads the <vt>.hrdps scalar dir and skips HRRR merging entirely. Timeline is sourced from ScalarFile.list_valid_times_hrdps/0 so HRDPS-only hours show up even when no HRRR file exists for the same valid_time. Also drops "— Unified" from the algo.md H1.
347 lines
11 KiB
Elixir
347 lines
11 KiB
Elixir
defmodule Microwaveprop.Weather.ScalarFileTest do
|
|
# async: false — mutates the global :propagation_scores_dir env to redirect
|
|
# the on-disk artifact tree to a private tmp directory.
|
|
use ExUnit.Case, async: false
|
|
|
|
alias Microwaveprop.Weather.ScalarFile
|
|
|
|
setup do
|
|
dir =
|
|
Path.join(
|
|
System.tmp_dir!(),
|
|
"weather_scalar_file_test_#{System.unique_integer([:positive])}"
|
|
)
|
|
|
|
File.mkdir_p!(dir)
|
|
|
|
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
|
|
|
|
on_exit(fn ->
|
|
File.rm_rf!(dir)
|
|
|
|
if prev do
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
|
|
else
|
|
Application.delete_env(:microwaveprop, :propagation_scores_dir)
|
|
end
|
|
end)
|
|
|
|
%{dir: dir}
|
|
end
|
|
|
|
defp sample_row(lat, lon, temp \\ 22.0) do
|
|
%{
|
|
lat: lat,
|
|
lon: lon,
|
|
valid_time: ~U[2026-04-28 12:00:00Z],
|
|
temperature: temp,
|
|
dewpoint_depression: 5.0,
|
|
surface_rh: 70.0,
|
|
surface_pressure_mb: 1013.0,
|
|
surface_refractivity: 330.0,
|
|
refractivity_gradient: -45.0,
|
|
bl_height: 800.0,
|
|
pwat: 25.0,
|
|
temp_850mb: 12.0,
|
|
dewpoint_850mb: 4.0,
|
|
temp_700mb: 0.0,
|
|
dewpoint_700mb: -10.0,
|
|
lapse_rate: 6.5,
|
|
mid_lapse_rate: 6.0,
|
|
inversion_strength: 0.5,
|
|
inversion_base_m: nil,
|
|
ducting: false,
|
|
duct_base_m: nil,
|
|
duct_strength: nil
|
|
}
|
|
end
|
|
|
|
describe "base_dir/0" do
|
|
test "is rooted under the configured scores dir", %{dir: dir} do
|
|
assert ScalarFile.base_dir() == Path.join(dir, "weather_scalars")
|
|
end
|
|
end
|
|
|
|
describe "write!/2 + read_bounds/2" do
|
|
test "round-trips the rows that fall inside the requested bounds" do
|
|
valid_time = ~U[2026-04-28 12:00:00Z]
|
|
|
|
rows = [
|
|
sample_row(33.0, -97.0, 22.0),
|
|
sample_row(33.5, -97.5, 23.0),
|
|
# Outside Texas bounds below
|
|
sample_row(45.0, -90.0, 5.0)
|
|
]
|
|
|
|
ScalarFile.write!(valid_time, rows)
|
|
|
|
result =
|
|
ScalarFile.read_bounds(valid_time, %{
|
|
"south" => 32.0,
|
|
"north" => 34.5,
|
|
"west" => -98.0,
|
|
"east" => -96.0
|
|
})
|
|
|
|
lat_lons = result |> Enum.map(&{&1.lat, &1.lon}) |> Enum.sort()
|
|
assert lat_lons == [{33.0, -97.0}, {33.5, -97.5}]
|
|
end
|
|
|
|
test "returns [] when the file does not exist" do
|
|
assert ScalarFile.read_bounds(~U[2026-04-28 12:00:00Z], nil) == []
|
|
end
|
|
|
|
test "read_bounds with nil bounds returns every persisted row" do
|
|
valid_time = ~U[2026-04-28 12:00:00Z]
|
|
rows = [sample_row(33.0, -97.0), sample_row(45.0, -90.0)]
|
|
|
|
ScalarFile.write!(valid_time, rows)
|
|
|
|
assert length(ScalarFile.read_bounds(valid_time, nil)) == 2
|
|
end
|
|
|
|
test "preserves the full set of derived layer fields on round-trip" do
|
|
valid_time = ~U[2026-04-28 12:00:00Z]
|
|
[row] = [sample_row(33.0, -97.0)]
|
|
|
|
ScalarFile.write!(valid_time, [row])
|
|
|
|
[round_tripped] = ScalarFile.read_bounds(valid_time, nil)
|
|
|
|
for key <- Map.keys(row) do
|
|
assert Map.fetch!(round_tripped, key) == Map.fetch!(row, key),
|
|
"key #{inspect(key)} mismatched after round-trip"
|
|
end
|
|
end
|
|
end
|
|
|
|
describe "read_point/3" do
|
|
test "returns the snapped row for the requested lat/lon" do
|
|
valid_time = ~U[2026-04-28 12:00:00Z]
|
|
row = sample_row(33.0, -97.0)
|
|
|
|
ScalarFile.write!(valid_time, [row])
|
|
|
|
assert {:ok, fetched} = ScalarFile.read_point(valid_time, 33.0, -97.0)
|
|
assert fetched.temperature == 22.0
|
|
end
|
|
|
|
test "returns :miss when no scalar file exists" do
|
|
assert ScalarFile.read_point(~U[2026-04-28 12:00:00Z], 33.0, -97.0) == :miss
|
|
end
|
|
|
|
test "returns :miss when the cell is not in the file" do
|
|
valid_time = ~U[2026-04-28 12:00:00Z]
|
|
ScalarFile.write!(valid_time, [sample_row(33.0, -97.0)])
|
|
|
|
assert ScalarFile.read_point(valid_time, 40.0, -80.0) == :miss
|
|
end
|
|
end
|
|
|
|
describe "exists?/1 and list_valid_times/0" do
|
|
test "reflects what has been written" do
|
|
vt1 = ~U[2026-04-28 12:00:00Z]
|
|
vt2 = ~U[2026-04-28 13:00:00Z]
|
|
|
|
refute ScalarFile.exists?(vt1)
|
|
assert ScalarFile.list_valid_times() == []
|
|
|
|
ScalarFile.write!(vt1, [sample_row(33.0, -97.0)])
|
|
ScalarFile.write!(vt2, [sample_row(33.0, -97.0)])
|
|
|
|
assert ScalarFile.exists?(vt1)
|
|
assert ScalarFile.list_valid_times() == [vt1, vt2]
|
|
end
|
|
end
|
|
|
|
describe "HRDPS sibling read merge" do
|
|
test "read_bounds returns HRRR + HRDPS rows when both dirs exist" do
|
|
vt = ~U[2026-04-29 12:00:00Z]
|
|
|
|
ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)])
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
|
|
|
|
rows = ScalarFile.read_bounds(vt, nil)
|
|
sorted = Enum.sort_by(rows, & &1.lat)
|
|
|
|
assert length(sorted) == 2
|
|
assert Enum.at(sorted, 0).lat == 33.0
|
|
assert Enum.at(sorted, 1).lat == 53.0
|
|
end
|
|
|
|
test "read_bounds returns HRDPS rows when only HRDPS dir exists" do
|
|
vt = ~U[2026-04-29 12:00:00Z]
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
|
|
|
|
assert [%{lat: 53.0, lon: -60.0}] = ScalarFile.read_bounds(vt, nil)
|
|
end
|
|
|
|
test "read_point falls through to HRDPS when the HRRR chunk is absent" do
|
|
vt = ~U[2026-04-29 12:00:00Z]
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
|
|
|
|
assert {:ok, %{lat: 53.0, lon: -60.0, temperature: 8.0}} =
|
|
ScalarFile.read_point(vt, 53.0, -60.0)
|
|
end
|
|
|
|
test "exists?/1 is true when only the HRDPS dir has chunks" do
|
|
vt = ~U[2026-04-29 12:00:00Z]
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
|
|
assert ScalarFile.exists?(vt)
|
|
end
|
|
|
|
test "merge prefers HRRR row when both dirs cover the same lat/lon" do
|
|
vt = ~U[2026-04-29 12:00:00Z]
|
|
|
|
ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)])
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 33.0, -97.0, 99.0)
|
|
|
|
[row] = ScalarFile.read_bounds(vt, nil)
|
|
assert row.temperature == 22.0
|
|
end
|
|
end
|
|
|
|
describe "HRDPS-only reads (for the /weather-ca endpoint)" do
|
|
test "read_bounds_hrdps returns only HRDPS chunks, ignoring co-located HRRR" do
|
|
vt = ~U[2026-04-29 12:00:00Z]
|
|
|
|
# Both dirs cover the same point. The HRRR-only read_bounds/2 would
|
|
# merge them (preferring HRRR); read_bounds_hrdps/2 must return ONLY
|
|
# the HRDPS row so /weather-ca shows Canadian-source data.
|
|
ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)])
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
|
|
|
|
assert [%{lat: 53.0, lon: -60.0, temperature: 8.0}] =
|
|
ScalarFile.read_bounds_hrdps(vt, nil)
|
|
end
|
|
|
|
test "read_bounds_hrdps respects bounds" do
|
|
vt = ~U[2026-04-29 12:00:00Z]
|
|
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0)
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 33.0, -97.0, 22.0)
|
|
|
|
bounds = %{"south" => 50.0, "north" => 60.0, "west" => -70.0, "east" => -50.0}
|
|
assert [%{lat: 53.0, lon: -60.0}] = ScalarFile.read_bounds_hrdps(vt, bounds)
|
|
end
|
|
|
|
test "read_bounds_hrdps returns [] when no HRDPS dir exists" do
|
|
vt = ~U[2026-04-29 12:00:00Z]
|
|
ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)])
|
|
|
|
assert ScalarFile.read_bounds_hrdps(vt, nil) == []
|
|
end
|
|
|
|
test "list_valid_times_hrdps returns times from HRDPS-suffixed dirs only" do
|
|
vt1 = ~U[2026-04-29 12:00:00Z]
|
|
vt2 = ~U[2026-04-29 13:00:00Z]
|
|
vt_hrrr_only = ~U[2026-04-29 14:00:00Z]
|
|
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt1), 53.0, -60.0, 8.0)
|
|
hand_write_chunk(ScalarFile.dir_for_hrdps(vt2), 53.0, -60.0, 9.0)
|
|
ScalarFile.write!(vt_hrrr_only, [sample_row(33.0, -97.0, 22.0)])
|
|
|
|
assert ScalarFile.list_valid_times_hrdps() == [vt1, vt2]
|
|
end
|
|
end
|
|
|
|
defp hand_write_chunk(dir, lat, lon, temp) do
|
|
File.mkdir_p!(dir)
|
|
lat_band = (lat / 5) |> Float.floor() |> trunc()
|
|
lon_band = (lon / 5) |> Float.floor() |> trunc()
|
|
path = Path.join(dir, "#{lat_band}_#{lon_band}.mp.gz")
|
|
|
|
payload =
|
|
Msgpax.pack!(
|
|
[
|
|
%{
|
|
"lat" => lat,
|
|
"lon" => lon,
|
|
"valid_time" => "2026-04-29T12:00:00Z",
|
|
"temperature" => temp
|
|
}
|
|
],
|
|
iodata: false
|
|
)
|
|
|
|
File.write!(path, :zlib.gzip(payload))
|
|
end
|
|
|
|
describe "Rust-writer compatibility (the cross-language wire format)" do
|
|
test "decodes a hand-rolled gzipped MessagePack chunk and atomizes whitelisted keys" do
|
|
valid_time = ~U[2026-04-29 12:00:00Z]
|
|
dir = Path.join(ScalarFile.dir_for(valid_time), "")
|
|
File.mkdir_p!(dir)
|
|
|
|
# Mirror exactly what `prop_grid_rs::weather_scalar_file::write_atomic`
|
|
# emits: gzipped MessagePack, top-level array of string-keyed maps.
|
|
# If this round-trip ever breaks, Elixir is reading something the
|
|
# Rust pipeline cannot produce — a wire-format regression.
|
|
payload =
|
|
Msgpax.pack!(
|
|
[
|
|
%{
|
|
"lat" => 33.0,
|
|
"lon" => -97.0,
|
|
"valid_time" => "2026-04-29T12:00:00Z",
|
|
"temperature" => 22.5,
|
|
"dewpoint_depression" => 10.0,
|
|
"surface_rh" => 50.0,
|
|
"ducting" => false,
|
|
# An out-of-whitelist key — the reader must NOT atomize this.
|
|
"unknown_extra_key" => 1.23
|
|
}
|
|
],
|
|
iodata: false
|
|
)
|
|
|
|
path = Path.join(dir, "6_-20.mp.gz")
|
|
File.write!(path, :zlib.gzip(payload))
|
|
|
|
[row] = ScalarFile.read_bounds(valid_time, nil)
|
|
|
|
# Whitelisted keys are atoms.
|
|
assert row.lat == 33.0
|
|
assert row.lon == -97.0
|
|
assert row.temperature == 22.5
|
|
assert row.ducting == false
|
|
|
|
# `valid_time` round-trips back to a DateTime.
|
|
assert row.valid_time == valid_time
|
|
|
|
# Out-of-whitelist keys stay as strings so a future field can be
|
|
# added without a deploy-order coordination problem.
|
|
assert row["unknown_extra_key"] == 1.23
|
|
end
|
|
end
|
|
|
|
describe "prune_older_than/1 and retain_window/2" do
|
|
test "prune_older_than removes files strictly before the cutoff" do
|
|
old = ~U[2026-04-27 12:00:00Z]
|
|
new = ~U[2026-04-28 12:00:00Z]
|
|
|
|
ScalarFile.write!(old, [sample_row(33.0, -97.0)])
|
|
ScalarFile.write!(new, [sample_row(33.0, -97.0)])
|
|
|
|
assert ScalarFile.prune_older_than(~U[2026-04-28 00:00:00Z]) == 1
|
|
refute ScalarFile.exists?(old)
|
|
assert ScalarFile.exists?(new)
|
|
end
|
|
|
|
test "retain_window keeps only files inside [run_time, run_time + max_forecast_hour * 3600]" do
|
|
run_time = ~U[2026-04-28 12:00:00Z]
|
|
inside = ~U[2026-04-28 14:00:00Z]
|
|
outside = ~U[2026-04-29 06:00:00Z]
|
|
|
|
ScalarFile.write!(run_time, [sample_row(33.0, -97.0)])
|
|
ScalarFile.write!(inside, [sample_row(33.0, -97.0)])
|
|
ScalarFile.write!(outside, [sample_row(33.0, -97.0)])
|
|
|
|
assert ScalarFile.retain_window(run_time, 4) == 1
|
|
assert ScalarFile.exists?(run_time)
|
|
assert ScalarFile.exists?(inside)
|
|
refute ScalarFile.exists?(outside)
|
|
end
|
|
end
|
|
end
|