prop/test/microwaveprop/weather/scalar_file_test.exs
Graham McIntire a14f14485e
perf(weather): serve /weather from chunked scalar artifacts
Move the /weather render path off raw HRRR profile decoding + per-cell
SoundingParams + WeatherLayers derivation. The dominant cost was
re-deriving the same scalar fields on every viewport pan and timeline
scrub.

Server side:

- New Microwaveprop.Weather.ScalarFile persists pre-derived scalar
  rows on NFS, bucketed into 5°×5° chunk files under
  <base>/weather_scalars/<iso>/<lat_band>_<lon_band>.etf.gz. Viewport
  reads only decode the chunks that overlap the requested bounds;
  point-detail clicks read exactly one chunk.
- weather_grid_at/2 and weather_point_detail/3 prefer ScalarFile;
  ProfilesFile is now the cold-start fallback only.
- warm_grid_cache_and_broadcast/1 and warm_grid_cache_from_latest_profile/0
  also persist the scalar artifact, and the cold-derive path kicks
  off a per-valid_time-locked async materialization so the next
  reader gets the cheap path.
- NotifyListener.handle_propagation_ready/1 fires
  Weather.materialize_scalar_file/1 in a detached Task on the Rust
  pipeline's NOTIFY propagation_ready, so steady-state forecast hours
  arrive with their scalar artifact already on disk.
- available_weather_valid_times/0 unions ScalarFile + ProfilesFile so
  the timeline survives an aggressive retention sweep on either side.

Browser side (finding #8):

- Mount the legend Leaflet control once and patch its inner content
  on layer changes instead of remove + re-add on every renderLayer.
- renderTimeline now keys off the rendered button set: selection-only
  updates take an applyTimelineSelection patch path that restyles
  existing buttons in place. Full innerHTML rebuild only fires when
  timelineData itself changes.

Also fixes a credo nesting-depth flag in tile_renderer.ex by extracting
interpolate_pair/2 from the inner anonymous function.
2026-04-28 17:15:36 -05:00

186 lines
5.4 KiB
Elixir

defmodule Microwaveprop.Weather.ScalarFileTest do
# async: false — mutates the global :propagation_scores_dir env to redirect
# the on-disk artifact tree to a private tmp directory.
use ExUnit.Case, async: false
alias Microwaveprop.Weather.ScalarFile
setup do
dir =
Path.join(
System.tmp_dir!(),
"weather_scalar_file_test_#{System.unique_integer([:positive])}"
)
File.mkdir_p!(dir)
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
on_exit(fn ->
File.rm_rf!(dir)
if prev do
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
else
Application.delete_env(:microwaveprop, :propagation_scores_dir)
end
end)
%{dir: dir}
end
defp sample_row(lat, lon, temp \\ 22.0) do
%{
lat: lat,
lon: lon,
valid_time: ~U[2026-04-28 12:00:00Z],
temperature: temp,
dewpoint_depression: 5.0,
surface_rh: 70.0,
surface_pressure_mb: 1013.0,
surface_refractivity: 330.0,
refractivity_gradient: -45.0,
bl_height: 800.0,
pwat: 25.0,
temp_850mb: 12.0,
dewpoint_850mb: 4.0,
temp_700mb: 0.0,
dewpoint_700mb: -10.0,
lapse_rate: 6.5,
mid_lapse_rate: 6.0,
inversion_strength: 0.5,
inversion_base_m: nil,
ducting: false,
duct_base_m: nil,
duct_strength: nil
}
end
describe "base_dir/0" do
test "is rooted under the configured scores dir", %{dir: dir} do
assert ScalarFile.base_dir() == Path.join(dir, "weather_scalars")
end
end
describe "write!/2 + read_bounds/2" do
test "round-trips the rows that fall inside the requested bounds" do
valid_time = ~U[2026-04-28 12:00:00Z]
rows = [
sample_row(33.0, -97.0, 22.0),
sample_row(33.5, -97.5, 23.0),
# Outside Texas bounds below
sample_row(45.0, -90.0, 5.0)
]
ScalarFile.write!(valid_time, rows)
result =
ScalarFile.read_bounds(valid_time, %{
"south" => 32.0,
"north" => 34.5,
"west" => -98.0,
"east" => -96.0
})
lat_lons = result |> Enum.map(&{&1.lat, &1.lon}) |> Enum.sort()
assert lat_lons == [{33.0, -97.0}, {33.5, -97.5}]
end
test "returns [] when the file does not exist" do
assert ScalarFile.read_bounds(~U[2026-04-28 12:00:00Z], nil) == []
end
test "read_bounds with nil bounds returns every persisted row" do
valid_time = ~U[2026-04-28 12:00:00Z]
rows = [sample_row(33.0, -97.0), sample_row(45.0, -90.0)]
ScalarFile.write!(valid_time, rows)
assert length(ScalarFile.read_bounds(valid_time, nil)) == 2
end
test "preserves the full set of derived layer fields on round-trip" do
valid_time = ~U[2026-04-28 12:00:00Z]
[row] = [sample_row(33.0, -97.0)]
ScalarFile.write!(valid_time, [row])
[round_tripped] = ScalarFile.read_bounds(valid_time, nil)
for key <- Map.keys(row) do
assert Map.fetch!(round_tripped, key) == Map.fetch!(row, key),
"key #{inspect(key)} mismatched after round-trip"
end
end
end
describe "read_point/3" do
test "returns the snapped row for the requested lat/lon" do
valid_time = ~U[2026-04-28 12:00:00Z]
row = sample_row(33.0, -97.0)
ScalarFile.write!(valid_time, [row])
assert {:ok, fetched} = ScalarFile.read_point(valid_time, 33.0, -97.0)
assert fetched.temperature == 22.0
end
test "returns :miss when no scalar file exists" do
assert ScalarFile.read_point(~U[2026-04-28 12:00:00Z], 33.0, -97.0) == :miss
end
test "returns :miss when the cell is not in the file" do
valid_time = ~U[2026-04-28 12:00:00Z]
ScalarFile.write!(valid_time, [sample_row(33.0, -97.0)])
assert ScalarFile.read_point(valid_time, 40.0, -80.0) == :miss
end
end
describe "exists?/1 and list_valid_times/0" do
test "reflects what has been written" do
vt1 = ~U[2026-04-28 12:00:00Z]
vt2 = ~U[2026-04-28 13:00:00Z]
refute ScalarFile.exists?(vt1)
assert ScalarFile.list_valid_times() == []
ScalarFile.write!(vt1, [sample_row(33.0, -97.0)])
ScalarFile.write!(vt2, [sample_row(33.0, -97.0)])
assert ScalarFile.exists?(vt1)
assert ScalarFile.list_valid_times() == [vt1, vt2]
end
end
describe "prune_older_than/1 and retain_window/2" do
test "prune_older_than removes files strictly before the cutoff" do
old = ~U[2026-04-27 12:00:00Z]
new = ~U[2026-04-28 12:00:00Z]
ScalarFile.write!(old, [sample_row(33.0, -97.0)])
ScalarFile.write!(new, [sample_row(33.0, -97.0)])
assert ScalarFile.prune_older_than(~U[2026-04-28 00:00:00Z]) == 1
refute ScalarFile.exists?(old)
assert ScalarFile.exists?(new)
end
test "retain_window keeps only files inside [run_time, run_time + max_forecast_hour * 3600]" do
run_time = ~U[2026-04-28 12:00:00Z]
inside = ~U[2026-04-28 14:00:00Z]
outside = ~U[2026-04-29 06:00:00Z]
ScalarFile.write!(run_time, [sample_row(33.0, -97.0)])
ScalarFile.write!(inside, [sample_row(33.0, -97.0)])
ScalarFile.write!(outside, [sample_row(33.0, -97.0)])
assert ScalarFile.retain_window(run_time, 4) == 1
assert ScalarFile.exists?(run_time)
assert ScalarFile.exists?(inside)
refute ScalarFile.exists?(outside)
end
end
end