defmodule Microwaveprop.Weather.ScalarFileTest do # async: false — mutates the global :propagation_scores_dir env to redirect # the on-disk artifact tree to a private tmp directory. use ExUnit.Case, async: false alias Microwaveprop.Weather.ScalarFile setup do dir = Path.join( System.tmp_dir!(), "weather_scalar_file_test_#{System.unique_integer([:positive])}" ) File.mkdir_p!(dir) prev = Application.get_env(:microwaveprop, :propagation_scores_dir) Application.put_env(:microwaveprop, :propagation_scores_dir, dir) on_exit(fn -> File.rm_rf!(dir) if prev do Application.put_env(:microwaveprop, :propagation_scores_dir, prev) else Application.delete_env(:microwaveprop, :propagation_scores_dir) end end) %{dir: dir} end defp sample_row(lat, lon, temp \\ 22.0) do %{ lat: lat, lon: lon, valid_time: ~U[2026-04-28 12:00:00Z], temperature: temp, dewpoint_depression: 5.0, surface_rh: 70.0, surface_pressure_mb: 1013.0, surface_refractivity: 330.0, refractivity_gradient: -45.0, bl_height: 800.0, pwat: 25.0, temp_850mb: 12.0, dewpoint_850mb: 4.0, temp_700mb: 0.0, dewpoint_700mb: -10.0, lapse_rate: 6.5, mid_lapse_rate: 6.0, inversion_strength: 0.5, inversion_base_m: nil, ducting: false, duct_base_m: nil, duct_strength: nil } end describe "base_dir/0" do test "is rooted under the configured scores dir", %{dir: dir} do assert ScalarFile.base_dir() == Path.join(dir, "weather_scalars") end end describe "write!/2 + read_bounds/2" do test "round-trips the rows that fall inside the requested bounds" do valid_time = ~U[2026-04-28 12:00:00Z] rows = [ sample_row(33.0, -97.0, 22.0), sample_row(33.5, -97.5, 23.0), # Outside Texas bounds below sample_row(45.0, -90.0, 5.0) ] ScalarFile.write!(valid_time, rows) result = ScalarFile.read_bounds(valid_time, %{ "south" => 32.0, "north" => 34.5, "west" => -98.0, "east" => -96.0 }) lat_lons = result |> Enum.map(&{&1.lat, &1.lon}) |> Enum.sort() assert lat_lons == [{33.0, -97.0}, {33.5, -97.5}] end test "returns [] when the file does not exist" do assert ScalarFile.read_bounds(~U[2026-04-28 12:00:00Z], nil) == [] end test "read_bounds with nil bounds returns every persisted row" do valid_time = ~U[2026-04-28 12:00:00Z] rows = [sample_row(33.0, -97.0), sample_row(45.0, -90.0)] ScalarFile.write!(valid_time, rows) assert length(ScalarFile.read_bounds(valid_time, nil)) == 2 end test "preserves the full set of derived layer fields on round-trip" do valid_time = ~U[2026-04-28 12:00:00Z] [row] = [sample_row(33.0, -97.0)] ScalarFile.write!(valid_time, [row]) [round_tripped] = ScalarFile.read_bounds(valid_time, nil) for key <- Map.keys(row) do assert Map.fetch!(round_tripped, key) == Map.fetch!(row, key), "key #{inspect(key)} mismatched after round-trip" end end end describe "read_point/3" do test "returns the snapped row for the requested lat/lon" do valid_time = ~U[2026-04-28 12:00:00Z] row = sample_row(33.0, -97.0) ScalarFile.write!(valid_time, [row]) assert {:ok, fetched} = ScalarFile.read_point(valid_time, 33.0, -97.0) assert fetched.temperature == 22.0 end test "returns :miss when no scalar file exists" do assert ScalarFile.read_point(~U[2026-04-28 12:00:00Z], 33.0, -97.0) == :miss end test "returns :miss when the cell is not in the file" do valid_time = ~U[2026-04-28 12:00:00Z] ScalarFile.write!(valid_time, [sample_row(33.0, -97.0)]) assert ScalarFile.read_point(valid_time, 40.0, -80.0) == :miss end end describe "exists?/1 and list_valid_times/0" do test "reflects what has been written" do vt1 = ~U[2026-04-28 12:00:00Z] vt2 = ~U[2026-04-28 13:00:00Z] refute ScalarFile.exists?(vt1) assert ScalarFile.list_valid_times() == [] ScalarFile.write!(vt1, [sample_row(33.0, -97.0)]) ScalarFile.write!(vt2, [sample_row(33.0, -97.0)]) assert ScalarFile.exists?(vt1) assert ScalarFile.list_valid_times() == [vt1, vt2] end end describe "HRDPS sibling read merge" do test "read_bounds returns HRRR + HRDPS rows when both dirs exist" do vt = ~U[2026-04-29 12:00:00Z] ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)]) hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0) rows = ScalarFile.read_bounds(vt, nil) sorted = Enum.sort_by(rows, & &1.lat) assert length(sorted) == 2 assert Enum.at(sorted, 0).lat == 33.0 assert Enum.at(sorted, 1).lat == 53.0 end test "read_bounds returns HRDPS rows when only HRDPS dir exists" do vt = ~U[2026-04-29 12:00:00Z] hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0) assert [%{lat: 53.0, lon: -60.0}] = ScalarFile.read_bounds(vt, nil) end test "read_point falls through to HRDPS when the HRRR chunk is absent" do vt = ~U[2026-04-29 12:00:00Z] hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0) assert {:ok, %{lat: 53.0, lon: -60.0, temperature: 8.0}} = ScalarFile.read_point(vt, 53.0, -60.0) end test "exists?/1 is true when only the HRDPS dir has chunks" do vt = ~U[2026-04-29 12:00:00Z] hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0) assert ScalarFile.exists?(vt) end test "merge prefers HRRR row when both dirs cover the same lat/lon" do vt = ~U[2026-04-29 12:00:00Z] ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)]) hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 33.0, -97.0, 99.0) [row] = ScalarFile.read_bounds(vt, nil) assert row.temperature == 22.0 end end describe "HRDPS-only reads (for the /weather-ca endpoint)" do test "read_bounds_hrdps returns only HRDPS chunks, ignoring co-located HRRR" do vt = ~U[2026-04-29 12:00:00Z] # Both dirs cover the same point. The HRRR-only read_bounds/2 would # merge them (preferring HRRR); read_bounds_hrdps/2 must return ONLY # the HRDPS row so /weather-ca shows Canadian-source data. ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)]) hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0) assert [%{lat: 53.0, lon: -60.0, temperature: 8.0}] = ScalarFile.read_bounds_hrdps(vt, nil) end test "read_bounds_hrdps respects bounds" do vt = ~U[2026-04-29 12:00:00Z] hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 53.0, -60.0, 8.0) hand_write_chunk(ScalarFile.dir_for_hrdps(vt), 33.0, -97.0, 22.0) bounds = %{"south" => 50.0, "north" => 60.0, "west" => -70.0, "east" => -50.0} assert [%{lat: 53.0, lon: -60.0}] = ScalarFile.read_bounds_hrdps(vt, bounds) end test "read_bounds_hrdps returns [] when no HRDPS dir exists" do vt = ~U[2026-04-29 12:00:00Z] ScalarFile.write!(vt, [sample_row(33.0, -97.0, 22.0)]) assert ScalarFile.read_bounds_hrdps(vt, nil) == [] end test "list_valid_times_hrdps returns times from HRDPS-suffixed dirs only" do vt1 = ~U[2026-04-29 12:00:00Z] vt2 = ~U[2026-04-29 13:00:00Z] vt_hrrr_only = ~U[2026-04-29 14:00:00Z] hand_write_chunk(ScalarFile.dir_for_hrdps(vt1), 53.0, -60.0, 8.0) hand_write_chunk(ScalarFile.dir_for_hrdps(vt2), 53.0, -60.0, 9.0) ScalarFile.write!(vt_hrrr_only, [sample_row(33.0, -97.0, 22.0)]) assert ScalarFile.list_valid_times_hrdps() == [vt1, vt2] end end defp hand_write_chunk(dir, lat, lon, temp) do File.mkdir_p!(dir) lat_band = (lat / 5) |> Float.floor() |> trunc() lon_band = (lon / 5) |> Float.floor() |> trunc() path = Path.join(dir, "#{lat_band}_#{lon_band}.mp.gz") payload = Msgpax.pack!( [ %{ "lat" => lat, "lon" => lon, "valid_time" => "2026-04-29T12:00:00Z", "temperature" => temp } ], iodata: false ) File.write!(path, :zlib.gzip(payload)) end describe "Rust-writer compatibility (the cross-language wire format)" do test "decodes a hand-rolled gzipped MessagePack chunk and atomizes whitelisted keys" do valid_time = ~U[2026-04-29 12:00:00Z] dir = Path.join(ScalarFile.dir_for(valid_time), "") File.mkdir_p!(dir) # Mirror exactly what `prop_grid_rs::weather_scalar_file::write_atomic` # emits: gzipped MessagePack, top-level array of string-keyed maps. # If this round-trip ever breaks, Elixir is reading something the # Rust pipeline cannot produce — a wire-format regression. payload = Msgpax.pack!( [ %{ "lat" => 33.0, "lon" => -97.0, "valid_time" => "2026-04-29T12:00:00Z", "temperature" => 22.5, "dewpoint_depression" => 10.0, "surface_rh" => 50.0, "ducting" => false, # An out-of-whitelist key — the reader must NOT atomize this. "unknown_extra_key" => 1.23 } ], iodata: false ) path = Path.join(dir, "6_-20.mp.gz") File.write!(path, :zlib.gzip(payload)) [row] = ScalarFile.read_bounds(valid_time, nil) # Whitelisted keys are atoms. assert row.lat == 33.0 assert row.lon == -97.0 assert row.temperature == 22.5 assert row.ducting == false # `valid_time` round-trips back to a DateTime. assert row.valid_time == valid_time # Out-of-whitelist keys stay as strings so a future field can be # added without a deploy-order coordination problem. assert row["unknown_extra_key"] == 1.23 end end describe "prune_older_than/1 and retain_window/2" do test "prune_older_than removes files strictly before the cutoff" do old = ~U[2026-04-27 12:00:00Z] new = ~U[2026-04-28 12:00:00Z] ScalarFile.write!(old, [sample_row(33.0, -97.0)]) ScalarFile.write!(new, [sample_row(33.0, -97.0)]) assert ScalarFile.prune_older_than(~U[2026-04-28 00:00:00Z]) == 1 refute ScalarFile.exists?(old) assert ScalarFile.exists?(new) end test "retain_window keeps only files inside [run_time, run_time + max_forecast_hour * 3600]" do run_time = ~U[2026-04-28 12:00:00Z] inside = ~U[2026-04-28 14:00:00Z] outside = ~U[2026-04-29 06:00:00Z] ScalarFile.write!(run_time, [sample_row(33.0, -97.0)]) ScalarFile.write!(inside, [sample_row(33.0, -97.0)]) ScalarFile.write!(outside, [sample_row(33.0, -97.0)]) assert ScalarFile.retain_window(run_time, 4) == 1 assert ScalarFile.exists?(run_time) assert ScalarFile.exists?(inside) refute ScalarFile.exists?(outside) end end end