defmodule Microwaveprop.WeatherGridTest do use Microwaveprop.DataCase, async: false alias Microwaveprop.Propagation.ProfilesFile alias Microwaveprop.Weather alias Microwaveprop.Weather.GridCache alias Microwaveprop.Weather.SoundingParams setup do # GridCache ETS state leaks between tests (it's a named table outside the # test sandbox). Clear it so each test gets a fresh cache and the disk # load path actually runs. GridCache.clear() # Microwaveprop.Cache memoizes ProfilesFile.read/1 keyed by # `(base_dir, valid_time)`; tests within this file share the scores_dir # and frequently reuse `DateTime.utc_now()` truncated to seconds, so a # prior test's cached read leaks into the next one's assertion. Microwaveprop.Cache.clear() dir = Path.join( System.tmp_dir!(), "weather_grid_test_#{System.unique_integer([:positive])}" ) File.mkdir_p!(Path.join(dir, "profiles")) prev = Application.get_env(:microwaveprop, :propagation_scores_dir) Application.put_env(:microwaveprop, :propagation_scores_dir, dir) on_exit(fn -> File.rm_rf!(dir) if prev do Application.put_env(:microwaveprop, :propagation_scores_dir, prev) else Application.delete_env(:microwaveprop, :propagation_scores_dir) end end) :ok end describe "load_weather_grid/1" do test "returns grid data for latest valid_time within bounds" do now = DateTime.truncate(DateTime.utc_now(), :second) insert_hrrr_profile(%{ lat: 32.125, lon: -97.375, valid_time: now, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0, min_refractivity_gradient: -280.0, ducting_detected: true }) insert_hrrr_profile(%{ lat: 32.250, lon: -97.375, valid_time: now, surface_temp_c: 26.0, surface_dewpoint_c: 19.0, surface_pressure_mb: 1012.0, hpbl_m: 600.0, pwat_mm: 32.0, min_refractivity_gradient: -150.0, ducting_detected: false }) bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0} result = Weather.load_weather_grid(bounds) assert length(result) == 2 point = Enum.find(result, &(&1.lat == 32.125)) assert point.temperature == 25.0 assert point.dewpoint_depression == 7.0 assert point.bl_height == 500.0 assert point.pwat == 30.0 assert point.refractivity_gradient == -280.0 assert point.ducting == true end test "returns empty list when no data exists" do bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0} assert Weather.load_weather_grid(bounds) == [] end test "uses most recent valid_time only" do old = DateTime.utc_now() |> DateTime.add(-7200, :second) |> DateTime.truncate(:second) new = DateTime.truncate(DateTime.utc_now(), :second) insert_hrrr_profile(%{ lat: 32.125, lon: -97.375, valid_time: old, surface_temp_c: 20.0, surface_dewpoint_c: 15.0, surface_pressure_mb: 1010.0, hpbl_m: 400.0, pwat_mm: 25.0 }) insert_hrrr_profile(%{ lat: 32.125, lon: -97.375, valid_time: new, surface_temp_c: 28.0, surface_dewpoint_c: 20.0, surface_pressure_mb: 1015.0, hpbl_m: 700.0, pwat_mm: 35.0 }) bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0} result = Weather.load_weather_grid(bounds) assert length(result) == 1 assert hd(result).temperature == 28.0 end test "includes derived upper-air fields and strips raw profile data" do now = DateTime.truncate(DateTime.utc_now(), :second) profile = [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}, %{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0}, %{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0} ] duct_chars = [ %{"base" => 200.0, "strength" => 50.0}, %{"base" => 800.0, "strength" => 30.0} ] insert_hrrr_profile(%{ lat: 32.125, lon: -97.375, valid_time: now, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, surface_refractivity: 320.0, hpbl_m: 500.0, pwat_mm: 30.0, min_refractivity_gradient: -280.0, ducting_detected: true, profile: profile, duct_characteristics: duct_chars }) bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0} [point] = Weather.load_weather_grid(bounds) # Derived fields should be present assert is_float(point.surface_rh) assert point.surface_refractivity == 320.0 assert point.temp_850mb == 15.0 assert point.dewpoint_850mb == 10.0 assert is_float(point.lapse_rate) assert point.duct_base_m == 200.0 assert point.duct_strength == 50.0 # Raw bulky fields should be stripped refute Map.has_key?(point, :profile) refute Map.has_key?(point, :duct_characteristics) refute Map.has_key?(point, :surface_temp_c) refute Map.has_key?(point, :surface_dewpoint_c) end end describe "build_grid_cache_rows/2" do test "returns an empty list for empty grid_data" do now = DateTime.truncate(DateTime.utc_now(), :second) assert Weather.build_grid_cache_rows(%{}, now) == [] end test "builds a derived row for each in-memory grid point without hitting the database" do now = DateTime.truncate(DateTime.utc_now(), :second) profile = [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}, %{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0}, %{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0} ] grid_data = %{ {32.125, -97.375} => %{ profile: profile, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 }, {32.250, -97.375} => %{ profile: profile, surface_temp_c: 26.0, surface_dewpoint_c: 19.0, surface_pressure_mb: 1012.0, hpbl_m: 600.0, pwat_mm: 32.0 } } result = Weather.build_grid_cache_rows(grid_data, now) assert length(result) == 2 point = Enum.find(result, &(&1.lat == 32.125)) assert point.lon == -97.375 assert point.valid_time == now assert point.temperature == 25.0 assert point.dewpoint_depression == 7.0 assert point.bl_height == 500.0 assert point.pwat == 30.0 assert point.surface_pressure_mb == 1013.0 end test "computes surface_refractivity and min_refractivity_gradient from the profile" do now = DateTime.truncate(DateTime.utc_now(), :second) profile = [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}, %{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0}, %{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0} ] grid_data = %{ {32.125, -97.375} => %{ profile: profile, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 } } [point] = Weather.build_grid_cache_rows(grid_data, now) assert is_float(point.surface_refractivity) assert is_float(point.refractivity_gradient) assert is_float(point.surface_rh) assert point.temp_850mb == 15.0 assert point.dewpoint_850mb == 10.0 end test "skips points with missing or physically impossible surface temperature" do now = DateTime.truncate(DateTime.utc_now(), :second) profile = [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0} ] grid_data = %{ {32.125, -97.375} => %{ profile: profile, surface_temp_c: nil, surface_dewpoint_c: nil, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 }, {32.250, -97.375} => %{ profile: profile, surface_temp_c: 250.0, surface_dewpoint_c: 200.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 }, {32.375, -97.375} => %{ profile: profile, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 } } result = Weather.build_grid_cache_rows(grid_data, now) assert length(result) == 1 assert hd(result).lat == 32.375 end test "with a bounds filter only derives in-bounds points" do # Filter-before-derive: when called with bounds, the builder must # skip the `SoundingParams.derive` work for every point outside # the viewport. Out-of-bounds points are left as raw profile maps # so the heavy per-point derivation cost is bounded by the # viewport size, not the full CONUS grid. now = DateTime.truncate(DateTime.utc_now(), :second) profile = [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0} ] base_profile = %{ profile: profile, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 } grid_data = %{ {32.125, -97.375} => base_profile, {40.000, -75.000} => base_profile, {45.000, -120.000} => base_profile } bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0} rows = Weather.build_grid_cache_rows(grid_data, now, bounds) assert length(rows) == 1 assert hd(rows).lat == 32.125 assert hd(rows).lon == -97.375 end test "with nil bounds derives every point (backwards compatible)" do now = DateTime.truncate(DateTime.utc_now(), :second) profile = [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0} ] grid_data = %{ {32.125, -97.375} => %{ profile: profile, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 }, {40.000, -75.000} => %{ profile: profile, surface_temp_c: 22.0, surface_dewpoint_c: 15.0, surface_pressure_mb: 1015.0, hpbl_m: 400.0, pwat_mm: 25.0 } } assert length(Weather.build_grid_cache_rows(grid_data, now, nil)) == 2 end test "explicit ducting_detected: false on the profile wins over derived true from sounding params" do # Regression guard: the `||` fallback used to clobber legitimate `false` # values written by the Rust ProfilesFile with whatever # `SoundingParams.derive/1` produced. The profile below has a steep # near-surface N drop (moist sfc → dry aloft) that derives ducts # = [_], so `sounding[:ducting_detected]` would be `true`. With the # explicit `false` on the profile map, the row must carry `false`. now = DateTime.truncate(DateTime.utc_now(), :second) strong_inversion_profile = [ %{"pres" => 1013.0, "tmpc" => 30.0, "dwpc" => 25.0, "hght" => 100.0}, %{"pres" => 1000.0, "tmpc" => 35.0, "dwpc" => -5.0, "hght" => 200.0}, %{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 0.0, "hght" => 800.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => -5.0, "hght" => 1500.0}, %{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0} ] # Sanity check: this profile does derive a duct from SoundingParams. derived = SoundingParams.derive(strong_inversion_profile) assert derived.ducting_detected == true grid_data = %{ {32.125, -97.375} => %{ profile: strong_inversion_profile, surface_temp_c: 30.0, surface_dewpoint_c: 25.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0, # Explicit false — persisted by the Rust worker. Must NOT be # clobbered by the derived-from-sounding true above. ducting_detected: false } } [row] = Weather.build_grid_cache_rows(grid_data, now) assert row.ducting == false end test "strips raw profile/duct_characteristics/surface_temp_c/surface_dewpoint_c fields" do now = DateTime.truncate(DateTime.utc_now(), :second) profile = [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0} ] grid_data = %{ {32.125, -97.375} => %{ profile: profile, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 } } [point] = Weather.build_grid_cache_rows(grid_data, now) refute Map.has_key?(point, :profile) refute Map.has_key?(point, :duct_characteristics) refute Map.has_key?(point, :surface_temp_c) refute Map.has_key?(point, :surface_dewpoint_c) end end describe "weather_point_detail/3" do test "returns all fields for a single grid point" do now = DateTime.truncate(DateTime.utc_now(), :second) insert_hrrr_profile(%{ lat: 32.125, lon: -97.375, valid_time: now, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0, min_refractivity_gradient: -280.0, ducting_detected: true }) result = Weather.weather_point_detail(32.15, -97.40, now) assert result.lat == 32.125 assert result.temperature == 25.0 assert result.surface_pressure_mb == 1013.0 end test "returns nil when no data at point" do now = DateTime.truncate(DateTime.utc_now(), :second) assert Weather.weather_point_detail(32.15, -97.40, now) == nil end test "falls back to ProfilesFile when GridCache is cold and DB has no grid points" do now = DateTime.truncate(DateTime.utc_now(), :second) grid_data = %{ {32.125, -97.375} => %{ surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0, profile: [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}, %{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0} ] } } ProfilesFile.write!(now, grid_data) result = Weather.weather_point_detail(32.15, -97.40, now) assert result.lat == 32.125 assert result.temperature == 25.0 assert result.surface_pressure_mb == 1013.0 end test "includes derived upper-air fields and strips raw profile data" do now = DateTime.truncate(DateTime.utc_now(), :second) profile = [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}, %{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0} ] duct_chars = [%{"base" => 300.0, "strength" => 40.0}] insert_hrrr_profile(%{ lat: 32.125, lon: -97.375, valid_time: now, surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, surface_refractivity: 320.0, hpbl_m: 500.0, pwat_mm: 30.0, min_refractivity_gradient: -280.0, ducting_detected: true, profile: profile, duct_characteristics: duct_chars }) result = Weather.weather_point_detail(32.15, -97.40, now) # Derived fields assert is_float(result.surface_rh) assert result.surface_refractivity == 320.0 assert result.temp_850mb == 15.0 assert result.duct_base_m == 300.0 assert result.duct_strength == 40.0 # Raw fields stripped refute Map.has_key?(result, :profile) refute Map.has_key?(result, :duct_characteristics) refute Map.has_key?(result, :surface_temp_c) refute Map.has_key?(result, :surface_dewpoint_c) end end describe "latest_grid_valid_time/0 + ProfilesFile fallback" do test "uses ProfilesFile.latest_valid_time when GridCache and DB are both empty" do vt = ~U[2026-04-14 18:00:00Z] ProfilesFile.write!(vt, %{ {32.125, -97.375} => %{surface_temp_c: 20.0, surface_dewpoint_c: 15.0} }) assert Weather.latest_grid_valid_time() == vt end end describe "load_weather_grid/1 + ProfilesFile fallback" do test "returns derived rows from ProfilesFile when GridCache is empty" do vt = ~U[2026-04-14 18:00:00Z] ProfilesFile.write!(vt, %{ {32.125, -97.375} => %{ surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0 }, {32.250, -97.375} => %{ surface_temp_c: 26.0, surface_dewpoint_c: 19.0, surface_pressure_mb: 1012.0, hpbl_m: 600.0, pwat_mm: 32.0 } }) bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0} rows = Weather.load_weather_grid(bounds) assert length(rows) == 2 assert Enum.any?(rows, &(&1.temperature == 25.0)) assert Enum.any?(rows, &(&1.temperature == 26.0)) end end describe "materialize_scalar_file/1" do alias Microwaveprop.Weather.ScalarFile test "writes a ScalarFile derived from the on-disk ProfilesFile" do vt = ~U[2026-04-28 12:00:00Z] ProfilesFile.write!(vt, %{ {32.125, -97.375} => %{ surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0, profile: [ %{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0}, %{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0} ] } }) refute ScalarFile.exists?(vt) :ok = Weather.materialize_scalar_file(vt) assert ScalarFile.exists?(vt) [row] = ScalarFile.read_bounds(vt, nil) assert row.temperature == 25.0 assert row.dewpoint_depression == 7.0 end test "is idempotent — does not re-write when the scalar file already exists" do vt = ~U[2026-04-28 13:00:00Z] ProfilesFile.write!(vt, %{ {32.125, -97.375} => %{ surface_temp_c: 22.0, surface_dewpoint_c: 12.0, surface_pressure_mb: 1010.0, hpbl_m: 600.0, pwat_mm: 24.0, profile: [] } }) :ok = Weather.materialize_scalar_file(vt) [first] = ScalarFile.read_bounds(vt, nil) # Mutate the underlying ProfilesFile, then call materialize again. # Idempotent skipping means the scalar file does not change. ProfilesFile.write!(vt, %{ {32.125, -97.375} => %{ surface_temp_c: 99.0, surface_dewpoint_c: 50.0, surface_pressure_mb: 999.0, hpbl_m: 999.0, pwat_mm: 99.0, profile: [] } }) :ok = Weather.materialize_scalar_file(vt) [second] = ScalarFile.read_bounds(vt, nil) assert second.temperature == first.temperature end test "is a no-op when the ProfilesFile does not exist" do vt = ~U[2026-04-28 14:00:00Z] assert Weather.materialize_scalar_file(vt) == :ok refute ScalarFile.exists?(vt) end end describe "weather_grid_at/2 → GridCache from ScalarFile" do alias Microwaveprop.Weather.ScalarFile setup do # Tests below assert end state in ETS. GridCache.clear() :ok end test "populates GridCache from ScalarFile on miss so the next read skips file IO" do vt = ~U[2026-04-28 15:00:00Z] ProfilesFile.write!(vt, %{ {32.125, -97.375} => %{ surface_temp_c: 25.0, surface_dewpoint_c: 18.0, surface_pressure_mb: 1013.0, hpbl_m: 500.0, pwat_mm: 30.0, profile: [] }, {32.250, -97.375} => %{ surface_temp_c: 26.0, surface_dewpoint_c: 19.0, surface_pressure_mb: 1012.0, hpbl_m: 600.0, pwat_mm: 32.0, profile: [] } }) # Materialize the ScalarFile so it's the source of truth. :ok = Weather.materialize_scalar_file(vt) assert ScalarFile.exists?(vt) assert GridCache.fetch(vt) == :miss # First call: miss → reads ScalarFile → populates GridCache. bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0} rows = Weather.weather_grid_at(vt, bounds) assert length(rows) == 2 # GridCache is now warm — confirms cache populate happened. assert {:ok, cached_rows} = GridCache.fetch(vt) assert length(cached_rows) == 2 # Wipe the on-disk ScalarFile. The next call should still return data # because GridCache is the hot path. _ = File.rm_rf(ScalarFile.dir_for(vt)) refute ScalarFile.exists?(vt) rows_again = Weather.weather_grid_at(vt, bounds) assert length(rows_again) == 2 end end # Merges `attrs` (a single HRRR grid cell) into the ProfilesFile for # `attrs.valid_time`. Re-reads any existing file so multiple calls for # the same valid_time accumulate cells, mirroring how the Rust worker # populates the grid in production. defp insert_hrrr_profile(%{lat: lat, lon: lon, valid_time: valid_time} = attrs) do existing = case ProfilesFile.read(valid_time) do {:ok, grid} -> grid {:error, _} -> %{} end profile_attrs = attrs |> Map.drop([:lat, :lon, :valid_time]) |> Map.put_new(:profile, []) grid = Map.put(existing, {lat, lon}, profile_attrs) ProfilesFile.write!(valid_time, grid) # write!/2 invalidates ProfilesFile's internal cache keyed by valid_time, # but the read helper is also cached separately here; clear on every write # so tests that rewrite the same valid_time see the new grid. Microwaveprop.Cache.clear() :ok end end