Graham McIntire
b67c0e88c0
Each /weather/tiles request was running the full ScalarFile decode path (File.read → gunzip → Msgpax.unpack → normalize) for any non-analysis valid_time, because GridCache deliberately skipped forecast hours to keep memory low. With 21 simultaneous viewport tiles all hitting the same few chunk files, each tile took 1.5-3.2s of contended IO + CPU. On a GridCache miss when a ScalarFile exists, hydrate GridCache from the file once (deduped via the existing claim_fill primitive) and serve all subsequent reads from ETS. Bound memory with prune_keep_latest/1 — keep the 24 most-recently-touched valid_times, which covers a full HRRR run (analysis + 18 forecasts) plus a few stragglers from the previous run. In the steady state, the first viewport read for a given forecast hour takes one full ScalarFile decode (~50-100 ms for 92k cells); every subsequent tile, point lookup, and viewport read for the same hour serves from ETS in <1 ms.
731 lines
23 KiB
Elixir
731 lines
23 KiB
Elixir
defmodule Microwaveprop.WeatherGridTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
|
|
alias Microwaveprop.Propagation.ProfilesFile
|
|
alias Microwaveprop.Weather
|
|
alias Microwaveprop.Weather.GridCache
|
|
alias Microwaveprop.Weather.SoundingParams
|
|
|
|
setup do
|
|
# GridCache ETS state leaks between tests (it's a named table outside the
|
|
# test sandbox). Clear it so each test gets a fresh cache and the disk
|
|
# load path actually runs.
|
|
GridCache.clear()
|
|
# Microwaveprop.Cache memoizes ProfilesFile.read/1 keyed by
|
|
# `(base_dir, valid_time)`; tests within this file share the scores_dir
|
|
# and frequently reuse `DateTime.utc_now()` truncated to seconds, so a
|
|
# prior test's cached read leaks into the next one's assertion.
|
|
Microwaveprop.Cache.clear()
|
|
|
|
dir =
|
|
Path.join(
|
|
System.tmp_dir!(),
|
|
"weather_grid_test_#{System.unique_integer([:positive])}"
|
|
)
|
|
|
|
File.mkdir_p!(Path.join(dir, "profiles"))
|
|
prev = Application.get_env(:microwaveprop, :propagation_scores_dir)
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, dir)
|
|
|
|
on_exit(fn ->
|
|
File.rm_rf!(dir)
|
|
|
|
if prev do
|
|
Application.put_env(:microwaveprop, :propagation_scores_dir, prev)
|
|
else
|
|
Application.delete_env(:microwaveprop, :propagation_scores_dir)
|
|
end
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
describe "load_weather_grid/1" do
|
|
test "returns grid data for latest valid_time within bounds" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true
|
|
})
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.250,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 26.0,
|
|
surface_dewpoint_c: 19.0,
|
|
surface_pressure_mb: 1012.0,
|
|
hpbl_m: 600.0,
|
|
pwat_mm: 32.0,
|
|
min_refractivity_gradient: -150.0,
|
|
ducting_detected: false
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
result = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(result) == 2
|
|
point = Enum.find(result, &(&1.lat == 32.125))
|
|
assert point.temperature == 25.0
|
|
assert point.dewpoint_depression == 7.0
|
|
assert point.bl_height == 500.0
|
|
assert point.pwat == 30.0
|
|
assert point.refractivity_gradient == -280.0
|
|
assert point.ducting == true
|
|
end
|
|
|
|
test "returns empty list when no data exists" do
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
assert Weather.load_weather_grid(bounds) == []
|
|
end
|
|
|
|
test "uses most recent valid_time only" do
|
|
old = DateTime.utc_now() |> DateTime.add(-7200, :second) |> DateTime.truncate(:second)
|
|
new = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: old,
|
|
surface_temp_c: 20.0,
|
|
surface_dewpoint_c: 15.0,
|
|
surface_pressure_mb: 1010.0,
|
|
hpbl_m: 400.0,
|
|
pwat_mm: 25.0
|
|
})
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: new,
|
|
surface_temp_c: 28.0,
|
|
surface_dewpoint_c: 20.0,
|
|
surface_pressure_mb: 1015.0,
|
|
hpbl_m: 700.0,
|
|
pwat_mm: 35.0
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
result = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(result) == 1
|
|
assert hd(result).temperature == 28.0
|
|
end
|
|
|
|
test "includes derived upper-air fields and strips raw profile data" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
duct_chars = [
|
|
%{"base" => 200.0, "strength" => 50.0},
|
|
%{"base" => 800.0, "strength" => 30.0}
|
|
]
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
surface_refractivity: 320.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true,
|
|
profile: profile,
|
|
duct_characteristics: duct_chars
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
[point] = Weather.load_weather_grid(bounds)
|
|
|
|
# Derived fields should be present
|
|
assert is_float(point.surface_rh)
|
|
assert point.surface_refractivity == 320.0
|
|
assert point.temp_850mb == 15.0
|
|
assert point.dewpoint_850mb == 10.0
|
|
assert is_float(point.lapse_rate)
|
|
assert point.duct_base_m == 200.0
|
|
assert point.duct_strength == 50.0
|
|
|
|
# Raw bulky fields should be stripped
|
|
refute Map.has_key?(point, :profile)
|
|
refute Map.has_key?(point, :duct_characteristics)
|
|
refute Map.has_key?(point, :surface_temp_c)
|
|
refute Map.has_key?(point, :surface_dewpoint_c)
|
|
end
|
|
end
|
|
|
|
describe "build_grid_cache_rows/2" do
|
|
test "returns an empty list for empty grid_data" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
assert Weather.build_grid_cache_rows(%{}, now) == []
|
|
end
|
|
|
|
test "builds a derived row for each in-memory grid point without hitting the database" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
},
|
|
{32.250, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 26.0,
|
|
surface_dewpoint_c: 19.0,
|
|
surface_pressure_mb: 1012.0,
|
|
hpbl_m: 600.0,
|
|
pwat_mm: 32.0
|
|
}
|
|
}
|
|
|
|
result = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
assert length(result) == 2
|
|
|
|
point = Enum.find(result, &(&1.lat == 32.125))
|
|
assert point.lon == -97.375
|
|
assert point.valid_time == now
|
|
assert point.temperature == 25.0
|
|
assert point.dewpoint_depression == 7.0
|
|
assert point.bl_height == 500.0
|
|
assert point.pwat == 30.0
|
|
assert point.surface_pressure_mb == 1013.0
|
|
end
|
|
|
|
test "computes surface_refractivity and min_refractivity_gradient from the profile" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 700.0, "tmpc" => 5.0, "dwpc" => -2.0, "hght" => 3100.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
}
|
|
}
|
|
|
|
[point] = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
assert is_float(point.surface_refractivity)
|
|
assert is_float(point.refractivity_gradient)
|
|
assert is_float(point.surface_rh)
|
|
assert point.temp_850mb == 15.0
|
|
assert point.dewpoint_850mb == 10.0
|
|
end
|
|
|
|
test "skips points with missing or physically impossible surface temperature" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: nil,
|
|
surface_dewpoint_c: nil,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
},
|
|
{32.250, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 250.0,
|
|
surface_dewpoint_c: 200.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
},
|
|
{32.375, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
}
|
|
}
|
|
|
|
result = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
assert length(result) == 1
|
|
assert hd(result).lat == 32.375
|
|
end
|
|
|
|
test "with a bounds filter only derives in-bounds points" do
|
|
# Filter-before-derive: when called with bounds, the builder must
|
|
# skip the `SoundingParams.derive` work for every point outside
|
|
# the viewport. Out-of-bounds points are left as raw profile maps
|
|
# so the heavy per-point derivation cost is bounded by the
|
|
# viewport size, not the full CONUS grid.
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
|
|
]
|
|
|
|
base_profile = %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
}
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => base_profile,
|
|
{40.000, -75.000} => base_profile,
|
|
{45.000, -120.000} => base_profile
|
|
}
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
rows = Weather.build_grid_cache_rows(grid_data, now, bounds)
|
|
|
|
assert length(rows) == 1
|
|
assert hd(rows).lat == 32.125
|
|
assert hd(rows).lon == -97.375
|
|
end
|
|
|
|
test "with nil bounds derives every point (backwards compatible)" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
},
|
|
{40.000, -75.000} => %{
|
|
profile: profile,
|
|
surface_temp_c: 22.0,
|
|
surface_dewpoint_c: 15.0,
|
|
surface_pressure_mb: 1015.0,
|
|
hpbl_m: 400.0,
|
|
pwat_mm: 25.0
|
|
}
|
|
}
|
|
|
|
assert length(Weather.build_grid_cache_rows(grid_data, now, nil)) == 2
|
|
end
|
|
|
|
test "explicit ducting_detected: false on the profile wins over derived true from sounding params" do
|
|
# Regression guard: the `||` fallback used to clobber legitimate `false`
|
|
# values written by the Rust ProfilesFile with whatever
|
|
# `SoundingParams.derive/1` produced. The profile below has a steep
|
|
# near-surface N drop (moist sfc → dry aloft) that derives ducts
|
|
# = [_], so `sounding[:ducting_detected]` would be `true`. With the
|
|
# explicit `false` on the profile map, the row must carry `false`.
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
strong_inversion_profile = [
|
|
%{"pres" => 1013.0, "tmpc" => 30.0, "dwpc" => 25.0, "hght" => 100.0},
|
|
%{"pres" => 1000.0, "tmpc" => 35.0, "dwpc" => -5.0, "hght" => 200.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 0.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => -5.0, "hght" => 1500.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
# Sanity check: this profile does derive a duct from SoundingParams.
|
|
derived = SoundingParams.derive(strong_inversion_profile)
|
|
assert derived.ducting_detected == true
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: strong_inversion_profile,
|
|
surface_temp_c: 30.0,
|
|
surface_dewpoint_c: 25.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
# Explicit false — persisted by the Rust worker. Must NOT be
|
|
# clobbered by the derived-from-sounding true above.
|
|
ducting_detected: false
|
|
}
|
|
}
|
|
|
|
[row] = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
assert row.ducting == false
|
|
end
|
|
|
|
test "strips raw profile/duct_characteristics/surface_temp_c/surface_dewpoint_c fields" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 925.0, "tmpc" => 20.0, "dwpc" => 14.0, "hght" => 800.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
|
|
]
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
profile: profile,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
}
|
|
}
|
|
|
|
[point] = Weather.build_grid_cache_rows(grid_data, now)
|
|
|
|
refute Map.has_key?(point, :profile)
|
|
refute Map.has_key?(point, :duct_characteristics)
|
|
refute Map.has_key?(point, :surface_temp_c)
|
|
refute Map.has_key?(point, :surface_dewpoint_c)
|
|
end
|
|
end
|
|
|
|
describe "weather_point_detail/3" do
|
|
test "returns all fields for a single grid point" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true
|
|
})
|
|
|
|
result = Weather.weather_point_detail(32.15, -97.40, now)
|
|
|
|
assert result.lat == 32.125
|
|
assert result.temperature == 25.0
|
|
assert result.surface_pressure_mb == 1013.0
|
|
end
|
|
|
|
test "returns nil when no data at point" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
assert Weather.weather_point_detail(32.15, -97.40, now) == nil
|
|
end
|
|
|
|
test "falls back to ProfilesFile when GridCache is cold and DB has no grid points" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
grid_data = %{
|
|
{32.125, -97.375} => %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
profile: [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
}
|
|
}
|
|
|
|
ProfilesFile.write!(now, grid_data)
|
|
|
|
result = Weather.weather_point_detail(32.15, -97.40, now)
|
|
|
|
assert result.lat == 32.125
|
|
assert result.temperature == 25.0
|
|
assert result.surface_pressure_mb == 1013.0
|
|
end
|
|
|
|
test "includes derived upper-air fields and strips raw profile data" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
profile = [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
|
|
duct_chars = [%{"base" => 300.0, "strength" => 40.0}]
|
|
|
|
insert_hrrr_profile(%{
|
|
lat: 32.125,
|
|
lon: -97.375,
|
|
valid_time: now,
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
surface_refractivity: 320.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
min_refractivity_gradient: -280.0,
|
|
ducting_detected: true,
|
|
profile: profile,
|
|
duct_characteristics: duct_chars
|
|
})
|
|
|
|
result = Weather.weather_point_detail(32.15, -97.40, now)
|
|
|
|
# Derived fields
|
|
assert is_float(result.surface_rh)
|
|
assert result.surface_refractivity == 320.0
|
|
assert result.temp_850mb == 15.0
|
|
assert result.duct_base_m == 300.0
|
|
assert result.duct_strength == 40.0
|
|
|
|
# Raw fields stripped
|
|
refute Map.has_key?(result, :profile)
|
|
refute Map.has_key?(result, :duct_characteristics)
|
|
refute Map.has_key?(result, :surface_temp_c)
|
|
refute Map.has_key?(result, :surface_dewpoint_c)
|
|
end
|
|
end
|
|
|
|
describe "latest_grid_valid_time/0 + ProfilesFile fallback" do
|
|
test "uses ProfilesFile.latest_valid_time when GridCache and DB are both empty" do
|
|
vt = ~U[2026-04-14 18:00:00Z]
|
|
|
|
ProfilesFile.write!(vt, %{
|
|
{32.125, -97.375} => %{surface_temp_c: 20.0, surface_dewpoint_c: 15.0}
|
|
})
|
|
|
|
assert Weather.latest_grid_valid_time() == vt
|
|
end
|
|
end
|
|
|
|
describe "load_weather_grid/1 + ProfilesFile fallback" do
|
|
test "returns derived rows from ProfilesFile when GridCache is empty" do
|
|
vt = ~U[2026-04-14 18:00:00Z]
|
|
|
|
ProfilesFile.write!(vt, %{
|
|
{32.125, -97.375} => %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0
|
|
},
|
|
{32.250, -97.375} => %{
|
|
surface_temp_c: 26.0,
|
|
surface_dewpoint_c: 19.0,
|
|
surface_pressure_mb: 1012.0,
|
|
hpbl_m: 600.0,
|
|
pwat_mm: 32.0
|
|
}
|
|
})
|
|
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
rows = Weather.load_weather_grid(bounds)
|
|
|
|
assert length(rows) == 2
|
|
assert Enum.any?(rows, &(&1.temperature == 25.0))
|
|
assert Enum.any?(rows, &(&1.temperature == 26.0))
|
|
end
|
|
end
|
|
|
|
describe "materialize_scalar_file/1" do
|
|
alias Microwaveprop.Weather.ScalarFile
|
|
|
|
test "writes a ScalarFile derived from the on-disk ProfilesFile" do
|
|
vt = ~U[2026-04-28 12:00:00Z]
|
|
|
|
ProfilesFile.write!(vt, %{
|
|
{32.125, -97.375} => %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
profile: [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 500.0, "tmpc" => -15.0, "dwpc" => -25.0, "hght" => 5600.0}
|
|
]
|
|
}
|
|
})
|
|
|
|
refute ScalarFile.exists?(vt)
|
|
|
|
:ok = Weather.materialize_scalar_file(vt)
|
|
|
|
assert ScalarFile.exists?(vt)
|
|
[row] = ScalarFile.read_bounds(vt, nil)
|
|
assert row.temperature == 25.0
|
|
assert row.dewpoint_depression == 7.0
|
|
end
|
|
|
|
test "is idempotent — does not re-write when the scalar file already exists" do
|
|
vt = ~U[2026-04-28 13:00:00Z]
|
|
|
|
ProfilesFile.write!(vt, %{
|
|
{32.125, -97.375} => %{
|
|
surface_temp_c: 22.0,
|
|
surface_dewpoint_c: 12.0,
|
|
surface_pressure_mb: 1010.0,
|
|
hpbl_m: 600.0,
|
|
pwat_mm: 24.0,
|
|
profile: []
|
|
}
|
|
})
|
|
|
|
:ok = Weather.materialize_scalar_file(vt)
|
|
[first] = ScalarFile.read_bounds(vt, nil)
|
|
|
|
# Mutate the underlying ProfilesFile, then call materialize again.
|
|
# Idempotent skipping means the scalar file does not change.
|
|
ProfilesFile.write!(vt, %{
|
|
{32.125, -97.375} => %{
|
|
surface_temp_c: 99.0,
|
|
surface_dewpoint_c: 50.0,
|
|
surface_pressure_mb: 999.0,
|
|
hpbl_m: 999.0,
|
|
pwat_mm: 99.0,
|
|
profile: []
|
|
}
|
|
})
|
|
|
|
:ok = Weather.materialize_scalar_file(vt)
|
|
[second] = ScalarFile.read_bounds(vt, nil)
|
|
|
|
assert second.temperature == first.temperature
|
|
end
|
|
|
|
test "is a no-op when the ProfilesFile does not exist" do
|
|
vt = ~U[2026-04-28 14:00:00Z]
|
|
assert Weather.materialize_scalar_file(vt) == :ok
|
|
refute ScalarFile.exists?(vt)
|
|
end
|
|
end
|
|
|
|
describe "weather_grid_at/2 → GridCache from ScalarFile" do
|
|
alias Microwaveprop.Weather.ScalarFile
|
|
|
|
setup do
|
|
# Tests below assert end state in ETS.
|
|
GridCache.clear()
|
|
:ok
|
|
end
|
|
|
|
test "populates GridCache from ScalarFile on miss so the next read skips file IO" do
|
|
vt = ~U[2026-04-28 15:00:00Z]
|
|
|
|
ProfilesFile.write!(vt, %{
|
|
{32.125, -97.375} => %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 30.0,
|
|
profile: []
|
|
},
|
|
{32.250, -97.375} => %{
|
|
surface_temp_c: 26.0,
|
|
surface_dewpoint_c: 19.0,
|
|
surface_pressure_mb: 1012.0,
|
|
hpbl_m: 600.0,
|
|
pwat_mm: 32.0,
|
|
profile: []
|
|
}
|
|
})
|
|
|
|
# Materialize the ScalarFile so it's the source of truth.
|
|
:ok = Weather.materialize_scalar_file(vt)
|
|
assert ScalarFile.exists?(vt)
|
|
assert GridCache.fetch(vt) == :miss
|
|
|
|
# First call: miss → reads ScalarFile → populates GridCache.
|
|
bounds = %{"south" => 32.0, "north" => 33.0, "west" => -98.0, "east" => -97.0}
|
|
rows = Weather.weather_grid_at(vt, bounds)
|
|
assert length(rows) == 2
|
|
|
|
# GridCache is now warm — confirms cache populate happened.
|
|
assert {:ok, cached_rows} = GridCache.fetch(vt)
|
|
assert length(cached_rows) == 2
|
|
|
|
# Wipe the on-disk ScalarFile. The next call should still return data
|
|
# because GridCache is the hot path.
|
|
_ = File.rm_rf(ScalarFile.dir_for(vt))
|
|
refute ScalarFile.exists?(vt)
|
|
|
|
rows_again = Weather.weather_grid_at(vt, bounds)
|
|
assert length(rows_again) == 2
|
|
end
|
|
end
|
|
|
|
# Merges `attrs` (a single HRRR grid cell) into the ProfilesFile for
|
|
# `attrs.valid_time`. Re-reads any existing file so multiple calls for
|
|
# the same valid_time accumulate cells, mirroring how the Rust worker
|
|
# populates the grid in production.
|
|
defp insert_hrrr_profile(%{lat: lat, lon: lon, valid_time: valid_time} = attrs) do
|
|
existing =
|
|
case ProfilesFile.read(valid_time) do
|
|
{:ok, grid} -> grid
|
|
{:error, _} -> %{}
|
|
end
|
|
|
|
profile_attrs =
|
|
attrs
|
|
|> Map.drop([:lat, :lon, :valid_time])
|
|
|> Map.put_new(:profile, [])
|
|
|
|
grid = Map.put(existing, {lat, lon}, profile_attrs)
|
|
|
|
ProfilesFile.write!(valid_time, grid)
|
|
# write!/2 invalidates ProfilesFile's internal cache keyed by valid_time,
|
|
# but the read helper is also cached separately here; clear on every write
|
|
# so tests that rewrite the same valid_time see the new grid.
|
|
Microwaveprop.Cache.clear()
|
|
|
|
:ok
|
|
end
|
|
end
|