Graham McIntire
805bbff330
AsosAdjustmentWorker fires every 10 minutes and loads every row of `hrrr_profiles` on the grid for the latest valid_time. The old query selected `profile: h.profile` — ~1.3 KB of JSONB × 92k grid points ≈ 120 MB of JSONB per tick. Postgrex's Jason.decode! ran inline for each row and blew past the 15 s pool checkout window, so every tick was killing connections with: DBConnection.ConnectionError: client timed out because it queued and checked out the connection for longer than 15000ms `score_grid_point/4` only touched the profile array to re-derive `min_refractivity_gradient`, but `hrrr_profiles` already persists that value as a scalar column at ingestion time. Teach `derive_from_hrrr/1` to honour the persisted scalar when it's present and drop `h.profile` from the worker's SELECT list. Net effect: same score math, ~1% of the JSONB transfer, tick stays under the pool deadline. Covered by a new scorer test that feeds a profile map with no `:profile` list and asserts the refractivity factor still reflects the persisted gradient instead of the neutral baseline.
358 lines
12 KiB
Elixir
358 lines
12 KiB
Elixir
defmodule Microwaveprop.PropagationTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
|
|
alias Microwaveprop.Propagation
|
|
alias Microwaveprop.Propagation.GridScore
|
|
alias Microwaveprop.Propagation.ScoreCache
|
|
|
|
setup do
|
|
ScoreCache.clear()
|
|
:ok
|
|
end
|
|
|
|
describe "score_grid_point/4" do
|
|
test "scores a single point for all bands" do
|
|
hrrr_profile = %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
wind_u: 3.0,
|
|
wind_v: 2.0,
|
|
cloud_cover_pct: 15.0,
|
|
precip_mm: 0.0,
|
|
profile: [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 975.0, "tmpc" => 22.0, "dwpc" => 15.0, "hght" => 350.0},
|
|
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
|
|
]
|
|
}
|
|
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
results = Propagation.score_grid_point(hrrr_profile, valid_time, 33.0, -97.0)
|
|
|
|
assert length(results) == 19
|
|
|
|
Enum.each(results, fn result ->
|
|
assert result.score >= 0 and result.score <= 100
|
|
assert is_map(result.factors)
|
|
assert is_integer(result.band_mhz)
|
|
end)
|
|
end
|
|
|
|
test "NEXRAD reflectivity drops the 24 GHz rain score when HRRR precip_mm is 0" do
|
|
# Same profile, once without NEXRAD and once with a heavy-rain dBZ. The
|
|
# 24 GHz rain-score factor must drop because NEXRAD can report rain that
|
|
# HRRR's hourly precip accumulation hasn't caught yet.
|
|
base_profile = %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
wind_u: 3.0,
|
|
wind_v: 2.0,
|
|
cloud_cover_pct: 15.0,
|
|
precip_mm: 0.0,
|
|
profile: [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
|
|
]
|
|
}
|
|
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
without = Propagation.score_grid_point(base_profile, valid_time, 33.0, -97.0)
|
|
|
|
with_nexrad =
|
|
base_profile
|
|
|> Map.put(:nexrad_max_reflectivity_dbz, 45.0)
|
|
|> Propagation.score_grid_point(valid_time, 33.0, -97.0)
|
|
|
|
rain_dry = Enum.find(without, &(&1.band_mhz == 24_000)).factors.rain
|
|
rain_wet = Enum.find(with_nexrad, &(&1.band_mhz == 24_000)).factors.rain
|
|
|
|
assert rain_wet < rain_dry
|
|
end
|
|
|
|
test "uses a persisted min_refractivity_gradient scalar without decoding the profile array" do
|
|
# AsosAdjustmentWorker loads 92k HRRR profile rows per 10-min tick and
|
|
# can't afford to pull the ~1 KB JSONB `profile` column for each one —
|
|
# Postgrex's Jason.decode! per row blew past the 15s pool checkout.
|
|
# hrrr_profiles persists min_refractivity_gradient as a scalar at
|
|
# ingestion time, so score_grid_point must honour it when the profile
|
|
# array is absent instead of trying to re-derive from a missing list.
|
|
hrrr_profile_without_array = %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
pwat_mm: 28.0,
|
|
wind_u: 3.0,
|
|
wind_v: 2.0,
|
|
cloud_cover_pct: 15.0,
|
|
precip_mm: 0.0,
|
|
min_refractivity_gradient: -400.0
|
|
}
|
|
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
results = Propagation.score_grid_point(hrrr_profile_without_array, valid_time, 33.0, -97.0)
|
|
|
|
assert length(results) == 19
|
|
|
|
Enum.each(results, fn result ->
|
|
assert result.score >= 0 and result.score <= 100
|
|
end)
|
|
|
|
result_10g = Enum.find(results, &(&1.band_mhz == 10_000))
|
|
# A strong negative refractivity gradient (< -300 N/km) should leave
|
|
# the refractivity factor visibly positive; if score_grid_point had
|
|
# silently used 0.0 we'd see the neutral baseline instead.
|
|
assert result_10g.factors.refractivity > 55
|
|
end
|
|
|
|
test "NEXRAD is ignored when HRRR precip_mm already reports heavier rain" do
|
|
base_profile = %{
|
|
surface_temp_c: 25.0,
|
|
surface_dewpoint_c: 18.0,
|
|
surface_pressure_mb: 1013.0,
|
|
hpbl_m: 500.0,
|
|
wind_u: 3.0,
|
|
wind_v: 2.0,
|
|
cloud_cover_pct: 15.0,
|
|
# 30 mm/hr → far above the ~3 mm/hr NEXRAD gives for 30 dBZ, so HRRR wins.
|
|
precip_mm: 30.0,
|
|
profile: [
|
|
%{"pres" => 1000.0, "tmpc" => 25.0, "dwpc" => 18.0, "hght" => 100.0},
|
|
%{"pres" => 950.0, "tmpc" => 19.0, "dwpc" => 10.0, "hght" => 600.0}
|
|
]
|
|
}
|
|
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
with_light_nexrad =
|
|
base_profile
|
|
|> Map.put(:nexrad_max_reflectivity_dbz, 30.0)
|
|
|> Propagation.score_grid_point(valid_time, 33.0, -97.0)
|
|
|
|
without = Propagation.score_grid_point(base_profile, valid_time, 33.0, -97.0)
|
|
|
|
rain_no_nx = Enum.find(without, &(&1.band_mhz == 24_000)).factors.rain
|
|
rain_with_nx = Enum.find(with_light_nexrad, &(&1.band_mhz == 24_000)).factors.rain
|
|
|
|
assert rain_with_nx == rain_no_nx
|
|
end
|
|
end
|
|
|
|
describe "upsert_scores/1" do
|
|
test "inserts scores" do
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
scores = [
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: %{humidity: 90}},
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 24_000, score: 60, factors: %{humidity: 40}}
|
|
]
|
|
|
|
assert {:ok, 2} = Propagation.upsert_scores(scores)
|
|
assert Repo.aggregate(GridScore, :count) == 2
|
|
end
|
|
|
|
test "upserts on conflict" do
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
scores = [
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: %{}}
|
|
]
|
|
|
|
Propagation.upsert_scores(scores)
|
|
Propagation.upsert_scores([%{hd(scores) | score: 80}])
|
|
|
|
assert Repo.aggregate(GridScore, :count) == 1
|
|
end
|
|
end
|
|
|
|
describe "latest_scores/1" do
|
|
test "returns latest scores for a band" do
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
scores = [
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: %{}},
|
|
%{lat: 36.0, lon: -96.0, valid_time: valid_time, band_mhz: 10_000, score: 80, factors: %{}}
|
|
]
|
|
|
|
Propagation.upsert_scores(scores)
|
|
results = Propagation.latest_scores(10_000)
|
|
assert length(results) == 2
|
|
end
|
|
|
|
test "returns empty list when no data" do
|
|
assert Propagation.latest_scores(10_000) == []
|
|
end
|
|
end
|
|
|
|
describe "scores_at/3 with cache" do
|
|
test "populates the cache on a DB miss" do
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
Propagation.upsert_scores([
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: %{}}
|
|
])
|
|
|
|
assert ScoreCache.fetch(10_000, valid_time) == :miss
|
|
_ = Propagation.scores_at(10_000, valid_time)
|
|
assert {:ok, [%{lat: 35.0, lon: -97.0, score: 75}]} = ScoreCache.fetch(10_000, valid_time)
|
|
end
|
|
|
|
test "returns cached bounds-filtered results without hitting the DB" do
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
# Seed the cache directly — DB is empty, so if the result matches the
|
|
# cached payload we know the DB was not consulted.
|
|
ScoreCache.put(10_000, valid_time, [
|
|
%{lat: 32.0, lon: -97.0, score: 75},
|
|
%{lat: 40.0, lon: -74.0, score: 50}
|
|
])
|
|
|
|
assert Repo.aggregate(GridScore, :count) == 0
|
|
|
|
bounds = %{"south" => 30.0, "north" => 35.0, "west" => -100.0, "east" => -95.0}
|
|
result = Propagation.scores_at(10_000, valid_time, bounds)
|
|
|
|
assert length(result) == 1
|
|
assert [%{lat: 32.0, lon: -97.0, score: 75}] = result
|
|
end
|
|
|
|
test "returns empty list when no data in cache or DB" do
|
|
assert Propagation.scores_at(10_000, ~U[2026-07-15 13:00:00Z]) == []
|
|
end
|
|
end
|
|
|
|
describe "point_forecast/3 with cache" do
|
|
test "returns forecast timeline from cached scores when warm" do
|
|
t1 = DateTime.truncate(DateTime.add(DateTime.utc_now(), 3600, :second), :second)
|
|
t2 = DateTime.add(t1, 3600, :second)
|
|
|
|
ScoreCache.put(10_000, t1, [%{lat: 32.875, lon: -97.0, score: 70}])
|
|
ScoreCache.put(10_000, t2, [%{lat: 32.875, lon: -97.0, score: 75}])
|
|
|
|
assert Repo.aggregate(GridScore, :count) == 0
|
|
|
|
result = Propagation.point_forecast(10_000, 32.875, -97.0)
|
|
|
|
assert [
|
|
%{valid_time: ^t1, score: 70},
|
|
%{valid_time: ^t2, score: 75}
|
|
] = result
|
|
end
|
|
|
|
test "filters past valid_times out" do
|
|
past = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
|
|
future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.truncate(:second)
|
|
|
|
ScoreCache.put(10_000, past, [%{lat: 32.875, lon: -97.0, score: 60}])
|
|
ScoreCache.put(10_000, future, [%{lat: 32.875, lon: -97.0, score: 80}])
|
|
|
|
result = Propagation.point_forecast(10_000, 32.875, -97.0)
|
|
assert [%{valid_time: ^future, score: 80}] = result
|
|
end
|
|
|
|
test "snaps coordinates to the nearest grid point" do
|
|
valid_time = DateTime.truncate(DateTime.add(DateTime.utc_now(), 3600, :second), :second)
|
|
ScoreCache.put(10_000, valid_time, [%{lat: 32.875, lon: -97.0, score: 65}])
|
|
|
|
# Off-grid query snaps to 32.875, -97.0
|
|
result = Propagation.point_forecast(10_000, 32.88, -96.98)
|
|
assert [%{valid_time: ^valid_time, score: 65}] = result
|
|
end
|
|
|
|
test "returns empty list when no cached or DB data for the point" do
|
|
assert Propagation.point_forecast(10_000, 32.875, -97.0) == []
|
|
end
|
|
end
|
|
|
|
describe "available_valid_times/1 with cache" do
|
|
test "returns cached valid_times without hitting the DB when cache is warm" do
|
|
t1 = DateTime.truncate(DateTime.utc_now(), :second)
|
|
t2 = DateTime.add(t1, 3600, :second)
|
|
|
|
ScoreCache.put(10_000, t1, [])
|
|
ScoreCache.put(10_000, t2, [])
|
|
|
|
assert Repo.aggregate(GridScore, :count) == 0
|
|
assert Propagation.available_valid_times(10_000) == [t1, t2]
|
|
end
|
|
|
|
test "filters cached times older than the 1-hour cutoff" do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
stale = DateTime.add(now, -7200, :second)
|
|
fresh = DateTime.add(now, 1800, :second)
|
|
|
|
ScoreCache.put(10_000, stale, [])
|
|
ScoreCache.put(10_000, fresh, [])
|
|
|
|
assert Propagation.available_valid_times(10_000) == [fresh]
|
|
end
|
|
|
|
test "falls back to DB on cold cache" do
|
|
valid_time = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
Propagation.upsert_scores([
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: %{}}
|
|
])
|
|
|
|
assert ScoreCache.valid_times(10_000) == []
|
|
assert Propagation.available_valid_times(10_000) == [valid_time]
|
|
end
|
|
end
|
|
|
|
describe "warm_cache_and_broadcast/2" do
|
|
test "loads scores from DB into the cache" do
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
Propagation.upsert_scores([
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: %{}},
|
|
%{lat: 36.0, lon: -96.0, valid_time: valid_time, band_mhz: 10_000, score: 80, factors: %{}}
|
|
])
|
|
|
|
assert ScoreCache.fetch(10_000, valid_time) == :miss
|
|
|
|
Propagation.warm_cache_and_broadcast(10_000, valid_time)
|
|
ScoreCache.sync()
|
|
|
|
assert {:ok, scores} = ScoreCache.fetch(10_000, valid_time)
|
|
assert length(scores) == 2
|
|
end
|
|
|
|
test "only warms the requested band" do
|
|
valid_time = ~U[2026-07-15 13:00:00Z]
|
|
|
|
Propagation.upsert_scores([
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 10_000, score: 75, factors: %{}},
|
|
%{lat: 35.0, lon: -97.0, valid_time: valid_time, band_mhz: 24_000, score: 30, factors: %{}}
|
|
])
|
|
|
|
Propagation.warm_cache_and_broadcast(10_000, valid_time)
|
|
ScoreCache.sync()
|
|
|
|
assert {:ok, [_]} = ScoreCache.fetch(10_000, valid_time)
|
|
assert ScoreCache.fetch(24_000, valid_time) == :miss
|
|
end
|
|
end
|
|
|
|
describe "latest_valid_time/0" do
|
|
test "returns nil when empty" do
|
|
assert Propagation.latest_valid_time() == nil
|
|
end
|
|
|
|
test "returns most recent time" do
|
|
t1 = ~U[2026-07-15 12:00:00Z]
|
|
t2 = ~U[2026-07-15 13:00:00Z]
|
|
|
|
Propagation.upsert_scores([
|
|
%{lat: 35.0, lon: -97.0, valid_time: t1, band_mhz: 10_000, score: 50, factors: %{}},
|
|
%{lat: 35.0, lon: -97.0, valid_time: t2, band_mhz: 10_000, score: 60, factors: %{}}
|
|
])
|
|
|
|
assert Propagation.latest_valid_time() == t2
|
|
end
|
|
end
|
|
end
|