Stop AsosAdjustmentWorker from yanking 120 MB of JSONB per tick
AsosAdjustmentWorker fires every 10 minutes and loads every row of `hrrr_profiles` on the grid for the latest valid_time. The old query selected `profile: h.profile` — ~1.3 KB of JSONB × 92k grid points ≈ 120 MB of JSONB per tick. Postgrex's Jason.decode! ran inline for each row and blew past the 15 s pool checkout window, so every tick was killing connections with: DBConnection.ConnectionError: client timed out because it queued and checked out the connection for longer than 15000ms `score_grid_point/4` only touched the profile array to re-derive `min_refractivity_gradient`, but `hrrr_profiles` already persists that value as a scalar column at ingestion time. Teach `derive_from_hrrr/1` to honour the persisted scalar when it's present and drop `h.profile` from the worker's SELECT list. Net effect: same score math, ~1% of the JSONB transfer, tick stays under the pool deadline. Covered by a new scorer test that feeds a profile map with no `:profile` list and asserts the refractivity factor still reflects the persisted gradient instead of the neutral baseline.
This commit is contained in:
parent
d49d9f0cc9
commit
805bbff330
3 changed files with 49 additions and 2 deletions
|
|
@ -439,6 +439,13 @@ defmodule Microwaveprop.Propagation do
|
|||
Repo.one(from(gs in GridScore, where: gs.band_mhz == ^band_mhz, select: max(gs.valid_time)))
|
||||
end
|
||||
|
||||
# Prefer the persisted scalar — `hrrr_profiles` already stored this at
|
||||
# ingestion time and AsosAdjustmentWorker loads 92k rows per tick without
|
||||
# the JSONB `profile` column to avoid a Jason.decode! storm on the DB pool.
|
||||
defp derive_from_hrrr(%{min_refractivity_gradient: grad}) when is_number(grad) do
|
||||
%{min_refractivity_gradient: grad * 1.0}
|
||||
end
|
||||
|
||||
defp derive_from_hrrr(%{profile: profile}) when is_list(profile) and length(profile) >= 3 do
|
||||
case SoundingParams.derive(profile) do
|
||||
nil -> %{}
|
||||
|
|
|
|||
|
|
@ -147,6 +147,11 @@ defmodule Microwaveprop.Workers.AsosAdjustmentWorker do
|
|||
end)
|
||||
end
|
||||
|
||||
# Deliberately DOES NOT select h.profile. At 92k grid points per tick that
|
||||
# column is ~120 MB of JSONB and Postgrex's per-row Jason.decode! blew past
|
||||
# the 15s pool checkout window. score_grid_point/4 already prefers the
|
||||
# persisted min_refractivity_gradient scalar over re-deriving from the
|
||||
# profile array, so leaving the array behind is functionally transparent.
|
||||
defp load_hrrr_profiles(valid_time) do
|
||||
Repo.all(
|
||||
from(h in HrrrProfile,
|
||||
|
|
@ -163,8 +168,7 @@ defmodule Microwaveprop.Workers.AsosAdjustmentWorker do
|
|||
hpbl_m: h.hpbl_m,
|
||||
pwat_mm: h.pwat_mm,
|
||||
ducting_detected: h.ducting_detected,
|
||||
duct_characteristics: h.duct_characteristics,
|
||||
profile: h.profile
|
||||
duct_characteristics: h.duct_characteristics
|
||||
}
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -74,6 +74,42 @@ defmodule Microwaveprop.PropagationTest do
|
|||
assert rain_wet < rain_dry
|
||||
end
|
||||
|
||||
test "uses a persisted min_refractivity_gradient scalar without decoding the profile array" do
|
||||
# AsosAdjustmentWorker loads 92k HRRR profile rows per 10-min tick and
|
||||
# can't afford to pull the ~1 KB JSONB `profile` column for each one —
|
||||
# Postgrex's Jason.decode! per row blew past the 15s pool checkout.
|
||||
# hrrr_profiles persists min_refractivity_gradient as a scalar at
|
||||
# ingestion time, so score_grid_point must honour it when the profile
|
||||
# array is absent instead of trying to re-derive from a missing list.
|
||||
hrrr_profile_without_array = %{
|
||||
surface_temp_c: 25.0,
|
||||
surface_dewpoint_c: 18.0,
|
||||
surface_pressure_mb: 1013.0,
|
||||
hpbl_m: 500.0,
|
||||
pwat_mm: 28.0,
|
||||
wind_u: 3.0,
|
||||
wind_v: 2.0,
|
||||
cloud_cover_pct: 15.0,
|
||||
precip_mm: 0.0,
|
||||
min_refractivity_gradient: -400.0
|
||||
}
|
||||
|
||||
valid_time = ~U[2026-07-15 13:00:00Z]
|
||||
results = Propagation.score_grid_point(hrrr_profile_without_array, valid_time, 33.0, -97.0)
|
||||
|
||||
assert length(results) == 19
|
||||
|
||||
Enum.each(results, fn result ->
|
||||
assert result.score >= 0 and result.score <= 100
|
||||
end)
|
||||
|
||||
result_10g = Enum.find(results, &(&1.band_mhz == 10_000))
|
||||
# A strong negative refractivity gradient (< -300 N/km) should leave
|
||||
# the refractivity factor visibly positive; if score_grid_point had
|
||||
# silently used 0.0 we'd see the neutral baseline instead.
|
||||
assert result_10g.factors.refractivity > 55
|
||||
end
|
||||
|
||||
test "NEXRAD is ignored when HRRR precip_mm already reports heavier rain" do
|
||||
base_profile = %{
|
||||
surface_temp_c: 25.0,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue