Split HRRR pressure levels for grid hot path vs per-contact profiles

The skew-T commit (30c1018) doubled @pressure_levels from 13 to 25 so
new contact fetches would cover the full troposphere. That list is
also what PropagationGridWorker pulls per forecast hour, which
doubled the GRIB footprint (~57 MB compressed + 92k points × 25
levels × 3 vars decoded through wgrib2) and pushed prod pods over
their 4 Gi OOMKill threshold. Every chain died during f00 and the
map timeline never got beyond now and now+1h because the .ntms files
for f02-f18 were never written.

Split the constant:

  * @profile_pressure_levels (25 levels, 1000-100 mb) drives the
    per-contact HrrrClient.fetch_profile path so the skew-T plot
    keeps its full-atmosphere trace.

  * @grid_pressure_levels (13 levels, 1000-700 mb) drives the grid
    hot path. That's the band SoundingParams.derive reads for
    min_refractivity_gradient, and native hybrid-sigma data
    (native_min_gradient) takes priority over the pressure-level
    fallback anyway, so upper-air levels contribute nothing to
    scoring — pure memory waste on this path.

build_profile/1 still iterates the full 25-level list; grid fetches
simply populate the 13 near-surface slots and skip the rest.

Makes HrrrClient.pressure_messages public with a :grid | :profile
variant so the split is testable from outside the module.
This commit is contained in:
Graham McIntire 2026-04-15 08:19:29 -05:00
parent e3fc09eee3
commit 4fa67984f3
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
2 changed files with 115 additions and 10 deletions

View file

@ -11,11 +11,10 @@ defmodule Microwaveprop.Weather.HrrrClient do
defp hrrr_base, do: Application.get_env(:microwaveprop, :hrrr_base_url, @hrrr_base_default)
# Fine-grained levels below 900mb (~1km) for duct detection, plus standard upper levels.
# Every 25mb from 1000-900 for ~80m vertical spacing near surface. Above 700mb
# the spacing loosens to 50/100mb — enough vertical resolution for the
# contact-detail skew-T log-P plot without ballooning byte-range fetches.
@pressure_levels [
# Per-point profile fetches (contact detail + skew-T): full troposphere +
# lower stratosphere. 25 levels from 1000→100 mb — fine-grained near the
# surface for duct detection, loosening above 700 mb for the skew-T plot.
@profile_pressure_levels [
1000,
975,
950,
@ -43,6 +42,30 @@ defmodule Microwaveprop.Weather.HrrrClient do
100
]
# Grid-wide fetches (PropagationGridWorker hot path): narrow to the
# near-surface band the scorer actually reads. `SoundingParams.derive`
# pulls `min_refractivity_gradient` from the lowest ~1 km, and native
# hybrid-sigma data (when present) takes priority for that metric
# anyway. Anything above 700 mb is dead weight that doubled the grid
# worker's peak memory when 92k points × levels × vars all decode
# through wgrib2 — enough to OOM the pod at 4 Gi. Keep the hot path
# on this slimmer list.
@grid_pressure_levels [
1000,
975,
950,
925,
900,
875,
850,
825,
800,
775,
750,
725,
700
]
@surface_messages [
%{var: "TMP", level: "2 m above ground"},
%{var: "DPT", level: "2 m above ground"},
@ -205,8 +228,12 @@ defmodule Microwaveprop.Weather.HrrrClient do
sfc_dpt_k = parsed["DPT:2 m above ground"]
sfc_pres_pa = parsed["PRES:surface"]
# Iterate the full profile list so per-contact fetches that include
# upper-air data still populate the whole skew-T trace. Grid fetches
# only populate the 1000→700 mb subset (see @grid_pressure_levels);
# the other levels just produce empty slots that this loop skips.
profile =
Enum.flat_map(@pressure_levels, fn level ->
Enum.flat_map(@profile_pressure_levels, fn level ->
level_str = "#{level} mb"
tmp = parsed["TMP:#{level_str}"]
dpt = parsed["DPT:#{level_str}"]
@ -258,7 +285,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
wanted =
case product do
:surface -> @surface_messages
:pressure -> pressure_messages()
:pressure -> pressure_messages(:profile)
end
Logger.info("HRRR fetching #{product} idx from #{idx_url}")
@ -280,7 +307,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
wanted =
case product do
:surface -> @surface_messages
:pressure -> pressure_messages()
:pressure -> pressure_messages(:grid)
end
Logger.info("HRRR grid fetching #{product} idx from #{idx_url}")
@ -321,8 +348,26 @@ defmodule Microwaveprop.Weather.HrrrClient do
end
end
defp pressure_messages do
for level <- @pressure_levels, var <- ["TMP", "DPT", "HGT"] do
@doc """
Pressure-level GRIB message descriptors for a fetch variant.
* `:grid` narrow list of 13 levels (1000700 mb) used by the
propagation grid hot path. Covers the ducting band without
ballooning the per-forecast-hour memory footprint.
* `:profile` full 25 levels (1000100 mb) used for per-contact
skew-T fetches.
"""
@spec pressure_messages(:grid | :profile) :: [%{var: String.t(), level: String.t()}]
def pressure_messages(:grid) do
build_pressure_messages(@grid_pressure_levels)
end
def pressure_messages(:profile) do
build_pressure_messages(@profile_pressure_levels)
end
defp build_pressure_messages(levels) do
for level <- levels, var <- ["TMP", "DPT", "HGT"] do
%{var: var, level: "#{level} mb"}
end
end

View file

@ -279,6 +279,66 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
end
end
describe "pressure_messages/1" do
test ":grid returns only the near-surface levels needed for scoring" do
messages = HrrrClient.pressure_messages(:grid)
# 13 levels × 3 vars (TMP/DPT/HGT)
assert length(messages) == 39
levels =
messages
|> Enum.map(fn m ->
[num_str, _] = String.split(m.level, " ")
String.to_integer(num_str)
end)
|> Enum.uniq()
|> Enum.sort()
# Narrow list: 1000 → 700 mb, every 25 mb
assert Enum.max(levels) == 1000
assert Enum.min(levels) == 700
assert 700 in levels
assert 900 in levels
# Upper-air levels must be absent — they're the whole reason the grid
# worker OOMs when this function returns the full profile list.
refute 500 in levels
refute 300 in levels
refute 100 in levels
end
test ":profile returns full troposphere + lower stratosphere for the skew-T" do
messages = HrrrClient.pressure_messages(:profile)
# 25 levels × 3 vars
assert length(messages) == 75
levels =
messages
|> Enum.map(fn m ->
[num_str, _] = String.split(m.level, " ")
String.to_integer(num_str)
end)
|> Enum.uniq()
|> Enum.sort()
assert Enum.max(levels) == 1000
assert Enum.min(levels) == 100
assert 500 in levels
assert 300 in levels
assert 100 in levels
end
test ":grid and :profile both request TMP, DPT and HGT at every level" do
for variant <- [:grid, :profile] do
messages = HrrrClient.pressure_messages(variant)
vars = messages |> Enum.map(& &1.var) |> Enum.uniq() |> Enum.sort()
assert vars == ["DPT", "HGT", "TMP"]
end
end
end
describe "surface_messages/0" do
test "returns list of surface message descriptors" do
messages = HrrrClient.surface_messages()