Split HRRR pressure levels for grid hot path vs per-contact profiles
The skew-T commit (30c1018) doubled @pressure_levels from 13 to 25 so
new contact fetches would cover the full troposphere. That list is
also what PropagationGridWorker pulls per forecast hour, which
doubled the GRIB footprint (~57 MB compressed + 92k points × 25
levels × 3 vars decoded through wgrib2) and pushed prod pods over
their 4 Gi OOMKill threshold. Every chain died during f00 and the
map timeline never got beyond now and now+1h because the .ntms files
for f02-f18 were never written.
Split the constant:
* @profile_pressure_levels (25 levels, 1000-100 mb) drives the
per-contact HrrrClient.fetch_profile path so the skew-T plot
keeps its full-atmosphere trace.
* @grid_pressure_levels (13 levels, 1000-700 mb) drives the grid
hot path. That's the band SoundingParams.derive reads for
min_refractivity_gradient, and native hybrid-sigma data
(native_min_gradient) takes priority over the pressure-level
fallback anyway, so upper-air levels contribute nothing to
scoring — pure memory waste on this path.
build_profile/1 still iterates the full 25-level list; grid fetches
simply populate the 13 near-surface slots and skip the rest.
Makes HrrrClient.pressure_messages public with a :grid | :profile
variant so the split is testable from outside the module.
This commit is contained in:
parent
e3fc09eee3
commit
4fa67984f3
2 changed files with 115 additions and 10 deletions
|
|
@ -11,11 +11,10 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
|
||||
defp hrrr_base, do: Application.get_env(:microwaveprop, :hrrr_base_url, @hrrr_base_default)
|
||||
|
||||
# Fine-grained levels below 900mb (~1km) for duct detection, plus standard upper levels.
|
||||
# Every 25mb from 1000-900 for ~80m vertical spacing near surface. Above 700mb
|
||||
# the spacing loosens to 50/100mb — enough vertical resolution for the
|
||||
# contact-detail skew-T log-P plot without ballooning byte-range fetches.
|
||||
@pressure_levels [
|
||||
# Per-point profile fetches (contact detail + skew-T): full troposphere +
|
||||
# lower stratosphere. 25 levels from 1000→100 mb — fine-grained near the
|
||||
# surface for duct detection, loosening above 700 mb for the skew-T plot.
|
||||
@profile_pressure_levels [
|
||||
1000,
|
||||
975,
|
||||
950,
|
||||
|
|
@ -43,6 +42,30 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
100
|
||||
]
|
||||
|
||||
# Grid-wide fetches (PropagationGridWorker hot path): narrow to the
|
||||
# near-surface band the scorer actually reads. `SoundingParams.derive`
|
||||
# pulls `min_refractivity_gradient` from the lowest ~1 km, and native
|
||||
# hybrid-sigma data (when present) takes priority for that metric
|
||||
# anyway. Anything above 700 mb is dead weight that doubled the grid
|
||||
# worker's peak memory when 92k points × levels × vars all decode
|
||||
# through wgrib2 — enough to OOM the pod at 4 Gi. Keep the hot path
|
||||
# on this slimmer list.
|
||||
@grid_pressure_levels [
|
||||
1000,
|
||||
975,
|
||||
950,
|
||||
925,
|
||||
900,
|
||||
875,
|
||||
850,
|
||||
825,
|
||||
800,
|
||||
775,
|
||||
750,
|
||||
725,
|
||||
700
|
||||
]
|
||||
|
||||
@surface_messages [
|
||||
%{var: "TMP", level: "2 m above ground"},
|
||||
%{var: "DPT", level: "2 m above ground"},
|
||||
|
|
@ -205,8 +228,12 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
sfc_dpt_k = parsed["DPT:2 m above ground"]
|
||||
sfc_pres_pa = parsed["PRES:surface"]
|
||||
|
||||
# Iterate the full profile list so per-contact fetches that include
|
||||
# upper-air data still populate the whole skew-T trace. Grid fetches
|
||||
# only populate the 1000→700 mb subset (see @grid_pressure_levels);
|
||||
# the other levels just produce empty slots that this loop skips.
|
||||
profile =
|
||||
Enum.flat_map(@pressure_levels, fn level ->
|
||||
Enum.flat_map(@profile_pressure_levels, fn level ->
|
||||
level_str = "#{level} mb"
|
||||
tmp = parsed["TMP:#{level_str}"]
|
||||
dpt = parsed["DPT:#{level_str}"]
|
||||
|
|
@ -258,7 +285,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
wanted =
|
||||
case product do
|
||||
:surface -> @surface_messages
|
||||
:pressure -> pressure_messages()
|
||||
:pressure -> pressure_messages(:profile)
|
||||
end
|
||||
|
||||
Logger.info("HRRR fetching #{product} idx from #{idx_url}")
|
||||
|
|
@ -280,7 +307,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
wanted =
|
||||
case product do
|
||||
:surface -> @surface_messages
|
||||
:pressure -> pressure_messages()
|
||||
:pressure -> pressure_messages(:grid)
|
||||
end
|
||||
|
||||
Logger.info("HRRR grid fetching #{product} idx from #{idx_url}")
|
||||
|
|
@ -321,8 +348,26 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
|||
end
|
||||
end
|
||||
|
||||
defp pressure_messages do
|
||||
for level <- @pressure_levels, var <- ["TMP", "DPT", "HGT"] do
|
||||
@doc """
|
||||
Pressure-level GRIB message descriptors for a fetch variant.
|
||||
|
||||
* `:grid` — narrow list of 13 levels (1000→700 mb) used by the
|
||||
propagation grid hot path. Covers the ducting band without
|
||||
ballooning the per-forecast-hour memory footprint.
|
||||
* `:profile` — full 25 levels (1000→100 mb) used for per-contact
|
||||
skew-T fetches.
|
||||
"""
|
||||
@spec pressure_messages(:grid | :profile) :: [%{var: String.t(), level: String.t()}]
|
||||
def pressure_messages(:grid) do
|
||||
build_pressure_messages(@grid_pressure_levels)
|
||||
end
|
||||
|
||||
def pressure_messages(:profile) do
|
||||
build_pressure_messages(@profile_pressure_levels)
|
||||
end
|
||||
|
||||
defp build_pressure_messages(levels) do
|
||||
for level <- levels, var <- ["TMP", "DPT", "HGT"] do
|
||||
%{var: var, level: "#{level} mb"}
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -279,6 +279,66 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
|
|||
end
|
||||
end
|
||||
|
||||
describe "pressure_messages/1" do
|
||||
test ":grid returns only the near-surface levels needed for scoring" do
|
||||
messages = HrrrClient.pressure_messages(:grid)
|
||||
|
||||
# 13 levels × 3 vars (TMP/DPT/HGT)
|
||||
assert length(messages) == 39
|
||||
|
||||
levels =
|
||||
messages
|
||||
|> Enum.map(fn m ->
|
||||
[num_str, _] = String.split(m.level, " ")
|
||||
String.to_integer(num_str)
|
||||
end)
|
||||
|> Enum.uniq()
|
||||
|> Enum.sort()
|
||||
|
||||
# Narrow list: 1000 → 700 mb, every 25 mb
|
||||
assert Enum.max(levels) == 1000
|
||||
assert Enum.min(levels) == 700
|
||||
assert 700 in levels
|
||||
assert 900 in levels
|
||||
|
||||
# Upper-air levels must be absent — they're the whole reason the grid
|
||||
# worker OOMs when this function returns the full profile list.
|
||||
refute 500 in levels
|
||||
refute 300 in levels
|
||||
refute 100 in levels
|
||||
end
|
||||
|
||||
test ":profile returns full troposphere + lower stratosphere for the skew-T" do
|
||||
messages = HrrrClient.pressure_messages(:profile)
|
||||
|
||||
# 25 levels × 3 vars
|
||||
assert length(messages) == 75
|
||||
|
||||
levels =
|
||||
messages
|
||||
|> Enum.map(fn m ->
|
||||
[num_str, _] = String.split(m.level, " ")
|
||||
String.to_integer(num_str)
|
||||
end)
|
||||
|> Enum.uniq()
|
||||
|> Enum.sort()
|
||||
|
||||
assert Enum.max(levels) == 1000
|
||||
assert Enum.min(levels) == 100
|
||||
assert 500 in levels
|
||||
assert 300 in levels
|
||||
assert 100 in levels
|
||||
end
|
||||
|
||||
test ":grid and :profile both request TMP, DPT and HGT at every level" do
|
||||
for variant <- [:grid, :profile] do
|
||||
messages = HrrrClient.pressure_messages(variant)
|
||||
vars = messages |> Enum.map(& &1.var) |> Enum.uniq() |> Enum.sort()
|
||||
assert vars == ["DPT", "HGT", "TMP"]
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "surface_messages/0" do
|
||||
test "returns list of surface message descriptors" do
|
||||
messages = HrrrClient.surface_messages()
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue