Split HRRR pressure levels for grid hot path vs per-contact profiles
The skew-T commit (30c1018) doubled @pressure_levels from 13 to 25 so
new contact fetches would cover the full troposphere. That list is
also what PropagationGridWorker pulls per forecast hour, which
doubled the GRIB footprint (~57 MB compressed + 92k points × 25
levels × 3 vars decoded through wgrib2) and pushed prod pods over
their 4 Gi OOMKill threshold. Every chain died during f00 and the
map timeline never got beyond now and now+1h because the .ntms files
for f02-f18 were never written.
Split the constant:
* @profile_pressure_levels (25 levels, 1000-100 mb) drives the
per-contact HrrrClient.fetch_profile path so the skew-T plot
keeps its full-atmosphere trace.
* @grid_pressure_levels (13 levels, 1000-700 mb) drives the grid
hot path. That's the band SoundingParams.derive reads for
min_refractivity_gradient, and native hybrid-sigma data
(native_min_gradient) takes priority over the pressure-level
fallback anyway, so upper-air levels contribute nothing to
scoring — pure memory waste on this path.
build_profile/1 still iterates the full 25-level list; grid fetches
simply populate the 13 near-surface slots and skip the rest.
Makes HrrrClient.pressure_messages public with a :grid | :profile
variant so the split is testable from outside the module.
This commit is contained in:
parent
e3fc09eee3
commit
4fa67984f3
2 changed files with 115 additions and 10 deletions
|
|
@ -11,11 +11,10 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
|
|
||||||
defp hrrr_base, do: Application.get_env(:microwaveprop, :hrrr_base_url, @hrrr_base_default)
|
defp hrrr_base, do: Application.get_env(:microwaveprop, :hrrr_base_url, @hrrr_base_default)
|
||||||
|
|
||||||
# Fine-grained levels below 900mb (~1km) for duct detection, plus standard upper levels.
|
# Per-point profile fetches (contact detail + skew-T): full troposphere +
|
||||||
# Every 25mb from 1000-900 for ~80m vertical spacing near surface. Above 700mb
|
# lower stratosphere. 25 levels from 1000→100 mb — fine-grained near the
|
||||||
# the spacing loosens to 50/100mb — enough vertical resolution for the
|
# surface for duct detection, loosening above 700 mb for the skew-T plot.
|
||||||
# contact-detail skew-T log-P plot without ballooning byte-range fetches.
|
@profile_pressure_levels [
|
||||||
@pressure_levels [
|
|
||||||
1000,
|
1000,
|
||||||
975,
|
975,
|
||||||
950,
|
950,
|
||||||
|
|
@ -43,6 +42,30 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
100
|
100
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Grid-wide fetches (PropagationGridWorker hot path): narrow to the
|
||||||
|
# near-surface band the scorer actually reads. `SoundingParams.derive`
|
||||||
|
# pulls `min_refractivity_gradient` from the lowest ~1 km, and native
|
||||||
|
# hybrid-sigma data (when present) takes priority for that metric
|
||||||
|
# anyway. Anything above 700 mb is dead weight that doubled the grid
|
||||||
|
# worker's peak memory when 92k points × levels × vars all decode
|
||||||
|
# through wgrib2 — enough to OOM the pod at 4 Gi. Keep the hot path
|
||||||
|
# on this slimmer list.
|
||||||
|
@grid_pressure_levels [
|
||||||
|
1000,
|
||||||
|
975,
|
||||||
|
950,
|
||||||
|
925,
|
||||||
|
900,
|
||||||
|
875,
|
||||||
|
850,
|
||||||
|
825,
|
||||||
|
800,
|
||||||
|
775,
|
||||||
|
750,
|
||||||
|
725,
|
||||||
|
700
|
||||||
|
]
|
||||||
|
|
||||||
@surface_messages [
|
@surface_messages [
|
||||||
%{var: "TMP", level: "2 m above ground"},
|
%{var: "TMP", level: "2 m above ground"},
|
||||||
%{var: "DPT", level: "2 m above ground"},
|
%{var: "DPT", level: "2 m above ground"},
|
||||||
|
|
@ -205,8 +228,12 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
sfc_dpt_k = parsed["DPT:2 m above ground"]
|
sfc_dpt_k = parsed["DPT:2 m above ground"]
|
||||||
sfc_pres_pa = parsed["PRES:surface"]
|
sfc_pres_pa = parsed["PRES:surface"]
|
||||||
|
|
||||||
|
# Iterate the full profile list so per-contact fetches that include
|
||||||
|
# upper-air data still populate the whole skew-T trace. Grid fetches
|
||||||
|
# only populate the 1000→700 mb subset (see @grid_pressure_levels);
|
||||||
|
# the other levels just produce empty slots that this loop skips.
|
||||||
profile =
|
profile =
|
||||||
Enum.flat_map(@pressure_levels, fn level ->
|
Enum.flat_map(@profile_pressure_levels, fn level ->
|
||||||
level_str = "#{level} mb"
|
level_str = "#{level} mb"
|
||||||
tmp = parsed["TMP:#{level_str}"]
|
tmp = parsed["TMP:#{level_str}"]
|
||||||
dpt = parsed["DPT:#{level_str}"]
|
dpt = parsed["DPT:#{level_str}"]
|
||||||
|
|
@ -258,7 +285,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
wanted =
|
wanted =
|
||||||
case product do
|
case product do
|
||||||
:surface -> @surface_messages
|
:surface -> @surface_messages
|
||||||
:pressure -> pressure_messages()
|
:pressure -> pressure_messages(:profile)
|
||||||
end
|
end
|
||||||
|
|
||||||
Logger.info("HRRR fetching #{product} idx from #{idx_url}")
|
Logger.info("HRRR fetching #{product} idx from #{idx_url}")
|
||||||
|
|
@ -280,7 +307,7 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
wanted =
|
wanted =
|
||||||
case product do
|
case product do
|
||||||
:surface -> @surface_messages
|
:surface -> @surface_messages
|
||||||
:pressure -> pressure_messages()
|
:pressure -> pressure_messages(:grid)
|
||||||
end
|
end
|
||||||
|
|
||||||
Logger.info("HRRR grid fetching #{product} idx from #{idx_url}")
|
Logger.info("HRRR grid fetching #{product} idx from #{idx_url}")
|
||||||
|
|
@ -321,8 +348,26 @@ defmodule Microwaveprop.Weather.HrrrClient do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp pressure_messages do
|
@doc """
|
||||||
for level <- @pressure_levels, var <- ["TMP", "DPT", "HGT"] do
|
Pressure-level GRIB message descriptors for a fetch variant.
|
||||||
|
|
||||||
|
* `:grid` — narrow list of 13 levels (1000→700 mb) used by the
|
||||||
|
propagation grid hot path. Covers the ducting band without
|
||||||
|
ballooning the per-forecast-hour memory footprint.
|
||||||
|
* `:profile` — full 25 levels (1000→100 mb) used for per-contact
|
||||||
|
skew-T fetches.
|
||||||
|
"""
|
||||||
|
@spec pressure_messages(:grid | :profile) :: [%{var: String.t(), level: String.t()}]
|
||||||
|
def pressure_messages(:grid) do
|
||||||
|
build_pressure_messages(@grid_pressure_levels)
|
||||||
|
end
|
||||||
|
|
||||||
|
def pressure_messages(:profile) do
|
||||||
|
build_pressure_messages(@profile_pressure_levels)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp build_pressure_messages(levels) do
|
||||||
|
for level <- levels, var <- ["TMP", "DPT", "HGT"] do
|
||||||
%{var: var, level: "#{level} mb"}
|
%{var: var, level: "#{level} mb"}
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
|
|
@ -279,6 +279,66 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe "pressure_messages/1" do
|
||||||
|
test ":grid returns only the near-surface levels needed for scoring" do
|
||||||
|
messages = HrrrClient.pressure_messages(:grid)
|
||||||
|
|
||||||
|
# 13 levels × 3 vars (TMP/DPT/HGT)
|
||||||
|
assert length(messages) == 39
|
||||||
|
|
||||||
|
levels =
|
||||||
|
messages
|
||||||
|
|> Enum.map(fn m ->
|
||||||
|
[num_str, _] = String.split(m.level, " ")
|
||||||
|
String.to_integer(num_str)
|
||||||
|
end)
|
||||||
|
|> Enum.uniq()
|
||||||
|
|> Enum.sort()
|
||||||
|
|
||||||
|
# Narrow list: 1000 → 700 mb, every 25 mb
|
||||||
|
assert Enum.max(levels) == 1000
|
||||||
|
assert Enum.min(levels) == 700
|
||||||
|
assert 700 in levels
|
||||||
|
assert 900 in levels
|
||||||
|
|
||||||
|
# Upper-air levels must be absent — they're the whole reason the grid
|
||||||
|
# worker OOMs when this function returns the full profile list.
|
||||||
|
refute 500 in levels
|
||||||
|
refute 300 in levels
|
||||||
|
refute 100 in levels
|
||||||
|
end
|
||||||
|
|
||||||
|
test ":profile returns full troposphere + lower stratosphere for the skew-T" do
|
||||||
|
messages = HrrrClient.pressure_messages(:profile)
|
||||||
|
|
||||||
|
# 25 levels × 3 vars
|
||||||
|
assert length(messages) == 75
|
||||||
|
|
||||||
|
levels =
|
||||||
|
messages
|
||||||
|
|> Enum.map(fn m ->
|
||||||
|
[num_str, _] = String.split(m.level, " ")
|
||||||
|
String.to_integer(num_str)
|
||||||
|
end)
|
||||||
|
|> Enum.uniq()
|
||||||
|
|> Enum.sort()
|
||||||
|
|
||||||
|
assert Enum.max(levels) == 1000
|
||||||
|
assert Enum.min(levels) == 100
|
||||||
|
assert 500 in levels
|
||||||
|
assert 300 in levels
|
||||||
|
assert 100 in levels
|
||||||
|
end
|
||||||
|
|
||||||
|
test ":grid and :profile both request TMP, DPT and HGT at every level" do
|
||||||
|
for variant <- [:grid, :profile] do
|
||||||
|
messages = HrrrClient.pressure_messages(variant)
|
||||||
|
vars = messages |> Enum.map(& &1.var) |> Enum.uniq() |> Enum.sort()
|
||||||
|
assert vars == ["DPT", "HGT", "TMP"]
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
describe "surface_messages/0" do
|
describe "surface_messages/0" do
|
||||||
test "returns list of surface message descriptors" do
|
test "returns list of surface message descriptors" do
|
||||||
messages = HrrrClient.surface_messages()
|
messages = HrrrClient.surface_messages()
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue