Extend skew-T plot to cover the full troposphere

Historical contacts showed a skew-T log-P diagram that stopped at 700 mb
because HrrrClient and Era5Client only fetched pressure-level data down
to the top of the boundary layer. The chart canvas already ran up to
100 mb, so the trace clipped mid-atmosphere.

Two complementary fixes:

1. Extend @pressure_levels in HrrrClient and Era5Client with
   650/600/550/500/450/400/350/300/250/200/150/100 mb so new fetches
   cover the full troposphere + lower stratosphere.

2. Prefer the native hybrid-sigma profile for the contact-detail
   skew-T when one has been backfilled for the contact's hour. The
   native profile already stores all 50 hybrid levels up to ~19 km,
   so historical contacts covered by the native backfill get a full
   trace without re-hitting S3. A new HrrrNativeProfile.to_skew_t_profile/1
   converts the parallel arrays into the %{"pres","tmpc","dwpc","hght"}
   list shape the renderer expects, deriving dewpoint from SPFH via the
   Magnus inverse. Weather.find_nearest_native_profile/3 mirrors
   find_nearest_hrrr/3 for the lookup.
This commit is contained in:
Graham McIntire 2026-04-14 17:25:40 -05:00
parent 9f5b427116
commit 30c1018400
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
10 changed files with 342 additions and 7 deletions

View file

@ -7,6 +7,7 @@ defmodule Microwaveprop.Weather do
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.GridCache
alias Microwaveprop.Weather.HrrrNativeProfile
alias Microwaveprop.Weather.HrrrProfile
alias Microwaveprop.Weather.IemClient
alias Microwaveprop.Weather.IemreObservation
@ -782,6 +783,37 @@ defmodule Microwaveprop.Weather do
|> Enum.reject(&is_nil/1)
end
@spec find_nearest_native_profile(float(), float(), DateTime.t()) ::
HrrrNativeProfile.t() | nil
def find_nearest_native_profile(lat, lon, timestamp) do
dlat = 0.07
dlon = 0.07
time_start = DateTime.add(timestamp, -3600, :second)
time_end = DateTime.add(timestamp, 3600, :second)
HrrrNativeProfile
|> where(
[n],
n.lat >= ^(lat - dlat) and n.lat <= ^(lat + dlat) and
n.lon >= ^(lon - dlon) and n.lon <= ^(lon + dlon) and
n.valid_time >= ^time_start and n.valid_time <= ^time_end
)
|> order_by([n],
asc:
fragment(
"ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))",
n.lat,
^lat,
n.lon,
^lon,
n.valid_time,
^timestamp
)
)
|> limit(1)
|> Repo.one()
end
@doc """
Find the best available atmospheric profile for a contact.
Tries HRRR first (3 km, hourly), falls back to ERA5 (0.25°, hourly).

View file

@ -21,7 +21,10 @@ defmodule Microwaveprop.Weather.Era5Client do
# bounds the worker but still gives CDS enough room.
@max_poll_attempts 720
@pressure_levels ~w(1000 975 950 925 900 875 850 825 800 775 750 725 700)
@pressure_levels ~w(
1000 975 950 925 900 875 850 825 800 775 750 725 700
650 600 550 500 450 400 350 300 250 200 150 100
)
@single_level_vars ~w(
2m_temperature

View file

@ -12,7 +12,9 @@ defmodule Microwaveprop.Weather.HrrrClient do
defp hrrr_base, do: Application.get_env(:microwaveprop, :hrrr_base_url, @hrrr_base_default)
# Fine-grained levels below 900mb (~1km) for duct detection, plus standard upper levels.
# Every 25mb from 1000-900 for ~80m vertical spacing near surface.
# Every 25mb from 1000-900 for ~80m vertical spacing near surface. Above 700mb
# the spacing loosens to 50/100mb — enough vertical resolution for the
# contact-detail skew-T log-P plot without ballooning byte-range fetches.
@pressure_levels [
1000,
975,
@ -26,7 +28,19 @@ defmodule Microwaveprop.Weather.HrrrClient do
775,
750,
725,
700
700,
650,
600,
550,
500,
450,
400,
350,
300,
250,
200,
150,
100
]
@surface_messages [

View file

@ -74,6 +74,50 @@ defmodule Microwaveprop.Weather.HrrrNativeProfile do
|> unique_constraint([:lat, :lon, :valid_time])
end
@doc """
Convert a native profile's parallel arrays into the list-of-maps
shape the skew-T renderer expects (`%{"pres", "tmpc", "dwpc", "hght"}`).
Dewpoint is derived from specific humidity, pressure, and temperature
via the Magnus inverse so the chart gets a full-atmosphere trace even
though the native file only carries SPFH. The result is sorted
surface-first (descending pressure).
"""
@spec to_skew_t_profile(t()) :: [map()]
def to_skew_t_profile(%__MODULE__{} = profile) do
heights = profile.heights_m || []
temps = profile.temp_k || []
spfhs = profile.spfh || []
pressures = profile.pressure_pa || []
[heights, temps, spfhs, pressures]
|> Enum.zip()
|> Enum.flat_map(&level_to_skew_t/1)
|> Enum.sort_by(& &1["pres"], :desc)
end
defp level_to_skew_t({h, t_k, q, p_pa}) when is_number(t_k) and is_number(p_pa) do
p_hpa = p_pa / 100.0
[
%{
"pres" => p_hpa,
"tmpc" => t_k - 273.15,
"dwpc" => dewpoint_c(q, p_pa),
"hght" => h
}
]
end
defp level_to_skew_t(_), do: []
defp dewpoint_c(q, p_pa) when is_number(q) and q > 0 and is_number(p_pa) do
e_hpa = q * p_pa / (0.622 + 0.378 * q) / 100.0
ln = :math.log(e_hpa / 6.112)
243.5 * ln / (17.67 - ln)
end
defp dewpoint_c(_, _), do: nil
# All level arrays must have the same length, matching level_count.
defp validate_array_lengths(changeset) do
level_count = get_field(changeset, :level_count)

View file

@ -10,6 +10,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
alias Microwaveprop.Terrain.TerrainAnalysis
alias Microwaveprop.Weather
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.HrrrNativeProfile
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
alias Microwaveprop.Workers.Era5FetchWorker
alias Microwaveprop.Workers.HrrrFetchWorker
@ -42,6 +43,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
solar: nil,
hrrr: nil,
hrrr_path: [],
native_profile: nil,
era5: nil,
era5_path: [],
iemre: nil,
@ -88,10 +90,21 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|> start_async(:solar, fn -> load_solar(contact) end)
|> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end)
|> start_async(:era5_path, fn -> Weather.era5_profiles_for_path(contact) end)
|> start_async(:native_profile, fn -> load_native_profile(contact) end)
|> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end)
|> start_async(:iemre, fn -> load_iemre(contact) end)
end
# The native HRRR profile gives the skew-T a full-atmosphere trace (50
# hybrid levels up to ~19 km) for historical hours we've backfilled.
# Falls back silently to nil when no native profile is on disk; the
# chart then uses the pressure-level profile instead.
defp load_native_profile(%{pos1: %{"lat" => lat, "lon" => lon}, qso_timestamp: ts}) do
Weather.find_nearest_native_profile(lat, lon, ts)
end
defp load_native_profile(_), do: nil
defp load_pending_edit(contact, socket) do
case socket.assigns do
%{current_scope: %{user: %{id: user_id}}} ->
@ -341,8 +354,12 @@ defmodule MicrowavepropWeb.ContactLive.Show do
{:noreply, assign(socket, :iemre, iemre)}
end
def handle_async(:native_profile, {:ok, native_profile}, socket) do
{:noreply, assign(socket, :native_profile, native_profile)}
end
def handle_async(slot, {:exit, reason}, socket)
when slot in [:weather, :solar, :hrrr_path, :era5_path, :terrain, :iemre] do
when slot in [:weather, :solar, :hrrr_path, :era5_path, :native_profile, :terrain, :iemre] do
Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
{:noreply, socket}
end
@ -1128,6 +1145,22 @@ defmodule MicrowavepropWeb.ContactLive.Show do
@era5 -> "ERA5 (0.25°)"
true -> nil
end %>
<% skew_t_levels =
cond do
@native_profile ->
HrrrNativeProfile.to_skew_t_profile(@native_profile)
profile && profile.profile ->
profile.profile
true ->
[]
end %>
<% skew_t_source =
cond do
@native_profile -> "HRRR native (50 hybrid levels)"
true -> profile_source
end %>
<%= if profile do %>
<div class="mb-6 border border-base-300 rounded-lg">
<button
@ -1174,8 +1207,13 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<% end %>
</div>
</div>
<%= if profile.profile && profile.profile != [] do %>
<.skew_t_chart profile={profile.profile} />
<%= if skew_t_levels != [] do %>
<%= if @native_profile do %>
<div class="text-xs text-base-content/70 mb-2">
Skew-T source: <span class="font-semibold">{skew_t_source}</span>
</div>
<% end %>
<.skew_t_chart profile={skew_t_levels} />
<div class="overflow-x-auto mt-4">
<table class="table table-xs table-zebra">
<thead>
@ -1187,7 +1225,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
</tr>
</thead>
<tbody>
<%= for level <- profile.profile do %>
<%= for level <- skew_t_levels do %>
<tr>
<td>{format_number(level["pres"])}</td>
<td>{format_number(level["hght"])}</td>

View file

@ -207,4 +207,19 @@ defmodule Microwaveprop.Weather.Era5ClientTest do
refute File.exists?(path)
end
end
describe "pressure_levels/0" do
test "covers upper troposphere and lower stratosphere for the skew-T plot" do
levels = Enum.map(Era5Client.pressure_levels(), &String.to_integer/1)
# Boundary-layer resolution near the surface stays intact.
assert 1000 in levels
assert 700 in levels
# Upper-air levels needed to fill out the skew-T above 700 mb.
for level <- [500, 300, 200, 100] do
assert level in levels, "expected #{level} mb in Era5Client.pressure_levels/0"
end
end
end
end

View file

@ -243,6 +243,40 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
result = HrrrClient.build_profile(parsed)
assert length(result.profile) == 1
end
test "includes upper-air levels above 700 mb for the skew-T plot" do
parsed = %{
"TMP:700 mb" => 275.0,
"DPT:700 mb" => 265.0,
"HGT:700 mb" => 3100.0,
"TMP:500 mb" => 253.0,
"DPT:500 mb" => 238.0,
"HGT:500 mb" => 5800.0,
"TMP:300 mb" => 228.0,
"DPT:300 mb" => 200.0,
"HGT:300 mb" => 9400.0,
"TMP:100 mb" => 205.0,
"DPT:100 mb" => 180.0,
"HGT:100 mb" => 16_300.0,
"TMP:2 m above ground" => 299.0,
"DPT:2 m above ground" => 292.0,
"PRES:surface" => 101_350.0,
"HPBL:surface" => 1500.0,
"PWAT:entire atmosphere (considered as a single layer)" => 25.0
}
result = HrrrClient.build_profile(parsed)
pressures = Enum.map(result.profile, & &1["pres"])
assert 500.0 in pressures
assert 300.0 in pressures
assert 100.0 in pressures
level_500 = Enum.find(result.profile, &(&1["pres"] == 500.0))
assert_in_delta level_500["tmpc"], 253.0 - 273.15, 0.001
assert_in_delta level_500["dwpc"], 238.0 - 273.15, 0.001
assert level_500["hght"] == 5800.0
end
end
describe "surface_messages/0" do

View file

@ -56,4 +56,56 @@ defmodule Microwaveprop.Weather.HrrrNativeProfileTest do
assert "has already been taken" in errors_on(changeset).lat
end
end
describe "to_skew_t_profile/1" do
test "converts native parallel arrays into skew-T map list sorted surface-first" do
profile = struct(HrrrNativeProfile, @valid_attrs)
levels = HrrrNativeProfile.to_skew_t_profile(profile)
assert length(levels) == 3
# Ordered surface-first (descending pressure).
pressures = Enum.map(levels, & &1["pres"])
assert pressures == Enum.sort(pressures, :desc)
[surface | _] = levels
# pressure Pa → hPa
assert_in_delta surface["pres"], 1010.0, 0.01
# temperature K → °C
assert_in_delta surface["tmpc"], 295.0 - 273.15, 0.01
# geopotential height passes through
assert surface["hght"] == 10.0
# Dewpoint derived from SPFH + pressure + temperature via Magnus.
# e = q*p/(0.622 + 0.378*q)/100 = 0.010*101000/0.62578/100 ≈ 16.140 hPa
# td = 243.5*ln(e/6.112)/(17.67 ln(e/6.112)) ≈ 14.16 °C
assert_in_delta surface["dwpc"], 14.16, 0.1
end
test "returns empty list when arrays are nil or empty" do
blank = %HrrrNativeProfile{
heights_m: nil,
temp_k: nil,
spfh: nil,
pressure_pa: nil
}
assert HrrrNativeProfile.to_skew_t_profile(blank) == []
end
test "drops levels with missing pressure or temperature" do
attrs =
@valid_attrs
|> Map.put(:temp_k, [295.0, nil, 285.0])
|> Map.put(:pressure_pa, [101_000.0, 99_800.0, nil])
profile = struct(HrrrNativeProfile, attrs)
levels = HrrrNativeProfile.to_skew_t_profile(profile)
assert length(levels) == 1
assert hd(levels)["hght"] == 10.0
end
end
end

View file

@ -2,6 +2,7 @@ defmodule Microwaveprop.WeatherTest do
use Microwaveprop.DataCase, async: true
alias Microwaveprop.Weather
alias Microwaveprop.Weather.HrrrNativeProfile
alias Microwaveprop.Weather.HrrrProfile
alias Microwaveprop.Weather.IemreObservation
@ -687,6 +688,50 @@ defmodule Microwaveprop.WeatherTest do
end
end
describe "find_nearest_native_profile/3" do
@native_attrs %{
valid_time: ~U[2026-03-28 18:00:00Z],
run_time: ~U[2026-03-28 18:00:00Z],
lat: 32.90,
lon: -97.04,
level_count: 2,
heights_m: [10.0, 500.0],
temp_k: [295.0, 285.0],
spfh: [0.010, 0.004],
pressure_pa: [101_000.0, 95_000.0],
u_wind_ms: [2.0, 3.0],
v_wind_ms: [1.0, 1.5],
tke_m2s2: [0.5, 0.1]
}
test "returns nearest native profile within bounding box + time window" do
{:ok, _} =
%HrrrNativeProfile{}
|> HrrrNativeProfile.changeset(@native_attrs)
|> Microwaveprop.Repo.insert()
profile = Weather.find_nearest_native_profile(32.91, -97.05, ~U[2026-03-28 18:20:00Z])
assert profile
assert profile.lat == 32.90
assert profile.level_count == 2
end
test "returns nil when no native profile exists nearby" do
assert Weather.find_nearest_native_profile(40.0, -80.0, ~U[2026-03-28 18:00:00Z]) == nil
end
test "returns nil when the nearest profile is outside the time window" do
{:ok, _} =
%HrrrNativeProfile{}
|> HrrrNativeProfile.changeset(@native_attrs)
|> Microwaveprop.Repo.insert()
# 3+ hours away from the stored 18:00Z profile.
assert Weather.find_nearest_native_profile(32.91, -97.05, ~U[2026-03-28 22:00:00Z]) ==
nil
end
end
describe "iemre_for_contact/1" do
test "returns IEMRE observation matching QSO pos1 and date" do
Weather.upsert_iemre_observation(@iemre_attrs)

View file

@ -6,6 +6,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Weather.HrrrNativeProfile
setup do
Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn ->
@ -191,5 +192,62 @@ defmodule MicrowavepropWeb.ContactLiveTest do
assert html =~ "1200"
assert html =~ "33.0"
end
test "skew-T uses native HRRR profile when backfilled", %{conn: conn} do
# Native backfill has reached this hour — use its full-atmosphere
# profile for the skew-T instead of the 700 mb-capped pressure-level one.
contact =
create_contact(%{
qso_timestamp: ~U[2024-06-15 18:00:00Z],
pos1: %{"lat" => 32.9, "lon" => -97.0},
pos2: %{"lat" => 30.3, "lon" => -97.7}
})
# ERA5 row keeps the atmospheric profile section rendering with its
# aggregate metadata (HPBL / PWAT / surface scalars).
{:ok, _} =
Repo.insert(%Era5Profile{
valid_time: ~U[2024-06-15 18:00:00Z],
lat: 32.9,
lon: -97.0,
profile: [%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}],
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1012.0,
hpbl_m: 1200.0,
pwat_mm: 30.0,
surface_refractivity: 320.0,
min_refractivity_gradient: -50.0,
ducting_detected: false
})
# Native profile at the same point/time. `pressure_pa: [..., 30_000.0]`
# corresponds to 300 mb — a level the pressure-level profile cannot
# cover because HRRR/ERA5 stopped at 700 mb historically.
{:ok, _} =
%HrrrNativeProfile{}
|> HrrrNativeProfile.changeset(%{
valid_time: ~U[2024-06-15 18:00:00Z],
run_time: ~U[2024-06-15 18:00:00Z],
lat: 32.9,
lon: -97.0,
level_count: 3,
heights_m: [10.0, 1500.0, 9500.0],
temp_k: [298.0, 288.0, 228.0],
spfh: [0.012, 0.006, 0.0001],
pressure_pa: [101_000.0, 85_000.0, 30_000.0],
u_wind_ms: [2.0, 5.0, 20.0],
v_wind_ms: [1.0, 2.0, 5.0],
tke_m2s2: [0.5, 0.2, 0.05]
})
|> Repo.insert()
{:ok, lv, _html} = live(conn, ~p"/contacts/#{contact.id}")
html = render_async(lv)
assert html =~ "HRRR native"
# 300 mb comes from the native profile — pressure-level data capped at 700.
assert html =~ "300.0"
end
end
end