Extend skew-T plot to cover the full troposphere

Historical contacts showed a skew-T log-P diagram that stopped at 700 mb
because HrrrClient and Era5Client only fetched pressure-level data down
to the top of the boundary layer. The chart canvas already ran up to
100 mb, so the trace clipped mid-atmosphere.

Two complementary fixes:

1. Extend @pressure_levels in HrrrClient and Era5Client with
   650/600/550/500/450/400/350/300/250/200/150/100 mb so new fetches
   cover the full troposphere + lower stratosphere.

2. Prefer the native hybrid-sigma profile for the contact-detail
   skew-T when one has been backfilled for the contact's hour. The
   native profile already stores all 50 hybrid levels up to ~19 km,
   so historical contacts covered by the native backfill get a full
   trace without re-hitting S3. A new HrrrNativeProfile.to_skew_t_profile/1
   converts the parallel arrays into the %{"pres","tmpc","dwpc","hght"}
   list shape the renderer expects, deriving dewpoint from SPFH via the
   Magnus inverse. Weather.find_nearest_native_profile/3 mirrors
   find_nearest_hrrr/3 for the lookup.

lib/microwaveprop/weather.ex

@@ -7,6 +7,7 @@ defmodule Microwaveprop.Weather do
   alias Microwaveprop.Repo
   alias Microwaveprop.Weather.Era5Profile
   alias Microwaveprop.Weather.GridCache
+  alias Microwaveprop.Weather.HrrrNativeProfile
   alias Microwaveprop.Weather.HrrrProfile
   alias Microwaveprop.Weather.IemClient
   alias Microwaveprop.Weather.IemreObservation
@@ -782,6 +783,37 @@ defmodule Microwaveprop.Weather do
     |> Enum.reject(&is_nil/1)
   end
 
+  @spec find_nearest_native_profile(float(), float(), DateTime.t()) ::
+          HrrrNativeProfile.t() | nil
+  def find_nearest_native_profile(lat, lon, timestamp) do
+    dlat = 0.07
+    dlon = 0.07
+    time_start = DateTime.add(timestamp, -3600, :second)
+    time_end = DateTime.add(timestamp, 3600, :second)
+
+    HrrrNativeProfile
+    |> where(
+      [n],
+      n.lat >= ^(lat - dlat) and n.lat <= ^(lat + dlat) and
+        n.lon >= ^(lon - dlon) and n.lon <= ^(lon + dlon) and
+        n.valid_time >= ^time_start and n.valid_time <= ^time_end
+    )
+    |> order_by([n],
+      asc:
+        fragment(
+          "ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))",
+          n.lat,
+          ^lat,
+          n.lon,
+          ^lon,
+          n.valid_time,
+          ^timestamp
+        )
+    )
+    |> limit(1)
+    |> Repo.one()
+  end
+
   @doc """
   Find the best available atmospheric profile for a contact.
   Tries HRRR first (3 km, hourly), falls back to ERA5 (0.25°, hourly).

lib/microwaveprop/weather/era5_client.ex

@@ -21,7 +21,10 @@ defmodule Microwaveprop.Weather.Era5Client do
   # bounds the worker but still gives CDS enough room.
   @max_poll_attempts 720
 
-  @pressure_levels ~w(1000 975 950 925 900 875 850 825 800 775 750 725 700)
+  @pressure_levels ~w(
+    1000 975 950 925 900 875 850 825 800 775 750 725 700
+    650 600 550 500 450 400 350 300 250 200 150 100
+  )
 
   @single_level_vars ~w(
     2m_temperature

lib/microwaveprop/weather/hrrr_client.ex

@@ -12,7 +12,9 @@ defmodule Microwaveprop.Weather.HrrrClient do
   defp hrrr_base, do: Application.get_env(:microwaveprop, :hrrr_base_url, @hrrr_base_default)
 
   # Fine-grained levels below 900mb (~1km) for duct detection, plus standard upper levels.
-  # Every 25mb from 1000-900 for ~80m vertical spacing near surface.
+  # Every 25mb from 1000-900 for ~80m vertical spacing near surface. Above 700mb
+  # the spacing loosens to 50/100mb — enough vertical resolution for the
+  # contact-detail skew-T log-P plot without ballooning byte-range fetches.
   @pressure_levels [
     1000,
     975,
@@ -26,7 +28,19 @@ defmodule Microwaveprop.Weather.HrrrClient do
     775,
     750,
     725,
-    700
+    700,
+    650,
+    600,
+    550,
+    500,
+    450,
+    400,
+    350,
+    300,
+    250,
+    200,
+    150,
+    100
   ]
 
   @surface_messages [

lib/microwaveprop/weather/hrrr_native_profile.ex

@@ -74,6 +74,50 @@ defmodule Microwaveprop.Weather.HrrrNativeProfile do
     |> unique_constraint([:lat, :lon, :valid_time])
   end
 
+  @doc """
+  Convert a native profile's parallel arrays into the list-of-maps
+  shape the skew-T renderer expects (`%{"pres", "tmpc", "dwpc", "hght"}`).
+  Dewpoint is derived from specific humidity, pressure, and temperature
+  via the Magnus inverse so the chart gets a full-atmosphere trace even
+  though the native file only carries SPFH. The result is sorted
+  surface-first (descending pressure).
+  """
+  @spec to_skew_t_profile(t()) :: [map()]
+  def to_skew_t_profile(%__MODULE__{} = profile) do
+    heights = profile.heights_m || []
+    temps = profile.temp_k || []
+    spfhs = profile.spfh || []
+    pressures = profile.pressure_pa || []
+
+    [heights, temps, spfhs, pressures]
+    |> Enum.zip()
+    |> Enum.flat_map(&level_to_skew_t/1)
+    |> Enum.sort_by(& &1["pres"], :desc)
+  end
+
+  defp level_to_skew_t({h, t_k, q, p_pa}) when is_number(t_k) and is_number(p_pa) do
+    p_hpa = p_pa / 100.0
+
+    [
+      %{
+        "pres" => p_hpa,
+        "tmpc" => t_k - 273.15,
+        "dwpc" => dewpoint_c(q, p_pa),
+        "hght" => h
+      }
+    ]
+  end
+
+  defp level_to_skew_t(_), do: []
+
+  defp dewpoint_c(q, p_pa) when is_number(q) and q > 0 and is_number(p_pa) do
+    e_hpa = q * p_pa / (0.622 + 0.378 * q) / 100.0
+    ln = :math.log(e_hpa / 6.112)
+    243.5 * ln / (17.67 - ln)
+  end
+
+  defp dewpoint_c(_, _), do: nil
+
   # All level arrays must have the same length, matching level_count.
   defp validate_array_lengths(changeset) do
     level_count = get_field(changeset, :level_count)

lib/microwaveprop_web/live/contact_live/show.ex

@@ -10,6 +10,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
   alias Microwaveprop.Terrain.TerrainAnalysis
   alias Microwaveprop.Weather
   alias Microwaveprop.Weather.HrrrClient
+  alias Microwaveprop.Weather.HrrrNativeProfile
   alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
   alias Microwaveprop.Workers.Era5FetchWorker
   alias Microwaveprop.Workers.HrrrFetchWorker
@@ -42,6 +43,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
         solar: nil,
         hrrr: nil,
         hrrr_path: [],
+        native_profile: nil,
         era5: nil,
         era5_path: [],
         iemre: nil,
@@ -88,10 +90,21 @@ defmodule MicrowavepropWeb.ContactLive.Show do
     |> start_async(:solar, fn -> load_solar(contact) end)
     |> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end)
     |> start_async(:era5_path, fn -> Weather.era5_profiles_for_path(contact) end)
+    |> start_async(:native_profile, fn -> load_native_profile(contact) end)
     |> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end)
     |> start_async(:iemre, fn -> load_iemre(contact) end)
   end
 
+  # The native HRRR profile gives the skew-T a full-atmosphere trace (50
+  # hybrid levels up to ~19 km) for historical hours we've backfilled.
+  # Falls back silently to nil when no native profile is on disk; the
+  # chart then uses the pressure-level profile instead.
+  defp load_native_profile(%{pos1: %{"lat" => lat, "lon" => lon}, qso_timestamp: ts}) do
+    Weather.find_nearest_native_profile(lat, lon, ts)
+  end
+
+  defp load_native_profile(_), do: nil
+
   defp load_pending_edit(contact, socket) do
     case socket.assigns do
       %{current_scope: %{user: %{id: user_id}}} ->
@@ -341,8 +354,12 @@ defmodule MicrowavepropWeb.ContactLive.Show do
     {:noreply, assign(socket, :iemre, iemre)}
   end
 
+  def handle_async(:native_profile, {:ok, native_profile}, socket) do
+    {:noreply, assign(socket, :native_profile, native_profile)}
+  end
+
   def handle_async(slot, {:exit, reason}, socket)
-      when slot in [:weather, :solar, :hrrr_path, :era5_path, :terrain, :iemre] do
+      when slot in [:weather, :solar, :hrrr_path, :era5_path, :native_profile, :terrain, :iemre] do
     Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
     {:noreply, socket}
   end
@@ -1128,6 +1145,22 @@ defmodule MicrowavepropWeb.ContactLive.Show do
           @era5 -> "ERA5 (0.25°)"
           true -> nil
         end %>
+      <% skew_t_levels =
+        cond do
+          @native_profile ->
+            HrrrNativeProfile.to_skew_t_profile(@native_profile)
+
+          profile && profile.profile ->
+            profile.profile
+
+          true ->
+            []
+        end %>
+      <% skew_t_source =
+        cond do
+          @native_profile -> "HRRR native (50 hybrid levels)"
+          true -> profile_source
+        end %>
       <%= if profile do %>
         <div class="mb-6 border border-base-300 rounded-lg">
           <button
@@ -1174,8 +1207,13 @@ defmodule MicrowavepropWeb.ContactLive.Show do
                   <% end %>
                 </div>
               </div>
-              <%= if profile.profile && profile.profile != [] do %>
+              <%= if skew_t_levels != [] do %>
-                <.skew_t_chart profile={profile.profile} />
+                <%= if @native_profile do %>
+                  <div class="text-xs text-base-content/70 mb-2">
+                    Skew-T source: <span class="font-semibold">{skew_t_source}</span>
+                  </div>
+                <% end %>
+                <.skew_t_chart profile={skew_t_levels} />
                 <div class="overflow-x-auto mt-4">
                   <table class="table table-xs table-zebra">
                     <thead>
@@ -1187,7 +1225,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
                       </tr>
                     </thead>
                     <tbody>
-                      <%= for level <- profile.profile do %>
+                      <%= for level <- skew_t_levels do %>
                         <tr>
                           <td>{format_number(level["pres"])}</td>
                           <td>{format_number(level["hght"])}</td>

test/microwaveprop/weather/era5_client_test.exs

@@ -207,4 +207,19 @@ defmodule Microwaveprop.Weather.Era5ClientTest do
       refute File.exists?(path)
     end
   end
+
+  describe "pressure_levels/0" do
+    test "covers upper troposphere and lower stratosphere for the skew-T plot" do
+      levels = Enum.map(Era5Client.pressure_levels(), &String.to_integer/1)
+
+      # Boundary-layer resolution near the surface stays intact.
+      assert 1000 in levels
+      assert 700 in levels
+
+      # Upper-air levels needed to fill out the skew-T above 700 mb.
+      for level <- [500, 300, 200, 100] do
+        assert level in levels, "expected #{level} mb in Era5Client.pressure_levels/0"
+      end
+    end
+  end
 end

test/microwaveprop/weather/hrrr_client_test.exs

@@ -243,6 +243,40 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
       result = HrrrClient.build_profile(parsed)
       assert length(result.profile) == 1
     end
+
+    test "includes upper-air levels above 700 mb for the skew-T plot" do
+      parsed = %{
+        "TMP:700 mb" => 275.0,
+        "DPT:700 mb" => 265.0,
+        "HGT:700 mb" => 3100.0,
+        "TMP:500 mb" => 253.0,
+        "DPT:500 mb" => 238.0,
+        "HGT:500 mb" => 5800.0,
+        "TMP:300 mb" => 228.0,
+        "DPT:300 mb" => 200.0,
+        "HGT:300 mb" => 9400.0,
+        "TMP:100 mb" => 205.0,
+        "DPT:100 mb" => 180.0,
+        "HGT:100 mb" => 16_300.0,
+        "TMP:2 m above ground" => 299.0,
+        "DPT:2 m above ground" => 292.0,
+        "PRES:surface" => 101_350.0,
+        "HPBL:surface" => 1500.0,
+        "PWAT:entire atmosphere (considered as a single layer)" => 25.0
+      }
+
+      result = HrrrClient.build_profile(parsed)
+      pressures = Enum.map(result.profile, & &1["pres"])
+
+      assert 500.0 in pressures
+      assert 300.0 in pressures
+      assert 100.0 in pressures
+
+      level_500 = Enum.find(result.profile, &(&1["pres"] == 500.0))
+      assert_in_delta level_500["tmpc"], 253.0 - 273.15, 0.001
+      assert_in_delta level_500["dwpc"], 238.0 - 273.15, 0.001
+      assert level_500["hght"] == 5800.0
+    end
   end
 
   describe "surface_messages/0" do

test/microwaveprop/weather/hrrr_native_profile_test.exs

@@ -56,4 +56,56 @@ defmodule Microwaveprop.Weather.HrrrNativeProfileTest do
       assert "has already been taken" in errors_on(changeset).lat
     end
   end
+
+  describe "to_skew_t_profile/1" do
+    test "converts native parallel arrays into skew-T map list sorted surface-first" do
+      profile = struct(HrrrNativeProfile, @valid_attrs)
+
+      levels = HrrrNativeProfile.to_skew_t_profile(profile)
+
+      assert length(levels) == 3
+
+      # Ordered surface-first (descending pressure).
+      pressures = Enum.map(levels, & &1["pres"])
+      assert pressures == Enum.sort(pressures, :desc)
+
+      [surface | _] = levels
+
+      # pressure Pa → hPa
+      assert_in_delta surface["pres"], 1010.0, 0.01
+      # temperature K → °C
+      assert_in_delta surface["tmpc"], 295.0 - 273.15, 0.01
+      # geopotential height passes through
+      assert surface["hght"] == 10.0
+
+      # Dewpoint derived from SPFH + pressure + temperature via Magnus.
+      # e = q*p/(0.622 + 0.378*q)/100 = 0.010*101000/0.62578/100 ≈ 16.140 hPa
+      # td = 243.5*ln(e/6.112)/(17.67 − ln(e/6.112)) ≈ 14.16 °C
+      assert_in_delta surface["dwpc"], 14.16, 0.1
+    end
+
+    test "returns empty list when arrays are nil or empty" do
+      blank = %HrrrNativeProfile{
+        heights_m: nil,
+        temp_k: nil,
+        spfh: nil,
+        pressure_pa: nil
+      }
+
+      assert HrrrNativeProfile.to_skew_t_profile(blank) == []
+    end
+
+    test "drops levels with missing pressure or temperature" do
+      attrs =
+        @valid_attrs
+        |> Map.put(:temp_k, [295.0, nil, 285.0])
+        |> Map.put(:pressure_pa, [101_000.0, 99_800.0, nil])
+
+      profile = struct(HrrrNativeProfile, attrs)
+
+      levels = HrrrNativeProfile.to_skew_t_profile(profile)
+      assert length(levels) == 1
+      assert hd(levels)["hght"] == 10.0
+    end
+  end
 end

test/microwaveprop/weather_test.exs

@@ -2,6 +2,7 @@ defmodule Microwaveprop.WeatherTest do
   use Microwaveprop.DataCase, async: true
 
   alias Microwaveprop.Weather
+  alias Microwaveprop.Weather.HrrrNativeProfile
   alias Microwaveprop.Weather.HrrrProfile
   alias Microwaveprop.Weather.IemreObservation
 
@@ -687,6 +688,50 @@ defmodule Microwaveprop.WeatherTest do
     end
   end
 
+  describe "find_nearest_native_profile/3" do
+    @native_attrs %{
+      valid_time: ~U[2026-03-28 18:00:00Z],
+      run_time: ~U[2026-03-28 18:00:00Z],
+      lat: 32.90,
+      lon: -97.04,
+      level_count: 2,
+      heights_m: [10.0, 500.0],
+      temp_k: [295.0, 285.0],
+      spfh: [0.010, 0.004],
+      pressure_pa: [101_000.0, 95_000.0],
+      u_wind_ms: [2.0, 3.0],
+      v_wind_ms: [1.0, 1.5],
+      tke_m2s2: [0.5, 0.1]
+    }
+
+    test "returns nearest native profile within bounding box + time window" do
+      {:ok, _} =
+        %HrrrNativeProfile{}
+        |> HrrrNativeProfile.changeset(@native_attrs)
+        |> Microwaveprop.Repo.insert()
+
+      profile = Weather.find_nearest_native_profile(32.91, -97.05, ~U[2026-03-28 18:20:00Z])
+      assert profile
+      assert profile.lat == 32.90
+      assert profile.level_count == 2
+    end
+
+    test "returns nil when no native profile exists nearby" do
+      assert Weather.find_nearest_native_profile(40.0, -80.0, ~U[2026-03-28 18:00:00Z]) == nil
+    end
+
+    test "returns nil when the nearest profile is outside the time window" do
+      {:ok, _} =
+        %HrrrNativeProfile{}
+        |> HrrrNativeProfile.changeset(@native_attrs)
+        |> Microwaveprop.Repo.insert()
+
+      # 3+ hours away from the stored 18:00Z profile.
+      assert Weather.find_nearest_native_profile(32.91, -97.05, ~U[2026-03-28 22:00:00Z]) ==
+               nil
+    end
+  end
+
   describe "iemre_for_contact/1" do
     test "returns IEMRE observation matching QSO pos1 and date" do
       Weather.upsert_iemre_observation(@iemre_attrs)

test/microwaveprop_web/live/contact_live_test.exs

@@ -6,6 +6,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
   alias Microwaveprop.Radio.Contact
   alias Microwaveprop.Repo
   alias Microwaveprop.Weather.Era5Profile
+  alias Microwaveprop.Weather.HrrrNativeProfile
 
   setup do
     Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn ->
@@ -191,5 +192,62 @@ defmodule MicrowavepropWeb.ContactLiveTest do
       assert html =~ "1200"
       assert html =~ "33.0"
     end
+
+    test "skew-T uses native HRRR profile when backfilled", %{conn: conn} do
+      # Native backfill has reached this hour — use its full-atmosphere
+      # profile for the skew-T instead of the 700 mb-capped pressure-level one.
+      contact =
+        create_contact(%{
+          qso_timestamp: ~U[2024-06-15 18:00:00Z],
+          pos1: %{"lat" => 32.9, "lon" => -97.0},
+          pos2: %{"lat" => 30.3, "lon" => -97.7}
+        })
+
+      # ERA5 row keeps the atmospheric profile section rendering with its
+      # aggregate metadata (HPBL / PWAT / surface scalars).
+      {:ok, _} =
+        Repo.insert(%Era5Profile{
+          valid_time: ~U[2024-06-15 18:00:00Z],
+          lat: 32.9,
+          lon: -97.0,
+          profile: [%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}],
+          surface_temp_c: 25.0,
+          surface_dewpoint_c: 18.0,
+          surface_pressure_mb: 1012.0,
+          hpbl_m: 1200.0,
+          pwat_mm: 30.0,
+          surface_refractivity: 320.0,
+          min_refractivity_gradient: -50.0,
+          ducting_detected: false
+        })
+
+      # Native profile at the same point/time. `pressure_pa: [..., 30_000.0]`
+      # corresponds to 300 mb — a level the pressure-level profile cannot
+      # cover because HRRR/ERA5 stopped at 700 mb historically.
+      {:ok, _} =
+        %HrrrNativeProfile{}
+        |> HrrrNativeProfile.changeset(%{
+          valid_time: ~U[2024-06-15 18:00:00Z],
+          run_time: ~U[2024-06-15 18:00:00Z],
+          lat: 32.9,
+          lon: -97.0,
+          level_count: 3,
+          heights_m: [10.0, 1500.0, 9500.0],
+          temp_k: [298.0, 288.0, 228.0],
+          spfh: [0.012, 0.006, 0.0001],
+          pressure_pa: [101_000.0, 85_000.0, 30_000.0],
+          u_wind_ms: [2.0, 5.0, 20.0],
+          v_wind_ms: [1.0, 2.0, 5.0],
+          tke_m2s2: [0.5, 0.2, 0.05]
+        })
+        |> Repo.insert()
+
+      {:ok, lv, _html} = live(conn, ~p"/contacts/#{contact.id}")
+      html = render_async(lv)
+
+      assert html =~ "HRRR native"
+      # 300 mb comes from the native profile — pressure-level data capped at 700.
+      assert html =~ "300.0"
+    end
   end
 end