graham/prop
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Show ERA5 fallback on contact detail, fix status page ERA5 progress

Browse source 
Contact detail page now renders an atmospheric profile section backed by
HRRR when available and ERA5 (reanalysis) otherwise, labeled with the
data source. When both sources are missing and HRRR is known-unavailable
(pre-2014 contacts), the Era5FetchWorker is enqueued so the month-tile
backfill gets triggered from a user visit.

Status page ERA5 progress previously computed "complete" as
(total_contacts - hrrr_unavailable_contacts), which treated every contact
with any HRRR status as ERA5-complete — inflating the bar to 76% while
era5_profiles had 0 rows. Replaced with a per-candidate EXISTS query
against era5_profiles using the same 0.15°/30-min tolerance
Weather.find_nearest_era5/3 uses at lookup time.
This commit is contained in:
Graham McIntire 2026-04-13 13:44:25 -05:00
parent 8f3cc77ff0
commit d4b849f976
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
4 changed files with 218 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

95
lib/microwaveprop_web/live/contact_live/show.ex
View file

@ -11,6 +11,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
alias Microwaveprop.Weather
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
alias Microwaveprop.Workers.Era5FetchWorker
alias Microwaveprop.Workers.HrrrFetchWorker
alias Microwaveprop.Workers.SolarIndexWorker
alias Microwaveprop.Workers.TerrainProfileWorker
@ -40,6 +41,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do
solar: nil,
hrrr: nil,
hrrr_path: [],
era5: nil,
era5_path: [],
iemre: nil,
terrain: nil,
elevation_profile: nil,
@ -83,6 +86,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|> start_async(:weather, fn -> load_weather(contact) end)
|> start_async(:solar, fn -> load_solar(contact) end)
|> start_async(:hrrr_path, fn -> Weather.hrrr_profiles_for_path(contact) end)
|> start_async(:era5_path, fn -> Weather.era5_profiles_for_path(contact) end)
|> start_async(:terrain, fn -> Terrain.get_terrain_profile(contact.id) end)
|> start_async(:iemre, fn -> load_iemre(contact) end)
end
@ -308,6 +312,16 @@ defmodule MicrowavepropWeb.ContactLive.Show do
{:noreply, socket}
end
def handle_async(:era5_path, {:ok, era5_path}, socket) do
era5 = List.first(era5_path)
contact = socket.assigns.contact
contact =
if socket.assigns.can_enqueue, do: maybe_enqueue_era5(era5, contact), else: contact
{:noreply, assign(socket, contact: contact, era5: era5, era5_path: era5_path)}
end
def handle_async(:terrain, {:ok, terrain}, socket) do
contact = socket.assigns.contact
@ -326,7 +340,8 @@ defmodule MicrowavepropWeb.ContactLive.Show do
{:noreply, assign(socket, :iemre, iemre)}
end
def handle_async(slot, {:exit, reason}, socket) when slot in [:weather, :solar, :hrrr_path, :terrain, :iemre] do
def handle_async(slot, {:exit, reason}, socket)
when slot in [:weather, :solar, :hrrr_path, :era5_path, :terrain, :iemre] do
Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
{:noreply, socket}
end
@ -626,6 +641,33 @@ defmodule MicrowavepropWeb.ContactLive.Show do
end
end
# ERA5 is a fallback for contacts where HRRR is unavailable (pre-2014 or
# missing from the archive). Only enqueue when HRRR is known-unavailable AND
# we don't already have an ERA5 profile — otherwise we'd waste CDS quota on
# contacts the HRRR path will cover.
defp maybe_enqueue_era5(era5, contact) when not is_nil(era5), do: contact
defp maybe_enqueue_era5(nil, %{hrrr_status: :unavailable} = contact) do
lat = contact.pos1 && contact.pos1["lat"]
lon = contact.pos1 && (contact.pos1["lon"] || contact.pos1["lng"])
if lat && lon do
valid_time = %{DateTime.truncate(contact.qso_timestamp, :second) | minute: 0, second: 0}
%{
"lat" => lat,
"lon" => lon,
"valid_time" => DateTime.to_iso8601(valid_time)
}
|> Era5FetchWorker.new()
|> Oban.insert()
end
contact
end
defp maybe_enqueue_era5(_nil, contact), do: contact
defp queue_info(counts, queue) do
case Map.get(counts, queue, 0) do
0 -> ""
@ -1077,23 +1119,31 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<div class="divider" />
<h2 class="text-base font-semibold mb-2">HRRR Model Profile</h2>
<%= if @hrrr do %>
<h2 class="text-base font-semibold mb-2">Atmospheric Profile</h2>
<% profile = @hrrr || @era5 %>
<% profile_source =
cond do
@hrrr -> "HRRR (3 km)"
@era5 -> "ERA5 (0.25°)"
true -> nil
end %>
<%= if profile do %>
<div class="mb-6 border border-base-300 rounded-lg">
<button
phx-click="toggle_hrrr_profile"
class="w-full flex items-center justify-between p-4 hover:bg-base-200 transition-colors"
>
<div class="flex items-center gap-4 text-sm">
<span class="badge badge-outline badge-sm">{profile_source}</span>
<span class="font-semibold">
{format_number(@hrrr.lat)}°, {format_number(@hrrr.lon)}°
{format_number(profile.lat)}°, {format_number(profile.lon)}°
</span>
<span>{Calendar.strftime(@hrrr.valid_time, "%H:%M UTC")}</span>
<span>HPBL: {format_number(@hrrr.hpbl_m)} m</span>
<span>PWAT: {format_number(@hrrr.pwat_mm)} mm</span>
<span>N: {format_number(@hrrr.surface_refractivity)}</span>
<span>dN/dh: {format_number(@hrrr.min_refractivity_gradient)}</span>
<%= if @hrrr.ducting_detected do %>
<span>{Calendar.strftime(profile.valid_time, "%H:%M UTC")}</span>
<span>HPBL: {format_number(profile.hpbl_m)} m</span>
<span>PWAT: {format_number(profile.pwat_mm)} mm</span>
<span>N: {format_number(profile.surface_refractivity)}</span>
<span>dN/dh: {format_number(profile.min_refractivity_gradient)}</span>
<%= if profile.ducting_detected do %>
<span class="badge badge-warning badge-sm">Ducting</span>
<% end %>
</div>
@ -1106,16 +1156,18 @@ defmodule MicrowavepropWeb.ContactLive.Show do
<%= if @hrrr_profile_expanded do %>
<div class="px-4 pb-4">
<div class="grid grid-cols-2 gap-x-8 gap-y-1 text-sm mb-4">
<div>Surface Temp: {format_number(@hrrr.surface_temp_c)}°C</div>
<div>Surface Dewpoint: {format_number(@hrrr.surface_dewpoint_c)}°C</div>
<div>Surface Pressure: {format_number(@hrrr.surface_pressure_mb)} mb</div>
<div>Surface Temp: {format_number(profile.surface_temp_c)}°C</div>
<div>Surface Dewpoint: {format_number(profile.surface_dewpoint_c)}°C</div>
<div>Surface Pressure: {format_number(profile.surface_pressure_mb)} mb</div>
<div>
Run Time: {if @hrrr.run_time,
do: Calendar.strftime(@hrrr.run_time, "%Y-%m-%d %H:%M UTC"),
else: "—"}
<%= if run_time = Map.get(profile, :run_time) do %>
Run Time: {Calendar.strftime(run_time, "%Y-%m-%d %H:%M UTC")}
<% else %>
Source: {profile_source} reanalysis
<% end %>
</div>
</div>
<%= if @hrrr.profile && @hrrr.profile != [] do %>
<%= if profile.profile && profile.profile != [] do %>
<div class="overflow-x-auto">
<table class="table table-xs table-zebra">
<thead>
@ -1127,7 +1179,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
</tr>
</thead>
<tbody>
<%= for level <- @hrrr.profile do %>
<%= for level <- profile.profile do %>
<tr>
<td>{format_number(level["pres"])}</td>
<td>{format_number(level["hght"])}</td>
@ -1151,9 +1203,12 @@ defmodule MicrowavepropWeb.ContactLive.Show do
Fetching HRRR atmospheric data {queue_info(@queue_counts, "hrrr")}
</span>
<% :unavailable -> %>
HRRR data unavailable for this contact time.
<span class="flex items-center gap-2">
<span class="loading loading-spinner loading-sm"></span>
HRRR unavailable — fetching ERA5 reanalysis {queue_info(@queue_counts, "era5_batch")}
</span>
<% _ -> %>
No HRRR profile available.
No atmospheric profile available.
<% end %>
</div>
<% end %>

44
lib/microwaveprop_web/live/status_live.ex
View file

@ -76,9 +76,11 @@ defmodule MicrowavepropWeb.StatusLive do
weather = count_incomplete(:weather_status, incomplete)
iemre = count_incomplete(:iemre_status, incomplete)
era5 =
era5_candidates =
Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count()))
era5_done = count_era5_done()
all_done = count_all_done(done)
%{
@ -86,13 +88,36 @@ defmodule MicrowavepropWeb.StatusLive do
hrrr: hrrr,
weather: weather,
iemre: iemre,
era5: era5,
era5_candidates: era5_candidates,
era5_done: era5_done,
remaining: max(terrain, max(hrrr, max(weather, iemre))),
complete: all_done,
total: total
}
end
# ERA5 is a fallback for pre-2014 contacts (HRRR unavailable). "Done" means
# there's an era5_profiles row within 0.15° + 30 min of the contact — the
# same tolerance `Weather.find_nearest_era5/3` uses at lookup time.
defp count_era5_done do
%{rows: [[done]]} =
Repo.query!("""
SELECT count(*)
FROM contacts c
WHERE c.hrrr_status = 'unavailable'
AND c.pos1 IS NOT NULL
AND EXISTS (
SELECT 1
FROM era5_profiles p
WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15)
AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15)
AND p.valid_time BETWEEN (c.qso_timestamp - INTERVAL '30 minutes') AND (c.qso_timestamp + INTERVAL '30 minutes')
)
""")
done
end
defp count_incomplete(status_field, statuses) do
Repo.one(
from(c in Contact,
@ -324,10 +349,15 @@ defmodule MicrowavepropWeb.StatusLive do
</div>
<div class="stat-desc">fully enriched</div>
<div class="flex flex-col gap-1.5 mt-3">
<%= for {label, complete} <- [{"HRRR", @unprocessed.total - @unprocessed.hrrr}, {"Weather", @unprocessed.total - @unprocessed.weather}, {"Terrain", @unprocessed.total - @unprocessed.terrain}, {"IEMRE", @unprocessed.total - @unprocessed.iemre}, {"ERA5", @unprocessed.total - @unprocessed.era5}] do %>
<% pct =
if @unprocessed.total > 0, do: round(complete * 100 / @unprocessed.total), else: 0 %>
<div class="flex items-center gap-2 text-xs">
<%= for {key, label, complete, denom} <- [
{"hrrr", "HRRR", @unprocessed.total - @unprocessed.hrrr, @unprocessed.total},
{"weather", "Weather", @unprocessed.total - @unprocessed.weather, @unprocessed.total},
{"terrain", "Terrain", @unprocessed.total - @unprocessed.terrain, @unprocessed.total},
{"iemre", "IEMRE", @unprocessed.total - @unprocessed.iemre, @unprocessed.total},
{"era5", "ERA5", @unprocessed.era5_done, @unprocessed.era5_candidates}
] do %>
<% pct = if denom > 0, do: round(complete * 100 / denom), else: 0 %>
<div class="flex items-center gap-2 text-xs" data-progress-key={key}>
<span class="w-14 shrink-0">{label}</span>
<progress
class="progress progress-primary flex-1 min-w-0"
@ -335,7 +365,7 @@ defmodule MicrowavepropWeb.StatusLive do
max={100}
/>
<span class="shrink-0 font-mono opacity-60 whitespace-nowrap">
{format_number(complete)} / {format_number(@unprocessed.total)}
{format_number(complete)} / {format_number(denom)}
</span>
</div>
<% end %>

38
test/microwaveprop_web/live/contact_live_test.exs
View file

@ -5,6 +5,7 @@ defmodule MicrowavepropWeb.ContactLiveTest do
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
setup do
Req.Test.stub(Microwaveprop.Terrain.ElevationClient, fn conn ->
@ -153,5 +154,42 @@ defmodule MicrowavepropWeb.ContactLiveTest do
live(conn, ~p"/contacts/#{Ecto.UUID.generate()}")
end
end
test "shows atmospheric profile section with ERA5 source for pre-HRRR contact", %{conn: conn} do
# Pre-2014 contact — HRRR is unavailable, ERA5 is the fallback source.
contact =
create_contact(%{
qso_timestamp: ~U[2010-06-15 18:00:00Z],
pos1: %{"lat" => 32.9, "lon" => -97.0},
pos2: %{"lat" => 30.3, "lon" => -97.7}
})
{:ok, _} =
Repo.insert(%Era5Profile{
valid_time: ~U[2010-06-15 18:00:00Z],
lat: 33.0,
lon: -97.0,
profile: [
%{"pres" => 850.0, "tmpc" => 15.0, "dwpc" => 10.0, "hght" => 1500.0}
],
surface_temp_c: 25.0,
surface_dewpoint_c: 18.0,
surface_pressure_mb: 1012.0,
hpbl_m: 1200.0,
pwat_mm: 30.0,
surface_refractivity: 320.0,
min_refractivity_gradient: -50.0,
ducting_detected: false
})
{:ok, lv, _html} = live(conn, ~p"/contacts/#{contact.id}")
html = render_async(lv)
assert html =~ "Atmospheric Profile"
assert html =~ "ERA5"
# Collapsed summary shows HPBL / PWAT / lat,lon from the injected ERA5 row.
assert html =~ "1200"
assert html =~ "33.0"
end
end
end

68
test/microwaveprop_web/live/status_live_test.exs Normal file
View file

@ -0,0 +1,68 @@
defmodule MicrowavepropWeb.StatusLiveTest do
use MicrowavepropWeb.ConnCase, async: false
import Microwaveprop.AccountsFixtures
import Phoenix.LiveViewTest
alias Microwaveprop.Cache
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Repo
alias Microwaveprop.Weather.Era5Profile
setup %{conn: conn} do
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
Cache.invalidate({MicrowavepropWeb.StatusLive, :stats})
Cache.invalidate({MicrowavepropWeb.StatusLive, :db_stats})
user = user_fixture()
{:ok, admin} = user |> Ecto.Changeset.change(%{is_admin: true}) |> Repo.update()
conn = log_in_user(conn, admin)
{:ok, conn: conn}
end
defp create_era5_candidate do
{:ok, contact} =
%Contact{}
|> Contact.changeset(%{
station1: "W5TEST",
station2: "K5TEST",
qso_timestamp: ~U[2010-06-15 18:00:00Z],
mode: "CW",
band: Decimal.new("1296"),
grid1: "EM12",
grid2: "EM00",
pos1: %{"lat" => 33.0, "lon" => -97.0},
pos2: %{"lat" => 30.3, "lon" => -97.7},
distance_km: Decimal.new("300")
})
|> Ecto.Changeset.change(%{hrrr_status: :unavailable})
|> Repo.insert()
contact
end
test "ERA5 progress bar numerator is actual era5 profile matches not total-minus-candidates", %{conn: conn} do
create_era5_candidate()
# No ERA5 profile yet — candidates = 1, done = 0
{:ok, lv, _html} = live(conn, ~p"/status")
era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render()
assert era5_row =~ "ERA5"
assert era5_row =~ "0 / 1"
# Insert a matching era5_profile row — done should become 1
{:ok, _} =
Repo.insert(%Era5Profile{
valid_time: ~U[2010-06-15 18:00:00Z],
lat: 33.0,
lon: -97.0,
ducting_detected: false
})
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
{:ok, lv, _html} = live(conn, ~p"/status")
era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render()
assert era5_row =~ "1 / 1"
end
end