Status page: relabel ERA5 → NARR, include NARR in backfill cron

- /status progress bar, status-by-type card, and internal map keys now
  read "NARR" (data-progress-key="narr", narr_candidates/narr_done).
  era5_profiles table stays as the DB landing spot (historical artifact).
- BackfillEnqueueWorker cron now passes "era5" in types so pre-2014
  contacts whose hrrr_status is :unavailable actually get NarrFetchWorker
  jobs enqueued every 30 min. The :era5 type key still maps internally
  to NarrFetchWorker via era5_jobs_for_contact/1.
This commit is contained in:
Graham McIntire 2026-04-16 08:56:57 -05:00
parent 17271b264a
commit 7857d3bc5a
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
3 changed files with 33 additions and 26 deletions

View file

@ -218,8 +218,11 @@ if config_env() == :prod do
# full grid-side JSONB write for.
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
# The :era5 type dispatches NarrFetchWorker for pre-2014 contacts
# (see ContactWeatherEnqueueWorker.era5_jobs_for_contact/1). The
# type key is a historical artifact — the data source is NARR.
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}},
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "era5"]}},
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
{"30 1 * * *", CanadianSoundingFetchWorker},
{"30 13 * * *", CanadianSoundingFetchWorker},

View file

@ -76,10 +76,10 @@ defmodule MicrowavepropWeb.StatusLive do
weather = count_incomplete(:weather_status, incomplete)
iemre = count_incomplete(:iemre_status, incomplete)
era5_candidates =
narr_candidates =
Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count()))
era5_done = count_era5_done()
narr_done = count_narr_done()
all_done = count_all_done(done)
@ -88,18 +88,19 @@ defmodule MicrowavepropWeb.StatusLive do
hrrr: hrrr,
weather: weather,
iemre: iemre,
era5_candidates: era5_candidates,
era5_done: era5_done,
narr_candidates: narr_candidates,
narr_done: narr_done,
remaining: max(terrain, max(hrrr, max(weather, iemre))),
complete: all_done,
total: total
}
end
# ERA5 is a fallback for pre-2014 contacts (HRRR unavailable). "Done" means
# there's an era5_profiles row within 0.15° + 30 min of the contact — the
# same tolerance `Weather.find_nearest_era5/3` uses at lookup time.
defp count_era5_done do
# NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
# Profiles land in the era5_profiles table (kept 1:1 after the ERA5/CDS
# pipeline was replaced). "Done" means a row within 0.15° + 30 min of the
# contact, matching `Weather.find_nearest_era5/3` lookup tolerance.
defp count_narr_done do
%{rows: [[done]]} =
Repo.query!("""
SELECT count(*)
@ -262,16 +263,17 @@ defmodule MicrowavepropWeb.StatusLive do
# Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*))
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
# ERA5 doesn't have its own status column — synthesize from HRRR unavailable + era5_profiles count
era5_count = Map.get(table_counts, "era5_profiles", 0)
# NARR doesn't have its own status column — synthesize from HRRR unavailable
# + era5_profiles row count (the table is reused as the NARR landing spot).
narr_count = Map.get(table_counts, "era5_profiles", 0)
hrrr_unavailable =
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
statuses =
Map.put(statuses, "era5", [
Map.put(statuses, "narr", [
{"candidates (HRRR unavail)", hrrr_unavailable},
{"profiles fetched", era5_count}
{"profiles fetched", narr_count}
])
hrrr_count = Map.get(table_counts, "hrrr_profiles", 0)
@ -287,7 +289,7 @@ defmodule MicrowavepropWeb.StatusLive do
statuses: statuses,
totals: %{
hrrr_profiles: hrrr_count,
era5_profiles: Map.get(table_counts, "era5_profiles", 0),
narr_profiles: Map.get(table_counts, "era5_profiles", 0),
terrain_profiles: terrain_count,
surface_observations: obs_count,
soundings: sounding_count,
@ -354,7 +356,7 @@ defmodule MicrowavepropWeb.StatusLive do
{"weather", "Weather", @unprocessed.total - @unprocessed.weather, @unprocessed.total},
{"terrain", "Terrain", @unprocessed.total - @unprocessed.terrain, @unprocessed.total},
{"iemre", "IEMRE", @unprocessed.total - @unprocessed.iemre, @unprocessed.total},
{"era5", "ERA5", @unprocessed.era5_done, @unprocessed.era5_candidates}
{"narr", "NARR", @unprocessed.narr_done, @unprocessed.narr_candidates}
] do %>
<% pct = if denom > 0, do: round(complete * 100 / denom), else: 0 %>
<div class="flex items-center gap-2 text-xs" data-progress-key={key}>
@ -433,7 +435,7 @@ defmodule MicrowavepropWeb.StatusLive do
<h2 class="text-base font-semibold mb-2">Enrichment Status by Type</h2>
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
<%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"era5", "ERA5"}] do %>
<%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"narr", "NARR"}] do %>
<div class="bg-base-200 rounded-box p-3 text-xs">
<div class="font-semibold mb-1">{label}</div>
<%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %>

View file

@ -20,7 +20,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
{:ok, conn: conn}
end
defp create_era5_candidate do
defp create_narr_candidate do
{:ok, contact} =
%Contact{}
|> Contact.changeset(%{
@ -41,16 +41,18 @@ defmodule MicrowavepropWeb.StatusLiveTest do
contact
end
test "ERA5 progress bar numerator is actual era5 profile matches not total-minus-candidates", %{conn: conn} do
create_era5_candidate()
# NARR fetches land in era5_profiles (table name kept as historical artifact);
# the status UI labels this "NARR" now.
test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do
create_narr_candidate()
# No ERA5 profile yet — candidates = 1, done = 0
# No NARR profile yet — candidates = 1, done = 0
{:ok, lv, _html} = live(conn, ~p"/status")
era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render()
assert era5_row =~ "ERA5"
assert era5_row =~ "0 / 1"
narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
assert narr_row =~ "NARR"
assert narr_row =~ "0 / 1"
# Insert a matching era5_profile row — done should become 1
# Insert a matching profile row — done should become 1
{:ok, _} =
Repo.insert(%Era5Profile{
valid_time: ~U[2010-06-15 18:00:00Z],
@ -62,7 +64,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
{:ok, lv, _html} = live(conn, ~p"/status")
era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render()
assert era5_row =~ "1 / 1"
narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
assert narr_row =~ "1 / 1"
end
end