From 7857d3bc5acd37c77497b1381f8717e9405a60e2 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 16 Apr 2026 08:56:57 -0500 Subject: [PATCH] =?UTF-8?q?Status=20page:=20relabel=20ERA5=20=E2=86=92=20N?= =?UTF-8?q?ARR,=20include=20NARR=20in=20backfill=20cron?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /status progress bar, status-by-type card, and internal map keys now read "NARR" (data-progress-key="narr", narr_candidates/narr_done). era5_profiles table stays as the DB landing spot (historical artifact). - BackfillEnqueueWorker cron now passes "era5" in types so pre-2014 contacts whose hrrr_status is :unavailable actually get NarrFetchWorker jobs enqueued every 30 min. The :era5 type key still maps internally to NarrFetchWorker via era5_jobs_for_contact/1. --- config/runtime.exs | 5 ++- lib/microwaveprop_web/live/status_live.ex | 32 ++++++++++--------- .../live/status_live_test.exs | 22 +++++++------ 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/config/runtime.exs b/config/runtime.exs index 1ac07f35..dcded4a9 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -218,8 +218,11 @@ if config_env() == :prod do # full grid-side JSONB write for. # {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, + # The :era5 type dispatches NarrFetchWorker for pre-2014 contacts + # (see ContactWeatherEnqueueWorker.era5_jobs_for_contact/1). The + # type key is a historical artifact — the data source is NARR. {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, - args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}}, + args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "era5"]}}, # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch {"30 1 * * *", CanadianSoundingFetchWorker}, {"30 13 * * *", CanadianSoundingFetchWorker}, diff --git a/lib/microwaveprop_web/live/status_live.ex b/lib/microwaveprop_web/live/status_live.ex index c2146ba5..1306ac95 100644 --- a/lib/microwaveprop_web/live/status_live.ex +++ b/lib/microwaveprop_web/live/status_live.ex @@ -76,10 +76,10 @@ defmodule MicrowavepropWeb.StatusLive do weather = count_incomplete(:weather_status, incomplete) iemre = count_incomplete(:iemre_status, incomplete) - era5_candidates = + narr_candidates = Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count())) - era5_done = count_era5_done() + narr_done = count_narr_done() all_done = count_all_done(done) @@ -88,18 +88,19 @@ defmodule MicrowavepropWeb.StatusLive do hrrr: hrrr, weather: weather, iemre: iemre, - era5_candidates: era5_candidates, - era5_done: era5_done, + narr_candidates: narr_candidates, + narr_done: narr_done, remaining: max(terrain, max(hrrr, max(weather, iemre))), complete: all_done, total: total } end - # ERA5 is a fallback for pre-2014 contacts (HRRR unavailable). "Done" means - # there's an era5_profiles row within 0.15° + 30 min of the contact — the - # same tolerance `Weather.find_nearest_era5/3` uses at lookup time. - defp count_era5_done do + # NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014). + # Profiles land in the era5_profiles table (kept 1:1 after the ERA5/CDS + # pipeline was replaced). "Done" means a row within 0.15° + 30 min of the + # contact, matching `Weather.find_nearest_era5/3` lookup tolerance. + defp count_narr_done do %{rows: [[done]]} = Repo.query!(""" SELECT count(*) @@ -262,16 +263,17 @@ defmodule MicrowavepropWeb.StatusLive do # Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*)) table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end) - # ERA5 doesn't have its own status column — synthesize from HRRR unavailable + era5_profiles count - era5_count = Map.get(table_counts, "era5_profiles", 0) + # NARR doesn't have its own status column — synthesize from HRRR unavailable + # + era5_profiles row count (the table is reused as the NARR landing spot). + narr_count = Map.get(table_counts, "era5_profiles", 0) hrrr_unavailable = statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end) statuses = - Map.put(statuses, "era5", [ + Map.put(statuses, "narr", [ {"candidates (HRRR unavail)", hrrr_unavailable}, - {"profiles fetched", era5_count} + {"profiles fetched", narr_count} ]) hrrr_count = Map.get(table_counts, "hrrr_profiles", 0) @@ -287,7 +289,7 @@ defmodule MicrowavepropWeb.StatusLive do statuses: statuses, totals: %{ hrrr_profiles: hrrr_count, - era5_profiles: Map.get(table_counts, "era5_profiles", 0), + narr_profiles: Map.get(table_counts, "era5_profiles", 0), terrain_profiles: terrain_count, surface_observations: obs_count, soundings: sounding_count, @@ -354,7 +356,7 @@ defmodule MicrowavepropWeb.StatusLive do {"weather", "Weather", @unprocessed.total - @unprocessed.weather, @unprocessed.total}, {"terrain", "Terrain", @unprocessed.total - @unprocessed.terrain, @unprocessed.total}, {"iemre", "IEMRE", @unprocessed.total - @unprocessed.iemre, @unprocessed.total}, - {"era5", "ERA5", @unprocessed.era5_done, @unprocessed.era5_candidates} + {"narr", "NARR", @unprocessed.narr_done, @unprocessed.narr_candidates} ] do %> <% pct = if denom > 0, do: round(complete * 100 / denom), else: 0 %>
@@ -433,7 +435,7 @@ defmodule MicrowavepropWeb.StatusLive do

Enrichment Status by Type

- <%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"era5", "ERA5"}] do %> + <%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"narr", "NARR"}] do %>
{label}
<%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %> diff --git a/test/microwaveprop_web/live/status_live_test.exs b/test/microwaveprop_web/live/status_live_test.exs index bd37b257..2618d2de 100644 --- a/test/microwaveprop_web/live/status_live_test.exs +++ b/test/microwaveprop_web/live/status_live_test.exs @@ -20,7 +20,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do {:ok, conn: conn} end - defp create_era5_candidate do + defp create_narr_candidate do {:ok, contact} = %Contact{} |> Contact.changeset(%{ @@ -41,16 +41,18 @@ defmodule MicrowavepropWeb.StatusLiveTest do contact end - test "ERA5 progress bar numerator is actual era5 profile matches not total-minus-candidates", %{conn: conn} do - create_era5_candidate() + # NARR fetches land in era5_profiles (table name kept as historical artifact); + # the status UI labels this "NARR" now. + test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do + create_narr_candidate() - # No ERA5 profile yet — candidates = 1, done = 0 + # No NARR profile yet — candidates = 1, done = 0 {:ok, lv, _html} = live(conn, ~p"/status") - era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render() - assert era5_row =~ "ERA5" - assert era5_row =~ "0 / 1" + narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render() + assert narr_row =~ "NARR" + assert narr_row =~ "0 / 1" - # Insert a matching era5_profile row — done should become 1 + # Insert a matching profile row — done should become 1 {:ok, _} = Repo.insert(%Era5Profile{ valid_time: ~U[2010-06-15 18:00:00Z], @@ -62,7 +64,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed}) {:ok, lv, _html} = live(conn, ~p"/status") - era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render() - assert era5_row =~ "1 / 1" + narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render() + assert narr_row =~ "1 / 1" end end