Status page: relabel ERA5 → NARR, include NARR in backfill cron
- /status progress bar, status-by-type card, and internal map keys now read "NARR" (data-progress-key="narr", narr_candidates/narr_done). era5_profiles table stays as the DB landing spot (historical artifact). - BackfillEnqueueWorker cron now passes "era5" in types so pre-2014 contacts whose hrrr_status is :unavailable actually get NarrFetchWorker jobs enqueued every 30 min. The :era5 type key still maps internally to NarrFetchWorker via era5_jobs_for_contact/1.
This commit is contained in:
parent
17271b264a
commit
7857d3bc5a
3 changed files with 33 additions and 26 deletions
|
|
@ -218,8 +218,11 @@ if config_env() == :prod do
|
|||
# full grid-side JSONB write for.
|
||||
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||
# The :era5 type dispatches NarrFetchWorker for pre-2014 contacts
|
||||
# (see ContactWeatherEnqueueWorker.era5_jobs_for_contact/1). The
|
||||
# type key is a historical artifact — the data source is NARR.
|
||||
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
|
||||
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}},
|
||||
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "era5"]}},
|
||||
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
||||
{"30 1 * * *", CanadianSoundingFetchWorker},
|
||||
{"30 13 * * *", CanadianSoundingFetchWorker},
|
||||
|
|
|
|||
|
|
@ -76,10 +76,10 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
weather = count_incomplete(:weather_status, incomplete)
|
||||
iemre = count_incomplete(:iemre_status, incomplete)
|
||||
|
||||
era5_candidates =
|
||||
narr_candidates =
|
||||
Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count()))
|
||||
|
||||
era5_done = count_era5_done()
|
||||
narr_done = count_narr_done()
|
||||
|
||||
all_done = count_all_done(done)
|
||||
|
||||
|
|
@ -88,18 +88,19 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
hrrr: hrrr,
|
||||
weather: weather,
|
||||
iemre: iemre,
|
||||
era5_candidates: era5_candidates,
|
||||
era5_done: era5_done,
|
||||
narr_candidates: narr_candidates,
|
||||
narr_done: narr_done,
|
||||
remaining: max(terrain, max(hrrr, max(weather, iemre))),
|
||||
complete: all_done,
|
||||
total: total
|
||||
}
|
||||
end
|
||||
|
||||
# ERA5 is a fallback for pre-2014 contacts (HRRR unavailable). "Done" means
|
||||
# there's an era5_profiles row within 0.15° + 30 min of the contact — the
|
||||
# same tolerance `Weather.find_nearest_era5/3` uses at lookup time.
|
||||
defp count_era5_done do
|
||||
# NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
|
||||
# Profiles land in the era5_profiles table (kept 1:1 after the ERA5/CDS
|
||||
# pipeline was replaced). "Done" means a row within 0.15° + 30 min of the
|
||||
# contact, matching `Weather.find_nearest_era5/3` lookup tolerance.
|
||||
defp count_narr_done do
|
||||
%{rows: [[done]]} =
|
||||
Repo.query!("""
|
||||
SELECT count(*)
|
||||
|
|
@ -262,16 +263,17 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
# Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*))
|
||||
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
|
||||
|
||||
# ERA5 doesn't have its own status column — synthesize from HRRR unavailable + era5_profiles count
|
||||
era5_count = Map.get(table_counts, "era5_profiles", 0)
|
||||
# NARR doesn't have its own status column — synthesize from HRRR unavailable
|
||||
# + era5_profiles row count (the table is reused as the NARR landing spot).
|
||||
narr_count = Map.get(table_counts, "era5_profiles", 0)
|
||||
|
||||
hrrr_unavailable =
|
||||
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
|
||||
|
||||
statuses =
|
||||
Map.put(statuses, "era5", [
|
||||
Map.put(statuses, "narr", [
|
||||
{"candidates (HRRR unavail)", hrrr_unavailable},
|
||||
{"profiles fetched", era5_count}
|
||||
{"profiles fetched", narr_count}
|
||||
])
|
||||
|
||||
hrrr_count = Map.get(table_counts, "hrrr_profiles", 0)
|
||||
|
|
@ -287,7 +289,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
statuses: statuses,
|
||||
totals: %{
|
||||
hrrr_profiles: hrrr_count,
|
||||
era5_profiles: Map.get(table_counts, "era5_profiles", 0),
|
||||
narr_profiles: Map.get(table_counts, "era5_profiles", 0),
|
||||
terrain_profiles: terrain_count,
|
||||
surface_observations: obs_count,
|
||||
soundings: sounding_count,
|
||||
|
|
@ -354,7 +356,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
{"weather", "Weather", @unprocessed.total - @unprocessed.weather, @unprocessed.total},
|
||||
{"terrain", "Terrain", @unprocessed.total - @unprocessed.terrain, @unprocessed.total},
|
||||
{"iemre", "IEMRE", @unprocessed.total - @unprocessed.iemre, @unprocessed.total},
|
||||
{"era5", "ERA5", @unprocessed.era5_done, @unprocessed.era5_candidates}
|
||||
{"narr", "NARR", @unprocessed.narr_done, @unprocessed.narr_candidates}
|
||||
] do %>
|
||||
<% pct = if denom > 0, do: round(complete * 100 / denom), else: 0 %>
|
||||
<div class="flex items-center gap-2 text-xs" data-progress-key={key}>
|
||||
|
|
@ -433,7 +435,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
|||
|
||||
<h2 class="text-base font-semibold mb-2">Enrichment Status by Type</h2>
|
||||
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
|
||||
<%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"era5", "ERA5"}] do %>
|
||||
<%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"narr", "NARR"}] do %>
|
||||
<div class="bg-base-200 rounded-box p-3 text-xs">
|
||||
<div class="font-semibold mb-1">{label}</div>
|
||||
<%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %>
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
|||
{:ok, conn: conn}
|
||||
end
|
||||
|
||||
defp create_era5_candidate do
|
||||
defp create_narr_candidate do
|
||||
{:ok, contact} =
|
||||
%Contact{}
|
||||
|> Contact.changeset(%{
|
||||
|
|
@ -41,16 +41,18 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
|||
contact
|
||||
end
|
||||
|
||||
test "ERA5 progress bar numerator is actual era5 profile matches not total-minus-candidates", %{conn: conn} do
|
||||
create_era5_candidate()
|
||||
# NARR fetches land in era5_profiles (table name kept as historical artifact);
|
||||
# the status UI labels this "NARR" now.
|
||||
test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do
|
||||
create_narr_candidate()
|
||||
|
||||
# No ERA5 profile yet — candidates = 1, done = 0
|
||||
# No NARR profile yet — candidates = 1, done = 0
|
||||
{:ok, lv, _html} = live(conn, ~p"/status")
|
||||
era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render()
|
||||
assert era5_row =~ "ERA5"
|
||||
assert era5_row =~ "0 / 1"
|
||||
narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
|
||||
assert narr_row =~ "NARR"
|
||||
assert narr_row =~ "0 / 1"
|
||||
|
||||
# Insert a matching era5_profile row — done should become 1
|
||||
# Insert a matching profile row — done should become 1
|
||||
{:ok, _} =
|
||||
Repo.insert(%Era5Profile{
|
||||
valid_time: ~U[2010-06-15 18:00:00Z],
|
||||
|
|
@ -62,7 +64,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
|||
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
|
||||
|
||||
{:ok, lv, _html} = live(conn, ~p"/status")
|
||||
era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render()
|
||||
assert era5_row =~ "1 / 1"
|
||||
narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
|
||||
assert narr_row =~ "1 / 1"
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue