Status page: relabel ERA5 → NARR, include NARR in backfill cron
- /status progress bar, status-by-type card, and internal map keys now read "NARR" (data-progress-key="narr", narr_candidates/narr_done). era5_profiles table stays as the DB landing spot (historical artifact). - BackfillEnqueueWorker cron now passes "era5" in types so pre-2014 contacts whose hrrr_status is :unavailable actually get NarrFetchWorker jobs enqueued every 30 min. The :era5 type key still maps internally to NarrFetchWorker via era5_jobs_for_contact/1.
This commit is contained in:
parent
17271b264a
commit
7857d3bc5a
3 changed files with 33 additions and 26 deletions
|
|
@ -218,8 +218,11 @@ if config_env() == :prod do
|
||||||
# full grid-side JSONB write for.
|
# full grid-side JSONB write for.
|
||||||
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
# {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
||||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||||
|
# The :era5 type dispatches NarrFetchWorker for pre-2014 contacts
|
||||||
|
# (see ContactWeatherEnqueueWorker.era5_jobs_for_contact/1). The
|
||||||
|
# type key is a historical artifact — the data source is NARR.
|
||||||
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
|
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
|
||||||
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}},
|
args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre", "era5"]}},
|
||||||
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
# UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
|
||||||
{"30 1 * * *", CanadianSoundingFetchWorker},
|
{"30 1 * * *", CanadianSoundingFetchWorker},
|
||||||
{"30 13 * * *", CanadianSoundingFetchWorker},
|
{"30 13 * * *", CanadianSoundingFetchWorker},
|
||||||
|
|
|
||||||
|
|
@ -76,10 +76,10 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
weather = count_incomplete(:weather_status, incomplete)
|
weather = count_incomplete(:weather_status, incomplete)
|
||||||
iemre = count_incomplete(:iemre_status, incomplete)
|
iemre = count_incomplete(:iemre_status, incomplete)
|
||||||
|
|
||||||
era5_candidates =
|
narr_candidates =
|
||||||
Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count()))
|
Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count()))
|
||||||
|
|
||||||
era5_done = count_era5_done()
|
narr_done = count_narr_done()
|
||||||
|
|
||||||
all_done = count_all_done(done)
|
all_done = count_all_done(done)
|
||||||
|
|
||||||
|
|
@ -88,18 +88,19 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
hrrr: hrrr,
|
hrrr: hrrr,
|
||||||
weather: weather,
|
weather: weather,
|
||||||
iemre: iemre,
|
iemre: iemre,
|
||||||
era5_candidates: era5_candidates,
|
narr_candidates: narr_candidates,
|
||||||
era5_done: era5_done,
|
narr_done: narr_done,
|
||||||
remaining: max(terrain, max(hrrr, max(weather, iemre))),
|
remaining: max(terrain, max(hrrr, max(weather, iemre))),
|
||||||
complete: all_done,
|
complete: all_done,
|
||||||
total: total
|
total: total
|
||||||
}
|
}
|
||||||
end
|
end
|
||||||
|
|
||||||
# ERA5 is a fallback for pre-2014 contacts (HRRR unavailable). "Done" means
|
# NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014).
|
||||||
# there's an era5_profiles row within 0.15° + 30 min of the contact — the
|
# Profiles land in the era5_profiles table (kept 1:1 after the ERA5/CDS
|
||||||
# same tolerance `Weather.find_nearest_era5/3` uses at lookup time.
|
# pipeline was replaced). "Done" means a row within 0.15° + 30 min of the
|
||||||
defp count_era5_done do
|
# contact, matching `Weather.find_nearest_era5/3` lookup tolerance.
|
||||||
|
defp count_narr_done do
|
||||||
%{rows: [[done]]} =
|
%{rows: [[done]]} =
|
||||||
Repo.query!("""
|
Repo.query!("""
|
||||||
SELECT count(*)
|
SELECT count(*)
|
||||||
|
|
@ -262,16 +263,17 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
# Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*))
|
# Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*))
|
||||||
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
|
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
|
||||||
|
|
||||||
# ERA5 doesn't have its own status column — synthesize from HRRR unavailable + era5_profiles count
|
# NARR doesn't have its own status column — synthesize from HRRR unavailable
|
||||||
era5_count = Map.get(table_counts, "era5_profiles", 0)
|
# + era5_profiles row count (the table is reused as the NARR landing spot).
|
||||||
|
narr_count = Map.get(table_counts, "era5_profiles", 0)
|
||||||
|
|
||||||
hrrr_unavailable =
|
hrrr_unavailable =
|
||||||
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
|
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
|
||||||
|
|
||||||
statuses =
|
statuses =
|
||||||
Map.put(statuses, "era5", [
|
Map.put(statuses, "narr", [
|
||||||
{"candidates (HRRR unavail)", hrrr_unavailable},
|
{"candidates (HRRR unavail)", hrrr_unavailable},
|
||||||
{"profiles fetched", era5_count}
|
{"profiles fetched", narr_count}
|
||||||
])
|
])
|
||||||
|
|
||||||
hrrr_count = Map.get(table_counts, "hrrr_profiles", 0)
|
hrrr_count = Map.get(table_counts, "hrrr_profiles", 0)
|
||||||
|
|
@ -287,7 +289,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
statuses: statuses,
|
statuses: statuses,
|
||||||
totals: %{
|
totals: %{
|
||||||
hrrr_profiles: hrrr_count,
|
hrrr_profiles: hrrr_count,
|
||||||
era5_profiles: Map.get(table_counts, "era5_profiles", 0),
|
narr_profiles: Map.get(table_counts, "era5_profiles", 0),
|
||||||
terrain_profiles: terrain_count,
|
terrain_profiles: terrain_count,
|
||||||
surface_observations: obs_count,
|
surface_observations: obs_count,
|
||||||
soundings: sounding_count,
|
soundings: sounding_count,
|
||||||
|
|
@ -354,7 +356,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
{"weather", "Weather", @unprocessed.total - @unprocessed.weather, @unprocessed.total},
|
{"weather", "Weather", @unprocessed.total - @unprocessed.weather, @unprocessed.total},
|
||||||
{"terrain", "Terrain", @unprocessed.total - @unprocessed.terrain, @unprocessed.total},
|
{"terrain", "Terrain", @unprocessed.total - @unprocessed.terrain, @unprocessed.total},
|
||||||
{"iemre", "IEMRE", @unprocessed.total - @unprocessed.iemre, @unprocessed.total},
|
{"iemre", "IEMRE", @unprocessed.total - @unprocessed.iemre, @unprocessed.total},
|
||||||
{"era5", "ERA5", @unprocessed.era5_done, @unprocessed.era5_candidates}
|
{"narr", "NARR", @unprocessed.narr_done, @unprocessed.narr_candidates}
|
||||||
] do %>
|
] do %>
|
||||||
<% pct = if denom > 0, do: round(complete * 100 / denom), else: 0 %>
|
<% pct = if denom > 0, do: round(complete * 100 / denom), else: 0 %>
|
||||||
<div class="flex items-center gap-2 text-xs" data-progress-key={key}>
|
<div class="flex items-center gap-2 text-xs" data-progress-key={key}>
|
||||||
|
|
@ -433,7 +435,7 @@ defmodule MicrowavepropWeb.StatusLive do
|
||||||
|
|
||||||
<h2 class="text-base font-semibold mb-2">Enrichment Status by Type</h2>
|
<h2 class="text-base font-semibold mb-2">Enrichment Status by Type</h2>
|
||||||
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
|
||||||
<%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"era5", "ERA5"}] do %>
|
<%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"narr", "NARR"}] do %>
|
||||||
<div class="bg-base-200 rounded-box p-3 text-xs">
|
<div class="bg-base-200 rounded-box p-3 text-xs">
|
||||||
<div class="font-semibold mb-1">{label}</div>
|
<div class="font-semibold mb-1">{label}</div>
|
||||||
<%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %>
|
<%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %>
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
||||||
{:ok, conn: conn}
|
{:ok, conn: conn}
|
||||||
end
|
end
|
||||||
|
|
||||||
defp create_era5_candidate do
|
defp create_narr_candidate do
|
||||||
{:ok, contact} =
|
{:ok, contact} =
|
||||||
%Contact{}
|
%Contact{}
|
||||||
|> Contact.changeset(%{
|
|> Contact.changeset(%{
|
||||||
|
|
@ -41,16 +41,18 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
||||||
contact
|
contact
|
||||||
end
|
end
|
||||||
|
|
||||||
test "ERA5 progress bar numerator is actual era5 profile matches not total-minus-candidates", %{conn: conn} do
|
# NARR fetches land in era5_profiles (table name kept as historical artifact);
|
||||||
create_era5_candidate()
|
# the status UI labels this "NARR" now.
|
||||||
|
test "NARR progress bar numerator is actual profile matches not total-minus-candidates", %{conn: conn} do
|
||||||
|
create_narr_candidate()
|
||||||
|
|
||||||
# No ERA5 profile yet — candidates = 1, done = 0
|
# No NARR profile yet — candidates = 1, done = 0
|
||||||
{:ok, lv, _html} = live(conn, ~p"/status")
|
{:ok, lv, _html} = live(conn, ~p"/status")
|
||||||
era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render()
|
narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
|
||||||
assert era5_row =~ "ERA5"
|
assert narr_row =~ "NARR"
|
||||||
assert era5_row =~ "0 / 1"
|
assert narr_row =~ "0 / 1"
|
||||||
|
|
||||||
# Insert a matching era5_profile row — done should become 1
|
# Insert a matching profile row — done should become 1
|
||||||
{:ok, _} =
|
{:ok, _} =
|
||||||
Repo.insert(%Era5Profile{
|
Repo.insert(%Era5Profile{
|
||||||
valid_time: ~U[2010-06-15 18:00:00Z],
|
valid_time: ~U[2010-06-15 18:00:00Z],
|
||||||
|
|
@ -62,7 +64,7 @@ defmodule MicrowavepropWeb.StatusLiveTest do
|
||||||
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
|
Cache.invalidate({MicrowavepropWeb.StatusLive, :count_unprocessed})
|
||||||
|
|
||||||
{:ok, lv, _html} = live(conn, ~p"/status")
|
{:ok, lv, _html} = live(conn, ~p"/status")
|
||||||
era5_row = lv |> element(~s|[data-progress-key="era5"]|) |> render()
|
narr_row = lv |> element(~s|[data-progress-key="narr"]|) |> render()
|
||||||
assert era5_row =~ "1 / 1"
|
assert narr_row =~ "1 / 1"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue