defmodule MicrowavepropWeb.StatusLive do @moduledoc "`/status` ops dashboard — enrichment progress, queue depths, DB size." use MicrowavepropWeb, :live_view import Ecto.Query alias Microwaveprop.Cache alias Microwaveprop.Format alias Microwaveprop.Radio.Contact alias Microwaveprop.Repo alias Microwaveprop.Weather.NarrClient @impl true def mount(_params, _session, socket) do _ = if connected?(socket) do :ok = Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "db:contact_status") :ok = Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "db:oban_jobs") # Rust prop-grid-rs emits NOTIFY propagation_ready on completion — # PropagationNotifyListener fans it out as `propagation:updated` so # the grid_tasks panel transitions from "running" → "done" without # a manual refresh. Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "propagation:updated") end stats = fetch_stats() unprocessed = count_unprocessed() db_stats = fetch_db_stats() grid_tasks = fetch_grid_tasks_stats() hrrr_point_tasks = fetch_hrrr_point_tasks_stats() {:ok, assign(socket, page_title: "Status", stats: stats, unprocessed: unprocessed, db_stats: db_stats, grid_tasks: grid_tasks, hrrr_point_tasks: hrrr_point_tasks, refresh_timer: nil )} end @impl true def handle_info({:contact_status_changed, _data}, socket) do {:noreply, schedule_refresh(socket)} end def handle_info({:oban_job_changed, _data}, socket) do {:noreply, schedule_refresh(socket)} end def handle_info({:propagation_updated, _valid_times}, socket) do {:noreply, schedule_refresh(socket)} end def handle_info(:refresh_stats, socket) do {:noreply, assign(socket, stats: fetch_stats(), unprocessed: count_unprocessed(), db_stats: fetch_db_stats(), grid_tasks: fetch_grid_tasks_stats(), hrrr_point_tasks: fetch_hrrr_point_tasks_stats(), refresh_timer: nil )} end defp schedule_refresh(%{assigns: %{refresh_timer: ref}} = socket) when is_reference(ref) do # Already scheduled, skip socket end defp schedule_refresh(socket) do # 2s debounce — bulk enrichment runs produce hundreds of status_changed # events per second, and each stats refresh fires 8+ count queries. ref = Process.send_after(self(), :refresh_stats, 2_000) assign(socket, refresh_timer: ref) end defp count_unprocessed do Cache.fetch_or_store({__MODULE__, :count_unprocessed}, 2_000, fn -> count_unprocessed_raw() end) end defp count_unprocessed_raw do # One scan over the contacts table returns every per-type pending # count via FILTER clauses. Cheaper than N separate COUNT queries # and cheaper than N index scans. Keep the FILTER list in lockstep # with the progress-bar list in the template — missing a field # here means the progress bar will deny-count its own column. %{rows: [[total, terrain, hrrr, weather, iemre, radar, mechanism, all_done]]} = Repo.query!(""" SELECT COUNT(*) FILTER (WHERE pos1 IS NOT NULL), COUNT(*) FILTER (WHERE pos1 IS NOT NULL AND terrain_status::text IN ('pending','queued','processing','failed')), COUNT(*) FILTER (WHERE pos1 IS NOT NULL AND hrrr_status::text IN ('pending','queued','processing','failed')), COUNT(*) FILTER (WHERE pos1 IS NOT NULL AND weather_status::text IN ('pending','queued','processing','failed')), COUNT(*) FILTER (WHERE pos1 IS NOT NULL AND iemre_status::text IN ('pending','queued','processing','failed')), COUNT(*) FILTER (WHERE pos1 IS NOT NULL AND radar_status::text IN ('pending','queued','processing','failed')), COUNT(*) FILTER (WHERE pos1 IS NOT NULL AND mechanism_status::text IN ('pending','queued','processing','failed')), COUNT(*) FILTER (WHERE pos1 IS NOT NULL AND hrrr_status::text IN ('complete','unavailable') AND weather_status::text IN ('complete','unavailable') AND terrain_status::text IN ('complete','unavailable') AND iemre_status::text IN ('complete','unavailable')) FROM contacts """) narr_candidates = count_narr_candidates() narr_done = count_narr_done() %{ terrain: terrain, hrrr: hrrr, weather: weather, iemre: iemre, radar: radar, mechanism: mechanism, narr_candidates: narr_candidates, narr_done: narr_done, remaining: max(terrain, max(hrrr, max(weather, iemre))), complete: all_done, total: total } end # NARR is the fallback for pre-2014 contacts (HRRR archive starts 2014). # Candidate eligibility is defined by qso_timestamp, not hrrr_status — # pre-2014 rows bounce between :queued and :unavailable every cron cycle # and the UI must count them either way. defp count_narr_candidates do coverage_end = NarrClient.coverage_end() Repo.one( from(c in Contact, where: c.qso_timestamp < ^coverage_end and not is_nil(c.pos1), select: count() ) ) end # "Done" means a narr_profiles row within 0.15° of the contact and # stamped at the floor-snapped 3-hour analysis mark. # # NARR analyses live at 00/03/06/09/12/15/18/21 UTC. The fetcher calls # `NarrClient.snap_to_analysis_hour/1`, which floors qso_timestamp to the # previous 3-hour mark — so a contact at 18:50Z yields a profile valid at # 18:00Z. The SQL snap below must stay in sync with that Elixir function. defp count_narr_done do coverage_end = NarrClient.coverage_end() %{rows: [[done]]} = Repo.query!( """ SELECT count(*) FROM contacts c WHERE c.qso_timestamp < $1 AND c.pos1 IS NOT NULL AND EXISTS ( SELECT 1 FROM narr_profiles p WHERE p.lat BETWEEN ((c.pos1->>'lat')::float - 0.15) AND ((c.pos1->>'lat')::float + 0.15) AND p.lon BETWEEN ((c.pos1->>'lon')::float - 0.15) AND ((c.pos1->>'lon')::float + 0.15) AND p.valid_time = date_trunc('hour', c.qso_timestamp) - make_interval(hours => CAST(EXTRACT(HOUR FROM c.qso_timestamp) AS integer) % 3) ) """, [coverage_end] ) done end defp fetch_grid_tasks_stats do Cache.fetch_or_store({__MODULE__, :grid_tasks_stats}, 2_000, fn -> fetch_grid_tasks_stats_raw() end) end # Snapshot of the Rust prop-grid-rs work queue. Mirrors the Oban panel — # "what's running, what's pending, what's retrying" — but pulls from the # grid_tasks hand-off table instead of oban_jobs. defp fetch_grid_tasks_stats_raw do running = Repo.all( from(t in "grid_tasks", where: t.status == "running", order_by: [asc: t.run_time, asc: t.forecast_hour], select: %{ run_time: t.run_time, forecast_hour: t.forecast_hour, attempt: t.attempt, claimed_at: t.claimed_at } ) ) counts_by_status = from(t in "grid_tasks", where: t.status in ["queued", "failed"], group_by: t.status, select: {t.status, count(t.id)} ) |> Repo.all() |> Map.new() done_1h = Repo.one( from(t in "grid_tasks", where: t.status == "done" and t.completed_at > ago(1, "hour"), select: count(t.id) ) ) %{ running: running, queued: Map.get(counts_by_status, "queued", 0), failed: Map.get(counts_by_status, "failed", 0), done_1h: done_1h } end defp fetch_hrrr_point_tasks_stats do Cache.fetch_or_store({__MODULE__, :hrrr_point_tasks_stats}, 2_000, fn -> fetch_hrrr_point_tasks_stats_raw() end) end # Snapshot of the Rust hrrr-point-rs work queue (per-QSO HRRR batches, # one row per valid_time). Same shape as grid_tasks_stats so the UI # can render both in a single table. defp fetch_hrrr_point_tasks_stats_raw do running_count = Repo.one( from(t in "hrrr_fetch_tasks", where: t.status == "running", select: count(t.id) ) ) counts_by_status = from(t in "hrrr_fetch_tasks", where: t.status in ["queued", "failed"], group_by: t.status, select: {t.status, count(t.id)} ) |> Repo.all() |> Map.new() done_1h = Repo.one( from(t in "hrrr_fetch_tasks", where: t.status == "done" and t.completed_at > ago(1, "hour"), select: count(t.id) ) ) %{ running: running_count, queued: Map.get(counts_by_status, "queued", 0), failed: Map.get(counts_by_status, "failed", 0), done_1h: done_1h } end defp fetch_stats do Cache.fetch_or_store({__MODULE__, :stats}, 2_000, fn -> fetch_stats_raw() end) end defp fetch_stats_raw do jobs = Repo.all( from(j in "oban_jobs", where: j.state in ["available", "executing", "scheduled", "retryable"], group_by: [j.worker, j.state], select: {j.worker, j.state, count(j.id)} ) ) by_worker = jobs |> Enum.group_by(&elem(&1, 0), fn {_, state, count} -> {state, count} end) |> Enum.map(&summarize_worker/1) |> Enum.sort_by(& &1.total, :desc) completed_1h = Repo.one( from(j in "oban_jobs", where: j.state == "completed" and j.completed_at > ago(1, "hour"), select: count(j.id) ) ) %{by_worker: by_worker, completed_1h: completed_1h} end defp summarize_worker({worker, states}) do short_name = worker |> String.split(".") |> List.last() %{ worker: short_name, total: Enum.reduce(states, 0, fn {_, c}, acc -> acc + c end), executing: state_count(states, "executing"), available: state_count(states, "available"), retryable: state_count(states, "retryable") } end defp state_count(states, name) do Enum.find_value(states, 0, fn {s, c} -> if s == name, do: c end) end defp fetch_db_stats do # pg_class + pg_stat_user_tables join is slow — 5s cache is fine since # table row counts barely change in a 5-second window. Cache.fetch_or_store({__MODULE__, :db_stats}, 5_000, fn -> fetch_db_stats_raw() end) end defp fetch_db_stats_raw do %{rows: rows} = Repo.query!(""" SELECT c.relname as table, s.n_live_tup as rows, s.n_dead_tup as dead, pg_total_relation_size(c.oid) as bytes FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace JOIN pg_stat_user_tables s ON s.relid = c.oid WHERE n.nspname = 'public' AND c.relkind = 'r' AND c.relname NOT LIKE 'oban_%' AND c.relname NOT LIKE 'schema_%' AND NOT EXISTS (SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid) UNION ALL SELECT parent.relname, COALESCE(SUM(cs.n_live_tup), 0)::bigint, COALESCE(SUM(cs.n_dead_tup), 0)::bigint, COALESCE(SUM(pg_total_relation_size(child.oid)), 0)::bigint FROM pg_class parent JOIN pg_namespace n ON n.oid = parent.relnamespace JOIN pg_inherits inh ON inh.inhparent = parent.oid JOIN pg_class child ON child.oid = inh.inhrelid LEFT JOIN pg_stat_user_tables cs ON cs.relid = child.oid WHERE n.nspname = 'public' AND parent.relkind = 'p' GROUP BY parent.relname ORDER BY bytes DESC LIMIT 20 """) tables = Enum.map(rows, fn [table, row_count, dead, bytes] -> %{table: table, rows: row_count, dead: dead, size: Format.bytes(bytes)} end) %{rows: total_rows} = Repo.query!("SELECT pg_database_size(current_database())") [[db_bytes]] = total_rows # Enrichment status breakdown %{rows: status_rows} = Repo.query!(""" SELECT 'hrrr' as type, hrrr_status as status, count(*) as cnt FROM contacts GROUP BY hrrr_status UNION ALL SELECT 'weather', weather_status, count(*) FROM contacts GROUP BY weather_status UNION ALL SELECT 'terrain', terrain_status, count(*) FROM contacts GROUP BY terrain_status UNION ALL SELECT 'iemre', iemre_status, count(*) FROM contacts GROUP BY iemre_status UNION ALL SELECT 'radar', radar_status, count(*) FROM contacts GROUP BY radar_status ORDER BY 1, 2 """) statuses = Enum.group_by(status_rows, fn [type, _, _] -> type end, fn [_, status, count] -> {status, count} end) # Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*)) table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end) # NARR doesn't have its own status column — synthesize from HRRR unavailable # + narr_profiles row count. narr_count = Map.get(table_counts, "narr_profiles", 0) hrrr_unavailable = statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end) statuses = Map.put(statuses, "narr", [ {"candidates (HRRR unavail)", hrrr_unavailable}, {"profiles fetched", narr_count} ]) hrrr_count = Map.get(table_counts, "hrrr_profiles", 0) terrain_count = Map.get(table_counts, "terrain_profiles", 0) obs_count = Map.get(table_counts, "surface_observations", 0) sounding_count = Map.get(table_counts, "soundings", 0) iemre_count = Map.get(table_counts, "iemre_observations", 0) contact_count = max(Map.get(table_counts, "contacts", 0), 1) %{ db_size: Format.bytes(db_bytes), tables: tables, statuses: statuses, totals: %{ hrrr_profiles: hrrr_count, narr_profiles: Map.get(table_counts, "narr_profiles", 0), terrain_profiles: terrain_count, surface_observations: obs_count, soundings: sounding_count, iemre_observations: iemre_count, contacts: contact_count }, averages: %{ hrrr_per_contact: Float.round(hrrr_count / contact_count, 1), obs_per_contact: Float.round(obs_count / contact_count, 1), soundings_per_contact: Float.round(sounding_count / contact_count, 1) } } end defp status_class("complete"), do: "text-success" defp status_class("pending"), do: "text-warning" defp status_class("queued"), do: "text-info" defp status_class("processing"), do: "text-info" defp status_class("failed"), do: "text-error" defp status_class("unavailable"), do: "opacity-40" defp status_class(_), do: "" @impl true def render(assigns) do ~H""" <.header> Status <:subtitle>Enrichment progress, job queues, and database health
Enrichment Progress
{Format.number(@unprocessed.complete)} / {Format.number(@unprocessed.total)}
fully enriched
<%= for {key, label, complete, denom} <- [ {"hrrr", "HRRR", @unprocessed.total - @unprocessed.hrrr, @unprocessed.total}, {"weather", "Weather", @unprocessed.total - @unprocessed.weather, @unprocessed.total}, {"terrain", "Terrain", @unprocessed.total - @unprocessed.terrain, @unprocessed.total}, {"iemre", "IEMRE", @unprocessed.total - @unprocessed.iemre, @unprocessed.total}, {"radar", "Radar", @unprocessed.total - @unprocessed.radar, @unprocessed.total}, {"mechanism", "Mechanism", @unprocessed.total - @unprocessed.mechanism, @unprocessed.total}, {"narr", "NARR", @unprocessed.narr_done, @unprocessed.narr_candidates} ] do %> <% pct = if denom > 0, do: round(complete * 100 / denom), else: 0 %>
{label} {Format.number(complete)} / {Format.number(denom)}
<% end %>
Active/Queued Jobs
{Format.number(Enum.reduce(@stats.by_worker, 0, fn w, acc -> acc + w.total end))}
Completed (last hour)
{Format.number(@stats.completed_1h)}

Job Queue Status

<%= if @stats.by_worker == [] do %>

No active jobs.

<% else %>
<%= for w <- @stats.by_worker do %> <% end %>
Worker Executing Pending Retryable Total
{w.worker} <%= if w.executing > 0 do %> {w.executing} <% else %> 0 <% end %> {w.available} <%= if w.retryable > 0 do %> {w.retryable} <% else %> 0 <% end %> {w.total}
<% end %>

Rust Workers

Worker Running Queued Failed Done (1h)
prop-grid-rs <%= if @grid_tasks.running == [] do %> 0 <% else %> <%= for r <- @grid_tasks.running do %> f{String.pad_leading(Integer.to_string(r.forecast_hour), 2, "0")} <% end %> running <% end %> {@grid_tasks.queued} <%= if @grid_tasks.failed > 0 do %> {@grid_tasks.failed} <% else %> 0 <% end %> {Format.number(@grid_tasks.done_1h)}
hrrr-point-rs <%= if @hrrr_point_tasks.running == 0 do %> 0 <% else %> {@hrrr_point_tasks.running} batches <% end %> {Format.number(@hrrr_point_tasks.queued)} <%= if @hrrr_point_tasks.failed > 0 do %> {@hrrr_point_tasks.failed} <% else %> 0 <% end %> {Format.number(@hrrr_point_tasks.done_1h)}

Enrichment Status by Type

<%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"radar", "Radar"}, {"narr", "NARR"}] do %>
{label}
<%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %>
{status} {Format.number(count)}
<% end %>
<% end %>

Database

Total DB Size
{@db_stats.db_size}
HRRR Profiles
{Format.number(@db_stats.totals.hrrr_profiles)}
{@db_stats.averages.hrrr_per_contact} per contact
Surface Observations
{Format.number(@db_stats.totals.surface_observations)}
{@db_stats.averages.obs_per_contact} per contact
Soundings
{Format.number(@db_stats.totals.soundings)}
{@db_stats.averages.soundings_per_contact} per contact

Table Sizes

<%= for t <- @db_stats.tables do %> <% end %>
Table Rows Dead Size
{t.table} {Format.number(t.rows)} 0 && "text-warning" ]}> {Format.number(t.dead)} {t.size}

Live updates via database notifications.

""" end end