From 2452f1f3977e07e04a710361fe7aa78aa5800d04 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 2 Apr 2026 13:24:43 -0500 Subject: [PATCH] Add detailed database stats to backfill dashboard - Enrichment status breakdown per type (HRRR/Weather/Terrain/IEMRE) with color-coded status counts (complete/pending/queued/failed) - Database overview: total DB size, HRRR profiles, observations, soundings with per-contact averages - Table sizes: rows and disk size per table, sorted by size - All stats auto-refresh every 2 seconds --- lib/microwaveprop_web/live/backfill_live.ex | 200 +++++++++++++++++++- 1 file changed, 192 insertions(+), 8 deletions(-) diff --git a/lib/microwaveprop_web/live/backfill_live.ex b/lib/microwaveprop_web/live/backfill_live.ex index 2683fd7b..154c111f 100644 --- a/lib/microwaveprop_web/live/backfill_live.ex +++ b/lib/microwaveprop_web/live/backfill_live.ex @@ -19,11 +19,11 @@ defmodule MicrowavepropWeb.BackfillLive do stats = fetch_stats() unprocessed = count_unprocessed() - total_contacts = Repo.aggregate(from(c in Contact, where: not is_nil(c.pos1)), :count) + db_stats = fetch_db_stats() case LiveStash.recover_state(socket) do {:recovered, socket} -> - {:ok, assign(socket, stats: stats, unprocessed: unprocessed, total_contacts: total_contacts)} + {:ok, assign(socket, stats: stats, unprocessed: unprocessed, db_stats: db_stats)} _ -> {:ok, @@ -32,7 +32,7 @@ defmodule MicrowavepropWeb.BackfillLive do limit: 500, stats: stats, unprocessed: unprocessed, - total_contacts: total_contacts, + db_stats: db_stats, last_enqueued: nil, enqueuing: false )} @@ -61,7 +61,8 @@ defmodule MicrowavepropWeb.BackfillLive do last_enqueued: count, enqueuing: false, stats: fetch_stats(), - unprocessed: count_unprocessed() + unprocessed: count_unprocessed(), + db_stats: fetch_db_stats() )} end @@ -71,7 +72,8 @@ defmodule MicrowavepropWeb.BackfillLive do {:noreply, assign(socket, stats: fetch_stats(), - unprocessed: count_unprocessed() + unprocessed: count_unprocessed(), + db_stats: fetch_db_stats() )} end @@ -154,6 +156,126 @@ defmodule MicrowavepropWeb.BackfillLive do %{by_worker: by_worker, completed_1h: completed_1h} end + defp fetch_db_stats do + %{rows: rows} = + Repo.query!(""" + SELECT + relname as table, + n_live_tup as rows, + pg_total_relation_size(c.oid) as bytes + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_stat_user_tables s ON s.relid = c.oid + WHERE n.nspname = 'public' + AND c.relkind = 'r' + AND relname NOT LIKE 'oban_%' + AND relname NOT LIKE 'schema_%' + AND n_live_tup > 0 + ORDER BY pg_total_relation_size(c.oid) DESC + LIMIT 20 + """) + + tables = + Enum.map(rows, fn [table, row_count, bytes] -> + %{table: table, rows: row_count, size: format_bytes(bytes)} + end) + + %{rows: total_rows} = + Repo.query!("SELECT pg_database_size(current_database())") + + [[db_bytes]] = total_rows + + # Enrichment status breakdown + %{rows: status_rows} = + Repo.query!(""" + SELECT + 'hrrr' as type, hrrr_status as status, count(*) as cnt FROM qsos GROUP BY hrrr_status + UNION ALL + SELECT + 'weather', weather_status, count(*) FROM qsos GROUP BY weather_status + UNION ALL + SELECT + 'terrain', terrain_status, count(*) FROM qsos GROUP BY terrain_status + UNION ALL + SELECT + 'iemre', iemre_status, count(*) FROM qsos GROUP BY iemre_status + ORDER BY 1, 2 + """) + + statuses = + Enum.group_by(status_rows, fn [type, _, _] -> type end, fn [_, status, count] -> + {status, count} + end) + + # Per-contact averages + %{rows: avg_rows} = + Repo.query!(""" + SELECT + (SELECT count(*) FROM hrrr_profiles) as hrrr_profiles, + (SELECT count(*) FROM terrain_profiles) as terrain_profiles, + (SELECT count(*) FROM surface_observations) as surface_obs, + (SELECT count(*) FROM soundings) as soundings, + (SELECT count(*) FROM iemre_observations) as iemre_obs, + (SELECT count(*) FROM qsos WHERE pos1 IS NOT NULL) as contacts + """) + + [[hrrr_count, terrain_count, obs_count, sounding_count, iemre_count, contact_count]] = avg_rows + + contact_count = max(contact_count, 1) + + %{ + db_size: format_bytes(db_bytes), + tables: tables, + statuses: statuses, + totals: %{ + hrrr_profiles: hrrr_count, + terrain_profiles: terrain_count, + surface_observations: obs_count, + soundings: sounding_count, + iemre_observations: iemre_count, + contacts: contact_count + }, + averages: %{ + hrrr_per_contact: Float.round(hrrr_count / contact_count, 1), + obs_per_contact: Float.round(obs_count / contact_count, 1), + soundings_per_contact: Float.round(sounding_count / contact_count, 1) + } + } + end + + defp format_number(n) when is_integer(n) do + n + |> Integer.to_string() + |> String.graphemes() + |> Enum.reverse() + |> Enum.chunk_every(3) + |> Enum.map_join(",", &Enum.reverse/1) + |> String.reverse() + |> then(fn s -> String.replace(s, ~r/^,/, "") end) + end + + defp format_number(n) when is_float(n), do: format_number(round(n)) + defp format_number(n), do: to_string(n) + + defp status_class("complete"), do: "text-success" + defp status_class("pending"), do: "text-warning" + defp status_class("queued"), do: "text-info" + defp status_class("processing"), do: "text-info" + defp status_class("failed"), do: "text-error" + defp status_class("unavailable"), do: "opacity-40" + defp status_class(_), do: "" + + defp format_bytes(bytes) when bytes >= 1_073_741_824, + do: "#{Float.round(bytes / 1_073_741_824, 1)} GB" + + defp format_bytes(bytes) when bytes >= 1_048_576, + do: "#{Float.round(bytes / 1_048_576, 1)} MB" + + defp format_bytes(bytes) when bytes >= 1024, + do: "#{Float.round(bytes / 1024, 1)} KB" + + defp format_bytes(bytes), do: "#{bytes} B" + @impl true def render(assigns) do ~H""" @@ -168,12 +290,12 @@ defmodule MicrowavepropWeb.BackfillLive do
Enrichment Progress
{@unprocessed.total} remaining
- {@total_contacts - @unprocessed.total} / {@total_contacts} complete + {@db_stats.totals.contacts - @unprocessed.total} / {@db_stats.totals.contacts} complete
Terrain: {@unprocessed.terrain} @@ -265,6 +387,68 @@ defmodule MicrowavepropWeb.BackfillLive do
<% end %> +
+ +

Enrichment Status by Type

+
+ <%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}] do %> +
+
{label}
+ <%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %> +
+ {status} + {format_number(count)} +
+ <% end %> +
+ <% end %> +
+ +

Database

+
+
+
Total DB Size
+
{@db_stats.db_size}
+
+
+
HRRR Profiles
+
{format_number(@db_stats.totals.hrrr_profiles)}
+
{@db_stats.averages.hrrr_per_contact} per contact
+
+
+
Surface Observations
+
{format_number(@db_stats.totals.surface_observations)}
+
{@db_stats.averages.obs_per_contact} per contact
+
+
+
Soundings
+
{format_number(@db_stats.totals.soundings)}
+
{@db_stats.averages.soundings_per_contact} per contact
+
+
+ +

Table Sizes

+
+ + + + + + + + + + <%= for t <- @db_stats.tables do %> + + + + + + <% end %> + +
TableRowsSize
{t.table}{format_number(t.rows)}{t.size}
+
+

Auto-refreshes every 2 seconds.

"""