diff --git a/lib/microwaveprop_web/live/backfill_live.ex b/lib/microwaveprop_web/live/backfill_live.ex index 2683fd7b..154c111f 100644 --- a/lib/microwaveprop_web/live/backfill_live.ex +++ b/lib/microwaveprop_web/live/backfill_live.ex @@ -19,11 +19,11 @@ defmodule MicrowavepropWeb.BackfillLive do stats = fetch_stats() unprocessed = count_unprocessed() - total_contacts = Repo.aggregate(from(c in Contact, where: not is_nil(c.pos1)), :count) + db_stats = fetch_db_stats() case LiveStash.recover_state(socket) do {:recovered, socket} -> - {:ok, assign(socket, stats: stats, unprocessed: unprocessed, total_contacts: total_contacts)} + {:ok, assign(socket, stats: stats, unprocessed: unprocessed, db_stats: db_stats)} _ -> {:ok, @@ -32,7 +32,7 @@ defmodule MicrowavepropWeb.BackfillLive do limit: 500, stats: stats, unprocessed: unprocessed, - total_contacts: total_contacts, + db_stats: db_stats, last_enqueued: nil, enqueuing: false )} @@ -61,7 +61,8 @@ defmodule MicrowavepropWeb.BackfillLive do last_enqueued: count, enqueuing: false, stats: fetch_stats(), - unprocessed: count_unprocessed() + unprocessed: count_unprocessed(), + db_stats: fetch_db_stats() )} end @@ -71,7 +72,8 @@ defmodule MicrowavepropWeb.BackfillLive do {:noreply, assign(socket, stats: fetch_stats(), - unprocessed: count_unprocessed() + unprocessed: count_unprocessed(), + db_stats: fetch_db_stats() )} end @@ -154,6 +156,126 @@ defmodule MicrowavepropWeb.BackfillLive do %{by_worker: by_worker, completed_1h: completed_1h} end + defp fetch_db_stats do + %{rows: rows} = + Repo.query!(""" + SELECT + relname as table, + n_live_tup as rows, + pg_total_relation_size(c.oid) as bytes + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_stat_user_tables s ON s.relid = c.oid + WHERE n.nspname = 'public' + AND c.relkind = 'r' + AND relname NOT LIKE 'oban_%' + AND relname NOT LIKE 'schema_%' + AND n_live_tup > 0 + ORDER BY pg_total_relation_size(c.oid) DESC + LIMIT 20 + """) + + tables = + Enum.map(rows, fn [table, row_count, bytes] -> + %{table: table, rows: row_count, size: format_bytes(bytes)} + end) + + %{rows: total_rows} = + Repo.query!("SELECT pg_database_size(current_database())") + + [[db_bytes]] = total_rows + + # Enrichment status breakdown + %{rows: status_rows} = + Repo.query!(""" + SELECT + 'hrrr' as type, hrrr_status as status, count(*) as cnt FROM qsos GROUP BY hrrr_status + UNION ALL + SELECT + 'weather', weather_status, count(*) FROM qsos GROUP BY weather_status + UNION ALL + SELECT + 'terrain', terrain_status, count(*) FROM qsos GROUP BY terrain_status + UNION ALL + SELECT + 'iemre', iemre_status, count(*) FROM qsos GROUP BY iemre_status + ORDER BY 1, 2 + """) + + statuses = + Enum.group_by(status_rows, fn [type, _, _] -> type end, fn [_, status, count] -> + {status, count} + end) + + # Per-contact averages + %{rows: avg_rows} = + Repo.query!(""" + SELECT + (SELECT count(*) FROM hrrr_profiles) as hrrr_profiles, + (SELECT count(*) FROM terrain_profiles) as terrain_profiles, + (SELECT count(*) FROM surface_observations) as surface_obs, + (SELECT count(*) FROM soundings) as soundings, + (SELECT count(*) FROM iemre_observations) as iemre_obs, + (SELECT count(*) FROM qsos WHERE pos1 IS NOT NULL) as contacts + """) + + [[hrrr_count, terrain_count, obs_count, sounding_count, iemre_count, contact_count]] = avg_rows + + contact_count = max(contact_count, 1) + + %{ + db_size: format_bytes(db_bytes), + tables: tables, + statuses: statuses, + totals: %{ + hrrr_profiles: hrrr_count, + terrain_profiles: terrain_count, + surface_observations: obs_count, + soundings: sounding_count, + iemre_observations: iemre_count, + contacts: contact_count + }, + averages: %{ + hrrr_per_contact: Float.round(hrrr_count / contact_count, 1), + obs_per_contact: Float.round(obs_count / contact_count, 1), + soundings_per_contact: Float.round(sounding_count / contact_count, 1) + } + } + end + + defp format_number(n) when is_integer(n) do + n + |> Integer.to_string() + |> String.graphemes() + |> Enum.reverse() + |> Enum.chunk_every(3) + |> Enum.map_join(",", &Enum.reverse/1) + |> String.reverse() + |> then(fn s -> String.replace(s, ~r/^,/, "") end) + end + + defp format_number(n) when is_float(n), do: format_number(round(n)) + defp format_number(n), do: to_string(n) + + defp status_class("complete"), do: "text-success" + defp status_class("pending"), do: "text-warning" + defp status_class("queued"), do: "text-info" + defp status_class("processing"), do: "text-info" + defp status_class("failed"), do: "text-error" + defp status_class("unavailable"), do: "opacity-40" + defp status_class(_), do: "" + + defp format_bytes(bytes) when bytes >= 1_073_741_824, + do: "#{Float.round(bytes / 1_073_741_824, 1)} GB" + + defp format_bytes(bytes) when bytes >= 1_048_576, + do: "#{Float.round(bytes / 1_048_576, 1)} MB" + + defp format_bytes(bytes) when bytes >= 1024, + do: "#{Float.round(bytes / 1024, 1)} KB" + + defp format_bytes(bytes), do: "#{bytes} B" + @impl true def render(assigns) do ~H""" @@ -168,12 +290,12 @@ defmodule MicrowavepropWeb.BackfillLive do
| Table | +Rows | +Size | +
|---|---|---|
| {t.table} | +{format_number(t.rows)} | +{t.size} | +
Auto-refreshes every 2 seconds.
"""