Graham McIntire
18e920fe54
Backfill page now sums child partition stats for partitioned tables like hrrr_profiles (relkind 'p') which were excluded by the relkind 'r' filter. Contacts map defers Leaflet init to requestAnimationFrame so the flex container has dimensions before map initialization.
538 lines
18 KiB
Elixir
538 lines
18 KiB
Elixir
defmodule MicrowavepropWeb.BackfillLive do
|
|
@moduledoc false
|
|
use MicrowavepropWeb, :live_view
|
|
use LiveStash
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Workers.BackfillEnqueueWorker
|
|
|
|
@impl true
|
|
def mount(_params, _session, socket) do
|
|
if connected?(socket) do
|
|
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "db:contact_status")
|
|
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "db:oban_jobs")
|
|
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "backfill:enqueue_complete")
|
|
end
|
|
|
|
stats = fetch_stats()
|
|
unprocessed = count_unprocessed()
|
|
db_stats = fetch_db_stats()
|
|
|
|
{limit, _} =
|
|
case LiveStash.recover_state(socket) do
|
|
{:recovered, socket} -> {socket.assigns[:limit] || 500, socket}
|
|
_ -> {500, socket}
|
|
end
|
|
|
|
{:ok,
|
|
assign(socket,
|
|
page_title: "Backfill",
|
|
limit: limit,
|
|
types: MapSet.new(~w(hrrr weather terrain iemre era5)),
|
|
stats: stats,
|
|
unprocessed: unprocessed,
|
|
db_stats: db_stats,
|
|
last_enqueued: nil,
|
|
enqueuing: false,
|
|
refresh_timer: nil
|
|
)}
|
|
end
|
|
|
|
@impl true
|
|
def handle_event("enqueue", %{"limit" => limit_str} = params, socket) do
|
|
limit = String.to_integer(limit_str)
|
|
|
|
types =
|
|
~w(hrrr weather terrain iemre era5)
|
|
|> Enum.filter(&(params[&1] == "true"))
|
|
|> then(fn
|
|
[] -> ~w(hrrr weather terrain iemre era5)
|
|
types -> types
|
|
end)
|
|
|
|
%{"limit" => limit, "types" => types}
|
|
|> BackfillEnqueueWorker.new()
|
|
|> Oban.insert()
|
|
|
|
{:noreply,
|
|
socket
|
|
|> assign(enqueuing: true, limit: limit, types: MapSet.new(types))
|
|
|> LiveStash.stash_assigns([:limit])}
|
|
end
|
|
|
|
@impl true
|
|
def handle_info({:enqueue_complete, count}, socket) do
|
|
{:noreply, assign(socket, last_enqueued: count, enqueuing: false)}
|
|
end
|
|
|
|
def handle_info({:contact_status_changed, _data}, socket) do
|
|
{:noreply, schedule_refresh(socket)}
|
|
end
|
|
|
|
def handle_info({:oban_job_changed, _data}, socket) do
|
|
{:noreply, schedule_refresh(socket)}
|
|
end
|
|
|
|
def handle_info(:refresh_stats, socket) do
|
|
{:noreply,
|
|
assign(socket,
|
|
stats: fetch_stats(),
|
|
unprocessed: count_unprocessed(),
|
|
db_stats: fetch_db_stats(),
|
|
refresh_timer: nil
|
|
)}
|
|
end
|
|
|
|
defp schedule_refresh(%{assigns: %{refresh_timer: ref}} = socket) when is_reference(ref) do
|
|
# Already scheduled, skip
|
|
socket
|
|
end
|
|
|
|
defp schedule_refresh(socket) do
|
|
ref = Process.send_after(self(), :refresh_stats, 1000)
|
|
assign(socket, refresh_timer: ref)
|
|
end
|
|
|
|
defp count_unprocessed do
|
|
incomplete = [:pending, :queued, :processing, :failed]
|
|
done = [:complete, :unavailable]
|
|
|
|
total = Repo.one(from(c in Contact, where: not is_nil(c.pos1), select: count()))
|
|
|
|
terrain =
|
|
Repo.one(from(c in Contact, where: c.terrain_status in ^incomplete and not is_nil(c.pos1), select: count()))
|
|
|
|
hrrr =
|
|
Repo.one(from(c in Contact, where: c.hrrr_status in ^incomplete and not is_nil(c.pos1), select: count()))
|
|
|
|
weather =
|
|
Repo.one(from(c in Contact, where: c.weather_status in ^incomplete and not is_nil(c.pos1), select: count()))
|
|
|
|
iemre =
|
|
Repo.one(from(c in Contact, where: c.iemre_status in ^incomplete and not is_nil(c.pos1), select: count()))
|
|
|
|
# ERA5 targets contacts where HRRR is unavailable
|
|
era5 =
|
|
Repo.one(from(c in Contact, where: c.hrrr_status == :unavailable and not is_nil(c.pos1), select: count()))
|
|
|
|
all_done =
|
|
Repo.one(
|
|
from(c in Contact,
|
|
where:
|
|
not is_nil(c.pos1) and
|
|
c.hrrr_status in ^done and
|
|
c.weather_status in ^done and
|
|
c.terrain_status in ^done and
|
|
c.iemre_status in ^done,
|
|
select: count()
|
|
)
|
|
)
|
|
|
|
%{
|
|
terrain: terrain,
|
|
hrrr: hrrr,
|
|
weather: weather,
|
|
iemre: iemre,
|
|
era5: era5,
|
|
remaining: max(terrain, max(hrrr, max(weather, iemre))),
|
|
complete: all_done,
|
|
total: total
|
|
}
|
|
end
|
|
|
|
defp fetch_stats do
|
|
jobs =
|
|
Repo.all(
|
|
from(j in "oban_jobs",
|
|
where: j.state in ["available", "executing", "scheduled", "retryable"],
|
|
group_by: [j.worker, j.state],
|
|
select: {j.worker, j.state, count(j.id)}
|
|
)
|
|
)
|
|
|
|
by_worker =
|
|
jobs
|
|
|> Enum.group_by(&elem(&1, 0), fn {_, state, count} -> {state, count} end)
|
|
|> Enum.map(fn {worker, states} ->
|
|
short_name = worker |> String.split(".") |> List.last()
|
|
total = Enum.reduce(states, 0, fn {_, c}, acc -> acc + c end)
|
|
executing = Enum.find_value(states, 0, fn {s, c} -> if s == "executing", do: c end)
|
|
available = Enum.find_value(states, 0, fn {s, c} -> if s == "available", do: c end)
|
|
retryable = Enum.find_value(states, 0, fn {s, c} -> if s == "retryable", do: c end)
|
|
|
|
%{
|
|
worker: short_name,
|
|
total: total,
|
|
executing: executing,
|
|
available: available,
|
|
retryable: retryable
|
|
}
|
|
end)
|
|
|> Enum.sort_by(& &1.total, :desc)
|
|
|
|
completed_1h =
|
|
Repo.one(
|
|
from(j in "oban_jobs",
|
|
where: j.state == "completed" and j.completed_at > ago(1, "hour"),
|
|
select: count(j.id)
|
|
)
|
|
)
|
|
|
|
%{by_worker: by_worker, completed_1h: completed_1h}
|
|
end
|
|
|
|
defp fetch_db_stats do
|
|
# Regular tables + partitioned tables (summing child partition stats)
|
|
%{rows: rows} =
|
|
Repo.query!("""
|
|
SELECT
|
|
c.relname as table,
|
|
s.n_live_tup as rows,
|
|
s.n_dead_tup as dead,
|
|
pg_total_relation_size(c.oid) as bytes
|
|
FROM pg_class c
|
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
JOIN pg_stat_user_tables s ON s.relid = c.oid
|
|
WHERE n.nspname = 'public'
|
|
AND c.relkind = 'r'
|
|
AND c.relname NOT LIKE 'oban_%'
|
|
AND c.relname NOT LIKE 'schema_%'
|
|
AND NOT EXISTS (SELECT 1 FROM pg_inherits WHERE inhrelid = c.oid)
|
|
UNION ALL
|
|
SELECT
|
|
parent.relname,
|
|
COALESCE(SUM(cs.n_live_tup), 0)::bigint,
|
|
COALESCE(SUM(cs.n_dead_tup), 0)::bigint,
|
|
COALESCE(SUM(pg_total_relation_size(child.oid)), 0)::bigint
|
|
FROM pg_class parent
|
|
JOIN pg_namespace n ON n.oid = parent.relnamespace
|
|
JOIN pg_inherits inh ON inh.inhparent = parent.oid
|
|
JOIN pg_class child ON child.oid = inh.inhrelid
|
|
LEFT JOIN pg_stat_user_tables cs ON cs.relid = child.oid
|
|
WHERE n.nspname = 'public' AND parent.relkind = 'p'
|
|
GROUP BY parent.relname
|
|
ORDER BY bytes DESC
|
|
LIMIT 20
|
|
""")
|
|
|
|
tables =
|
|
Enum.map(rows, fn [table, row_count, dead, bytes] ->
|
|
%{table: table, rows: row_count, dead: dead, size: format_bytes(bytes)}
|
|
end)
|
|
|
|
%{rows: total_rows} =
|
|
Repo.query!("SELECT pg_database_size(current_database())")
|
|
|
|
[[db_bytes]] = total_rows
|
|
|
|
# Enrichment status breakdown
|
|
%{rows: status_rows} =
|
|
Repo.query!("""
|
|
SELECT
|
|
'hrrr' as type, hrrr_status as status, count(*) as cnt FROM contacts GROUP BY hrrr_status
|
|
UNION ALL
|
|
SELECT
|
|
'weather', weather_status, count(*) FROM contacts GROUP BY weather_status
|
|
UNION ALL
|
|
SELECT
|
|
'terrain', terrain_status, count(*) FROM contacts GROUP BY terrain_status
|
|
UNION ALL
|
|
SELECT
|
|
'iemre', iemre_status, count(*) FROM contacts GROUP BY iemre_status
|
|
ORDER BY 1, 2
|
|
""")
|
|
|
|
statuses =
|
|
Enum.group_by(status_rows, fn [type, _, _] -> type end, fn [_, status, count] ->
|
|
{status, count}
|
|
end)
|
|
|
|
# Per-table row counts from pg_stat estimates (already fetched above, avoids slow count(*))
|
|
table_counts = Map.new(tables, fn %{table: t, rows: r} -> {t, r} end)
|
|
|
|
# ERA5 doesn't have its own status column — synthesize from HRRR unavailable + era5_profiles count
|
|
era5_count = Map.get(table_counts, "era5_profiles", 0)
|
|
|
|
hrrr_unavailable =
|
|
statuses |> Map.get("hrrr", []) |> Enum.find_value(0, fn {s, c} -> if s == "unavailable", do: c end)
|
|
|
|
statuses =
|
|
Map.put(statuses, "era5", [
|
|
{"candidates (HRRR unavail)", hrrr_unavailable},
|
|
{"profiles fetched", era5_count}
|
|
])
|
|
|
|
hrrr_count = Map.get(table_counts, "hrrr_profiles", 0)
|
|
terrain_count = Map.get(table_counts, "terrain_profiles", 0)
|
|
obs_count = Map.get(table_counts, "surface_observations", 0)
|
|
sounding_count = Map.get(table_counts, "soundings", 0)
|
|
iemre_count = Map.get(table_counts, "iemre_observations", 0)
|
|
contact_count = max(Map.get(table_counts, "contacts", 0), 1)
|
|
|
|
%{
|
|
db_size: format_bytes(db_bytes),
|
|
tables: tables,
|
|
statuses: statuses,
|
|
totals: %{
|
|
hrrr_profiles: hrrr_count,
|
|
era5_profiles: Map.get(table_counts, "era5_profiles", 0),
|
|
terrain_profiles: terrain_count,
|
|
surface_observations: obs_count,
|
|
soundings: sounding_count,
|
|
iemre_observations: iemre_count,
|
|
contacts: contact_count
|
|
},
|
|
averages: %{
|
|
hrrr_per_contact: Float.round(hrrr_count / contact_count, 1),
|
|
obs_per_contact: Float.round(obs_count / contact_count, 1),
|
|
soundings_per_contact: Float.round(sounding_count / contact_count, 1)
|
|
}
|
|
}
|
|
end
|
|
|
|
defp format_number(n) when is_integer(n) do
|
|
n
|
|
|> Integer.to_string()
|
|
|> String.graphemes()
|
|
|> Enum.reverse()
|
|
|> Enum.chunk_every(3)
|
|
|> Enum.map_join(",", &Enum.join/1)
|
|
|> String.reverse()
|
|
|> String.replace(~r/^,/, "")
|
|
end
|
|
|
|
defp format_number(n) when is_float(n), do: format_number(round(n))
|
|
defp format_number(n), do: to_string(n)
|
|
|
|
defp status_class("complete"), do: "text-success"
|
|
defp status_class("pending"), do: "text-warning"
|
|
defp status_class("queued"), do: "text-info"
|
|
defp status_class("processing"), do: "text-info"
|
|
defp status_class("failed"), do: "text-error"
|
|
defp status_class("unavailable"), do: "opacity-40"
|
|
defp status_class(_), do: ""
|
|
|
|
defp format_bytes(bytes) when bytes >= 1_073_741_824, do: "#{Float.round(bytes / 1_073_741_824, 1)} GB"
|
|
|
|
defp format_bytes(bytes) when bytes >= 1_048_576, do: "#{Float.round(bytes / 1_048_576, 1)} MB"
|
|
|
|
defp format_bytes(bytes) when bytes >= 1024, do: "#{Float.round(bytes / 1024, 1)} KB"
|
|
|
|
defp format_bytes(bytes), do: "#{bytes} B"
|
|
|
|
@impl true
|
|
def render(assigns) do
|
|
~H"""
|
|
<Layouts.app flash={@flash} current_scope={@current_scope}>
|
|
<.header>
|
|
Backfill Dashboard
|
|
<:subtitle>Enqueue and monitor contact enrichment jobs</:subtitle>
|
|
</.header>
|
|
|
|
<div class="grid grid-cols-1 md:grid-cols-4 gap-4 my-6">
|
|
<div class="bg-base-200 rounded-box p-4 md:col-span-2">
|
|
<div class="text-xs opacity-60">Enrichment Progress</div>
|
|
<div class="text-2xl font-bold">
|
|
{format_number(@unprocessed.complete)} / {format_number(@unprocessed.total)} fully enriched
|
|
</div>
|
|
<div class="flex flex-col gap-1.5 mt-3">
|
|
<%= for {label, complete} <- [{"HRRR", @unprocessed.total - @unprocessed.hrrr}, {"Weather", @unprocessed.total - @unprocessed.weather}, {"Terrain", @unprocessed.total - @unprocessed.terrain}, {"IEMRE", @unprocessed.total - @unprocessed.iemre}, {"ERA5", @unprocessed.total - @unprocessed.era5}] do %>
|
|
<% pct =
|
|
if @unprocessed.total > 0, do: round(complete * 100 / @unprocessed.total), else: 0 %>
|
|
<div class="flex items-center gap-2 text-xs">
|
|
<span class="w-14 shrink-0">{label}</span>
|
|
<progress
|
|
class="progress progress-primary flex-1 min-w-0"
|
|
value={pct}
|
|
max={100}
|
|
/>
|
|
<span class="shrink-0 font-mono opacity-60 whitespace-nowrap">
|
|
{format_number(complete)} / {format_number(@unprocessed.total)}
|
|
</span>
|
|
</div>
|
|
<% end %>
|
|
</div>
|
|
</div>
|
|
<div class="bg-base-200 rounded-box p-4 md:col-span-2">
|
|
<div class="flex gap-8">
|
|
<div>
|
|
<div class="text-xs opacity-60">Active/Queued Jobs</div>
|
|
<div class="text-2xl font-bold">
|
|
{Enum.reduce(@stats.by_worker, 0, fn w, acc -> acc + w.total end)}
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<div class="text-xs opacity-60">Completed (last hour)</div>
|
|
<div class="text-2xl font-bold">{format_number(@stats.completed_1h)}</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="bg-base-200 rounded-box p-4 mb-6">
|
|
<form phx-submit="enqueue" class="flex items-end gap-4 flex-wrap">
|
|
<div>
|
|
<label class="label text-sm">Contacts to enqueue</label>
|
|
<input
|
|
type="number"
|
|
name="limit"
|
|
value={@limit}
|
|
min="1"
|
|
max="5000"
|
|
class="input input-bordered w-32"
|
|
/>
|
|
</div>
|
|
<div class="flex gap-3 items-center">
|
|
<%= for type <- ~w(hrrr weather terrain iemre era5) do %>
|
|
<label class="label cursor-pointer gap-1">
|
|
<input
|
|
type="checkbox"
|
|
name={type}
|
|
value="true"
|
|
checked={MapSet.member?(@types, type)}
|
|
class="checkbox checkbox-sm"
|
|
/>
|
|
<span class="text-xs uppercase">{type}</span>
|
|
</label>
|
|
<% end %>
|
|
</div>
|
|
<button class="btn btn-primary" disabled={@enqueuing}>
|
|
<%= if @enqueuing do %>
|
|
<span class="loading loading-spinner loading-sm"></span> Enqueuing...
|
|
<% else %>
|
|
Enqueue Backfill
|
|
<% end %>
|
|
</button>
|
|
<%= if @last_enqueued do %>
|
|
<span class="text-sm text-success">
|
|
Enqueued {@last_enqueued} contacts
|
|
</span>
|
|
<% end %>
|
|
</form>
|
|
</div>
|
|
|
|
<h2 class="text-base font-semibold mb-2">Job Queue Status</h2>
|
|
<%= if @stats.by_worker == [] do %>
|
|
<p class="text-sm text-base-content/50 italic">No active jobs.</p>
|
|
<% else %>
|
|
<div class="overflow-x-auto">
|
|
<table class="table table-sm table-zebra">
|
|
<thead>
|
|
<tr>
|
|
<th>Worker</th>
|
|
<th>Executing</th>
|
|
<th>Pending</th>
|
|
<th>Retryable</th>
|
|
<th>Total</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<%= for w <- @stats.by_worker do %>
|
|
<tr>
|
|
<td class="font-semibold">{w.worker}</td>
|
|
<td>
|
|
<%= if w.executing > 0 do %>
|
|
<span class="flex items-center gap-1">
|
|
<span class="loading loading-spinner loading-xs"></span> {w.executing}
|
|
</span>
|
|
<% else %>
|
|
0
|
|
<% end %>
|
|
</td>
|
|
<td>{w.available}</td>
|
|
<td>
|
|
<%= if w.retryable > 0 do %>
|
|
<span class="text-warning">{w.retryable}</span>
|
|
<% else %>
|
|
0
|
|
<% end %>
|
|
</td>
|
|
<td>{w.total}</td>
|
|
</tr>
|
|
<% end %>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
<% end %>
|
|
|
|
<div class="divider" />
|
|
|
|
<h2 class="text-base font-semibold mb-2">Enrichment Status by Type</h2>
|
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-6">
|
|
<%= for {type, label} <- [{"hrrr", "HRRR"}, {"weather", "Weather"}, {"terrain", "Terrain"}, {"iemre", "IEMRE"}, {"era5", "ERA5"}] do %>
|
|
<div class="bg-base-200 rounded-box p-3 text-xs">
|
|
<div class="font-semibold mb-1">{label}</div>
|
|
<%= for {status, count} <- Map.get(@db_stats.statuses, type, []) do %>
|
|
<div class="flex justify-between">
|
|
<span class={status_class(status)}>{status}</span>
|
|
<span class="font-mono">{format_number(count)}</span>
|
|
</div>
|
|
<% end %>
|
|
</div>
|
|
<% end %>
|
|
</div>
|
|
|
|
<h2 class="text-base font-semibold mb-2">Database</h2>
|
|
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mb-4">
|
|
<div class="bg-base-200 rounded-box p-3">
|
|
<div class="text-xs opacity-60">Total DB Size</div>
|
|
<div class="text-lg font-bold font-mono">{@db_stats.db_size}</div>
|
|
</div>
|
|
<div class="bg-base-200 rounded-box p-3">
|
|
<div class="text-xs opacity-60">HRRR Profiles</div>
|
|
<div class="text-lg font-bold font-mono">
|
|
{format_number(@db_stats.totals.hrrr_profiles)}
|
|
</div>
|
|
<div class="text-xs opacity-60">{@db_stats.averages.hrrr_per_contact} per contact</div>
|
|
</div>
|
|
<div class="bg-base-200 rounded-box p-3">
|
|
<div class="text-xs opacity-60">Surface Observations</div>
|
|
<div class="text-lg font-bold font-mono">
|
|
{format_number(@db_stats.totals.surface_observations)}
|
|
</div>
|
|
<div class="text-xs opacity-60">{@db_stats.averages.obs_per_contact} per contact</div>
|
|
</div>
|
|
<div class="bg-base-200 rounded-box p-3">
|
|
<div class="text-xs opacity-60">Soundings</div>
|
|
<div class="text-lg font-bold font-mono">{format_number(@db_stats.totals.soundings)}</div>
|
|
<div class="text-xs opacity-60">{@db_stats.averages.soundings_per_contact} per contact</div>
|
|
</div>
|
|
</div>
|
|
|
|
<h2 class="text-base font-semibold mb-2">Table Sizes</h2>
|
|
<div class="overflow-x-auto mb-4">
|
|
<table class="table table-xs table-zebra">
|
|
<thead>
|
|
<tr>
|
|
<th>Table</th>
|
|
<th class="text-right">Rows</th>
|
|
<th class="text-right">Dead</th>
|
|
<th class="text-right">Size</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
<%= for t <- @db_stats.tables do %>
|
|
<tr>
|
|
<td class="font-mono text-xs">{t.table}</td>
|
|
<td class="text-right font-mono text-xs">{format_number(t.rows)}</td>
|
|
<td class={[
|
|
"text-right font-mono text-xs",
|
|
t.dead > 0 && "text-warning"
|
|
]}>
|
|
{format_number(t.dead)}
|
|
</td>
|
|
<td class="text-right font-mono text-xs">{t.size}</td>
|
|
</tr>
|
|
<% end %>
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
|
|
<p class="text-xs text-base-content/50 mt-4">Live updates via database notifications.</p>
|
|
</Layouts.app>
|
|
"""
|
|
end
|
|
end
|