Add /backfill dashboard for enqueuing and monitoring enrichment jobs
- Input field to enqueue N contacts for backfill - Live stats: unprocessed contacts, active/queued jobs, completed/hour - Per-worker breakdown: executing (with spinner), available, retryable - Auto-refreshes every 2 seconds
This commit is contained in:
parent
9e975aa64b
commit
c95da8ccdb
2 changed files with 240 additions and 0 deletions
239
lib/microwaveprop_web/live/backfill_live.ex
Normal file
239
lib/microwaveprop_web/live/backfill_live.ex
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
defmodule MicrowavepropWeb.BackfillLive do
|
||||
@moduledoc false
|
||||
use MicrowavepropWeb, :live_view
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
||||
|
||||
@refresh_interval 2_000
|
||||
|
||||
@impl true
|
||||
def mount(_params, _session, socket) do
|
||||
if connected?(socket) do
|
||||
Process.send_after(self(), :refresh_stats, @refresh_interval)
|
||||
end
|
||||
|
||||
stats = fetch_stats()
|
||||
unprocessed = count_unprocessed()
|
||||
|
||||
{:ok,
|
||||
assign(socket,
|
||||
page_title: "Backfill",
|
||||
limit: 500,
|
||||
stats: stats,
|
||||
unprocessed: unprocessed,
|
||||
last_enqueued: nil,
|
||||
enqueuing: false
|
||||
)}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_event("enqueue", %{"limit" => limit_str}, socket) do
|
||||
limit = String.to_integer(limit_str)
|
||||
|
||||
# Run in a task to not block the LiveView
|
||||
pid = self()
|
||||
|
||||
Task.start(fn ->
|
||||
count = enqueue_batch(limit)
|
||||
send(pid, {:enqueued, count})
|
||||
end)
|
||||
|
||||
{:noreply, assign(socket, enqueuing: true)}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:enqueued, count}, socket) do
|
||||
{:noreply,
|
||||
assign(socket,
|
||||
last_enqueued: count,
|
||||
enqueuing: false,
|
||||
stats: fetch_stats(),
|
||||
unprocessed: count_unprocessed()
|
||||
)}
|
||||
end
|
||||
|
||||
def handle_info(:refresh_stats, socket) do
|
||||
Process.send_after(self(), :refresh_stats, @refresh_interval)
|
||||
|
||||
{:noreply,
|
||||
assign(socket,
|
||||
stats: fetch_stats(),
|
||||
unprocessed: count_unprocessed()
|
||||
)}
|
||||
end
|
||||
|
||||
defp enqueue_batch(limit) do
|
||||
contacts =
|
||||
from(c in Contact,
|
||||
where:
|
||||
not is_nil(c.pos1) and
|
||||
(c.hrrr_queued == false or c.weather_queued == false or
|
||||
c.terrain_queued == false or c.iemre_queued == false),
|
||||
order_by: [desc: c.qso_timestamp],
|
||||
limit: ^limit
|
||||
)
|
||||
|> Repo.all()
|
||||
|
||||
Enum.each(contacts, &ContactWeatherEnqueueWorker.enqueue_for_contact/1)
|
||||
length(contacts)
|
||||
end
|
||||
|
||||
defp count_unprocessed do
|
||||
Repo.one(
|
||||
from(c in Contact,
|
||||
where:
|
||||
not is_nil(c.pos1) and
|
||||
(c.hrrr_queued == false or c.weather_queued == false or
|
||||
c.terrain_queued == false or c.iemre_queued == false),
|
||||
select: count(c.id)
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
defp fetch_stats do
|
||||
jobs =
|
||||
Repo.all(
|
||||
from(j in "oban_jobs",
|
||||
where: j.state in ["available", "executing", "scheduled", "retryable"],
|
||||
group_by: [j.worker, j.state],
|
||||
select: {j.worker, j.state, count(j.id)}
|
||||
)
|
||||
)
|
||||
|
||||
by_worker =
|
||||
Enum.group_by(jobs, &elem(&1, 0), fn {_, state, count} -> {state, count} end)
|
||||
|> Enum.map(fn {worker, states} ->
|
||||
short_name = worker |> String.split(".") |> List.last()
|
||||
total = Enum.reduce(states, 0, fn {_, c}, acc -> acc + c end)
|
||||
executing = Enum.find_value(states, 0, fn {s, c} -> if s == "executing", do: c end)
|
||||
available = Enum.find_value(states, 0, fn {s, c} -> if s == "available", do: c end)
|
||||
retryable = Enum.find_value(states, 0, fn {s, c} -> if s == "retryable", do: c end)
|
||||
|
||||
%{
|
||||
worker: short_name,
|
||||
total: total,
|
||||
executing: executing,
|
||||
available: available,
|
||||
retryable: retryable
|
||||
}
|
||||
end)
|
||||
|> Enum.sort_by(& &1.total, :desc)
|
||||
|
||||
completed_1h =
|
||||
Repo.one(
|
||||
from(j in "oban_jobs",
|
||||
where: j.state == "completed" and j.completed_at > ago(1, "hour"),
|
||||
select: count(j.id)
|
||||
)
|
||||
)
|
||||
|
||||
%{by_worker: by_worker, completed_1h: completed_1h}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def render(assigns) do
|
||||
~H"""
|
||||
<Layouts.app flash={@flash}>
|
||||
<.header>
|
||||
Backfill Dashboard
|
||||
<:subtitle>Enqueue and monitor contact enrichment jobs</:subtitle>
|
||||
</.header>
|
||||
|
||||
<div class="grid grid-cols-1 md:grid-cols-3 gap-4 my-6">
|
||||
<div class="stat bg-base-200 rounded-box p-4">
|
||||
<div class="text-xs opacity-60">Contacts Needing Enrichment</div>
|
||||
<div class="text-2xl font-bold">{@unprocessed}</div>
|
||||
</div>
|
||||
<div class="stat bg-base-200 rounded-box p-4">
|
||||
<div class="text-xs opacity-60">Active/Queued Jobs</div>
|
||||
<div class="text-2xl font-bold">
|
||||
{Enum.reduce(@stats.by_worker, 0, fn w, acc -> acc + w.total end)}
|
||||
</div>
|
||||
</div>
|
||||
<div class="stat bg-base-200 rounded-box p-4">
|
||||
<div class="text-xs opacity-60">Completed (last hour)</div>
|
||||
<div class="text-2xl font-bold">{@stats.completed_1h}</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="bg-base-200 rounded-box p-4 mb-6">
|
||||
<form phx-submit="enqueue" class="flex items-end gap-4">
|
||||
<div>
|
||||
<label class="label text-sm">Contacts to enqueue</label>
|
||||
<input
|
||||
type="number"
|
||||
name="limit"
|
||||
value={@limit}
|
||||
min="1"
|
||||
max="5000"
|
||||
class="input input-bordered w-32"
|
||||
/>
|
||||
</div>
|
||||
<button class="btn btn-primary" disabled={@enqueuing}>
|
||||
<%= if @enqueuing do %>
|
||||
<span class="loading loading-spinner loading-sm"></span> Enqueuing...
|
||||
<% else %>
|
||||
Enqueue Backfill
|
||||
<% end %>
|
||||
</button>
|
||||
<%= if @last_enqueued do %>
|
||||
<span class="text-sm text-success">
|
||||
Enqueued {@last_enqueued} contacts
|
||||
</span>
|
||||
<% end %>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<h2 class="text-base font-semibold mb-2">Job Queue Status</h2>
|
||||
<%= if @stats.by_worker == [] do %>
|
||||
<p class="text-sm text-base-content/50 italic">No active jobs.</p>
|
||||
<% else %>
|
||||
<div class="overflow-x-auto">
|
||||
<table class="table table-sm table-zebra">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Worker</th>
|
||||
<th>Executing</th>
|
||||
<th>Available</th>
|
||||
<th>Retryable</th>
|
||||
<th>Total</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<%= for w <- @stats.by_worker do %>
|
||||
<tr>
|
||||
<td class="font-semibold">{w.worker}</td>
|
||||
<td>
|
||||
<%= if w.executing > 0 do %>
|
||||
<span class="flex items-center gap-1">
|
||||
<span class="loading loading-spinner loading-xs"></span> {w.executing}
|
||||
</span>
|
||||
<% else %>
|
||||
0
|
||||
<% end %>
|
||||
</td>
|
||||
<td>{w.available}</td>
|
||||
<td>
|
||||
<%= if w.retryable > 0 do %>
|
||||
<span class="text-warning">{w.retryable}</span>
|
||||
<% else %>
|
||||
0
|
||||
<% end %>
|
||||
</td>
|
||||
<td>{w.total}</td>
|
||||
</tr>
|
||||
<% end %>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<% end %>
|
||||
|
||||
<p class="text-xs text-base-content/50 mt-4">Auto-refreshes every 2 seconds.</p>
|
||||
</Layouts.app>
|
||||
"""
|
||||
end
|
||||
end
|
||||
|
|
@ -25,6 +25,7 @@ defmodule MicrowavepropWeb.Router do
|
|||
live "/contacts/map", ContactMapLive
|
||||
live "/contacts/:id", ContactLive.Show
|
||||
live "/algo", AlgoLive
|
||||
live "/backfill", BackfillLive
|
||||
|
||||
# Redirect old /qsos routes
|
||||
get "/qsos", PageController, :redirect_contacts
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue