Show aggregated propagation pipeline status on the map
Adds a small chip below the NTMS title in both the mobile and
desktop sidebars of /map that collapses PropagationGridWorker +
AsosAdjustmentWorker Oban state into one of four states —
running / idle / stale / unknown — with a plain-language label
("Updating propagation (HRRR run)…", "Up to date · 12m ago").
The chip refreshes on a 15s timer and whenever the pipeline
broadcasts propagation:updated, so a long HRRR sweep is visible
from the moment a visitor opens the page.
This commit is contained in:
parent
03e7448935
commit
554bc9f339
4 changed files with 320 additions and 1 deletions
113
lib/microwaveprop/propagation/pipeline_status.ex
Normal file
113
lib/microwaveprop/propagation/pipeline_status.ex
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
defmodule Microwaveprop.Propagation.PipelineStatus do
|
||||||
|
@moduledoc """
|
||||||
|
Aggregated status for the propagation update pipeline.
|
||||||
|
|
||||||
|
The pipeline is driven by two Oban workers:
|
||||||
|
|
||||||
|
* `PropagationGridWorker` — hourly, fetches HRRR f00–f18, scores
|
||||||
|
the CONUS grid, and broadcasts `propagation:updated`.
|
||||||
|
* `AsosAdjustmentWorker` — every 10 minutes, nudges scores with
|
||||||
|
recent ASOS surface observations.
|
||||||
|
|
||||||
|
`current/0` returns one struct so the map page can render a single
|
||||||
|
status chip ("Updating…", "Up to date · 8m ago", "Stale · 4h ago")
|
||||||
|
that reflects the whole pipeline rather than tailing a single job.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import Ecto.Query
|
||||||
|
|
||||||
|
alias Microwaveprop.Repo
|
||||||
|
|
||||||
|
@grid_worker "Microwaveprop.Workers.PropagationGridWorker"
|
||||||
|
@asos_worker "Microwaveprop.Workers.AsosAdjustmentWorker"
|
||||||
|
|
||||||
|
@workers [@grid_worker, @asos_worker]
|
||||||
|
|
||||||
|
# Scores older than this flip the chip to :stale. Matches the
|
||||||
|
# FreshnessMonitor threshold — anything past 2h is already a gap
|
||||||
|
# the pipeline should have caught.
|
||||||
|
@stale_after_minutes 120
|
||||||
|
|
||||||
|
@type state :: :running | :idle | :stale | :unknown
|
||||||
|
|
||||||
|
@type t :: %{
|
||||||
|
state: state(),
|
||||||
|
label: String.t(),
|
||||||
|
last_update_at: DateTime.t() | nil
|
||||||
|
}
|
||||||
|
|
||||||
|
@spec current() :: t()
|
||||||
|
def current do
|
||||||
|
case running_worker() do
|
||||||
|
worker when is_binary(worker) ->
|
||||||
|
%{
|
||||||
|
state: :running,
|
||||||
|
label: running_label(worker),
|
||||||
|
last_update_at: latest_completed_at()
|
||||||
|
}
|
||||||
|
|
||||||
|
nil ->
|
||||||
|
build_idle_or_stale(latest_completed_at())
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp running_worker do
|
||||||
|
Repo.one(
|
||||||
|
from j in "oban_jobs",
|
||||||
|
where: j.state == "executing" and j.worker in ^@workers,
|
||||||
|
order_by: [desc: j.attempted_at],
|
||||||
|
limit: 1,
|
||||||
|
select: j.worker
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
|
defp latest_completed_at do
|
||||||
|
case Repo.one(
|
||||||
|
from j in "oban_jobs",
|
||||||
|
where: j.state == "completed" and j.worker in ^@workers,
|
||||||
|
order_by: [desc: j.completed_at],
|
||||||
|
limit: 1,
|
||||||
|
select: j.completed_at
|
||||||
|
) do
|
||||||
|
nil -> nil
|
||||||
|
%NaiveDateTime{} = naive -> DateTime.from_naive!(naive, "Etc/UTC")
|
||||||
|
%DateTime{} = dt -> dt
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp build_idle_or_stale(nil) do
|
||||||
|
%{state: :unknown, label: "Propagation status unknown", last_update_at: nil}
|
||||||
|
end
|
||||||
|
|
||||||
|
defp build_idle_or_stale(%DateTime{} = last) do
|
||||||
|
age_minutes = max(DateTime.diff(DateTime.utc_now(), last, :minute), 0)
|
||||||
|
|
||||||
|
if age_minutes > @stale_after_minutes do
|
||||||
|
%{
|
||||||
|
state: :stale,
|
||||||
|
label: "Propagation data stale · last update #{format_age(age_minutes)} ago",
|
||||||
|
last_update_at: last
|
||||||
|
}
|
||||||
|
else
|
||||||
|
%{
|
||||||
|
state: :idle,
|
||||||
|
label: "Up to date · #{format_age(age_minutes)} ago",
|
||||||
|
last_update_at: last
|
||||||
|
}
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
defp running_label(@grid_worker), do: "Updating propagation (HRRR run)…"
|
||||||
|
defp running_label(@asos_worker), do: "Updating propagation (ASOS nudge)…"
|
||||||
|
defp running_label(_), do: "Updating propagation…"
|
||||||
|
|
||||||
|
defp format_age(0), do: "just now"
|
||||||
|
defp format_age(1), do: "1m"
|
||||||
|
defp format_age(n) when n < 60, do: "#{n}m"
|
||||||
|
|
||||||
|
defp format_age(n) do
|
||||||
|
hours = div(n, 60)
|
||||||
|
mins = rem(n, 60)
|
||||||
|
if mins == 0, do: "#{hours}h", else: "#{hours}h #{mins}m"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
@ -5,6 +5,7 @@ defmodule MicrowavepropWeb.MapLive do
|
||||||
|
|
||||||
alias Microwaveprop.Propagation
|
alias Microwaveprop.Propagation
|
||||||
alias Microwaveprop.Propagation.BandConfig
|
alias Microwaveprop.Propagation.BandConfig
|
||||||
|
alias Microwaveprop.Propagation.PipelineStatus
|
||||||
alias Microwaveprop.Terrain.Viewshed
|
alias Microwaveprop.Terrain.Viewshed
|
||||||
|
|
||||||
require Logger
|
require Logger
|
||||||
|
|
@ -14,13 +15,22 @@ defmodule MicrowavepropWeb.MapLive do
|
||||||
@default_center %{lat: 32.897, lon: -97.038}
|
@default_center %{lat: 32.897, lon: -97.038}
|
||||||
@default_zoom 7
|
@default_zoom 7
|
||||||
|
|
||||||
|
# How often to re-query oban_jobs for pipeline state while the map is
|
||||||
|
# open. Short enough that the "Updating…" chip shows up within a page-
|
||||||
|
# visit for an in-progress hourly run, long enough to be cheap.
|
||||||
|
@pipeline_status_refresh_ms 15_000
|
||||||
|
|
||||||
@impl true
|
@impl true
|
||||||
def mount(_params, session, socket) do
|
def mount(_params, session, socket) do
|
||||||
if connected?(socket) do
|
if connected?(socket) do
|
||||||
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "propagation:updated")
|
Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "propagation:updated")
|
||||||
|
Process.send_after(self(), :refresh_pipeline_status, @pipeline_status_refresh_ms)
|
||||||
end
|
end
|
||||||
|
|
||||||
socket = assign(socket, :initial_utc_clock, Calendar.strftime(DateTime.utc_now(), "%H:%M UTC"))
|
socket =
|
||||||
|
socket
|
||||||
|
|> assign(:initial_utc_clock, Calendar.strftime(DateTime.utc_now(), "%H:%M UTC"))
|
||||||
|
|> assign(:pipeline_status, PipelineStatus.current())
|
||||||
|
|
||||||
# LiveStash only persists the keys passed to `stash_assigns/2`
|
# LiveStash only persists the keys passed to `stash_assigns/2`
|
||||||
# (selected_band + selected_time). On reconnect the recovered socket has
|
# (selected_band + selected_time). On reconnect the recovered socket has
|
||||||
|
|
@ -264,12 +274,18 @@ defmodule MicrowavepropWeb.MapLive do
|
||||||
socket =
|
socket =
|
||||||
socket
|
socket
|
||||||
|> assign(valid_times: valid_times, selected_time: selected_time)
|
|> assign(valid_times: valid_times, selected_time: selected_time)
|
||||||
|
|> assign(:pipeline_status, PipelineStatus.current())
|
||||||
|> push_event("update_scores", %{scores: scores})
|
|> push_event("update_scores", %{scores: scores})
|
||||||
|> push_timeline()
|
|> push_timeline()
|
||||||
|
|
||||||
{:noreply, socket}
|
{:noreply, socket}
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def handle_info(:refresh_pipeline_status, socket) do
|
||||||
|
Process.send_after(self(), :refresh_pipeline_status, @pipeline_status_refresh_ms)
|
||||||
|
{:noreply, assign(socket, :pipeline_status, PipelineStatus.current())}
|
||||||
|
end
|
||||||
|
|
||||||
@impl true
|
@impl true
|
||||||
def handle_async(:viewshed, {:ok, result}, socket) do
|
def handle_async(:viewshed, {:ok, result}, socket) do
|
||||||
points = Enum.map(result.boundary, fn p -> %{lat: p.lat, lon: p.lon} end)
|
points = Enum.map(result.boundary, fn p -> %{lat: p.lat, lon: p.lon} end)
|
||||||
|
|
@ -344,6 +360,38 @@ defmodule MicrowavepropWeb.MapLive do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
attr :id, :string, required: true
|
||||||
|
attr :status, :map, required: true
|
||||||
|
|
||||||
|
defp pipeline_status_chip(assigns) do
|
||||||
|
~H"""
|
||||||
|
<div
|
||||||
|
id={@id}
|
||||||
|
data-pipeline-state={@status.state}
|
||||||
|
class="flex items-center gap-1.5 text-xs px-1 py-1"
|
||||||
|
title={@status.label}
|
||||||
|
>
|
||||||
|
<%= case @status.state do %>
|
||||||
|
<% :running -> %>
|
||||||
|
<.icon
|
||||||
|
name="hero-arrow-path"
|
||||||
|
class="size-3.5 shrink-0 text-info motion-safe:animate-spin"
|
||||||
|
/>
|
||||||
|
<span class="opacity-90 truncate">{@status.label}</span>
|
||||||
|
<% :idle -> %>
|
||||||
|
<span class="size-2 rounded-full bg-success inline-block shrink-0"></span>
|
||||||
|
<span class="opacity-70 truncate">{@status.label}</span>
|
||||||
|
<% :stale -> %>
|
||||||
|
<span class="size-2 rounded-full bg-warning inline-block shrink-0"></span>
|
||||||
|
<span class="opacity-80 truncate">{@status.label}</span>
|
||||||
|
<% :unknown -> %>
|
||||||
|
<span class="size-2 rounded-full bg-base-content/30 inline-block shrink-0"></span>
|
||||||
|
<span class="opacity-60 truncate">{@status.label}</span>
|
||||||
|
<% end %>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
end
|
||||||
|
|
||||||
@impl true
|
@impl true
|
||||||
def render(assigns) do
|
def render(assigns) do
|
||||||
~H"""
|
~H"""
|
||||||
|
|
@ -405,6 +453,8 @@ defmodule MicrowavepropWeb.MapLive do
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<.pipeline_status_chip id="pipeline-status-mobile" status={@pipeline_status} />
|
||||||
|
|
||||||
<div class="dropdown">
|
<div class="dropdown">
|
||||||
<div tabindex="0" role="button" class="btn btn-sm w-full justify-between">
|
<div tabindex="0" role="button" class="btn btn-sm w-full justify-between">
|
||||||
{selected_label(@bands, @selected_band)}
|
{selected_label(@bands, @selected_band)}
|
||||||
|
|
@ -558,6 +608,8 @@ defmodule MicrowavepropWeb.MapLive do
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<.pipeline_status_chip id="pipeline-status-desktop" status={@pipeline_status} />
|
||||||
|
|
||||||
<%!-- Band selector --%>
|
<%!-- Band selector --%>
|
||||||
<div class="dropdown">
|
<div class="dropdown">
|
||||||
<div tabindex="0" role="button" class="btn btn-sm w-full justify-between">
|
<div tabindex="0" role="button" class="btn btn-sm w-full justify-between">
|
||||||
|
|
|
||||||
143
test/microwaveprop/propagation/pipeline_status_test.exs
Normal file
143
test/microwaveprop/propagation/pipeline_status_test.exs
Normal file
|
|
@ -0,0 +1,143 @@
|
||||||
|
defmodule Microwaveprop.Propagation.PipelineStatusTest do
|
||||||
|
use Microwaveprop.DataCase, async: false
|
||||||
|
|
||||||
|
alias Microwaveprop.Propagation.PipelineStatus
|
||||||
|
alias Microwaveprop.Repo
|
||||||
|
|
||||||
|
@grid_worker "Microwaveprop.Workers.PropagationGridWorker"
|
||||||
|
@asos_worker "Microwaveprop.Workers.AsosAdjustmentWorker"
|
||||||
|
|
||||||
|
# Bypass Oban's normal insert pipeline (which executes inline in test)
|
||||||
|
# by writing directly to the Oban.Job schema. The SQL sandbox keeps
|
||||||
|
# Oban's executor from seeing these rows.
|
||||||
|
defp insert_oban_job(overrides) do
|
||||||
|
now = DateTime.utc_now()
|
||||||
|
|
||||||
|
defaults = %{
|
||||||
|
state: "available",
|
||||||
|
queue: "propagation",
|
||||||
|
worker: @grid_worker,
|
||||||
|
args: %{},
|
||||||
|
attempt: 0,
|
||||||
|
max_attempts: 3,
|
||||||
|
inserted_at: now,
|
||||||
|
scheduled_at: now
|
||||||
|
}
|
||||||
|
|
||||||
|
attrs = Map.merge(defaults, Map.new(overrides))
|
||||||
|
Repo.insert!(struct!(Oban.Job, attrs))
|
||||||
|
end
|
||||||
|
|
||||||
|
defp minutes_ago(n) do
|
||||||
|
DateTime.add(DateTime.utc_now(), -n * 60, :second)
|
||||||
|
end
|
||||||
|
|
||||||
|
describe "current/0" do
|
||||||
|
test "returns :unknown when no pipeline jobs have ever run" do
|
||||||
|
status = PipelineStatus.current()
|
||||||
|
|
||||||
|
assert status.state == :unknown
|
||||||
|
assert status.last_update_at == nil
|
||||||
|
assert is_binary(status.label)
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns :running with grid worker label when PropagationGridWorker is executing" do
|
||||||
|
insert_oban_job(%{
|
||||||
|
state: "executing",
|
||||||
|
worker: @grid_worker,
|
||||||
|
attempted_at: minutes_ago(5)
|
||||||
|
})
|
||||||
|
|
||||||
|
status = PipelineStatus.current()
|
||||||
|
|
||||||
|
assert status.state == :running
|
||||||
|
assert status.label =~ "HRRR"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns :running with ASOS label when AsosAdjustmentWorker is executing" do
|
||||||
|
insert_oban_job(%{
|
||||||
|
state: "executing",
|
||||||
|
worker: @asos_worker,
|
||||||
|
attempted_at: minutes_ago(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
status = PipelineStatus.current()
|
||||||
|
|
||||||
|
assert status.state == :running
|
||||||
|
assert status.label =~ "ASOS"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns :idle with Up to date label when last grid run completed recently" do
|
||||||
|
completed = minutes_ago(15)
|
||||||
|
|
||||||
|
insert_oban_job(%{
|
||||||
|
state: "completed",
|
||||||
|
worker: @grid_worker,
|
||||||
|
attempted_at: completed,
|
||||||
|
completed_at: completed
|
||||||
|
})
|
||||||
|
|
||||||
|
status = PipelineStatus.current()
|
||||||
|
|
||||||
|
assert status.state == :idle
|
||||||
|
assert status.last_update_at
|
||||||
|
assert status.label =~ "Up to date"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "returns :stale when last completed was more than 120 minutes ago" do
|
||||||
|
stale = minutes_ago(180)
|
||||||
|
|
||||||
|
insert_oban_job(%{
|
||||||
|
state: "completed",
|
||||||
|
worker: @grid_worker,
|
||||||
|
attempted_at: stale,
|
||||||
|
completed_at: stale
|
||||||
|
})
|
||||||
|
|
||||||
|
status = PipelineStatus.current()
|
||||||
|
|
||||||
|
assert status.state == :stale
|
||||||
|
assert status.label =~ "stale"
|
||||||
|
end
|
||||||
|
|
||||||
|
test "prefers :running over :idle when a job is executing even if a recent job completed" do
|
||||||
|
recent = minutes_ago(5)
|
||||||
|
|
||||||
|
insert_oban_job(%{
|
||||||
|
state: "completed",
|
||||||
|
worker: @grid_worker,
|
||||||
|
attempted_at: recent,
|
||||||
|
completed_at: recent
|
||||||
|
})
|
||||||
|
|
||||||
|
insert_oban_job(%{
|
||||||
|
state: "executing",
|
||||||
|
worker: @grid_worker,
|
||||||
|
attempted_at: minutes_ago(1)
|
||||||
|
})
|
||||||
|
|
||||||
|
status = PipelineStatus.current()
|
||||||
|
|
||||||
|
assert status.state == :running
|
||||||
|
end
|
||||||
|
|
||||||
|
test "ignores completed jobs from unrelated workers" do
|
||||||
|
# WeatherFetchWorker enriches individual QSOs — it's not part of
|
||||||
|
# the CONUS grid update pipeline, so its recent success should
|
||||||
|
# not make the map page show 'Up to date'.
|
||||||
|
completed = minutes_ago(5)
|
||||||
|
|
||||||
|
insert_oban_job(%{
|
||||||
|
state: "completed",
|
||||||
|
worker: "Microwaveprop.Workers.WeatherFetchWorker",
|
||||||
|
queue: "weather",
|
||||||
|
attempted_at: completed,
|
||||||
|
completed_at: completed
|
||||||
|
})
|
||||||
|
|
||||||
|
status = PipelineStatus.current()
|
||||||
|
|
||||||
|
assert status.state == :unknown
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
@ -97,4 +97,15 @@ defmodule MicrowavepropWeb.MapLiveTest do
|
||||||
refute html =~ "checked"
|
refute html =~ "checked"
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
describe "pipeline status chip" do
|
||||||
|
test "renders the aggregated pipeline status chip on mount", %{conn: conn} do
|
||||||
|
{:ok, _lv, html} = live(conn, ~p"/map")
|
||||||
|
# Both the mobile and desktop sidebars carry their own chip.
|
||||||
|
assert html =~ ~s(id="pipeline-status-mobile")
|
||||||
|
assert html =~ ~s(id="pipeline-status-desktop")
|
||||||
|
# With a clean test DB no pipeline workers have ever run.
|
||||||
|
assert html =~ ~s(data-pipeline-state="unknown")
|
||||||
|
end
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue