The :propagation Oban queue runs with concurrency 2, so the grid
worker chain step and the 10-minute ASOS nudge can (and do) execute
in parallel. PipelineStatus.current/0 used to return only the
most-recently-started worker, which made the chip label flip to
"ASOS nudge" at every :00 / :10 tick even though HRRR was still
chugging through forecast hours behind it.
Replace the single :detail field with a list of running_detail
maps, each tagged with :grid or :asos so the chip can:
- Render one sub-line per currently-executing worker
- Sort them deterministically (grid before asos) so the eye
doesn't see rows swap
- Still override the grid entry with the forecast-hour progress
label when a propagation_pipeline_progress broadcast arrives
141 lines
4.2 KiB
Elixir
141 lines
4.2 KiB
Elixir
defmodule Microwaveprop.Propagation.PipelineStatus do
|
||
@moduledoc """
|
||
Aggregated status for the propagation update pipeline.
|
||
|
||
The pipeline is driven by two Oban workers:
|
||
|
||
* `PropagationGridWorker` — hourly, fetches HRRR f00–f18, scores
|
||
the CONUS grid, and broadcasts `propagation:updated`.
|
||
* `AsosAdjustmentWorker` — every 10 minutes, nudges scores with
|
||
recent ASOS surface observations.
|
||
|
||
`current/0` returns one struct so the map page can render a single
|
||
status chip ("Updating…", "Up to date · 8m ago", "Stale · 4h ago")
|
||
that reflects the whole pipeline rather than tailing a single job.
|
||
"""
|
||
|
||
import Ecto.Query
|
||
|
||
alias Microwaveprop.Repo
|
||
|
||
@grid_worker "Microwaveprop.Workers.PropagationGridWorker"
|
||
@asos_worker "Microwaveprop.Workers.AsosAdjustmentWorker"
|
||
|
||
@workers [@grid_worker, @asos_worker]
|
||
|
||
# Scores older than this flip the chip to :stale. Matches the
|
||
# FreshnessMonitor threshold — anything past 2h is already a gap
|
||
# the pipeline should have caught.
|
||
@stale_after_minutes 120
|
||
|
||
@type state :: :running | :idle | :stale | :unknown
|
||
|
||
@type worker_tag :: :grid | :asos
|
||
|
||
@type running_detail :: %{worker: worker_tag(), label: String.t()}
|
||
|
||
@type t :: %{
|
||
state: state(),
|
||
label: String.t(),
|
||
details: [running_detail()],
|
||
last_update_at: DateTime.t() | nil
|
||
}
|
||
|
||
@doc """
|
||
Format a forecast-progress payload (as broadcast by
|
||
`PropagationGridWorker`) into a short detail line for the running
|
||
chip ("now", "+3h", …).
|
||
|
||
Returns `nil` when the payload has no `forecast_hour` so callers can
|
||
fall back to the base detail (e.g. "HRRR run") from `current/0`.
|
||
"""
|
||
@spec running_detail(map() | nil) :: String.t() | nil
|
||
def running_detail(%{forecast_hour: 0}), do: "now"
|
||
def running_detail(%{forecast_hour: fh}) when is_integer(fh) and fh > 0, do: "+#{fh}h"
|
||
def running_detail(_), do: nil
|
||
|
||
@spec current() :: t()
|
||
def current do
|
||
case running_workers() do
|
||
[] ->
|
||
build_idle_or_stale(latest_completed_at())
|
||
|
||
details ->
|
||
%{
|
||
state: :running,
|
||
label: "Updating propagation",
|
||
details: details,
|
||
last_update_at: latest_completed_at()
|
||
}
|
||
end
|
||
end
|
||
|
||
# Returns every pipeline worker currently in executing state, in a
|
||
# stable grid-before-asos order so the chip sub-lines don't flicker
|
||
# as jobs come and go.
|
||
defp running_workers do
|
||
from(j in "oban_jobs",
|
||
where: j.state == "executing" and j.worker in ^@workers,
|
||
distinct: j.worker,
|
||
select: j.worker
|
||
)
|
||
|> Repo.all()
|
||
|> Enum.sort_by(&worker_sort_key/1)
|
||
|> Enum.map(&to_running_detail/1)
|
||
end
|
||
|
||
defp to_running_detail(@grid_worker), do: %{worker: :grid, label: "HRRR run"}
|
||
defp to_running_detail(@asos_worker), do: %{worker: :asos, label: "ASOS nudge"}
|
||
|
||
defp worker_sort_key(@grid_worker), do: 0
|
||
defp worker_sort_key(@asos_worker), do: 1
|
||
defp worker_sort_key(_), do: 99
|
||
|
||
defp latest_completed_at do
|
||
case Repo.one(
|
||
from j in "oban_jobs",
|
||
where: j.state == "completed" and j.worker in ^@workers,
|
||
order_by: [desc: j.completed_at],
|
||
limit: 1,
|
||
select: j.completed_at
|
||
) do
|
||
nil -> nil
|
||
%NaiveDateTime{} = naive -> DateTime.from_naive!(naive, "Etc/UTC")
|
||
%DateTime{} = dt -> dt
|
||
end
|
||
end
|
||
|
||
defp build_idle_or_stale(nil) do
|
||
%{state: :unknown, label: "Propagation status unknown", details: [], last_update_at: nil}
|
||
end
|
||
|
||
defp build_idle_or_stale(%DateTime{} = last) do
|
||
age_minutes = max(DateTime.diff(DateTime.utc_now(), last, :minute), 0)
|
||
|
||
if age_minutes > @stale_after_minutes do
|
||
%{
|
||
state: :stale,
|
||
label: "Propagation data stale · last update #{format_age(age_minutes)} ago",
|
||
details: [],
|
||
last_update_at: last
|
||
}
|
||
else
|
||
%{
|
||
state: :idle,
|
||
label: "Up to date · #{format_age(age_minutes)} ago",
|
||
details: [],
|
||
last_update_at: last
|
||
}
|
||
end
|
||
end
|
||
|
||
defp format_age(0), do: "just now"
|
||
defp format_age(1), do: "1m"
|
||
defp format_age(n) when n < 60, do: "#{n}m"
|
||
|
||
defp format_age(n) do
|
||
hours = div(n, 60)
|
||
mins = rem(n, 60)
|
||
if mins == 0, do: "#{hours}h", else: "#{hours}h #{mins}m"
|
||
end
|
||
end
|