Graham McIntire
fac604b289
Era5SubmitWorker was counting era5_cds_jobs rows 1:1 against the CDS
150-job ceiling, but each row holds TWO CDS job IDs (single-level +
pressure-level). Prod hit 74 rows = 148 in-flight jobs and got stuck
in a reject→resubmit spiral. The cap guard now multiplies row count
by 2 to match what CDS sees.
Era5PollWorker @stuck_after_seconds goes from 4h to 18h. CDS
single-level routinely sits at 'accepted' for 12+h under load while
pressure finishes in ~90 min; the 4h threshold was firing on normal
slow runs and feeding the same spiral.
PipelineStatus now returns `label` + `detail` separately so the map
chip can render "Updating propagation" on one line and the
sub-detail ("HRRR run" / "ASOS nudge" / "+3h" / "now") on a second
line below it. running_label/1 is renamed to running_detail/1 and
returns just the sub-part.
130 lines
3.8 KiB
Elixir
130 lines
3.8 KiB
Elixir
defmodule Microwaveprop.Propagation.PipelineStatus do
|
||
@moduledoc """
|
||
Aggregated status for the propagation update pipeline.
|
||
|
||
The pipeline is driven by two Oban workers:
|
||
|
||
* `PropagationGridWorker` — hourly, fetches HRRR f00–f18, scores
|
||
the CONUS grid, and broadcasts `propagation:updated`.
|
||
* `AsosAdjustmentWorker` — every 10 minutes, nudges scores with
|
||
recent ASOS surface observations.
|
||
|
||
`current/0` returns one struct so the map page can render a single
|
||
status chip ("Updating…", "Up to date · 8m ago", "Stale · 4h ago")
|
||
that reflects the whole pipeline rather than tailing a single job.
|
||
"""
|
||
|
||
import Ecto.Query
|
||
|
||
alias Microwaveprop.Repo
|
||
|
||
@grid_worker "Microwaveprop.Workers.PropagationGridWorker"
|
||
@asos_worker "Microwaveprop.Workers.AsosAdjustmentWorker"
|
||
|
||
@workers [@grid_worker, @asos_worker]
|
||
|
||
# Scores older than this flip the chip to :stale. Matches the
|
||
# FreshnessMonitor threshold — anything past 2h is already a gap
|
||
# the pipeline should have caught.
|
||
@stale_after_minutes 120
|
||
|
||
@type state :: :running | :idle | :stale | :unknown
|
||
|
||
@type t :: %{
|
||
state: state(),
|
||
label: String.t(),
|
||
detail: String.t() | nil,
|
||
last_update_at: DateTime.t() | nil
|
||
}
|
||
|
||
@doc """
|
||
Format a forecast-progress payload (as broadcast by
|
||
`PropagationGridWorker`) into a short detail line for the running
|
||
chip ("now", "+3h", …).
|
||
|
||
Returns `nil` when the payload has no `forecast_hour` so callers can
|
||
fall back to the base detail (e.g. "HRRR run") from `current/0`.
|
||
"""
|
||
@spec running_detail(map() | nil) :: String.t() | nil
|
||
def running_detail(%{forecast_hour: 0}), do: "now"
|
||
def running_detail(%{forecast_hour: fh}) when is_integer(fh) and fh > 0, do: "+#{fh}h"
|
||
def running_detail(_), do: nil
|
||
|
||
@spec current() :: t()
|
||
def current do
|
||
case running_worker() do
|
||
worker when is_binary(worker) ->
|
||
%{
|
||
state: :running,
|
||
label: "Updating propagation",
|
||
detail: base_running_detail(worker),
|
||
last_update_at: latest_completed_at()
|
||
}
|
||
|
||
nil ->
|
||
build_idle_or_stale(latest_completed_at())
|
||
end
|
||
end
|
||
|
||
defp running_worker do
|
||
Repo.one(
|
||
from j in "oban_jobs",
|
||
where: j.state == "executing" and j.worker in ^@workers,
|
||
order_by: [desc: j.attempted_at],
|
||
limit: 1,
|
||
select: j.worker
|
||
)
|
||
end
|
||
|
||
defp latest_completed_at do
|
||
case Repo.one(
|
||
from j in "oban_jobs",
|
||
where: j.state == "completed" and j.worker in ^@workers,
|
||
order_by: [desc: j.completed_at],
|
||
limit: 1,
|
||
select: j.completed_at
|
||
) do
|
||
nil -> nil
|
||
%NaiveDateTime{} = naive -> DateTime.from_naive!(naive, "Etc/UTC")
|
||
%DateTime{} = dt -> dt
|
||
end
|
||
end
|
||
|
||
defp build_idle_or_stale(nil) do
|
||
%{state: :unknown, label: "Propagation status unknown", detail: nil, last_update_at: nil}
|
||
end
|
||
|
||
defp build_idle_or_stale(%DateTime{} = last) do
|
||
age_minutes = max(DateTime.diff(DateTime.utc_now(), last, :minute), 0)
|
||
|
||
if age_minutes > @stale_after_minutes do
|
||
%{
|
||
state: :stale,
|
||
label: "Propagation data stale · last update #{format_age(age_minutes)} ago",
|
||
detail: nil,
|
||
last_update_at: last
|
||
}
|
||
else
|
||
%{
|
||
state: :idle,
|
||
label: "Up to date · #{format_age(age_minutes)} ago",
|
||
detail: nil,
|
||
last_update_at: last
|
||
}
|
||
end
|
||
end
|
||
|
||
defp base_running_detail(@grid_worker), do: "HRRR run"
|
||
defp base_running_detail(@asos_worker), do: "ASOS nudge"
|
||
defp base_running_detail(_), do: nil
|
||
|
||
defp format_age(0), do: "just now"
|
||
defp format_age(1), do: "1m"
|
||
defp format_age(n) when n < 60, do: "#{n}m"
|
||
|
||
defp format_age(n) do
|
||
hours = div(n, 60)
|
||
mins = rem(n, 60)
|
||
if mins == 0, do: "#{hours}h", else: "#{hours}h #{mins}m"
|
||
end
|
||
end
|