PropagationGridWorker.merge_commercial_link_data/2 called
Commercial.link_degradation_at twice per grid cell (once to count,
once to merge), and each call re-ran enabled_links plus two sample
queries per in-range link. That was ~500k SQL queries per forecast
hour — fine for the DB but catastrophic for log volume in dev
iex sessions trying to watch a chain step run.
Split into two new functions:
* build_link_lookup/2 does all the DB work up front — one
enabled_links query + one link_degradation per enabled link
(~10 queries total).
* link_degradation_from_lookup/3 is pure: takes a (lat, lon)
and the precomputed lookup, returns the aggregate or nil.
The worker now calls build_link_lookup once per forecast hour and
the per-cell path is haversine-only. Net: 500k queries → ~10.
PipelineStatus freshness detection moves off oban_jobs.completed_at
onto ScoresFile.latest_valid_time(). The on-disk files ARE the data
the map renders from, so the chip's "Up to date · Nm ago" and the
2h stale threshold now track actual data presence. Running-state
detection still comes from oban_jobs since the chain step is still
an Oban row. Tests updated to seed a ScoresFile instead of a
completed Oban row for the idle/stale cases.
135 lines
4.1 KiB
Elixir
135 lines
4.1 KiB
Elixir
defmodule Microwaveprop.Propagation.PipelineStatus do
|
|
@moduledoc """
|
|
Aggregated status for the propagation update pipeline.
|
|
|
|
Two inputs drive the chip:
|
|
|
|
* **Running detection** — queries `oban_jobs` for executing
|
|
PropagationGridWorker / AsosAdjustmentWorker rows so the chip
|
|
knows when a chain step is in flight.
|
|
* **Freshness detection** — reads the newest valid_time from
|
|
`Microwaveprop.Propagation.ScoresFile` on disk. The on-disk
|
|
files ARE the propagation data now, so the chip's "Up to date ·
|
|
Nm ago" and the 2h stale threshold track actual data presence
|
|
instead of Oban's `completed_at` timestamp.
|
|
"""
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Propagation.ScoresFile
|
|
alias Microwaveprop.Repo
|
|
|
|
@grid_worker "Microwaveprop.Workers.PropagationGridWorker"
|
|
@asos_worker "Microwaveprop.Workers.AsosAdjustmentWorker"
|
|
|
|
@workers [@grid_worker, @asos_worker]
|
|
|
|
# Scores older than this flip the chip to :stale. Matches the
|
|
# FreshnessMonitor threshold — anything past 2h is already a gap
|
|
# the pipeline should have caught.
|
|
@stale_after_minutes 120
|
|
|
|
@type state :: :running | :idle | :stale | :unknown
|
|
|
|
@type worker_tag :: :grid | :asos
|
|
|
|
@type running_detail :: %{worker: worker_tag(), label: String.t()}
|
|
|
|
@type t :: %{
|
|
state: state(),
|
|
label: String.t(),
|
|
details: [running_detail()],
|
|
last_update_at: DateTime.t() | nil
|
|
}
|
|
|
|
@doc """
|
|
Format a forecast-progress payload (as broadcast by
|
|
`PropagationGridWorker`) into a short detail line for the running
|
|
chip ("now", "+3h", …).
|
|
|
|
Returns `nil` when the payload has no `forecast_hour` so callers can
|
|
fall back to the base detail (e.g. "HRRR run") from `current/0`.
|
|
"""
|
|
@spec running_detail(map() | nil) :: String.t() | nil
|
|
def running_detail(%{forecast_hour: 0}), do: "now"
|
|
def running_detail(%{forecast_hour: fh}) when is_integer(fh) and fh > 0, do: "+#{fh}h"
|
|
def running_detail(_), do: nil
|
|
|
|
@spec current() :: t()
|
|
def current do
|
|
case running_workers() do
|
|
[] ->
|
|
build_idle_or_stale(latest_data_at())
|
|
|
|
details ->
|
|
%{
|
|
state: :running,
|
|
label: "Updating propagation",
|
|
details: details,
|
|
last_update_at: latest_data_at()
|
|
}
|
|
end
|
|
end
|
|
|
|
# Returns every pipeline worker currently in executing state, in a
|
|
# stable grid-before-asos order so the chip sub-lines don't flicker
|
|
# as jobs come and go.
|
|
defp running_workers do
|
|
from(j in "oban_jobs",
|
|
where: j.state == "executing" and j.worker in ^@workers,
|
|
distinct: j.worker,
|
|
select: j.worker
|
|
)
|
|
|> Repo.all()
|
|
|> Enum.sort_by(&worker_sort_key/1)
|
|
|> Enum.map(&to_running_detail/1)
|
|
end
|
|
|
|
defp to_running_detail(@grid_worker), do: %{worker: :grid, label: "HRRR run"}
|
|
defp to_running_detail(@asos_worker), do: %{worker: :asos, label: "ASOS nudge"}
|
|
|
|
defp worker_sort_key(@grid_worker), do: 0
|
|
defp worker_sort_key(@asos_worker), do: 1
|
|
defp worker_sort_key(_), do: 99
|
|
|
|
# Freshness is derived from the newest ScoresFile on disk. That file
|
|
# is the thing the map actually renders from, so its valid_time is
|
|
# the most honest answer to "when was the data last updated?".
|
|
defp latest_data_at do
|
|
ScoresFile.latest_valid_time()
|
|
end
|
|
|
|
defp build_idle_or_stale(nil) do
|
|
%{state: :unknown, label: "Propagation status unknown", details: [], last_update_at: nil}
|
|
end
|
|
|
|
defp build_idle_or_stale(%DateTime{} = last) do
|
|
age_minutes = max(DateTime.diff(DateTime.utc_now(), last, :minute), 0)
|
|
|
|
if age_minutes > @stale_after_minutes do
|
|
%{
|
|
state: :stale,
|
|
label: "Propagation data stale · last update #{format_age(age_minutes)} ago",
|
|
details: [],
|
|
last_update_at: last
|
|
}
|
|
else
|
|
%{
|
|
state: :idle,
|
|
label: "Up to date · #{format_age(age_minutes)} ago",
|
|
details: [],
|
|
last_update_at: last
|
|
}
|
|
end
|
|
end
|
|
|
|
defp format_age(0), do: "just now"
|
|
defp format_age(1), do: "1m"
|
|
defp format_age(n) when n < 60, do: "#{n}m"
|
|
|
|
defp format_age(n) do
|
|
hours = div(n, 60)
|
|
mins = rem(n, 60)
|
|
if mins == 0, do: "#{hours}h", else: "#{hours}h #{mins}m"
|
|
end
|
|
end
|