96 lines
3.3 KiB
Elixir
96 lines
3.3 KiB
Elixir
defmodule Microwaveprop.Workers.PropagationGridWorker do
|
|
@moduledoc """
|
|
Hourly seed worker for the Rust `prop-grid-rs` chain.
|
|
|
|
Post-Phase-3-cutover, Elixir no longer runs any fetch/decode/score
|
|
work for the propagation grid. The hourly cron fires this worker
|
|
with empty args, and it inserts grid_tasks rows (1 analysis
|
|
f00 + forecast f01..f18 hourly, f21..f48 3-hourly) for Rust to drain. Rust owns everything
|
|
from there: HRRR fetch, wgrib2 decode, native-level duct merge,
|
|
NEXRAD composite, commercial-link degradation, band scoring, and
|
|
both the ProfilesFile (MessagePack) and the per-band score files.
|
|
|
|
The old chain-step perform/2 clauses and the merge/score helpers
|
|
moved to `rust/prop_grid_rs/src/pipeline.rs`. `Propagation.record_run_timing`
|
|
is still called from this module when the Rust worker reports a
|
|
chain step, through the PropagationNotifyListener path.
|
|
"""
|
|
|
|
use Oban.Pro.Worker,
|
|
queue: :propagation,
|
|
priority: 0,
|
|
max_attempts: 5,
|
|
# Deduplicate seed jobs in a 1-hour window. The hourly cron fires
|
|
# `%{}` args; unique protects against FreshnessMonitor re-fires
|
|
# during an outage stacking multiple chains for the same run.
|
|
unique: [period: 3600, states: :incomplete]
|
|
|
|
alias Microwaveprop.Instrument
|
|
alias Microwaveprop.Propagation.GridTaskEnqueuer
|
|
alias Microwaveprop.Weather.HrrrClient
|
|
|
|
require Logger
|
|
|
|
@impl Oban.Pro.Worker
|
|
def process(%Oban.Job{args: args}) when args == %{} do
|
|
Instrument.span([:propagation, :grid_worker, :perform], %{}, fn ->
|
|
seed_chain()
|
|
end)
|
|
end
|
|
|
|
defp seed_chain do
|
|
# Reclaim any `running` grid_tasks rows orphaned by a Rust worker
|
|
# that died mid-claim (SIGKILL, OOM, node drain). Without this
|
|
# the status panel shows a permanent-looking spinner and the
|
|
# rows are never re-worked.
|
|
_ = GridTaskEnqueuer.reclaim_stale_running()
|
|
|
|
run_time = pick_run_time(DateTime.utc_now())
|
|
|
|
Logger.info(
|
|
"PropagationGrid: seeding chain run_time=#{run_time} (f00 analysis + f01-f18 hourly, f21-f48 3-hourly forecasts via grid_tasks)"
|
|
)
|
|
|
|
case GridTaskEnqueuer.seed_with_analysis(run_time) do
|
|
{:ok, count} ->
|
|
:telemetry.execute(
|
|
[:microwaveprop, :propagation, :grid_worker, :seeded],
|
|
%{queued_tasks: count},
|
|
%{run_time: run_time}
|
|
)
|
|
|
|
:ok
|
|
|
|
{:error, reason} ->
|
|
:telemetry.execute(
|
|
[:microwaveprop, :propagation, :grid_worker, :exception],
|
|
%{},
|
|
%{run_time: run_time, reason: inspect(reason)}
|
|
)
|
|
|
|
{:error, reason}
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Pick the freshest HRRR cycle that's published.
|
|
|
|
Probes `now-1h` first via `HrrrClient.cycle_available?/1`. If the
|
|
pressure-level f18 idx file is on the mirror/S3, the cycle is fully
|
|
uploaded and we use it. Otherwise we fall back to the conservative
|
|
`now-2h` slot — NOAA's stated max publishing latency for HRRR.
|
|
|
|
Public so the test suite can exercise the selection logic without
|
|
going through Oban.
|
|
"""
|
|
@spec pick_run_time(DateTime.t()) :: DateTime.t()
|
|
def pick_run_time(%DateTime{} = now) do
|
|
fresh = now |> DateTime.add(-1, :hour) |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second)
|
|
|
|
if HrrrClient.cycle_available?(fresh) do
|
|
fresh
|
|
else
|
|
now |> DateTime.add(-2, :hour) |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second)
|
|
end
|
|
end
|
|
end
|