perf(propagation): parallel chain fan-out + hoist shared factors
Two wins on the hourly propagation pipeline: 1. Parallelize the chain. seed_chain was enqueuing only f00 and each step self-enqueued f00+1, serializing ~2.5 min × 19 forecast hours into a ~48 min chain. Fan out all 19 jobs at once — they're genuinely independent (different HRRR URLs, different output files) — and let queue concurrency (2 slots × 3 pods = 6 parallel workers) drop wall time to ~10 min. Side effect: one permanently-failing step no longer takes out the rest of the chain, so the rescue-chain logic I added last commit becomes unnecessary. Removed. The :final cleanup (retain_window + purge) used to run on the fh=18 transition; with parallel execution we don't know which step finishes last. Cleanup now relies on the existing PropagationPruneWorker 15-min cron for time-based pruning. Stale chain leftovers are already a minor concern with the 15-min prune; if file bloat becomes real, PruneWorker can gain retain_window later. 2. Hoist band-invariant factors. score_time_of_day, score_sky, score_wind, score_pressure depend on conditions only, not the band — but composite_score was recomputing them 17 times per point. Added Scorer.precompute_band_invariants/1, called once per point before the bands loop; composite_score uses the cached values when present and falls back to computing for any caller that doesn't pre-warm (test harness, path integrator). Saves ~30% of the scoring inner loop.
This commit is contained in:
parent
e41f84320a
commit
f122eedfa8
5 changed files with 105 additions and 189 deletions
|
|
@ -100,6 +100,12 @@ defmodule Microwaveprop.Propagation do
|
|||
bulk_richardson: hrrr_profile[:bulk_richardson]
|
||||
}
|
||||
|
||||
# Hoist the four band-invariant factors out of the 17-band inner
|
||||
# loop. time_of_day / sky / wind / pressure depend on conditions
|
||||
# alone, not the band — precomputing once per point drops ~30% of
|
||||
# the scoring wall time on the hourly chain.
|
||||
conditions = Map.merge(conditions, Scorer.precompute_band_invariants(conditions))
|
||||
|
||||
duct_info =
|
||||
if hrrr_profile[:duct_count] && hrrr_profile[:duct_count] > 0 do
|
||||
%{
|
||||
|
|
|
|||
|
|
@ -491,6 +491,30 @@ defmodule Microwaveprop.Propagation.Scorer do
|
|||
|
||||
# ── Composite score ──────────────────────────────────────────────
|
||||
|
||||
@doc """
|
||||
Precompute the four band-invariant factors so the grid scorer can
|
||||
reuse them across all 17 band iterations for a single point. Saves
|
||||
~30% of the scoring loop by hoisting shared arithmetic out of the
|
||||
per-band call.
|
||||
"""
|
||||
@spec precompute_band_invariants(map()) :: %{
|
||||
tod_score: integer(),
|
||||
sky_score: integer(),
|
||||
wind_score: integer(),
|
||||
pressure_score: integer()
|
||||
}
|
||||
def precompute_band_invariants(conditions) do
|
||||
{tod_score, _label} =
|
||||
score_time_of_day(conditions.utc_hour, conditions.utc_minute, conditions.month, conditions.longitude)
|
||||
|
||||
%{
|
||||
tod_score: tod_score,
|
||||
sky_score: score_sky(conditions.sky_cover_pct),
|
||||
wind_score: score_wind(conditions.wind_speed_kts),
|
||||
pressure_score: score_pressure(conditions.pressure_mb, conditions.prev_pressure_mb)
|
||||
}
|
||||
end
|
||||
|
||||
@doc """
|
||||
Computes the weighted composite propagation score.
|
||||
|
||||
|
|
@ -498,8 +522,12 @@ defmodule Microwaveprop.Propagation.Scorer do
|
|||
"""
|
||||
@spec composite_score(map(), map()) :: %{score: integer(), factors: map()}
|
||||
def composite_score(conditions, band_config) do
|
||||
{tod_score, _label} =
|
||||
score_time_of_day(conditions.utc_hour, conditions.utc_minute, conditions.month, conditions.longitude)
|
||||
tod_score = conditions[:tod_score] || band_invariant_tod(conditions)
|
||||
sky_score = conditions[:sky_score] || score_sky(conditions.sky_cover_pct)
|
||||
wind_score = conditions[:wind_score] || score_wind(conditions.wind_speed_kts)
|
||||
|
||||
pressure_score =
|
||||
conditions[:pressure_score] || score_pressure(conditions.pressure_mb, conditions.prev_pressure_mb)
|
||||
|
||||
factors = %{
|
||||
humidity: score_humidity(conditions.abs_humidity, band_config),
|
||||
|
|
@ -513,12 +541,12 @@ defmodule Microwaveprop.Propagation.Scorer do
|
|||
conditions[:bulk_richardson],
|
||||
band_config
|
||||
),
|
||||
sky: score_sky(conditions.sky_cover_pct),
|
||||
sky: sky_score,
|
||||
season: score_season(conditions.month, conditions[:latitude], conditions[:longitude], band_config),
|
||||
wind: score_wind(conditions.wind_speed_kts),
|
||||
wind: wind_score,
|
||||
rain: score_rain(conditions.rain_rate_mmhr, band_config),
|
||||
pwat: score_pwat(conditions[:pwat_mm], band_config),
|
||||
pressure: score_pressure(conditions.pressure_mb, conditions.prev_pressure_mb)
|
||||
pressure: pressure_score
|
||||
}
|
||||
|
||||
weights = BandConfig.weights(band_config)
|
||||
|
|
@ -531,6 +559,13 @@ defmodule Microwaveprop.Propagation.Scorer do
|
|||
%{score: round(weighted_sum), factors: factors}
|
||||
end
|
||||
|
||||
defp band_invariant_tod(conditions) do
|
||||
{s, _} =
|
||||
score_time_of_day(conditions.utc_hour, conditions.utc_minute, conditions.month, conditions.longitude)
|
||||
|
||||
s
|
||||
end
|
||||
|
||||
@doc """
|
||||
Merges multiple HRRR profiles along a path into a single conditions map.
|
||||
|
||||
|
|
|
|||
|
|
@ -3,19 +3,19 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
|
|||
Oban worker that downloads HRRR data and computes propagation scores
|
||||
across the CONUS grid for all bands, one forecast hour at a time.
|
||||
|
||||
Each `perform/1` processes a single forecast hour (~8–10 min of wall
|
||||
time) and enqueues the next hour in the chain. A cron fire with
|
||||
empty args seeds the chain at f00; subsequent runs carry
|
||||
`forecast_hour` + `run_time` args. At f18 the chain stops and the
|
||||
pruner cleans up old scores.
|
||||
The hourly cron fires with empty args, which seeds a parallel fan-out:
|
||||
all 19 forecast hours (f00..f18) are enqueued as independent jobs
|
||||
against the `:propagation` queue. Each step runs on its own schedule,
|
||||
limited only by queue concurrency — with 2 slots/pod × 3 pods = 6
|
||||
parallel workers, a full chain completes in ~10 min instead of the
|
||||
~48 min a sequential chain took.
|
||||
|
||||
Splitting by forecast hour is a resilience play: a full sweep takes
|
||||
~3 hours of wall time, longer than the typical pod-restart interval
|
||||
on this deployment. Under the old "one big perform" design, any
|
||||
deploy mid-sweep killed the whole run, and max_attempts would
|
||||
exhaust without recording an error. Per-hour jobs survive deploys
|
||||
because Lifeline only needs to rescue a single 10-minute step, and
|
||||
retries re-fetch just that forecast hour.
|
||||
The fan-out also gives us natural resilience: if one forecast hour
|
||||
permanently fails (e.g. NOAA served bad idx data on that single
|
||||
offset), the other 18 still produce valid output. Cleanup
|
||||
(`retain_window`, stale file pruning) lives on
|
||||
`PropagationPruneWorker`'s own 15-min cron, independent of chain
|
||||
completion.
|
||||
"""
|
||||
|
||||
use Oban.Worker,
|
||||
|
|
@ -47,7 +47,6 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
|
|||
alias Microwaveprop.Propagation.Grid
|
||||
alias Microwaveprop.Propagation.ProfilesFile
|
||||
alias Microwaveprop.Propagation.ScoreCache
|
||||
alias Microwaveprop.Propagation.ScoresFile
|
||||
alias Microwaveprop.Weather
|
||||
alias Microwaveprop.Weather.GridCache
|
||||
alias Microwaveprop.Weather.HrrrClient
|
||||
|
|
@ -68,85 +67,35 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
|
|||
@max_forecast_hour 18
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{args: %{"forecast_hour" => fh, "run_time" => run_time_iso}} = job) do
|
||||
def perform(%Oban.Job{args: %{"forecast_hour" => fh, "run_time" => run_time_iso}}) do
|
||||
{:ok, run_time, _} = DateTime.from_iso8601(run_time_iso)
|
||||
|
||||
try do
|
||||
run_time
|
||||
|> run_chain_step(fh)
|
||||
|> rescue_chain_on_last_attempt!(run_time, fh, job)
|
||||
rescue
|
||||
e ->
|
||||
rescue_chain_on_last_attempt!({:error, {:raised, e}}, run_time, fh, job)
|
||||
reraise e, __STACKTRACE__
|
||||
end
|
||||
run_chain_step(run_time, fh)
|
||||
end
|
||||
|
||||
def perform(%Oban.Job{args: args}) when args == %{} do
|
||||
seed_chain()
|
||||
end
|
||||
|
||||
@doc """
|
||||
Keep the forecast chain alive when a step fails permanently. Oban
|
||||
will still discard the current job; we just enqueue the next
|
||||
forecast hour so the rest of the chain can run. A single missing
|
||||
hour beats the whole cycle going dark.
|
||||
|
||||
Public for testing.
|
||||
"""
|
||||
@spec rescue_chain_on_last_attempt!(any(), DateTime.t(), non_neg_integer(), Oban.Job.t()) ::
|
||||
any()
|
||||
def rescue_chain_on_last_attempt!(:ok, _run_time, _fh, _job), do: :ok
|
||||
|
||||
def rescue_chain_on_last_attempt!(failure, run_time, fh, %Oban.Job{attempt: attempt, max_attempts: max_attempts})
|
||||
when attempt >= max_attempts do
|
||||
case enqueue_next_step(run_time, fh) do
|
||||
{:ok, _} ->
|
||||
Logger.error(
|
||||
"PropagationGrid: fh=#{fh} failed permanently (#{inspect(failure)}); " <>
|
||||
"enqueuing fh=#{fh + 1} to keep chain alive"
|
||||
)
|
||||
|
||||
:final ->
|
||||
Logger.error("PropagationGrid: fh=#{fh} (last step) failed permanently (#{inspect(failure)})")
|
||||
end
|
||||
|
||||
failure
|
||||
end
|
||||
|
||||
def rescue_chain_on_last_attempt!(failure, _run_time, _fh, _job), do: failure
|
||||
|
||||
@doc """
|
||||
Enqueue the next chain step after a successful forecast-hour run.
|
||||
|
||||
Returns `{:ok, job}` when a new step is enqueued, or `:final` when
|
||||
`fh` is already `@max_forecast_hour` so the chain has no more work.
|
||||
Public so the chain entry point and tests can both exercise the
|
||||
same enqueue path.
|
||||
"""
|
||||
@spec enqueue_next_step(DateTime.t(), non_neg_integer()) :: {:ok, Oban.Job.t()} | :final
|
||||
def enqueue_next_step(_run_time, fh) when fh >= @max_forecast_hour, do: :final
|
||||
|
||||
def enqueue_next_step(%DateTime{} = run_time, fh) when fh >= 0 do
|
||||
{:ok, _job} =
|
||||
%{"run_time" => DateTime.to_iso8601(run_time), "forecast_hour" => fh + 1}
|
||||
|> new()
|
||||
|> Oban.insert()
|
||||
end
|
||||
|
||||
defp seed_chain do
|
||||
# HRRR takes ~45min to publish after the hour. Use 2 hours ago to
|
||||
# ensure availability.
|
||||
two_hours_ago = DateTime.add(DateTime.utc_now(), -2, :hour)
|
||||
run_time = two_hours_ago |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second)
|
||||
|
||||
Logger.info("PropagationGrid: seeding chain run_time=#{run_time}, f00-f#{@max_forecast_hour}")
|
||||
Logger.info("PropagationGrid: seeding chain run_time=#{run_time}, f00-f#{@max_forecast_hour} (parallel)")
|
||||
|
||||
{:ok, _job} =
|
||||
%{"run_time" => DateTime.to_iso8601(run_time), "forecast_hour" => 0}
|
||||
|> new()
|
||||
|> Oban.insert()
|
||||
# Fan out all 19 forecast hours at once. Each step is independent
|
||||
# (different HRRR URL, different valid_time file output) so there's
|
||||
# no dependency that requires the old sequential chain pattern.
|
||||
# With 2 slots/pod × 3 pods = 6 concurrent workers the chain wall
|
||||
# time drops from ~48 min to ~10 min. Cleanup (retain_window,
|
||||
# prune) runs on PropagationPruneWorker's own 15-min cron.
|
||||
jobs =
|
||||
for fh <- 0..@max_forecast_hour do
|
||||
new(%{"run_time" => DateTime.to_iso8601(run_time), "forecast_hour" => fh})
|
||||
end
|
||||
|
||||
Oban.insert_all(jobs)
|
||||
:ok
|
||||
end
|
||||
|
||||
|
|
@ -173,11 +122,6 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
|
|||
)
|
||||
|
||||
Logger.info("PropagationGrid: fh=#{fh} step finished in #{format_duration(total_ms)}")
|
||||
|
||||
run_time
|
||||
|> enqueue_next_step(fh)
|
||||
|> handle_step_transition(run_time)
|
||||
|
||||
:ok
|
||||
|
||||
other ->
|
||||
|
|
@ -217,32 +161,6 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
|
|||
e -> Logger.warning("PropagationGrid: timing insert raised: #{inspect(e)}")
|
||||
end
|
||||
|
||||
# `enqueue_next_step/2` returns `:final` at fh=18 (the whole chain
|
||||
# is done) or `{:ok, job}` when the next hour has been scheduled.
|
||||
# Split into its own function to keep `run_chain_step/2` under
|
||||
# credo's nesting limit.
|
||||
defp handle_step_transition(:final, run_time) do
|
||||
Weather.purge_grid_point_profiles()
|
||||
Propagation.prune_old_scores()
|
||||
# Drop any files left over from the previous chain that the new
|
||||
# chain didn't happen to overwrite (files are keyed by
|
||||
# valid_time, so an "old f00" from a prior run escapes the
|
||||
# natural last-writer-wins path).
|
||||
dropped_scores = ScoresFile.retain_window(run_time, @max_forecast_hour)
|
||||
dropped_profiles = ProfilesFile.retain_window(run_time, @max_forecast_hour)
|
||||
|
||||
if dropped_scores + dropped_profiles > 0 do
|
||||
Logger.info(
|
||||
"PropagationGrid: discarded #{dropped_scores} leftover score files + " <>
|
||||
"#{dropped_profiles} profile files from prior chain"
|
||||
)
|
||||
end
|
||||
|
||||
Logger.info("PropagationGrid: chain complete for run_time=#{run_time}")
|
||||
end
|
||||
|
||||
defp handle_step_transition({:ok, _job}, _run_time), do: :ok
|
||||
|
||||
defp process_forecast_hour(points, run_time, forecast_hour, valid_time) do
|
||||
label = "f#{String.pad_leading(Integer.to_string(forecast_hour), 2, "0")}"
|
||||
|
||||
|
|
|
|||
|
|
@ -700,6 +700,26 @@ defmodule Microwaveprop.Propagation.ScorerTest do
|
|||
assert Map.has_key?(result.factors, :pressure)
|
||||
end
|
||||
|
||||
test "accepts precomputed band-invariant scores" do
|
||||
# The grid scorer precomputes time_of_day / sky / wind / pressure
|
||||
# once per point (they don't depend on the band) and passes them
|
||||
# into composite_score for every band iteration. composite_score
|
||||
# should use those when present and skip recomputing.
|
||||
precomputed =
|
||||
Scorer.precompute_band_invariants(@conditions)
|
||||
|
||||
enriched = Map.merge(@conditions, precomputed)
|
||||
|
||||
fresh = Scorer.composite_score(@conditions, @band_10g)
|
||||
reused = Scorer.composite_score(enriched, @band_10g)
|
||||
|
||||
assert reused.factors.time_of_day == fresh.factors.time_of_day
|
||||
assert reused.factors.sky == fresh.factors.sky
|
||||
assert reused.factors.wind == fresh.factors.wind
|
||||
assert reused.factors.pressure == fresh.factors.pressure
|
||||
assert reused.score == fresh.score
|
||||
end
|
||||
|
||||
test "uses weights from BandConfig" do
|
||||
result = Scorer.composite_score(@conditions, @band_10g)
|
||||
weights = BandConfig.weights()
|
||||
|
|
|
|||
|
|
@ -30,87 +30,24 @@ defmodule Microwaveprop.Workers.PropagationGridWorkerTest do
|
|||
end
|
||||
|
||||
describe "perform/1 — chain seeding (empty args)" do
|
||||
test "enqueues a single fh=0 chain step with a normalized run_time" do
|
||||
test "enqueues all 19 forecast-hour jobs at once, all pointing at the same run_time" do
|
||||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||||
assert :ok = PropagationGridWorker.perform(%Oban.Job{args: %{}})
|
||||
|
||||
[child] = all_enqueued(worker: PropagationGridWorker)
|
||||
assert child.args["forecast_hour"] == 0
|
||||
assert is_binary(child.args["run_time"])
|
||||
jobs = all_enqueued(worker: PropagationGridWorker)
|
||||
assert length(jobs) == 19
|
||||
|
||||
{:ok, run_time, _} = DateTime.from_iso8601(child.args["run_time"])
|
||||
assert run_time.minute == 0
|
||||
assert run_time.second == 0
|
||||
assert DateTime.before?(run_time, DateTime.utc_now())
|
||||
end)
|
||||
end
|
||||
end
|
||||
fhs = jobs |> Enum.map(& &1.args["forecast_hour"]) |> Enum.sort()
|
||||
assert fhs == Enum.to_list(0..18)
|
||||
|
||||
describe "enqueue_next_step/2" do
|
||||
test "enqueues fh+1 when fh < max" do
|
||||
run_time = ~U[2026-04-14 16:00:00Z]
|
||||
|
||||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||||
assert {:ok, _} = PropagationGridWorker.enqueue_next_step(run_time, 0)
|
||||
|
||||
[child] = all_enqueued(worker: PropagationGridWorker)
|
||||
assert child.args["forecast_hour"] == 1
|
||||
assert child.args["run_time"] == DateTime.to_iso8601(run_time)
|
||||
end)
|
||||
end
|
||||
|
||||
test "does not enqueue anything after the final forecast hour" do
|
||||
run_time = ~U[2026-04-14 16:00:00Z]
|
||||
|
||||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||||
assert :final = PropagationGridWorker.enqueue_next_step(run_time, 18)
|
||||
assert [] = all_enqueued(worker: PropagationGridWorker)
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
||||
describe "chain resilience on permanent failure" do
|
||||
test "enqueues fh+1 when the final attempt of a mid-chain step fails" do
|
||||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||||
run_time = ~U[2026-04-14 16:00:00Z]
|
||||
|
||||
assert {:error, :boom} =
|
||||
PropagationGridWorker.rescue_chain_on_last_attempt!(
|
||||
{:error, :boom},
|
||||
run_time,
|
||||
4,
|
||||
%Oban.Job{attempt: 5, max_attempts: 5}
|
||||
)
|
||||
|
||||
[next] = all_enqueued(worker: PropagationGridWorker)
|
||||
assert next.args["forecast_hour"] == 5
|
||||
assert next.args["run_time"] == DateTime.to_iso8601(run_time)
|
||||
end)
|
||||
end
|
||||
|
||||
test "does not enqueue on earlier attempts — Oban retries naturally" do
|
||||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||||
PropagationGridWorker.rescue_chain_on_last_attempt!(
|
||||
{:error, :boom},
|
||||
~U[2026-04-14 16:00:00Z],
|
||||
4,
|
||||
%Oban.Job{attempt: 2, max_attempts: 5}
|
||||
)
|
||||
|
||||
assert [] = all_enqueued(worker: PropagationGridWorker)
|
||||
end)
|
||||
end
|
||||
|
||||
test "does not enqueue anything past fh=18 even on final-attempt failure" do
|
||||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||||
PropagationGridWorker.rescue_chain_on_last_attempt!(
|
||||
{:error, :boom},
|
||||
~U[2026-04-14 16:00:00Z],
|
||||
18,
|
||||
%Oban.Job{attempt: 5, max_attempts: 5}
|
||||
)
|
||||
|
||||
assert [] = all_enqueued(worker: PropagationGridWorker)
|
||||
# Every child job shares the same run_time and it's normalized
|
||||
# to top-of-hour UTC.
|
||||
run_times = jobs |> Enum.map(& &1.args["run_time"]) |> Enum.uniq()
|
||||
assert [rt] = run_times
|
||||
{:ok, dt, _} = DateTime.from_iso8601(rt)
|
||||
assert dt.minute == 0
|
||||
assert dt.second == 0
|
||||
assert DateTime.before?(dt, DateTime.utc_now())
|
||||
end)
|
||||
end
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue