diff --git a/config/config.exs b/config/config.exs index 9c7b4d63..50950ff4 100644 --- a/config/config.exs +++ b/config/config.exs @@ -74,6 +74,12 @@ config :microwaveprop, Oban, enqueue: 1, gefs: 2, terrain: 4, + # RoverPathProfileWorker runs the same full HRRR/terrain/score + # /loss/forecast pipeline as the propagation grid chain. Kept on + # its own queue at 1 slot/pod so a backfill flood can't stack + # multiple heavy path-computes in one BEAM heap (the OOM that + # took down hot pods on 2026-05-03). + rover_path: 1, commercial: 2, iemre: 10, # 2 slots so PropagationPruneWorker can run alongside the diff --git a/config/runtime.exs b/config/runtime.exs index 2ec4fc2d..4f73b6be 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -172,6 +172,15 @@ if config_env() == :prod do # :hrrr queue removed — per-QSO HRRR fetches flow through # hrrr_fetch_tasks and the Rust hrrr-point-worker (Phase 3 Stream C). terrain: 3, + # 1 slot per pod, same reasoning as :propagation. Each + # RoverPathProfileWorker job allocates a full path-compute heap + # (terrain points + 9 HRRR profiles + sounding + ionosphere + + # scoring + loss + forecast); 3 slots × 4 hot pods on the + # :terrain queue OOM-killed pods on 2026-05-03 when a backfill + # flood landed. Cluster-wide capacity is now (hot replicas + 1 + # backfill) jobs at a time, which still drains a 200-job + # backlog inside ~30 min. + rover_path: 1, iemre: 3, backfill_enqueue: 1, admin: 1, diff --git a/lib/microwaveprop/workers/rover_path_profile_worker.ex b/lib/microwaveprop/workers/rover_path_profile_worker.ex index 2e7f6d9b..cdaa93af 100644 --- a/lib/microwaveprop/workers/rover_path_profile_worker.ex +++ b/lib/microwaveprop/workers/rover_path_profile_worker.ex @@ -16,7 +16,7 @@ defmodule Microwaveprop.Workers.RoverPathProfileWorker do handed `?rover_path_id=UUID` and renders without recomputing. """ use Oban.Worker, - queue: :terrain, + queue: :rover_path, max_attempts: 5, unique: [ period: 300,