From f7eb888123ff2b0c72c2523b8133056fdd1f7b4b Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 3 May 2026 15:47:42 -0500 Subject: [PATCH] docs(oban): expand :rover_path slot-1 rationale with memory budget Future-proof the queue config so a "let's bump concurrency for throughput" change has the per-pod RSS math in front of it. Includes the actual memory limit (6 GiB), observed steady state (3-5 GiB), the ~500 MiB peak per path-compute, and the explicit recommendation to add hot replicas via HPA or offload HRRR fetch to hrrr-point-rs before touching this slot count. --- config/config.exs | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/config/config.exs b/config/config.exs index 50950ff4..ec74fb73 100644 --- a/config/config.exs +++ b/config/config.exs @@ -74,11 +74,21 @@ config :microwaveprop, Oban, enqueue: 1, gefs: 2, terrain: 4, - # RoverPathProfileWorker runs the same full HRRR/terrain/score - # /loss/forecast pipeline as the propagation grid chain. Kept on - # its own queue at 1 slot/pod so a backfill flood can't stack - # multiple heavy path-computes in one BEAM heap (the OOM that - # took down hot pods on 2026-05-03). + # DO NOT raise above 1 without measuring per-pod RSS first. + # RoverPathProfileWorker runs the full PathCompute pipeline + # (terrain sample analysis + 9 HRRR profile fetches/decodes + + # sounding readout + ionosphere readout + per-band scoring + + # loss + power + 18 h forecast). Per-job heap is roughly the + # same shape as the :propagation grid chain — ~500 MiB peak. + # With the deployment.yaml memory limit at 6 GiB and steady-state + # hot pods sitting at 3-5 GiB (ScoreCache + GridCache + LiveView + # state), even 2 concurrent path-computes can push a hot pod past + # the limit and trigger OOMKill. Verified on 2026-05-03: a + # backfill flood ran 3 slots/pod × 4 hot pods on the :terrain + # queue and OOMKilled two pods in a CrashLoopBackOff cycle. + # If you need more throughput, prefer adding hot replicas (HPA + # max is 4 in k8s/hpa.yaml) over raising this slot count, OR + # offload the HRRR profile fetch to hrrr-point-rs first. rover_path: 1, commercial: 2, iemre: 10,