diff --git a/config/config.exs b/config/config.exs index 50950ff4..ec74fb73 100644 --- a/config/config.exs +++ b/config/config.exs @@ -74,11 +74,21 @@ config :microwaveprop, Oban, enqueue: 1, gefs: 2, terrain: 4, - # RoverPathProfileWorker runs the same full HRRR/terrain/score - # /loss/forecast pipeline as the propagation grid chain. Kept on - # its own queue at 1 slot/pod so a backfill flood can't stack - # multiple heavy path-computes in one BEAM heap (the OOM that - # took down hot pods on 2026-05-03). + # DO NOT raise above 1 without measuring per-pod RSS first. + # RoverPathProfileWorker runs the full PathCompute pipeline + # (terrain sample analysis + 9 HRRR profile fetches/decodes + + # sounding readout + ionosphere readout + per-band scoring + + # loss + power + 18 h forecast). Per-job heap is roughly the + # same shape as the :propagation grid chain — ~500 MiB peak. + # With the deployment.yaml memory limit at 6 GiB and steady-state + # hot pods sitting at 3-5 GiB (ScoreCache + GridCache + LiveView + # state), even 2 concurrent path-computes can push a hot pod past + # the limit and trigger OOMKill. Verified on 2026-05-03: a + # backfill flood ran 3 slots/pod × 4 hot pods on the :terrain + # queue and OOMKilled two pods in a CrashLoopBackOff cycle. + # If you need more throughput, prefer adding hot replicas (HPA + # max is 4 in k8s/hpa.yaml) over raising this slot count, OR + # offload the HRRR profile fetch to hrrr-point-rs first. rover_path: 1, commercial: 2, iemre: 10,