From 66fef4f6da98e98d06d8a2b6460ba981ba923abe Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Thu, 23 Apr 2026 10:53:21 -0500
Subject: [PATCH] perf(oban): move IEM-heavy queues off hot pods
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Hot pods were running weather/narr/rtma/nexrad/contact_import
alongside LiveView and the hourly propagation chain. The weather
queue in particular hammers IEM with ASOS backfills; each Req
retry against a 429 stacks an in-worker Process.sleep, and enough
concurrent retries saturated the Ecto pool. Once DB checkouts
started queueing past 15 s the cascade was always the same:
Postgrex client timeouts → Oban.Notifier/Peer 5 s health checks
failing → /health readiness timeout → kubelet flipping the pod
out of the service endpoint set. One hot replica accumulated 11
restarts over 16 h.

Split the shared queue block: anything that's truly per-QSO
enrichment on demand (terrain, iemre, radar, mechanism, gefs,
ionosphere, space_weather, admin, exports, backfill_enqueue) stays
on both tiers, while weather/narr/rtma/nexrad/contact_import run
only on prop-backfill. The hot pods mark those as paused so the
queue entries still exist cluster-wide (Oban Pro's Smart engine
rejects `limit: 0`). Cluster capacity for the backfill queues is
unchanged; they just execute on the dedicated t620.
---
 config/runtime.exs | 46 +++++++++++++++++++++++++++++++---------------
 1 file changed, 31 insertions(+), 15 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index 17fd6ff6..a8f29c5d 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -167,16 +167,33 @@ if config_env() == :prod do
   ]
 
   shared_queues = [
-    # 1 slot per pod (3 cluster-wide). Any higher than this and the
-    # ASOS backfill hammers IEM hard enough to get HTTP 429s, which
-    # fill the retryable queue and thrash pod CPU without making
-    # forward progress.
-    weather: 1,
-    gefs: 1,
     # :hrrr queue removed — per-QSO HRRR fetches flow through
     # hrrr_fetch_tasks and the Rust hrrr-point-worker (Phase 3 Stream C).
     terrain: 3,
     iemre: 3,
+    backfill_enqueue: 1,
+    admin: 1,
+    radar: 2,
+    ionosphere: 1,
+    space_weather: 1,
+    mechanism: 4,
+    exports: 1,
+    gefs: 1
+  ]
+
+  # Queues that are only allowed to run on the dedicated backfill pod.
+  # Historical backfill (weather/narr/rtma/nexrad/contact_import) spins
+  # against rate-limited upstreams (IEM especially) and stacks Req
+  # exponential-backoff sleeps under its workers; running those slots
+  # on the hot pods was thrashing the DB pool and tripping the /health
+  # readiness probe. Keeping them on prop-backfill alone lets the hot
+  # pods serve LiveView + cron reliably while still processing
+  # enrichment cluster-wide.
+  backfill_only_queues = [
+    # 1 slot — see earlier note: any higher and the ASOS backfill
+    # hammers IEM hard enough to get HTTP 429s, which fill the
+    # retryable queue and thrash pod CPU without forward progress.
+    weather: 1,
     # Historical backfill for pre-2014 contacts (pre-HRRR archive).
     # NARR is fetched anonymously from NCEI with no quota, no job
     # queue. Replaced the ERA5/CDS pipeline which never produced a
@@ -185,14 +202,7 @@ if config_env() == :prod do
     # docs/plans/2026-04-15-merra2-historical-backfill.md.
     narr: 6,
     rtma: 2,
-    backfill_enqueue: 1,
-    admin: 1,
     nexrad: 2,
-    radar: 2,
-    ionosphere: 1,
-    space_weather: 1,
-    mechanism: 4,
-    exports: 1,
     contact_import: 4
   ]
 
@@ -203,6 +213,12 @@ if config_env() == :prod do
   # each queue with `paused: true` (queue exists, processes nothing).
   paused_queue = [local_limit: 1, paused: true]
 
+  # Pause the backfill-only queues on hot pods so the queue exists
+  # (base config in config/config.exs declares it) but processes
+  # nothing here. Same `:paused` trick as disabled_hot_queues below.
+  paused_backfill_queues =
+    Enum.map(backfill_only_queues, fn {queue, _} -> {queue, paused_queue} end)
+
   disabled_hot_queues = [
     propagation: paused_queue,
     commercial: paused_queue,
@@ -212,8 +228,8 @@ if config_env() == :prod do
 
   oban_queues =
     case prop_role do
-      "backfill" -> disabled_hot_queues ++ shared_queues
-      _ -> hot_only_queues ++ shared_queues
+      "backfill" -> disabled_hot_queues ++ shared_queues ++ backfill_only_queues
+      _ -> hot_only_queues ++ shared_queues ++ paused_backfill_queues
     end
 
   cron_plugin =