diff --git a/config/runtime.exs b/config/runtime.exs index 17fd6ff6..a8f29c5d 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -167,16 +167,33 @@ if config_env() == :prod do ] shared_queues = [ - # 1 slot per pod (3 cluster-wide). Any higher than this and the - # ASOS backfill hammers IEM hard enough to get HTTP 429s, which - # fill the retryable queue and thrash pod CPU without making - # forward progress. - weather: 1, - gefs: 1, # :hrrr queue removed — per-QSO HRRR fetches flow through # hrrr_fetch_tasks and the Rust hrrr-point-worker (Phase 3 Stream C). terrain: 3, iemre: 3, + backfill_enqueue: 1, + admin: 1, + radar: 2, + ionosphere: 1, + space_weather: 1, + mechanism: 4, + exports: 1, + gefs: 1 + ] + + # Queues that are only allowed to run on the dedicated backfill pod. + # Historical backfill (weather/narr/rtma/nexrad/contact_import) spins + # against rate-limited upstreams (IEM especially) and stacks Req + # exponential-backoff sleeps under its workers; running those slots + # on the hot pods was thrashing the DB pool and tripping the /health + # readiness probe. Keeping them on prop-backfill alone lets the hot + # pods serve LiveView + cron reliably while still processing + # enrichment cluster-wide. + backfill_only_queues = [ + # 1 slot — see earlier note: any higher and the ASOS backfill + # hammers IEM hard enough to get HTTP 429s, which fill the + # retryable queue and thrash pod CPU without forward progress. + weather: 1, # Historical backfill for pre-2014 contacts (pre-HRRR archive). # NARR is fetched anonymously from NCEI with no quota, no job # queue. Replaced the ERA5/CDS pipeline which never produced a @@ -185,14 +202,7 @@ if config_env() == :prod do # docs/plans/2026-04-15-merra2-historical-backfill.md. narr: 6, rtma: 2, - backfill_enqueue: 1, - admin: 1, nexrad: 2, - radar: 2, - ionosphere: 1, - space_weather: 1, - mechanism: 4, - exports: 1, contact_import: 4 ] @@ -203,6 +213,12 @@ if config_env() == :prod do # each queue with `paused: true` (queue exists, processes nothing). paused_queue = [local_limit: 1, paused: true] + # Pause the backfill-only queues on hot pods so the queue exists + # (base config in config/config.exs declares it) but processes + # nothing here. Same `:paused` trick as disabled_hot_queues below. + paused_backfill_queues = + Enum.map(backfill_only_queues, fn {queue, _} -> {queue, paused_queue} end) + disabled_hot_queues = [ propagation: paused_queue, commercial: paused_queue, @@ -212,8 +228,8 @@ if config_env() == :prod do oban_queues = case prop_role do - "backfill" -> disabled_hot_queues ++ shared_queues - _ -> hot_only_queues ++ shared_queues + "backfill" -> disabled_hot_queues ++ shared_queues ++ backfill_only_queues + _ -> hot_only_queues ++ shared_queues ++ paused_backfill_queues end cron_plugin =