From 2c2b0170cc383b02245fca52abc48ed10eb471a6 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 13 Apr 2026 15:28:45 -0500 Subject: [PATCH] =?UTF-8?q?Drop=20era5=5Fbatch=20global=5Flimit=20?= =?UTF-8?q?=E2=80=94=20tracker=20leaks=20across=20deploys?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Pro Smart engine's global_limit tracker is stored inside oban_producers.meta and is not reliably released when a pod exits between job start and completion. After today's rollout two pods died holding tracked slots for running era5_batch jobs; the survivors refused to start any new ones because the cluster-wide tracker was stuck at 4/4 — with zero actual jobs executing — and the whole backfill froze for 2+ hours. Keeping rate_limit (10/hr) is enough to stay polite to Copernicus CDS. local_limit stays at 2 per pod, so effective cluster concurrency just scales with replica count instead of being capped by a global value whose bookkeeping can't survive a restart. --- config/runtime.exs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index a73d4d76..9dcd92c5 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -167,9 +167,11 @@ if config_env() == :prod do terrain: 3, iemre: 3, era5: 2, + # No global_limit: the Pro tracker leaked phantom slots across deploys + # and stalled the whole backfill. The rate_limit is enough to protect + # CDS and lets the per-pod local_limit scale naturally with replicas. era5_batch: [ local_limit: 2, - global_limit: 4, rate_limit: [allowed: 10, period: {1, :hour}] ], rtma: 2,