From 485676887a97369f1f3a31bdfe8c1a042d4ff9cc Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 9 Apr 2026 14:12:22 -0500 Subject: [PATCH] Move Era5MonthBatchWorker to its own queue The slow CDS submit/poll/download cycle of the month-batch worker was sharing the era5 queue with the cheap Era5FetchWorker router, and its 2 concurrent slots were permanently pinned by long-running batch jobs while hundreds of router jobs starved. Give the batch worker its own era5_batch queue (also 2 concurrent per pod) so the router has dedicated capacity. --- config/runtime.exs | 1 + lib/microwaveprop/workers/era5_month_batch_worker.ex | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/config/runtime.exs b/config/runtime.exs index 4575cdef..da797fdf 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -141,6 +141,7 @@ if config_env() == :prod do terrain: 3, iemre: 3, era5: 2, + era5_batch: 2, rtma: 2, backfill_enqueue: 1 ], diff --git a/lib/microwaveprop/workers/era5_month_batch_worker.ex b/lib/microwaveprop/workers/era5_month_batch_worker.ex index 910928e7..2533b796 100644 --- a/lib/microwaveprop/workers/era5_month_batch_worker.ex +++ b/lib/microwaveprop/workers/era5_month_batch_worker.ex @@ -13,9 +13,13 @@ defmodule Microwaveprop.Workers.Era5MonthBatchWorker do Uniqueness on the full arg set means Oban collapses duplicate enqueues for the same tile-month — multiple `Era5FetchWorker` requests for the same region collapse into a single batch. + + Runs on its own `:era5_batch` queue so its slow CDS poll cycles don't + starve the fast `Era5FetchWorker` router that also lives in the ERA5 + namespace. """ use Oban.Worker, - queue: :era5, + queue: :era5_batch, max_attempts: 3, unique: [ period: :infinity,