diff --git a/config/runtime.exs b/config/runtime.exs index 9dcd92c5..2fab9561 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -168,11 +168,17 @@ if config_env() == :prod do iemre: 3, era5: 2, # No global_limit: the Pro tracker leaked phantom slots across deploys - # and stalled the whole backfill. The rate_limit is enough to protect - # CDS and lets the per-pod local_limit scale naturally with replicas. + # and stalled the whole backfill. The rate_limit protects CDS and the + # per-pod local_limit scales naturally with replicas. + # + # Parallelism sizing: each job makes 2 CDS requests *serially* (single- + # level then pressure-level), so each active worker holds 1 CDS slot at + # a time. CDS-Beta's per-user ceiling is ~16 concurrent. local_limit 4 + # × 3 pods = 12 concurrent, leaving headroom. rate_limit 30/hour stays + # above steady-state throughput so it doesn't re-become the bottleneck. era5_batch: [ - local_limit: 2, - rate_limit: [allowed: 10, period: {1, :hour}] + local_limit: 4, + rate_limit: [allowed: 30, period: {1, :hour}] ], rtma: 2, backfill_enqueue: 1, diff --git a/lib/microwaveprop/workers/era5_month_batch_worker.ex b/lib/microwaveprop/workers/era5_month_batch_worker.ex index e33d9437..1e5f1454 100644 --- a/lib/microwaveprop/workers/era5_month_batch_worker.ex +++ b/lib/microwaveprop/workers/era5_month_batch_worker.ex @@ -18,9 +18,9 @@ defmodule Microwaveprop.Workers.Era5MonthBatchWorker do starve the fast `Era5FetchWorker` router that also lives in the ERA5 namespace. """ - # The :era5_batch queue is rate-limited to 10/hour so a single failed run - # eats a meaningful chunk of throughput. Retrying 5×, combined with the - # generous backoff below, gives CDS up to a day to come back without + # The :era5_batch queue is rate-limited (see config/runtime.exs) to stay + # under CDS-Beta's per-user concurrent ceiling. Retrying 5×, combined with + # the generous backoff below, gives CDS up to a day to come back without # discarding the tile and silently dropping the historical contact. use Oban.Worker, queue: :era5_batch,