From fb5b1f06f82829b13622caa0607455135a8286b7 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sun, 19 Apr 2026 14:36:03 -0500
Subject: [PATCH] fix(infra): actually drop hot queues + cron on backfill pods

config.exs registers Oban queues + the Cron plugin at compile time,
and Config deep-merges with runtime.exs by key. Simply omitting
propagation/commercial/solar from runtime.exs left them inheriting
from config.exs, so backfill pods were still processing hot-path
queues and holding the Cron plugin. Override the hot queues to
limit: 0 and re-register Cron with an empty crontab in backfill
mode so the compile-time defaults get neutralised.
---
 config/runtime.exs | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/config/runtime.exs b/config/runtime.exs
index 9954ae9e..ea2844dd 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -17,6 +17,7 @@ import Config
 # Alternatively, you can use `mix phx.gen.release` to generate a `bin/server`
 # script that automatically sets the env var above.
 alias Microwaveprop.Workers.CanadianSoundingFetchWorker
+alias Oban.Plugins.Cron
 
 if System.get_env("PHX_SERVER") do
   config :microwaveprop, MicrowavepropWeb.Endpoint, server: true
@@ -164,14 +165,21 @@ if config_env() == :prod do
     contact_import: 4
   ]
 
+  # config/config.exs sets a base Oban config (queues + cron) that Config
+  # deep-merges with runtime.exs. To actually drop the hot-path queues on
+  # backfill pods we have to explicitly override them to `limit: 0` rather
+  # than just leaving them out — Config deep merge by key means omitted
+  # values fall through to the compile-time defaults.
+  disabled_hot_queues = [propagation: 0, commercial: 0, solar: 0, enqueue: 0]
+
   oban_queues =
     case prop_role do
-      "backfill" -> shared_queues
+      "backfill" -> disabled_hot_queues ++ shared_queues
       _ -> hot_only_queues ++ shared_queues
     end
 
   cron_plugin =
-    {Oban.Plugins.Cron,
+    {Cron,
      crontab: [
        {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
        {"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
@@ -234,13 +242,16 @@ if config_env() == :prod do
     {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)}
   ]
 
-  # Backfill pods skip the cron plugin entirely — no risk of a
-  # backfill node becoming cron leader and enqueueing work targeted
-  # at queues it doesn't even run (propagation, commercial). Hot pods
-  # own scheduling; backfill pods just drain the queues.
+  # Backfill pods must not fire the hourly cron. Config deep-merge by
+  # module key means we can't just omit the Cron plugin — config.exs
+  # registers it and that registration survives. Override it with an
+  # empty crontab on backfill pods so the plugin exists but schedules
+  # nothing.
+  empty_cron_plugin = {Cron, crontab: []}
+
   oban_plugins =
     case prop_role do
-      "backfill" -> base_plugins
+      "backfill" -> base_plugins ++ [empty_cron_plugin]
       _ -> base_plugins ++ [cron_plugin]
     end