From 2abe66c0b449ce081c9dbecbe442f62da0c556fb Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Tue, 14 Apr 2026 10:01:49 -0500
Subject: [PATCH] Unbreak propagation grid runs in prod
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Oban's Lifeline plugin was rescuing PropagationGridWorker from
executing back to retryable every 10 minutes (runtime.exs) or
30 minutes (config.exs / dev.exs), but one full f00–f18 sweep
takes ~95 minutes and the worker's own timeout/1 is already
90 minutes. The rescue fired mid-run, the retry restarted at
f00, and after 3 rescues the job was discarded with errors: []
(Lifeline rescues don't write an error summary). prod's
propagation_scores never advanced beyond ~5 valid_times, so
the forecast timeline on /map only showed 3–4 hours instead of
the intended ~17.

Raise rescue_after to 120 minutes in all three configs — past
the worker's own 90-minute cap with headroom — and document
the contract so the next concurrency tweak doesn't regress it.

Also restore MrmsFetchWorker and AsosAdjustmentWorker to the
prod cron in runtime.exs: runtime.exs replaces config.exs's
plugin list wholesale and those two had been silently dropped
in the last rewrite. AsosAdjustmentWorker depends on the MRMS
precip cache, so both move together.
---
 config/config.exs  |  3 ++-
 config/dev.exs     |  3 ++-
 config/runtime.exs | 16 +++++++++++++++-
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/config/config.exs b/config/config.exs
index 716dfdce..aac35991 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -69,7 +69,8 @@ config :microwaveprop, Oban,
   ],
   plugins: [
     {Oban.Plugins.Pruner, max_age: 3600 * 24},
-    {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 30)},
+    # See runtime.exs — must exceed PropagationGridWorker's 90-min timeout.
+    {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 120)},
     {Oban.Plugins.Cron,
      crontab: [
        {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
diff --git a/config/dev.exs b/config/dev.exs
index ce9724ef..de8779ec 100644
--- a/config/dev.exs
+++ b/config/dev.exs
@@ -84,7 +84,8 @@ config :microwaveprop, Oban,
   ],
   plugins: [
     {Oban.Plugins.Pruner, max_age: 3600 * 24},
-    {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 30)},
+    # See runtime.exs — must exceed PropagationGridWorker's 90-min timeout.
+    {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 120)},
     {Oban.Plugins.Cron,
      crontab: [
        {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker},
diff --git a/config/runtime.exs b/config/runtime.exs
index 900df5b7..e534bcbc 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -197,7 +197,16 @@ if config_env() == :prod do
     ],
     plugins: [
       {Oban.Plugins.Pruner, max_age: 3600 * 24},
-      {Oban.Plugins.Lifeline, rescue_after: 10 * 60 * 1000},
+      # Lifeline rescues jobs stuck in `executing` after this window.
+      # MUST be larger than the longest `timeout/1` callback on any worker
+      # or Lifeline races the job's own deadline: PropagationGridWorker
+      # caps itself at 90 min and a real run is ~95 min of wall time, so
+      # 120 min gives enough headroom for the hourly sweep to finish
+      # before the safety net trips. Setting it shorter caused the grid
+      # job to be rescued mid-run and retried from f00, which discarded
+      # with empty errors after 3 rescues and left only 4-5 valid_times
+      # in propagation_scores.
+      {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 120)},
       {Oban.Plugins.Cron,
        crontab: [
          {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
@@ -206,6 +215,11 @@ if config_env() == :prod do
          # Hourly cron caused overlapping/failed runs because one full sweep
          # exceeds a 60-minute window.
          {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker},
+         # MRMS refreshes the PrecipRate cache that AsosAdjustmentWorker
+         # overlays onto the score grid. Both must run together — an ASOS
+         # nudge without a fresh MRMS frame reverts to HRRR-only rain.
+         {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker},
+         {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
          {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
          {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
           args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}},