Switch to DynamicLifeline for ~30s orphan rescue on deploys

Replace the timer-based Oban.Plugins.Lifeline (45 min rescue_after)
with Oban.Pro.Plugins.DynamicLifeline, which watches producer
records and rescues orphaned jobs within one rescue_interval (30s)
after the owning pod disappears. This cuts PropagationGridWorker
chain recovery from up to 55 min (wait for next hourly cron) down
to ~30s after a rolling deploy kills a mid-flight step. Bump the
worker's max_attempts 3 → 5 so a couple of rescues during a deploy
don't exhaust the chain's retry budget.

config/config.exs

@@ -71,8 +71,9 @@ config :microwaveprop, Oban,
   ],
   plugins: [
     {Oban.Plugins.Pruner, max_age: 3600 * 24},
-    # See runtime.exs — must exceed the worker's timeout/1 callback.
-    {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 45)},
+    # See runtime.exs — producer-record-based orphan rescue, not
+    # timer-based. Fast recovery from rolling deploys.
+    {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)},
     {Oban.Plugins.Cron,
      crontab: [
        {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},

config/dev.exs

@@ -86,8 +86,8 @@ config :microwaveprop, Oban,
   ],
   plugins: [
     {Oban.Plugins.Pruner, max_age: 3600 * 24},
-    # See runtime.exs — must exceed the worker's timeout/1 callback.
-    {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 45)},
+    # See runtime.exs — producer-record-based orphan rescue.
+    {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)},
     {Oban.Plugins.Cron,
      crontab: [
        {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},

config/runtime.exs

@@ -199,14 +199,15 @@ if config_env() == :prod do
     ],
     plugins: [
       {Oban.Plugins.Pruner, max_age: 3600 * 24},
-      # Lifeline rescues jobs stuck in `executing` after this window.
-      # MUST be larger than the longest `timeout/1` callback on any worker
-      # or Lifeline races the job's own deadline. The chain-style
-      # PropagationGridWorker caps a single forecast-hour step at 20
-      # min and real steps run ~8-10 min, so 45 min gives comfortable
-      # headroom for a slow step without letting a truly stuck job
-      # linger for hours before the safety net trips.
-      {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 45)},
+      # DynamicLifeline uses producer records (heartbeats from each
+      # live node) to rescue orphans, not a timer. When a pod dies
+      # mid-deploy its Producer row disappears and any in-flight job
+      # with that producer in `attempted_by` gets moved back to
+      # `available` on the next rescue cycle — within ~30s instead of
+      # waiting for a timer-based `rescue_after` to expire. Critical
+      # for PropagationGridWorker chain steps to recover from rolling
+      # deploys before the next hourly cron fire.
+      {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)},
       {Oban.Plugins.Cron,
        crontab: [
          {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},

lib/microwaveprop/workers/propagation_grid_worker.ex

@@ -20,7 +20,11 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
 
   use Oban.Worker,
     queue: :propagation,
-    max_attempts: 3
+    # Higher than the default 3 so a few DynamicLifeline rescues
+    # (e.g., a rolling deploy that kills a mid-flight chain step)
+    # don't exhaust the chain's retry budget and discard the whole
+    # run. Legitimate scoring errors still give up after 5 attempts.
+    max_attempts: 5
 
   alias Microwaveprop.Commercial
   alias Microwaveprop.Propagation