diff --git a/config/config.exs b/config/config.exs index c3912f2d..91b10a2c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -71,8 +71,9 @@ config :microwaveprop, Oban, ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, - # See runtime.exs — must exceed the worker's timeout/1 callback. - {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 45)}, + # See runtime.exs — producer-record-based orphan rescue, not + # timer-based. Fast recovery from rolling deploys. + {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)}, {Oban.Plugins.Cron, crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, diff --git a/config/dev.exs b/config/dev.exs index 4f164d76..f74c0a2b 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -86,8 +86,8 @@ config :microwaveprop, Oban, ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, - # See runtime.exs — must exceed the worker's timeout/1 callback. - {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 45)}, + # See runtime.exs — producer-record-based orphan rescue. + {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)}, {Oban.Plugins.Cron, crontab: [ {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}, diff --git a/config/runtime.exs b/config/runtime.exs index 7fe2e5c8..e5a3f83a 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -199,14 +199,15 @@ if config_env() == :prod do ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, - # Lifeline rescues jobs stuck in `executing` after this window. - # MUST be larger than the longest `timeout/1` callback on any worker - # or Lifeline races the job's own deadline. The chain-style - # PropagationGridWorker caps a single forecast-hour step at 20 - # min and real steps run ~8-10 min, so 45 min gives comfortable - # headroom for a slow step without letting a truly stuck job - # linger for hours before the safety net trips. - {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 45)}, + # DynamicLifeline uses producer records (heartbeats from each + # live node) to rescue orphans, not a timer. When a pod dies + # mid-deploy its Producer row disappears and any in-flight job + # with that producer in `attempted_by` gets moved back to + # `available` on the next rescue cycle — within ~30s instead of + # waiting for a timer-based `rescue_after` to expire. Critical + # for PropagationGridWorker chain steps to recover from rolling + # deploys before the next hourly cron fire. + {Oban.Pro.Plugins.DynamicLifeline, rescue_interval: to_timeout(second: 30)}, {Oban.Plugins.Cron, crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, diff --git a/lib/microwaveprop/workers/propagation_grid_worker.ex b/lib/microwaveprop/workers/propagation_grid_worker.ex index 40fe293b..87f92ab9 100644 --- a/lib/microwaveprop/workers/propagation_grid_worker.ex +++ b/lib/microwaveprop/workers/propagation_grid_worker.ex @@ -20,7 +20,11 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do use Oban.Worker, queue: :propagation, - max_attempts: 3 + # Higher than the default 3 so a few DynamicLifeline rescues + # (e.g., a rolling deploy that kills a mid-flight chain step) + # don't exhaust the chain's retry budget and discard the whole + # run. Legitimate scoring errors still give up after 5 attempts. + max_attempts: 5 alias Microwaveprop.Commercial alias Microwaveprop.Propagation