diff --git a/config/config.exs b/config/config.exs index 716dfdce..aac35991 100644 --- a/config/config.exs +++ b/config/config.exs @@ -69,7 +69,8 @@ config :microwaveprop, Oban, ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, - {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 30)}, + # See runtime.exs — must exceed PropagationGridWorker's 90-min timeout. + {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 120)}, {Oban.Plugins.Cron, crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, diff --git a/config/dev.exs b/config/dev.exs index ce9724ef..de8779ec 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -84,7 +84,8 @@ config :microwaveprop, Oban, ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, - {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 30)}, + # See runtime.exs — must exceed PropagationGridWorker's 90-min timeout. + {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 120)}, {Oban.Plugins.Cron, crontab: [ {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker}, diff --git a/config/runtime.exs b/config/runtime.exs index 900df5b7..e534bcbc 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -197,7 +197,16 @@ if config_env() == :prod do ], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, - {Oban.Plugins.Lifeline, rescue_after: 10 * 60 * 1000}, + # Lifeline rescues jobs stuck in `executing` after this window. + # MUST be larger than the longest `timeout/1` callback on any worker + # or Lifeline races the job's own deadline: PropagationGridWorker + # caps itself at 90 min and a real run is ~95 min of wall time, so + # 120 min gives enough headroom for the hourly sweep to finish + # before the safety net trips. Setting it shorter caused the grid + # job to be rescued mid-run and retried from f00, which discarded + # with empty errors after 3 rescues and left only 4-5 valid_times + # in propagation_scores. + {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 120)}, {Oban.Plugins.Cron, crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, @@ -206,6 +215,11 @@ if config_env() == :prod do # Hourly cron caused overlapping/failed runs because one full sweep # exceeds a 60-minute window. {"5 */3 * * *", Microwaveprop.Workers.PropagationGridWorker}, + # MRMS refreshes the PrecipRate cache that AsosAdjustmentWorker + # overlays onto the score grid. Both must run together — an ASOS + # nudge without a fresh MRMS frame reverts to HRRR-only rain. + {"*/2 * * * *", Microwaveprop.Workers.MrmsFetchWorker}, + {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker, args: %{"limit" => 500, "types" => ["hrrr", "weather", "terrain", "iemre"]}},