From ead34990b00229252d8d105e8630de55b3988b38 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Mon, 30 Mar 2026 09:16:07 -0500 Subject: [PATCH] Fix HRRR pipeline: single concurrency, cancel bad GRIB, rescue stuck jobs - Reduce HRRR queue to 1 worker to avoid overwhelming upstream - Add Oban Lifeline plugin to rescue orphaned executing jobs after 30m - Treat GRIB2 decode errors (malformed section, missing sections, etc.) as permanent failures instead of retrying indefinitely --- config/config.exs | 3 ++- lib/microwaveprop/workers/hrrr_fetch_worker.ex | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/config/config.exs b/config/config.exs index d4302683..6abf6172 100644 --- a/config/config.exs +++ b/config/config.exs @@ -44,9 +44,10 @@ config :microwaveprop, MicrowavepropWeb.Endpoint, config :microwaveprop, Oban, repo: Microwaveprop.Repo, - queues: [solar: 1, weather: 3, enqueue: 1, hrrr: 2, terrain: 1], + queues: [solar: 1, weather: 3, enqueue: 1, hrrr: 1, terrain: 1], plugins: [ {Oban.Plugins.Pruner, max_age: 3600 * 24}, + {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 30)}, {Oban.Plugins.Cron, crontab: [ {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, diff --git a/lib/microwaveprop/workers/hrrr_fetch_worker.ex b/lib/microwaveprop/workers/hrrr_fetch_worker.ex index 1b5bcb35..4f855d80 100644 --- a/lib/microwaveprop/workers/hrrr_fetch_worker.ex +++ b/lib/microwaveprop/workers/hrrr_fetch_worker.ex @@ -56,10 +56,15 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do end end - # 404 means the HRRR data doesn't exist (pre-2014, etc.) — will never succeed defp permanent_failure?("HRRR idx HTTP 404"), do: true defp permanent_failure?("HRRR grib HTTP 404"), do: true defp permanent_failure?(:outside_grid), do: true + defp permanent_failure?("malformed section"), do: true + defp permanent_failure?("missing sections: " <> _), do: true + defp permanent_failure?("not a GRIB2 message"), do: true + defp permanent_failure?("unsupported GRIB edition " <> _), do: true + defp permanent_failure?("GRIB2 extraction failed: " <> _), do: true + defp permanent_failure?("GRIB2 complex packing decode failed: " <> _), do: true defp permanent_failure?(_), do: false defp maybe_add_derived_params(attrs, nil), do: attrs