Fix HRRR pipeline: single concurrency, cancel bad GRIB, rescue stuck jobs

- Reduce HRRR queue to 1 worker to avoid overwhelming upstream
- Add Oban Lifeline plugin to rescue orphaned executing jobs after 30m
- Treat GRIB2 decode errors (malformed section, missing sections, etc.)
  as permanent failures instead of retrying indefinitely

config/config.exs

@@ -44,9 +44,10 @@ config :microwaveprop, MicrowavepropWeb.Endpoint,
 
 config :microwaveprop, Oban,
   repo: Microwaveprop.Repo,
-  queues: [solar: 1, weather: 3, enqueue: 1, hrrr: 2, terrain: 1],
+  queues: [solar: 1, weather: 3, enqueue: 1, hrrr: 1, terrain: 1],
   plugins: [
     {Oban.Plugins.Pruner, max_age: 3600 * 24},
+    {Oban.Plugins.Lifeline, rescue_after: to_timeout(minute: 30)},
     {Oban.Plugins.Cron,
      crontab: [
        {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},

lib/microwaveprop/workers/hrrr_fetch_worker.ex

@@ -56,10 +56,15 @@ defmodule Microwaveprop.Workers.HrrrFetchWorker do
     end
   end
 
-  # 404 means the HRRR data doesn't exist (pre-2014, etc.) — will never succeed
   defp permanent_failure?("HRRR idx HTTP 404"), do: true
   defp permanent_failure?("HRRR grib HTTP 404"), do: true
   defp permanent_failure?(:outside_grid), do: true
+  defp permanent_failure?("malformed section"), do: true
+  defp permanent_failure?("missing sections: " <> _), do: true
+  defp permanent_failure?("not a GRIB2 message"), do: true
+  defp permanent_failure?("unsupported GRIB edition " <> _), do: true
+  defp permanent_failure?("GRIB2 extraction failed: " <> _), do: true
+  defp permanent_failure?("GRIB2 complex packing decode failed: " <> _), do: true
   defp permanent_failure?(_), do: false
 
   defp maybe_add_derived_params(attrs, nil), do: attrs