fix(enrichment): stub IEMRE on permanent failure + bump hot CPU

IemreFetchWorker now writes an empty stub observation when IEM returns
a permanent-failure status (404, 422), mirroring the existing {:ok, []}
path. The backfill cron's next tick sees the stub via
has_iemre_observation?/3, generates no job, and mark_status!/3 flips
the contact's iemre_status from :queued to :complete — draining the
51 contacts that have been stuck behind cancelled out-of-grid IEMRE
jobs.

Raise hot pod CPU limit 2 → 3 so BEAM gets 3 schedulers online.
Observed run queues of [2, 10, 0, 0, 0, 0] on the 2-scheduler
configuration during the :05 propagation chain, starving /live and
/health probe handlers and tripping intermittent liveness timeouts.

k8s/deployment.yaml

@@ -120,7 +120,12 @@ spec:
               cpu: 100m
               memory: 512Mi
             limits:
-              cpu: "2"
+              # Bumped 2 → 3 on 2026-04-23: BEAM was getting 2 schedulers
+              # online on a 2-core limit, and the :05 propagation chain
+              # plus ScoreCache warming spiked the second run queue to
+              # ~10, starving /live and /health probe handlers and
+              # tripping liveness/readiness timeouts.
+              cpu: "3"
               memory: 6Gi
           securityContext:
             allowPrivilegeEscalation: false

lib/microwaveprop/workers/iemre_fetch_worker.ex

@@ -50,16 +50,23 @@ defmodule Microwaveprop.Workers.IemreFetchWorker do
         :ok
 
       {:error, reason} ->
-        handle_error(reason, lat, lon, date_str)
+        handle_error(reason, lat, lon, date, date_str)
     end
   end
 
-  defp handle_error(reason, lat, lon, date_str) do
+  defp handle_error(reason, lat, lon, date, date_str) do
     if transient_failure?(reason) do
       Logger.error("IEMRE transient error for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}")
       {:error, reason}
     else
-      Logger.warning("IEMRE permanent failure for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}")
+      # Permanent upstream failure (e.g. 404, 422 out-of-grid). Stub the
+      # bucket so future backfill enqueues skip the fetch and let
+      # ContactWeatherEnqueueWorker reconcile the contacts' :queued →
+      # :complete transition via the empty-jobs branch.
+      _ = Weather.upsert_iemre_observation(%{lat: lat, lon: lon, date: date, hourly: []})
+
+      Logger.warning("IEMRE permanent failure for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}, stored stub")
+
       {:cancel, reason}
     end
   end

test/microwaveprop/workers/iemre_fetch_worker_test.exs

@@ -98,7 +98,7 @@ defmodule Microwaveprop.Workers.IemreFetchWorkerTest do
       assert {:error, "IEM IEMRE HTTP 503"} = IemreFetchWorker.perform(job)
     end
 
-    test "cancels on permanent failure (404)" do
+    test "cancels on permanent failure (404) and stores stub so backfill reconciles" do
       Req.Test.stub(IemClient, fn conn ->
         Plug.Conn.send_resp(conn, 404, "not found")
       end)
@@ -112,6 +112,33 @@ defmodule Microwaveprop.Workers.IemreFetchWorkerTest do
       }
 
       assert {:cancel, "IEM IEMRE HTTP 404"} = IemreFetchWorker.perform(job)
+      assert Repo.aggregate(IemreObservation, :count) == 1
+      obs = Repo.one!(IemreObservation)
+      assert obs.hourly == []
+      assert obs.lat == 32.875
+      assert obs.lon == -97.0
+    end
+
+    test "cancels on permanent failure (422 out-of-grid) and stores stub" do
+      Req.Test.stub(IemClient, fn conn ->
+        Plug.Conn.send_resp(conn, 422, "out of grid")
+      end)
+
+      job = %Oban.Job{
+        args: %{
+          "lat" => 60.75,
+          "lon" => -151.25,
+          "date" => "2019-08-03"
+        }
+      }
+
+      assert {:cancel, "IEM IEMRE HTTP 422"} = IemreFetchWorker.perform(job)
+      assert Repo.aggregate(IemreObservation, :count) == 1
+      obs = Repo.one!(IemreObservation)
+      assert obs.hourly == []
+      assert obs.lat == 60.75
+      assert obs.lon == -151.25
+      assert obs.date == ~D[2019-08-03]
     end
   end