From 47ba8d3b6d837201173641857a80490a79339018 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Thu, 23 Apr 2026 12:26:02 -0500
Subject: [PATCH] fix(enrichment): stub IEMRE on permanent failure + bump hot
 CPU
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

IemreFetchWorker now writes an empty stub observation when IEM returns
a permanent-failure status (404, 422), mirroring the existing {:ok, []}
path. The backfill cron's next tick sees the stub via
has_iemre_observation?/3, generates no job, and mark_status!/3 flips
the contact's iemre_status from :queued to :complete — draining the
51 contacts that have been stuck behind cancelled out-of-grid IEMRE
jobs.

Raise hot pod CPU limit 2 → 3 so BEAM gets 3 schedulers online.
Observed run queues of [2, 10, 0, 0, 0, 0] on the 2-scheduler
configuration during the :05 propagation chain, starving /live and
/health probe handlers and tripping intermittent liveness timeouts.
---
 k8s/deployment.yaml                           |  7 ++++-
 .../workers/iemre_fetch_worker.ex             | 13 +++++++--
 .../workers/iemre_fetch_worker_test.exs       | 29 ++++++++++++++++++-
 3 files changed, 44 insertions(+), 5 deletions(-)

diff --git a/k8s/deployment.yaml b/k8s/deployment.yaml
index 8533395f..b8e4f75d 100644
--- a/k8s/deployment.yaml
+++ b/k8s/deployment.yaml
@@ -120,7 +120,12 @@ spec:
               cpu: 100m
               memory: 512Mi
             limits:
-              cpu: "2"
+              # Bumped 2 → 3 on 2026-04-23: BEAM was getting 2 schedulers
+              # online on a 2-core limit, and the :05 propagation chain
+              # plus ScoreCache warming spiked the second run queue to
+              # ~10, starving /live and /health probe handlers and
+              # tripping liveness/readiness timeouts.
+              cpu: "3"
               memory: 6Gi
           securityContext:
             allowPrivilegeEscalation: false
diff --git a/lib/microwaveprop/workers/iemre_fetch_worker.ex b/lib/microwaveprop/workers/iemre_fetch_worker.ex
index 9412797c..111e3756 100644
--- a/lib/microwaveprop/workers/iemre_fetch_worker.ex
+++ b/lib/microwaveprop/workers/iemre_fetch_worker.ex
@@ -50,16 +50,23 @@ defmodule Microwaveprop.Workers.IemreFetchWorker do
         :ok
 
       {:error, reason} ->
-        handle_error(reason, lat, lon, date_str)
+        handle_error(reason, lat, lon, date, date_str)
     end
   end
 
-  defp handle_error(reason, lat, lon, date_str) do
+  defp handle_error(reason, lat, lon, date, date_str) do
     if transient_failure?(reason) do
       Logger.error("IEMRE transient error for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}")
       {:error, reason}
     else
-      Logger.warning("IEMRE permanent failure for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}")
+      # Permanent upstream failure (e.g. 404, 422 out-of-grid). Stub the
+      # bucket so future backfill enqueues skip the fetch and let
+      # ContactWeatherEnqueueWorker reconcile the contacts' :queued →
+      # :complete transition via the empty-jobs branch.
+      _ = Weather.upsert_iemre_observation(%{lat: lat, lon: lon, date: date, hourly: []})
+
+      Logger.warning("IEMRE permanent failure for #{lat},#{lon} @ #{date_str}: #{inspect(reason)}, stored stub")
+
       {:cancel, reason}
     end
   end
diff --git a/test/microwaveprop/workers/iemre_fetch_worker_test.exs b/test/microwaveprop/workers/iemre_fetch_worker_test.exs
index 0de0d747..1c2d9063 100644
--- a/test/microwaveprop/workers/iemre_fetch_worker_test.exs
+++ b/test/microwaveprop/workers/iemre_fetch_worker_test.exs
@@ -98,7 +98,7 @@ defmodule Microwaveprop.Workers.IemreFetchWorkerTest do
       assert {:error, "IEM IEMRE HTTP 503"} = IemreFetchWorker.perform(job)
     end
 
-    test "cancels on permanent failure (404)" do
+    test "cancels on permanent failure (404) and stores stub so backfill reconciles" do
       Req.Test.stub(IemClient, fn conn ->
         Plug.Conn.send_resp(conn, 404, "not found")
       end)
@@ -112,6 +112,33 @@ defmodule Microwaveprop.Workers.IemreFetchWorkerTest do
       }
 
       assert {:cancel, "IEM IEMRE HTTP 404"} = IemreFetchWorker.perform(job)
+      assert Repo.aggregate(IemreObservation, :count) == 1
+      obs = Repo.one!(IemreObservation)
+      assert obs.hourly == []
+      assert obs.lat == 32.875
+      assert obs.lon == -97.0
+    end
+
+    test "cancels on permanent failure (422 out-of-grid) and stores stub" do
+      Req.Test.stub(IemClient, fn conn ->
+        Plug.Conn.send_resp(conn, 422, "out of grid")
+      end)
+
+      job = %Oban.Job{
+        args: %{
+          "lat" => 60.75,
+          "lon" => -151.25,
+          "date" => "2019-08-03"
+        }
+      }
+
+      assert {:cancel, "IEM IEMRE HTTP 422"} = IemreFetchWorker.perform(job)
+      assert Repo.aggregate(IemreObservation, :count) == 1
+      obs = Repo.one!(IemreObservation)
+      assert obs.hourly == []
+      assert obs.lat == 60.75
+      assert obs.lon == -151.25
+      assert obs.date == ~D[2019-08-03]
     end
   end