From 662d7232e520a5e940b0d1ea4d7fc1b5f0c535d3 Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Wed, 15 Apr 2026 13:02:50 -0500
Subject: [PATCH] ERA5 poll worker: snooze on transient HTTP errors instead of
 discarding
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A brief CDS network blip on 2026-04-15 returned HTTP 5xx from check_status
for multiple concurrently-polling workers. The old `{:error, _}` branches
consumed attempts, max_attempts=10 burned out in seconds, and 13 rows
were discarded with no cleanup — orphaned with no active poll worker,
including two rows that had both CDS legs done and were ready to decode.
Result: zero era5_profiles ever inserted despite 59 tile-months in flight.

Transient HTTP errors are now a `{:snooze, 60}` so a CDS outage pauses
polling instead of killing it. Terminal states (:failed, :rejected,
:not_found, :stuck) still drive the fail/resubmit paths.
---
 lib/microwaveprop/workers/era5_poll_worker.ex | 12 ++++-
 .../workers/era5_poll_worker_test.exs         | 53 +++++++++++++++++++
 2 files changed, 63 insertions(+), 2 deletions(-)

diff --git a/lib/microwaveprop/workers/era5_poll_worker.ex b/lib/microwaveprop/workers/era5_poll_worker.ex
index fcca9246..75a4a2fb 100644
--- a/lib/microwaveprop/workers/era5_poll_worker.ex
+++ b/lib/microwaveprop/workers/era5_poll_worker.ex
@@ -32,6 +32,12 @@ defmodule Microwaveprop.Workers.Era5PollWorker do
   # so 5 minutes strikes a balance between responsiveness and noise.
   @snooze_seconds 5 * 60
 
+  # Snooze delay after a transient CDS HTTP error (5xx, network blip).
+  # Shorter than the normal poll interval so recovery is fast, but a
+  # snooze instead of an attempt so a sustained outage doesn't burn
+  # through max_attempts and orphan the row.
+  @error_snooze_seconds 60
+
   # A CDS job that's been in 'accepted' state for more than this is stuck
   # in the server's queue and almost certainly won't run. Observed in
   # prod: pressure-level completes in ~90 min, but single-level
@@ -102,10 +108,12 @@ defmodule Microwaveprop.Workers.Era5PollWorker do
         fail_row(row, "pressure-level job failed: #{reason}")
 
       {{:error, reason}, _} ->
-        {:error, "single-level check_status error: #{reason}"}
+        Logger.warning("Era5Poll: transient single-level check_status error (snoozing): #{reason}")
+        {:snooze, @error_snooze_seconds}
 
       {_, {:error, reason}} ->
-        {:error, "pressure-level check_status error: #{reason}"}
+        Logger.warning("Era5Poll: transient pressure-level check_status error (snoozing): #{reason}")
+        {:snooze, @error_snooze_seconds}
 
       _both_running_or_one_done ->
         bump_poll_count(row)
diff --git a/test/microwaveprop/workers/era5_poll_worker_test.exs b/test/microwaveprop/workers/era5_poll_worker_test.exs
index 716211b3..6c31171e 100644
--- a/test/microwaveprop/workers/era5_poll_worker_test.exs
+++ b/test/microwaveprop/workers/era5_poll_worker_test.exs
@@ -247,6 +247,59 @@ defmodule Microwaveprop.Workers.Era5PollWorkerTest do
     end
   end
 
+  describe "perform/1 — transient CDS HTTP error" do
+    # Observed in prod 2026-04-15: a brief CDS network blip returned HTTP
+    # 5xx on check_status. The old {:error, _} branches consumed attempts
+    # and max_attempts=10 burned out in seconds, so 13 rows became
+    # orphaned (discarded with no cleanup, no active poll worker). The
+    # fix: treat HTTP errors as transient — snooze without counting an
+    # attempt. Terminal states (:failed, :rejected, :not_found) still
+    # drive the fail/resubmit paths.
+    test "snoozes and keeps the row when single-level check_status returns a transient HTTP error" do
+      row = insert_cds_job()
+
+      Req.Test.stub(Era5Client, fn conn ->
+        cond do
+          conn.method == "GET" and String.contains?(conn.request_path, "cds-single-123") ->
+            conn
+            |> Plug.Conn.put_resp_content_type("application/json")
+            |> Plug.Conn.resp(503, ~s({"error":"service unavailable"}))
+
+          conn.method == "GET" and String.contains?(conn.request_path, "cds-pressure-456") ->
+            Req.Test.json(conn, %{"status" => "running"})
+        end
+      end)
+
+      assert {:snooze, snooze_s} =
+               Era5PollWorker.perform(%Oban.Job{args: %{"era5_cds_job_id" => row.id}})
+
+      assert snooze_s > 0
+      assert Repo.get(Era5CdsJob, row.id)
+    end
+
+    test "snoozes and keeps the row when pressure-level check_status returns a transient HTTP error" do
+      row = insert_cds_job()
+
+      Req.Test.stub(Era5Client, fn conn ->
+        cond do
+          conn.method == "GET" and String.contains?(conn.request_path, "cds-single-123") ->
+            Req.Test.json(conn, %{"status" => "running"})
+
+          conn.method == "GET" and String.contains?(conn.request_path, "cds-pressure-456") ->
+            conn
+            |> Plug.Conn.put_resp_content_type("application/json")
+            |> Plug.Conn.resp(502, ~s({"error":"bad gateway"}))
+        end
+      end)
+
+      assert {:snooze, snooze_s} =
+               Era5PollWorker.perform(%Oban.Job{args: %{"era5_cds_job_id" => row.id}})
+
+      assert snooze_s > 0
+      assert Repo.get(Era5CdsJob, row.id)
+    end
+  end
+
   describe "perform/1 — failure" do
     test "returns {:error, _}, deletes the row, and deletes both CDS jobs when a leg fails" do
       row = insert_cds_job()