diff --git a/lib/microwaveprop/workers/era5_poll_worker.ex b/lib/microwaveprop/workers/era5_poll_worker.ex index fcca9246..75a4a2fb 100644 --- a/lib/microwaveprop/workers/era5_poll_worker.ex +++ b/lib/microwaveprop/workers/era5_poll_worker.ex @@ -32,6 +32,12 @@ defmodule Microwaveprop.Workers.Era5PollWorker do # so 5 minutes strikes a balance between responsiveness and noise. @snooze_seconds 5 * 60 + # Snooze delay after a transient CDS HTTP error (5xx, network blip). + # Shorter than the normal poll interval so recovery is fast, but a + # snooze instead of an attempt so a sustained outage doesn't burn + # through max_attempts and orphan the row. + @error_snooze_seconds 60 + # A CDS job that's been in 'accepted' state for more than this is stuck # in the server's queue and almost certainly won't run. Observed in # prod: pressure-level completes in ~90 min, but single-level @@ -102,10 +108,12 @@ defmodule Microwaveprop.Workers.Era5PollWorker do fail_row(row, "pressure-level job failed: #{reason}") {{:error, reason}, _} -> - {:error, "single-level check_status error: #{reason}"} + Logger.warning("Era5Poll: transient single-level check_status error (snoozing): #{reason}") + {:snooze, @error_snooze_seconds} {_, {:error, reason}} -> - {:error, "pressure-level check_status error: #{reason}"} + Logger.warning("Era5Poll: transient pressure-level check_status error (snoozing): #{reason}") + {:snooze, @error_snooze_seconds} _both_running_or_one_done -> bump_poll_count(row) diff --git a/test/microwaveprop/workers/era5_poll_worker_test.exs b/test/microwaveprop/workers/era5_poll_worker_test.exs index 716211b3..6c31171e 100644 --- a/test/microwaveprop/workers/era5_poll_worker_test.exs +++ b/test/microwaveprop/workers/era5_poll_worker_test.exs @@ -247,6 +247,59 @@ defmodule Microwaveprop.Workers.Era5PollWorkerTest do end end + describe "perform/1 — transient CDS HTTP error" do + # Observed in prod 2026-04-15: a brief CDS network blip returned HTTP + # 5xx on check_status. The old {:error, _} branches consumed attempts + # and max_attempts=10 burned out in seconds, so 13 rows became + # orphaned (discarded with no cleanup, no active poll worker). The + # fix: treat HTTP errors as transient — snooze without counting an + # attempt. Terminal states (:failed, :rejected, :not_found) still + # drive the fail/resubmit paths. + test "snoozes and keeps the row when single-level check_status returns a transient HTTP error" do + row = insert_cds_job() + + Req.Test.stub(Era5Client, fn conn -> + cond do + conn.method == "GET" and String.contains?(conn.request_path, "cds-single-123") -> + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.resp(503, ~s({"error":"service unavailable"})) + + conn.method == "GET" and String.contains?(conn.request_path, "cds-pressure-456") -> + Req.Test.json(conn, %{"status" => "running"}) + end + end) + + assert {:snooze, snooze_s} = + Era5PollWorker.perform(%Oban.Job{args: %{"era5_cds_job_id" => row.id}}) + + assert snooze_s > 0 + assert Repo.get(Era5CdsJob, row.id) + end + + test "snoozes and keeps the row when pressure-level check_status returns a transient HTTP error" do + row = insert_cds_job() + + Req.Test.stub(Era5Client, fn conn -> + cond do + conn.method == "GET" and String.contains?(conn.request_path, "cds-single-123") -> + Req.Test.json(conn, %{"status" => "running"}) + + conn.method == "GET" and String.contains?(conn.request_path, "cds-pressure-456") -> + conn + |> Plug.Conn.put_resp_content_type("application/json") + |> Plug.Conn.resp(502, ~s({"error":"bad gateway"})) + end + end) + + assert {:snooze, snooze_s} = + Era5PollWorker.perform(%Oban.Job{args: %{"era5_cds_job_id" => row.id}}) + + assert snooze_s > 0 + assert Repo.get(Era5CdsJob, row.id) + end + end + describe "perform/1 — failure" do test "returns {:error, _}, deletes the row, and deletes both CDS jobs when a leg fails" do row = insert_cds_job()