prop/test/microwaveprop/workers/era5_submit_worker_test.exs
Graham McIntire 6842a6dd42
Snooze ERA5 submits when CDS in-flight ceiling is near
CDS enforces a per-user cap of 150 queued jobs. Each successful submit
adds 2 rows to era5_cds_jobs (single-level + pressure-level), so when
the current in-flight count + 2 would exceed (150 - 10 headroom), the
worker snoozes for 5 minutes instead of POSTing to CDS. Stops 'Number
of queued requests is limited to 150' rejects from burning Oban
attempts.
2026-04-13 17:30:50 -05:00

218 lines
7.4 KiB
Elixir

defmodule Microwaveprop.Workers.Era5SubmitWorkerTest do
use Microwaveprop.DataCase, async: false
use Oban.Testing, repo: Microwaveprop.Repo
import Ecto.Query
alias Microwaveprop.Weather.Era5CdsJob
alias Microwaveprop.Weather.Era5Client
alias Microwaveprop.Weather.Era5Profile
alias Microwaveprop.Workers.Era5PollWorker
alias Microwaveprop.Workers.Era5SubmitWorker
@default_args %{"year" => 2014, "month" => 3, "tile_lat" => 32, "tile_lon" => -98}
setup do
System.put_env("ERA5_CDS_API_KEY", "test-key")
on_exit(fn -> System.delete_env("ERA5_CDS_API_KEY") end)
:ok
end
describe "perform/1 — short-circuit paths" do
test "returns {:ok, :cached} when the month-tile already has era5_profiles" do
# Seed a profile inside the (2014-03, tile 32,-98) window.
%Era5Profile{}
|> Era5Profile.changeset(%{
lat: 32.5,
lon: -97.25,
valid_time: ~U[2014-03-15 12:00:00Z],
profile: []
})
|> Repo.insert!()
Req.Test.stub(Era5Client, fn _ -> raise "CDS should not be contacted on cache hit" end)
Oban.Testing.with_testing_mode(:manual, fn ->
assert {:ok, :cached} = Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
assert [] = all_enqueued(worker: Era5PollWorker)
end)
end
test "returns {:ok, :already_submitted} when an era5_cds_jobs row already exists" do
{:ok, _} =
%Era5CdsJob{}
|> Era5CdsJob.changeset(%{
year: 2014,
month: 3,
tile_lat: 32,
tile_lon: -98,
single_job_id: "pre-existing-single",
pressure_job_id: "pre-existing-pressure",
submitted_at: DateTime.truncate(DateTime.utc_now(), :second)
})
|> Repo.insert()
Req.Test.stub(Era5Client, fn _ -> raise "CDS should not be contacted on duplicate submit" end)
Oban.Testing.with_testing_mode(:manual, fn ->
assert {:ok, :already_submitted} =
Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
end)
end
end
describe "perform/1 — happy path" do
test "submits both CDS jobs, persists the row, and enqueues a poll worker" do
Req.Test.stub(Era5Client, fn conn ->
assert conn.method == "POST"
dataset_id =
cond do
String.contains?(conn.request_path, "single-levels") -> "single-job-123"
String.contains?(conn.request_path, "pressure-levels") -> "pressure-job-456"
end
Req.Test.json(conn, %{"jobID" => dataset_id})
end)
Oban.Testing.with_testing_mode(:manual, fn ->
assert {:ok, %Era5CdsJob{} = row} =
Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
assert row.single_job_id == "single-job-123"
assert row.pressure_job_id == "pressure-job-456"
assert row.year == 2014
assert row.month == 3
assert row.tile_lat == 32
assert row.tile_lon == -98
# Row is persisted in the DB.
assert Repo.aggregate(
from(j in Era5CdsJob,
where: j.year == 2014 and j.month == 3 and j.tile_lat == 32 and j.tile_lon == -98
),
:count,
:id
) == 1
# A poll worker was enqueued for this row, scheduled ~5 min from now.
[poll_job] = all_enqueued(worker: Era5PollWorker)
assert poll_job.args["era5_cds_job_id"] == row.id
assert DateTime.after?(poll_job.scheduled_at, DateTime.utc_now())
end)
end
test "returns {:error, _} and persists no row if either CDS submit fails" do
Req.Test.stub(Era5Client, fn conn ->
if String.contains?(conn.request_path, "pressure-levels") do
Plug.Conn.resp(conn, 400, ~s({"detail":"bad pressure request"}))
else
Req.Test.json(conn, %{"jobID" => "single-ok"})
end
end)
Oban.Testing.with_testing_mode(:manual, fn ->
assert {:error, reason} = Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
assert is_binary(reason)
# No orphaned row.
assert Repo.aggregate(Era5CdsJob, :count, :id) == 0
# No poll worker enqueued.
assert [] = all_enqueued(worker: Era5PollWorker)
end)
end
end
describe "CDS in-flight cap" do
# CDS rejects with "Number of queued requests is limited to 150" when
# a user has too many pending jobs. Each submit adds 2 rows to
# era5_cds_jobs (single-level + pressure-level), so we snooze when
# the in-flight count is close to the ceiling.
@cap_headroom 10
@cds_ceiling 150
test "snoozes when era5_cds_jobs count is at or above the ceiling - headroom" do
# Seed enough in-flight rows to push us over the snooze threshold.
seed_count = @cds_ceiling - @cap_headroom
rows =
for i <- 1..seed_count do
%{
id: Ecto.UUID.generate(),
year: 2014,
month: 3,
tile_lat: 32,
tile_lon: -98 - i,
single_job_id: "seed-single-#{i}",
pressure_job_id: "seed-pressure-#{i}",
submitted_at: DateTime.truncate(DateTime.utc_now(), :second),
inserted_at: DateTime.truncate(DateTime.utc_now(), :second),
updated_at: DateTime.truncate(DateTime.utc_now(), :second),
poll_count: 0
}
end
Repo.insert_all(Era5CdsJob, rows)
Req.Test.stub(Era5Client, fn _ -> raise "CDS should not be contacted when capped" end)
Oban.Testing.with_testing_mode(:manual, fn ->
assert {:snooze, seconds} = Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
assert is_integer(seconds) and seconds > 0
end)
# No new era5_cds_jobs row inserted for the requested tile.
refute Repo.one(
from j in Era5CdsJob,
where: j.year == 2014 and j.month == 3 and j.tile_lat == 32 and j.tile_lon == -98
)
end
test "submits normally when in-flight count is well below the ceiling" do
# 10 in-flight — comfortably under the cap.
rows =
for i <- 1..10 do
%{
id: Ecto.UUID.generate(),
year: 2013,
month: i,
tile_lat: 40,
tile_lon: -90,
single_job_id: "warm-single-#{i}",
pressure_job_id: "warm-pressure-#{i}",
submitted_at: DateTime.truncate(DateTime.utc_now(), :second),
inserted_at: DateTime.truncate(DateTime.utc_now(), :second),
updated_at: DateTime.truncate(DateTime.utc_now(), :second),
poll_count: 0
}
end
Repo.insert_all(Era5CdsJob, rows)
Req.Test.stub(Era5Client, fn conn ->
dataset_id =
cond do
String.contains?(conn.request_path, "single-levels") -> "fresh-single"
String.contains?(conn.request_path, "pressure-levels") -> "fresh-pressure"
end
Req.Test.json(conn, %{"jobID" => dataset_id})
end)
Oban.Testing.with_testing_mode(:manual, fn ->
assert {:ok, %Era5CdsJob{}} = Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
end)
end
end
describe "unique constraint" do
test "collapses duplicate (year, month, tile_lat, tile_lon) Oban enqueues" do
Oban.Testing.with_testing_mode(:manual, fn ->
{:ok, j1} = Oban.insert(Era5SubmitWorker.new(@default_args))
{:ok, j2} = Oban.insert(Era5SubmitWorker.new(@default_args))
assert j1.id == j2.id
end)
end
end
end