CDS enforces a per-user cap of 150 queued jobs. Each successful submit adds 2 rows to era5_cds_jobs (single-level + pressure-level), so when the current in-flight count + 2 would exceed (150 - 10 headroom), the worker snoozes for 5 minutes instead of POSTing to CDS. Stops 'Number of queued requests is limited to 150' rejects from burning Oban attempts.
218 lines
7.4 KiB
Elixir
218 lines
7.4 KiB
Elixir
defmodule Microwaveprop.Workers.Era5SubmitWorkerTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
use Oban.Testing, repo: Microwaveprop.Repo
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Weather.Era5CdsJob
|
|
alias Microwaveprop.Weather.Era5Client
|
|
alias Microwaveprop.Weather.Era5Profile
|
|
alias Microwaveprop.Workers.Era5PollWorker
|
|
alias Microwaveprop.Workers.Era5SubmitWorker
|
|
|
|
@default_args %{"year" => 2014, "month" => 3, "tile_lat" => 32, "tile_lon" => -98}
|
|
|
|
setup do
|
|
System.put_env("ERA5_CDS_API_KEY", "test-key")
|
|
on_exit(fn -> System.delete_env("ERA5_CDS_API_KEY") end)
|
|
:ok
|
|
end
|
|
|
|
describe "perform/1 — short-circuit paths" do
|
|
test "returns {:ok, :cached} when the month-tile already has era5_profiles" do
|
|
# Seed a profile inside the (2014-03, tile 32,-98) window.
|
|
%Era5Profile{}
|
|
|> Era5Profile.changeset(%{
|
|
lat: 32.5,
|
|
lon: -97.25,
|
|
valid_time: ~U[2014-03-15 12:00:00Z],
|
|
profile: []
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
Req.Test.stub(Era5Client, fn _ -> raise "CDS should not be contacted on cache hit" end)
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
assert {:ok, :cached} = Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
|
|
assert [] = all_enqueued(worker: Era5PollWorker)
|
|
end)
|
|
end
|
|
|
|
test "returns {:ok, :already_submitted} when an era5_cds_jobs row already exists" do
|
|
{:ok, _} =
|
|
%Era5CdsJob{}
|
|
|> Era5CdsJob.changeset(%{
|
|
year: 2014,
|
|
month: 3,
|
|
tile_lat: 32,
|
|
tile_lon: -98,
|
|
single_job_id: "pre-existing-single",
|
|
pressure_job_id: "pre-existing-pressure",
|
|
submitted_at: DateTime.truncate(DateTime.utc_now(), :second)
|
|
})
|
|
|> Repo.insert()
|
|
|
|
Req.Test.stub(Era5Client, fn _ -> raise "CDS should not be contacted on duplicate submit" end)
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
assert {:ok, :already_submitted} =
|
|
Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
|
|
end)
|
|
end
|
|
end
|
|
|
|
describe "perform/1 — happy path" do
|
|
test "submits both CDS jobs, persists the row, and enqueues a poll worker" do
|
|
Req.Test.stub(Era5Client, fn conn ->
|
|
assert conn.method == "POST"
|
|
|
|
dataset_id =
|
|
cond do
|
|
String.contains?(conn.request_path, "single-levels") -> "single-job-123"
|
|
String.contains?(conn.request_path, "pressure-levels") -> "pressure-job-456"
|
|
end
|
|
|
|
Req.Test.json(conn, %{"jobID" => dataset_id})
|
|
end)
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
assert {:ok, %Era5CdsJob{} = row} =
|
|
Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
|
|
|
|
assert row.single_job_id == "single-job-123"
|
|
assert row.pressure_job_id == "pressure-job-456"
|
|
assert row.year == 2014
|
|
assert row.month == 3
|
|
assert row.tile_lat == 32
|
|
assert row.tile_lon == -98
|
|
|
|
# Row is persisted in the DB.
|
|
assert Repo.aggregate(
|
|
from(j in Era5CdsJob,
|
|
where: j.year == 2014 and j.month == 3 and j.tile_lat == 32 and j.tile_lon == -98
|
|
),
|
|
:count,
|
|
:id
|
|
) == 1
|
|
|
|
# A poll worker was enqueued for this row, scheduled ~5 min from now.
|
|
[poll_job] = all_enqueued(worker: Era5PollWorker)
|
|
assert poll_job.args["era5_cds_job_id"] == row.id
|
|
assert DateTime.after?(poll_job.scheduled_at, DateTime.utc_now())
|
|
end)
|
|
end
|
|
|
|
test "returns {:error, _} and persists no row if either CDS submit fails" do
|
|
Req.Test.stub(Era5Client, fn conn ->
|
|
if String.contains?(conn.request_path, "pressure-levels") do
|
|
Plug.Conn.resp(conn, 400, ~s({"detail":"bad pressure request"}))
|
|
else
|
|
Req.Test.json(conn, %{"jobID" => "single-ok"})
|
|
end
|
|
end)
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
assert {:error, reason} = Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
|
|
assert is_binary(reason)
|
|
|
|
# No orphaned row.
|
|
assert Repo.aggregate(Era5CdsJob, :count, :id) == 0
|
|
|
|
# No poll worker enqueued.
|
|
assert [] = all_enqueued(worker: Era5PollWorker)
|
|
end)
|
|
end
|
|
end
|
|
|
|
describe "CDS in-flight cap" do
|
|
# CDS rejects with "Number of queued requests is limited to 150" when
|
|
# a user has too many pending jobs. Each submit adds 2 rows to
|
|
# era5_cds_jobs (single-level + pressure-level), so we snooze when
|
|
# the in-flight count is close to the ceiling.
|
|
@cap_headroom 10
|
|
@cds_ceiling 150
|
|
|
|
test "snoozes when era5_cds_jobs count is at or above the ceiling - headroom" do
|
|
# Seed enough in-flight rows to push us over the snooze threshold.
|
|
seed_count = @cds_ceiling - @cap_headroom
|
|
|
|
rows =
|
|
for i <- 1..seed_count do
|
|
%{
|
|
id: Ecto.UUID.generate(),
|
|
year: 2014,
|
|
month: 3,
|
|
tile_lat: 32,
|
|
tile_lon: -98 - i,
|
|
single_job_id: "seed-single-#{i}",
|
|
pressure_job_id: "seed-pressure-#{i}",
|
|
submitted_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
inserted_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
updated_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
poll_count: 0
|
|
}
|
|
end
|
|
|
|
Repo.insert_all(Era5CdsJob, rows)
|
|
|
|
Req.Test.stub(Era5Client, fn _ -> raise "CDS should not be contacted when capped" end)
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
assert {:snooze, seconds} = Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
|
|
assert is_integer(seconds) and seconds > 0
|
|
end)
|
|
|
|
# No new era5_cds_jobs row inserted for the requested tile.
|
|
refute Repo.one(
|
|
from j in Era5CdsJob,
|
|
where: j.year == 2014 and j.month == 3 and j.tile_lat == 32 and j.tile_lon == -98
|
|
)
|
|
end
|
|
|
|
test "submits normally when in-flight count is well below the ceiling" do
|
|
# 10 in-flight — comfortably under the cap.
|
|
rows =
|
|
for i <- 1..10 do
|
|
%{
|
|
id: Ecto.UUID.generate(),
|
|
year: 2013,
|
|
month: i,
|
|
tile_lat: 40,
|
|
tile_lon: -90,
|
|
single_job_id: "warm-single-#{i}",
|
|
pressure_job_id: "warm-pressure-#{i}",
|
|
submitted_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
inserted_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
updated_at: DateTime.truncate(DateTime.utc_now(), :second),
|
|
poll_count: 0
|
|
}
|
|
end
|
|
|
|
Repo.insert_all(Era5CdsJob, rows)
|
|
|
|
Req.Test.stub(Era5Client, fn conn ->
|
|
dataset_id =
|
|
cond do
|
|
String.contains?(conn.request_path, "single-levels") -> "fresh-single"
|
|
String.contains?(conn.request_path, "pressure-levels") -> "fresh-pressure"
|
|
end
|
|
|
|
Req.Test.json(conn, %{"jobID" => dataset_id})
|
|
end)
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
assert {:ok, %Era5CdsJob{}} = Era5SubmitWorker.perform(%Oban.Job{args: @default_args})
|
|
end)
|
|
end
|
|
end
|
|
|
|
describe "unique constraint" do
|
|
test "collapses duplicate (year, month, tile_lat, tile_lon) Oban enqueues" do
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
{:ok, j1} = Oban.insert(Era5SubmitWorker.new(@default_args))
|
|
{:ok, j2} = Oban.insert(Era5SubmitWorker.new(@default_args))
|
|
assert j1.id == j2.id
|
|
end)
|
|
end
|
|
end
|
|
end
|