Graham McIntire
1ec10bec1f
The single Era5MonthBatchWorker pinned an Oban slot for the full 30-45
min a CDS month-tile takes to assemble, and lost the work entirely on a
rolling deploy because the in-flight Task died with the pod. This splits
the flow into two tiny workers and persists the CDS job IDs so deploys
survive.
New pipeline:
1. Era5SubmitWorker (:era5_submit queue) — POSTs both CDS requests in
parallel, writes an `era5_cds_jobs` row with the returned job IDs,
and enqueues an Era5PollWorker scheduled +5 min. ~1s of real work.
Short-circuits when the month-tile is already cached or when a row
already exists (in-flight from a previous attempt).
2. Era5PollWorker (:era5_poll queue) — reads the row, calls
Era5Client.check_status/1 for both CDS job IDs, and:
- returns {:snooze, 300} if either job is still running (Oban
re-schedules without counting an attempt and releases the slot
immediately — a pod can keep dozens of tile-months in flight
without pinning workers)
- streams both GRIB files to disk via Req into: File.stream!,
decodes via Wgrib2.extract_grid_messages_from_file, bulk-inserts
via Era5BatchClient.decode_and_insert/6, deletes the row, and
DELETEs both completed jobs from CDS to free server-side quota
- if either leg CDS-reports failed, deletes the row + both CDS
jobs and returns {:error, reason}
Era5Client gains four testable building blocks:
submit_job/2 (bare POST → {:ok, job_id})
check_status/1 (GET → :running | {:done, src} | {:failed, reason})
download_source_to_file/3 (streams {:url, href} or writes {:body, bin})
delete_job/1 (DELETE /jobs/:id, treats 200/202/204/404 as :ok)
All Req calls now route through `era5_req_options` so tests can stub
CDS responses via Req.Test.stub(Era5Client, fn).
Era5MonthBatchWorker is retained as a thin forwarder to Era5SubmitWorker
so any jobs already in the :era5_batch queue on prod pods drain cleanly
on the next rolling deploy. Safe to delete in a follow-up.
Adds era5_cds_jobs table with a unique index on
(year, month, tile_lat, tile_lon) so duplicate submits collapse.
New queue config in runtime.exs:
era5_submit: local_limit 4, rate_limit 30/hour (burst protection)
era5_poll: local_limit 20 (polls are cheap GETs)
era5_batch: kept at 1 for legacy job drain, delete next cycle
82 lines
2.2 KiB
Elixir
82 lines
2.2 KiB
Elixir
defmodule Microwaveprop.Weather.Era5CdsJobTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.Weather.Era5CdsJob
|
|
|
|
describe "changeset/2" do
|
|
@valid_attrs %{
|
|
year: 2014,
|
|
month: 3,
|
|
tile_lat: 32,
|
|
tile_lon: -98,
|
|
single_job_id: "abc-single",
|
|
pressure_job_id: "abc-pressure",
|
|
submitted_at: ~U[2026-04-13 21:00:00Z]
|
|
}
|
|
|
|
test "accepts a complete attrs map" do
|
|
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, @valid_attrs)
|
|
assert changeset.valid?
|
|
end
|
|
|
|
test "requires year, month, tile coordinates, and both CDS job IDs" do
|
|
changeset = Era5CdsJob.changeset(%Era5CdsJob{}, %{})
|
|
|
|
refute changeset.valid?
|
|
|
|
required = ~w(year month tile_lat tile_lon single_job_id pressure_job_id submitted_at)a
|
|
errors = Keyword.keys(changeset.errors)
|
|
|
|
for field <- required do
|
|
assert field in errors, "expected #{inspect(field)} to be required"
|
|
end
|
|
end
|
|
|
|
test "last_polled_at and poll_count default to nil/0 and can be updated" do
|
|
{:ok, row} =
|
|
%Era5CdsJob{}
|
|
|> Era5CdsJob.changeset(@valid_attrs)
|
|
|> Repo.insert()
|
|
|
|
assert row.poll_count == 0
|
|
assert is_nil(row.last_polled_at)
|
|
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
{:ok, bumped} =
|
|
row
|
|
|> Era5CdsJob.changeset(%{last_polled_at: now, poll_count: 1})
|
|
|> Repo.update()
|
|
|
|
assert bumped.poll_count == 1
|
|
assert bumped.last_polled_at == now
|
|
end
|
|
end
|
|
|
|
describe "uniqueness" do
|
|
test "collapses duplicate (year, month, tile_lat, tile_lon) inserts" do
|
|
attrs = %{
|
|
year: 2014,
|
|
month: 3,
|
|
tile_lat: 32,
|
|
tile_lon: -98,
|
|
single_job_id: "first-single",
|
|
pressure_job_id: "first-pressure",
|
|
submitted_at: ~U[2026-04-13 21:00:00Z]
|
|
}
|
|
|
|
{:ok, _} =
|
|
%Era5CdsJob{}
|
|
|> Era5CdsJob.changeset(attrs)
|
|
|> Repo.insert()
|
|
|
|
{:error, changeset} =
|
|
%Era5CdsJob{}
|
|
|> Era5CdsJob.changeset(%{attrs | single_job_id: "second-single", pressure_job_id: "second-pressure"})
|
|
|> Repo.insert()
|
|
|
|
refute changeset.valid?
|
|
assert {:tile_month, _} = List.keyfind(changeset.errors, :tile_month, 0) || {:tile_month, nil}
|
|
end
|
|
end
|
|
end
|