Graham McIntire
8637253fda
Per-point ERA5 fetches were tragically slow because every point-hour triggered its own asynchronous CDS job (submit → poll → assemble → download). For backfill this meant thousands of independent jobs queued against Copernicus. The new path groups requests by calendar month and a 2° × 2° lat/lon tile so one CDS cycle populates ~60k profiles at once, and Oban uniqueness on (year, month, tile_lat, tile_lon) collapses every duplicate enqueue. - Era5BatchClient builds the monthly CDS requests, extracts every (lat, lon, hour) from the GRIB2 blob with wgrib2, derives refractivity params, and bulk-inserts in 2k-row chunks with on_conflict: :nothing. fetch_month_into_db/1 short-circuits when the month-tile already has any cached profile. - Era5MonthBatchWorker runs the batch on the :era5 queue with a generous backoff (10m → 1d) and the uniqueness key above. - Era5FetchWorker is now a thin router: cache hit → :ok, cache miss → enqueue the month-batch for the point's tile-month and return. No more per-point CDS calls. - Wgrib2 grows extract_grid_messages/3 which preserves per-message datetimes by parsing `d=YYYYMMDDHH[MMSS]` from the inventory, so a single GRIB2 file carrying a whole month decodes correctly. - The era5_backfill mix task enqueues month-tile batches directly.
100 lines
3.5 KiB
Elixir
100 lines
3.5 KiB
Elixir
defmodule Microwaveprop.Workers.Era5FetchWorkerTest do
|
|
use Microwaveprop.DataCase
|
|
use Oban.Testing, repo: Microwaveprop.Repo
|
|
|
|
alias Microwaveprop.Weather.Era5Profile
|
|
alias Microwaveprop.Workers.Era5FetchWorker
|
|
alias Microwaveprop.Workers.Era5MonthBatchWorker
|
|
|
|
describe "perform/1" do
|
|
test "enqueues a month-tile batch worker when no profile is cached" do
|
|
args = %{"lat" => 32.75, "lon" => -97.25, "valid_time" => "2014-01-15T12:00:00Z"}
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
assert :ok = Era5FetchWorker.perform(%Oban.Job{args: args})
|
|
|
|
jobs = all_enqueued(worker: Era5MonthBatchWorker)
|
|
assert [job] = jobs
|
|
assert job.args["year"] == 2014
|
|
assert job.args["month"] == 1
|
|
# 32.75 → tile 32 (floor to even degree), -97.25 → tile -98
|
|
assert job.args["tile_lat"] == 32
|
|
assert job.args["tile_lon"] == -98
|
|
end)
|
|
end
|
|
|
|
test "is a no-op when the profile is already cached" do
|
|
valid_time = ~U[2014-01-15 12:00:00Z]
|
|
|
|
%Era5Profile{}
|
|
|> Era5Profile.changeset(%{
|
|
lat: 32.75,
|
|
lon: -97.25,
|
|
valid_time: valid_time,
|
|
profile: []
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
args = %{"lat" => 32.75, "lon" => -97.25, "valid_time" => DateTime.to_iso8601(valid_time)}
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
assert :ok = Era5FetchWorker.perform(%Oban.Job{args: args})
|
|
assert [] = all_enqueued(worker: Era5MonthBatchWorker)
|
|
end)
|
|
end
|
|
end
|
|
|
|
describe "unique constraint" do
|
|
test "deduplicates identical (lat, lon, valid_time) jobs across inserts" do
|
|
args = %{"lat" => 32.75, "lon" => -97.0, "valid_time" => "2014-01-01T00:00:00Z"}
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
{:ok, job1} = Oban.insert(Era5FetchWorker.new(args))
|
|
{:ok, job2} = Oban.insert(Era5FetchWorker.new(args))
|
|
|
|
# Second insert should hit the unique constraint and return the
|
|
# existing job (same id) instead of creating a new one.
|
|
assert job1.id == job2.id
|
|
|
|
count = Repo.aggregate(Oban.Job, :count, :id)
|
|
assert count == 1
|
|
end)
|
|
end
|
|
|
|
test "allows distinct hours on the same date" do
|
|
args_00 = %{"lat" => 32.75, "lon" => -97.0, "valid_time" => "2014-01-01T00:00:00Z"}
|
|
args_06 = %{"lat" => 32.75, "lon" => -97.0, "valid_time" => "2014-01-01T06:00:00Z"}
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
{:ok, job1} = Oban.insert(Era5FetchWorker.new(args_00))
|
|
{:ok, job2} = Oban.insert(Era5FetchWorker.new(args_06))
|
|
|
|
assert job1.id != job2.id
|
|
assert Repo.aggregate(Oban.Job, :count, :id) == 2
|
|
end)
|
|
end
|
|
|
|
test "allows distinct grid points for the same valid_time" do
|
|
args_a = %{"lat" => 32.75, "lon" => -97.0, "valid_time" => "2014-01-01T00:00:00Z"}
|
|
args_b = %{"lat" => 33.00, "lon" => -97.0, "valid_time" => "2014-01-01T00:00:00Z"}
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
{:ok, job1} = Oban.insert(Era5FetchWorker.new(args_a))
|
|
{:ok, job2} = Oban.insert(Era5FetchWorker.new(args_b))
|
|
|
|
assert job1.id != job2.id
|
|
assert Repo.aggregate(Oban.Job, :count, :id) == 2
|
|
end)
|
|
end
|
|
|
|
test "deduplicates across repeated Oban.insert calls" do
|
|
args = %{"lat" => 32.75, "lon" => -97.0, "valid_time" => "2014-01-01T00:00:00Z"}
|
|
|
|
Oban.Testing.with_testing_mode(:manual, fn ->
|
|
Enum.each(1..5, fn _ -> Oban.insert(Era5FetchWorker.new(args)) end)
|
|
|
|
assert Repo.aggregate(Oban.Job, :count, :id) == 1
|
|
end)
|
|
end
|
|
end
|
|
end
|