Graham McIntire
8637253fda
Per-point ERA5 fetches were tragically slow because every point-hour triggered its own asynchronous CDS job (submit → poll → assemble → download). For backfill this meant thousands of independent jobs queued against Copernicus. The new path groups requests by calendar month and a 2° × 2° lat/lon tile so one CDS cycle populates ~60k profiles at once, and Oban uniqueness on (year, month, tile_lat, tile_lon) collapses every duplicate enqueue. - Era5BatchClient builds the monthly CDS requests, extracts every (lat, lon, hour) from the GRIB2 blob with wgrib2, derives refractivity params, and bulk-inserts in 2k-row chunks with on_conflict: :nothing. fetch_month_into_db/1 short-circuits when the month-tile already has any cached profile. - Era5MonthBatchWorker runs the batch on the :era5 queue with a generous backoff (10m → 1d) and the uniqueness key above. - Era5FetchWorker is now a thin router: cache hit → :ok, cache miss → enqueue the month-batch for the point's tile-month and return. No more per-point CDS calls. - Wgrib2 grows extract_grid_messages/3 which preserves per-message datetimes by parsing `d=YYYYMMDDHH[MMSS]` from the inventory, so a single GRIB2 file carrying a whole month decodes correctly. - The era5_backfill mix task enqueues month-tile batches directly.
78 lines
2.6 KiB
Elixir
78 lines
2.6 KiB
Elixir
defmodule Microwaveprop.Weather.Era5BatchClientTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.Weather.Era5BatchClient
|
|
alias Microwaveprop.Weather.Era5Profile
|
|
|
|
describe "tile_for_point/2" do
|
|
test "floors positive coordinates to a 2° lattice" do
|
|
assert {32, 96} = Era5BatchClient.tile_for_point(32.75, 97.5)
|
|
assert {32, 96} = Era5BatchClient.tile_for_point(33.99, 97.99)
|
|
assert {34, 98} = Era5BatchClient.tile_for_point(34.0, 98.0)
|
|
end
|
|
|
|
test "floors negative longitudes toward -infinity" do
|
|
assert {32, -98} = Era5BatchClient.tile_for_point(32.75, -97.25)
|
|
assert {32, -98} = Era5BatchClient.tile_for_point(33.0, -97.01)
|
|
assert {32, -100} = Era5BatchClient.tile_for_point(33.0, -99.0)
|
|
end
|
|
|
|
test "floors negative latitudes toward -infinity" do
|
|
assert {-2, 0} = Era5BatchClient.tile_for_point(-1.25, 0.5)
|
|
assert {-34, -60} = Era5BatchClient.tile_for_point(-33.5, -59.5)
|
|
end
|
|
|
|
test "edges land on the tile to the right/north" do
|
|
# 30.0 exactly → tile 30 (south/west edge)
|
|
assert {30, 96} = Era5BatchClient.tile_for_point(30.0, 97.5)
|
|
end
|
|
end
|
|
|
|
describe "fetch_month_into_db/1 idempotency" do
|
|
test "returns {:ok, 0} when the month-tile already has any profile" do
|
|
# Seed one profile inside the 2014-03, tile (32, -98) window.
|
|
%Era5Profile{}
|
|
|> Era5Profile.changeset(%{
|
|
lat: 32.5,
|
|
lon: -97.25,
|
|
valid_time: ~U[2014-03-15 12:00:00Z],
|
|
profile: []
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
# No CDS API key configured in tests → if the cache check fails, the
|
|
# function would return an ERA5_CDS_API_KEY error. {:ok, 0} means the
|
|
# short-circuit path was taken.
|
|
assert {:ok, 0} =
|
|
Era5BatchClient.fetch_month_into_db(%{
|
|
year: 2014,
|
|
month: 3,
|
|
tile_lat: 32,
|
|
tile_lon: -98
|
|
})
|
|
end
|
|
|
|
test "a profile outside the tile window does not count as cached" do
|
|
# This profile is in tile (30, -100), not (32, -98).
|
|
%Era5Profile{}
|
|
|> Era5Profile.changeset(%{
|
|
lat: 30.5,
|
|
lon: -99.25,
|
|
valid_time: ~U[2014-03-15 12:00:00Z],
|
|
profile: []
|
|
})
|
|
|> Repo.insert!()
|
|
|
|
# The (32, -98) tile is still uncached, so fetch_month_into_db tries
|
|
# to reach CDS and fails with missing API key, which bubbles up
|
|
# (rather than :ok, 0).
|
|
assert {:error, _} =
|
|
Era5BatchClient.fetch_month_into_db(%{
|
|
year: 2014,
|
|
month: 3,
|
|
tile_lat: 32,
|
|
tile_lon: -98
|
|
})
|
|
end
|
|
end
|
|
end
|