Graham McIntire
d5726a89d5
The HRDPS pipeline as written was unusable in production: each chain step took ~5 hours (not 30-90s as the dev-bench claim) because every `wgrib2 -lon` batch redundantly JPEG2000-decodes ~150 records. With 57 batches × 18 forecast hours, a single cycle would take days to drain. The /weather Canadian view only needs the current hour, so the cheapest fix is to stop seeding the rest of the chain entirely: * Rust grid: HRDPS_STEP=0.5 (was 0.125) drops point count 16× to ~3.5k. Coarser cells than HRRR, but /weather gets visible Canadian coverage inside one chain step (~20 min) instead of never. * GridTaskEnqueuer.seed_current_hour/2 inserts exactly one forecast row whose valid_time is closest to `now`. fh is clamped to 1..18 so the Rust F00Reserved branch never fires. * HrdpsGridWorker switches to seed_current_hour and drops the 6-hour-cycle cron in favor of HH:35 hourly fires. Reseeds are idempotent on the 4-column unique key. Also fixes a pre-existing credo "nested too deep" in ScalarFile.read_point by extracting lookup_in_chunk + hit_or_false helpers — happened to be on the read path that surfaces this work, so folding it into the same commit.
76 lines
2.4 KiB
Elixir
76 lines
2.4 KiB
Elixir
defmodule Microwaveprop.Workers.HrdpsGridWorkerTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
use Oban.Testing, repo: Microwaveprop.Repo
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Workers.HrdpsGridWorker
|
|
|
|
setup do
|
|
original = Application.get_env(:microwaveprop, :hrdps_cycle_available_fn)
|
|
|
|
on_exit(fn ->
|
|
if original,
|
|
do: Application.put_env(:microwaveprop, :hrdps_cycle_available_fn, original),
|
|
else: Application.delete_env(:microwaveprop, :hrdps_cycle_available_fn)
|
|
end)
|
|
end
|
|
|
|
describe "pick_run_time/1" do
|
|
test "returns the now-4h cycle when it's published" do
|
|
Application.put_env(:microwaveprop, :hrdps_cycle_available_fn, fn _dt -> true end)
|
|
|
|
now = ~U[2026-04-29 16:30:00Z]
|
|
run_time = HrdpsGridWorker.pick_run_time(now)
|
|
|
|
# now - 4h = 12:30Z; snap down to cycle = 12:00Z
|
|
assert run_time == ~U[2026-04-29 12:00:00Z]
|
|
end
|
|
|
|
test "falls back to the previous cycle when fresh isn't yet published" do
|
|
Application.put_env(:microwaveprop, :hrdps_cycle_available_fn, fn _dt -> false end)
|
|
|
|
now = ~U[2026-04-29 16:30:00Z]
|
|
run_time = HrdpsGridWorker.pick_run_time(now)
|
|
|
|
# now - 10h = 06:30Z; snap down to cycle = 06:00Z
|
|
assert run_time == ~U[2026-04-29 06:00:00Z]
|
|
end
|
|
end
|
|
|
|
describe "perform/1" do
|
|
test "seeds exactly one current-hour grid_task row tagged with source='hrdps'" do
|
|
Application.put_env(:microwaveprop, :hrdps_cycle_available_fn, fn _dt -> true end)
|
|
|
|
assert :ok = perform_job(HrdpsGridWorker, %{})
|
|
|
|
rows =
|
|
Repo.all(
|
|
from(g in "grid_tasks",
|
|
where: g.source == "hrdps",
|
|
select: %{kind: g.kind, forecast_hour: g.forecast_hour}
|
|
)
|
|
)
|
|
|
|
# One forecast row, no analysis. /weather only consumes current
|
|
# hour so we don't pay the wgrib2 cost for the rest of the chain.
|
|
assert length(rows) == 1
|
|
[%{kind: kind, forecast_hour: fh}] = rows
|
|
assert kind == "forecast"
|
|
assert fh >= 1 and fh <= 18
|
|
end
|
|
|
|
test "is idempotent — re-running the worker for the same cycle inserts nothing extra" do
|
|
Application.put_env(:microwaveprop, :hrdps_cycle_available_fn, fn _dt -> true end)
|
|
|
|
assert :ok = perform_job(HrdpsGridWorker, %{})
|
|
assert :ok = perform_job(HrdpsGridWorker, %{})
|
|
|
|
total =
|
|
Repo.one(from(g in "grid_tasks", where: g.source == "hrdps", select: count()))
|
|
|
|
assert total == 1
|
|
end
|
|
end
|
|
end
|