Graham McIntire
5cfb9e6c8e
Telemetry showed ~66 PropagationGridWorker exceptions per 6h with 55 ArgumentErrors and 11 TimeoutErrors, producing ~13 discarded chain steps. Each discard broke the chain: subsequent forecast hours were never enqueued, leaving the score store with huge gaps (e.g. at 14:11 UTC the earliest available forecast was 18:00, because f00-f05 all failed somewhere upstream and nothing ran after them). Three changes: 1. PropagationGridWorker: on the final attempt, still enqueue fh+1 even when this step failed. Oban discards the current job normally — but the rest of the chain keeps running, so one bad hour doesn't take out the remaining 12-18. The rescue is factored into a tested public helper. 2. HrrrClient.parse_idx: skip malformed idx lines instead of raising. NOAA S3 occasionally serves an HTML error page as the idx body, and the old strict String.to_integer path raised ArgumentError on the first non-numeric line and took down the chain step. This is the root cause of the 55 ArgumentErrors. 3. JS renderTimeline: when no forecast hour is at-or-before wall-clock (all times are future — the gap scenario the fixes above are designed to prevent), stop labeling the earliest future slot "Now". Lets the user see honest "+Nh" offsets instead of a lie on the pill.
117 lines
4.2 KiB
Elixir
117 lines
4.2 KiB
Elixir
defmodule Microwaveprop.Workers.PropagationGridWorkerTest do
|
||
@moduledoc """
|
||
Tests the chain-orchestration behavior of PropagationGridWorker.
|
||
|
||
The worker processes forecast hours f00–f18 across the CONUS grid,
|
||
but a single full sweep takes ~2 hours of wall time — longer than a
|
||
typical pod restart window. To survive deploys, the worker processes
|
||
ONE forecast hour per `perform/1` call and enqueues the next hour as
|
||
a fresh Oban job. The tests here cover the dispatch + chain logic
|
||
without mocking the full HRRR / scoring stack.
|
||
"""
|
||
use Microwaveprop.DataCase, async: false
|
||
use Oban.Testing, repo: Microwaveprop.Repo
|
||
|
||
alias Microwaveprop.Workers.MrmsFetchWorker
|
||
alias Microwaveprop.Workers.PropagationGridWorker
|
||
alias Microwaveprop.Workers.PropagationPruneWorker
|
||
|
||
describe "queue priority" do
|
||
test "PropagationGridWorker runs at the highest priority on :propagation" do
|
||
assert PropagationGridWorker.__opts__()[:priority] == 0
|
||
end
|
||
|
||
test "MrmsFetchWorker and PropagationPruneWorker yield to the grid chain" do
|
||
# Same :propagation queue — must be lower priority so hourly chain
|
||
# steps jump ahead of a MRMS or pruner backlog.
|
||
assert MrmsFetchWorker.__opts__()[:priority] > 0
|
||
assert PropagationPruneWorker.__opts__()[:priority] > 0
|
||
end
|
||
end
|
||
|
||
describe "perform/1 — chain seeding (empty args)" do
|
||
test "enqueues a single fh=0 chain step with a normalized run_time" do
|
||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||
assert :ok = PropagationGridWorker.perform(%Oban.Job{args: %{}})
|
||
|
||
[child] = all_enqueued(worker: PropagationGridWorker)
|
||
assert child.args["forecast_hour"] == 0
|
||
assert is_binary(child.args["run_time"])
|
||
|
||
{:ok, run_time, _} = DateTime.from_iso8601(child.args["run_time"])
|
||
assert run_time.minute == 0
|
||
assert run_time.second == 0
|
||
assert DateTime.before?(run_time, DateTime.utc_now())
|
||
end)
|
||
end
|
||
end
|
||
|
||
describe "enqueue_next_step/2" do
|
||
test "enqueues fh+1 when fh < max" do
|
||
run_time = ~U[2026-04-14 16:00:00Z]
|
||
|
||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||
assert {:ok, _} = PropagationGridWorker.enqueue_next_step(run_time, 0)
|
||
|
||
[child] = all_enqueued(worker: PropagationGridWorker)
|
||
assert child.args["forecast_hour"] == 1
|
||
assert child.args["run_time"] == DateTime.to_iso8601(run_time)
|
||
end)
|
||
end
|
||
|
||
test "does not enqueue anything after the final forecast hour" do
|
||
run_time = ~U[2026-04-14 16:00:00Z]
|
||
|
||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||
assert :final = PropagationGridWorker.enqueue_next_step(run_time, 18)
|
||
assert [] = all_enqueued(worker: PropagationGridWorker)
|
||
end)
|
||
end
|
||
end
|
||
|
||
describe "chain resilience on permanent failure" do
|
||
test "enqueues fh+1 when the final attempt of a mid-chain step fails" do
|
||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||
run_time = ~U[2026-04-14 16:00:00Z]
|
||
|
||
assert {:error, :boom} =
|
||
PropagationGridWorker.rescue_chain_on_last_attempt!(
|
||
{:error, :boom},
|
||
run_time,
|
||
4,
|
||
%Oban.Job{attempt: 5, max_attempts: 5}
|
||
)
|
||
|
||
[next] = all_enqueued(worker: PropagationGridWorker)
|
||
assert next.args["forecast_hour"] == 5
|
||
assert next.args["run_time"] == DateTime.to_iso8601(run_time)
|
||
end)
|
||
end
|
||
|
||
test "does not enqueue on earlier attempts — Oban retries naturally" do
|
||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||
PropagationGridWorker.rescue_chain_on_last_attempt!(
|
||
{:error, :boom},
|
||
~U[2026-04-14 16:00:00Z],
|
||
4,
|
||
%Oban.Job{attempt: 2, max_attempts: 5}
|
||
)
|
||
|
||
assert [] = all_enqueued(worker: PropagationGridWorker)
|
||
end)
|
||
end
|
||
|
||
test "does not enqueue anything past fh=18 even on final-attempt failure" do
|
||
Oban.Testing.with_testing_mode(:manual, fn ->
|
||
PropagationGridWorker.rescue_chain_on_last_attempt!(
|
||
{:error, :boom},
|
||
~U[2026-04-14 16:00:00Z],
|
||
18,
|
||
%Oban.Job{attempt: 5, max_attempts: 5}
|
||
)
|
||
|
||
assert [] = all_enqueued(worker: PropagationGridWorker)
|
||
end)
|
||
end
|
||
end
|
||
end
|