prop/test/microwaveprop/workers/pskr_calibration_worker_test.exs
Graham McIntire fe2ad2df15
fix(pskr): self-healing follow-up sampler runs for hours with missing HRRR
The producer-only path left older hours stuck at 0% HRRR coverage
forever once their fetch tasks drained: the rolling-window cron only
re-passes the prior hour, so manual lookback runs (or any
out-of-window hour) wrote NULL samples + enqueued tasks, then never
re-ran to UPSERT the now-landed HRRR data.

Fix: split the responsibility cleanly.

* CalibrationSampler.build_for_hour/1 now returns
  %{upserted: int, missing_hrrr_cells: int} so callers know whether
  the producer enqueued anything (i.e. whether a follow-up makes
  sense). Spec-tightened with a new @type result.
* PskrCalibrationWorker, after each hour's sampler pass, schedules a
  follow-up of itself for that exact hour 10 min later when missing
  > 0. The follow-up carries args %{"hour_utc" => ..., "_follow_up"
  => true}; the sentinel prevents follow-ups from chaining further
  follow-ups (rolling-window cron is the safety net).

Tests:
* Updated 4 existing sampler tests to the new map return shape.
* Moved the follow-up assertions from the sampler suite into the
  worker suite where they belong (sampler is now scheduling-free).
* Added 3 worker tests: schedules-when-missing, no-schedule-when-
  covered, follow-up-doesn't-chain.

3313 tests + 228 properties, 0 failures.
2026-05-05 18:00:10 -05:00

179 lines
6 KiB
Elixir

defmodule Microwaveprop.Workers.PskrCalibrationWorkerTest do
use Microwaveprop.DataCase, async: true
use Oban.Testing, repo: Microwaveprop.Repo
alias Microwaveprop.Pskr.CalibrationSample
alias Microwaveprop.Pskr.SpotHourly
alias Microwaveprop.Repo
alias Microwaveprop.Weather.HrrrProfile
alias Microwaveprop.Workers.PskrCalibrationWorker
defp insert_spot!(hour_utc, opts \\ []) do
sender_grid = Keyword.get(opts, :sender_grid, "EM12KL")
midpoint_lat = Keyword.get(opts, :midpoint_lat, 33.0)
midpoint_lon = Keyword.get(opts, :midpoint_lon, -97.0)
{:ok, _} =
%SpotHourly{}
|> SpotHourly.changeset(%{
hour_utc: hour_utc,
band: "10000",
sender_grid: sender_grid,
receiver_grid: "DM43ST",
spot_count: 1,
midpoint_lat: midpoint_lat,
midpoint_lon: midpoint_lon,
distance_km: 200.0,
modes: ["FT8"]
})
|> Repo.insert()
end
defp truncate_to_hour(dt), do: %{dt | minute: 0, second: 0, microsecond: {0, 0}}
describe "perform/1 with explicit hour_utc" do
test "samples just that hour" do
hour = ~U[2026-05-04 18:00:00Z]
insert_spot!(hour)
assert :ok =
perform_job(PskrCalibrationWorker, %{"hour_utc" => DateTime.to_iso8601(hour)})
[sample] = Repo.all(CalibrationSample)
assert sample.hour_utc == hour
end
test "succeeds when the hour has no spots" do
assert :ok =
perform_job(PskrCalibrationWorker, %{"hour_utc" => "2026-05-04T03:00:00Z"})
assert Repo.aggregate(CalibrationSample, :count) == 0
end
end
describe "perform/1 default rolling window" do
test "samples both the current hour and the previous hour" do
# Frequent processing means a single fire must cover the hour
# boundary: spots in the just-ended hour and spots in the
# in-progress hour both get sample rows. Otherwise spots that
# arrive in the last few minutes of an hour wait up to a full
# cron interval before they're sampled.
now = truncate_to_hour(DateTime.utc_now())
previous = DateTime.add(now, -3600, :second)
insert_spot!(now)
insert_spot!(previous, sender_grid: "EM10AA")
assert :ok = perform_job(PskrCalibrationWorker, %{})
hours = CalibrationSample |> Repo.all() |> Enum.map(& &1.hour_utc) |> Enum.sort()
assert hours == [previous, now]
end
test "is idempotent — re-running merges into the same rows" do
now = truncate_to_hour(DateTime.utc_now())
insert_spot!(now)
assert :ok = perform_job(PskrCalibrationWorker, %{})
first_count = Repo.aggregate(CalibrationSample, :count)
assert :ok = perform_job(PskrCalibrationWorker, %{})
assert Repo.aggregate(CalibrationSample, :count) == first_count
end
end
describe "perform/1 with explicit lookback_hours" do
test "extends the rolling window to backfill recent gaps" do
# Operator sets lookback_hours=3 to recover from a short pod
# outage. The worker fires once and produces samples for the
# current hour plus the three before it.
now = truncate_to_hour(DateTime.utc_now())
Enum.each(0..3, fn offset ->
hour = DateTime.add(now, -offset * 3600, :second)
insert_spot!(hour, sender_grid: "EM1#{offset}AA")
end)
assert :ok = perform_job(PskrCalibrationWorker, %{"lookback_hours" => 3})
assert Repo.aggregate(CalibrationSample, :count) == 4
end
end
describe "follow-up scheduling" do
# Self-healing pipeline: when the sampler reports cells whose HRRR
# data isn't yet in the DB (so it just enqueued hrrr_fetch_tasks
# for the Rust hrrr-point-worker to drain), the worker schedules
# a re-run of itself for that exact hour ~10 min later. By that
# point the Rust worker has typically drained the batch, and the
# follow-up re-run UPSERTs the samples with non-NULL HRRR.
test "schedules a follow-up for hours with missing HRRR data" do
Oban.Testing.with_testing_mode(:manual, fn ->
hour = ~U[2026-05-04 18:00:00Z]
insert_spot!(hour)
assert :ok =
perform_job(PskrCalibrationWorker, %{"hour_utc" => DateTime.to_iso8601(hour)})
assert_enqueued(
worker: PskrCalibrationWorker,
args: %{
"hour_utc" => DateTime.to_iso8601(hour),
"_follow_up" => true
}
)
end)
end
test "does not schedule a follow-up when no cells are missing HRRR" do
# Plant the HRRR profile so the sampler finds it on first pass.
hour = ~U[2026-05-04 18:00:00Z]
insert_spot!(hour)
{:ok, _} =
%HrrrProfile{}
|> HrrrProfile.changeset(%{
valid_time: hour,
lat: 33.0,
lon: -97.0,
run_time: hour,
surface_temp_c: 20.0,
surface_dewpoint_c: 15.0,
pwat_mm: 25.0,
surface_pressure_mb: 1013.0,
min_refractivity_gradient: -100.0,
hpbl_m: 500.0,
ducting_detected: false,
profile: []
})
|> Repo.insert()
Oban.Testing.with_testing_mode(:manual, fn ->
assert :ok =
perform_job(PskrCalibrationWorker, %{"hour_utc" => DateTime.to_iso8601(hour)})
refute_enqueued(worker: PskrCalibrationWorker)
end)
end
test "a follow-up re-run does not chain another follow-up (loop terminator)" do
# Even if the follow-up re-run STILL finds missing HRRR (e.g.
# the Rust worker is slow), the `_follow_up: true` sentinel
# prevents chaining a second follow-up. The rolling-window
# cron is the safety net for that case.
Oban.Testing.with_testing_mode(:manual, fn ->
hour = ~U[2026-05-04 18:00:00Z]
insert_spot!(hour)
assert :ok =
perform_job(PskrCalibrationWorker, %{
"hour_utc" => DateTime.to_iso8601(hour),
"_follow_up" => true
})
refute_enqueued(worker: PskrCalibrationWorker)
end)
end
end
end