Graham McIntire
95c9ac3dcd
- IemRateLimiter gains AIMD-style adaptive spacing. signal_429/0 widens the current gap (*= 1.5, capped at max_interval_ms default 10s); signal_success/0 narrows it back toward the configured base (*= 0.92, floored at interval_ms). Self-tunes to IEM's moving ceiling without needing the manual "safe for 4 pods" constant. - IemClient now routes every response through a central handle_response helper that fires the widen/narrow feedback signals, eliminating the four near-identical case blocks. - Mix.Tasks.Weather.RebatchAsos collapses any already-queued single- station "asos" jobs into the batched "asos_batch" shape the enqueuer now emits, so the pending backfill queue converts to the new per-request-efficient path instead of draining at the old rate. Idempotent; supports --dry-run. 2833 tests + credo green.
89 lines
3.2 KiB
Elixir
89 lines
3.2 KiB
Elixir
defmodule Mix.Tasks.Weather.RebatchAsosTest do
|
|
use Microwaveprop.DataCase, async: false
|
|
|
|
alias Microwaveprop.Weather.IemClient
|
|
alias Microwaveprop.Workers.WeatherFetchWorker
|
|
alias Mix.Tasks.Weather.RebatchAsos
|
|
|
|
setup do
|
|
# Oban runs inline in tests, so the batch job the task inserts
|
|
# actually executes. Stub IEM so that execution is a quiet no-op
|
|
# (stations don't exist, so the batch handler just logs "missing"
|
|
# and returns :ok).
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
Req.Test.text(conn, "#DEBUG\nstation,valid,tmpf\n")
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
# Oban runs `testing: :inline` in this project, so Oban.insert! would
|
|
# execute the job immediately. We stash raw Oban.Job rows so we can
|
|
# assert on the queue state the rebatch task would see in prod.
|
|
defp insert_asos_job(station_id, station_code, start_dt, end_dt) do
|
|
changeset =
|
|
WeatherFetchWorker.new(%{
|
|
"fetch_type" => "asos",
|
|
"station_id" => station_id,
|
|
"station_code" => station_code,
|
|
"start_dt" => start_dt,
|
|
"end_dt" => end_dt
|
|
})
|
|
|
|
# Force state=available so the task picks it up; Oban.Job defaults
|
|
# to :available already but we set it explicitly for clarity.
|
|
changeset
|
|
|> Ecto.Changeset.put_change(:state, "available")
|
|
|> Repo.insert!()
|
|
end
|
|
|
|
defp pending_count_by_type do
|
|
import Ecto.Query
|
|
|
|
from(j in Oban.Job,
|
|
where: j.state in ["available", "scheduled", "retryable"],
|
|
select: {fragment("?->>'fetch_type'", j.args), count(j.id)},
|
|
group_by: fragment("?->>'fetch_type'", j.args)
|
|
)
|
|
|> Repo.all()
|
|
|> Map.new()
|
|
end
|
|
|
|
test "collapses N single-station jobs sharing a window into one batch" do
|
|
insert_asos_job(Ecto.UUID.generate(), "KDFW", "2026-03-28T16:00:00Z", "2026-03-28T20:00:00Z")
|
|
insert_asos_job(Ecto.UUID.generate(), "KFTW", "2026-03-28T16:00:00Z", "2026-03-28T20:00:00Z")
|
|
insert_asos_job(Ecto.UUID.generate(), "KAFW", "2026-03-28T16:00:00Z", "2026-03-28T20:00:00Z")
|
|
|
|
output = ExUnit.CaptureIO.capture_io(fn -> RebatchAsos.run([]) end)
|
|
|
|
assert output =~ "3 pending `asos` jobs across 1 time windows"
|
|
assert output =~ "inserted 1 batch jobs"
|
|
assert output =~ "cancelled 3 single-station jobs"
|
|
end
|
|
|
|
test "groups per (start_dt, end_dt) window" do
|
|
insert_asos_job(Ecto.UUID.generate(), "KDFW", "2026-03-28T16:00:00Z", "2026-03-28T20:00:00Z")
|
|
insert_asos_job(Ecto.UUID.generate(), "KFTW", "2026-03-28T16:00:00Z", "2026-03-28T20:00:00Z")
|
|
insert_asos_job(Ecto.UUID.generate(), "KAUS", "2026-03-29T10:00:00Z", "2026-03-29T14:00:00Z")
|
|
|
|
output = ExUnit.CaptureIO.capture_io(fn -> RebatchAsos.run([]) end)
|
|
|
|
assert output =~ "3 pending `asos` jobs across 2 time windows"
|
|
assert output =~ "inserted 2 batch jobs"
|
|
end
|
|
|
|
test "dry-run does not mutate the queue" do
|
|
insert_asos_job(Ecto.UUID.generate(), "KDFW", "2026-03-28T16:00:00Z", "2026-03-28T20:00:00Z")
|
|
|
|
ExUnit.CaptureIO.capture_io(fn -> RebatchAsos.run(["--dry-run"]) end)
|
|
|
|
counts = pending_count_by_type()
|
|
assert counts["asos"] == 1
|
|
assert counts["asos_batch"] in [nil, 0]
|
|
end
|
|
|
|
test "no pending asos jobs — short-circuits" do
|
|
output = ExUnit.CaptureIO.capture_io(fn -> RebatchAsos.run([]) end)
|
|
assert output =~ "Nothing to do"
|
|
end
|
|
end
|