Graham McIntire
95c9ac3dcd
- IemRateLimiter gains AIMD-style adaptive spacing. signal_429/0 widens the current gap (*= 1.5, capped at max_interval_ms default 10s); signal_success/0 narrows it back toward the configured base (*= 0.92, floored at interval_ms). Self-tunes to IEM's moving ceiling without needing the manual "safe for 4 pods" constant. - IemClient now routes every response through a central handle_response helper that fires the widen/narrow feedback signals, eliminating the four near-identical case blocks. - Mix.Tasks.Weather.RebatchAsos collapses any already-queued single- station "asos" jobs into the batched "asos_batch" shape the enqueuer now emits, so the pending backfill queue converts to the new per-request-efficient path instead of draining at the old rate. Idempotent; supports --dry-run. 2833 tests + credo green.
112 lines
3.5 KiB
Elixir
112 lines
3.5 KiB
Elixir
defmodule Mix.Tasks.Weather.RebatchAsos do
|
|
@shortdoc "Collapse pending single-station ASOS jobs into batched jobs"
|
|
|
|
@moduledoc """
|
|
Converts every pending single-station `WeatherFetchWorker` ASOS job
|
|
into the batched (`asos_batch`) shape — one job per unique
|
|
`(start_dt, end_dt)` window covering every station that had a
|
|
pending job for that window.
|
|
|
|
Why: the worker still accepts the old `"asos"` fetch_type for
|
|
backward compat, but every such job pays the full `IemRateLimiter`
|
|
gap + IEM 429-retry tail for a single station's rows. Collapsing
|
|
them into batched jobs gets the queue through the backlog at the
|
|
new per-request efficiency.
|
|
|
|
Usage (idempotent — running it twice is a no-op):
|
|
|
|
mix weather.rebatch_asos # rewrites everything
|
|
mix weather.rebatch_asos --dry-run # report only
|
|
|
|
"""
|
|
|
|
use Mix.Task
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Workers.WeatherFetchWorker
|
|
|
|
require Logger
|
|
|
|
@impl Mix.Task
|
|
def run(args) do
|
|
{opts, _, _} = OptionParser.parse(args, strict: [dry_run: :boolean])
|
|
Mix.Task.run("app.start")
|
|
dry_run? = Keyword.get(opts, :dry_run, false)
|
|
|
|
jobs = load_pending_asos_jobs()
|
|
|
|
if jobs == [] do
|
|
Mix.shell().info("No pending `asos` jobs. Nothing to do.")
|
|
else
|
|
do_rebatch(jobs, dry_run?)
|
|
end
|
|
end
|
|
|
|
@doc false
|
|
def load_pending_asos_jobs do
|
|
query =
|
|
from(j in Oban.Job,
|
|
where:
|
|
j.worker == "Microwaveprop.Workers.WeatherFetchWorker" and
|
|
j.state in ["available", "scheduled", "retryable"] and
|
|
fragment("?->>'fetch_type' = ?", j.args, "asos"),
|
|
select: %{id: j.id, args: j.args}
|
|
)
|
|
|
|
Repo.all(query)
|
|
end
|
|
|
|
defp do_rebatch(jobs, dry_run?) do
|
|
groups =
|
|
jobs
|
|
|> Enum.group_by(fn %{args: a} ->
|
|
{Map.get(a, "start_dt"), Map.get(a, "end_dt")}
|
|
end)
|
|
|> Enum.reject(fn {{s, e}, _} -> is_nil(s) or is_nil(e) end)
|
|
|
|
Mix.shell().info("Found #{length(jobs)} pending `asos` jobs across #{length(groups)} time windows.")
|
|
|
|
batch_args =
|
|
Enum.map(groups, fn {{start_dt, end_dt}, members} ->
|
|
# Deterministic sort (matches the enqueuer's ordering) so a
|
|
# re-run with identical membership hits Oban's unique dedup.
|
|
sorted = Enum.sort_by(members, fn %{args: a} -> a["station_code"] end)
|
|
|
|
%{
|
|
"fetch_type" => "asos_batch",
|
|
"station_ids" => Enum.map(sorted, fn %{args: a} -> a["station_id"] end),
|
|
"station_codes" => Enum.map(sorted, fn %{args: a} -> a["station_code"] end),
|
|
"start_dt" => start_dt,
|
|
"end_dt" => end_dt,
|
|
"_source_ids" => Enum.map(sorted, & &1.id)
|
|
}
|
|
end)
|
|
|
|
if dry_run? do
|
|
Enum.each(batch_args, fn a ->
|
|
Mix.shell().info("[dry-run] #{length(a["station_codes"])} stations for #{a["start_dt"]}..#{a["end_dt"]}")
|
|
end)
|
|
else
|
|
apply_rebatch(batch_args)
|
|
end
|
|
end
|
|
|
|
defp apply_rebatch(batch_args) do
|
|
{inserted, cancelled} =
|
|
Enum.reduce(batch_args, {0, 0}, fn a, {ins, can} ->
|
|
source_ids = a["_source_ids"]
|
|
job_args = Map.delete(a, "_source_ids")
|
|
|
|
Repo.transaction(fn ->
|
|
_ = job_args |> WeatherFetchWorker.new() |> Oban.insert!()
|
|
{:ok, _n} = Oban.cancel_all_jobs(from(j in Oban.Job, where: j.id in ^source_ids))
|
|
end)
|
|
|
|
{ins + 1, can + length(source_ids)}
|
|
end)
|
|
|
|
Mix.shell().info("Rebatched: inserted #{inserted} batch jobs, cancelled #{cancelled} single-station jobs.")
|
|
end
|
|
end
|