Graham McIntire
e4668790a4
Mix.Task.run isn't available in a compiled release, so the in-prod `bin/microwaveprop eval 'Mix.Tasks.Weather.RebatchAsos.run(...)'` dispatch crashes. Moves the real logic into a plain module (`Microwaveprop.Weather.RebatchAsos`) that both the Mix task and a release eval can call. Uses IO.puts over Mix.shell for the same reason.
111 lines
3.5 KiB
Elixir
111 lines
3.5 KiB
Elixir
defmodule Microwaveprop.Weather.RebatchAsos do
|
|
@moduledoc """
|
|
Converts pending single-station `WeatherFetchWorker` ASOS jobs into
|
|
the batched `asos_batch` shape — one job per unique
|
|
`(start_dt, end_dt)` window covering every station that had a
|
|
pending job for that window.
|
|
|
|
Callable from both a Mix task (`mix weather.rebatch_asos`) and a
|
|
release `eval` shell (`bin/microwaveprop eval
|
|
'Microwaveprop.Weather.RebatchAsos.run()'`). In prod the Mix API
|
|
isn't available, so this module is Mix-free.
|
|
|
|
Idempotent: a second run with no pending "asos" jobs is a no-op.
|
|
"""
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Workers.WeatherFetchWorker
|
|
|
|
@doc """
|
|
Run the rebatch. Options:
|
|
* `:dry_run` (boolean) — log the plan but don't mutate the queue.
|
|
|
|
Returns a summary map: `%{found: n, windows: n, inserted: n, cancelled: n}`.
|
|
"""
|
|
@spec run(keyword()) :: %{
|
|
found: non_neg_integer(),
|
|
windows: non_neg_integer(),
|
|
inserted: non_neg_integer(),
|
|
cancelled: non_neg_integer()
|
|
}
|
|
def run(opts \\ []) do
|
|
dry_run? = Keyword.get(opts, :dry_run, false)
|
|
jobs = load_pending_asos_jobs()
|
|
|
|
if jobs == [] do
|
|
IO.puts("No pending `asos` jobs. Nothing to do.")
|
|
%{found: 0, windows: 0, inserted: 0, cancelled: 0}
|
|
else
|
|
do_rebatch(jobs, dry_run?)
|
|
end
|
|
end
|
|
|
|
defp load_pending_asos_jobs do
|
|
query =
|
|
from(j in Oban.Job,
|
|
where:
|
|
j.worker == "Microwaveprop.Workers.WeatherFetchWorker" and
|
|
j.state in ["available", "scheduled", "retryable"] and
|
|
fragment("?->>'fetch_type' = ?", j.args, "asos"),
|
|
select: %{id: j.id, args: j.args}
|
|
)
|
|
|
|
Repo.all(query)
|
|
end
|
|
|
|
defp do_rebatch(jobs, dry_run?) do
|
|
groups =
|
|
jobs
|
|
|> Enum.group_by(fn %{args: a} ->
|
|
{Map.get(a, "start_dt"), Map.get(a, "end_dt")}
|
|
end)
|
|
|> Enum.reject(fn {{s, e}, _} -> is_nil(s) or is_nil(e) end)
|
|
|
|
IO.puts("Found #{length(jobs)} pending `asos` jobs across #{length(groups)} time windows.")
|
|
|
|
batch_args =
|
|
Enum.map(groups, fn {{start_dt, end_dt}, members} ->
|
|
sorted = Enum.sort_by(members, fn %{args: a} -> a["station_code"] end)
|
|
|
|
%{
|
|
"fetch_type" => "asos_batch",
|
|
"station_ids" => Enum.map(sorted, fn %{args: a} -> a["station_id"] end),
|
|
"station_codes" => Enum.map(sorted, fn %{args: a} -> a["station_code"] end),
|
|
"start_dt" => start_dt,
|
|
"end_dt" => end_dt,
|
|
"_source_ids" => Enum.map(sorted, & &1.id)
|
|
}
|
|
end)
|
|
|
|
if dry_run? do
|
|
Enum.each(batch_args, fn a ->
|
|
IO.puts("[dry-run] #{length(a["station_codes"])} stations for #{a["start_dt"]}..#{a["end_dt"]}")
|
|
end)
|
|
|
|
%{found: length(jobs), windows: length(groups), inserted: 0, cancelled: 0}
|
|
else
|
|
apply_rebatch(batch_args, length(jobs))
|
|
end
|
|
end
|
|
|
|
defp apply_rebatch(batch_args, found) do
|
|
{inserted, cancelled} =
|
|
Enum.reduce(batch_args, {0, 0}, fn a, {ins, can} ->
|
|
source_ids = a["_source_ids"]
|
|
job_args = Map.delete(a, "_source_ids")
|
|
|
|
Repo.transaction(fn ->
|
|
_ = job_args |> WeatherFetchWorker.new() |> Oban.insert!()
|
|
{:ok, _n} = Oban.cancel_all_jobs(from(j in Oban.Job, where: j.id in ^source_ids))
|
|
end)
|
|
|
|
{ins + 1, can + length(source_ids)}
|
|
end)
|
|
|
|
IO.puts("Rebatched: inserted #{inserted} batch jobs, cancelled #{cancelled} single-station jobs.")
|
|
|
|
%{found: found, windows: length(batch_args), inserted: inserted, cancelled: cancelled}
|
|
end
|
|
end
|