defmodule Microwaveprop.Weather.RebatchAsos do @moduledoc """ Converts pending single-station `WeatherFetchWorker` ASOS jobs into the batched `asos_batch` shape — one job per unique `(start_dt, end_dt)` window covering every station that had a pending job for that window. Callable from both a Mix task (`mix weather.rebatch_asos`) and a release `eval` shell (`bin/microwaveprop eval 'Microwaveprop.Weather.RebatchAsos.run()'`). In prod the Mix API isn't available, so this module is Mix-free. Idempotent: a second run with no pending "asos" jobs is a no-op. """ import Ecto.Query alias Microwaveprop.Repo alias Microwaveprop.Workers.WeatherFetchWorker @doc """ Run the rebatch. Options: * `:dry_run` (boolean) — log the plan but don't mutate the queue. Returns a summary map: `%{found: n, windows: n, inserted: n, cancelled: n}`. """ @spec run(keyword()) :: %{ found: non_neg_integer(), windows: non_neg_integer(), inserted: non_neg_integer(), cancelled: non_neg_integer() } def run(opts \\ []) do dry_run? = Keyword.get(opts, :dry_run, false) jobs = load_pending_asos_jobs() if jobs == [] do IO.puts("No pending `asos` jobs. Nothing to do.") %{found: 0, windows: 0, inserted: 0, cancelled: 0} else do_rebatch(jobs, dry_run?) end end defp load_pending_asos_jobs do query = from(j in Oban.Job, where: j.worker == "Microwaveprop.Workers.WeatherFetchWorker" and j.state in ["available", "scheduled", "retryable"] and fragment("?->>'fetch_type' = ?", j.args, "asos"), select: %{id: j.id, args: j.args} ) Repo.all(query) end defp do_rebatch(jobs, dry_run?) do groups = jobs |> Enum.group_by(fn %{args: a} -> {Map.get(a, "start_dt"), Map.get(a, "end_dt")} end) |> Enum.reject(fn {{s, e}, _} -> is_nil(s) or is_nil(e) end) IO.puts("Found #{length(jobs)} pending `asos` jobs across #{length(groups)} time windows.") batch_args = Enum.map(groups, fn {{start_dt, end_dt}, members} -> sorted = Enum.sort_by(members, fn %{args: a} -> a["station_code"] end) %{ "fetch_type" => "asos_batch", "station_ids" => Enum.map(sorted, fn %{args: a} -> a["station_id"] end), "station_codes" => Enum.map(sorted, fn %{args: a} -> a["station_code"] end), "start_dt" => start_dt, "end_dt" => end_dt, "_source_ids" => Enum.map(sorted, & &1.id) } end) if dry_run? do Enum.each(batch_args, fn a -> IO.puts("[dry-run] #{length(a["station_codes"])} stations for #{a["start_dt"]}..#{a["end_dt"]}") end) %{found: length(jobs), windows: length(groups), inserted: 0, cancelled: 0} else apply_rebatch(batch_args, length(jobs)) end end defp apply_rebatch(batch_args, found) do {inserted, cancelled} = Enum.reduce(batch_args, {0, 0}, fn a, {ins, can} -> source_ids = a["_source_ids"] job_args = Map.delete(a, "_source_ids") Repo.transaction(fn -> _ = job_args |> WeatherFetchWorker.new() |> Oban.insert!() {:ok, _n} = Oban.cancel_all_jobs(from(j in Oban.Job, where: j.id in ^source_ids)) end) {ins + 1, can + length(source_ids)} end) IO.puts("Rebatched: inserted #{inserted} batch jobs, cancelled #{cancelled} single-station jobs.") %{found: found, windows: length(batch_args), inserted: inserted, cancelled: cancelled} end end