defmodule Mix.Tasks.HrrrNativeBackfill do @shortdoc "Enqueue HrrrNativeGridWorker jobs for the top-N hours by contact count" @moduledoc """ Backfills the `hrrr_native_profiles` table by enqueueing one `HrrrNativeGridWorker` job per distinct `(year, month, day, hour)` where we have contacts, prioritized by contact count so the most data-dense hours land first. Native HRRR files are ~566 MB each, so think about the total bandwidth before running with a large limit. Typical plan: mix hrrr_native_backfill --limit 50 # one-time smoke backfill mix hrrr_native_backfill --limit 500 # full Phase 2 backfill (~280 GB) Jobs are deduplicated by Oban's unique constraint on `{year, month, day, hour}` so running the task twice is safe. """ use Mix.Task import Ecto.Query alias Microwaveprop.Radio.Contact alias Microwaveprop.Repo alias Microwaveprop.Workers.HrrrNativeGridWorker @impl Mix.Task def run(argv) do Mix.Task.run("app.start") _ = Oban.pause_all_queues(Oban) {opts, _, _} = OptionParser.parse(argv, switches: [limit: :integer]) limit = Keyword.get(opts, :limit, 50) hours = top_hours_by_contact_count(limit) Mix.shell().info("Enqueueing #{length(hours)} HrrrNativeGridWorker jobs") Enum.each(hours, fn %{year: y, month: m, day: d, hour: h, contacts: n} -> args = %{"year" => y, "month" => m, "day" => d, "hour" => h} case Oban.insert(HrrrNativeGridWorker.new(args)) do {:ok, _job} -> Mix.shell().info(" #{y}-#{pad(m)}-#{pad(d)} #{pad(h)}Z (#{n} contacts)") {:error, reason} -> Mix.shell().error(" #{y}-#{pad(m)}-#{pad(d)} #{pad(h)}Z failed: #{inspect(reason)}") end end) end defp pad(n), do: n |> Integer.to_string() |> String.pad_leading(2, "0") defp top_hours_by_contact_count(limit) do Repo.all( from(c in Contact, where: not is_nil(c.pos1), select: %{ year: fragment("EXTRACT(YEAR FROM ?)::int", c.qso_timestamp), month: fragment("EXTRACT(MONTH FROM ?)::int", c.qso_timestamp), day: fragment("EXTRACT(DAY FROM ?)::int", c.qso_timestamp), hour: fragment("EXTRACT(HOUR FROM ?)::int", c.qso_timestamp), contacts: count(c.id) }, group_by: [ fragment("EXTRACT(YEAR FROM ?)::int", c.qso_timestamp), fragment("EXTRACT(MONTH FROM ?)::int", c.qso_timestamp), fragment("EXTRACT(DAY FROM ?)::int", c.qso_timestamp), fragment("EXTRACT(HOUR FROM ?)::int", c.qso_timestamp) ], order_by: [desc: count(c.id)], limit: ^limit ) ) end end