Graham McIntire
900685aa06
- Spike docs at docs/research/hrrr_native_levels.md confirming files
are on AWS S3 for 5+ years, 50 hybrid levels, and include TKE and
SPFH needed for Phase 2 turbulence features. Architectural finding:
per-point on-demand fetching is impractical (~530 MB/file), so
the ingestion worker batches per (date, hour) instead.
- hrrr_native_profiles schema: arrays per level plus cached surface
scalars and placeholder columns for Phase 2/4 derived fields.
Strictly additive — the existing hrrr_profiles table is untouched.
- HrrrNativeClient: pure URL/message-list helpers, build_native_profile/1
that turns a parsed wgrib2 map into the schema shape (TDD'd).
- Exposed HrrrClient.download_grib_ranges/2 so the native client
reuses the existing parallel byte-range download + disk cache.
- HrrrNativeGridWorker: Oban worker keyed on {year, month, day, hour},
unique at :infinity, pulls distinct (lat, lon) points from contacts
in the ±30 min window, downloads the native grib2, extracts per
point, bulk-upserts.
- mix hrrr_native_backfill --limit N enqueues the top-N hours by
contact count.
Phase 1 gate still pending Task 1.6 (sanity-check backtest after
live data lands).
69 lines
2.5 KiB
Elixir
69 lines
2.5 KiB
Elixir
defmodule Mix.Tasks.HrrrNativeBackfill do
|
|
@shortdoc "Enqueue HrrrNativeGridWorker jobs for the top-N hours by contact count"
|
|
@moduledoc """
|
|
Backfills the `hrrr_native_profiles` table by enqueueing one
|
|
`HrrrNativeGridWorker` job per distinct `(year, month, day, hour)`
|
|
where we have contacts, prioritized by contact count so the most
|
|
data-dense hours land first.
|
|
|
|
Native HRRR files are ~566 MB each, so think about the total
|
|
bandwidth before running with a large limit. Typical plan:
|
|
|
|
mix hrrr_native_backfill --limit 50 # one-time smoke backfill
|
|
mix hrrr_native_backfill --limit 500 # full Phase 2 backfill (~280 GB)
|
|
|
|
Jobs are deduplicated by Oban's unique constraint on
|
|
`{year, month, day, hour}` so running the task twice is safe.
|
|
"""
|
|
use Mix.Task
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Workers.HrrrNativeGridWorker
|
|
|
|
@impl Mix.Task
|
|
def run(argv) do
|
|
Mix.Task.run("app.start")
|
|
|
|
{opts, _, _} = OptionParser.parse(argv, switches: [limit: :integer])
|
|
limit = Keyword.get(opts, :limit, 50)
|
|
|
|
hours = top_hours_by_contact_count(limit)
|
|
Mix.shell().info("Enqueueing #{length(hours)} HrrrNativeGridWorker jobs")
|
|
|
|
Enum.each(hours, fn %{year: y, month: m, day: d, hour: h, contacts: n} ->
|
|
args = %{"year" => y, "month" => m, "day" => d, "hour" => h}
|
|
|
|
case Oban.insert(HrrrNativeGridWorker.new(args)) do
|
|
{:ok, _job} -> Mix.shell().info(" #{y}-#{pad(m)}-#{pad(d)} #{pad(h)}Z (#{n} contacts)")
|
|
{:error, reason} -> Mix.shell().error(" #{y}-#{pad(m)}-#{pad(d)} #{pad(h)}Z failed: #{inspect(reason)}")
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp pad(n), do: n |> Integer.to_string() |> String.pad_leading(2, "0")
|
|
|
|
defp top_hours_by_contact_count(limit) do
|
|
from(c in Contact,
|
|
where: not is_nil(c.pos1),
|
|
select: %{
|
|
year: fragment("EXTRACT(YEAR FROM ?)::int", c.qso_timestamp),
|
|
month: fragment("EXTRACT(MONTH FROM ?)::int", c.qso_timestamp),
|
|
day: fragment("EXTRACT(DAY FROM ?)::int", c.qso_timestamp),
|
|
hour: fragment("EXTRACT(HOUR FROM ?)::int", c.qso_timestamp),
|
|
contacts: count(c.id)
|
|
},
|
|
group_by: [
|
|
fragment("EXTRACT(YEAR FROM ?)::int", c.qso_timestamp),
|
|
fragment("EXTRACT(MONTH FROM ?)::int", c.qso_timestamp),
|
|
fragment("EXTRACT(DAY FROM ?)::int", c.qso_timestamp),
|
|
fragment("EXTRACT(HOUR FROM ?)::int", c.qso_timestamp)
|
|
],
|
|
order_by: [desc: count(c.id)],
|
|
limit: ^limit
|
|
)
|
|
|> Repo.all()
|
|
end
|
|
end
|