defmodule Microwaveprop.Workers.HrrrNativeGridWorker do @moduledoc """ Fetches one hour's worth of HRRR native hybrid-sigma profiles for every point of interest (contact location) and bulk-inserts them into `hrrr_native_profiles`. Jobs are unique on `{year, month, day, hour}` so a backfill sweep that enqueues duplicate hours collapses automatically. This is the batch companion to the existing `HrrrFetchWorker`. The native-level product is ~566 MB per run hour, so per-point on-demand fetching is impractical — see `docs/research/hrrr_native_levels.md`. Instead, each job grabs one file once and extracts native profiles for every point of interest in a single pass. """ use Oban.Worker, queue: :hrrr, max_attempts: 3, unique: [ period: :infinity, states: [:available, :scheduled, :executing, :retryable], keys: [:year, :month, :day, :hour] ] import Ecto.Query alias Microwaveprop.Radio.Contact alias Microwaveprop.Repo alias Microwaveprop.Weather.HrrrClient alias Microwaveprop.Weather.HrrrNativeClient alias Microwaveprop.Weather.HrrrNativeProfile require Logger @impl Oban.Worker def perform(%Oban.Job{args: args}) do %{"year" => year, "month" => month, "day" => day, "hour" => hour} = args {:ok, date} = Date.new(year, month, day) {:ok, valid_time} = DateTime.new(date, Time.new!(hour, 0, 0), "Etc/UTC") points = points_of_interest_for_hour(valid_time) cond do points == [] -> Logger.info("HrrrNativeGridWorker: no points for #{valid_time}, skipping") :ok already_ingested?(points, valid_time) -> Logger.info("HrrrNativeGridWorker: #{valid_time} already ingested, skipping") :ok true -> fetch_and_upsert(date, hour, valid_time, points) end end @doc false def points_of_interest_for_hour(valid_time) do time_start = DateTime.add(valid_time, -1800, :second) time_end = DateTime.add(valid_time, 1800, :second) Contact |> where([c], not is_nil(c.pos1)) |> where([c], c.qso_timestamp >= ^time_start and c.qso_timestamp <= ^time_end) |> select([c], c.pos1) |> Repo.all() |> Enum.flat_map(fn pos -> case {pos["lat"], pos["lon"] || pos["lng"]} do {lat, lon} when is_number(lat) and is_number(lon) -> [{snap(lat), snap(lon)}] _ -> [] end end) |> Enum.uniq() end defp snap(x), do: Float.round(x * 1.0, 3) defp already_ingested?(points, valid_time) do # Short-circuit: if we already have a profile for every point at # this valid_time, the worker is a no-op. Any missing point # triggers a full download (partial fills are rare; the cost of # re-downloading for a handful of gaps is acceptable). count = HrrrNativeProfile |> where([p], p.valid_time == ^valid_time) |> where( [p], fragment( "ROW(?, ?) IN (SELECT * FROM UNNEST(?::float[], ?::float[]))", p.lat, p.lon, ^Enum.map(points, &elem(&1, 0)), ^Enum.map(points, &elem(&1, 1)) ) ) |> select([p], count(p.id)) |> Repo.one() count >= length(points) end defp fetch_and_upsert(date, hour, valid_time, points) do url = HrrrNativeClient.hrrr_native_url(date, hour) idx_url = url <> ".idx" with {:ok, idx_text} <- fetch_idx(idx_url), idx_entries = HrrrClient.parse_idx(idx_text), ranges = HrrrNativeClient.essential_byte_ranges(idx_entries), _ = Logger.info("HRRR native downloading #{length(ranges)} ranges for #{valid_time}"), {:ok, grib_binary} <- HrrrClient.download_grib_ranges(url, ranges), _ = Logger.info("HRRR native downloaded #{byte_size(grib_binary)} bytes, extracting #{length(points)} points..."), {:ok, profiles} <- HrrrNativeClient.extract_native_profiles(grib_binary, points) do rows = Enum.flat_map(profiles, fn {{lat, lon}, profile} -> if profile[:level_count] && profile.level_count > 0 do now = DateTime.truncate(DateTime.utc_now(), :second) [ Map.merge(profile, %{ id: Ecto.UUID.bingenerate(), valid_time: valid_time, run_time: valid_time, lat: lat, lon: lon, inserted_at: now, updated_at: now }) ] else [] end end) {inserted, _} = Repo.insert_all( HrrrNativeProfile, rows, on_conflict: {:replace_all_except, [:id, :inserted_at]}, conflict_target: [:lat, :lon, :valid_time] ) Logger.info("HrrrNativeGridWorker: upserted #{inserted} profiles for #{valid_time}") :ok else {:error, reason} -> Logger.warning("HrrrNativeGridWorker failed for #{valid_time}: #{inspect(reason)}") {:error, reason} end end defp fetch_idx(url) do case Req.get(url, receive_timeout: 120_000) do {:ok, %{status: 200, body: body}} -> {:ok, body} {:ok, %{status: status}} -> {:error, "HRRR native idx HTTP #{status}"} {:error, reason} -> {:error, reason} end end end