prop/lib/microwaveprop/workers/hrrr_native_grid_worker.ex

defmodule Microwaveprop.Workers.HrrrNativeGridWorker do
  @moduledoc """
  Fetches one hour's worth of HRRR native hybrid-sigma profiles for
  every point of interest (contact location) and bulk-inserts them
  into `hrrr_native_profiles`.

  Jobs are unique on `{year, month, day, hour}` so a backfill sweep
  that enqueues duplicate hours collapses automatically.

  This is the batch companion to the existing `HrrrFetchWorker`. The
  native-level product is ~566 MB per run hour, so per-point
  on-demand fetching is impractical — see
  `docs/research/hrrr_native_levels.md`. Instead, each job grabs one
  file once and extracts native profiles for every point of interest
  in a single pass.
  """

  use Oban.Pro.Worker,
    queue: :hrrr,
    max_attempts: 3,
    unique: [
      period: :infinity,
      states: :incomplete,
      keys: [:year, :month, :day, :hour]
    ]

  import Ecto.Query

  alias Microwaveprop.Radio.Contact
  alias Microwaveprop.Repo
  alias Microwaveprop.Weather.HrrrClient
  alias Microwaveprop.Weather.HrrrNativeClient
  alias Microwaveprop.Weather.HrrrNativeProfile

  require Logger

  @impl Oban.Pro.Worker
  def process(%Oban.Job{args: args}) do
    %{"year" => year, "month" => month, "day" => day, "hour" => hour} = args
    {:ok, date} = Date.new(year, month, day)
    {:ok, valid_time} = DateTime.new(date, Time.new!(hour, 0, 0), "Etc/UTC")

    points = points_of_interest_for_hour(valid_time)

    cond do
      points == [] ->
        Logger.info("HrrrNativeGridWorker: no points for #{valid_time}, skipping")
        :ok

      already_ingested?(points, valid_time) ->
        Logger.info("HrrrNativeGridWorker: #{valid_time} already ingested, skipping")
        :ok

      true ->
        fetch_and_upsert(date, hour, valid_time, points)
    end
  end

  @doc false
  @spec points_of_interest_for_hour(DateTime.t()) :: [{float(), float()}]
  @batch_size 1000

  def points_of_interest_for_hour(valid_time) do
    time_start = DateTime.add(valid_time, -1800, :second)
    time_end = DateTime.add(valid_time, 1800, :second)

    Contact
    |> where([c], not is_nil(c.pos1))
    |> where([c], c.qso_timestamp >= ^time_start and c.qso_timestamp <= ^time_end)
    |> select([c], c.pos1)
    |> order_by([c], c.id)
    |> stream_batches(@batch_size)
    |> Enum.flat_map(fn pos ->
      case {pos["lat"], pos["lon"]} do
        {lat, lon} when is_number(lat) and is_number(lon) -> [{snap(lat), snap(lon)}]
        _ -> []
      end
    end)
    |> Enum.uniq()
  end

  defp stream_batches(query, batch_size) do
    Stream.resource(
      fn -> {query, 0} end,
      fn {query, offset} ->
        batch =
          query
          |> limit(^batch_size)
          |> offset(^offset)
          |> Repo.all()

        if batch == [] do
          {:halt, nil}
        else
          {batch, {query, offset + batch_size}}
        end
      end,
      fn _ -> :ok end
    )
  end

  defp snap(x), do: Float.round(x * 1.0, 3)

  defp already_ingested?(points, valid_time) do
    # Short-circuit: if we already have a profile for every point at
    # this valid_time, the worker is a no-op. Any missing point
    # triggers a full download (partial fills are rare; the cost of
    # re-downloading for a handful of gaps is acceptable).
    count =
      HrrrNativeProfile
      |> where([p], p.valid_time == ^valid_time)
      |> where(
        [p],
        fragment(
          "ROW(?, ?) IN (SELECT * FROM UNNEST(?::float[], ?::float[]))",
          p.lat,
          p.lon,
          ^Enum.map(points, &elem(&1, 0)),
          ^Enum.map(points, &elem(&1, 1))
        )
      )
      |> select([p], count(p.id))
      |> Repo.one()

    count >= length(points)
  end

  defp fetch_and_upsert(date, hour, valid_time, points) do
    url = HrrrNativeClient.hrrr_native_url(date, hour)
    idx_url = url <> ".idx"

    tmp_grib = Path.join(System.tmp_dir!(), "hrrr_native_#{System.unique_integer([:positive])}.grib2")

    try do
      fetch_and_upsert_with_file(date, hour, valid_time, points, url, idx_url, tmp_grib)
    after
      File.rm(tmp_grib)
    end
  end

  defp fetch_and_upsert_with_file(_date, _hour, valid_time, points, url, idx_url, tmp_grib) do
    with {:ok, idx_text} <- fetch_idx(idx_url),
         idx_entries = HrrrClient.parse_idx(idx_text),
         ranges = HrrrNativeClient.essential_byte_ranges(idx_entries),
         _ = Logger.info("HRRR native downloading #{length(ranges)} ranges for #{valid_time}"),
         :ok <- HrrrClient.download_grib_ranges_to_file(url, ranges, tmp_grib),
         _ = Logger.info("HRRR native downloaded to #{tmp_grib}, extracting #{length(points)} points..."),
         {:ok, profiles} <- HrrrNativeClient.extract_native_profiles_from_file(tmp_grib, points) do
      upsert_native_profiles(profiles, valid_time)
    else
      {:error, reason} ->
        Logger.warning("HrrrNativeGridWorker failed for #{valid_time}: #{inspect(reason)}")
        {:error, reason}
    end
  end

  defp upsert_native_profiles(profiles, valid_time) do
    now = DateTime.truncate(DateTime.utc_now(), :second)

    rows =
      Enum.flat_map(profiles, fn {{lat, lon}, profile} ->
        if profile[:level_count] && profile.level_count > 0 do
          [
            Map.merge(profile, %{
              id: Ecto.UUID.generate(),
              valid_time: valid_time,
              run_time: valid_time,
              lat: lat,
              lon: lon,
              inserted_at: now,
              updated_at: now
            })
          ]
        else
          []
        end
      end)

    {inserted, _} =
      Repo.insert_all(
        HrrrNativeProfile,
        rows,
        on_conflict: {:replace, ~w(run_time level_count heights_m temp_k spfh pressure_pa
              u_wind_ms v_wind_ms tke_m2s2
              surface_temp_k surface_spfh surface_pressure_pa
              inversion_top_m bulk_richardson theta_e_jump_k shear_at_top_ms
              ducts best_duct_band_ghz updated_at)a},
        conflict_target: [:lat, :lon, :valid_time]
      )

    Logger.info("HrrrNativeGridWorker: upserted #{inserted} profiles for #{valid_time}")
    :ok
  end

  defp fetch_idx(url) do
    case Req.get(url, receive_timeout: 120_000) do
      {:ok, %{status: 200, body: body}} -> {:ok, body}
      {:ok, %{status: status}} -> {:error, "HRRR native idx HTTP #{status}"}
      {:error, reason} -> {:error, reason}
    end
  end
end