prop/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex

defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
  @moduledoc false
  use Oban.Pro.Worker, queue: :enqueue, max_attempts: 3

  alias Microwaveprop.Propagation.Grid
  alias Microwaveprop.Radio
  alias Microwaveprop.Radio.Contact
  alias Microwaveprop.Weather
  alias Microwaveprop.Weather.HrrrPointEnqueuer
  alias Microwaveprop.Weather.NarrClient
  alias Microwaveprop.Weather.NexradClient
  alias Microwaveprop.Workers.IemreFetchWorker
  alias Microwaveprop.Workers.MechanismClassifyWorker
  alias Microwaveprop.Workers.NarrFetchWorker
  alias Microwaveprop.Workers.RadarFrameWorker
  alias Microwaveprop.Workers.TerrainProfileWorker
  alias Microwaveprop.Workers.WeatherFetchWorker

  require Logger

  @asos_radius_km 150
  @sounding_radius_km 300
  # Oban jobs have ~149 params each; PG limit is 65535 params per query
  @insert_batch_size 400

  @doc """
  Enqueue all enrichment jobs (weather, HRRR, terrain, IEMRE) for a single contact.
  Called directly from submission flow — no Oban indirection.
  Optionally pass a list of types to limit which enrichments run.
  """
  @spec enqueue_for_contact(Contact.t(), [atom()]) :: :ok
  def enqueue_for_contact(contact, types \\ [:hrrr, :weather, :terrain, :iemre, :radar, :mechanism]) do
    contact = Radio.ensure_positions!(contact)
    # BackfillEnqueueWorker selects any contact with at least one
    # *_status in [:pending, :queued, :failed] and calls this with all
    # types. Filter out types that are already :complete for this
    # contact so mark_status!/3 doesn't demote them back to :queued on
    # every cron tick — that caused tens of thousands of
    # mechanism/terrain rows to ping-pong between :complete and :queued.
    types = Enum.reject(types, &already_complete?(contact, &1))
    jobs_by_type = build_jobs_by_type(contact, types)

    bulk_jobs =
      jobs_by_type[:weather] ++
        jobs_by_type[:terrain] ++
        jobs_by_type[:iemre] ++
        jobs_by_type[:radar] ++
        jobs_by_type[:mechanism]

    if bulk_jobs != [] do
      insert_all_chunked(bulk_jobs)
    end

    # HRRR enrichment runs via the Rust hrrr-point-worker via the
    # hrrr_fetch_tasks table (Phase 3 Stream C). Grouping by valid_time
    # and UPSERT-union means a backfill tick that adds a new contact in
    # an already-scheduled hour collapses into the existing row instead
    # of creating a duplicate fetch.
    _ =
      if :hrrr in types do
        {:ok, _} = HrrrPointEnqueuer.enqueue_for_contacts([contact])
      end

    # NARR jobs use Oban.insert_all/1 for batching. Oban OSS insert_all
    # does not check uniqueness, but the worker's unique fields on
    # (lat, lon, valid_time) de-duplicate at execution time.
    _ = Oban.insert_all(jobs_by_type[:narr])

    _ = mark_enrichment_statuses(contact, types, jobs_by_type)

    :ok
  end

  defp build_jobs_by_type(contact, types) do
    %{
      weather: if(:weather in types, do: build_weather_jobs([contact]), else: []),
      # hrrr jobs are routed to hrrr_fetch_tasks (not Oban) but the
      # status-marking path still wants to know whether any HRRR work
      # was created for this contact, so we mirror the list-shape.
      hrrr: if(:hrrr in types, do: hrrr_placeholder_jobs([contact]), else: []),
      terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
      iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
      narr: if(:narr in types, do: build_narr_jobs([contact]), else: []),
      radar: if(:radar in types, do: build_radar_jobs([contact]), else: []),
      mechanism: if(:mechanism in types, do: build_mechanism_jobs([contact]), else: [])
    }
  end

  # Placeholder for the HRRR job-list signal. Returns a truthy singleton
  # if the contact has at least one HRRR-eligible path point so
  # mark_hrrr_status!/3 flips hrrr_queued on the contact row. Actual
  # scheduling happens via HrrrPointEnqueuer.enqueue_for_contacts/1
  # above. Returns [] for pre-coverage contacts (NARR's territory) so
  # the status stays :unavailable, and [] for contacts whose HRRR
  # profiles are already fully present in the DB so mark_hrrr_status!/3
  # flips them directly to :complete — otherwise backfill re-flags the
  # contact as :queued every cron cycle forever (HrrrPointEnqueuer
  # sees 0 missing points, inserts nothing, but the status stays
  # :queued and the contact is picked up again next tick).
  defp hrrr_placeholder_jobs(contacts) do
    # Collect all unique HRRR grid points across all contacts, query
    # presence once, then map back per-contact — avoids N+1 per-contact
    # queries through hrrr_data_fully_present?/1.
    {point_map, present_set} = build_hrrr_present_set(contacts)

    Enum.flat_map(contacts, fn contact ->
      hrrr_placeholder_for_contact(contact, point_map, present_set)
    end)
  end

  defp hrrr_placeholder_for_contact(contact, point_map, present_set) do
    cond do
      is_nil(contact.pos1) ->
        []

      not Grid.contains?(contact.pos1) ->
        []

      NarrClient.in_coverage?(contact.qso_timestamp) ->
        []

      true ->
        points = Map.get(point_map, contact.id, [])
        if Enum.all?(points, &MapSet.member?(present_set, &1)), do: [], else: [contact.id]
    end
  end

  defp build_hrrr_present_set(contacts) do
    contacts
    |> Enum.filter(&(&1.pos1 != nil and not NarrClient.in_coverage?(&1.qso_timestamp)))
    |> Enum.reduce({%{}, []}, fn c, {pm, pts} ->
      rounded = Weather.HrrrClient.nearest_hrrr_hour(c.qso_timestamp)

      points =
        c
        |> Radio.contact_path_points()
        |> Enum.map(fn {lat, lon} -> {Weather.round_to_hrrr_grid(lat, lon), rounded} end)

      {Map.put(pm, c.id, points), points ++ pts}
    end)
    |> then(fn {pm, all_pts} ->
      unique = Enum.uniq(all_pts)
      present = Weather.hrrr_points_present_batch(unique)
      {pm, present}
    end)
  end

  defp mark_enrichment_statuses(contact, types, jobs_by_type) do
    ids = [contact.id]

    _ =
      if contact.pos1 do
        mark_pos1_statuses(contact, ids, types, jobs_by_type)
      end

    if contact.pos1 && contact.pos2 do
      mark_path_statuses(contact, ids, types, jobs_by_type)
    else
      # Path enrichments (terrain/radar/mechanism) need both endpoints.
      # Marking :unavailable stops BackfillEnqueueWorker from re-scanning
      # these contacts every cron cycle. Radio.reset_enrichment_statuses/2
      # will flip them back to :pending if pos2 is added later.
      mark_missing_path_statuses(ids, types)
    end
  end

  defp mark_missing_path_statuses(ids, types) do
    _ = if :terrain in types, do: Radio.set_enrichment_status!(ids, :terrain_status, :unavailable)
    _ = if :radar in types, do: Radio.set_enrichment_status!(ids, :radar_status, :unavailable)

    if :mechanism in types,
      do: Radio.set_enrichment_status!(ids, :mechanism_status, :unavailable)
  end

  defp mark_pos1_statuses(contact, ids, types, jobs_by_type) do
    _ = if :weather in types, do: mark_status!(ids, :weather_status, jobs_by_type[:weather])
    _ = if :hrrr in types, do: mark_hrrr_status!(contact, ids, jobs_by_type[:hrrr])
    if :iemre in types, do: mark_status!(ids, :iemre_status, jobs_by_type[:iemre])
  end

  defp mark_path_statuses(contact, ids, types, jobs_by_type) do
    _ = if :terrain in types, do: mark_status!(ids, :terrain_status, jobs_by_type[:terrain])
    _ = if :radar in types, do: mark_radar_status!(contact, ids, jobs_by_type[:radar])

    if :mechanism in types,
      do: mark_status!(ids, :mechanism_status, jobs_by_type[:mechanism])
  end

  # IEM n0q composite reflectivity archive only has meaningful CONUS
  # coverage post-2014 (pre that, the archive is spotty mosaics). For
  # pre-2014 contacts we can't get radar, so pin to :unavailable rather
  # than leaving it :pending forever.
  defp mark_radar_status!(contact, ids, []) do
    if NarrClient.in_coverage?(contact.qso_timestamp) do
      Radio.set_enrichment_status!(ids, :radar_status, :unavailable)
    else
      Radio.set_enrichment_status!(ids, :radar_status, :queued)
    end
  end

  defp mark_radar_status!(_contact, ids, [_ | _]), do: Radio.set_enrichment_status!(ids, :radar_status, :queued)

  # Empty-job-list HRRR path has three terminal states:
  #
  # 1. pos1 outside the CONUS grid → :unavailable. hrrr_point_rs silently
  #    returns zero profiles for OCONUS points and the backfill cron
  #    would otherwise re-flag :queued every tick forever.
  # 2. pre-2014 contacts (NarrClient.in_coverage?) → :unavailable so the
  #    :narr filter in BackfillEnqueueWorker picks them up.
  # 3. modern CONUS contact whose profiles already exist → :complete.
  defp mark_hrrr_status!(contact, ids, []) do
    cond do
      not Grid.contains?(contact.pos1) ->
        Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable)

      NarrClient.in_coverage?(contact.qso_timestamp) ->
        Radio.set_enrichment_status!(ids, :hrrr_status, :unavailable)

      true ->
        Radio.set_enrichment_status!(ids, :hrrr_status, :complete)
    end
  end

  defp mark_hrrr_status!(_contact, ids, [_ | _]), do: Radio.set_enrichment_status!(ids, :hrrr_status, :queued)

  @impl Oban.Pro.Worker
  def process(%Oban.Job{}) do
    enqueue_weather_jobs()
    enqueue_hrrr_jobs()
    enqueue_terrain_jobs()
    enqueue_iemre_jobs()

    # After enqueuing, sweep any :queued contacts whose data has
    # already landed (from a prior run's ingestion) but whose
    # <type>_status was never flipped. Without these, contacts stayed
    # visible to the backfill scan forever.
    Enum.each(
      [
        {"weather", &Weather.reconcile_weather_statuses/0},
        {"iemre", &Weather.reconcile_iemre_statuses/0},
        {"hrrr", &Weather.reconcile_hrrr_statuses/0}
      ],
      fn {label, fun} ->
        case fun.() do
          {:ok, n} when n > 0 ->
            Logger.info("ContactWeatherEnqueueWorker: reconciled #{n} #{label}_status → :complete")

          _ ->
            :ok
        end
      end
    )

    :ok
  end

  defp enqueue_weather_jobs do
    case Radio.unprocessed_contacts() do
      [] ->
        :ok

      contacts ->
        _ = Radio.backfill_distances(contacts)

        jobs = build_weather_jobs(contacts)

        if jobs != [] do
          insert_all_chunked(jobs)
        end

        ids = Enum.map(contacts, & &1.id)
        _ = Radio.mark_weather_queued!(ids)

        enqueue_weather_jobs()
    end
  end

  defp enqueue_hrrr_jobs do
    case Radio.unprocessed_hrrr_contacts() do
      [] ->
        :ok

      contacts ->
        # Rust Stream C path: route into hrrr_fetch_tasks instead of
        # Oban. The enqueuer groups by valid_time and UPSERTs so a
        # backfill cron sweep that rediscovers the same contacts
        # simply refreshes the existing hourly row.
        {:ok, _} = HrrrPointEnqueuer.enqueue_for_contacts(contacts)

        ids = Enum.map(contacts, & &1.id)
        _ = Radio.mark_hrrr_queued!(ids)

        enqueue_hrrr_jobs()
    end
  end

  defp enqueue_terrain_jobs do
    case Radio.unprocessed_terrain_contacts() do
      [] ->
        :ok

      contacts ->
        jobs = build_terrain_jobs(contacts)

        if jobs != [] do
          insert_all_chunked(jobs)
        end

        ids = Enum.map(contacts, & &1.id)
        _ = Radio.mark_terrain_queued!(ids)

        enqueue_terrain_jobs()
    end
  end

  @spec build_terrain_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
  def build_terrain_jobs(contacts) do
    contacts
    |> Enum.filter(fn contact -> contact.pos1 && contact.pos2 end)
    |> Enum.map(fn contact ->
      TerrainProfileWorker.new(%{"contact_id" => contact.id})
    end)
    |> Enum.uniq_by(fn changeset -> changeset.changes.args end)
  end

  @spec build_radar_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
  def build_radar_jobs(contacts) do
    # Group contacts by their 5-min NEXRAD frame so the worker can
    # fetch+decode the ~5 MB PNG once per frame and iterate every
    # contact in that frame against the in-memory pixel buffer.
    # Historical backlog averages ~10 contacts per distinct frame,
    # collapsing the per-job cost by roughly that factor.
    contacts
    |> Enum.filter(fn contact ->
      contact.pos1 && contact.pos2 && not NarrClient.in_coverage?(contact.qso_timestamp)
    end)
    |> Enum.group_by(fn contact ->
      contact.qso_timestamp
      |> DateTime.from_naive!("Etc/UTC")
      |> NexradClient.round_to_5min()
    end)
    |> Enum.map(fn {frame_ts, batch} ->
      RadarFrameWorker.new(%{
        "frame_ts" => DateTime.to_iso8601(frame_ts),
        "contact_ids" => Enum.map(batch, & &1.id)
      })
    end)
  end

  @spec build_mechanism_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
  def build_mechanism_jobs(contacts) do
    contacts
    |> Enum.filter(fn contact -> contact.pos1 && contact.pos2 end)
    |> Enum.map(fn contact ->
      MechanismClassifyWorker.new(%{"contact_id" => contact.id})
    end)
    |> Enum.uniq_by(fn changeset -> changeset.changes.args end)
  end

  @spec build_weather_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
  def build_weather_jobs(contacts) do
    contacts
    |> Enum.flat_map(&jobs_for_contact/1)
    |> Enum.uniq_by(fn changeset -> changeset.changes.args end)
  end

  @spec build_iemre_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
  def build_iemre_jobs(contacts) do
    contacts
    |> Enum.flat_map(&iemre_job_for_contact/1)
    |> Enum.uniq_by(fn changeset -> changeset.changes.args end)
  end

  @spec build_narr_jobs([Contact.t()]) :: [Ecto.Changeset.t()]
  def build_narr_jobs(contacts) do
    contacts
    |> Enum.flat_map(&narr_jobs_for_contact/1)
    |> Enum.uniq_by(fn changeset -> changeset.changes.args end)
  end

  defp narr_jobs_for_contact(%{pos1: nil}), do: []

  defp narr_jobs_for_contact(contact) do
    # NARR analyses are 3-hourly (00/03/06/09/12/15/18/21 UTC). NarrFetchWorker
    # rejects anything else, so snap here in the caller.
    valid_time = NarrClient.snap_to_analysis_hour(contact.qso_timestamp)

    # NCEI's NARR archive covers 1979-01-01 → 2014-10-02. Post-coverage contacts
    # (hrrr_status = :unavailable but qso_timestamp >= 2014-10-02) are HRRR's
    # responsibility — NARR would just 404.
    if NarrClient.in_coverage?(valid_time) do
      narr_jobs_for_path(contact, valid_time)
    else
      []
    end
  end

  defp narr_jobs_for_path(contact, valid_time) do
    contact
    |> Radio.contact_path_points()
    |> Enum.flat_map(fn {lat, lon} ->
      rlat = Float.round(lat * 4) / 4
      rlon = Float.round(lon * 4) / 4

      if Weather.find_nearest_narr(rlat, rlon, valid_time) do
        []
      else
        [
          NarrFetchWorker.new(%{
            "lat" => rlat,
            "lon" => rlon,
            "valid_time" => DateTime.to_iso8601(valid_time)
          })
        ]
      end
    end)
  end

  defp jobs_for_contact(contact) do
    contact
    |> Radio.contact_path_points()
    |> Enum.flat_map(fn {lat, lon} ->
      build_asos_jobs(lat, lon, contact.qso_timestamp) ++
        build_raob_jobs(lat, lon, contact.qso_timestamp)
    end)
  end

  defp enqueue_iemre_jobs do
    case Radio.unprocessed_iemre_contacts() do
      [] ->
        :ok

      contacts ->
        jobs = build_iemre_jobs(contacts)

        if jobs != [] do
          insert_all_chunked(jobs)
        end

        ids = Enum.map(contacts, & &1.id)
        _ = Radio.mark_iemre_queued!(ids)

        enqueue_iemre_jobs()
    end
  end

  defp build_asos_jobs(lat, lon, timestamp) do
    start_dt = DateTime.add(timestamp, -2 * 3600, :second)
    end_dt = DateTime.add(timestamp, 2 * 3600, :second)

    stations = Weather.nearby_stations(lat, lon, "asos", @asos_radius_km)

    # A ±2h window around the QSO can straddle UTC midnight, so enumerate
    # every date the window touches. Each (station, date) becomes one
    # asos_day job — Oban's unique-args dedup collapses cross-QSO
    # repeats (contact A's KDFW/2021-08-21 and contact B's KDFW/2021-08-21
    # share args → one job). A full-day fetch returns ~24 hourly rows,
    # amortizing the IemRateLimiter gap over 24× the previous payload.
    dates = window_dates(start_dt, end_dt)

    pairs =
      for station <- stations, date <- dates do
        {station, date}
      end

    covered =
      pairs
      |> Enum.map(fn {s, d} -> {s.id, d} end)
      |> Weather.station_day_pairs_covered()

    pairs
    |> Enum.reject(fn {s, d} -> MapSet.member?(covered, {s.id, d}) end)
    |> Enum.map(fn {station, date} ->
      WeatherFetchWorker.new(%{
        "fetch_type" => "asos_day",
        "station_id" => station.id,
        "station_code" => station.station_code,
        "date" => Date.to_iso8601(date)
      })
    end)
  end

  defp window_dates(start_dt, end_dt) do
    start_date = DateTime.to_date(start_dt)
    end_date = DateTime.to_date(end_dt)

    if Date.compare(start_date, end_date) == :eq do
      [start_date]
    else
      [start_date, end_date]
    end
  end

  defp iemre_job_for_contact(%{pos1: nil}), do: []

  defp iemre_job_for_contact(contact) do
    date = DateTime.to_date(contact.qso_timestamp)

    contact
    |> Radio.contact_path_points()
    |> Enum.flat_map(fn {lat, lon} ->
      {rlat, rlon} = Weather.round_to_iemre_grid(lat, lon)

      if Weather.has_iemre_observation?(rlat, rlon, date) do
        []
      else
        [
          IemreFetchWorker.new(%{
            "lat" => rlat,
            "lon" => rlon,
            "date" => Date.to_iso8601(date)
          })
        ]
      end
    end)
  end

  defp build_raob_jobs(lat, lon, timestamp) do
    build_raob_jobs(lat, lon, timestamp, @sounding_radius_km, nil)
  end

  defp build_raob_jobs(lat, lon, timestamp, radius_km, contact_id) do
    sounding_times = Weather.sounding_times_around(timestamp)
    stations = Weather.nearby_stations(lat, lon, "sounding", radius_km)
    station_ids = Enum.map(stations, & &1.id)
    covered = Weather.station_ids_with_soundings(station_ids, sounding_times)

    for station <- stations,
        sounding_time <- sounding_times,
        not MapSet.member?(covered, {station.id, sounding_time}) do
      args =
        Map.merge(
          %{
            "fetch_type" => "raob",
            "station_id" => station.id,
            "station_code" => station.station_code,
            "sounding_time" => DateTime.to_iso8601(sounding_time)
          },
          if(contact_id, do: %{"contact_id" => contact_id}, else: %{})
        )

      WeatherFetchWorker.new(args)
    end
  end

  @doc """
  Enqueue RAOB fetch jobs for the given contact at a progressively wider
  radius until the radius yields at least one sounding station. Passes
  the contact id through so `WeatherFetchWorker` can PubSub the contact
  detail page when each sounding lands.

  Returns `{:ok, %{radius_km, jobs_enqueued}}`, or `{:error, :no_stations}`
  when even 1000 km finds nothing.
  """
  @spec enqueue_raob_fetch_with_widening(Contact.t()) ::
          {:ok, %{radius_km: pos_integer(), jobs_enqueued: non_neg_integer()}}
          | {:error, :no_stations}
          | {:error, :no_coords}
  def enqueue_raob_fetch_with_widening(contact) do
    case Radio.contact_path_points(contact) do
      [] ->
        {:error, :no_coords}

      [{lat, lon} | _] ->
        do_widening_raob(lat, lon, contact)
    end
  end

  defp do_widening_raob(lat, lon, contact) do
    Enum.reduce_while([150, 300, 600, 1000], {:error, :no_stations}, fn radius, _acc ->
      stations = Weather.nearby_stations(lat, lon, "sounding", radius)

      if stations == [] do
        {:cont, {:error, :no_stations}}
      else
        {:halt, enqueue_raob_at_radius(lat, lon, contact, radius)}
      end
    end)
  end

  defp enqueue_raob_at_radius(lat, lon, contact, radius) do
    jobs = build_raob_jobs(lat, lon, contact.qso_timestamp, radius, contact.id)
    if jobs != [], do: insert_all_chunked(jobs)
    {:ok, %{radius_km: radius, jobs_enqueued: length(jobs)}}
  end

  defp insert_all_chunked(jobs) do
    jobs
    |> Enum.chunk_every(@insert_batch_size)
    |> Enum.each(&Oban.insert_all/1)
  end

  defp mark_status!(ids, field, [_ | _]), do: Radio.set_enrichment_status!(ids, field, :queued)
  defp mark_status!(ids, field, []), do: Radio.set_enrichment_status!(ids, field, :complete)

  # :narr is a virtual type synthesized from hrrr_status = :unavailable;
  # it has no column of its own, so there's nothing to "already be
  # complete."
  defp already_complete?(_contact, :narr), do: false

  defp already_complete?(contact, type) do
    Map.get(contact, String.to_existing_atom("#{type}_status")) == :complete
  end
end