prop/lib/microwaveprop/workers/backfill_enqueue_worker.ex

defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
  @moduledoc """
  Runs backfill enrichment enqueue as an Oban job so the backfill dashboard
  returns immediately instead of blocking on the enqueue loop.
  """
  use Oban.Pro.Worker, queue: :backfill_enqueue, max_attempts: 1

  import Ecto.Query

  alias Microwaveprop.Radio.Contact
  alias Microwaveprop.Repo
  alias Microwaveprop.Weather.NarrClient
  alias Microwaveprop.Workers.ContactWeatherEnqueueWorker

  require Logger

  @enrichable [:pending, :queued, :failed]

  # How long a contact must sit at :unavailable before the backfill
  # will re-probe it. :unavailable is reached either because a worker
  # gave up (upstream 404/5xx, no nearby station, pre-2014 archive
  # gap) or because the stale-queued reconciler flipped it after 3
  # days of :queued without data. Upstream conditions change over
  # time (new ASOS stations, IEM backfills, Rust worker new
  # capabilities), so "Partial" contacts must eventually get a second
  # look — without pounding the upstream every 30 min. 24 h strikes
  # a balance.
  @unavailable_reprobe_cooldown_seconds 86_400
  @valid_types ~w(hrrr weather terrain iemre narr radar mechanism)

  # Virtual types have no <type>_status column on contacts; they're synthesized
  # from other status fields (e.g. :narr candidates are `hrrr_status = :unavailable`).
  # Skip these in any reconcile / ordering pass that reads a <type>_status field.
  @virtual_types [:narr]

  # A :queued contact whose updated_at is older than this is one the cron has
  # already tried ~144 times over 3 days without the underlying worker landing
  # data. At that point the data is effectively unreachable (dead ASOS station,
  # pre-2014 HRRR, out-of-grid IEMRE) and the cron should stop re-enqueueing.
  # set_enrichment_status!/3 is a no-op when the status is unchanged, so a
  # stuck :queued row retains a stale updated_at across cron cycles — which is
  # exactly what we need as the "given up" signal.
  @stale_queued_cutoff_seconds 3 * 86_400

  @impl Oban.Pro.Worker
  def process(%Oban.Job{args: args}) do
    types = parse_types(args)
    limit = Map.get(args, "limit")

    reconciled = reconcile_stale_queued(types)

    if reconciled > 0 do
      Logger.info("BackfillEnqueue: reconciled #{reconciled} stale queued contacts")
    end

    failed_task_unavail = reconcile_failed_hrrr_task_contacts(types)

    if failed_task_unavail > 0 do
      Logger.info(
        "BackfillEnqueue: marked #{failed_task_unavail} hrrr-queued contacts as unavailable (upstream archive gap)"
      )
    end

    stale_unavail = reconcile_stale_queued_to_unavailable(types)

    if stale_unavail > 0 do
      Logger.info("BackfillEnqueue: marked #{stale_unavail} stuck-queued contacts as unavailable")
    end

    contacts =
      Contact
      |> where([c], not is_nil(c.pos1) or not is_nil(c.grid1))
      |> where(^type_filter(types))
      |> order_by(^status_priority_order(types))
      |> maybe_limit(limit)
      |> Repo.all()

    count = length(contacts)
    Logger.info("BackfillEnqueue: processing #{count} contacts (types: #{inspect(types)})")

    Enum.each(contacts, &ContactWeatherEnqueueWorker.enqueue_for_contact(&1, types))

    _ =
      Phoenix.PubSub.broadcast(
        Microwaveprop.PubSub,
        "backfill:enqueue_complete",
        {:enqueue_complete, count}
      )

    :ok
  end

  defp maybe_limit(query, nil), do: query
  defp maybe_limit(query, n) when is_integer(n), do: limit(query, ^n)

  defp parse_types(%{"types" => types}) when is_list(types) do
    types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
  end

  defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :narr, :radar, :mechanism]

  defp status_priority_order(types) do
    # Prioritize pending/failed over queued so real work happens first.
    # Virtual types (e.g. :narr) have no *_status column — fall back to
    # time-only ordering when every requested type is virtual.
    case Enum.find(types, &(&1 not in @virtual_types)) do
      nil ->
        [desc: dynamic([c], c.qso_timestamp)]

      type ->
        field = status_column(type)

        [
          asc:
            dynamic(
              [c],
              fragment("CASE ? WHEN 'pending' THEN 0 WHEN 'failed' THEN 1 ELSE 2 END", field(c, ^field))
            ),
          desc: dynamic([c], c.qso_timestamp)
        ]
    end
  end

  defp status_column(type), do: String.to_existing_atom("#{type}_status")

  # Flip contacts that have been stuck in :queued for longer than
  # @stale_queued_cutoff_seconds to :unavailable. One Repo.update_all per type,
  # scoped to only the types passed in args so a partial cron (e.g. hrrr-only)
  # doesn't touch other statuses. Virtual types (e.g. :narr) have no *_status
  # column and are skipped. Fast: hits the (<field>, updated_at) combo with a
  # simple range scan.
  defp reconcile_stale_queued_to_unavailable(types) do
    cutoff =
      DateTime.utc_now()
      |> DateTime.add(-@stale_queued_cutoff_seconds, :second)
      |> DateTime.truncate(:second)

    now = DateTime.truncate(DateTime.utc_now(), :second)

    types
    |> Enum.reject(&(&1 in @virtual_types))
    |> Enum.reduce(0, fn type, acc ->
      field = status_column(type)

      # Bump updated_at to now when flipping so the
      # @unavailable_reprobe_cooldown window starts fresh — otherwise
      # the main loop's :unavailable-reprobe filter would pick this
      # contact back up on the same cron tick (stale timestamp < any
      # reasonable cooldown) and immediately mark it :complete on the
      # empty-jobs branch, defeating the point of flagging :unavailable.
      {n, _} =
        Contact
        |> where([c], field(c, ^field) == :queued and c.updated_at < ^cutoff)
        |> Repo.update_all(set: [{field, :unavailable}, {:updated_at, now}])

      acc + n
    end)
  end

  # A `hrrr_fetch_tasks` row in `failed` state is one the Rust worker
  # gave up on after retrying — either wgrib2 decode errors (very old
  # HRRR format) or `idx HTTP 404` from the upstream archive. There's
  # no path to profiles for any contact whose rounded HRRR hour lands
  # on that valid_time, so flip them to `:unavailable` immediately
  # rather than letting them wait 3 days for the generic stale-queue
  # reconcile. Matches `HrrrClient.nearest_hrrr_hour/1` rounding in SQL:
  # drop minutes, bump the hour up when the original minute was >= 30.
  defp reconcile_failed_hrrr_task_contacts(types) do
    if :hrrr in types do
      now = DateTime.truncate(DateTime.utc_now(), :second)

      failed_vts_sub =
        from(t in "hrrr_fetch_tasks",
          where: t.status == "failed",
          select: t.valid_time
        )

      {n, _} =
        Contact
        |> where([c], c.hrrr_status == :queued)
        |> where(
          [c],
          fragment(
            "date_trunc('hour', ?) + (CASE WHEN date_part('minute', ?) >= 30 THEN interval '1 hour' ELSE interval '0' END)",
            c.qso_timestamp,
            c.qso_timestamp
          ) in subquery(failed_vts_sub)
        )
        |> Repo.update_all(set: [hrrr_status: :unavailable, updated_at: now])

      n
    else
      0
    end
  end

  # Fast SQL reconciliation for contacts stuck in "queued" where data already exists.
  # Terrain profiles have a direct contact_id FK so we can reconcile in bulk.
  defp reconcile_stale_queued(types) do
    terrain_count =
      if :terrain in types do
        {count, _} =
          Repo.update_all(
            from(c in Contact,
              where: c.terrain_status == :queued,
              where: c.id in subquery(from(tp in "terrain_profiles", select: tp.contact_id))
            ),
            set: [terrain_status: :complete]
          )

        count
      else
        0
      end

    # MechanismClassifyWorker sets propagation_mechanism + mechanism_status
    # atomically, so a row with a populated propagation_mechanism was
    # successfully classified. A prior re-enqueue bug clobbered status
    # back to :queued on subsequent cron cycles; the next worker run
    # will just re-classify and set :complete again. Reconcile here so
    # the backlog drains in one tick instead of waiting for each
    # contact's worker to run a second time.
    mechanism_count =
      if :mechanism in types do
        {count, _} =
          Repo.update_all(
            from(c in Contact,
              where: c.mechanism_status == :queued and not is_nil(c.propagation_mechanism)
            ),
            set: [mechanism_status: :complete]
          )

        count
      else
        0
      end

    # Radar reconciliation: contact_common_volume_radar has a direct
    # contact_id FK (same shape as terrain_profiles).
    radar_count =
      if :radar in types do
        {count, _} =
          Repo.update_all(
            from(c in Contact,
              where: c.radar_status == :queued,
              where: c.id in subquery(from(r in "contact_common_volume_radar", select: r.contact_id))
            ),
            set: [radar_status: :complete]
          )

        count
      else
        0
      end

    terrain_count + mechanism_count + radar_count
  end

  defp type_filter(types) do
    reprobe_cutoff =
      DateTime.utc_now()
      |> DateTime.add(-@unavailable_reprobe_cooldown_seconds, :second)
      |> DateTime.truncate(:second)

    Enum.reduce(types, dynamic(false), fn
      :narr, acc ->
        # NARR targets pre-2014 contacts where HRRR is unavailable. The NCEI
        # archive ends 2014-10-02 — post-cutoff fetches would 404, so scope
        # the candidate set here to keep the cron from scanning contacts we
        # can't serve.
        coverage_end = NarrClient.coverage_end()
        dynamic([c], ^acc or (c.hrrr_status == :unavailable and c.qso_timestamp < ^coverage_end))

      type, acc ->
        field = status_column(type)

        # Include :unavailable rows older than the cooldown so "Partial"
        # contacts get another chance when upstream archives heal.
        dynamic(
          [c],
          ^acc or field(c, ^field) in ^@enrichable or
            (field(c, ^field) == :unavailable and c.updated_at < ^reprobe_cutoff)
        )
    end)
  end
end