prop/lib/microwaveprop/propagation/grid_task_enqueuer.ex

defmodule Microwaveprop.Propagation.GridTaskEnqueuer do
  @moduledoc """
  Seeds `grid_tasks` rows for the Rust `prop-grid-rs` worker.

  Called from `PropagationGridWorker.seed_chain/0`. Rust claims
  kind='forecast' and kind='analysis' rows, with analysis lanes
  taking priority (see `claim_next_analysis` in the Rust db module).

  Inserts are idempotent via the `(run_time, forecast_hour, kind)`
  unique index — re-seeding the same cycle is a no-op.
  """

  import Ecto.Query

  alias Microwaveprop.Repo

  require Logger

  @max_forecast_hour 18

  # A `running` row older than this is an orphan: the Rust worker that
  # claimed it was SIGKILLed / OOMed / evicted before it could emit a
  # terminal status. `claim_next` uses `FOR UPDATE SKIP LOCKED` so the
  # row stays `running` indefinitely until something reclaims it.
  # 15 min comfortably exceeds the ~3-4 min wall time of a healthy
  # forecast-hour chain step; any row past that is stuck.
  @stale_running_cutoff_seconds 15 * 60

  # After this many claim/reclaim cycles, a row is permanently flipped
  # to `failed` so the status-page spinner stops and the next hourly
  # reseed can replace it. Rust's `claim_next` increments `attempt` on
  # each pickup, so this is a total-attempts ceiling.
  @max_reclaim_attempts 5

  @doc """
  Seed one kind='analysis' row (f00) plus 18 kind='forecast' rows
  (f01..f18) for `run_time`. This is the Phase 3 Stream A cutover
  shape: Rust owns the entire chain end-to-end.
  """
  @spec seed_with_analysis(DateTime.t()) :: {:ok, non_neg_integer()} | {:error, term()}
  def seed_with_analysis(%DateTime{} = run_time) do
    run_time = DateTime.truncate(run_time, :second)
    now = DateTime.truncate(DateTime.utc_now(), :microsecond)

    analysis_row = %{
      id: Ecto.UUID.bingenerate(),
      run_time: run_time,
      forecast_hour: 0,
      valid_time: run_time,
      status: "queued",
      attempt: 0,
      kind: "analysis",
      claimed_at: nil,
      completed_at: nil,
      error: nil,
      inserted_at: now,
      updated_at: now
    }

    forecast_rows =
      for fh <- 1..@max_forecast_hour do
        %{
          id: Ecto.UUID.bingenerate(),
          run_time: run_time,
          forecast_hour: fh,
          valid_time: DateTime.add(run_time, fh * 3600, :second),
          status: "queued",
          attempt: 0,
          kind: "forecast",
          claimed_at: nil,
          completed_at: nil,
          error: nil,
          inserted_at: now,
          updated_at: now
        }
      end

    do_insert([analysis_row | forecast_rows], run_time)
  end

  @doc """
  Legacy: seed only kind='forecast' rows (f01..f18). Retained for
  tooling that needs to re-seed the forecast lane without touching
  the analysis row. `seed_with_analysis/1` is the production path.
  """
  @spec seed(DateTime.t()) :: {:ok, non_neg_integer()} | {:error, term()}
  def seed(%DateTime{} = run_time) do
    run_time = DateTime.truncate(run_time, :second)
    now = DateTime.truncate(DateTime.utc_now(), :microsecond)

    rows =
      for fh <- 1..@max_forecast_hour do
        %{
          id: Ecto.UUID.bingenerate(),
          run_time: run_time,
          forecast_hour: fh,
          valid_time: DateTime.add(run_time, fh * 3600, :second),
          status: "queued",
          attempt: 0,
          kind: "forecast",
          claimed_at: nil,
          completed_at: nil,
          error: nil,
          inserted_at: now,
          updated_at: now
        }
      end

    do_insert(rows, run_time)
  end

  @doc """
  Reclaim orphan `running` rows — Rust workers that died mid-claim
  (SIGKILL, OOM, node drain) leave `status='running'` sticky, which
  makes `claim_next` skip the row forever and the status panel shows
  a permanent-looking spinner. Flip rows whose `claimed_at` is older
  than `cutoff_seconds` back to `queued`. Rows that have already
  burned through `@max_reclaim_attempts` become `failed` so the next
  hourly seed replaces them instead of looping.

  Called from `PropagationGridWorker.seed_chain/0` at :05 each hour.
  """
  @spec reclaim_stale_running(pos_integer()) :: %{requeued: non_neg_integer(), failed: non_neg_integer()}
  def reclaim_stale_running(cutoff_seconds \\ @stale_running_cutoff_seconds) do
    cutoff = DateTime.utc_now() |> DateTime.add(-cutoff_seconds, :second) |> DateTime.truncate(:second)
    now = DateTime.truncate(DateTime.utc_now(), :second)

    {failed_n, _} =
      "grid_tasks"
      |> where(
        [t],
        t.status == "running" and t.claimed_at < ^cutoff and t.attempt >= ^@max_reclaim_attempts
      )
      |> Repo.update_all(
        set: [
          status: "failed",
          error: "reclaim-orphan: exceeded max_reclaim_attempts",
          claimed_at: nil,
          updated_at: now
        ]
      )

    {requeued_n, _} =
      "grid_tasks"
      |> where([t], t.status == "running" and t.claimed_at < ^cutoff)
      |> Repo.update_all(set: [status: "queued", claimed_at: nil, updated_at: now])

    if failed_n > 0 or requeued_n > 0 do
      Logger.info("GridTaskEnqueuer: reclaimed stale-running grid_tasks (requeued=#{requeued_n}, failed=#{failed_n})")
    end

    %{requeued: requeued_n, failed: failed_n}
  end

  defp do_insert(rows, run_time) do
    {count, _} =
      Repo.insert_all("grid_tasks", rows,
        on_conflict: :nothing,
        conflict_target: [:run_time, :forecast_hour, :kind]
      )

    if count > 0 do
      Logger.info("GridTaskEnqueuer: seeded #{count} grid_tasks for run_time=#{run_time}")
    else
      Logger.info("GridTaskEnqueuer: run_time=#{run_time} already seeded")
    end

    {:ok, count}
  rescue
    e ->
      Logger.error("GridTaskEnqueuer: seed failed: #{inspect(e)}")
      {:error, e}
  end
end