diff --git a/lib/microwaveprop/propagation/grid_task_enqueuer.ex b/lib/microwaveprop/propagation/grid_task_enqueuer.ex index 1a52cc9e..238cd957 100644 --- a/lib/microwaveprop/propagation/grid_task_enqueuer.ex +++ b/lib/microwaveprop/propagation/grid_task_enqueuer.ex @@ -10,12 +10,28 @@ defmodule Microwaveprop.Propagation.GridTaskEnqueuer do unique index — re-seeding the same cycle is a no-op. """ + import Ecto.Query + alias Microwaveprop.Repo require Logger @max_forecast_hour 18 + # A `running` row older than this is an orphan: the Rust worker that + # claimed it was SIGKILLed / OOMed / evicted before it could emit a + # terminal status. `claim_next` uses `FOR UPDATE SKIP LOCKED` so the + # row stays `running` indefinitely until something reclaims it. + # 15 min comfortably exceeds the ~3-4 min wall time of a healthy + # forecast-hour chain step; any row past that is stuck. + @stale_running_cutoff_seconds 15 * 60 + + # After this many claim/reclaim cycles, a row is permanently flipped + # to `failed` so the status-page spinner stops and the next hourly + # reseed can replace it. Rust's `claim_next` increments `attempt` on + # each pickup, so this is a total-attempts ceiling. + @max_reclaim_attempts 5 + @doc """ Seed one kind='analysis' row (f00) plus 18 kind='forecast' rows (f01..f18) for `run_time`. This is the Phase 3 Stream A cutover @@ -93,6 +109,49 @@ defmodule Microwaveprop.Propagation.GridTaskEnqueuer do do_insert(rows, run_time) end + @doc """ + Reclaim orphan `running` rows — Rust workers that died mid-claim + (SIGKILL, OOM, node drain) leave `status='running'` sticky, which + makes `claim_next` skip the row forever and the status panel shows + a permanent-looking spinner. Flip rows whose `claimed_at` is older + than `cutoff_seconds` back to `queued`. Rows that have already + burned through `@max_reclaim_attempts` become `failed` so the next + hourly seed replaces them instead of looping. + + Called from `PropagationGridWorker.seed_chain/0` at :05 each hour. + """ + @spec reclaim_stale_running(pos_integer()) :: %{requeued: non_neg_integer(), failed: non_neg_integer()} + def reclaim_stale_running(cutoff_seconds \\ @stale_running_cutoff_seconds) do + cutoff = DateTime.utc_now() |> DateTime.add(-cutoff_seconds, :second) |> DateTime.truncate(:second) + now = DateTime.truncate(DateTime.utc_now(), :second) + + {failed_n, _} = + "grid_tasks" + |> where( + [t], + t.status == "running" and t.claimed_at < ^cutoff and t.attempt >= ^@max_reclaim_attempts + ) + |> Repo.update_all( + set: [ + status: "failed", + error: "reclaim-orphan: exceeded max_reclaim_attempts", + claimed_at: nil, + updated_at: now + ] + ) + + {requeued_n, _} = + "grid_tasks" + |> where([t], t.status == "running" and t.claimed_at < ^cutoff) + |> Repo.update_all(set: [status: "queued", claimed_at: nil, updated_at: now]) + + if failed_n > 0 or requeued_n > 0 do + Logger.info("GridTaskEnqueuer: reclaimed stale-running grid_tasks (requeued=#{requeued_n}, failed=#{failed_n})") + end + + %{requeued: requeued_n, failed: failed_n} + end + defp do_insert(rows, run_time) do {count, _} = Repo.insert_all("grid_tasks", rows, diff --git a/lib/microwaveprop/workers/propagation_grid_worker.ex b/lib/microwaveprop/workers/propagation_grid_worker.ex index 1ce9a964..5de8f6ba 100644 --- a/lib/microwaveprop/workers/propagation_grid_worker.ex +++ b/lib/microwaveprop/workers/propagation_grid_worker.ex @@ -39,6 +39,12 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do end defp seed_chain do + # Reclaim any `running` grid_tasks rows orphaned by a Rust worker + # that died mid-claim (SIGKILL, OOM, node drain). Without this + # the status panel shows a permanent-looking spinner and the + # rows are never re-worked. + _ = GridTaskEnqueuer.reclaim_stale_running() + # HRRR takes ~45min to publish after the hour. Use 2 hours ago to # ensure availability. two_hours_ago = DateTime.add(DateTime.utc_now(), -2, :hour) diff --git a/lib/microwaveprop_web/live/status_live.ex b/lib/microwaveprop_web/live/status_live.ex index 92402f75..cec75103 100644 --- a/lib/microwaveprop_web/live/status_live.ex +++ b/lib/microwaveprop_web/live/status_live.ex @@ -551,7 +551,7 @@ defmodule MicrowavepropWeb.StatusLive do