From c2a23d18407b7d13cd2f588f1269c8b722c3bbcb Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sun, 3 May 2026 13:52:27 -0500
Subject: [PATCH] feat(rover-planning): hourly backfill worker recovers stuck
 paths
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds RoverPlanning.backfill_paths/0 that walks every mission and
re-enqueues RoverPathProfileWorker jobs for any (rover × station) pair
whose Path is missing or not :complete. The path-profile worker
short-circuits on :complete, so this is idempotent.

Wires a new RoverMissionBackfillWorker into both dev (config.exs) and
prod (runtime.exs) crons at HH:30 — heals missions stuck in :pending
or :failed from transient elevation-API errors or interrupted
create_mission enqueues without operator action.
---
 config/config.exs                             |  3 ++
 config/runtime.exs                            |  3 ++
 lib/microwaveprop/rover_planning.ex           | 20 ++++++++
 .../workers/rover_mission_backfill_worker.ex  | 26 ++++++++++
 test/microwaveprop/rover_planning_test.exs    | 41 +++++++++++++++
 .../rover_mission_backfill_worker_test.exs    | 50 +++++++++++++++++++
 6 files changed, 143 insertions(+)
 create mode 100644 lib/microwaveprop/workers/rover_mission_backfill_worker.ex
 create mode 100644 test/microwaveprop/workers/rover_mission_backfill_worker_test.exs

diff --git a/config/config.exs b/config/config.exs
index b5c3af61..9c7b4d63 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -105,6 +105,9 @@ config :microwaveprop, Oban,
        {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
        {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
        {"15 * * * *", Microwaveprop.Workers.GridCachePruneWorker},
+       # Heals rover-planning missions whose paths are stuck in :pending
+       # or :failed (transient elevation-API errors, interrupted enqueues).
+       {"30 * * * *", Microwaveprop.Workers.RoverMissionBackfillWorker},
        # UWYO publishes the 00Z/12Z Canadian radiosondes ~90 minutes after launch
        {"30 1 * * *", CanadianSoundingFetchWorker},
        {"30 13 * * *", CanadianSoundingFetchWorker},
diff --git a/config/runtime.exs b/config/runtime.exs
index 4e2d5f0f..2ec4fc2d 100644
--- a/config/runtime.exs
+++ b/config/runtime.exs
@@ -267,6 +267,9 @@ if config_env() == :prod do
        {"30 5,11,17,23 * * *", Microwaveprop.Workers.GefsFetchWorker},
        {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
        {"15 * * * *", Microwaveprop.Workers.GridCachePruneWorker},
+       # Re-enqueues any rover-planning Path stuck in :pending/:failed.
+       # Idempotent — RoverPathProfileWorker no-ops on :complete.
+       {"30 * * * *", Microwaveprop.Workers.RoverMissionBackfillWorker},
        # The :narr type is virtual: it targets contacts with hrrr_status =
        # :unavailable (pre-2014, missing from the HRRR archive) and dispatches
        # NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
diff --git a/lib/microwaveprop/rover_planning.ex b/lib/microwaveprop/rover_planning.ex
index 187387f2..9ab3cebe 100644
--- a/lib/microwaveprop/rover_planning.ex
+++ b/lib/microwaveprop/rover_planning.ex
@@ -122,6 +122,26 @@ defmodule Microwaveprop.RoverPlanning do
     enqueue_paths_for(mission)
   end
 
+  @doc """
+  Walks every mission and re-enqueues any (rover_location × station)
+  pair whose `Path` is missing or not `:complete`. Existing complete
+  rows are left alone — the worker also short-circuits on `:complete`,
+  so this is idempotent and safe to run on a cron.
+
+  Use case: a rolling deploy or a transient elevation-API failure left
+  some missions with `:pending` / `:failed` entries; the next backfill
+  tick recovers them without operator action.
+  """
+  @spec backfill_paths() :: :ok
+  def backfill_paths do
+    Mission
+    |> Repo.all()
+    |> Repo.preload(stations: from(s in Station, order_by: s.position))
+    |> Enum.each(&enqueue_paths_for/1)
+
+    :ok
+  end
+
   defp enqueue_paths_for(%Mission{} = mission) do
     rovers = candidate_rover_locations(mission)
     stations = mission.stations || []
diff --git a/lib/microwaveprop/workers/rover_mission_backfill_worker.ex b/lib/microwaveprop/workers/rover_mission_backfill_worker.ex
new file mode 100644
index 00000000..c0f3c647
--- /dev/null
+++ b/lib/microwaveprop/workers/rover_mission_backfill_worker.ex
@@ -0,0 +1,26 @@
+defmodule Microwaveprop.Workers.RoverMissionBackfillWorker do
+  @moduledoc """
+  Periodic safety net for the `/rover-planning` matrix. Walks every
+  mission and re-enqueues `RoverPathProfileWorker` jobs for any
+  `(rover_location × station)` pair whose `Path` is missing or not
+  `:complete`. The path-profile worker short-circuits on `:complete`,
+  so this is idempotent across runs.
+
+  Recovers from transient failures (elevation API blips, rolling
+  deploys that interrupted the original `create_mission` enqueue)
+  without operator action.
+  """
+
+  use Oban.Worker,
+    queue: :terrain,
+    priority: 5,
+    max_attempts: 3,
+    unique: [period: 300, states: [:available, :scheduled, :executing, :retryable]]
+
+  alias Microwaveprop.RoverPlanning
+
+  @impl Oban.Worker
+  def perform(%Oban.Job{}) do
+    RoverPlanning.backfill_paths()
+  end
+end
diff --git a/test/microwaveprop/rover_planning_test.exs b/test/microwaveprop/rover_planning_test.exs
index 2c54f249..0f1ee25b 100644
--- a/test/microwaveprop/rover_planning_test.exs
+++ b/test/microwaveprop/rover_planning_test.exs
@@ -84,6 +84,47 @@ defmodule Microwaveprop.RoverPlanningTest do
       assert Map.has_key?(path.result, "propagation_score")
     end
 
+    test "backfill_paths/0 re-runs failed paths across all missions" do
+      user = AccountsFixtures.user_fixture()
+      _loc = create_rover_location(%{lat: 32.0, lon: -97.0, status: :good})
+
+      assert {:ok, mission} = RoverPlanning.create_mission(user, valid_attrs())
+      [path] = RoverPlanning.list_paths(mission)
+      assert path.status == :complete
+
+      # Simulate a path that was stuck in :failed (e.g. transient
+      # elevation API error during the original computation).
+      {:ok, _} =
+        path
+        |> Ecto.Changeset.change(%{status: :failed, error: "transient", result: nil})
+        |> Repo.update()
+
+      assert :ok = RoverPlanning.backfill_paths()
+
+      # Oban inline → backfill enqueues a job, the worker reruns
+      # immediately, and the path flips back to :complete.
+      [reloaded] = RoverPlanning.list_paths(mission)
+      assert reloaded.status == :complete
+      assert is_map(reloaded.result)
+    end
+
+    test "backfill_paths/0 inserts missing rows for under-populated missions" do
+      user = AccountsFixtures.user_fixture()
+      _loc = create_rover_location(%{lat: 32.0, lon: -97.0, status: :good})
+
+      assert {:ok, mission} = RoverPlanning.create_mission(user, valid_attrs())
+      # Simulate a half-broken mission: paths were never persisted (e.g.
+      # the original create_mission Multi succeeded but the Oban insert
+      # was lost due to a crash). Backfill must repopulate the matrix.
+      Repo.delete_all(Ecto.Query.from(p in RoverPlanning.Path, where: p.mission_id == ^mission.id))
+      assert RoverPlanning.list_paths(mission) == []
+
+      assert :ok = RoverPlanning.backfill_paths()
+
+      assert [path] = RoverPlanning.list_paths(mission)
+      assert path.status == :complete
+    end
+
     test "with only_known_good=false, all rover-locations get a path entry" do
       user = AccountsFixtures.user_fixture()
       _ideal = create_rover_location(%{lat: 32.0, lon: -97.0, status: :good})
diff --git a/test/microwaveprop/workers/rover_mission_backfill_worker_test.exs b/test/microwaveprop/workers/rover_mission_backfill_worker_test.exs
new file mode 100644
index 00000000..1165b533
--- /dev/null
+++ b/test/microwaveprop/workers/rover_mission_backfill_worker_test.exs
@@ -0,0 +1,50 @@
+defmodule Microwaveprop.Workers.RoverMissionBackfillWorkerTest do
+  use Microwaveprop.DataCase, async: true
+
+  alias Microwaveprop.AccountsFixtures
+  alias Microwaveprop.Repo
+  alias Microwaveprop.Rover
+  alias Microwaveprop.RoverPlanning
+  alias Microwaveprop.Terrain.ElevationClient
+  alias Microwaveprop.Workers.RoverMissionBackfillWorker
+
+  setup do
+    Req.Test.stub(ElevationClient, fn conn ->
+      params = Plug.Conn.fetch_query_params(conn).query_params
+      lat_count = params["latitude"] |> String.split(",") |> length()
+      Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
+    end)
+
+    :ok
+  end
+
+  describe "perform/1" do
+    test "re-runs failed paths" do
+      user = AccountsFixtures.user_fixture()
+      {:ok, _loc} = Rover.create_location(user, %{lat: 32.0, lon: -97.0, status: :good})
+
+      {:ok, mission} =
+        RoverPlanning.create_mission(user, %{
+          "name" => "Backfill mission",
+          "band_mhz" => 10_000,
+          "only_known_good" => true,
+          "rover_height_ft" => 8.0,
+          "station_height_ft" => 30.0,
+          "stations" => %{"0" => %{"input" => "EM12kp", "position" => 0}}
+        })
+
+      [path] = RoverPlanning.list_paths(mission)
+      assert path.status == :complete
+
+      {:ok, _} =
+        path
+        |> Ecto.Changeset.change(%{status: :failed, error: "transient", result: nil})
+        |> Repo.update()
+
+      assert :ok = RoverMissionBackfillWorker.perform(%Oban.Job{})
+
+      [reloaded] = RoverPlanning.list_paths(mission)
+      assert reloaded.status == :complete
+    end
+  end
+end