feat(prop): adaptive HRRR cycle selection — prefer fresh, fall back if late

PropagationGridWorker hard-coded `now-2h` for run_time selection on
the conservative theory that NOAA always finishes publishing within
two hours. In practice f18 is on the mirror by HH:55–HH:00 of the
next hour, so the cron at HH:05 was always picking a cycle that was
already an hour stale.

`HrrrClient.cycle_available?/1` HEADs the wrfprsf18 idx (the slowest
file in the cycle) — a 200 means every earlier forecast hour is also
on disk. `pick_run_time/1` probes `now-1h` first; if the probe
returns true we use that cycle, otherwise we fall back to the
existing `now-2h` slot. Test hook via :hrrr_cycle_available_fn env
keeps the behaviour fully exercisable without hitting NOAA.

End-to-end: weather/score data lag drops from ~2 h to ~1 h on every
hour where NOAA delivers on time. The fallback path keeps the chain
running on slow-publish hours.

config/test.exs

@@ -68,6 +68,11 @@ config :microwaveprop, cache_contact_map: false
 config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false]
 config :microwaveprop, gefs_req_options: [plug: {Req.Test, Microwaveprop.Weather.GefsClient}, retry: false]
 config :microwaveprop, giro_req_options: [plug: {Req.Test, Microwaveprop.Ionosphere.GiroClient}, retry: false]
+
+# Default the HRRR cycle freshness probe to "always available" in tests.
+# Individual cases that exercise the fallback path override via
+# `Application.put_env/3` in their setup.
+config :microwaveprop, hrrr_cycle_available_fn: fn _run_time -> true end
 config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false]
 config :microwaveprop, iem_rate_limiter_interval_ms: 0
 config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false]
@@ -79,7 +84,6 @@ config :microwaveprop, rtma_req_options: [plug: {Req.Test, Microwaveprop.Weather
 config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}]
 config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false]
 config :microwaveprop, start_freshness_monitor: false
-config :microwaveprop, start_score_cache_reconciler: false
 config :microwaveprop, swpc_req_options: [plug: {Req.Test, Microwaveprop.SpaceWeather.SwpcClient}, retry: false]
 config :microwaveprop, uwyo_req_options: [plug: {Req.Test, Microwaveprop.Weather.UwyoSoundingClient}, retry: false]
 

lib/microwaveprop/weather/hrrr_client.ex

@@ -177,6 +177,44 @@ defmodule Microwaveprop.Weather.HrrrClient do
     end
   end
 
+  @doc """
+  Returns true if the HRRR cycle for `run_time` is fully published.
+  Probes the `wrfprsf18.grib2.idx` file (last forecast hour of the slower
+  pressure-level product — last to land for a given cycle). A 200 means
+  every earlier forecast hour is also on disk; anything else means the
+  cycle is still being uploaded or NOAA hasn't started it yet.
+
+  Used by `PropagationGridWorker` to prefer a fresh `now-1h` cycle and
+  fall back to the conservative `now-2h` slot only when the fresher
+  cycle hasn't fully published.
+
+  Override the probe in tests via the
+  `:microwaveprop, :hrrr_cycle_available_fn` Application env (a 1-arity
+  function from `DateTime` to `boolean`).
+  """
+  @spec cycle_available?(DateTime.t()) :: boolean()
+  def cycle_available?(run_time) do
+    rounded = nearest_hrrr_hour(run_time)
+
+    case Application.get_env(:microwaveprop, :hrrr_cycle_available_fn) do
+      fun when is_function(fun, 1) -> fun.(rounded)
+      _ -> probe_cycle(rounded)
+    end
+  end
+
+  defp probe_cycle(%DateTime{} = run_time) do
+    url =
+      run_time
+      |> DateTime.to_date()
+      |> hrrr_url(run_time.hour, :pressure, 18)
+      |> Kernel.<>(".idx")
+
+    case Req.head(url, req_options()) do
+      {:ok, %{status: 200}} -> true
+      _ -> false
+    end
+  end
+
   @spec nearest_hrrr_hour(DateTime.t()) :: DateTime.t()
   def nearest_hrrr_hour(dt) do
     total_seconds = dt.minute * 60 + dt.second

lib/microwaveprop/workers/propagation_grid_worker.ex

@@ -45,10 +45,7 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
     # rows are never re-worked.
     _ = GridTaskEnqueuer.reclaim_stale_running()
 
-    # HRRR takes ~45min to publish after the hour. Use 2 hours ago to
-    # ensure availability.
-    two_hours_ago = DateTime.add(DateTime.utc_now(), -2, :hour)
-    run_time = two_hours_ago |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second)
+    run_time = pick_run_time(DateTime.utc_now())
 
     Logger.info("PropagationGrid: seeding chain run_time=#{run_time} (f00 analysis + f01-f18 forecasts via grid_tasks)")
 
@@ -72,4 +69,26 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do
         {:error, reason}
     end
   end
+
+  @doc """
+  Pick the freshest HRRR cycle that's published.
+
+  Probes `now-1h` first via `HrrrClient.cycle_available?/1`. If the
+  pressure-level f18 idx file is on the mirror/S3, the cycle is fully
+  uploaded and we use it. Otherwise we fall back to the conservative
+  `now-2h` slot — NOAA's stated max publishing latency for HRRR.
+
+  Public so the test suite can exercise the selection logic without
+  going through Oban.
+  """
+  @spec pick_run_time(DateTime.t()) :: DateTime.t()
+  def pick_run_time(%DateTime{} = now) do
+    fresh = now |> DateTime.add(-1, :hour) |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second)
+
+    if HrrrClient.cycle_available?(fresh) do
+      fresh
+    else
+      now |> DateTime.add(-2, :hour) |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second)
+    end
+  end
 end

test/microwaveprop/weather/hrrr_client_test.exs

@@ -3,6 +3,38 @@ defmodule Microwaveprop.Weather.HrrrClientTest do
 
   alias Microwaveprop.Weather.HrrrClient
 
+  describe "cycle_available?/1 (stubbed probe)" do
+    setup do
+      original = Application.get_env(:microwaveprop, :hrrr_cycle_available_fn)
+
+      on_exit(fn ->
+        if original,
+          do: Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, original),
+          else: Application.delete_env(:microwaveprop, :hrrr_cycle_available_fn)
+      end)
+    end
+
+    test "returns whatever the configured probe function returns" do
+      Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn _dt -> true end)
+      assert HrrrClient.cycle_available?(~U[2026-04-25 00:00:00Z])
+
+      Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn _dt -> false end)
+      refute HrrrClient.cycle_available?(~U[2026-04-25 00:00:00Z])
+    end
+
+    test "passes the rounded run_time to the probe" do
+      test_pid = self()
+
+      Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn dt ->
+        send(test_pid, {:probed, dt})
+        true
+      end)
+
+      HrrrClient.cycle_available?(~U[2026-04-25 00:38:00Z])
+      assert_receive {:probed, ~U[2026-04-25 01:00:00Z]}
+    end
+  end
+
   describe "nearest_hrrr_hour/1" do
     test "rounds down when within first 30 minutes" do
       assert HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:15:00Z]) ==

test/microwaveprop/workers/propagation_grid_worker_test.exs

@@ -15,6 +15,47 @@ defmodule Microwaveprop.Workers.PropagationGridWorkerTest do
   alias Microwaveprop.Workers.PropagationGridWorker
   alias Microwaveprop.Workers.PropagationPruneWorker
 
+  describe "pick_run_time/1 (HRRR cycle freshness selection)" do
+    setup do
+      original = Application.get_env(:microwaveprop, :hrrr_cycle_available_fn)
+
+      on_exit(fn ->
+        if original,
+          do: Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, original),
+          else: Application.delete_env(:microwaveprop, :hrrr_cycle_available_fn)
+      end)
+    end
+
+    test "picks now-1h when that cycle has fully published" do
+      Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn _dt -> true end)
+
+      # Cron fires at HH:05, so use that as `now`. now-1h = 23:05 rounds
+      # to 23:00 (the freshly-published cycle).
+      now = ~U[2026-04-25 00:05:00Z]
+      assert PropagationGridWorker.pick_run_time(now) == ~U[2026-04-24 23:00:00Z]
+    end
+
+    test "falls back to now-2h when the now-1h cycle is not yet available" do
+      Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn _dt -> false end)
+
+      now = ~U[2026-04-25 00:05:00Z]
+      assert PropagationGridWorker.pick_run_time(now) == ~U[2026-04-24 22:00:00Z]
+    end
+
+    test "probes the rounded now-1h value, not the raw `now`" do
+      test_pid = self()
+
+      Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn dt ->
+        send(test_pid, {:probed, dt})
+        true
+      end)
+
+      # 00:05 UTC: -1h = 23:05 UTC, rounds to 23:00 UTC.
+      PropagationGridWorker.pick_run_time(~U[2026-04-25 00:05:00Z])
+      assert_receive {:probed, ~U[2026-04-24 23:00:00Z]}
+    end
+  end
+
   describe "queue priority" do
     test "PropagationGridWorker runs at the highest priority on :propagation" do
       assert PropagationGridWorker.__opts__()[:priority] == 0