From 3b25add3f232d67100eecd4dc5627bb8c94ce5e2 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 24 Apr 2026 19:44:03 -0500 Subject: [PATCH] =?UTF-8?q?feat(prop):=20adaptive=20HRRR=20cycle=20selecti?= =?UTF-8?q?on=20=E2=80=94=20prefer=20fresh,=20fall=20back=20if=20late?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PropagationGridWorker hard-coded `now-2h` for run_time selection on the conservative theory that NOAA always finishes publishing within two hours. In practice f18 is on the mirror by HH:55–HH:00 of the next hour, so the cron at HH:05 was always picking a cycle that was already an hour stale. `HrrrClient.cycle_available?/1` HEADs the wrfprsf18 idx (the slowest file in the cycle) — a 200 means every earlier forecast hour is also on disk. `pick_run_time/1` probes `now-1h` first; if the probe returns true we use that cycle, otherwise we fall back to the existing `now-2h` slot. Test hook via :hrrr_cycle_available_fn env keeps the behaviour fully exercisable without hitting NOAA. End-to-end: weather/score data lag drops from ~2 h to ~1 h on every hour where NOAA delivers on time. The fallback path keeps the chain running on slow-publish hours. --- config/test.exs | 6 ++- lib/microwaveprop/weather/hrrr_client.ex | 38 +++++++++++++++++ .../workers/propagation_grid_worker.ex | 27 ++++++++++-- .../weather/hrrr_client_test.exs | 32 +++++++++++++++ .../workers/propagation_grid_worker_test.exs | 41 +++++++++++++++++++ 5 files changed, 139 insertions(+), 5 deletions(-) diff --git a/config/test.exs b/config/test.exs index 6c27f380..21392632 100644 --- a/config/test.exs +++ b/config/test.exs @@ -68,6 +68,11 @@ config :microwaveprop, cache_contact_map: false config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false] config :microwaveprop, gefs_req_options: [plug: {Req.Test, Microwaveprop.Weather.GefsClient}, retry: false] config :microwaveprop, giro_req_options: [plug: {Req.Test, Microwaveprop.Ionosphere.GiroClient}, retry: false] + +# Default the HRRR cycle freshness probe to "always available" in tests. +# Individual cases that exercise the fallback path override via +# `Application.put_env/3` in their setup. +config :microwaveprop, hrrr_cycle_available_fn: fn _run_time -> true end config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false] config :microwaveprop, iem_rate_limiter_interval_ms: 0 config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false] @@ -79,7 +84,6 @@ config :microwaveprop, rtma_req_options: [plug: {Req.Test, Microwaveprop.Weather config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}] config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false] config :microwaveprop, start_freshness_monitor: false -config :microwaveprop, start_score_cache_reconciler: false config :microwaveprop, swpc_req_options: [plug: {Req.Test, Microwaveprop.SpaceWeather.SwpcClient}, retry: false] config :microwaveprop, uwyo_req_options: [plug: {Req.Test, Microwaveprop.Weather.UwyoSoundingClient}, retry: false] diff --git a/lib/microwaveprop/weather/hrrr_client.ex b/lib/microwaveprop/weather/hrrr_client.ex index f9b93c83..985578be 100644 --- a/lib/microwaveprop/weather/hrrr_client.ex +++ b/lib/microwaveprop/weather/hrrr_client.ex @@ -177,6 +177,44 @@ defmodule Microwaveprop.Weather.HrrrClient do end end + @doc """ + Returns true if the HRRR cycle for `run_time` is fully published. + Probes the `wrfprsf18.grib2.idx` file (last forecast hour of the slower + pressure-level product — last to land for a given cycle). A 200 means + every earlier forecast hour is also on disk; anything else means the + cycle is still being uploaded or NOAA hasn't started it yet. + + Used by `PropagationGridWorker` to prefer a fresh `now-1h` cycle and + fall back to the conservative `now-2h` slot only when the fresher + cycle hasn't fully published. + + Override the probe in tests via the + `:microwaveprop, :hrrr_cycle_available_fn` Application env (a 1-arity + function from `DateTime` to `boolean`). + """ + @spec cycle_available?(DateTime.t()) :: boolean() + def cycle_available?(run_time) do + rounded = nearest_hrrr_hour(run_time) + + case Application.get_env(:microwaveprop, :hrrr_cycle_available_fn) do + fun when is_function(fun, 1) -> fun.(rounded) + _ -> probe_cycle(rounded) + end + end + + defp probe_cycle(%DateTime{} = run_time) do + url = + run_time + |> DateTime.to_date() + |> hrrr_url(run_time.hour, :pressure, 18) + |> Kernel.<>(".idx") + + case Req.head(url, req_options()) do + {:ok, %{status: 200}} -> true + _ -> false + end + end + @spec nearest_hrrr_hour(DateTime.t()) :: DateTime.t() def nearest_hrrr_hour(dt) do total_seconds = dt.minute * 60 + dt.second diff --git a/lib/microwaveprop/workers/propagation_grid_worker.ex b/lib/microwaveprop/workers/propagation_grid_worker.ex index 5de8f6ba..5dbbc7ec 100644 --- a/lib/microwaveprop/workers/propagation_grid_worker.ex +++ b/lib/microwaveprop/workers/propagation_grid_worker.ex @@ -45,10 +45,7 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do # rows are never re-worked. _ = GridTaskEnqueuer.reclaim_stale_running() - # HRRR takes ~45min to publish after the hour. Use 2 hours ago to - # ensure availability. - two_hours_ago = DateTime.add(DateTime.utc_now(), -2, :hour) - run_time = two_hours_ago |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second) + run_time = pick_run_time(DateTime.utc_now()) Logger.info("PropagationGrid: seeding chain run_time=#{run_time} (f00 analysis + f01-f18 forecasts via grid_tasks)") @@ -72,4 +69,26 @@ defmodule Microwaveprop.Workers.PropagationGridWorker do {:error, reason} end end + + @doc """ + Pick the freshest HRRR cycle that's published. + + Probes `now-1h` first via `HrrrClient.cycle_available?/1`. If the + pressure-level f18 idx file is on the mirror/S3, the cycle is fully + uploaded and we use it. Otherwise we fall back to the conservative + `now-2h` slot — NOAA's stated max publishing latency for HRRR. + + Public so the test suite can exercise the selection logic without + going through Oban. + """ + @spec pick_run_time(DateTime.t()) :: DateTime.t() + def pick_run_time(%DateTime{} = now) do + fresh = now |> DateTime.add(-1, :hour) |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second) + + if HrrrClient.cycle_available?(fresh) do + fresh + else + now |> DateTime.add(-2, :hour) |> HrrrClient.nearest_hrrr_hour() |> DateTime.truncate(:second) + end + end end diff --git a/test/microwaveprop/weather/hrrr_client_test.exs b/test/microwaveprop/weather/hrrr_client_test.exs index 3ffb04ba..dc1cbc3e 100644 --- a/test/microwaveprop/weather/hrrr_client_test.exs +++ b/test/microwaveprop/weather/hrrr_client_test.exs @@ -3,6 +3,38 @@ defmodule Microwaveprop.Weather.HrrrClientTest do alias Microwaveprop.Weather.HrrrClient + describe "cycle_available?/1 (stubbed probe)" do + setup do + original = Application.get_env(:microwaveprop, :hrrr_cycle_available_fn) + + on_exit(fn -> + if original, + do: Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, original), + else: Application.delete_env(:microwaveprop, :hrrr_cycle_available_fn) + end) + end + + test "returns whatever the configured probe function returns" do + Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn _dt -> true end) + assert HrrrClient.cycle_available?(~U[2026-04-25 00:00:00Z]) + + Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn _dt -> false end) + refute HrrrClient.cycle_available?(~U[2026-04-25 00:00:00Z]) + end + + test "passes the rounded run_time to the probe" do + test_pid = self() + + Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn dt -> + send(test_pid, {:probed, dt}) + true + end) + + HrrrClient.cycle_available?(~U[2026-04-25 00:38:00Z]) + assert_receive {:probed, ~U[2026-04-25 01:00:00Z]} + end + end + describe "nearest_hrrr_hour/1" do test "rounds down when within first 30 minutes" do assert HrrrClient.nearest_hrrr_hour(~U[2026-03-28 18:15:00Z]) == diff --git a/test/microwaveprop/workers/propagation_grid_worker_test.exs b/test/microwaveprop/workers/propagation_grid_worker_test.exs index 56548582..1958df39 100644 --- a/test/microwaveprop/workers/propagation_grid_worker_test.exs +++ b/test/microwaveprop/workers/propagation_grid_worker_test.exs @@ -15,6 +15,47 @@ defmodule Microwaveprop.Workers.PropagationGridWorkerTest do alias Microwaveprop.Workers.PropagationGridWorker alias Microwaveprop.Workers.PropagationPruneWorker + describe "pick_run_time/1 (HRRR cycle freshness selection)" do + setup do + original = Application.get_env(:microwaveprop, :hrrr_cycle_available_fn) + + on_exit(fn -> + if original, + do: Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, original), + else: Application.delete_env(:microwaveprop, :hrrr_cycle_available_fn) + end) + end + + test "picks now-1h when that cycle has fully published" do + Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn _dt -> true end) + + # Cron fires at HH:05, so use that as `now`. now-1h = 23:05 rounds + # to 23:00 (the freshly-published cycle). + now = ~U[2026-04-25 00:05:00Z] + assert PropagationGridWorker.pick_run_time(now) == ~U[2026-04-24 23:00:00Z] + end + + test "falls back to now-2h when the now-1h cycle is not yet available" do + Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn _dt -> false end) + + now = ~U[2026-04-25 00:05:00Z] + assert PropagationGridWorker.pick_run_time(now) == ~U[2026-04-24 22:00:00Z] + end + + test "probes the rounded now-1h value, not the raw `now`" do + test_pid = self() + + Application.put_env(:microwaveprop, :hrrr_cycle_available_fn, fn dt -> + send(test_pid, {:probed, dt}) + true + end) + + # 00:05 UTC: -1h = 23:05 UTC, rounds to 23:00 UTC. + PropagationGridWorker.pick_run_time(~U[2026-04-25 00:05:00Z]) + assert_receive {:probed, ~U[2026-04-24 23:00:00Z]} + end + end + describe "queue priority" do test "PropagationGridWorker runs at the highest priority on :propagation" do assert PropagationGridWorker.__opts__()[:priority] == 0