defmodule Microwaveprop.Workers.GefsFetchWorker do @moduledoc """ Fetches one GEFS ensemble-mean forecast hour, extracts the CONUS grid, and upserts into `gefs_profiles`. Each `perform/1` handles a single `(run_time, forecast_hour)` pair so a failure at one hour doesn't block the others — the cron seeder enqueues the full `f024..f168` set per run. Per-hour independence also keeps memory bounded: the wgrib2 extraction step for a GEFS 0.5° file is small (<1 MB after byte-range slicing) but the target 0.125° grid still carries ~200k cells. """ use Oban.Pro.Worker, queue: :gefs, max_attempts: 5, unique: [period: 3600, states: :incomplete] alias Microwaveprop.Propagation alias Microwaveprop.Weather alias Microwaveprop.Weather.GefsClient alias Microwaveprop.Weather.SoundingParams require Logger @valid_run_hours [0, 6, 12, 18] @impl Oban.Pro.Worker def process(%Oban.Job{args: %{"run_time" => run_time_iso, "forecast_hour" => fh}}) when is_integer(fh) do with {:ok, run_time, _} <- DateTime.from_iso8601(run_time_iso), :ok <- validate_run_hour(run_time.hour) do run_date = DateTime.to_date(run_time) run_hour = run_time.hour Logger.info("GefsFetch: run_time=#{run_time_iso} fh=#{fh}") case GefsClient.fetch_grid_profiles(run_date, run_hour, fh) do {:ok, profiles} -> valid_time = run_time |> DateTime.add(fh * 3600, :second) |> DateTime.truncate(:second) score_and_persist(profiles, run_time, fh, valid_time) :ok {:error, :wgrib2_not_available} -> Logger.warning("GefsFetch: wgrib2 missing on host — cancelling job") {:cancel, :wgrib2_not_available} {:error, reason} -> handle_error(reason, "fh=#{fh} @ #{run_time_iso}") end else _ -> {:cancel, :invalid_args} end end def process(%Oban.Job{args: args}) when args == %{} do seed_extended_horizon() end def process(%Oban.Job{args: _args}), do: {:cancel, :invalid_args} @doc """ Forecast hours enqueued per GEFS run: f024 through f168 at the 6-hour cadence GEFS publishes at. Covers Day 2-7 — the window HRRR can't reach. Starts at f024 so the near-term (f00-f18) stays HRRR-owned and the handoff lines up on the f018 boundary. """ @spec extended_horizon_hours() :: [non_neg_integer()] def extended_horizon_hours, do: Enum.to_list(24..168//6) @doc """ Enqueues one fetch job per extended-horizon forecast hour for the most recent GEFS run that should already be published on NOMADS (run publication lags ~3-4 hours behind the run time; the seeder targets runs 5+ hours in the past for safety). """ @spec seed_extended_horizon() :: :ok def seed_extended_horizon do run_time = most_recent_available_run(DateTime.utc_now()) Logger.info("GefsFetch: seeding extended-horizon chain for run_time=#{run_time}") Enum.each(extended_horizon_hours(), fn fh -> %{"run_time" => DateTime.to_iso8601(run_time), "forecast_hour" => fh} |> new() |> Oban.insert() end) :ok end @doc """ Picks the most-recent 00/06/12/18Z GEFS run whose file set should be fully published given an ~5-hour publication lag. Exposed so tests can pin a deterministic "now". """ @spec most_recent_available_run(DateTime.t()) :: DateTime.t() def most_recent_available_run(%DateTime{} = now) do now |> DateTime.add(-5 * 3600, :second) |> GefsClient.nearest_run() end @doc """ Converts a `GefsClient.build_profile/1` result (already tagged with `lat` and `lon`) into a full `gefs_profiles` row attr map. Exposed for unit testing. """ @spec build_profile_attrs(DateTime.t(), non_neg_integer(), map()) :: map() def build_profile_attrs(%DateTime{} = run_time, forecast_hour, profile) when is_integer(forecast_hour) do valid_time = DateTime.add(run_time, forecast_hour * 3600, :second) derived = SoundingParams.derive(profile.profile || []) base = %{ run_time: DateTime.truncate(run_time, :second), forecast_hour: forecast_hour, valid_time: DateTime.truncate(valid_time, :second), lat: profile.lat, lon: profile.lon, profile: profile.profile, surface_temp_c: profile.surface_temp_c, surface_dewpoint_c: profile.surface_dewpoint_c, surface_pressure_mb: profile.surface_pressure_mb, pwat_mm: profile[:pwat_mm], wind_u_mps: profile[:wind_u] || profile[:wind_u_mps], wind_v_mps: profile[:wind_v] || profile[:wind_v_mps], cloud_cover_pct: profile[:cloud_cover_pct], precip_mm: profile[:precip_mm] } case derived do nil -> base params -> Map.merge(base, %{ surface_refractivity: params.surface_refractivity, min_refractivity_gradient: params.min_refractivity_gradient, ducting_detected: params.ducting_detected, duct_characteristics: params.duct_characteristics }) end end # Runs the propagation scorer across the fetched GEFS grid and # persists scores to the `.prop` file for this `valid_time` so the # extended-horizon outlook picks them up via the usual # `Propagation.scores_at/3` path. Also upserts raw profiles to # `gefs_profiles` for future per-contact extended-horizon lookups. defp score_and_persist(profiles, run_time, fh, valid_time) do grid_data = profiles_to_grid(profiles) supervisor = {:via, PartitionSupervisor, {Microwaveprop.TaskSupervisor, self()}} scores = supervisor |> Task.Supervisor.async_stream_nolink( grid_data, fn {{lat, lon}, profile} -> profile |> Propagation.score_grid_point(valid_time, lat, lon) |> Enum.map(fn r -> %{ lat: lat, lon: lon, valid_time: valid_time, band_mhz: r.band_mhz, score: r.score, factors: nil } end) end, max_concurrency: System.schedulers_online() * 2, timeout: 30_000 ) |> Stream.flat_map(fn {:ok, results} -> results {:exit, reason} -> Logger.error( "GefsFetch scoring task crashed: fh=#{fh} valid_time=#{inspect(valid_time)} reason=#{inspect(reason)}" ) [] end) |> Enum.to_list() _ = case Propagation.replace_scores(scores, valid_time) do {:ok, count} -> Logger.info("GefsFetch: wrote #{count} scores for fh=#{fh} @ #{valid_time}") broadcast_updated(valid_time) error -> Logger.error("GefsFetch: replace_scores failed fh=#{fh}: #{inspect(error)}") end attrs = Enum.map(profiles, &build_profile_attrs(run_time, fh, &1)) {count, _} = Weather.upsert_gefs_profiles_batch(attrs) Logger.info("GefsFetch: upserted #{count}/#{length(attrs)} profile rows for fh=#{fh}") end defp profiles_to_grid(profiles) do Map.new(profiles, fn profile -> {{profile.lat, profile.lon}, profile} end) end defp broadcast_updated(valid_time) do Phoenix.PubSub.broadcast( Microwaveprop.PubSub, "propagation:updated", {:propagation_updated, [valid_time]} ) end defp validate_run_hour(hour) when hour in @valid_run_hours, do: :ok defp validate_run_hour(_), do: {:error, :invalid_run_hour} defp handle_error(reason, label) do if transient?(reason) do Logger.error("GefsFetch transient error for #{label}: #{inspect(reason)}") {:error, reason} else Logger.warning("GefsFetch permanent failure for #{label}: #{inspect(reason)}") {:cancel, reason} end end defp transient?(%{__exception__: true}), do: true defp transient?("GEFS idx HTTP " <> status), do: server_error?(status) defp transient?("GEFS grib HTTP " <> status), do: server_error?(status) defp transient?(_), do: false defp server_error?(status) do case Integer.parse(status) do {code, _} when code in [429, 500, 502, 503, 504] -> true _ -> false end end end