diff --git a/config/config.exs b/config/config.exs index ac4bc184..01163797 100644 --- a/config/config.exs +++ b/config/config.exs @@ -59,6 +59,7 @@ config :microwaveprop, Oban, solar: 1, weather: 20, enqueue: 1, + gefs: 2, hrrr: 5, terrain: 4, commercial: 2, diff --git a/config/dev.exs b/config/dev.exs index ba7ea9bb..95161cdf 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -78,6 +78,7 @@ config :microwaveprop, Oban, solar: 1, weather: 20, enqueue: 1, + gefs: 2, hrrr: 5, terrain: 4, commercial: 2, diff --git a/config/runtime.exs b/config/runtime.exs index f915c6ee..bd1b0599 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -165,6 +165,7 @@ if config_env() == :prod do commercial: 1, solar: 1, weather: 3, + gefs: 1, hrrr: 1, terrain: 3, iemre: 3, diff --git a/config/test.exs b/config/test.exs index 1bd74983..4b728a91 100644 --- a/config/test.exs +++ b/config/test.exs @@ -58,6 +58,7 @@ config :microwaveprop, config :microwaveprop, cache_contact_count: false config :microwaveprop, cache_contact_map: false config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false] +config :microwaveprop, gefs_req_options: [plug: {Req.Test, Microwaveprop.Weather.GefsClient}, retry: false] config :microwaveprop, giro_req_options: [plug: {Req.Test, Microwaveprop.Ionosphere.GiroClient}, retry: false] config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false] config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false] diff --git a/lib/microwaveprop/weather/gefs_client.ex b/lib/microwaveprop/weather/gefs_client.ex index 542c3676..b2546e67 100644 --- a/lib/microwaveprop/weather/gefs_client.ex +++ b/lib/microwaveprop/weather/gefs_client.ex @@ -38,6 +38,12 @@ defmodule Microwaveprop.Weather.GefsClient do %{var: "APCP", level: "surface"} ] + # Pressure levels where GEFS pgrb2a publishes all of TMP, RH, and HGT + # (the subset needed to build a temperature / dewpoint / height trace + # the scorer can consume). RH is missing at 300/400/600 mb even though + # HGT and UGRD/VGRD publish there — those levels are excluded. + @profile_pressure_levels [1000, 925, 850, 700, 500, 250, 200, 100, 50, 10] + @doc """ Rounds a wall-clock `DateTime` down to the nearest GEFS run cycle (00/06/12/18Z) on the same UTC day. @@ -107,6 +113,180 @@ defmodule Microwaveprop.Weather.GefsClient do b * gamma / (a - gamma) end + @doc """ + Converts a raw `%{"VAR:LEVEL" => float}` cell map (as returned by the + GRIB2 extractor) into a scorer-shaped profile map. + + Returns Celsius temperatures, millibar pressure, and a pressure-level + profile list with dewpoint derived from relative humidity. Missing + fields fall through as `nil` where the scorer can tolerate it. + """ + @spec build_profile(%{String.t() => number()}) :: map() + def build_profile(parsed) when is_map(parsed) do + profile = build_pressure_profile(parsed) + + sfc_temp_c = + case parsed["TMP:2 m above ground"] do + k when is_number(k) -> k - 273.15 + _ -> lowest_profile_value(profile, "tmpc") + end + + sfc_dewpoint_c = + case {sfc_temp_c, parsed["RH:2 m above ground"]} do + {t, rh} when is_number(t) and is_number(rh) -> dewpoint_from_rh(t, rh) + _ -> lowest_profile_value(profile, "dwpc") + end + + sfc_pressure_mb = + case parsed["PRES:surface"] do + pa when is_number(pa) -> pa / 100.0 + _ -> nil + end + + %{ + surface_temp_c: sfc_temp_c, + surface_dewpoint_c: sfc_dewpoint_c, + surface_pressure_mb: sfc_pressure_mb, + pwat_mm: parsed["PWAT:entire atmosphere (considered as a single layer)"], + wind_u_mps: parsed["UGRD:10 m above ground"], + wind_v_mps: parsed["VGRD:10 m above ground"], + cloud_cover_pct: parsed["TCDC:entire atmosphere"], + precip_mm: parsed["APCP:surface"], + profile: profile + } + end + + defp build_pressure_profile(parsed) do + Enum.flat_map(@profile_pressure_levels, fn level -> + key = "#{level} mb" + tmp_k = parsed["TMP:#{key}"] + rh = parsed["RH:#{key}"] + hgt = parsed["HGT:#{key}"] + + if is_number(tmp_k) and is_number(rh) and is_number(hgt) do + tmpc = tmp_k - 273.15 + + [ + %{ + "pres" => level * 1.0, + "tmpc" => tmpc, + "dwpc" => dewpoint_from_rh(tmpc, rh), + "hght" => hgt + } + ] + else + [] + end + end) + end + + defp lowest_profile_value([], _key), do: nil + defp lowest_profile_value([first | _], key), do: first[key] + + @doc """ + Fetches GEFS ensemble-mean (`geavg`) grid data for one forecast hour + and returns a list of scorer-ready profile attribute maps, one per + CONUS grid cell (0.125° matching `Microwaveprop.Propagation.Grid`). + + Uses NOMADS byte-range requests against the `.idx` sidecar to pull + only the surface messages the scorer needs, then hands the slim + GRIB2 binary to `wgrib2 -lola` for bilinear regridding from GEFS's + native 0.5° lat/lon grid onto the CONUS 0.125° grid. + + Returns `{:error, :wgrib2_not_available}` if the `wgrib2` binary + isn't on the host — all callers should degrade gracefully. + """ + @spec fetch_grid_profiles(Date.t(), non_neg_integer(), non_neg_integer()) :: + {:ok, [map()]} | {:error, term()} + def fetch_grid_profiles(%Date{} = run_date, run_hour, forecast_hour) do + alias Microwaveprop.Propagation.Grid + alias Microwaveprop.Weather.Grib2.Wgrib2 + alias Microwaveprop.Weather.HrrrClient + + url = ensmean_url(run_date, run_hour, forecast_hour) + + with {:ok, idx_text} <- fetch_idx(idx_url(url)), + idx_entries = HrrrClient.parse_idx(idx_text), + ranges = HrrrClient.byte_ranges_for_messages(idx_entries, @surface_messages), + {:ok, grib_binary} <- download_grib_ranges(url, ranges), + {:ok, grid} <- Wgrib2.extract_grid(grib_binary, wgrib2_match_pattern(), Grid.wgrib2_grid_spec()) do + profiles = + Enum.map(grid, fn {{lat, lon}, cell_values} -> + cell_values + |> build_profile() + |> Map.merge(%{lat: lat, lon: lon}) + end) + + {:ok, profiles} + end + end + + defp wgrib2_match_pattern do + vars = @surface_messages |> Enum.map(& &1.var) |> Enum.uniq() |> Enum.join("|") + ":(#{vars}):" + end + + defp fetch_idx(url) do + case Req.get(url, req_options()) do + {:ok, %{status: 200, body: body}} -> {:ok, body} + {:ok, %{status: status}} -> {:error, "GEFS idx HTTP #{status}"} + {:error, reason} -> {:error, reason} + end + end + + defp download_grib_ranges(_url, []), do: {:ok, <<>>} + + defp download_grib_ranges(url, ranges) do + merged = + ranges + |> Enum.sort_by(&elem(&1, 0)) + |> Enum.reduce([], fn {s, e}, acc -> + case acc do + [{ps, pe} | rest] when s <= pe + 1 -> [{ps, max(pe, e)} | rest] + _ -> [{s, e} | acc] + end + end) + |> Enum.reverse() + + results = + merged + |> Task.async_stream( + fn {start, stop} -> + case Req.get(url, [{:headers, [{"Range", "bytes=#{start}-#{stop}"}]} | req_options()]) do + {:ok, %{status: 206, body: body}} -> {:ok, start, body} + {:ok, %{status: status}} -> {:error, "GEFS grib HTTP #{status}"} + {:error, reason} -> {:error, reason} + end + end, + max_concurrency: 8, + timeout: 120_000 + ) + |> Enum.map(fn + {:ok, result} -> result + {:exit, reason} -> {:error, reason} + end) + + case Enum.find(results, &match?({:error, _}, &1)) do + {:error, reason} -> + {:error, reason} + + nil -> + binary = + results + |> Enum.sort_by(fn {:ok, offset, _} -> offset end) + |> Enum.map(fn {:ok, _, body} -> body end) + |> IO.iodata_to_binary() + + {:ok, binary} + end + end + + defp req_options do + defaults = [receive_timeout: 120_000, max_retries: 3] + overrides = Application.get_env(:microwaveprop, :gefs_req_options, []) + Keyword.merge(defaults, overrides) + end + defp pad2(n), do: n |> Integer.to_string() |> String.pad_leading(2, "0") defp pad3(n), do: n |> Integer.to_string() |> String.pad_leading(3, "0") end diff --git a/lib/microwaveprop/workers/gefs_fetch_worker.ex b/lib/microwaveprop/workers/gefs_fetch_worker.ex new file mode 100644 index 00000000..74a4b2eb --- /dev/null +++ b/lib/microwaveprop/workers/gefs_fetch_worker.ex @@ -0,0 +1,122 @@ +defmodule Microwaveprop.Workers.GefsFetchWorker do + @moduledoc """ + Fetches one GEFS ensemble-mean forecast hour, extracts the CONUS + grid, and upserts into `gefs_profiles`. + + Each `perform/1` handles a single `(run_time, forecast_hour)` pair so + a failure at one hour doesn't block the others — the cron seeder + enqueues the full `f024..f168` set per run. Per-hour independence + also keeps memory bounded: the wgrib2 extraction step for a GEFS + 0.5° file is small (<1 MB after byte-range slicing) but the target + 0.125° grid still carries ~200k cells. + """ + + use Oban.Worker, + queue: :gefs, + max_attempts: 5, + unique: [period: 3600, states: [:available, :scheduled, :executing, :retryable]] + + alias Microwaveprop.Weather + alias Microwaveprop.Weather.GefsClient + alias Microwaveprop.Weather.SoundingParams + + require Logger + + @valid_run_hours [0, 6, 12, 18] + + @impl Oban.Worker + def perform(%Oban.Job{args: %{"run_time" => run_time_iso, "forecast_hour" => fh}}) when is_integer(fh) do + with {:ok, run_time, _} <- DateTime.from_iso8601(run_time_iso), + :ok <- validate_run_hour(run_time.hour) do + run_date = DateTime.to_date(run_time) + run_hour = run_time.hour + + Logger.info("GefsFetch: run_time=#{run_time_iso} fh=#{fh}") + + case GefsClient.fetch_grid_profiles(run_date, run_hour, fh) do + {:ok, profiles} -> + attrs = Enum.map(profiles, &build_profile_attrs(run_time, fh, &1)) + {count, _} = Weather.upsert_gefs_profiles_batch(attrs) + Logger.info("GefsFetch: stored #{count}/#{length(attrs)} profiles for fh=#{fh}") + :ok + + {:error, :wgrib2_not_available} -> + Logger.warning("GefsFetch: wgrib2 missing on host — cancelling job") + {:cancel, :wgrib2_not_available} + + {:error, reason} -> + handle_error(reason, "fh=#{fh} @ #{run_time_iso}") + end + else + _ -> {:cancel, :invalid_args} + end + end + + def perform(%Oban.Job{args: _args}), do: {:cancel, :invalid_args} + + @doc """ + Converts a `GefsClient.build_profile/1` result (already tagged with + `lat` and `lon`) into a full `gefs_profiles` row attr map. Exposed + for unit testing. + """ + @spec build_profile_attrs(DateTime.t(), non_neg_integer(), map()) :: map() + def build_profile_attrs(%DateTime{} = run_time, forecast_hour, profile) when is_integer(forecast_hour) do + valid_time = DateTime.add(run_time, forecast_hour * 3600, :second) + derived = SoundingParams.derive(profile.profile || []) + + base = %{ + run_time: DateTime.truncate(run_time, :second), + forecast_hour: forecast_hour, + valid_time: DateTime.truncate(valid_time, :second), + lat: profile.lat, + lon: profile.lon, + profile: profile.profile, + surface_temp_c: profile.surface_temp_c, + surface_dewpoint_c: profile.surface_dewpoint_c, + surface_pressure_mb: profile.surface_pressure_mb, + pwat_mm: profile[:pwat_mm], + wind_u_mps: profile[:wind_u_mps], + wind_v_mps: profile[:wind_v_mps], + cloud_cover_pct: profile[:cloud_cover_pct], + precip_mm: profile[:precip_mm] + } + + case derived do + nil -> + base + + params -> + Map.merge(base, %{ + surface_refractivity: params.surface_refractivity, + min_refractivity_gradient: params.min_refractivity_gradient, + ducting_detected: params.ducting_detected, + duct_characteristics: params.duct_characteristics + }) + end + end + + defp validate_run_hour(hour) when hour in @valid_run_hours, do: :ok + defp validate_run_hour(_), do: {:error, :invalid_run_hour} + + defp handle_error(reason, label) do + if transient?(reason) do + Logger.error("GefsFetch transient error for #{label}: #{inspect(reason)}") + {:error, reason} + else + Logger.warning("GefsFetch permanent failure for #{label}: #{inspect(reason)}") + {:cancel, reason} + end + end + + defp transient?(%{__exception__: true}), do: true + defp transient?("GEFS idx HTTP " <> status), do: server_error?(status) + defp transient?("GEFS grib HTTP " <> status), do: server_error?(status) + defp transient?(_), do: false + + defp server_error?(status) do + case Integer.parse(status) do + {code, _} when code in [429, 500, 502, 503, 504] -> true + _ -> false + end + end +end diff --git a/test/microwaveprop/weather/gefs_client_test.exs b/test/microwaveprop/weather/gefs_client_test.exs index 77a66bd4..571657eb 100644 --- a/test/microwaveprop/weather/gefs_client_test.exs +++ b/test/microwaveprop/weather/gefs_client_test.exs @@ -95,6 +95,90 @@ defmodule Microwaveprop.Weather.GefsClientTest do end end + describe "build_profile/1" do + @raw %{ + "TMP:2 m above ground" => 293.15, + "RH:2 m above ground" => 50.0, + "PRES:surface" => 101_325.0, + "PWAT:entire atmosphere (considered as a single layer)" => 25.4, + "UGRD:10 m above ground" => 3.0, + "VGRD:10 m above ground" => -1.5, + "TCDC:entire atmosphere" => 62.0, + "APCP:surface" => 0.0, + "TMP:1000 mb" => 293.15, + "RH:1000 mb" => 50.0, + "HGT:1000 mb" => 110.0, + "TMP:925 mb" => 288.15, + "RH:925 mb" => 40.0, + "HGT:925 mb" => 780.0 + } + + test "converts surface TMP from Kelvin to Celsius" do + p = GefsClient.build_profile(@raw) + assert_in_delta p.surface_temp_c, 20.0, 0.01 + end + + test "converts surface PRES from Pa to mb" do + p = GefsClient.build_profile(@raw) + assert_in_delta p.surface_pressure_mb, 1013.25, 0.1 + end + + test "derives surface dewpoint from RH via Magnus" do + p = GefsClient.build_profile(@raw) + assert_in_delta p.surface_dewpoint_c, 9.27, 0.3 + end + + test "carries PWAT, winds, cloud cover, precip through untouched" do + p = GefsClient.build_profile(@raw) + assert p.pwat_mm == 25.4 + assert p.wind_u_mps == 3.0 + assert p.wind_v_mps == -1.5 + assert p.cloud_cover_pct == 62.0 + assert p.precip_mm == 0.0 + end + + test "builds a pressure-level profile with Td derived from RH at each level" do + p = GefsClient.build_profile(@raw) + assert length(p.profile) == 2 + + l1000 = Enum.find(p.profile, &(&1["pres"] == 1000.0)) + assert_in_delta l1000["tmpc"], 20.0, 0.01 + assert_in_delta l1000["dwpc"], 9.27, 0.3 + assert l1000["hght"] == 110.0 + + l925 = Enum.find(p.profile, &(&1["pres"] == 925.0)) + assert_in_delta l925["tmpc"], 15.0, 0.01 + assert l925["dwpc"] < l925["tmpc"] + end + + test "falls back to the lowest pressure-level values when surface fields are missing" do + raw = %{ + "TMP:1000 mb" => 293.15, + "RH:1000 mb" => 60.0, + "HGT:1000 mb" => 100.0 + } + + p = GefsClient.build_profile(raw) + assert_in_delta p.surface_temp_c, 20.0, 0.01 + assert is_float(p.surface_dewpoint_c) + assert p.surface_pressure_mb == nil + end + + test "skips pressure levels that lack TMP or HGT" do + raw = + Map.drop(@raw, [ + "TMP:925 mb", + "RH:925 mb", + "HGT:925 mb" + ]) + + p = GefsClient.build_profile(raw) + pressures = Enum.map(p.profile, & &1["pres"]) + assert 1000.0 in pressures + refute 925.0 in pressures + end + end + describe "dewpoint_from_rh/2" do test "returns the input temperature when RH is 100%" do # At saturation, Td == T. Magnus returns T exactly. diff --git a/test/microwaveprop/workers/gefs_fetch_worker_test.exs b/test/microwaveprop/workers/gefs_fetch_worker_test.exs new file mode 100644 index 00000000..62c568b3 --- /dev/null +++ b/test/microwaveprop/workers/gefs_fetch_worker_test.exs @@ -0,0 +1,114 @@ +defmodule Microwaveprop.Workers.GefsFetchWorkerTest do + use Microwaveprop.DataCase, async: false + + alias Microwaveprop.Weather + alias Microwaveprop.Weather.GefsProfile + alias Microwaveprop.Workers.GefsFetchWorker + + describe "perform/1" do + test "rejects args missing run_time or forecast_hour" do + job = %Oban.Job{args: %{}} + assert {:cancel, _} = GefsFetchWorker.perform(job) + end + + test "cancels when run_hour is not a valid GEFS cycle" do + args = %{ + "run_time" => "2026-04-18T05:00:00Z", + "forecast_hour" => 24 + } + + assert {:cancel, _} = GefsFetchWorker.perform(%Oban.Job{args: args}) + end + end + + describe "build_profile_attrs/3" do + test "merges lat/lon/valid_time onto a GefsClient profile map" do + run_time = ~U[2026-04-18 12:00:00Z] + + profile = %{ + lat: 32.9, + lon: -97.0, + surface_temp_c: 20.0, + surface_dewpoint_c: 10.0, + surface_pressure_mb: 1013.0, + pwat_mm: 20.0, + wind_u_mps: 2.0, + wind_v_mps: -1.0, + cloud_cover_pct: 50.0, + precip_mm: 0.0, + profile: [%{"pres" => 1000.0, "tmpc" => 20.0, "dwpc" => 10.0, "hght" => 100.0}] + } + + attrs = GefsFetchWorker.build_profile_attrs(run_time, 24, profile) + assert attrs.run_time == run_time + assert attrs.forecast_hour == 24 + assert attrs.valid_time == ~U[2026-04-19 12:00:00Z] + assert attrs.lat == 32.9 + assert attrs.surface_temp_c == 20.0 + end + + test "derives SoundingParams-style refractivity metrics when profile is populated" do + run_time = ~U[2026-04-18 12:00:00Z] + + profile = %{ + lat: 32.9, + lon: -97.0, + surface_temp_c: 20.0, + surface_dewpoint_c: 15.0, + surface_pressure_mb: 1013.0, + pwat_mm: 25.0, + profile: [ + %{"pres" => 1000.0, "tmpc" => 20.0, "dwpc" => 15.0, "hght" => 100.0}, + %{"pres" => 925.0, "tmpc" => 18.0, "dwpc" => 12.0, "hght" => 770.0}, + %{"pres" => 850.0, "tmpc" => 14.0, "dwpc" => 8.0, "hght" => 1500.0} + ] + } + + attrs = GefsFetchWorker.build_profile_attrs(run_time, 24, profile) + assert is_float(attrs.surface_refractivity) + assert is_float(attrs.min_refractivity_gradient) + assert is_boolean(attrs.ducting_detected) + end + end + + describe "storing profiles" do + test "round-trips a full batch through the context" do + run_time = ~U[2026-04-18 12:00:00Z] + fh = 24 + valid_time = DateTime.add(run_time, fh * 3600, :second) + + batch = [ + %{ + run_time: run_time, + forecast_hour: fh, + valid_time: valid_time, + lat: 32.9, + lon: -97.0, + surface_temp_c: 20.0, + surface_dewpoint_c: 10.0, + surface_pressure_mb: 1013.0, + pwat_mm: 20.0, + profile: [] + }, + %{ + run_time: run_time, + forecast_hour: fh, + valid_time: valid_time, + lat: 33.0, + lon: -97.0, + surface_temp_c: 19.5, + surface_dewpoint_c: 9.5, + surface_pressure_mb: 1012.5, + pwat_mm: 19.0, + profile: [] + } + ] + + {count, _} = Weather.upsert_gefs_profiles_batch(batch) + assert count == 2 + + persisted = Microwaveprop.Repo.all(GefsProfile) + assert length(persisted) == 2 + end + end +end