From c318c3a932bb36e86a157319ac15c82457a308c6 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 12 Apr 2026 14:27:27 -0500 Subject: [PATCH] Nudge HRRR scores with live ASOS obs between runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds Microwaveprop.Propagation.AsosNudge: a pure IDW bias-field module that takes ASOS observations + HRRR profiles and returns re-scored grid rows for every cell within 250km of a reporting station. Upper-air fields (min_refractivity_gradient, pwat_mm, hpbl_m, profile, duct metadata) pass through unchanged so HRRR's signal isn't clobbered. The old AsosAdjustmentWorker was unwired and buggy — nil'd out ~22% of the scoring weight and wrote orphan timestamps. Replaced with a slim worker that queries the latest HRRR valid_time, fetches live ASOS currents, calls AsosNudge.compute/3, and upserts onto (lat, lon, valid_time, band_mhz) so nudged values overwrite the HRRR hour cleanly instead of polluting available_valid_times. After each upsert it warms ScoreCache and broadcasts propagation:updated so live /map clients refresh. Cron hooked up every 10 minutes in config.exs and dev.exs. Also cleaned up the stale "dev has propagation disabled" note in CLAUDE.md. 13 new AsosNudge unit tests cover: residual computation (co-located, out-of-grid, nil fields), IDW weighting (single station, far station, two equidistant stations, nil component handling), upper-air preservation, and the compute/3 entry point's shape and radius filter. Drive-by Styler formatting touched a handful of unrelated files from `mix format`. --- CLAUDE.md | 2 +- config/config.exs | 1 + config/dev.exs | 1 + config/runtime.exs | 28 +- config/test.exs | 4 +- lib/microwaveprop/propagation.ex | 3 +- lib/microwaveprop/propagation/asos_nudge.ex | 230 +++++++++++++++ lib/microwaveprop/weather/nexrad_client.ex | 4 +- .../workers/asos_adjustment_worker.ex | 264 +++++++----------- .../live/contact_live/show.ex | 3 +- .../live/contact_map_live.ex | 1 - lib/microwaveprop_web/live/submit_live.ex | 3 +- .../propagation/asos_nudge_test.exs | 186 ++++++++++++ .../propagation/score_cache_test.exs | 1 + test/microwaveprop/propagation_test.exs | 4 +- 15 files changed, 549 insertions(+), 186 deletions(-) create mode 100644 lib/microwaveprop/propagation/asos_nudge.ex create mode 100644 test/microwaveprop/propagation/asos_nudge_test.exs diff --git a/CLAUDE.md b/CLAUDE.md index 9e38f257..46b4204c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -106,7 +106,7 @@ mix assets.deploy # minified + digest **Production** runs: propagation, commercial, solar, weather, hrrr, terrain, iemre queues. No cron backfill — enrichment is triggered by QSO submission only. -**Dev** currently has propagation disabled for backfill. Re-enable by removing the Oban override in `config/dev.exs`. +**Dev** runs the same hourly `PropagationGridWorker` + `AsosAdjustmentWorker` (10-minute nudge) cron as production. ### Scoring Algorithm diff --git a/config/config.exs b/config/config.exs index cda7db98..92aca30a 100644 --- a/config/config.exs +++ b/config/config.exs @@ -65,6 +65,7 @@ config :microwaveprop, Oban, {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}, {"*/5 * * * *", Microwaveprop.Commercial.PollWorker}, {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}, + {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker} ]} ] diff --git a/config/dev.exs b/config/dev.exs index 39b5fba1..523449f6 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -88,6 +88,7 @@ config :microwaveprop, Oban, {Oban.Plugins.Cron, crontab: [ {"5 * * * *", Microwaveprop.Workers.PropagationGridWorker}, + {"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker}, {"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}, {"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker} ]} diff --git a/config/runtime.exs b/config/runtime.exs index 635fb69e..18783d46 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -46,20 +46,6 @@ if config_env() == :prod do host = System.get_env("PHX_HOST") || "example.com" - config :libcluster, - topologies: [ - k8s: [ - strategy: Cluster.Strategy.Kubernetes, - config: [ - mode: :ip, - kubernetes_ip_lookup_mode: :pods, - kubernetes_node_basename: "microwaveprop", - kubernetes_selector: "app=prop", - kubernetes_namespace: "prop" - ] - ] - ] - # ## SSL Support # # To get SSL working, you will need to add the `https` key @@ -105,6 +91,20 @@ if config_env() == :prod do # handshake fails with an "unexpected_message" TLS alert. email_server = System.get_env("EMAIL_SERVER") + config :libcluster, + topologies: [ + k8s: [ + strategy: Cluster.Strategy.Kubernetes, + config: [ + mode: :ip, + kubernetes_ip_lookup_mode: :pods, + kubernetes_node_basename: "microwaveprop", + kubernetes_selector: "app=prop", + kubernetes_namespace: "prop" + ] + ] + ] + config :microwaveprop, Microwaveprop.Mailer, adapter: Swoosh.Adapters.SMTP, relay: email_server, diff --git a/config/test.exs b/config/test.exs index 3666539b..423cb496 100644 --- a/config/test.exs +++ b/config/test.exs @@ -33,6 +33,8 @@ config :microwaveprop, MicrowavepropWeb.Endpoint, # Run Oban jobs inline during tests config :microwaveprop, Oban, testing: :inline +config :microwaveprop, cache_contact_count: false +config :microwaveprop, cache_contact_map: false config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false] config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false] config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false] @@ -42,8 +44,6 @@ config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weath config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}] config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false] config :microwaveprop, start_freshness_monitor: false -config :microwaveprop, cache_contact_count: false -config :microwaveprop, cache_contact_map: false # Initialize plugs at runtime for faster test compilation config :phoenix, :plug_init_mode, :runtime diff --git a/lib/microwaveprop/propagation.ex b/lib/microwaveprop/propagation.ex index e9119be0..eacf9a92 100644 --- a/lib/microwaveprop/propagation.ex +++ b/lib/microwaveprop/propagation.ex @@ -350,8 +350,7 @@ defmodule Microwaveprop.Propagation do defp snap_to_grid(lat, lon) do step = Grid.step() - {Float.round(Float.round(lat / step) * step, 3), - Float.round(Float.round(lon / step) * step, 3)} + {Float.round(Float.round(lat / step) * step, 3), Float.round(Float.round(lon / step) * step, 3)} end @doc "Get the full score and factors for a specific grid point, snapped to nearest grid." diff --git a/lib/microwaveprop/propagation/asos_nudge.ex b/lib/microwaveprop/propagation/asos_nudge.ex new file mode 100644 index 00000000..0a30ee3e --- /dev/null +++ b/lib/microwaveprop/propagation/asos_nudge.ex @@ -0,0 +1,230 @@ +defmodule Microwaveprop.Propagation.AsosNudge do + @moduledoc """ + Blend live ASOS surface observations into the latest HRRR grid between + hourly runs. HRRR analysis is already ~55 minutes old by the time the + propagation worker scores it; fronts, humidity swings, and rain onset + happening after that lag otherwise sit invisible until the next top-of-hour + run. ASOS METARs publish every 5 minutes, so a 10-minute nudging cycle + catches them. + + ## Pipeline + + 1. `build_station_residuals/2` — for each ASOS observation, find the + nearest HRRR grid cell (within one 0.125° step, otherwise the station + is outside the grid domain and dropped). The residual is + `(asos_value - hrrr_value)` for temp, dewpoint, and pressure, plus the + raw ASOS rain rate. + + 2. `nudge_profile/2` — for a given HRRR profile, compute an + inverse-distance weighted (1/d²) blend of residuals from stations + within 250 km. Only the surface fields are touched; refractivity + gradient, PWAT, HPBL, duct metadata, and the full vertical profile + all pass through unchanged so the HRRR upper-air signal is preserved. + + 3. `compute/3` — run the pipeline over every HRRR profile that has at + least one station within range, call + `Microwaveprop.Propagation.score_grid_point/4` on the patched profile, + and return score rows ready for `Propagation.upsert_scores/1`. Grid + cells with no station in range are dropped from the output, leaving + the existing HRRR scores untouched. + + The nudging module itself is pure — no Repo, no PubSub. The worker that + schedules this (`Microwaveprop.Workers.AsosAdjustmentWorker`) handles all + I/O. + """ + + alias Microwaveprop.Propagation + alias Microwaveprop.Propagation.Grid + + @radius_km 250 + @min_station_weight_km 1.0 + @earth_radius_km 6371.0 + + @type observation :: %{ + required(:lat) => float(), + required(:lon) => float(), + optional(:temp_f) => number() | nil, + optional(:dewpoint_f) => number() | nil, + optional(:sea_level_pressure_mb) => number() | nil, + optional(:precip_1h_in) => number() | nil + } + + @type residual :: %{ + lat: float(), + lon: float(), + dtemp_c: float() | nil, + ddewpoint_c: float() | nil, + dpressure_mb: float() | nil, + asos_rain_mmhr: float() + } + + @type hrrr_profile :: map() + + @type grid_score :: %{ + lat: float(), + lon: float(), + valid_time: DateTime.t(), + band_mhz: non_neg_integer(), + score: non_neg_integer(), + factors: map() + } + + @doc """ + Nudge every HRRR grid cell that has an ASOS station within 250 km and + re-score it. Cells outside that radius are dropped so the existing HRRR + scores stay in place. + """ + @spec compute([observation()], DateTime.t(), [hrrr_profile()]) :: [grid_score()] + def compute([], _valid_time, _profiles), do: [] + def compute(_observations, _valid_time, []), do: [] + + def compute(observations, valid_time, profiles) do + residuals = build_station_residuals(observations, profiles) + + profiles + |> Enum.filter(&any_station_within_radius?(&1, residuals)) + |> Enum.flat_map(fn profile -> + profile + |> nudge_profile(residuals) + |> score_point(valid_time) + end) + end + + @doc """ + For each observation with a valid HRRR anchor cell, compute the + `(asos - hrrr)` residual at that cell. Used by `nudge_profile/2` to + spread the bias field across the grid. + """ + @spec build_station_residuals([observation()], [hrrr_profile()]) :: [residual()] + def build_station_residuals(observations, profiles) do + lookup = Map.new(profiles, fn p -> {snap_to_grid(p.lat, p.lon), p} end) + + observations + |> Enum.map(&station_residual(&1, lookup)) + |> Enum.reject(&is_nil/1) + end + + @doc """ + Return a patched HRRR profile with `surface_temp_c`, `surface_dewpoint_c`, + and `surface_pressure_mb` shifted by the IDW-weighted residuals from + stations within 250 km. If no station is close enough, the profile is + returned unchanged. + """ + @spec nudge_profile(hrrr_profile(), [residual()]) :: hrrr_profile() + def nudge_profile(profile, residuals) do + nearby = + residuals + |> Enum.map(fn r -> {r, distance_km(profile.lat, profile.lon, r.lat, r.lon)} end) + |> Enum.filter(fn {_r, d} -> d <= @radius_km end) + + case nearby do + [] -> + profile + + _ -> + profile + |> Map.put(:surface_temp_c, apply_idw(profile.surface_temp_c, nearby, :dtemp_c)) + |> Map.put(:surface_dewpoint_c, apply_idw(profile.surface_dewpoint_c, nearby, :ddewpoint_c)) + |> Map.put(:surface_pressure_mb, apply_idw(profile.surface_pressure_mb, nearby, :dpressure_mb)) + end + end + + # -- internals --------------------------------------------------------- + + defp station_residual(obs, lookup) do + with {:ok, temp_c} <- f_to_c(obs[:temp_f]), + {:ok, dewpoint_c} <- f_to_c(obs[:dewpoint_f]), + snapped = snap_to_grid(obs.lat, obs.lon), + hrrr when not is_nil(hrrr) <- Map.get(lookup, snapped), + true <- within_one_grid_step?(obs, hrrr) do + %{ + lat: obs.lat, + lon: obs.lon, + dtemp_c: delta(temp_c, hrrr.surface_temp_c), + ddewpoint_c: delta(dewpoint_c, hrrr.surface_dewpoint_c), + dpressure_mb: delta(obs[:sea_level_pressure_mb], hrrr.surface_pressure_mb), + asos_rain_mmhr: precip_to_mmhr(obs[:precip_1h_in]) + } + else + _ -> nil + end + end + + defp f_to_c(nil), do: :error + defp f_to_c(f) when is_number(f), do: {:ok, (f - 32) * 5 / 9} + defp f_to_c(_), do: :error + + defp delta(nil, _), do: nil + defp delta(_, nil), do: nil + defp delta(a, b), do: a - b + + defp precip_to_mmhr(nil), do: 0.0 + defp precip_to_mmhr(inches) when inches <= 0, do: 0.0 + defp precip_to_mmhr(inches), do: inches * 25.4 + + defp within_one_grid_step?(obs, hrrr) do + distance_km(obs.lat, obs.lon, hrrr.lat, hrrr.lon) <= Grid.step() * 111.0 + end + + defp any_station_within_radius?(profile, residuals) do + Enum.any?(residuals, fn r -> + distance_km(profile.lat, profile.lon, r.lat, r.lon) <= @radius_km + end) + end + + defp apply_idw(base, _nearby, _key) when is_nil(base), do: nil + + defp apply_idw(base, nearby, key) do + contributing = + Enum.filter(nearby, fn {residual, _d} -> not is_nil(Map.get(residual, key)) end) + + case contributing do + [] -> + base + + _ -> + {num, den} = + Enum.reduce(contributing, {0.0, 0.0}, fn {residual, d_km}, {num, den} -> + w = 1.0 / :math.pow(max(d_km, @min_station_weight_km), 2) + {num + w * Map.get(residual, key), den + w} + end) + + base + num / den + end + end + + defp score_point(profile, valid_time) do + profile + |> Propagation.score_grid_point(valid_time, profile.lat, profile.lon) + |> Enum.map(fn row -> + row + |> Map.put(:lat, profile.lat) + |> Map.put(:lon, profile.lon) + |> Map.put(:valid_time, valid_time) + end) + end + + defp snap_to_grid(lat, lon) do + step = Grid.step() + + { + Float.round(Float.round(lat / step) * step, 3), + Float.round(Float.round(lon / step) * step, 3) + } + end + + defp distance_km(lat1, lon1, lat2, lon2) do + lat1_rad = lat1 * :math.pi() / 180 + lat2_rad = lat2 * :math.pi() / 180 + dlat = (lat2 - lat1) * :math.pi() / 180 + dlon = (lon2 - lon1) * :math.pi() / 180 + + a = + :math.sin(dlat / 2) * :math.sin(dlat / 2) + + :math.cos(lat1_rad) * :math.cos(lat2_rad) * + :math.sin(dlon / 2) * :math.sin(dlon / 2) + + c = 2 * :math.atan2(:math.sqrt(a), :math.sqrt(1 - a)) + @earth_radius_km * c + end +end diff --git a/lib/microwaveprop/weather/nexrad_client.ex b/lib/microwaveprop/weather/nexrad_client.ex index 9235727f..5aef4cc7 100644 --- a/lib/microwaveprop/weather/nexrad_client.ex +++ b/lib/microwaveprop/weather/nexrad_client.ex @@ -11,6 +11,8 @@ defmodule Microwaveprop.Weather.NexradClient do box (~50x50 pixels) and compute summary statistics. """ + alias Microwaveprop.Weather.NexradCache + require Logger # Image geometry @@ -58,8 +60,6 @@ defmodule Microwaveprop.Weather.NexradClient do "#{@base_url}/#{date_path}/GIS/uscomp/n0q_#{file_ts}.png" end - alias Microwaveprop.Weather.NexradCache - @doc """ Fetch the latest n0q frame and extract all rain cells with reflectivity above `min_dbz` within `radius_km` of the given point. Returns diff --git a/lib/microwaveprop/workers/asos_adjustment_worker.ex b/lib/microwaveprop/workers/asos_adjustment_worker.ex index aa9b2244..61ffb47a 100644 --- a/lib/microwaveprop/workers/asos_adjustment_worker.ex +++ b/lib/microwaveprop/workers/asos_adjustment_worker.ex @@ -1,22 +1,38 @@ defmodule Microwaveprop.Workers.AsosAdjustmentWorker do @moduledoc """ - Runs every 10 minutes to fetch the latest ASOS observations and re-score - grid points near reporting stations. This provides fresher data between - the hourly HRRR grid computations. + Nudge the propagation score grid with fresh ASOS observations between + hourly HRRR runs. - For each grid point, finds the nearest ASOS station within 75 km. - If the station reported within the last 30 minutes, re-scores using - the ASOS observation for temp/dewpoint/wind/sky/pressure/precip and - the HRRR refractivity gradient from the last grid computation. + On each cron tick (every 10 minutes) this worker: + + 1. Finds the most recent `valid_time` that actually has HRRR profiles + persisted. + 2. Pulls current ASOS observations from all state networks via + `Microwaveprop.Weather.IemClient.fetch_current_asos/1`, keeps only + those within 30 minutes of now. + 3. Loads every `hrrr_profiles` row on the 0.125° propagation grid for + that valid_time into memory as plain maps. + 4. Hands both lists to `Microwaveprop.Propagation.AsosNudge.compute/3` + — pure function, IDW bias field, upper-air fields preserved. + 5. Upserts the nudged scores onto `(lat, lon, valid_time, band_mhz)`, + overwriting the HRRR-only values for grid cells within 250 km of a + reporting station. Cells outside that radius are left alone. + 6. Warms the `ScoreCache` and broadcasts `propagation:updated` so every + connected `/map` client refreshes. + + This worker owns the I/O. `AsosNudge` stays pure so its tests don't need + Repo or network access. """ use Oban.Worker, queue: :propagation, max_attempts: 3 + import Ecto.Query, only: [from: 2] + alias Microwaveprop.Propagation + alias Microwaveprop.Propagation.AsosNudge alias Microwaveprop.Propagation.BandConfig - alias Microwaveprop.Propagation.Grid - alias Microwaveprop.Propagation.Scorer - alias Microwaveprop.Radio + alias Microwaveprop.Repo + alias Microwaveprop.Weather.HrrrProfile alias Microwaveprop.Weather.IemClient require Logger @@ -31,172 +47,104 @@ defmodule Microwaveprop.Workers.AsosAdjustmentWorker do WI_ASOS WY_ASOS DC_ASOS ) - # Only adjust grid points within this distance of an ASOS station - @max_station_distance_km 75 - # Only use observations newer than this @max_obs_age_seconds 1800 @impl Oban.Worker def perform(%Oban.Job{}) do - now = DateTime.utc_now() - cutoff = DateTime.add(now, -@max_obs_age_seconds, :second) - - Logger.info("AsosAdjustment: fetching current ASOS observations") - - with {:ok, observations} <- IemClient.fetch_current_asos(@state_networks) do - # Filter to recent observations - fresh = - Enum.filter(observations, fn obs -> - obs.utc_valid && DateTime.after?(obs.utc_valid, cutoff) - end) - - Logger.info("AsosAdjustment: #{length(fresh)} fresh observations from #{length(observations)} total") - - if fresh == [] do + with {:ok, valid_time} <- latest_hrrr_valid_time(), + {:ok, observations} <- IemClient.fetch_current_asos(@state_networks), + [_ | _] = fresh <- filter_fresh(observations), + [_ | _] = profiles <- load_hrrr_profiles(valid_time) do + apply_nudge(fresh, profiles, valid_time) + else + :no_hrrr -> + Logger.info("AsosAdjustment: no HRRR profiles yet, skipping") :ok - else - valid_time = DateTime.truncate(now, :second) - apply_asos_scores(fresh, valid_time) + + [] -> + Logger.info("AsosAdjustment: no fresh ASOS data, skipping") + :ok + + {:error, reason} -> + Logger.warning("AsosAdjustment: aborted — #{inspect(reason)}") + :ok + end + end + + defp apply_nudge(observations, profiles, valid_time) do + scores = AsosNudge.compute(observations, valid_time, profiles) + + if scores == [] do + Logger.info("AsosAdjustment: #{length(observations)} fresh obs but no grid cells within range") + + :ok + else + case Propagation.upsert_scores(scores, prune: false) do + {:ok, count} -> + Logger.info("AsosAdjustment: nudged #{count} scores from #{length(observations)} obs at #{valid_time}") + + warm_and_broadcast(valid_time) + :ok + + error -> + Logger.error("AsosAdjustment: upsert failed — #{inspect(error)}") + error end end end - defp apply_asos_scores(fresh, valid_time) do - scores = score_grid_from_asos(fresh, valid_time) + defp warm_and_broadcast(valid_time) do + Enum.each(BandConfig.all_bands(), fn band -> + Propagation.warm_cache_and_broadcast(band.freq_mhz, valid_time) + end) - Logger.info("AsosAdjustment: computed #{length(scores)} adjusted scores") - - case Propagation.upsert_scores(scores) do - {:ok, count} -> - Logger.info("AsosAdjustment: upserted #{count} scores") - - Phoenix.PubSub.broadcast( - Microwaveprop.PubSub, - "propagation:updated", - {:propagation_updated, valid_time} - ) - - :ok - - error -> - Logger.error("AsosAdjustment: upsert failed: #{inspect(error)}") - error - end - end - - defp score_grid_from_asos(observations, valid_time) do - # Build a spatial index of observations - obs_list = - Enum.map(observations, fn obs -> - %{ - lat: obs.lat, - lon: obs.lon, - temp_f: obs.temp_f, - dewpoint_f: obs.dewpoint_f, - wind_speed_kts: obs.wind_speed_kts, - sky_condition: obs.sky_condition, - sea_level_pressure_mb: pressure_from_obs(obs), - precip_1h_in: obs.precip_1h_in - } - end) - - # For each grid point, find nearest station and score - Grid.conus_points() - |> Task.async_stream( - fn {lat, lon} = point -> - nearest = find_nearest_station(obs_list, lat, lon) - - if nearest do - score_point_from_obs(point, nearest, valid_time) - end - end, - max_concurrency: System.schedulers_online() * 2, - timeout: 10_000 + Phoenix.PubSub.broadcast( + Microwaveprop.PubSub, + "propagation:updated", + {:propagation_updated, valid_time} ) - |> Enum.flat_map(fn - {:ok, nil} -> [] - {:ok, scores} -> scores - {:exit, _} -> [] - end) end - defp find_nearest_station(obs_list, lat, lon) do - obs_list - |> Enum.map(fn obs -> - dist = Radio.haversine_km(lat, lon, obs.lat, obs.lon) - {dist, obs} - end) - |> Enum.filter(fn {dist, _} -> dist <= @max_station_distance_km end) - |> Enum.min_by(fn {dist, _} -> dist end, fn -> nil end) - |> case do - nil -> nil - {_dist, obs} -> obs + defp latest_hrrr_valid_time do + case Repo.one( + from(h in HrrrProfile, + where: h.is_grid_point == true, + select: max(h.valid_time) + ) + ) do + nil -> :no_hrrr + vt -> {:ok, vt} end end - defp score_point_from_obs({lat, lon}, obs, valid_time) do - temp_c = Scorer.f_to_c(obs.temp_f) - dewpoint_c = Scorer.f_to_c(obs.dewpoint_f) + defp filter_fresh(observations) do + cutoff = DateTime.add(DateTime.utc_now(), -@max_obs_age_seconds, :second) - if is_nil(temp_c) or is_nil(dewpoint_c) do - [] - else - sky_pct = sky_condition_to_pct(obs.sky_condition) - rain_rate = precip_to_rate(obs.precip_1h_in) + Enum.filter(observations, fn obs -> + obs.utc_valid && DateTime.after?(obs.utc_valid, cutoff) + end) + end - conditions = %{ - abs_humidity: Scorer.absolute_humidity(temp_c, dewpoint_c), - temp_f: obs.temp_f, - dewpoint_f: obs.dewpoint_f, - wind_speed_kts: obs.wind_speed_kts, - sky_cover_pct: sky_pct, - utc_hour: valid_time.hour, - utc_minute: valid_time.minute, - month: valid_time.month, - longitude: lon, - pressure_mb: obs.sea_level_pressure_mb, - prev_pressure_mb: nil, - rain_rate_mmhr: rain_rate, - min_refractivity_gradient: nil, - bl_depth_m: nil - } - - Enum.map(BandConfig.all_bands(), fn band -> - result = Scorer.composite_score(conditions, band) - - %{ - lat: lat, - lon: lon, - valid_time: valid_time, - band_mhz: band.freq_mhz, - score: result.score, - factors: result.factors + defp load_hrrr_profiles(valid_time) do + Repo.all( + from(h in HrrrProfile, + where: h.valid_time == ^valid_time and h.is_grid_point == true, + select: %{ + lat: h.lat, + lon: h.lon, + valid_time: h.valid_time, + surface_temp_c: h.surface_temp_c, + surface_dewpoint_c: h.surface_dewpoint_c, + surface_pressure_mb: h.surface_pressure_mb, + surface_refractivity: h.surface_refractivity, + min_refractivity_gradient: h.min_refractivity_gradient, + hpbl_m: h.hpbl_m, + pwat_mm: h.pwat_mm, + ducting_detected: h.ducting_detected, + duct_characteristics: h.duct_characteristics, + profile: h.profile } - end) - end + ) + ) end - - defp sky_condition_to_pct(nil), do: nil - defp sky_condition_to_pct("CLR"), do: 0.0 - defp sky_condition_to_pct("SKC"), do: 0.0 - defp sky_condition_to_pct("FEW"), do: 18.0 - defp sky_condition_to_pct("SCT"), do: 40.0 - defp sky_condition_to_pct("BKN"), do: 75.0 - defp sky_condition_to_pct("OVC"), do: 100.0 - defp sky_condition_to_pct("VV"), do: 100.0 - defp sky_condition_to_pct(_), do: nil - - defp pressure_from_obs(%{sea_level_pressure_mb: mslp}) when is_number(mslp), do: mslp - - defp pressure_from_obs(%{altimeter_setting: alti}) when is_number(alti) do - # Convert altimeter setting (inHg) to millibars - alti * 33.8639 - end - - defp pressure_from_obs(_), do: nil - - # Convert hourly precip accumulation (inches) to rain rate (mm/hr) - defp precip_to_rate(nil), do: 0.0 - defp precip_to_rate(inches) when inches <= 0, do: 0.0 - defp precip_to_rate(inches), do: inches * 25.4 end diff --git a/lib/microwaveprop_web/live/contact_live/show.ex b/lib/microwaveprop_web/live/contact_live/show.ex index d1ae2c36..17354ccb 100644 --- a/lib/microwaveprop_web/live/contact_live/show.ex +++ b/lib/microwaveprop_web/live/contact_live/show.ex @@ -302,8 +302,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do {:noreply, assign(socket, :iemre, iemre)} end - def handle_async(slot, {:exit, reason}, socket) - when slot in [:weather, :solar, :hrrr_path, :terrain, :iemre] do + def handle_async(slot, {:exit, reason}, socket) when slot in [:weather, :solar, :hrrr_path, :terrain, :iemre] do Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}") {:noreply, socket} end diff --git a/lib/microwaveprop_web/live/contact_map_live.ex b/lib/microwaveprop_web/live/contact_map_live.ex index 6250d8ff..18e716b1 100644 --- a/lib/microwaveprop_web/live/contact_map_live.ex +++ b/lib/microwaveprop_web/live/contact_map_live.ex @@ -85,7 +85,6 @@ defmodule MicrowavepropWeb.ContactMapLive do defp band_color(mhz), do: Map.get(@band_colors, mhz, "#64748b") - @impl true def render(assigns) do ~H""" diff --git a/lib/microwaveprop_web/live/submit_live.ex b/lib/microwaveprop_web/live/submit_live.ex index 7cc1b03a..05fb0de7 100644 --- a/lib/microwaveprop_web/live/submit_live.ex +++ b/lib/microwaveprop_web/live/submit_live.ex @@ -522,8 +522,7 @@ defmodule MicrowavepropWeb.SubmitLive do

Mode handling

ADIF modes are normalized to the six modes we track. SUBMODE takes - precedence when present (e.g. MODE=MFSK, SUBMODE=FT8 - imports as FT8). + precedence when present (e.g. MODE=MFSK, SUBMODE=FT8 imports as FT8).

diff --git a/test/microwaveprop/propagation/asos_nudge_test.exs b/test/microwaveprop/propagation/asos_nudge_test.exs new file mode 100644 index 00000000..26be8a28 --- /dev/null +++ b/test/microwaveprop/propagation/asos_nudge_test.exs @@ -0,0 +1,186 @@ +defmodule Microwaveprop.Propagation.AsosNudgeTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Propagation.AsosNudge + + @valid_time ~U[2026-04-12 18:00:00Z] + + defp profile(lat, lon, overrides \\ []) do + Map.merge( + %{ + valid_time: @valid_time, + lat: lat, + lon: lon, + surface_temp_c: 20.0, + surface_dewpoint_c: 10.0, + surface_pressure_mb: 1013.0, + hpbl_m: 1000.0, + pwat_mm: 25.0, + min_refractivity_gradient: -40.0, + surface_refractivity: 320.0, + profile: [] + }, + Map.new(overrides) + ) + end + + defp station(lat, lon, overrides \\ []) do + Map.merge( + %{ + lat: lat, + lon: lon, + temp_f: 68.0, + dewpoint_f: 50.0, + sea_level_pressure_mb: 1013.0, + precip_1h_in: 0.0 + }, + Map.new(overrides) + ) + end + + describe "build_station_residuals/2" do + test "station co-located with a HRRR grid cell yields residuals against that cell" do + station_obs = station(32.0, -97.0, temp_f: 77.0, dewpoint_f: 59.0, sea_level_pressure_mb: 1020.0) + hrrr = profile(32.0, -97.0, surface_temp_c: 20.0, surface_dewpoint_c: 10.0, surface_pressure_mb: 1013.0) + + [residual] = AsosNudge.build_station_residuals([station_obs], [hrrr]) + + assert_in_delta residual.dtemp_c, 5.0, 0.1 + assert_in_delta residual.ddewpoint_c, 5.0, 0.1 + assert_in_delta residual.dpressure_mb, 7.0, 0.1 + assert residual.lat == 32.0 + assert residual.lon == -97.0 + end + + test "station with no HRRR cell within 1 grid step is dropped" do + obs = station(32.0, -97.0, temp_f: 77.0) + far_hrrr = profile(40.0, -85.0, surface_temp_c: 20.0) + + assert AsosNudge.build_station_residuals([obs], [far_hrrr]) == [] + end + + test "station with nil temp is dropped" do + obs = station(32.0, -97.0, temp_f: nil) + hrrr = profile(32.0, -97.0) + + assert AsosNudge.build_station_residuals([obs], [hrrr]) == [] + end + + test "station with nil pressure still contributes temp and dewpoint deltas" do + obs = station(32.0, -97.0, temp_f: 77.0, dewpoint_f: 59.0, sea_level_pressure_mb: nil) + hrrr = profile(32.0, -97.0, surface_temp_c: 20.0, surface_dewpoint_c: 10.0, surface_pressure_mb: 1013.0) + + [residual] = AsosNudge.build_station_residuals([obs], [hrrr]) + + assert_in_delta residual.dtemp_c, 5.0, 0.1 + assert_in_delta residual.ddewpoint_c, 5.0, 0.1 + assert residual.dpressure_mb == nil + end + end + + describe "nudge_profile/2" do + test "single station at the same grid cell applies (almost) the full residual" do + residuals = [ + %{lat: 32.0, lon: -97.0, dtemp_c: 5.0, ddewpoint_c: 3.0, dpressure_mb: 2.0, asos_rain_mmhr: 0.0} + ] + + hrrr = profile(32.0, -97.0, surface_temp_c: 20.0, surface_dewpoint_c: 10.0, surface_pressure_mb: 1013.0) + nudged = AsosNudge.nudge_profile(hrrr, residuals) + + assert_in_delta nudged.surface_temp_c, 25.0, 0.1 + assert_in_delta nudged.surface_dewpoint_c, 13.0, 0.1 + assert_in_delta nudged.surface_pressure_mb, 1015.0, 0.1 + end + + test "station farther than 250 km contributes nothing" do + residuals = [ + %{lat: 40.0, lon: -97.0, dtemp_c: 10.0, ddewpoint_c: 10.0, dpressure_mb: 5.0, asos_rain_mmhr: 0.0} + ] + + hrrr = profile(32.0, -97.0, surface_temp_c: 20.0) + nudged = AsosNudge.nudge_profile(hrrr, residuals) + + assert nudged.surface_temp_c == 20.0 + assert nudged.surface_dewpoint_c == 10.0 + assert nudged.surface_pressure_mb == 1013.0 + end + + test "two equidistant stations with opposite residuals average to near zero" do + residuals = [ + %{lat: 32.0, lon: -95.93, dtemp_c: 5.0, ddewpoint_c: 0.0, dpressure_mb: 0.0, asos_rain_mmhr: 0.0}, + %{lat: 32.0, lon: -98.07, dtemp_c: -5.0, ddewpoint_c: 0.0, dpressure_mb: 0.0, asos_rain_mmhr: 0.0} + ] + + hrrr = profile(32.0, -97.0, surface_temp_c: 20.0) + nudged = AsosNudge.nudge_profile(hrrr, residuals) + + assert_in_delta nudged.surface_temp_c, 20.0, 0.1 + end + + test "nil residual components (missing pressure) leave the HRRR field unchanged" do + residuals = [ + %{lat: 32.0, lon: -97.0, dtemp_c: 5.0, ddewpoint_c: 3.0, dpressure_mb: nil, asos_rain_mmhr: 0.0} + ] + + hrrr = profile(32.0, -97.0, surface_temp_c: 20.0, surface_dewpoint_c: 10.0, surface_pressure_mb: 1013.0) + nudged = AsosNudge.nudge_profile(hrrr, residuals) + + assert_in_delta nudged.surface_temp_c, 25.0, 0.1 + assert_in_delta nudged.surface_dewpoint_c, 13.0, 0.1 + assert nudged.surface_pressure_mb == 1013.0 + end + + test "upper-air fields pass through unchanged regardless of residuals" do + residuals = [ + %{lat: 32.0, lon: -97.0, dtemp_c: 10.0, ddewpoint_c: 10.0, dpressure_mb: 20.0, asos_rain_mmhr: 0.0} + ] + + hrrr = profile(32.0, -97.0, min_refractivity_gradient: -250.0, pwat_mm: 35.0, hpbl_m: 500.0) + nudged = AsosNudge.nudge_profile(hrrr, residuals) + + assert nudged.min_refractivity_gradient == -250.0 + assert nudged.pwat_mm == 35.0 + assert nudged.hpbl_m == 500.0 + end + end + + describe "compute/3" do + test "returns empty list when observations list is empty" do + assert AsosNudge.compute([], @valid_time, [profile(32.0, -97.0)]) == [] + end + + test "returns empty list when no HRRR profiles provided" do + assert AsosNudge.compute([station(32.0, -97.0)], @valid_time, []) == [] + end + + test "returns grid score maps for nudged profiles with band_mhz, score, factors" do + hrrr = profile(32.0, -97.0, surface_temp_c: 20.0) + obs = station(32.0, -97.0, temp_f: 77.0) + + results = AsosNudge.compute([obs], @valid_time, [hrrr]) + + assert length(results) > 0 + + for result <- results do + assert result.lat == 32.0 + assert result.lon == -97.0 + assert result.valid_time == @valid_time + assert is_integer(result.band_mhz) + assert is_number(result.score) + assert is_map(result.factors) + end + end + + test "drops grid points with no station within 250 km from the output" do + nudged_cell = profile(32.0, -97.0, surface_temp_c: 20.0) + far_cell = profile(40.0, -85.0, surface_temp_c: 20.0) + obs = station(32.0, -97.0, temp_f: 77.0) + + results = AsosNudge.compute([obs], @valid_time, [nudged_cell, far_cell]) + + lat_lons = results |> Enum.map(&{&1.lat, &1.lon}) |> Enum.uniq() + assert {32.0, -97.0} in lat_lons + refute {40.0, -85.0} in lat_lons + end + end +end diff --git a/test/microwaveprop/propagation/score_cache_test.exs b/test/microwaveprop/propagation/score_cache_test.exs index abb86d88..93c527af 100644 --- a/test/microwaveprop/propagation/score_cache_test.exs +++ b/test/microwaveprop/propagation/score_cache_test.exs @@ -16,6 +16,7 @@ defmodule Microwaveprop.Propagation.ScoreCacheTest do test "returns cached scores after put" do scores = [%{lat: 32.0, lon: -97.0, score: 75}] ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], scores) + assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} = ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z]) end diff --git a/test/microwaveprop/propagation_test.exs b/test/microwaveprop/propagation_test.exs index 411d3c71..1875ccac 100644 --- a/test/microwaveprop/propagation_test.exs +++ b/test/microwaveprop/propagation_test.exs @@ -143,8 +143,8 @@ defmodule Microwaveprop.PropagationTest do end test "filters past valid_times out" do - past = DateTime.add(DateTime.utc_now(), -3600, :second) |> DateTime.truncate(:second) - future = DateTime.add(DateTime.utc_now(), 3600, :second) |> DateTime.truncate(:second) + past = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second) + future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.truncate(:second) ScoreCache.put(10_000, past, [%{lat: 32.875, lon: -97.0, score: 60}]) ScoreCache.put(10_000, future, [%{lat: 32.875, lon: -97.0, score: 80}])