Nudge HRRR scores with live ASOS obs between runs
Adds Microwaveprop.Propagation.AsosNudge: a pure IDW bias-field module that takes ASOS observations + HRRR profiles and returns re-scored grid rows for every cell within 250km of a reporting station. Upper-air fields (min_refractivity_gradient, pwat_mm, hpbl_m, profile, duct metadata) pass through unchanged so HRRR's signal isn't clobbered. The old AsosAdjustmentWorker was unwired and buggy — nil'd out ~22% of the scoring weight and wrote orphan timestamps. Replaced with a slim worker that queries the latest HRRR valid_time, fetches live ASOS currents, calls AsosNudge.compute/3, and upserts onto (lat, lon, valid_time, band_mhz) so nudged values overwrite the HRRR hour cleanly instead of polluting available_valid_times. After each upsert it warms ScoreCache and broadcasts propagation:updated so live /map clients refresh. Cron hooked up every 10 minutes in config.exs and dev.exs. Also cleaned up the stale "dev has propagation disabled" note in CLAUDE.md. 13 new AsosNudge unit tests cover: residual computation (co-located, out-of-grid, nil fields), IDW weighting (single station, far station, two equidistant stations, nil component handling), upper-air preservation, and the compute/3 entry point's shape and radius filter. Drive-by Styler formatting touched a handful of unrelated files from `mix format`.
This commit is contained in:
parent
8225242697
commit
c318c3a932
15 changed files with 549 additions and 186 deletions
|
|
@ -106,7 +106,7 @@ mix assets.deploy # minified + digest
|
|||
|
||||
**Production** runs: propagation, commercial, solar, weather, hrrr, terrain, iemre queues. No cron backfill — enrichment is triggered by QSO submission only.
|
||||
|
||||
**Dev** currently has propagation disabled for backfill. Re-enable by removing the Oban override in `config/dev.exs`.
|
||||
**Dev** runs the same hourly `PropagationGridWorker` + `AsosAdjustmentWorker` (10-minute nudge) cron as production.
|
||||
|
||||
### Scoring Algorithm
|
||||
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ config :microwaveprop, Oban,
|
|||
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker},
|
||||
{"*/5 * * * *", Microwaveprop.Commercial.PollWorker},
|
||||
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
|
||||
{"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker}
|
||||
]}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -88,6 +88,7 @@ config :microwaveprop, Oban,
|
|||
{Oban.Plugins.Cron,
|
||||
crontab: [
|
||||
{"5 * * * *", Microwaveprop.Workers.PropagationGridWorker},
|
||||
{"*/10 * * * *", Microwaveprop.Workers.AsosAdjustmentWorker},
|
||||
{"*/15 * * * *", Microwaveprop.Workers.PropagationPruneWorker},
|
||||
{"0 8 * * *", Microwaveprop.Workers.SolarIndexWorker}
|
||||
]}
|
||||
|
|
|
|||
|
|
@ -46,20 +46,6 @@ if config_env() == :prod do
|
|||
|
||||
host = System.get_env("PHX_HOST") || "example.com"
|
||||
|
||||
config :libcluster,
|
||||
topologies: [
|
||||
k8s: [
|
||||
strategy: Cluster.Strategy.Kubernetes,
|
||||
config: [
|
||||
mode: :ip,
|
||||
kubernetes_ip_lookup_mode: :pods,
|
||||
kubernetes_node_basename: "microwaveprop",
|
||||
kubernetes_selector: "app=prop",
|
||||
kubernetes_namespace: "prop"
|
||||
]
|
||||
]
|
||||
]
|
||||
|
||||
# ## SSL Support
|
||||
#
|
||||
# To get SSL working, you will need to add the `https` key
|
||||
|
|
@ -105,6 +91,20 @@ if config_env() == :prod do
|
|||
# handshake fails with an "unexpected_message" TLS alert.
|
||||
email_server = System.get_env("EMAIL_SERVER")
|
||||
|
||||
config :libcluster,
|
||||
topologies: [
|
||||
k8s: [
|
||||
strategy: Cluster.Strategy.Kubernetes,
|
||||
config: [
|
||||
mode: :ip,
|
||||
kubernetes_ip_lookup_mode: :pods,
|
||||
kubernetes_node_basename: "microwaveprop",
|
||||
kubernetes_selector: "app=prop",
|
||||
kubernetes_namespace: "prop"
|
||||
]
|
||||
]
|
||||
]
|
||||
|
||||
config :microwaveprop, Microwaveprop.Mailer,
|
||||
adapter: Swoosh.Adapters.SMTP,
|
||||
relay: email_server,
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ config :microwaveprop, MicrowavepropWeb.Endpoint,
|
|||
|
||||
# Run Oban jobs inline during tests
|
||||
config :microwaveprop, Oban, testing: :inline
|
||||
config :microwaveprop, cache_contact_count: false
|
||||
config :microwaveprop, cache_contact_map: false
|
||||
config :microwaveprop, elevation_req_options: [plug: {Req.Test, Microwaveprop.Terrain.ElevationClient}, retry: false]
|
||||
config :microwaveprop, hrrr_req_options: [plug: {Req.Test, Microwaveprop.Weather.HrrrClient}, retry: false]
|
||||
config :microwaveprop, iem_req_options: [plug: {Req.Test, Microwaveprop.Weather.IemClient}, retry: false]
|
||||
|
|
@ -42,8 +44,6 @@ config :microwaveprop, nexrad_req_options: [plug: {Req.Test, Microwaveprop.Weath
|
|||
config :microwaveprop, solar_req_options: [plug: {Req.Test, Microwaveprop.Weather.SolarClient}]
|
||||
config :microwaveprop, srtm_req_options: [plug: {Req.Test, Microwaveprop.Terrain.Srtm}, retry: false]
|
||||
config :microwaveprop, start_freshness_monitor: false
|
||||
config :microwaveprop, cache_contact_count: false
|
||||
config :microwaveprop, cache_contact_map: false
|
||||
|
||||
# Initialize plugs at runtime for faster test compilation
|
||||
config :phoenix, :plug_init_mode, :runtime
|
||||
|
|
|
|||
|
|
@ -350,8 +350,7 @@ defmodule Microwaveprop.Propagation do
|
|||
|
||||
defp snap_to_grid(lat, lon) do
|
||||
step = Grid.step()
|
||||
{Float.round(Float.round(lat / step) * step, 3),
|
||||
Float.round(Float.round(lon / step) * step, 3)}
|
||||
{Float.round(Float.round(lat / step) * step, 3), Float.round(Float.round(lon / step) * step, 3)}
|
||||
end
|
||||
|
||||
@doc "Get the full score and factors for a specific grid point, snapped to nearest grid."
|
||||
|
|
|
|||
230
lib/microwaveprop/propagation/asos_nudge.ex
Normal file
230
lib/microwaveprop/propagation/asos_nudge.ex
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
defmodule Microwaveprop.Propagation.AsosNudge do
|
||||
@moduledoc """
|
||||
Blend live ASOS surface observations into the latest HRRR grid between
|
||||
hourly runs. HRRR analysis is already ~55 minutes old by the time the
|
||||
propagation worker scores it; fronts, humidity swings, and rain onset
|
||||
happening after that lag otherwise sit invisible until the next top-of-hour
|
||||
run. ASOS METARs publish every 5 minutes, so a 10-minute nudging cycle
|
||||
catches them.
|
||||
|
||||
## Pipeline
|
||||
|
||||
1. `build_station_residuals/2` — for each ASOS observation, find the
|
||||
nearest HRRR grid cell (within one 0.125° step, otherwise the station
|
||||
is outside the grid domain and dropped). The residual is
|
||||
`(asos_value - hrrr_value)` for temp, dewpoint, and pressure, plus the
|
||||
raw ASOS rain rate.
|
||||
|
||||
2. `nudge_profile/2` — for a given HRRR profile, compute an
|
||||
inverse-distance weighted (1/d²) blend of residuals from stations
|
||||
within 250 km. Only the surface fields are touched; refractivity
|
||||
gradient, PWAT, HPBL, duct metadata, and the full vertical profile
|
||||
all pass through unchanged so the HRRR upper-air signal is preserved.
|
||||
|
||||
3. `compute/3` — run the pipeline over every HRRR profile that has at
|
||||
least one station within range, call
|
||||
`Microwaveprop.Propagation.score_grid_point/4` on the patched profile,
|
||||
and return score rows ready for `Propagation.upsert_scores/1`. Grid
|
||||
cells with no station in range are dropped from the output, leaving
|
||||
the existing HRRR scores untouched.
|
||||
|
||||
The nudging module itself is pure — no Repo, no PubSub. The worker that
|
||||
schedules this (`Microwaveprop.Workers.AsosAdjustmentWorker`) handles all
|
||||
I/O.
|
||||
"""
|
||||
|
||||
alias Microwaveprop.Propagation
|
||||
alias Microwaveprop.Propagation.Grid
|
||||
|
||||
@radius_km 250
|
||||
@min_station_weight_km 1.0
|
||||
@earth_radius_km 6371.0
|
||||
|
||||
@type observation :: %{
|
||||
required(:lat) => float(),
|
||||
required(:lon) => float(),
|
||||
optional(:temp_f) => number() | nil,
|
||||
optional(:dewpoint_f) => number() | nil,
|
||||
optional(:sea_level_pressure_mb) => number() | nil,
|
||||
optional(:precip_1h_in) => number() | nil
|
||||
}
|
||||
|
||||
@type residual :: %{
|
||||
lat: float(),
|
||||
lon: float(),
|
||||
dtemp_c: float() | nil,
|
||||
ddewpoint_c: float() | nil,
|
||||
dpressure_mb: float() | nil,
|
||||
asos_rain_mmhr: float()
|
||||
}
|
||||
|
||||
@type hrrr_profile :: map()
|
||||
|
||||
@type grid_score :: %{
|
||||
lat: float(),
|
||||
lon: float(),
|
||||
valid_time: DateTime.t(),
|
||||
band_mhz: non_neg_integer(),
|
||||
score: non_neg_integer(),
|
||||
factors: map()
|
||||
}
|
||||
|
||||
@doc """
|
||||
Nudge every HRRR grid cell that has an ASOS station within 250 km and
|
||||
re-score it. Cells outside that radius are dropped so the existing HRRR
|
||||
scores stay in place.
|
||||
"""
|
||||
@spec compute([observation()], DateTime.t(), [hrrr_profile()]) :: [grid_score()]
|
||||
def compute([], _valid_time, _profiles), do: []
|
||||
def compute(_observations, _valid_time, []), do: []
|
||||
|
||||
def compute(observations, valid_time, profiles) do
|
||||
residuals = build_station_residuals(observations, profiles)
|
||||
|
||||
profiles
|
||||
|> Enum.filter(&any_station_within_radius?(&1, residuals))
|
||||
|> Enum.flat_map(fn profile ->
|
||||
profile
|
||||
|> nudge_profile(residuals)
|
||||
|> score_point(valid_time)
|
||||
end)
|
||||
end
|
||||
|
||||
@doc """
|
||||
For each observation with a valid HRRR anchor cell, compute the
|
||||
`(asos - hrrr)` residual at that cell. Used by `nudge_profile/2` to
|
||||
spread the bias field across the grid.
|
||||
"""
|
||||
@spec build_station_residuals([observation()], [hrrr_profile()]) :: [residual()]
|
||||
def build_station_residuals(observations, profiles) do
|
||||
lookup = Map.new(profiles, fn p -> {snap_to_grid(p.lat, p.lon), p} end)
|
||||
|
||||
observations
|
||||
|> Enum.map(&station_residual(&1, lookup))
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Return a patched HRRR profile with `surface_temp_c`, `surface_dewpoint_c`,
|
||||
and `surface_pressure_mb` shifted by the IDW-weighted residuals from
|
||||
stations within 250 km. If no station is close enough, the profile is
|
||||
returned unchanged.
|
||||
"""
|
||||
@spec nudge_profile(hrrr_profile(), [residual()]) :: hrrr_profile()
|
||||
def nudge_profile(profile, residuals) do
|
||||
nearby =
|
||||
residuals
|
||||
|> Enum.map(fn r -> {r, distance_km(profile.lat, profile.lon, r.lat, r.lon)} end)
|
||||
|> Enum.filter(fn {_r, d} -> d <= @radius_km end)
|
||||
|
||||
case nearby do
|
||||
[] ->
|
||||
profile
|
||||
|
||||
_ ->
|
||||
profile
|
||||
|> Map.put(:surface_temp_c, apply_idw(profile.surface_temp_c, nearby, :dtemp_c))
|
||||
|> Map.put(:surface_dewpoint_c, apply_idw(profile.surface_dewpoint_c, nearby, :ddewpoint_c))
|
||||
|> Map.put(:surface_pressure_mb, apply_idw(profile.surface_pressure_mb, nearby, :dpressure_mb))
|
||||
end
|
||||
end
|
||||
|
||||
# -- internals ---------------------------------------------------------
|
||||
|
||||
defp station_residual(obs, lookup) do
|
||||
with {:ok, temp_c} <- f_to_c(obs[:temp_f]),
|
||||
{:ok, dewpoint_c} <- f_to_c(obs[:dewpoint_f]),
|
||||
snapped = snap_to_grid(obs.lat, obs.lon),
|
||||
hrrr when not is_nil(hrrr) <- Map.get(lookup, snapped),
|
||||
true <- within_one_grid_step?(obs, hrrr) do
|
||||
%{
|
||||
lat: obs.lat,
|
||||
lon: obs.lon,
|
||||
dtemp_c: delta(temp_c, hrrr.surface_temp_c),
|
||||
ddewpoint_c: delta(dewpoint_c, hrrr.surface_dewpoint_c),
|
||||
dpressure_mb: delta(obs[:sea_level_pressure_mb], hrrr.surface_pressure_mb),
|
||||
asos_rain_mmhr: precip_to_mmhr(obs[:precip_1h_in])
|
||||
}
|
||||
else
|
||||
_ -> nil
|
||||
end
|
||||
end
|
||||
|
||||
defp f_to_c(nil), do: :error
|
||||
defp f_to_c(f) when is_number(f), do: {:ok, (f - 32) * 5 / 9}
|
||||
defp f_to_c(_), do: :error
|
||||
|
||||
defp delta(nil, _), do: nil
|
||||
defp delta(_, nil), do: nil
|
||||
defp delta(a, b), do: a - b
|
||||
|
||||
defp precip_to_mmhr(nil), do: 0.0
|
||||
defp precip_to_mmhr(inches) when inches <= 0, do: 0.0
|
||||
defp precip_to_mmhr(inches), do: inches * 25.4
|
||||
|
||||
defp within_one_grid_step?(obs, hrrr) do
|
||||
distance_km(obs.lat, obs.lon, hrrr.lat, hrrr.lon) <= Grid.step() * 111.0
|
||||
end
|
||||
|
||||
defp any_station_within_radius?(profile, residuals) do
|
||||
Enum.any?(residuals, fn r ->
|
||||
distance_km(profile.lat, profile.lon, r.lat, r.lon) <= @radius_km
|
||||
end)
|
||||
end
|
||||
|
||||
defp apply_idw(base, _nearby, _key) when is_nil(base), do: nil
|
||||
|
||||
defp apply_idw(base, nearby, key) do
|
||||
contributing =
|
||||
Enum.filter(nearby, fn {residual, _d} -> not is_nil(Map.get(residual, key)) end)
|
||||
|
||||
case contributing do
|
||||
[] ->
|
||||
base
|
||||
|
||||
_ ->
|
||||
{num, den} =
|
||||
Enum.reduce(contributing, {0.0, 0.0}, fn {residual, d_km}, {num, den} ->
|
||||
w = 1.0 / :math.pow(max(d_km, @min_station_weight_km), 2)
|
||||
{num + w * Map.get(residual, key), den + w}
|
||||
end)
|
||||
|
||||
base + num / den
|
||||
end
|
||||
end
|
||||
|
||||
defp score_point(profile, valid_time) do
|
||||
profile
|
||||
|> Propagation.score_grid_point(valid_time, profile.lat, profile.lon)
|
||||
|> Enum.map(fn row ->
|
||||
row
|
||||
|> Map.put(:lat, profile.lat)
|
||||
|> Map.put(:lon, profile.lon)
|
||||
|> Map.put(:valid_time, valid_time)
|
||||
end)
|
||||
end
|
||||
|
||||
defp snap_to_grid(lat, lon) do
|
||||
step = Grid.step()
|
||||
|
||||
{
|
||||
Float.round(Float.round(lat / step) * step, 3),
|
||||
Float.round(Float.round(lon / step) * step, 3)
|
||||
}
|
||||
end
|
||||
|
||||
defp distance_km(lat1, lon1, lat2, lon2) do
|
||||
lat1_rad = lat1 * :math.pi() / 180
|
||||
lat2_rad = lat2 * :math.pi() / 180
|
||||
dlat = (lat2 - lat1) * :math.pi() / 180
|
||||
dlon = (lon2 - lon1) * :math.pi() / 180
|
||||
|
||||
a =
|
||||
:math.sin(dlat / 2) * :math.sin(dlat / 2) +
|
||||
:math.cos(lat1_rad) * :math.cos(lat2_rad) *
|
||||
:math.sin(dlon / 2) * :math.sin(dlon / 2)
|
||||
|
||||
c = 2 * :math.atan2(:math.sqrt(a), :math.sqrt(1 - a))
|
||||
@earth_radius_km * c
|
||||
end
|
||||
end
|
||||
|
|
@ -11,6 +11,8 @@ defmodule Microwaveprop.Weather.NexradClient do
|
|||
box (~50x50 pixels) and compute summary statistics.
|
||||
"""
|
||||
|
||||
alias Microwaveprop.Weather.NexradCache
|
||||
|
||||
require Logger
|
||||
|
||||
# Image geometry
|
||||
|
|
@ -58,8 +60,6 @@ defmodule Microwaveprop.Weather.NexradClient do
|
|||
"#{@base_url}/#{date_path}/GIS/uscomp/n0q_#{file_ts}.png"
|
||||
end
|
||||
|
||||
alias Microwaveprop.Weather.NexradCache
|
||||
|
||||
@doc """
|
||||
Fetch the latest n0q frame and extract all rain cells with reflectivity
|
||||
above `min_dbz` within `radius_km` of the given point. Returns
|
||||
|
|
|
|||
|
|
@ -1,22 +1,38 @@
|
|||
defmodule Microwaveprop.Workers.AsosAdjustmentWorker do
|
||||
@moduledoc """
|
||||
Runs every 10 minutes to fetch the latest ASOS observations and re-score
|
||||
grid points near reporting stations. This provides fresher data between
|
||||
the hourly HRRR grid computations.
|
||||
Nudge the propagation score grid with fresh ASOS observations between
|
||||
hourly HRRR runs.
|
||||
|
||||
For each grid point, finds the nearest ASOS station within 75 km.
|
||||
If the station reported within the last 30 minutes, re-scores using
|
||||
the ASOS observation for temp/dewpoint/wind/sky/pressure/precip and
|
||||
the HRRR refractivity gradient from the last grid computation.
|
||||
On each cron tick (every 10 minutes) this worker:
|
||||
|
||||
1. Finds the most recent `valid_time` that actually has HRRR profiles
|
||||
persisted.
|
||||
2. Pulls current ASOS observations from all state networks via
|
||||
`Microwaveprop.Weather.IemClient.fetch_current_asos/1`, keeps only
|
||||
those within 30 minutes of now.
|
||||
3. Loads every `hrrr_profiles` row on the 0.125° propagation grid for
|
||||
that valid_time into memory as plain maps.
|
||||
4. Hands both lists to `Microwaveprop.Propagation.AsosNudge.compute/3`
|
||||
— pure function, IDW bias field, upper-air fields preserved.
|
||||
5. Upserts the nudged scores onto `(lat, lon, valid_time, band_mhz)`,
|
||||
overwriting the HRRR-only values for grid cells within 250 km of a
|
||||
reporting station. Cells outside that radius are left alone.
|
||||
6. Warms the `ScoreCache` and broadcasts `propagation:updated` so every
|
||||
connected `/map` client refreshes.
|
||||
|
||||
This worker owns the I/O. `AsosNudge` stays pure so its tests don't need
|
||||
Repo or network access.
|
||||
"""
|
||||
|
||||
use Oban.Worker, queue: :propagation, max_attempts: 3
|
||||
|
||||
import Ecto.Query, only: [from: 2]
|
||||
|
||||
alias Microwaveprop.Propagation
|
||||
alias Microwaveprop.Propagation.AsosNudge
|
||||
alias Microwaveprop.Propagation.BandConfig
|
||||
alias Microwaveprop.Propagation.Grid
|
||||
alias Microwaveprop.Propagation.Scorer
|
||||
alias Microwaveprop.Radio
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.HrrrProfile
|
||||
alias Microwaveprop.Weather.IemClient
|
||||
|
||||
require Logger
|
||||
|
|
@ -31,172 +47,104 @@ defmodule Microwaveprop.Workers.AsosAdjustmentWorker do
|
|||
WI_ASOS WY_ASOS DC_ASOS
|
||||
)
|
||||
|
||||
# Only adjust grid points within this distance of an ASOS station
|
||||
@max_station_distance_km 75
|
||||
# Only use observations newer than this
|
||||
@max_obs_age_seconds 1800
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{}) do
|
||||
now = DateTime.utc_now()
|
||||
cutoff = DateTime.add(now, -@max_obs_age_seconds, :second)
|
||||
with {:ok, valid_time} <- latest_hrrr_valid_time(),
|
||||
{:ok, observations} <- IemClient.fetch_current_asos(@state_networks),
|
||||
[_ | _] = fresh <- filter_fresh(observations),
|
||||
[_ | _] = profiles <- load_hrrr_profiles(valid_time) do
|
||||
apply_nudge(fresh, profiles, valid_time)
|
||||
else
|
||||
:no_hrrr ->
|
||||
Logger.info("AsosAdjustment: no HRRR profiles yet, skipping")
|
||||
:ok
|
||||
|
||||
Logger.info("AsosAdjustment: fetching current ASOS observations")
|
||||
[] ->
|
||||
Logger.info("AsosAdjustment: no fresh ASOS data, skipping")
|
||||
:ok
|
||||
|
||||
with {:ok, observations} <- IemClient.fetch_current_asos(@state_networks) do
|
||||
# Filter to recent observations
|
||||
fresh =
|
||||
Enum.filter(observations, fn obs ->
|
||||
obs.utc_valid && DateTime.after?(obs.utc_valid, cutoff)
|
||||
end)
|
||||
{:error, reason} ->
|
||||
Logger.warning("AsosAdjustment: aborted — #{inspect(reason)}")
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
Logger.info("AsosAdjustment: #{length(fresh)} fresh observations from #{length(observations)} total")
|
||||
defp apply_nudge(observations, profiles, valid_time) do
|
||||
scores = AsosNudge.compute(observations, valid_time, profiles)
|
||||
|
||||
if scores == [] do
|
||||
Logger.info("AsosAdjustment: #{length(observations)} fresh obs but no grid cells within range")
|
||||
|
||||
if fresh == [] do
|
||||
:ok
|
||||
else
|
||||
valid_time = DateTime.truncate(now, :second)
|
||||
apply_asos_scores(fresh, valid_time)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp apply_asos_scores(fresh, valid_time) do
|
||||
scores = score_grid_from_asos(fresh, valid_time)
|
||||
|
||||
Logger.info("AsosAdjustment: computed #{length(scores)} adjusted scores")
|
||||
|
||||
case Propagation.upsert_scores(scores) do
|
||||
case Propagation.upsert_scores(scores, prune: false) do
|
||||
{:ok, count} ->
|
||||
Logger.info("AsosAdjustment: upserted #{count} scores")
|
||||
Logger.info("AsosAdjustment: nudged #{count} scores from #{length(observations)} obs at #{valid_time}")
|
||||
|
||||
warm_and_broadcast(valid_time)
|
||||
:ok
|
||||
|
||||
error ->
|
||||
Logger.error("AsosAdjustment: upsert failed — #{inspect(error)}")
|
||||
error
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
defp warm_and_broadcast(valid_time) do
|
||||
Enum.each(BandConfig.all_bands(), fn band ->
|
||||
Propagation.warm_cache_and_broadcast(band.freq_mhz, valid_time)
|
||||
end)
|
||||
|
||||
Phoenix.PubSub.broadcast(
|
||||
Microwaveprop.PubSub,
|
||||
"propagation:updated",
|
||||
{:propagation_updated, valid_time}
|
||||
)
|
||||
|
||||
:ok
|
||||
|
||||
error ->
|
||||
Logger.error("AsosAdjustment: upsert failed: #{inspect(error)}")
|
||||
error
|
||||
end
|
||||
end
|
||||
|
||||
defp score_grid_from_asos(observations, valid_time) do
|
||||
# Build a spatial index of observations
|
||||
obs_list =
|
||||
Enum.map(observations, fn obs ->
|
||||
%{
|
||||
lat: obs.lat,
|
||||
lon: obs.lon,
|
||||
temp_f: obs.temp_f,
|
||||
dewpoint_f: obs.dewpoint_f,
|
||||
wind_speed_kts: obs.wind_speed_kts,
|
||||
sky_condition: obs.sky_condition,
|
||||
sea_level_pressure_mb: pressure_from_obs(obs),
|
||||
precip_1h_in: obs.precip_1h_in
|
||||
}
|
||||
end)
|
||||
|
||||
# For each grid point, find nearest station and score
|
||||
Grid.conus_points()
|
||||
|> Task.async_stream(
|
||||
fn {lat, lon} = point ->
|
||||
nearest = find_nearest_station(obs_list, lat, lon)
|
||||
|
||||
if nearest do
|
||||
score_point_from_obs(point, nearest, valid_time)
|
||||
end
|
||||
end,
|
||||
max_concurrency: System.schedulers_online() * 2,
|
||||
timeout: 10_000
|
||||
defp latest_hrrr_valid_time do
|
||||
case Repo.one(
|
||||
from(h in HrrrProfile,
|
||||
where: h.is_grid_point == true,
|
||||
select: max(h.valid_time)
|
||||
)
|
||||
|> Enum.flat_map(fn
|
||||
{:ok, nil} -> []
|
||||
{:ok, scores} -> scores
|
||||
{:exit, _} -> []
|
||||
) do
|
||||
nil -> :no_hrrr
|
||||
vt -> {:ok, vt}
|
||||
end
|
||||
end
|
||||
|
||||
defp filter_fresh(observations) do
|
||||
cutoff = DateTime.add(DateTime.utc_now(), -@max_obs_age_seconds, :second)
|
||||
|
||||
Enum.filter(observations, fn obs ->
|
||||
obs.utc_valid && DateTime.after?(obs.utc_valid, cutoff)
|
||||
end)
|
||||
end
|
||||
|
||||
defp find_nearest_station(obs_list, lat, lon) do
|
||||
obs_list
|
||||
|> Enum.map(fn obs ->
|
||||
dist = Radio.haversine_km(lat, lon, obs.lat, obs.lon)
|
||||
{dist, obs}
|
||||
end)
|
||||
|> Enum.filter(fn {dist, _} -> dist <= @max_station_distance_km end)
|
||||
|> Enum.min_by(fn {dist, _} -> dist end, fn -> nil end)
|
||||
|> case do
|
||||
nil -> nil
|
||||
{_dist, obs} -> obs
|
||||
end
|
||||
end
|
||||
|
||||
defp score_point_from_obs({lat, lon}, obs, valid_time) do
|
||||
temp_c = Scorer.f_to_c(obs.temp_f)
|
||||
dewpoint_c = Scorer.f_to_c(obs.dewpoint_f)
|
||||
|
||||
if is_nil(temp_c) or is_nil(dewpoint_c) do
|
||||
[]
|
||||
else
|
||||
sky_pct = sky_condition_to_pct(obs.sky_condition)
|
||||
rain_rate = precip_to_rate(obs.precip_1h_in)
|
||||
|
||||
conditions = %{
|
||||
abs_humidity: Scorer.absolute_humidity(temp_c, dewpoint_c),
|
||||
temp_f: obs.temp_f,
|
||||
dewpoint_f: obs.dewpoint_f,
|
||||
wind_speed_kts: obs.wind_speed_kts,
|
||||
sky_cover_pct: sky_pct,
|
||||
utc_hour: valid_time.hour,
|
||||
utc_minute: valid_time.minute,
|
||||
month: valid_time.month,
|
||||
longitude: lon,
|
||||
pressure_mb: obs.sea_level_pressure_mb,
|
||||
prev_pressure_mb: nil,
|
||||
rain_rate_mmhr: rain_rate,
|
||||
min_refractivity_gradient: nil,
|
||||
bl_depth_m: nil
|
||||
defp load_hrrr_profiles(valid_time) do
|
||||
Repo.all(
|
||||
from(h in HrrrProfile,
|
||||
where: h.valid_time == ^valid_time and h.is_grid_point == true,
|
||||
select: %{
|
||||
lat: h.lat,
|
||||
lon: h.lon,
|
||||
valid_time: h.valid_time,
|
||||
surface_temp_c: h.surface_temp_c,
|
||||
surface_dewpoint_c: h.surface_dewpoint_c,
|
||||
surface_pressure_mb: h.surface_pressure_mb,
|
||||
surface_refractivity: h.surface_refractivity,
|
||||
min_refractivity_gradient: h.min_refractivity_gradient,
|
||||
hpbl_m: h.hpbl_m,
|
||||
pwat_mm: h.pwat_mm,
|
||||
ducting_detected: h.ducting_detected,
|
||||
duct_characteristics: h.duct_characteristics,
|
||||
profile: h.profile
|
||||
}
|
||||
|
||||
Enum.map(BandConfig.all_bands(), fn band ->
|
||||
result = Scorer.composite_score(conditions, band)
|
||||
|
||||
%{
|
||||
lat: lat,
|
||||
lon: lon,
|
||||
valid_time: valid_time,
|
||||
band_mhz: band.freq_mhz,
|
||||
score: result.score,
|
||||
factors: result.factors
|
||||
}
|
||||
end)
|
||||
)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defp sky_condition_to_pct(nil), do: nil
|
||||
defp sky_condition_to_pct("CLR"), do: 0.0
|
||||
defp sky_condition_to_pct("SKC"), do: 0.0
|
||||
defp sky_condition_to_pct("FEW"), do: 18.0
|
||||
defp sky_condition_to_pct("SCT"), do: 40.0
|
||||
defp sky_condition_to_pct("BKN"), do: 75.0
|
||||
defp sky_condition_to_pct("OVC"), do: 100.0
|
||||
defp sky_condition_to_pct("VV"), do: 100.0
|
||||
defp sky_condition_to_pct(_), do: nil
|
||||
|
||||
defp pressure_from_obs(%{sea_level_pressure_mb: mslp}) when is_number(mslp), do: mslp
|
||||
|
||||
defp pressure_from_obs(%{altimeter_setting: alti}) when is_number(alti) do
|
||||
# Convert altimeter setting (inHg) to millibars
|
||||
alti * 33.8639
|
||||
end
|
||||
|
||||
defp pressure_from_obs(_), do: nil
|
||||
|
||||
# Convert hourly precip accumulation (inches) to rain rate (mm/hr)
|
||||
defp precip_to_rate(nil), do: 0.0
|
||||
defp precip_to_rate(inches) when inches <= 0, do: 0.0
|
||||
defp precip_to_rate(inches), do: inches * 25.4
|
||||
end
|
||||
|
|
|
|||
|
|
@ -302,8 +302,7 @@ defmodule MicrowavepropWeb.ContactLive.Show do
|
|||
{:noreply, assign(socket, :iemre, iemre)}
|
||||
end
|
||||
|
||||
def handle_async(slot, {:exit, reason}, socket)
|
||||
when slot in [:weather, :solar, :hrrr_path, :terrain, :iemre] do
|
||||
def handle_async(slot, {:exit, reason}, socket) when slot in [:weather, :solar, :hrrr_path, :terrain, :iemre] do
|
||||
Logger.warning("contact hydrate task #{slot} exited: #{inspect(reason)}")
|
||||
{:noreply, socket}
|
||||
end
|
||||
|
|
|
|||
|
|
@ -85,7 +85,6 @@ defmodule MicrowavepropWeb.ContactMapLive do
|
|||
|
||||
defp band_color(mhz), do: Map.get(@band_colors, mhz, "#64748b")
|
||||
|
||||
|
||||
@impl true
|
||||
def render(assigns) do
|
||||
~H"""
|
||||
|
|
|
|||
|
|
@ -522,8 +522,7 @@ defmodule MicrowavepropWeb.SubmitLive do
|
|||
<p class="font-semibold mb-2">Mode handling</p>
|
||||
<p class="mb-2 text-base-content/70">
|
||||
ADIF modes are normalized to the six modes we track. SUBMODE takes
|
||||
precedence when present (e.g. <code>MODE=MFSK, SUBMODE=FT8</code>
|
||||
imports as FT8).
|
||||
precedence when present (e.g. <code>MODE=MFSK, SUBMODE=FT8</code> imports as FT8).
|
||||
</p>
|
||||
<table class="table table-xs mt-2">
|
||||
<thead>
|
||||
|
|
|
|||
186
test/microwaveprop/propagation/asos_nudge_test.exs
Normal file
186
test/microwaveprop/propagation/asos_nudge_test.exs
Normal file
|
|
@ -0,0 +1,186 @@
|
|||
defmodule Microwaveprop.Propagation.AsosNudgeTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Microwaveprop.Propagation.AsosNudge
|
||||
|
||||
@valid_time ~U[2026-04-12 18:00:00Z]
|
||||
|
||||
defp profile(lat, lon, overrides \\ []) do
|
||||
Map.merge(
|
||||
%{
|
||||
valid_time: @valid_time,
|
||||
lat: lat,
|
||||
lon: lon,
|
||||
surface_temp_c: 20.0,
|
||||
surface_dewpoint_c: 10.0,
|
||||
surface_pressure_mb: 1013.0,
|
||||
hpbl_m: 1000.0,
|
||||
pwat_mm: 25.0,
|
||||
min_refractivity_gradient: -40.0,
|
||||
surface_refractivity: 320.0,
|
||||
profile: []
|
||||
},
|
||||
Map.new(overrides)
|
||||
)
|
||||
end
|
||||
|
||||
defp station(lat, lon, overrides \\ []) do
|
||||
Map.merge(
|
||||
%{
|
||||
lat: lat,
|
||||
lon: lon,
|
||||
temp_f: 68.0,
|
||||
dewpoint_f: 50.0,
|
||||
sea_level_pressure_mb: 1013.0,
|
||||
precip_1h_in: 0.0
|
||||
},
|
||||
Map.new(overrides)
|
||||
)
|
||||
end
|
||||
|
||||
describe "build_station_residuals/2" do
|
||||
test "station co-located with a HRRR grid cell yields residuals against that cell" do
|
||||
station_obs = station(32.0, -97.0, temp_f: 77.0, dewpoint_f: 59.0, sea_level_pressure_mb: 1020.0)
|
||||
hrrr = profile(32.0, -97.0, surface_temp_c: 20.0, surface_dewpoint_c: 10.0, surface_pressure_mb: 1013.0)
|
||||
|
||||
[residual] = AsosNudge.build_station_residuals([station_obs], [hrrr])
|
||||
|
||||
assert_in_delta residual.dtemp_c, 5.0, 0.1
|
||||
assert_in_delta residual.ddewpoint_c, 5.0, 0.1
|
||||
assert_in_delta residual.dpressure_mb, 7.0, 0.1
|
||||
assert residual.lat == 32.0
|
||||
assert residual.lon == -97.0
|
||||
end
|
||||
|
||||
test "station with no HRRR cell within 1 grid step is dropped" do
|
||||
obs = station(32.0, -97.0, temp_f: 77.0)
|
||||
far_hrrr = profile(40.0, -85.0, surface_temp_c: 20.0)
|
||||
|
||||
assert AsosNudge.build_station_residuals([obs], [far_hrrr]) == []
|
||||
end
|
||||
|
||||
test "station with nil temp is dropped" do
|
||||
obs = station(32.0, -97.0, temp_f: nil)
|
||||
hrrr = profile(32.0, -97.0)
|
||||
|
||||
assert AsosNudge.build_station_residuals([obs], [hrrr]) == []
|
||||
end
|
||||
|
||||
test "station with nil pressure still contributes temp and dewpoint deltas" do
|
||||
obs = station(32.0, -97.0, temp_f: 77.0, dewpoint_f: 59.0, sea_level_pressure_mb: nil)
|
||||
hrrr = profile(32.0, -97.0, surface_temp_c: 20.0, surface_dewpoint_c: 10.0, surface_pressure_mb: 1013.0)
|
||||
|
||||
[residual] = AsosNudge.build_station_residuals([obs], [hrrr])
|
||||
|
||||
assert_in_delta residual.dtemp_c, 5.0, 0.1
|
||||
assert_in_delta residual.ddewpoint_c, 5.0, 0.1
|
||||
assert residual.dpressure_mb == nil
|
||||
end
|
||||
end
|
||||
|
||||
describe "nudge_profile/2" do
|
||||
test "single station at the same grid cell applies (almost) the full residual" do
|
||||
residuals = [
|
||||
%{lat: 32.0, lon: -97.0, dtemp_c: 5.0, ddewpoint_c: 3.0, dpressure_mb: 2.0, asos_rain_mmhr: 0.0}
|
||||
]
|
||||
|
||||
hrrr = profile(32.0, -97.0, surface_temp_c: 20.0, surface_dewpoint_c: 10.0, surface_pressure_mb: 1013.0)
|
||||
nudged = AsosNudge.nudge_profile(hrrr, residuals)
|
||||
|
||||
assert_in_delta nudged.surface_temp_c, 25.0, 0.1
|
||||
assert_in_delta nudged.surface_dewpoint_c, 13.0, 0.1
|
||||
assert_in_delta nudged.surface_pressure_mb, 1015.0, 0.1
|
||||
end
|
||||
|
||||
test "station farther than 250 km contributes nothing" do
|
||||
residuals = [
|
||||
%{lat: 40.0, lon: -97.0, dtemp_c: 10.0, ddewpoint_c: 10.0, dpressure_mb: 5.0, asos_rain_mmhr: 0.0}
|
||||
]
|
||||
|
||||
hrrr = profile(32.0, -97.0, surface_temp_c: 20.0)
|
||||
nudged = AsosNudge.nudge_profile(hrrr, residuals)
|
||||
|
||||
assert nudged.surface_temp_c == 20.0
|
||||
assert nudged.surface_dewpoint_c == 10.0
|
||||
assert nudged.surface_pressure_mb == 1013.0
|
||||
end
|
||||
|
||||
test "two equidistant stations with opposite residuals average to near zero" do
|
||||
residuals = [
|
||||
%{lat: 32.0, lon: -95.93, dtemp_c: 5.0, ddewpoint_c: 0.0, dpressure_mb: 0.0, asos_rain_mmhr: 0.0},
|
||||
%{lat: 32.0, lon: -98.07, dtemp_c: -5.0, ddewpoint_c: 0.0, dpressure_mb: 0.0, asos_rain_mmhr: 0.0}
|
||||
]
|
||||
|
||||
hrrr = profile(32.0, -97.0, surface_temp_c: 20.0)
|
||||
nudged = AsosNudge.nudge_profile(hrrr, residuals)
|
||||
|
||||
assert_in_delta nudged.surface_temp_c, 20.0, 0.1
|
||||
end
|
||||
|
||||
test "nil residual components (missing pressure) leave the HRRR field unchanged" do
|
||||
residuals = [
|
||||
%{lat: 32.0, lon: -97.0, dtemp_c: 5.0, ddewpoint_c: 3.0, dpressure_mb: nil, asos_rain_mmhr: 0.0}
|
||||
]
|
||||
|
||||
hrrr = profile(32.0, -97.0, surface_temp_c: 20.0, surface_dewpoint_c: 10.0, surface_pressure_mb: 1013.0)
|
||||
nudged = AsosNudge.nudge_profile(hrrr, residuals)
|
||||
|
||||
assert_in_delta nudged.surface_temp_c, 25.0, 0.1
|
||||
assert_in_delta nudged.surface_dewpoint_c, 13.0, 0.1
|
||||
assert nudged.surface_pressure_mb == 1013.0
|
||||
end
|
||||
|
||||
test "upper-air fields pass through unchanged regardless of residuals" do
|
||||
residuals = [
|
||||
%{lat: 32.0, lon: -97.0, dtemp_c: 10.0, ddewpoint_c: 10.0, dpressure_mb: 20.0, asos_rain_mmhr: 0.0}
|
||||
]
|
||||
|
||||
hrrr = profile(32.0, -97.0, min_refractivity_gradient: -250.0, pwat_mm: 35.0, hpbl_m: 500.0)
|
||||
nudged = AsosNudge.nudge_profile(hrrr, residuals)
|
||||
|
||||
assert nudged.min_refractivity_gradient == -250.0
|
||||
assert nudged.pwat_mm == 35.0
|
||||
assert nudged.hpbl_m == 500.0
|
||||
end
|
||||
end
|
||||
|
||||
describe "compute/3" do
|
||||
test "returns empty list when observations list is empty" do
|
||||
assert AsosNudge.compute([], @valid_time, [profile(32.0, -97.0)]) == []
|
||||
end
|
||||
|
||||
test "returns empty list when no HRRR profiles provided" do
|
||||
assert AsosNudge.compute([station(32.0, -97.0)], @valid_time, []) == []
|
||||
end
|
||||
|
||||
test "returns grid score maps for nudged profiles with band_mhz, score, factors" do
|
||||
hrrr = profile(32.0, -97.0, surface_temp_c: 20.0)
|
||||
obs = station(32.0, -97.0, temp_f: 77.0)
|
||||
|
||||
results = AsosNudge.compute([obs], @valid_time, [hrrr])
|
||||
|
||||
assert length(results) > 0
|
||||
|
||||
for result <- results do
|
||||
assert result.lat == 32.0
|
||||
assert result.lon == -97.0
|
||||
assert result.valid_time == @valid_time
|
||||
assert is_integer(result.band_mhz)
|
||||
assert is_number(result.score)
|
||||
assert is_map(result.factors)
|
||||
end
|
||||
end
|
||||
|
||||
test "drops grid points with no station within 250 km from the output" do
|
||||
nudged_cell = profile(32.0, -97.0, surface_temp_c: 20.0)
|
||||
far_cell = profile(40.0, -85.0, surface_temp_c: 20.0)
|
||||
obs = station(32.0, -97.0, temp_f: 77.0)
|
||||
|
||||
results = AsosNudge.compute([obs], @valid_time, [nudged_cell, far_cell])
|
||||
|
||||
lat_lons = results |> Enum.map(&{&1.lat, &1.lon}) |> Enum.uniq()
|
||||
assert {32.0, -97.0} in lat_lons
|
||||
refute {40.0, -85.0} in lat_lons
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -16,6 +16,7 @@ defmodule Microwaveprop.Propagation.ScoreCacheTest do
|
|||
test "returns cached scores after put" do
|
||||
scores = [%{lat: 32.0, lon: -97.0, score: 75}]
|
||||
ScoreCache.put(10_000, ~U[2026-04-12 12:00:00Z], scores)
|
||||
|
||||
assert {:ok, [%{lat: 32.0, lon: -97.0, score: 75}]} =
|
||||
ScoreCache.fetch(10_000, ~U[2026-04-12 12:00:00Z])
|
||||
end
|
||||
|
|
|
|||
|
|
@ -143,8 +143,8 @@ defmodule Microwaveprop.PropagationTest do
|
|||
end
|
||||
|
||||
test "filters past valid_times out" do
|
||||
past = DateTime.add(DateTime.utc_now(), -3600, :second) |> DateTime.truncate(:second)
|
||||
future = DateTime.add(DateTime.utc_now(), 3600, :second) |> DateTime.truncate(:second)
|
||||
past = DateTime.utc_now() |> DateTime.add(-3600, :second) |> DateTime.truncate(:second)
|
||||
future = DateTime.utc_now() |> DateTime.add(3600, :second) |> DateTime.truncate(:second)
|
||||
|
||||
ScoreCache.put(10_000, past, [%{lat: 32.875, lon: -97.0, score: 60}])
|
||||
ScoreCache.put(10_000, future, [%{lat: 32.875, lon: -97.0, score: 80}])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue