perf: finish audit follow-ups (path conditions, recalibrator parallelism)
Path calculator conditions map was missing :latitude and :best_duct_band_ghz. Scorer read them with bracket-notation so the behaviour was defensively correct but we were leaving signal on the floor: - latitude: midpoint of src/dst so `score_season` picks up the right latitude-based seasonal shift for northern vs southern paths - best_duct_band_ghz: queried via new Weather.nearest_native_duct_ghz/3 at the midpoint so score_refractivity/4 applies the 1.15× boost when a native-resolution duct supports the target band Recalibrator.fit/1 was serialising ~10k per-contact HRRR lookups (5000 positives + 5000 negatives, one Weather.find_nearest_hrrr each). Parallelised both sides with Task.async_stream at 20 concurrency — well under the prod db pool of 30. Cuts a band recalibration from ~minutes of query round-trips to seconds.
This commit is contained in:
parent
1deab8b83d
commit
06feda7110
3 changed files with 82 additions and 12 deletions
|
|
@ -175,10 +175,23 @@ defmodule Microwaveprop.Propagation.Recalibrator do
|
||||||
|
|
||||||
factor_band = band_mhz || 10_000
|
factor_band = band_mhz || 10_000
|
||||||
|
|
||||||
|
# Each contact_to_factors/2 calls `Weather.find_nearest_hrrr/3` —
|
||||||
|
# serial execution over ~5000 contacts burns a minute+ just on
|
||||||
|
# query round-trips. Parallelize at 20 concurrent queries, well
|
||||||
|
# under the :db pool size (30 in prod), so the recalibration
|
||||||
|
# finishes in seconds instead of minutes.
|
||||||
factor_vectors =
|
factor_vectors =
|
||||||
contacts
|
contacts
|
||||||
|> Enum.map(&contact_to_factors(&1, factor_band))
|
|> Task.async_stream(&contact_to_factors(&1, factor_band),
|
||||||
|> Enum.reject(&is_nil/1)
|
max_concurrency: 20,
|
||||||
|
timeout: 30_000,
|
||||||
|
on_timeout: :kill_task
|
||||||
|
)
|
||||||
|
|> Enum.flat_map(fn
|
||||||
|
{:ok, nil} -> []
|
||||||
|
{:ok, vec} -> [vec]
|
||||||
|
{:exit, _} -> []
|
||||||
|
end)
|
||||||
|
|
||||||
{factor_vectors, contacts}
|
{factor_vectors, contacts}
|
||||||
end
|
end
|
||||||
|
|
@ -201,13 +214,22 @@ defmodule Microwaveprop.Propagation.Recalibrator do
|
||||||
factor_band = band_mhz || 10_000
|
factor_band = band_mhz || 10_000
|
||||||
|
|
||||||
baselines
|
baselines
|
||||||
|> Enum.map(fn {lat, lon, time} ->
|
|> Task.async_stream(
|
||||||
case Weather.find_nearest_hrrr(lat, lon, time) do
|
fn {lat, lon, time} ->
|
||||||
nil -> nil
|
case Weather.find_nearest_hrrr(lat, lon, time) do
|
||||||
profile -> compute_factors(profile, time, factor_band)
|
nil -> nil
|
||||||
end
|
profile -> compute_factors(profile, time, factor_band)
|
||||||
|
end
|
||||||
|
end,
|
||||||
|
max_concurrency: 20,
|
||||||
|
timeout: 30_000,
|
||||||
|
on_timeout: :kill_task
|
||||||
|
)
|
||||||
|
|> Enum.flat_map(fn
|
||||||
|
{:ok, nil} -> []
|
||||||
|
{:ok, vec} -> [vec]
|
||||||
|
{:exit, _} -> []
|
||||||
end)
|
end)
|
||||||
|> Enum.reject(&is_nil/1)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
defp run_training(positives, negatives, learning_rate, epochs) do
|
defp run_training(positives, negatives, learning_rate, epochs) do
|
||||||
|
|
|
||||||
|
|
@ -795,6 +795,45 @@ defmodule Microwaveprop.Weather do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
@doc """
|
||||||
|
Returns the nearest `hrrr_native_profiles.best_duct_band_ghz` to
|
||||||
|
(`lat`, `lon`) at `timestamp`, or nil if no native profile covers
|
||||||
|
the cell at that time. Uses the same ±0.07° / ±1h tolerance as
|
||||||
|
HRRR pressure-level lookups.
|
||||||
|
|
||||||
|
The native product resolves thin trapping layers the pressure-level
|
||||||
|
product misses (see algo.md Part 2c), so this value is what the
|
||||||
|
refractivity scoring should consult when deciding whether to apply
|
||||||
|
the 1.15× native-duct boost.
|
||||||
|
"""
|
||||||
|
@spec nearest_native_duct_ghz(float(), float(), DateTime.t()) :: float() | nil
|
||||||
|
def nearest_native_duct_ghz(lat, lon, %DateTime{} = timestamp) do
|
||||||
|
dlat = 0.07
|
||||||
|
dlon = 0.07
|
||||||
|
time_start = DateTime.add(timestamp, -3600, :second)
|
||||||
|
time_end = DateTime.add(timestamp, 3600, :second)
|
||||||
|
|
||||||
|
Repo.one(
|
||||||
|
from(h in HrrrNativeProfile,
|
||||||
|
where:
|
||||||
|
h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and
|
||||||
|
h.valid_time >= ^time_start and h.valid_time <= ^time_end,
|
||||||
|
order_by:
|
||||||
|
fragment(
|
||||||
|
"ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))",
|
||||||
|
h.lat,
|
||||||
|
^lat,
|
||||||
|
h.lon,
|
||||||
|
^lon,
|
||||||
|
h.valid_time,
|
||||||
|
^timestamp
|
||||||
|
),
|
||||||
|
limit: 1,
|
||||||
|
select: h.best_duct_band_ghz
|
||||||
|
)
|
||||||
|
)
|
||||||
|
end
|
||||||
|
|
||||||
@doc """
|
@doc """
|
||||||
Nearest sounding to (`lat`, `lon`) within `radius_km` km and a ±3-hour
|
Nearest sounding to (`lat`, `lon`) within `radius_km` km and a ±3-hour
|
||||||
window around `timestamp`. Joins `weather_stations` to `soundings` so
|
window around `timestamp`. Joins `weather_stations` to `soundings` so
|
||||||
|
|
|
||||||
|
|
@ -271,8 +271,15 @@ defmodule MicrowavepropWeb.PathLive do
|
||||||
# surface ducts that sounding data resolves.
|
# surface ducts that sounding data resolves.
|
||||||
sounding = build_sounding_readout(midlat, midlon, now)
|
sounding = build_sounding_readout(midlat, midlon, now)
|
||||||
|
|
||||||
|
# Native-resolution HRRR duct band near the midpoint. Resolves
|
||||||
|
# thin trapping layers HRRR's 13 pressure levels miss; feeds the
|
||||||
|
# 1.15× boost in Scorer.score_refractivity when the duct supports
|
||||||
|
# the target band.
|
||||||
|
native_duct_ghz = Weather.nearest_native_duct_ghz(midlat, midlon, now)
|
||||||
|
|
||||||
# Build conditions and score
|
# Build conditions and score
|
||||||
{conditions, scoring} = build_scoring(hrrr_profiles, src, now, band_config)
|
{conditions, scoring} =
|
||||||
|
build_scoring(hrrr_profiles, src, dst, now, band_config, native_duct_ghz)
|
||||||
|
|
||||||
# Loss and power budgets
|
# Loss and power budgets
|
||||||
loss_budget = compute_loss_budget(dist_km, freq_ghz, band_config, terrain_result, conditions)
|
loss_budget = compute_loss_budget(dist_km, freq_ghz, band_config, terrain_result, conditions)
|
||||||
|
|
@ -436,9 +443,9 @@ defmodule MicrowavepropWeb.PathLive do
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
defp build_scoring([], _src, _now, _band_config), do: {nil, nil}
|
defp build_scoring([], _src, _dst, _now, _band_config, _native_duct_ghz), do: {nil, nil}
|
||||||
|
|
||||||
defp build_scoring(profiles, src, now, band_config) do
|
defp build_scoring(profiles, src, dst, now, band_config, native_duct_ghz) do
|
||||||
temps = profiles |> Enum.map(& &1.surface_temp_c) |> Enum.reject(&is_nil/1)
|
temps = profiles |> Enum.map(& &1.surface_temp_c) |> Enum.reject(&is_nil/1)
|
||||||
dewpoints = profiles |> Enum.map(& &1.surface_dewpoint_c) |> Enum.reject(&is_nil/1)
|
dewpoints = profiles |> Enum.map(& &1.surface_dewpoint_c) |> Enum.reject(&is_nil/1)
|
||||||
|
|
||||||
|
|
@ -464,13 +471,15 @@ defmodule MicrowavepropWeb.PathLive do
|
||||||
utc_hour: now.hour,
|
utc_hour: now.hour,
|
||||||
utc_minute: now.minute,
|
utc_minute: now.minute,
|
||||||
month: now.month,
|
month: now.month,
|
||||||
|
latitude: (src.lat + dst.lat) / 2,
|
||||||
longitude: src.lon,
|
longitude: src.lon,
|
||||||
pressure_mb: if(pressures != [], do: Enum.min(pressures)),
|
pressure_mb: if(pressures != [], do: Enum.min(pressures)),
|
||||||
prev_pressure_mb: nil,
|
prev_pressure_mb: nil,
|
||||||
rain_rate_mmhr: 0.0,
|
rain_rate_mmhr: 0.0,
|
||||||
min_refractivity_gradient: if(gradients != [], do: Enum.min(gradients)),
|
min_refractivity_gradient: if(gradients != [], do: Enum.min(gradients)),
|
||||||
bl_depth_m: if(bl_depths != [], do: Enum.sum(bl_depths) / length(bl_depths)),
|
bl_depth_m: if(bl_depths != [], do: Enum.sum(bl_depths) / length(bl_depths)),
|
||||||
pwat_mm: if(pwats != [], do: Enum.sum(pwats) / length(pwats))
|
pwat_mm: if(pwats != [], do: Enum.sum(pwats) / length(pwats)),
|
||||||
|
best_duct_band_ghz: native_duct_ghz
|
||||||
}
|
}
|
||||||
|
|
||||||
scoring = Scorer.composite_score(conditions, band_config)
|
scoring = Scorer.composite_score(conditions, band_config)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue