perf: finish audit follow-ups (path conditions, recalibrator parallelism)
Path calculator conditions map was missing :latitude and :best_duct_band_ghz. Scorer read them with bracket-notation so the behaviour was defensively correct but we were leaving signal on the floor: - latitude: midpoint of src/dst so `score_season` picks up the right latitude-based seasonal shift for northern vs southern paths - best_duct_band_ghz: queried via new Weather.nearest_native_duct_ghz/3 at the midpoint so score_refractivity/4 applies the 1.15× boost when a native-resolution duct supports the target band Recalibrator.fit/1 was serialising ~10k per-contact HRRR lookups (5000 positives + 5000 negatives, one Weather.find_nearest_hrrr each). Parallelised both sides with Task.async_stream at 20 concurrency — well under the prod db pool of 30. Cuts a band recalibration from ~minutes of query round-trips to seconds.
This commit is contained in:
parent
1deab8b83d
commit
06feda7110
3 changed files with 82 additions and 12 deletions
|
|
@ -175,10 +175,23 @@ defmodule Microwaveprop.Propagation.Recalibrator do
|
|||
|
||||
factor_band = band_mhz || 10_000
|
||||
|
||||
# Each contact_to_factors/2 calls `Weather.find_nearest_hrrr/3` —
|
||||
# serial execution over ~5000 contacts burns a minute+ just on
|
||||
# query round-trips. Parallelize at 20 concurrent queries, well
|
||||
# under the :db pool size (30 in prod), so the recalibration
|
||||
# finishes in seconds instead of minutes.
|
||||
factor_vectors =
|
||||
contacts
|
||||
|> Enum.map(&contact_to_factors(&1, factor_band))
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> Task.async_stream(&contact_to_factors(&1, factor_band),
|
||||
max_concurrency: 20,
|
||||
timeout: 30_000,
|
||||
on_timeout: :kill_task
|
||||
)
|
||||
|> Enum.flat_map(fn
|
||||
{:ok, nil} -> []
|
||||
{:ok, vec} -> [vec]
|
||||
{:exit, _} -> []
|
||||
end)
|
||||
|
||||
{factor_vectors, contacts}
|
||||
end
|
||||
|
|
@ -201,13 +214,22 @@ defmodule Microwaveprop.Propagation.Recalibrator do
|
|||
factor_band = band_mhz || 10_000
|
||||
|
||||
baselines
|
||||
|> Enum.map(fn {lat, lon, time} ->
|
||||
case Weather.find_nearest_hrrr(lat, lon, time) do
|
||||
nil -> nil
|
||||
profile -> compute_factors(profile, time, factor_band)
|
||||
end
|
||||
|> Task.async_stream(
|
||||
fn {lat, lon, time} ->
|
||||
case Weather.find_nearest_hrrr(lat, lon, time) do
|
||||
nil -> nil
|
||||
profile -> compute_factors(profile, time, factor_band)
|
||||
end
|
||||
end,
|
||||
max_concurrency: 20,
|
||||
timeout: 30_000,
|
||||
on_timeout: :kill_task
|
||||
)
|
||||
|> Enum.flat_map(fn
|
||||
{:ok, nil} -> []
|
||||
{:ok, vec} -> [vec]
|
||||
{:exit, _} -> []
|
||||
end)
|
||||
|> Enum.reject(&is_nil/1)
|
||||
end
|
||||
|
||||
defp run_training(positives, negatives, learning_rate, epochs) do
|
||||
|
|
|
|||
|
|
@ -795,6 +795,45 @@ defmodule Microwaveprop.Weather do
|
|||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Returns the nearest `hrrr_native_profiles.best_duct_band_ghz` to
|
||||
(`lat`, `lon`) at `timestamp`, or nil if no native profile covers
|
||||
the cell at that time. Uses the same ±0.07° / ±1h tolerance as
|
||||
HRRR pressure-level lookups.
|
||||
|
||||
The native product resolves thin trapping layers the pressure-level
|
||||
product misses (see algo.md Part 2c), so this value is what the
|
||||
refractivity scoring should consult when deciding whether to apply
|
||||
the 1.15× native-duct boost.
|
||||
"""
|
||||
@spec nearest_native_duct_ghz(float(), float(), DateTime.t()) :: float() | nil
|
||||
def nearest_native_duct_ghz(lat, lon, %DateTime{} = timestamp) do
|
||||
dlat = 0.07
|
||||
dlon = 0.07
|
||||
time_start = DateTime.add(timestamp, -3600, :second)
|
||||
time_end = DateTime.add(timestamp, 3600, :second)
|
||||
|
||||
Repo.one(
|
||||
from(h in HrrrNativeProfile,
|
||||
where:
|
||||
h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and
|
||||
h.valid_time >= ^time_start and h.valid_time <= ^time_end,
|
||||
order_by:
|
||||
fragment(
|
||||
"ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))",
|
||||
h.lat,
|
||||
^lat,
|
||||
h.lon,
|
||||
^lon,
|
||||
h.valid_time,
|
||||
^timestamp
|
||||
),
|
||||
limit: 1,
|
||||
select: h.best_duct_band_ghz
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
@doc """
|
||||
Nearest sounding to (`lat`, `lon`) within `radius_km` km and a ±3-hour
|
||||
window around `timestamp`. Joins `weather_stations` to `soundings` so
|
||||
|
|
|
|||
|
|
@ -271,8 +271,15 @@ defmodule MicrowavepropWeb.PathLive do
|
|||
# surface ducts that sounding data resolves.
|
||||
sounding = build_sounding_readout(midlat, midlon, now)
|
||||
|
||||
# Native-resolution HRRR duct band near the midpoint. Resolves
|
||||
# thin trapping layers HRRR's 13 pressure levels miss; feeds the
|
||||
# 1.15× boost in Scorer.score_refractivity when the duct supports
|
||||
# the target band.
|
||||
native_duct_ghz = Weather.nearest_native_duct_ghz(midlat, midlon, now)
|
||||
|
||||
# Build conditions and score
|
||||
{conditions, scoring} = build_scoring(hrrr_profiles, src, now, band_config)
|
||||
{conditions, scoring} =
|
||||
build_scoring(hrrr_profiles, src, dst, now, band_config, native_duct_ghz)
|
||||
|
||||
# Loss and power budgets
|
||||
loss_budget = compute_loss_budget(dist_km, freq_ghz, band_config, terrain_result, conditions)
|
||||
|
|
@ -436,9 +443,9 @@ defmodule MicrowavepropWeb.PathLive do
|
|||
end
|
||||
end
|
||||
|
||||
defp build_scoring([], _src, _now, _band_config), do: {nil, nil}
|
||||
defp build_scoring([], _src, _dst, _now, _band_config, _native_duct_ghz), do: {nil, nil}
|
||||
|
||||
defp build_scoring(profiles, src, now, band_config) do
|
||||
defp build_scoring(profiles, src, dst, now, band_config, native_duct_ghz) do
|
||||
temps = profiles |> Enum.map(& &1.surface_temp_c) |> Enum.reject(&is_nil/1)
|
||||
dewpoints = profiles |> Enum.map(& &1.surface_dewpoint_c) |> Enum.reject(&is_nil/1)
|
||||
|
||||
|
|
@ -464,13 +471,15 @@ defmodule MicrowavepropWeb.PathLive do
|
|||
utc_hour: now.hour,
|
||||
utc_minute: now.minute,
|
||||
month: now.month,
|
||||
latitude: (src.lat + dst.lat) / 2,
|
||||
longitude: src.lon,
|
||||
pressure_mb: if(pressures != [], do: Enum.min(pressures)),
|
||||
prev_pressure_mb: nil,
|
||||
rain_rate_mmhr: 0.0,
|
||||
min_refractivity_gradient: if(gradients != [], do: Enum.min(gradients)),
|
||||
bl_depth_m: if(bl_depths != [], do: Enum.sum(bl_depths) / length(bl_depths)),
|
||||
pwat_mm: if(pwats != [], do: Enum.sum(pwats) / length(pwats))
|
||||
pwat_mm: if(pwats != [], do: Enum.sum(pwats) / length(pwats)),
|
||||
best_duct_band_ghz: native_duct_ghz
|
||||
}
|
||||
|
||||
scoring = Scorer.composite_score(conditions, band_config)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue