458K-record upsert held a connection for the entire transaction, exceeding the 15s Postgrex timeout on prod. Set transaction timeout to infinity and increase prod pool from 10 to 20.
196 lines
5.8 KiB
Elixir
196 lines
5.8 KiB
Elixir
defmodule Microwaveprop.Propagation do
|
|
@moduledoc false
|
|
|
|
import Ecto.Query
|
|
|
|
alias Microwaveprop.Propagation.BandConfig
|
|
alias Microwaveprop.Propagation.Grid
|
|
alias Microwaveprop.Propagation.GridScore
|
|
alias Microwaveprop.Propagation.Scorer
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Weather.SoundingParams
|
|
|
|
require Logger
|
|
|
|
@doc """
|
|
Score a single grid point across all bands using HRRR profile data.
|
|
Returns a list of %{band_mhz, score, factors} maps.
|
|
"""
|
|
def score_grid_point(hrrr_profile, valid_time) do
|
|
derived = derive_from_hrrr(hrrr_profile)
|
|
|
|
temp_c = hrrr_profile.surface_temp_c
|
|
dewpoint_c = hrrr_profile.surface_dewpoint_c
|
|
|
|
# Skip points with missing or physically impossible surface data
|
|
if is_nil(temp_c) or is_nil(dewpoint_c) or temp_c < -80 or temp_c > 60 or
|
|
dewpoint_c < -80 or dewpoint_c > 50 do
|
|
[]
|
|
else
|
|
score_grid_point_with_data(hrrr_profile, valid_time, temp_c, dewpoint_c, derived)
|
|
end
|
|
end
|
|
|
|
defp score_grid_point_with_data(hrrr_profile, valid_time, temp_c, dewpoint_c, derived) do
|
|
temp_f = Scorer.c_to_f(temp_c)
|
|
dewpoint_f = Scorer.c_to_f(dewpoint_c)
|
|
|
|
conditions = %{
|
|
abs_humidity: Scorer.absolute_humidity(temp_c, dewpoint_c),
|
|
temp_f: temp_f,
|
|
dewpoint_f: dewpoint_f,
|
|
wind_speed_kts: Scorer.wind_speed_kts(hrrr_profile[:wind_u], hrrr_profile[:wind_v]),
|
|
sky_cover_pct: hrrr_profile[:cloud_cover_pct],
|
|
utc_hour: valid_time.hour,
|
|
utc_minute: valid_time.minute,
|
|
month: valid_time.month,
|
|
pressure_mb: hrrr_profile.surface_pressure_mb,
|
|
prev_pressure_mb: nil,
|
|
rain_rate_mmhr: Scorer.precip_to_rate_mmhr(hrrr_profile[:precip_mm]),
|
|
min_refractivity_gradient: derived[:min_refractivity_gradient],
|
|
bl_depth_m: hrrr_profile[:hpbl_m]
|
|
}
|
|
|
|
Enum.map(BandConfig.all_bands(), fn band_config ->
|
|
result = Scorer.composite_score(conditions, band_config)
|
|
Map.put(result, :band_mhz, band_config.freq_mhz)
|
|
end)
|
|
end
|
|
|
|
@doc "Upsert propagation scores in batches within a transaction so readers see all-or-nothing."
|
|
def upsert_scores(scores) do
|
|
now = DateTime.truncate(DateTime.utc_now(), :second)
|
|
|
|
entries =
|
|
Enum.map(scores, fn s ->
|
|
%{
|
|
id: Ecto.UUID.generate(),
|
|
lat: s.lat,
|
|
lon: s.lon,
|
|
valid_time: s.valid_time,
|
|
band_mhz: s.band_mhz,
|
|
score: s.score,
|
|
factors: s.factors,
|
|
inserted_at: now,
|
|
updated_at: now
|
|
}
|
|
end)
|
|
|
|
result =
|
|
Repo.transaction(
|
|
fn ->
|
|
entries
|
|
|> Enum.chunk_every(500)
|
|
|> Enum.reduce(0, fn chunk, acc ->
|
|
{count, _} =
|
|
Repo.insert_all(GridScore, chunk,
|
|
on_conflict: {:replace, [:score, :factors, :updated_at]},
|
|
conflict_target: [:lat, :lon, :valid_time, :band_mhz]
|
|
)
|
|
|
|
acc + count
|
|
end)
|
|
end,
|
|
timeout: :infinity
|
|
)
|
|
|
|
case result do
|
|
{:ok, _count} -> prune_old_scores()
|
|
_ -> :ok
|
|
end
|
|
|
|
result
|
|
end
|
|
|
|
defp prune_old_scores do
|
|
keep_times =
|
|
Repo.all(
|
|
from(gs in GridScore, select: gs.valid_time, distinct: gs.valid_time, order_by: [desc: gs.valid_time], limit: 2)
|
|
)
|
|
|
|
if length(keep_times) == 2 do
|
|
oldest_kept = List.last(keep_times)
|
|
|
|
{deleted, _} = Repo.delete_all(from(gs in GridScore, where: gs.valid_time < ^oldest_kept))
|
|
|
|
if deleted > 0 do
|
|
Logger.info("PropagationScores: pruned #{deleted} old scores, keeping 2 most recent valid_times")
|
|
end
|
|
end
|
|
end
|
|
|
|
@doc "Get the latest scores for a band, optionally within a bounding box. Excludes factors for performance."
|
|
def latest_scores(band_mhz, bounds \\ nil) do
|
|
latest_time = latest_valid_time(band_mhz)
|
|
|
|
case latest_time do
|
|
nil ->
|
|
[]
|
|
|
|
_ ->
|
|
from(gs in GridScore,
|
|
where: gs.band_mhz == ^band_mhz and gs.valid_time == ^latest_time,
|
|
select: %{lat: gs.lat, lon: gs.lon, score: gs.score, valid_time: gs.valid_time}
|
|
)
|
|
|> maybe_filter_bounds(bounds)
|
|
|> Repo.all()
|
|
end
|
|
end
|
|
|
|
@doc "Get the full score and factors for a specific grid point, snapped to nearest grid."
|
|
def point_detail(band_mhz, lat, lon) do
|
|
step = Grid.step()
|
|
snapped_lat = Float.round(Float.round(lat / step) * step, 3)
|
|
snapped_lon = Float.round(Float.round(lon / step) * step, 3)
|
|
|
|
case latest_valid_time(band_mhz) do
|
|
nil ->
|
|
nil
|
|
|
|
latest_time ->
|
|
Repo.one(
|
|
from(gs in GridScore,
|
|
where:
|
|
gs.band_mhz == ^band_mhz and
|
|
gs.valid_time == ^latest_time and
|
|
gs.lat == ^snapped_lat and
|
|
gs.lon == ^snapped_lon,
|
|
select: %{
|
|
lat: gs.lat,
|
|
lon: gs.lon,
|
|
score: gs.score,
|
|
factors: gs.factors,
|
|
valid_time: gs.valid_time
|
|
}
|
|
)
|
|
)
|
|
end
|
|
end
|
|
|
|
defp maybe_filter_bounds(query, nil), do: query
|
|
|
|
defp maybe_filter_bounds(query, %{"south" => s, "north" => n, "west" => w, "east" => e}) do
|
|
from(gs in query,
|
|
where: gs.lat >= ^s and gs.lat <= ^n and gs.lon >= ^w and gs.lon <= ^e
|
|
)
|
|
end
|
|
|
|
@doc "Get the latest valid_time across all scores."
|
|
def latest_valid_time do
|
|
Repo.one(from(gs in GridScore, select: max(gs.valid_time)))
|
|
end
|
|
|
|
@doc "Get the latest valid_time for a specific band."
|
|
def latest_valid_time(band_mhz) do
|
|
Repo.one(from(gs in GridScore, where: gs.band_mhz == ^band_mhz, select: max(gs.valid_time)))
|
|
end
|
|
|
|
defp derive_from_hrrr(%{profile: profile}) when is_list(profile) and length(profile) >= 3 do
|
|
case SoundingParams.derive(profile) do
|
|
nil -> %{}
|
|
derived -> %{min_refractivity_gradient: derived.min_refractivity_gradient}
|
|
end
|
|
end
|
|
|
|
defp derive_from_hrrr(_), do: %{}
|
|
end
|