From 5eeac23e83fc6463cd71ea5e6a37b8d255ca2588 Mon Sep 17 00:00:00 2001 From: Graham McInitre Date: Thu, 16 Jul 2026 07:47:47 -0500 Subject: [PATCH] =?UTF-8?q?refactor:=20extract=20ProfileLookup=20module=20?= =?UTF-8?q?from=20weather.ex=20(2072=E2=86=921696=20lines,=20-376)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/microwaveprop/weather.ex | 370 +-------------- lib/microwaveprop/weather/profile_lookup.ex | 428 ++++++++++++++++++ .../workers/contact_weather_enqueue_worker.ex | 42 +- .../live/weather_ca_map_live.ex | 365 +-------------- .../live/weather_map_component.ex | 428 ++++++++++++++++++ .../live/weather_map_live.ex | 416 +---------------- 6 files changed, 940 insertions(+), 1109 deletions(-) create mode 100644 lib/microwaveprop/weather/profile_lookup.ex create mode 100644 lib/microwaveprop_web/live/weather_map_component.ex diff --git a/lib/microwaveprop/weather.ex b/lib/microwaveprop/weather.ex index 3750bd73..c564ec0b 100644 --- a/lib/microwaveprop/weather.ex +++ b/lib/microwaveprop/weather.ex @@ -11,12 +11,12 @@ defmodule Microwaveprop.Weather do alias Microwaveprop.Repo alias Microwaveprop.Weather.GefsProfile alias Microwaveprop.Weather.GridCache - alias Microwaveprop.Weather.HrrrClient alias Microwaveprop.Weather.HrrrNativeProfile alias Microwaveprop.Weather.HrrrProfile alias Microwaveprop.Weather.IemClient alias Microwaveprop.Weather.IemreObservation alias Microwaveprop.Weather.NarrProfile + alias Microwaveprop.Weather.ProfileLookup alias Microwaveprop.Weather.ScalarFile alias Microwaveprop.Weather.SolarIndex alias Microwaveprop.Weather.Sounding @@ -30,6 +30,26 @@ defmodule Microwaveprop.Weather do # Approximate km per degree latitude @km_per_deg_lat 111.0 + # Profile-lookup delegates — implementations live in ProfileLookup + defdelegate find_nearest_hrrr(lat, lon, timestamp), to: ProfileLookup + defdelegate hrrr_profiles_for_path(contact), to: ProfileLookup + defdelegate hrrr_profiles_for_contacts(contacts), to: ProfileLookup + defdelegate find_nearest_native_profile(lat, lon, timestamp), to: ProfileLookup + defdelegate narr_profiles_for_path(contact), to: ProfileLookup + defdelegate find_nearest_narr(lat, lon, timestamp), to: ProfileLookup + defdelegate best_profile_for_contact(contact), to: ProfileLookup + defdelegate hrrr_for_contact(contact), to: ProfileLookup + defdelegate narr_for_contact(contact), to: ProfileLookup + defdelegate hrrr_data_fully_present?(contact), to: ProfileLookup + defdelegate has_hrrr_profile?(lat, lon, valid_time), to: ProfileLookup + defdelegate hrrr_points_present_batch(points), to: ProfileLookup + defdelegate iemre_for_contact(contact), to: ProfileLookup + defdelegate iemre_for_path(contact), to: ProfileLookup + defdelegate find_nearest_iemre(lat, lon, timestamp), to: ProfileLookup + defdelegate round_to_hrrr_grid(lat, lon), to: ProfileLookup + defdelegate round_to_iemre_grid(lat, lon), to: ProfileLookup + defdelegate purge_grid_point_profiles(), to: ProfileLookup + @spec find_or_create_station(map()) :: {:ok, Station.t()} | {:error, Ecto.Changeset.t()} def find_or_create_station(attrs) do code = attrs[:station_code] || attrs["station_code"] @@ -1530,68 +1550,6 @@ defmodule Microwaveprop.Weather do end) end - @spec has_hrrr_profile?(float(), float(), DateTime.t()) :: boolean() - def has_hrrr_profile?(lat, lon, valid_time) do - dlat = 0.07 - dlon = 0.07 - - HrrrProfile - |> where( - [h], - h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and - h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and - h.valid_time == ^valid_time - ) - |> Repo.exists?() - end - - @doc """ - Returns `true` when every path point for `contact` (pos1 → midpoint → pos2, - or just pos1 for one-endpoint contacts) already has an HRRR profile at the - nearest HRRR hour. Lets the enrichment enqueuer skip :queued → :queued - churn when the backing data is already present. - - Returns `false` if `pos1` or `qso_timestamp` is nil — a contact with no - position or no timestamp can't be looked up. - """ - @spec hrrr_data_fully_present?( - Contact.t() - | %{ - required(:pos1) => term(), - required(:qso_timestamp) => DateTime.t() | nil, - optional(:pos2) => term() - } - ) :: boolean() - def hrrr_data_fully_present?(%{pos1: nil}), do: false - def hrrr_data_fully_present?(%{qso_timestamp: nil}), do: false - - def hrrr_data_fully_present?(contact) do - rounded = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp) - - case Radio.contact_path_points(contact) do - [] -> - false - - points -> - Enum.all?(points, fn {lat, lon} -> - {rlat, rlon} = round_to_hrrr_grid(lat, lon) - has_hrrr_profile?(rlat, rlon, rounded) - end) - end - end - - @spec hrrr_for_contact(map()) :: HrrrProfile.t() | nil - def hrrr_for_contact(%{pos1: nil}), do: nil - - def hrrr_for_contact(contact) do - lat = contact.pos1["lat"] - lon = contact.pos1["lon"] - - if lat && lon do - find_nearest_hrrr(lat, lon, contact.qso_timestamp) - end - end - @doc """ Returns just `hrrr_native_profiles.best_duct_band_ghz` near the given cell. Convenience wrapper around `nearest_native_duct_info/3` @@ -1698,160 +1656,6 @@ defmodule Microwaveprop.Weather do end end - @spec find_nearest_hrrr(float(), float(), DateTime.t()) :: HrrrProfile.t() | nil - def find_nearest_hrrr(lat, lon, timestamp) do - dlat = 0.07 - dlon = 0.07 - time_start = DateTime.add(timestamp, -3600, :second) - time_end = DateTime.add(timestamp, 3600, :second) - - HrrrProfile - |> where( - [h], - h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and - h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and - h.valid_time >= ^time_start and h.valid_time <= ^time_end - ) - |> order_by([h], - asc: - fragment( - "ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))", - h.lat, - ^lat, - h.lon, - ^lon, - h.valid_time, - ^timestamp - ) - ) - |> limit(1) - |> Repo.one() - end - - @spec hrrr_profiles_for_path(map()) :: [HrrrProfile.t()] - def hrrr_profiles_for_path(%{pos1: nil}), do: [] - - def hrrr_profiles_for_path(contact) do - contact - |> Radio.contact_path_points() - |> Enum.map(fn {lat, lon} -> find_nearest_hrrr(lat, lon, contact.qso_timestamp) end) - |> Enum.reject(&is_nil/1) - end - - @doc """ - Batch version of `hrrr_profiles_for_path/1`. Queries all HRRR profiles - for a list of contacts in a single DB round-trip, then maps results - back per-contact. Returns a list of `{contact, [profile]}` tuples. - """ - @spec hrrr_profiles_for_contacts([map()]) :: [{map(), [HrrrProfile.t()]}] - def hrrr_profiles_for_contacts(contacts) do - for_result = - for c <- contacts, c.pos1 != nil, {lat, lon} <- Radio.contact_path_points(c) do - {{lat, lon, c.qso_timestamp}, c} - end - - points = - Enum.uniq_by(for_result, fn {key, _} -> key end) - - {keys, contact_map} = - Enum.reduce(points, {[], %{}}, fn {{lat, lon, ts} = key, c}, {ks, cm} -> - {[key | ks], Map.update(cm, c, [key], &[key | &1])} - end) - - index = find_nearest_hrrrs_batch(keys) - - Enum.map(contacts, fn c -> - point_keys = Map.get(contact_map, c, []) - profiles = point_keys |> Enum.map(&Map.get(index, &1)) |> Enum.reject(&is_nil/1) - {c, profiles} - end) - end - - # Batch nearest-HRRR lookup: query all profiles in the union bounding - # box once, then match each point to its nearest profile in Elixir. - @spec find_nearest_hrrrs_batch([{float(), float(), DateTime.t()}]) :: %{ - {float(), float(), DateTime.t()} => HrrrProfile.t() - } - defp find_nearest_hrrrs_batch([]), do: %{} - - defp find_nearest_hrrrs_batch(points) do - d = 0.07 - margin_s = 3600 - - {lats, lons, times} = Enum.unzip(points) - lat_min = Enum.min(lats) - d - lat_max = Enum.max(lats) + d - lon_min = Enum.min(lons) - d - lon_max = Enum.max(lons) + d - time_min = times |> Enum.min_by(&DateTime.to_unix/1) |> DateTime.add(-margin_s, :second) - time_max = times |> Enum.max_by(&DateTime.to_unix/1) |> DateTime.add(margin_s, :second) - - profiles = - Repo.all( - from(h in HrrrProfile, - where: - h.lat >= ^lat_min and h.lat <= ^lat_max and - h.lon >= ^lon_min and h.lon <= ^lon_max and - h.valid_time >= ^time_min and h.valid_time <= ^time_max, - select: %{lat: h.lat, lon: h.lon, valid_time: h.valid_time, profile: h} - ) - ) - - Map.new(points, fn {lat, lon, ts} -> - nearest = - Enum.min_by( - profiles, - fn p -> - abs(p.lat - lat) + abs(p.lon - lon) + abs(DateTime.diff(p.valid_time, ts)) - end, - fn -> nil end - ) - - {{lat, lon, ts}, nearest && nearest.profile} - end) - end - - @spec find_nearest_native_profile(float(), float(), DateTime.t()) :: - HrrrNativeProfile.t() | nil - def find_nearest_native_profile(lat, lon, timestamp) do - dlat = 0.07 - dlon = 0.07 - time_start = DateTime.add(timestamp, -3600, :second) - time_end = DateTime.add(timestamp, 3600, :second) - - HrrrNativeProfile - |> where( - [n], - n.lat >= ^(lat - dlat) and n.lat <= ^(lat + dlat) and - n.lon >= ^(lon - dlon) and n.lon <= ^(lon + dlon) and - n.valid_time >= ^time_start and n.valid_time <= ^time_end - ) - |> order_by([n], - asc: - fragment( - "ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))", - n.lat, - ^lat, - n.lon, - ^lon, - n.valid_time, - ^timestamp - ) - ) - |> limit(1) - |> Repo.one() - end - - @doc """ - Find the best available atmospheric profile for a contact. - Tries HRRR first (3 km, hourly), falls back to NARR (32 km, 3-hourly). - Returns the profile struct or nil. - """ - @spec best_profile_for_contact(map()) :: HrrrProfile.t() | NarrProfile.t() | nil - def best_profile_for_contact(contact) do - hrrr_for_contact(contact) || narr_for_contact(contact) - end - @doc "Find all atmospheric profiles along a contact's path, from any source." @spec profiles_along_path(map()) :: [HrrrProfile.t() | NarrProfile.t()] def profiles_along_path(contact) do @@ -1864,61 +1668,6 @@ defmodule Microwaveprop.Weather do end end - @spec narr_for_contact(map()) :: NarrProfile.t() | nil - def narr_for_contact(%{pos1: nil}), do: nil - - def narr_for_contact(contact) do - lat = contact.pos1["lat"] - lon = contact.pos1["lon"] - - if lat && lon do - find_nearest_narr(lat, lon, contact.qso_timestamp) - end - end - - @spec narr_profiles_for_path(map()) :: [NarrProfile.t()] - def narr_profiles_for_path(%{pos1: nil}), do: [] - - def narr_profiles_for_path(contact) do - contact - |> Radio.contact_path_points() - |> Enum.map(fn {lat, lon} -> find_nearest_narr(lat, lon, contact.qso_timestamp) end) - |> Enum.reject(&is_nil/1) - end - - @spec find_nearest_narr(float(), float(), DateTime.t()) :: NarrProfile.t() | nil - def find_nearest_narr(lat, lon, timestamp) do - dlat = 0.15 - dlon = 0.15 - time_start = DateTime.add(timestamp, -1800, :second) - time_end = DateTime.add(timestamp, 1800, :second) - - NarrProfile - |> where( - [p], - p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and - p.lon >= ^(lon - dlon) and p.lon <= ^(lon + dlon) and - p.valid_time >= ^time_start and p.valid_time <= ^time_end - ) - |> order_by([p], - asc: - fragment( - "ABS(? - ?) + ABS(? - ?)", - p.lat, - ^lat, - p.lon, - ^lon - ) - ) - |> limit(1) - |> Repo.one() - end - - @spec round_to_hrrr_grid(float(), float()) :: {float(), float()} - def round_to_hrrr_grid(lat, lon) do - {Float.round(lat / 1.0, 2), Float.round(lon / 1.0, 2)} - end - @doc """ Delete every `is_grid_point = true` row from `hrrr_profiles`, regardless of age. Grid-point profiles are historical — the propagation grid now lives in @@ -1928,51 +1677,6 @@ defmodule Microwaveprop.Weather do Preserves QSO-linked rows (`is_grid_point = false`), which remain the data path for contact enrichment and the `/path` calculator. """ - @spec purge_grid_point_profiles() :: non_neg_integer() - def purge_grid_point_profiles do - deleted = - Enum.reduce(hrrr_profile_partitions(), 0, fn partition, acc -> - %{num_rows: n} = - Repo.query!( - ~s(DELETE FROM "#{partition}" WHERE is_grid_point = true), - [], - timeout: 600_000 - ) - - if n > 0 do - Logger.info("Purged #{n} grid-point rows from #{partition}") - end - - acc + n - end) - - deleted - end - - @spec hrrr_profile_partitions() :: [String.t()] - defp hrrr_profile_partitions do - {:ok, %{rows: rows}} = - Repo.query( - """ - SELECT child.relname - FROM pg_inherits - JOIN pg_class parent ON pg_inherits.inhparent = parent.oid - JOIN pg_class child ON pg_inherits.inhrelid = child.oid - WHERE parent.relname = 'hrrr_profiles' - ORDER BY child.relname - """, - [], - timeout: 60_000 - ) - - Enum.map(rows, fn [name] -> name end) - end - - @spec round_to_iemre_grid(float(), float()) :: {float(), float()} - def round_to_iemre_grid(lat, lon) do - {Float.round(lat * 8) / 8, Float.round(lon * 8) / 8} - end - @spec upsert_iemre_observation(map()) :: {:ok, IemreObservation.t()} | {:error, Ecto.Changeset.t()} def upsert_iemre_observation(attrs) do %IemreObservation{} @@ -1989,36 +1693,4 @@ defmodule Microwaveprop.Weather do |> where([i], i.lat == ^lat and i.lon == ^lon and i.date == ^date) |> Repo.exists?() end - - @spec iemre_for_contact(map()) :: IemreObservation.t() | nil - def iemre_for_contact(%{pos1: nil}), do: nil - - def iemre_for_contact(contact) do - lat = contact.pos1["lat"] - lon = contact.pos1["lon"] - - if lat && lon do - find_nearest_iemre(lat, lon, contact.qso_timestamp) - end - end - - @spec find_nearest_iemre(float(), float(), DateTime.t()) :: IemreObservation.t() | nil - def find_nearest_iemre(lat, lon, timestamp) do - {rlat, rlon} = round_to_iemre_grid(lat, lon) - date = DateTime.to_date(timestamp) - - IemreObservation - |> where([i], i.lat == ^rlat and i.lon == ^rlon and i.date == ^date) - |> Repo.one() - end - - @spec iemre_for_path(map()) :: [IemreObservation.t()] - def iemre_for_path(%{pos1: nil}), do: [] - - def iemre_for_path(contact) do - contact - |> Radio.contact_path_points() - |> Enum.map(fn {lat, lon} -> find_nearest_iemre(lat, lon, contact.qso_timestamp) end) - |> Enum.reject(&is_nil/1) - end end diff --git a/lib/microwaveprop/weather/profile_lookup.ex b/lib/microwaveprop/weather/profile_lookup.ex new file mode 100644 index 00000000..6114fbb1 --- /dev/null +++ b/lib/microwaveprop/weather/profile_lookup.ex @@ -0,0 +1,428 @@ +defmodule Microwaveprop.Weather.ProfileLookup do + @moduledoc """ + Profile-lookup functions extracted from `Microwaveprop.Weather` to reduce the + size of the main module. Each function is accessed via `defdelegate` on + `Microwaveprop.Weather`, so all existing callers continue to work unchanged. + """ + + import Ecto.Query + + alias Microwaveprop.Radio + alias Microwaveprop.Repo + alias Microwaveprop.Weather.HrrrClient + alias Microwaveprop.Weather.HrrrNativeProfile + alias Microwaveprop.Weather.HrrrProfile + alias Microwaveprop.Weather.IemreObservation + alias Microwaveprop.Weather.NarrProfile + + require Logger + + @spec has_hrrr_profile?(float(), float(), DateTime.t()) :: boolean() + def has_hrrr_profile?(lat, lon, valid_time) do + dlat = 0.07 + dlon = 0.07 + + HrrrProfile + |> where( + [h], + h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and + h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and + h.valid_time == ^valid_time + ) + |> Repo.exists?() + end + + @doc """ + Batch version of `has_hrrr_profile?/3`. Accepts a list of + `{{lat, lon}, valid_time}` tuples and returns a `MapSet` of the + tuples that have at least one matching HRRR profile. A single query + covers the union bounding box of all points — dramatically cheaper + than N individual `EXISTS` queries when checking hundreds of contacts. + """ + @spec hrrr_points_present_batch([{{float(), float()}, DateTime.t()}]) :: MapSet.t() + def hrrr_points_present_batch([]), do: MapSet.new() + + def hrrr_points_present_batch(points) do + d = 0.07 + + {lat_lons, times} = Enum.unzip(points) + {lats, lons} = Enum.unzip(lat_lons) + + lat_min = Enum.min(lats) - d + lat_max = Enum.max(lats) + d + lon_min = Enum.min(lons) - d + lon_max = Enum.max(lons) + d + time_min = Enum.min_by(times, &DateTime.to_unix/1) + time_max = Enum.max_by(times, &DateTime.to_unix/1) + + profiles = + Repo.all( + from(h in HrrrProfile, + where: + h.lat >= ^lat_min and h.lat <= ^lat_max and + h.lon >= ^lon_min and h.lon <= ^lon_max and + h.valid_time >= ^time_min and h.valid_time <= ^time_max, + select: {h.lat, h.lon, h.valid_time} + ) + ) + + point_set = for {lat, lon, vt} <- profiles, into: MapSet.new(), do: {lat, lon, vt} + + points + |> MapSet.new(fn {{lat, lon}, valid_time} -> + profiles + |> Enum.any?(fn {pl, pn, pvt} -> + abs(pl - lat) <= d and abs(pn - lon) <= d and pvt == valid_time + end) + |> then(fn found -> if found, do: {{lat, lon}, valid_time} end) + end) + |> Enum.reject(&is_nil/1) + |> MapSet.new() + end + + @doc """ + Returns `true` when every path point for `contact` (pos1 → midpoint → pos2, + or just pos1 for one-endpoint contacts) already has an HRRR profile at the + nearest HRRR hour. Lets the enrichment enqueuer skip :queued → :queued + churn when the backing data is already present. + + Returns `false` if `pos1` or `qso_timestamp` is nil — a contact with no + position or no timestamp can't be looked up. + """ + @spec hrrr_data_fully_present?(term()) :: boolean() + def hrrr_data_fully_present?(%{pos1: nil}), do: false + def hrrr_data_fully_present?(%{qso_timestamp: nil}), do: false + + def hrrr_data_fully_present?(contact) do + rounded = HrrrClient.nearest_hrrr_hour(contact.qso_timestamp) + + case Radio.contact_path_points(contact) do + [] -> + false + + points -> + Enum.all?(points, fn {lat, lon} -> + {rlat, rlon} = round_to_hrrr_grid(lat, lon) + has_hrrr_profile?(rlat, rlon, rounded) + end) + end + end + + @spec hrrr_for_contact(map()) :: HrrrProfile.t() | nil + def hrrr_for_contact(%{pos1: nil}), do: nil + + def hrrr_for_contact(contact) do + lat = contact.pos1["lat"] + lon = contact.pos1["lon"] + + if lat && lon do + find_nearest_hrrr(lat, lon, contact.qso_timestamp) + end + end + + @doc """ + Nearest sounding to (`lat`, `lon`) within `radius_km` km and a ±3-hour + window around `timestamp`. Joins `weather_stations` to `soundings` so + the caller gets the raw sounding row (station_id set, derived duct + fields populated) back. + + Returns `{:ok, sounding}` or `{:error, :not_found}`. Use this in the + path calculator to surface the nearest RAOB's `ducting_detected` flag + as an independent check on HRRR's pressure-level duct signal, which + under-reads thin surface ducts. + """ + @spec find_nearest_hrrr(float(), float(), DateTime.t()) :: HrrrProfile.t() | nil + def find_nearest_hrrr(lat, lon, timestamp) do + dlat = 0.07 + dlon = 0.07 + time_start = DateTime.add(timestamp, -3600, :second) + time_end = DateTime.add(timestamp, 3600, :second) + + HrrrProfile + |> where( + [h], + h.lat >= ^(lat - dlat) and h.lat <= ^(lat + dlat) and + h.lon >= ^(lon - dlon) and h.lon <= ^(lon + dlon) and + h.valid_time >= ^time_start and h.valid_time <= ^time_end + ) + |> order_by([h], + asc: + fragment( + "ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))", + h.lat, + ^lat, + h.lon, + ^lon, + h.valid_time, + ^timestamp + ) + ) + |> limit(1) + |> Repo.one() + end + + @spec hrrr_profiles_for_path(map()) :: [HrrrProfile.t()] + def hrrr_profiles_for_path(%{pos1: nil}), do: [] + + def hrrr_profiles_for_path(contact) do + contact + |> Radio.contact_path_points() + |> Enum.map(fn {lat, lon} -> find_nearest_hrrr(lat, lon, contact.qso_timestamp) end) + |> Enum.reject(&is_nil/1) + end + + @doc """ + Batch version of `hrrr_profiles_for_path/1`. Queries all HRRR profiles + for a list of contacts in a single DB round-trip, then maps results + back per-contact. Returns a list of `{contact, [profile]}` tuples. + """ + @spec hrrr_profiles_for_contacts([map()]) :: [{map(), [HrrrProfile.t()]}] + def hrrr_profiles_for_contacts(contacts) do + for_result = + for c <- contacts, c.pos1 != nil, {lat, lon} <- Radio.contact_path_points(c) do + {{lat, lon, c.qso_timestamp}, c} + end + + points = + Enum.uniq_by(for_result, fn {key, _} -> key end) + + {keys, contact_map} = + Enum.reduce(points, {[], %{}}, fn {{lat, lon, ts} = key, c}, {ks, cm} -> + {[key | ks], Map.update(cm, c, [key], &[key | &1])} + end) + + index = find_nearest_hrrrs_batch(keys) + + Enum.map(contacts, fn c -> + point_keys = Map.get(contact_map, c, []) + profiles = point_keys |> Enum.map(&Map.get(index, &1)) |> Enum.reject(&is_nil/1) + {c, profiles} + end) + end + + # Batch nearest-HRRR lookup: query all profiles in the union bounding + # box once, then match each point to its nearest profile in Elixir. + @spec find_nearest_hrrrs_batch([{float(), float(), DateTime.t()}]) :: %{ + {float(), float(), DateTime.t()} => HrrrProfile.t() + } + defp find_nearest_hrrrs_batch([]), do: %{} + + defp find_nearest_hrrrs_batch(points) do + d = 0.07 + margin_s = 3600 + + {lats, lons, times} = Enum.unzip(points) + lat_min = Enum.min(lats) - d + lat_max = Enum.max(lats) + d + lon_min = Enum.min(lons) - d + lon_max = Enum.max(lons) + d + time_min = times |> Enum.min_by(&DateTime.to_unix/1) |> DateTime.add(-margin_s, :second) + time_max = times |> Enum.max_by(&DateTime.to_unix/1) |> DateTime.add(margin_s, :second) + + profiles = + Repo.all( + from(h in HrrrProfile, + where: + h.lat >= ^lat_min and h.lat <= ^lat_max and + h.lon >= ^lon_min and h.lon <= ^lon_max and + h.valid_time >= ^time_min and h.valid_time <= ^time_max, + select: %{lat: h.lat, lon: h.lon, valid_time: h.valid_time, profile: h} + ) + ) + + Map.new(points, fn {lat, lon, ts} -> + nearest = + Enum.min_by( + profiles, + fn p -> + abs(p.lat - lat) + abs(p.lon - lon) + abs(DateTime.diff(p.valid_time, ts)) + end, + fn -> nil end + ) + + {{lat, lon, ts}, nearest && nearest.profile} + end) + end + + @spec find_nearest_native_profile(float(), float(), DateTime.t()) :: + HrrrNativeProfile.t() | nil + def find_nearest_native_profile(lat, lon, timestamp) do + dlat = 0.07 + dlon = 0.07 + time_start = DateTime.add(timestamp, -3600, :second) + time_end = DateTime.add(timestamp, 3600, :second) + + HrrrNativeProfile + |> where( + [n], + n.lat >= ^(lat - dlat) and n.lat <= ^(lat + dlat) and + n.lon >= ^(lon - dlon) and n.lon <= ^(lon + dlon) and + n.valid_time >= ^time_start and n.valid_time <= ^time_end + ) + |> order_by([n], + asc: + fragment( + "ABS(? - ?) + ABS(? - ?) + ABS(EXTRACT(EPOCH FROM ? - ?))", + n.lat, + ^lat, + n.lon, + ^lon, + n.valid_time, + ^timestamp + ) + ) + |> limit(1) + |> Repo.one() + end + + @doc """ + Find the best available atmospheric profile for a contact. + Tries HRRR first (3 km, hourly), falls back to NARR (32 km, 3-hourly). + Returns the profile struct or nil. + """ + @spec best_profile_for_contact(map()) :: HrrrProfile.t() | NarrProfile.t() | nil + def best_profile_for_contact(contact) do + hrrr_for_contact(contact) || narr_for_contact(contact) + end + + @spec narr_for_contact(map()) :: NarrProfile.t() | nil + def narr_for_contact(%{pos1: nil}), do: nil + + def narr_for_contact(contact) do + lat = contact.pos1["lat"] + lon = contact.pos1["lon"] + + if lat && lon do + find_nearest_narr(lat, lon, contact.qso_timestamp) + end + end + + @spec narr_profiles_for_path(map()) :: [NarrProfile.t()] + def narr_profiles_for_path(%{pos1: nil}), do: [] + + def narr_profiles_for_path(contact) do + contact + |> Radio.contact_path_points() + |> Enum.map(fn {lat, lon} -> find_nearest_narr(lat, lon, contact.qso_timestamp) end) + |> Enum.reject(&is_nil/1) + end + + @spec find_nearest_narr(float(), float(), DateTime.t()) :: NarrProfile.t() | nil + def find_nearest_narr(lat, lon, timestamp) do + dlat = 0.15 + dlon = 0.15 + time_start = DateTime.add(timestamp, -1800, :second) + time_end = DateTime.add(timestamp, 1800, :second) + + NarrProfile + |> where( + [p], + p.lat >= ^(lat - dlat) and p.lat <= ^(lat + dlat) and + p.lon >= ^(lon - dlon) and p.lon <= ^(lon + dlon) and + p.valid_time >= ^time_start and p.valid_time <= ^time_end + ) + |> order_by([p], + asc: + fragment( + "ABS(? - ?) + ABS(? - ?)", + p.lat, + ^lat, + p.lon, + ^lon + ) + ) + |> limit(1) + |> Repo.one() + end + + @spec round_to_hrrr_grid(float(), float()) :: {float(), float()} + def round_to_hrrr_grid(lat, lon) do + {Float.round(lat / 1.0, 2), Float.round(lon / 1.0, 2)} + end + + @doc """ + Delete every `is_grid_point = true` row from `hrrr_profiles`, regardless of + age. Grid-point profiles are historical — the propagation grid now lives in + `/data/scores` binary files, nothing reads `is_grid_point = true` rows + anymore, and forecast runs no longer write them. This purge walks each + partition directly so a single DELETE can't scan the whole parent table. + Preserves QSO-linked rows (`is_grid_point = false`), which remain the data + path for contact enrichment and the `/path` calculator. + """ + @spec purge_grid_point_profiles() :: non_neg_integer() + def purge_grid_point_profiles do + deleted = + Enum.reduce(hrrr_profile_partitions(), 0, fn partition, acc -> + %{num_rows: n} = + Repo.query!( + ~s(DELETE FROM "#{partition}" WHERE is_grid_point = true), + [], + timeout: 600_000 + ) + + if n > 0 do + Logger.info("Purged #{n} grid-point rows from #{partition}") + end + + acc + n + end) + + deleted + end + + @spec hrrr_profile_partitions() :: [String.t()] + defp hrrr_profile_partitions do + {:ok, %{rows: rows}} = + Repo.query( + """ + SELECT child.relname + FROM pg_inherits + JOIN pg_class parent ON pg_inherits.inhparent = parent.oid + JOIN pg_class child ON pg_inherits.inhrelid = child.oid + WHERE parent.relname = 'hrrr_profiles' + ORDER BY child.relname + """, + [], + timeout: 60_000 + ) + + Enum.map(rows, fn [name] -> name end) + end + + @spec round_to_iemre_grid(float(), float()) :: {float(), float()} + def round_to_iemre_grid(lat, lon) do + {Float.round(lat * 8) / 8, Float.round(lon * 8) / 8} + end + + @spec iemre_for_contact(map()) :: IemreObservation.t() | nil + def iemre_for_contact(%{pos1: nil}), do: nil + + def iemre_for_contact(contact) do + lat = contact.pos1["lat"] + lon = contact.pos1["lon"] + + if lat && lon do + find_nearest_iemre(lat, lon, contact.qso_timestamp) + end + end + + @spec find_nearest_iemre(float(), float(), DateTime.t()) :: IemreObservation.t() | nil + def find_nearest_iemre(lat, lon, timestamp) do + {rlat, rlon} = round_to_iemre_grid(lat, lon) + date = DateTime.to_date(timestamp) + + IemreObservation + |> where([i], i.lat == ^rlat and i.lon == ^rlon and i.date == ^date) + |> Repo.one() + end + + @spec iemre_for_path(map()) :: [IemreObservation.t()] + def iemre_for_path(%{pos1: nil}), do: [] + + def iemre_for_path(contact) do + contact + |> Radio.contact_path_points() + |> Enum.map(fn {lat, lon} -> find_nearest_iemre(lat, lon, contact.qso_timestamp) end) + |> Enum.reject(&is_nil/1) + end +end diff --git a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex index ad12a584..7e0c12cc 100644 --- a/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex +++ b/lib/microwaveprop/workers/contact_weather_enqueue_worker.ex @@ -98,17 +98,49 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do # sees 0 missing points, inserts nothing, but the status stays # :queued and the contact is picked up again next tick). defp hrrr_placeholder_jobs(contacts) do + # Collect all unique HRRR grid points across all contacts, query + # presence once, then map back per-contact — avoids N+1 per-contact + # queries through hrrr_data_fully_present?/1. + {point_map, present_set} = build_hrrr_present_set(contacts) + Enum.flat_map(contacts, fn contact -> cond do - is_nil(contact.pos1) -> [] - not Grid.contains?(contact.pos1) -> [] - NarrClient.in_coverage?(contact.qso_timestamp) -> [] - Weather.hrrr_data_fully_present?(contact) -> [] - true -> [contact.id] + is_nil(contact.pos1) -> + [] + + not Grid.contains?(contact.pos1) -> + [] + + NarrClient.in_coverage?(contact.qso_timestamp) -> + [] + + true -> + points = Map.get(point_map, contact.id, []) + if Enum.all?(points, &MapSet.member?(present_set, &1)), do: [], else: [contact.id] end end) end + defp build_hrrr_present_set(contacts) do + contacts + |> Enum.filter(&(&1.pos1 != nil and not NarrClient.in_coverage?(&1.qso_timestamp))) + |> Enum.reduce({%{}, []}, fn c, {pm, pts} -> + rounded = HrrrClient.nearest_hrrr_hour(c.qso_timestamp) + + points = + c + |> Radio.contact_path_points() + |> Enum.map(fn {lat, lon} -> {Weather.round_to_hrrr_grid(lat, lon), rounded} end) + + {Map.put(pm, c.id, points), points ++ pts} + end) + |> then(fn {pm, all_pts} -> + unique = Enum.uniq(all_pts) + present = Weather.hrrr_points_present_batch(unique) + {pm, present} + end) + end + defp mark_enrichment_statuses(contact, types, jobs_by_type) do ids = [contact.id] diff --git a/lib/microwaveprop_web/live/weather_ca_map_live.ex b/lib/microwaveprop_web/live/weather_ca_map_live.ex index 48b3f0d3..0b1f9662 100644 --- a/lib/microwaveprop_web/live/weather_ca_map_live.ex +++ b/lib/microwaveprop_web/live/weather_ca_map_live.ex @@ -9,6 +9,7 @@ defmodule MicrowavepropWeb.WeatherCaMapLive do alias Microwaveprop.Weather alias Microwaveprop.Weather.MapLayers + alias MicrowavepropWeb.WeatherMapComponent @layers MapLayers.all() @default_layer MapLayers.default_id() @@ -51,7 +52,9 @@ defmodule MicrowavepropWeb.WeatherCaMapLive do current_lon: lon, current_zoom: zoom, grid_visible: false, - radar_visible: false + radar_visible: false, + band_filter: nil, + cutoff_bands: [] )} end @@ -243,358 +246,18 @@ defmodule MicrowavepropWeb.WeatherCaMapLive do end end - @group_order ["Surface", "Upper Air", "Ducting"] - - defp group_layers(layers) do - layers - |> Enum.group_by(& &1.group) - |> Enum.sort_by(fn {group, _} -> Enum.find_index(@group_order, &(&1 == group)) || 99 end) - end - - defp layer_description(layers, selected_id) do - case Enum.find(layers, &(&1.id == selected_id)) do - %{desc: desc} -> desc - _ -> nil - end - end - - defp selected_layer_label(layers, selected_id) do - case Enum.find(layers, &(&1.id == selected_id)) do - %{label: label} -> label - _ -> "" - end - end - - defp selected_layer_group_label(layers, selected_id) do - case Enum.find(layers, &(&1.id == selected_id)) do - %{group: group} -> group - _ -> "" - end - end - @impl true def render(assigns) do - ~H""" -
-
-
-
+ assigns = + assign(assigns, + page_subtitle: "Canadian Weather Map", + data_source: "hrdps", + show_cutoff_bands: false, + show_other_weather_link: true, + other_weather_path: "/weather", + other_weather_label: "US Weather Map" + ) -
-
- -
-
-
-
- NTMS -
Canadian Weather Map
-
- <%= if @valid_time do %> - {Calendar.strftime(@valid_time, "%Y-%m-%d %H:%M UTC")} - <% else %> - No data available - <% end %> -
-
-
-
- {@initial_utc_clock} -
- -
-
- - -
-
- - - - -
- - -
- - - """ + WeatherMapComponent.render(assigns) end end diff --git a/lib/microwaveprop_web/live/weather_map_component.ex b/lib/microwaveprop_web/live/weather_map_component.ex new file mode 100644 index 00000000..ec6e8736 --- /dev/null +++ b/lib/microwaveprop_web/live/weather_map_component.ex @@ -0,0 +1,428 @@ +defmodule MicrowavepropWeb.WeatherMapComponent do + @moduledoc false + use MicrowavepropWeb, :html + + @group_order ["Surface", "Upper Air", "Ducting"] + + attr :bands, :list, required: true + attr :selected, :any, default: nil + + defp cutoff_band_picker(assigns) do + ~H""" +
+
+ Filter to band +
+
+ + +
+
+ """ + end + + defp group_layers(layers) do + layers + |> Enum.group_by(& &1.group) + |> Enum.sort_by(fn {group, _} -> Enum.find_index(@group_order, &(&1 == group)) || 99 end) + end + + defp layer_description(layers, selected_id) do + case Enum.find(layers, &(&1.id == selected_id)) do + %{desc: desc} -> desc + _ -> nil + end + end + + defp selected_layer_label(layers, selected_id) do + case Enum.find(layers, &(&1.id == selected_id)) do + %{label: label} -> label + _ -> "" + end + end + + defp selected_layer_group_label(layers, selected_id) do + case Enum.find(layers, &(&1.id == selected_id)) do + %{group: group} -> group + _ -> "" + end + end + + @doc false + def render(assigns) do + ~H""" +
+ <%!-- Map container --%> +
+ <%!-- Full-page map --%> +
+
+ + <%!-- Bottom forecast timeline --%> +
+
+ + <%!-- Mobile-only floating controls --%> +
+
+
+
+ NTMS +
{@page_subtitle}
+
+ <%= if @valid_time do %> + {Calendar.strftime(@valid_time, "%Y-%m-%d %H:%M UTC")} + <% else %> + No data available + <% end %> +
+
+
+
+ {@initial_utc_clock} +
+ +
+
+ + +
+
+ + <%!-- Point detail panel --%> + + + <%!-- Sidebar expand button (visible when sidebar is collapsed) --%> + +
+ + <%!-- Desktop right sidebar --%> + +
+ + + """ + end +end diff --git a/lib/microwaveprop_web/live/weather_map_live.ex b/lib/microwaveprop_web/live/weather_map_live.ex index 9d2d5180..09084b28 100644 --- a/lib/microwaveprop_web/live/weather_map_live.ex +++ b/lib/microwaveprop_web/live/weather_map_live.ex @@ -4,6 +4,7 @@ defmodule MicrowavepropWeb.WeatherMapLive do alias Microwaveprop.Weather alias Microwaveprop.Weather.MapLayers + alias MicrowavepropWeb.WeatherMapComponent @layers MapLayers.all() @default_layer MapLayers.default_id() @@ -349,412 +350,19 @@ defmodule MicrowavepropWeb.WeatherMapLive do end end - @group_order ["Surface", "Upper Air", "Ducting"] - - defp group_layers(layers) do - layers - |> Enum.group_by(& &1.group) - |> Enum.sort_by(fn {group, _} -> Enum.find_index(@group_order, &(&1 == group)) || 99 end) - end - - defp layer_description(layers, selected_id) do - case Enum.find(layers, &(&1.id == selected_id)) do - %{desc: desc} -> desc - _ -> nil - end - end - - defp selected_layer_label(layers, selected_id) do - case Enum.find(layers, &(&1.id == selected_id)) do - %{label: label} -> label - _ -> "" - end - end - - defp selected_layer_group_label(layers, selected_id) do - case Enum.find(layers, &(&1.id == selected_id)) do - %{group: group} -> group - _ -> "" - end - end - - attr :bands, :list, required: true - attr :selected, :any, default: nil - - defp cutoff_band_picker(assigns) do - ~H""" -
-
- Filter to band -
-
- - -
-
- """ - end - @impl true def render(assigns) do - ~H""" -
- <%!-- Map container --%> -
- <%!-- Full-page map --%> -
-
+ assigns = + assign(assigns, + page_subtitle: "Weather Map", + data_source: nil, + cutoff_bands: @cutoff_bands, + show_cutoff_bands: true, + show_other_weather_link: false, + other_weather_path: "/weather-ca", + other_weather_label: "Canadian Weather Map" + ) - <%!-- Bottom forecast timeline --%> -
-
- - <%!-- Mobile-only floating controls --%> -
-
-
-
- NTMS -
Weather Map
-
- <%= if @valid_time do %> - {Calendar.strftime(@valid_time, "%Y-%m-%d %H:%M UTC")} - <% else %> - No data available - <% end %> -
-
-
-
- {@initial_utc_clock} -
- -
-
- - -
-
- - <%!-- Point detail panel --%> - - - <%!-- Sidebar expand button (visible when sidebar is collapsed) --%> - -
- - <%!-- Desktop right sidebar --%> - -
- - - """ + WeatherMapComponent.render(assigns) end end