diff --git a/assets/js/propagation_map_hook.js b/assets/js/propagation_map_hook.js index 0b6fbaf2..88ac0c00 100644 --- a/assets/js/propagation_map_hook.js +++ b/assets/js/propagation_map_hook.js @@ -353,9 +353,18 @@ function buildPopupHTML(detail, viewshedLoading) { export const PropagationMap = { mounted() { + const centerLat = parseFloat(this.el.dataset.centerLat) + const centerLon = parseFloat(this.el.dataset.centerLon) + const zoom = parseInt(this.el.dataset.zoom, 10) + + const center = + Number.isFinite(centerLat) && Number.isFinite(centerLon) + ? [centerLat, centerLon] + : [32.897, -97.038] + this.map = L.map(this.el, { - center: [32.897, -97.038], - zoom: 7, + center, + zoom: Number.isFinite(zoom) ? zoom : 7, minZoom: 4, maxZoom: 10 }) diff --git a/lib/microwaveprop_web/live/about_live.ex b/lib/microwaveprop_web/live/about_live.ex index eae3fb0e..b71c33ab 100644 --- a/lib/microwaveprop_web/live/about_live.ex +++ b/lib/microwaveprop_web/live/about_live.ex @@ -2,19 +2,223 @@ defmodule MicrowavepropWeb.AboutLive do @moduledoc false use MicrowavepropWeb, :live_view + alias Microwaveprop.Repo + @impl true def mount(_params, _session, socket) do - {:ok, assign(socket, page_title: "About")} + {:ok, + socket + |> assign(:page_title, "About") + |> assign(:stats, fetch_stats())} end @impl true def render(assigns) do ~H""" - + <.header> - About + About NTMS Propagation Prediction + <:subtitle>What we're building and why. + +
+

What we're trying to do

+

+ Microwave propagation above 10 GHz is dominated by the lower atmosphere: + humidity gradients, temperature inversions, ducting layers, rain cells, + and hyper-local refractivity structure. At 10, 24, 47, 76, 122, and 241 GHz + these effects decide whether a contact happens at 50 km or 500 km — and the + window often lasts minutes, not hours. +

+

+ Traditionally microwave contacts are ruled by line of sight, but we've seen that is incorrect. This project is an attempt to build a data-driven + propagation prediction model specifically for the amateur microwave bands, + using hourly numerical weather forecasts, historical contact records, and + eventually calibrated beacon measurements. +

+ +

The approach

+

+ We pair two things that most propagation tools keep separate: +

+
    +
  1. + Atmospheric state — hourly 3 km HRRR forecasts + (surface fields plus pressure-level profiles), hourly surface + observations from ASOS, 12-hourly upper-air soundings, and gridded + IEMRE reanalysis. From these we derive refractivity profiles, + ducting detection, minimum refractivity gradient, precipitable water, + boundary-layer depth, and a 9-factor composite score for every + 0.125° cell on a CONUS grid, for each hour of an 18-hour forecast. +
    2. +
  2. + Historical contacts — 58k+ amateur microwave QSOs + tagged with the atmospheric conditions at both ends and along the + path at the exact time they happened. That gives us a ground-truth + dataset we can use to calibrate the scoring weights and, eventually, + train a machine learning model that predicts contact success given + conditions. +
    3. +
+

+ The current scoring algorithm is a weighted sum of nine factors + (humidity, time of day, T–Td depression, refractivity gradient, sky + cover, season, rain, wind, pressure) with band-dependent weights — + humidity helps at 10 GHz via enhanced refractivity but hurts at + 24+ GHz via absorption, for example. Every coefficient in that + formula is a hypothesis waiting to be tested against the contact + and beacon data. +

+ +

What we've collected

+

Live counts from the production database:

+ +
+ <.stat_card label="Contacts" value={@stats.contacts} /> + <.stat_card label="Weather stations" value={@stats.weather_stations} /> + <.stat_card label="Surface observations" value={@stats.surface_observations} /> + <.stat_card label="Upper-air soundings" value={@stats.soundings} /> + <.stat_card label="HRRR profiles" value={@stats.hrrr_profiles} /> + <.stat_card label="IEMRE gridded obs" value={@stats.iemre_observations} /> + <.stat_card label="Terrain profiles" value={@stats.terrain_profiles} /> + <.stat_card label="Propagation scores" value={@stats.propagation_scores} /> + <.stat_card label="Beacons" value={@stats.beacons} /> + <.stat_card label="Commercial link samples" value={@stats.commercial_samples} /> +
+ +

The stack

+
    +
  • + Backend: Elixir / Phoenix 1.8 with LiveView for the + real-time map, Ecto on PostgreSQL for storage, Oban for the + background data pipelines (HRRR fetch, terrain, ASOS, soundings, + IEMRE, solar indices, commercial links), Bandit as the HTTP server. +
    • +
  • + Frontend: LiveView with Leaflet for maps, Canvas + tile layers for the HRRR heatmap (we render 0.125° cells at + interactive frame rates), Tailwind v4 + daisyUI for layout, esbuild + for JS bundling. +
    • +
  • + Physics: ITU-R P.526-16 knife-edge + Deygout + 3-edge terrain diffraction, ITU-R P.838-3 rain attenuation, + dynamic k-factor from live HRRR refractivity gradients, great-circle + geometry for link budgets, Free-Space Path Loss with frequency- + dependent O₂ and H₂O absorption per band. +
    • +
  • + Machine learning: Nx / Axon / EXLA scaffolding for + a 13-feature (8 atmospheric + 4 cyclical temporal + 1 log-frequency) + feed-forward network, 64 → 32 → 1 sigmoid. Not trained yet — + waiting on a larger calibration dataset. +
    • +
  • + Data sources: NOAA HRRR model (AWS S3, hourly + analysis + 18 h forecasts), Iowa Environmental Mesonet (ASOS & + upper-air soundings), IEMRE gridded reanalysis, SRTM 90 m terrain + tiles, SNMP polling of seven commercial microwave links near DFW + at 5-minute intervals, Copernicus ERA5 for pre-2014 contact + enrichment. +
    • +
+ +

Where this goes next

+
    +
  • + Beacon calibration. A distributed network of + amateur receivers continuously reporting CW beacon signal levels, + feeding ground-truth measurements back into the scoring algorithm. + Beacon submissions are now open to anyone via the Beacons page. +
    • +
  • + Scoring weight calibration. Fit the 9-factor + weights against recorded QSO distances / counts per band so the + composite score reflects real propagation rather than intuition. +
    • +
  • + ML model training. Once we have enough labeled + contact+condition pairs, train the Axon model to replace or augment + the hand-tuned scoring function. +
    • +
  • + Better inputs. MRMS for precipitation at + 24+ GHz (where rain attenuation dominates), RTMA/URMA surface + analysis blending, GOES total precipitable water. +
    • +
+ +

+ Built by and for the North Texas Microwave Society. +

+
""" end + + attr :label, :string, required: true + attr :value, :integer, required: true + + defp stat_card(assigns) do + ~H""" +
+
{@label}
+
{format_count(@value)}
+
+ """ + end + + # For tables that get huge (tens of millions of rows), using count(*) on + # every About page load is wasteful. We use PostgreSQL's pg_class.reltuples + # estimate — it's maintained by ANALYZE and is accurate to within a few + # percent, which is plenty for "big number on a marketing page." + @estimate_tables ~w(hrrr_profiles propagation_scores surface_observations) + + @stat_keys ~w( + contacts weather_stations surface_observations soundings hrrr_profiles + iemre_observations terrain_profiles propagation_scores beacons + commercial_samples + )a + + defp fetch_stats do + %{ + contacts: count_exact("contacts"), + weather_stations: count_exact("weather_stations"), + surface_observations: count_estimate("surface_observations"), + soundings: count_exact("soundings"), + hrrr_profiles: count_estimate("hrrr_profiles"), + iemre_observations: count_exact("iemre_observations"), + terrain_profiles: count_exact("terrain_profiles"), + propagation_scores: count_estimate("propagation_scores"), + beacons: count_exact("beacons"), + commercial_samples: count_exact("commercial_samples") + } + rescue + _ -> Map.new(@stat_keys, fn k -> {k, 0} end) + end + + defp count_exact(table) do + %Postgrex.Result{rows: [[count]]} = + Repo.query!("SELECT count(*) FROM #{table}") + + count + end + + defp count_estimate(table) when table in @estimate_tables do + %Postgrex.Result{rows: [[estimate]]} = + Repo.query!("SELECT reltuples::bigint FROM pg_class WHERE relname = $1", [table]) + + max(estimate || 0, 0) + end + + defp format_count(n) when is_integer(n) and n >= 1_000_000 do + "#{Float.round(n / 1_000_000, 1)}M" + end + + defp format_count(n) when is_integer(n) and n >= 1_000 do + "#{Float.round(n / 1_000, 1)}k" + end + + defp format_count(n) when is_integer(n), do: Integer.to_string(n) + defp format_count(_), do: "—" end diff --git a/lib/microwaveprop_web/live/map_live.ex b/lib/microwaveprop_web/live/map_live.ex index 87f92397..92d9c76c 100644 --- a/lib/microwaveprop_web/live/map_live.ex +++ b/lib/microwaveprop_web/live/map_live.ex @@ -10,15 +10,12 @@ defmodule MicrowavepropWeb.MapLive do require Logger @default_band 10_000 - @initial_bounds %{ - "south" => 29.5, - "north" => 36.3, - "west" => -101.5, - "east" => -92.5 - } + # Default map center when no visitor geolocation is available — DFW (EM12). + @default_center %{lat: 32.897, lon: -97.038} + @default_zoom 7 @impl true - def mount(_params, _session, socket) do + def mount(_params, session, socket) do if connected?(socket) do Phoenix.PubSub.subscribe(Microwaveprop.PubSub, "propagation:updated") end @@ -31,23 +28,46 @@ defmodule MicrowavepropWeb.MapLive do bands = BandConfig.all_bands() valid_times = Propagation.available_valid_times(@default_band) selected_time = closest_to_now(valid_times) - initial_scores = Propagation.scores_at(@default_band, selected_time, @initial_bounds) + center = initial_center(session) + bounds = bounds_around(center) + initial_scores = Propagation.scores_at(@default_band, selected_time, bounds) {:ok, assign(socket, - page_title: "Propagation Map", + page_title: "Propagation Prediction Map", bands: bands, selected_band: @default_band, initial_scores_json: Jason.encode!(initial_scores), valid_times: valid_times, selected_time: selected_time, - bounds: @initial_bounds, + bounds: bounds, + initial_center: center, + initial_zoom: @default_zoom, grid_visible: false, antenna_height_ft: 33 )} end end + # Prefer the visitor's Cloudflare geolocation when present; fall back to DFW. + defp initial_center(%{"cf_lat" => lat, "cf_lon" => lon}) when is_float(lat) and is_float(lon) do + %{lat: lat, lon: lon} + end + + defp initial_center(_session), do: @default_center + + # Build a bounding box roughly matching the hardcoded default (~3.4° × 9°) + # around a given center so the initial HRRR score query still returns a + # useful tile set for the visible area. + defp bounds_around(%{lat: lat, lon: lon}) do + %{ + "south" => lat - 3.4, + "north" => lat + 3.4, + "west" => lon - 4.5, + "east" => lon + 4.5 + } + end + @impl true def handle_event("select_band", %{"value" => band}, socket) do band = if is_binary(band), do: String.to_integer(band), else: band @@ -274,6 +294,9 @@ defmodule MicrowavepropWeb.MapLive do ) } data-selected-time={if @selected_time, do: DateTime.to_iso8601(@selected_time), else: ""} + data-center-lat={@initial_center.lat} + data-center-lon={@initial_center.lon} + data-zoom={@initial_zoom} class="absolute inset-0 z-0" > @@ -287,7 +310,7 @@ defmodule MicrowavepropWeb.MapLive do
NTMS -
Propagation Map
+
Propagation Prediction Map
Contact Map + <.link navigate="/about" class="btn btn-xs btn-ghost justify-start"> + About +
@@ -423,7 +449,7 @@ defmodule MicrowavepropWeb.MapLive do
NTMS -
Propagation Map
+
Propagation Prediction Map
<.link navigate="/contacts/map">Contact Map +
  • + <.link navigate="/about">About +
    • <%!-- Auth --%> diff --git a/lib/microwaveprop_web/live/weather_map_live.ex b/lib/microwaveprop_web/live/weather_map_live.ex index 1705441d..efc25dfe 100644 --- a/lib/microwaveprop_web/live/weather_map_live.ex +++ b/lib/microwaveprop_web/live/weather_map_live.ex @@ -282,7 +282,7 @@ defmodule MicrowavepropWeb.WeatherMapLive do <%!-- Links --%>
    <.link navigate="/map" class="btn btn-xs btn-ghost justify-start"> - Propagation Map + Propagation Prediction Map
    diff --git a/lib/microwaveprop_web/router.ex b/lib/microwaveprop_web/router.ex index 3414831f..858beedd 100644 --- a/lib/microwaveprop_web/router.ex +++ b/lib/microwaveprop_web/router.ex @@ -8,6 +8,7 @@ defmodule MicrowavepropWeb.Router do plug :accepts, ["html"] plug :fetch_session plug :store_remote_ip + plug :store_cf_geo plug :fetch_live_flash plug :put_root_layout, html: {MicrowavepropWeb.Layouts, :root} plug :protect_from_forgery @@ -20,6 +21,23 @@ defmodule MicrowavepropWeb.Router do Plug.Conn.put_session(conn, :remote_ip, ip_str) end + # Extract Cloudflare visitor geolocation headers (set by a Cloudflare + # Managed Transform) and stash them in the session so LiveViews can use + # them to center maps on the visitor's approximate location. + defp store_cf_geo(conn, _opts) do + lat = conn |> Plug.Conn.get_req_header("cf-iplatitude") |> List.first() + lon = conn |> Plug.Conn.get_req_header("cf-iplongitude") |> List.first() + + with {lat_f, ""} when is_float(lat_f) <- lat && Float.parse(lat), + {lon_f, ""} when is_float(lon_f) <- lon && Float.parse(lon) do + conn + |> Plug.Conn.put_session(:cf_lat, lat_f) + |> Plug.Conn.put_session(:cf_lon, lon_f) + else + _ -> conn + end + end + pipeline :api do plug :accepts, ["json"] end