228 lines
9.9 KiB
Elixir
228 lines
9.9 KiB
Elixir
defmodule MicrowavepropWeb.AboutLive do
|
||
@moduledoc false
|
||
use MicrowavepropWeb, :live_view
|
||
|
||
alias Microwaveprop.Repo
|
||
|
||
@impl true
|
||
def mount(_params, _session, socket) do
|
||
{:ok,
|
||
socket
|
||
|> assign(:page_title, "About")
|
||
|> assign(:stats, fetch_stats())}
|
||
end
|
||
|
||
@impl true
|
||
def render(assigns) do
|
||
~H"""
|
||
<Layouts.app flash={@flash} current_scope={@current_scope} max_width="max-w-4xl">
|
||
<.header>
|
||
About NTMS Propagation Prediction
|
||
<:subtitle>What we're building and why.</:subtitle>
|
||
</.header>
|
||
|
||
<section class="space-y-8 text-sm leading-relaxed">
|
||
<div class="space-y-3">
|
||
<h2 class="text-xl font-bold">What we're trying to do</h2>
|
||
<p>
|
||
Microwave propagation above 10 GHz is dominated by the lower atmosphere:
|
||
humidity gradients, temperature inversions, ducting layers, rain cells,
|
||
and hyper-local refractivity structure. At 10, 24, 47, 76, 122, and 241 GHz
|
||
these effects decide whether a contact happens at 50 km or 500 km. This project was
|
||
started by Jim KM5PO, then picked up by Graham W5ISP.
|
||
</p>
|
||
<p>
|
||
Traditionally microwave contacts are ruled by line of sight, but we've
|
||
seen that is incorrect. This project is an attempt to build a data-driven
|
||
propagation prediction model specifically for the amateur microwave bands,
|
||
using hourly numerical weather forecasts, historical contact records, and
|
||
eventually calibrated beacon measurements.
|
||
</p>
|
||
</div>
|
||
|
||
<div class="space-y-3">
|
||
<h2 class="text-xl font-bold">The approach</h2>
|
||
<p>We pair two things that most propagation tools keep separate:</p>
|
||
<ol class="list-decimal list-outside pl-5 space-y-2">
|
||
<li>
|
||
<strong>Atmospheric state</strong> — hourly 3 km HRRR forecasts
|
||
(surface fields plus pressure-level profiles), hourly surface
|
||
observations from ASOS, 12-hourly upper-air soundings, and gridded
|
||
IEMRE reanalysis. From these we derive refractivity profiles,
|
||
ducting detection, minimum refractivity gradient, precipitable water,
|
||
boundary-layer depth, and a 9-factor composite score for every
|
||
0.125° cell on a CONUS grid, for each hour of an 18-hour forecast.
|
||
</li>
|
||
<li>
|
||
<strong>Historical contacts</strong> — 58k+ amateur microwave QSOs
|
||
tagged with the atmospheric conditions at both ends and along the
|
||
path at the exact time they happened. That gives us a ground-truth
|
||
dataset we can use to calibrate the scoring weights and, eventually,
|
||
train a machine learning model that predicts contact success given
|
||
conditions.
|
||
</li>
|
||
</ol>
|
||
<p>
|
||
The current scoring algorithm is a weighted sum of nine factors
|
||
(humidity, time of day, T–Td depression, refractivity gradient, sky
|
||
cover, season, rain, wind, pressure) with band-dependent weights —
|
||
humidity helps at 10 GHz via enhanced refractivity but hurts at
|
||
24+ GHz via absorption, for example. Every coefficient in that
|
||
formula is a hypothesis waiting to be tested against the contact
|
||
and beacon data.
|
||
</p>
|
||
</div>
|
||
|
||
<div class="space-y-3">
|
||
<h2 class="text-xl font-bold">What we've collected</h2>
|
||
<p>Live counts from the production database:</p>
|
||
<div class="grid grid-cols-2 md:grid-cols-3 gap-3">
|
||
<.stat_card label="Contacts" value={@stats.contacts} />
|
||
<.stat_card label="Weather stations" value={@stats.weather_stations} />
|
||
<.stat_card label="Surface observations" value={@stats.surface_observations} />
|
||
<.stat_card label="Upper-air soundings" value={@stats.soundings} />
|
||
<.stat_card label="HRRR profiles" value={@stats.hrrr_profiles} />
|
||
<.stat_card label="IEMRE gridded obs" value={@stats.iemre_observations} />
|
||
<.stat_card label="Terrain profiles" value={@stats.terrain_profiles} />
|
||
<.stat_card label="Propagation scores" value={@stats.propagation_scores} />
|
||
<.stat_card label="Beacons" value={@stats.beacons} />
|
||
<.stat_card label="Commercial link samples" value={@stats.commercial_samples} />
|
||
</div>
|
||
</div>
|
||
|
||
<div class="space-y-3">
|
||
<h2 class="text-xl font-bold">The stack</h2>
|
||
<ul class="list-disc list-outside pl-5 space-y-2">
|
||
<li>
|
||
<strong>Backend:</strong> Elixir / Phoenix 1.8 with LiveView for the
|
||
real-time map, Ecto on PostgreSQL for storage, Oban for the
|
||
background data pipelines (HRRR fetch, terrain, ASOS, soundings,
|
||
IEMRE, solar indices, commercial links), Bandit as the HTTP server.
|
||
</li>
|
||
<li>
|
||
<strong>Frontend:</strong> LiveView with Leaflet for maps, Canvas
|
||
tile layers for the HRRR heatmap (we render 0.125° cells at
|
||
interactive frame rates), Tailwind v4 + daisyUI for layout, esbuild
|
||
for JS bundling.
|
||
</li>
|
||
<li>
|
||
<strong>Physics:</strong> ITU-R P.526-16 knife-edge + Deygout
|
||
3-edge terrain diffraction, ITU-R P.838-3 rain attenuation,
|
||
dynamic k-factor from live HRRR refractivity gradients, great-circle
|
||
geometry for link budgets, Free-Space Path Loss with frequency-
|
||
dependent O₂ and H₂O absorption per band.
|
||
</li>
|
||
<li>
|
||
<strong>Machine learning:</strong> Nx / Axon / EXLA scaffolding for
|
||
a 13-feature (8 atmospheric + 4 cyclical temporal + 1 log-frequency)
|
||
feed-forward network, 64 → 32 → 1 sigmoid. Not trained yet —
|
||
waiting on a larger calibration dataset.
|
||
</li>
|
||
<li>
|
||
<strong>Data sources:</strong> NOAA HRRR model (AWS S3, hourly
|
||
analysis + 18 h forecasts), Iowa Environmental Mesonet (ASOS &
|
||
upper-air soundings), IEMRE gridded reanalysis, SRTM 90 m terrain
|
||
tiles, SNMP polling of seven commercial microwave links near DFW
|
||
at 5-minute intervals, Copernicus ERA5 for pre-2014 contact
|
||
enrichment.
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
|
||
<div class="space-y-3">
|
||
<h2 class="text-xl font-bold">Where this goes next</h2>
|
||
<ul class="list-disc list-outside pl-5 space-y-2">
|
||
<li>
|
||
<strong>Beacon calibration.</strong> A distributed network of
|
||
amateur receivers continuously reporting CW beacon signal levels,
|
||
feeding ground-truth measurements back into the scoring algorithm.
|
||
Beacon submissions are now open to anyone via the Beacons page.
|
||
</li>
|
||
<li>
|
||
<strong>Scoring weight calibration.</strong> Fit the 9-factor
|
||
weights against recorded QSO distances / counts per band so the
|
||
composite score reflects real propagation rather than intuition.
|
||
</li>
|
||
<li>
|
||
<strong>ML model training.</strong> Once we have enough labeled
|
||
contact+condition pairs, train the Axon model to replace or augment
|
||
the hand-tuned scoring function.
|
||
</li>
|
||
<li>
|
||
<strong>Better inputs.</strong> MRMS for precipitation at
|
||
24+ GHz (where rain attenuation dominates), RTMA/URMA surface
|
||
analysis blending, GOES total precipitable water.
|
||
</li>
|
||
</ul>
|
||
</div>
|
||
</section>
|
||
</Layouts.app>
|
||
"""
|
||
end
|
||
|
||
attr :label, :string, required: true
|
||
attr :value, :integer, required: true
|
||
|
||
defp stat_card(assigns) do
|
||
~H"""
|
||
<div class="rounded-lg border border-base-300 bg-base-200/40 p-3">
|
||
<div class="text-xs opacity-70 uppercase tracking-wide">{@label}</div>
|
||
<div class="text-xl font-bold tabular-nums">{format_count(@value)}</div>
|
||
</div>
|
||
"""
|
||
end
|
||
|
||
# For tables that get huge (tens of millions of rows), using count(*) on
|
||
# every About page load is wasteful. We use PostgreSQL's pg_class.reltuples
|
||
# estimate — it's maintained by ANALYZE and is accurate to within a few
|
||
# percent, which is plenty for "big number on a marketing page."
|
||
@estimate_tables ~w(hrrr_profiles propagation_scores surface_observations)
|
||
|
||
@stat_keys ~w(
|
||
contacts weather_stations surface_observations soundings hrrr_profiles
|
||
iemre_observations terrain_profiles propagation_scores beacons
|
||
commercial_samples
|
||
)a
|
||
|
||
defp fetch_stats do
|
||
%{
|
||
contacts: count_exact("contacts"),
|
||
weather_stations: count_exact("weather_stations"),
|
||
surface_observations: count_estimate("surface_observations"),
|
||
soundings: count_exact("soundings"),
|
||
hrrr_profiles: count_estimate("hrrr_profiles"),
|
||
iemre_observations: count_exact("iemre_observations"),
|
||
terrain_profiles: count_exact("terrain_profiles"),
|
||
propagation_scores: count_estimate("propagation_scores"),
|
||
beacons: count_exact("beacons"),
|
||
commercial_samples: count_exact("commercial_samples")
|
||
}
|
||
rescue
|
||
_ -> Map.new(@stat_keys, fn k -> {k, 0} end)
|
||
end
|
||
|
||
defp count_exact(table) do
|
||
%Postgrex.Result{rows: [[count]]} =
|
||
Repo.query!("SELECT count(*) FROM #{table}")
|
||
|
||
count
|
||
end
|
||
|
||
defp count_estimate(table) when table in @estimate_tables do
|
||
%Postgrex.Result{rows: [[estimate]]} =
|
||
Repo.query!("SELECT reltuples::bigint FROM pg_class WHERE relname = $1", [table])
|
||
|
||
max(estimate || 0, 0)
|
||
end
|
||
|
||
defp format_count(n) when is_integer(n) and n >= 1_000_000 do
|
||
"#{Float.round(n / 1_000_000, 1)}M"
|
||
end
|
||
|
||
defp format_count(n) when is_integer(n) and n >= 1_000 do
|
||
"#{Float.round(n / 1_000, 1)}k"
|
||
end
|
||
|
||
defp format_count(n) when is_integer(n), do: Integer.to_string(n)
|
||
defp format_count(_), do: "—"
|
||
end
|