feat(propagation): per-contact mechanism classification
Classifies every contact's likely non-LOS propagation mechanism and persists the result on contacts.propagation_mechanism. Mechanism is determined in priority order: 1. user_declared_prop_mode (ADIF PROP_MODE from the operator log) 2. EME — moon-ephemeris check, ≥2m band, >1800 km path 3. aurora — Kp≥5 + high-lat path, 50-432 MHz 4. sporadic-E — foEs × 5 ≥ band_mhz, 400-2500 km path 5. meteor_scatter — ±3 days of a shower peak, VHF/UHF 6. rain_scatter — common-volume radar heavy rain, 5-11 GHz ≤800 km 7. tropo_duct — HRRR native_best_duct ≥ band or ducting_detected 8. line_of_sight — ≤50 km path 9. troposcatter — default Persisted via MechanismClassifyWorker (queue: :mechanism, unique on contact_id). Submit-time enqueue path includes :mechanism by default; BackfillEnqueueWorker cron now handles :mechanism alongside existing types so prod continuously backfills any contact with propagation_mechanism_status in (:pending, :queued, :failed). Also added :radar to the cron's type list so common-volume radar backfill runs automatically rather than only via `mix radar_backfill`. New modules: - Microwaveprop.Propagation.MoonEphemeris — Meeus low-precision moon position, accuracy ±1° — enough for the mutual-visibility EME test - Microwaveprop.Propagation.MechanismClassifier — plug-in priority chain over the evidence map - Microwaveprop.Workers.MechanismClassifyWorker — assembles inputs from HRRR / native profiles / common-volume radar / solar_indices / ionosonde + calls the classifier ADIF importer now reads PROP_MODE into user_declared_prop_mode so operator-tagged mechanisms (EME/ES/MS/RS/AS/AUR) become ground truth.
This commit is contained in:
parent
5cffbec165
commit
622edee180
14 changed files with 1231 additions and 17 deletions
|
|
@ -71,6 +71,7 @@ config :microwaveprop, Oban,
|
|||
radar: 2,
|
||||
ionosphere: 1,
|
||||
space_weather: 1,
|
||||
mechanism: 4,
|
||||
contact_import: 4
|
||||
],
|
||||
plugins: [
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ config :microwaveprop, Oban,
|
|||
iemre: 10,
|
||||
narr: 6,
|
||||
radar: 2,
|
||||
mechanism: 4,
|
||||
backfill_enqueue: 1,
|
||||
exports: 1,
|
||||
contact_import: 4
|
||||
|
|
|
|||
|
|
@ -182,6 +182,7 @@ if config_env() == :prod do
|
|||
radar: 2,
|
||||
ionosphere: 1,
|
||||
space_weather: 1,
|
||||
mechanism: 4,
|
||||
exports: 1,
|
||||
contact_import: 4
|
||||
],
|
||||
|
|
@ -224,7 +225,7 @@ if config_env() == :prod do
|
|||
# :unavailable (pre-2014, missing from the HRRR archive) and dispatches
|
||||
# NarrFetchWorker against NCEI. See narr_jobs_for_contact/1.
|
||||
{"*/30 * * * *", Microwaveprop.Workers.BackfillEnqueueWorker,
|
||||
args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr"]}},
|
||||
args: %{"types" => ["hrrr", "weather", "terrain", "iemre", "narr", "radar", "mechanism"]}},
|
||||
# Hourly safety net for pos1/pos2/distance_km. Normally every contact
|
||||
# gets positions at insert time via Radio.resolve_grids_and_insert/1,
|
||||
# but direct DB writes (manual fixes, bulk imports) can bypass that.
|
||||
|
|
|
|||
320
lib/microwaveprop/propagation/mechanism_classifier.ex
Normal file
320
lib/microwaveprop/propagation/mechanism_classifier.ex
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
defmodule Microwaveprop.Propagation.MechanismClassifier do
|
||||
@moduledoc """
|
||||
Classifies the propagation mechanism of an individual contact from
|
||||
whatever evidence we have: user-declared ADIF PROP_MODE (if any),
|
||||
moon ephemeris, HRRR / native-duct profile, common-volume radar,
|
||||
ionosonde foEs, Kp index, and active meteor showers.
|
||||
|
||||
Evaluates mechanisms in priority order and returns the first match.
|
||||
User-declared PROP_MODE always wins when present — operator ground
|
||||
truth beats any inference we can do post-hoc.
|
||||
|
||||
## Input map
|
||||
|
||||
%{
|
||||
band_mhz: integer(),
|
||||
distance_km: float(),
|
||||
qso_timestamp: DateTime.t(),
|
||||
pos1: %{"lat" => float(), "lon" => float()},
|
||||
pos2: %{"lat" => float(), "lon" => float()},
|
||||
user_declared_prop_mode: String.t() | nil,
|
||||
radar: nil | %{max_dbz: float() | nil, heavy_rain_pixel_count: integer(), coverage_pct: float() | nil},
|
||||
duct_either_endpoint: boolean(),
|
||||
native_best_duct_ghz: float() | nil,
|
||||
kp_index: integer() | nil,
|
||||
foes_mhz: float() | nil,
|
||||
active_meteor_shower: String.t() | nil
|
||||
}
|
||||
|
||||
## Output
|
||||
|
||||
%{mechanism: atom(), confidence: :high | :medium | :low}
|
||||
|
||||
Mechanisms (atoms) returned:
|
||||
|
||||
- `:eme` — moon bounce
|
||||
- `:aurora` — aurora scatter
|
||||
- `:sporadic_e` — sporadic-E ionospheric
|
||||
- `:meteor_scatter` — meteor trail ionization
|
||||
- `:aircraft_scatter` — bistatic radar off aircraft
|
||||
- `:rain_scatter` — precipitation scatter
|
||||
- `:rain_scatter_possible` — light rain in the common volume
|
||||
- `:tropo_duct` — surface or elevated refractivity duct
|
||||
- `:line_of_sight` — within radio horizon
|
||||
- `:troposcatter` — forward scatter off tropospheric refractive fluctuations (default)
|
||||
- `:unknown` — not enough data to classify
|
||||
"""
|
||||
|
||||
alias Microwaveprop.Propagation.MoonEphemeris
|
||||
|
||||
# ADIF PROP_MODE values we recognize. Anything else falls through to
|
||||
# physics-based classification.
|
||||
@user_declared_map %{
|
||||
"EME" => :eme,
|
||||
"ES" => :sporadic_e,
|
||||
"SPE" => :sporadic_e,
|
||||
"F2" => :f2,
|
||||
"MS" => :meteor_scatter,
|
||||
"METEOR" => :meteor_scatter,
|
||||
"RS" => :rain_scatter,
|
||||
"RAIN" => :rain_scatter,
|
||||
"AS" => :aircraft_scatter,
|
||||
"AIRCRAFT" => :aircraft_scatter,
|
||||
"AUR" => :aurora,
|
||||
"AURE" => :aurora,
|
||||
"TR" => :troposcatter,
|
||||
"TROPO" => :troposcatter,
|
||||
"LOS" => :line_of_sight
|
||||
}
|
||||
|
||||
# Band windows for each mechanism. "2 m" means practical EME floor;
|
||||
# "below 432" means we drop EME for 6 m and below.
|
||||
@eme_min_band_mhz 144
|
||||
@eme_min_distance_km 1_800.0
|
||||
|
||||
# Es: 50-432 MHz typical. foEs → MUF ≈ 5 × foEs (single hop at 1500 km).
|
||||
# For a band to propagate via Es, foEs must support it.
|
||||
@es_band_min_mhz 50
|
||||
@es_band_max_mhz 432
|
||||
# Single-hop Es path range (geometry-limited by reflection from ~100 km ionosphere).
|
||||
@es_min_distance_km 400.0
|
||||
@es_max_distance_km 2_500.0
|
||||
|
||||
# Aurora: VHF primarily. Kp threshold conventionally ≥ 5 for auroral
|
||||
# scatter to be plausible; stronger events (Kp ≥ 7) make it likely.
|
||||
@aurora_band_min_mhz 50
|
||||
@aurora_band_max_mhz 432
|
||||
@aurora_kp_floor 5
|
||||
@aurora_min_latitude 40.0
|
||||
|
||||
# Meteor scatter: VHF/UHF range, peaks during active showers.
|
||||
@ms_band_min_mhz 50
|
||||
@ms_band_max_mhz 432
|
||||
@ms_min_distance_km 600.0
|
||||
@ms_max_distance_km 2_300.0
|
||||
|
||||
# Aircraft scatter thresholds intentionally omitted until we land ADS-B
|
||||
# history ingestion. Without flight data we can only classify aircraft
|
||||
# scatter when the operator declared PROP_MODE=AS.
|
||||
|
||||
# Rain scatter: see `RainScatterClassifier` for derivation.
|
||||
@rs_band_min_mhz 5_000
|
||||
@rs_band_max_mhz 11_000
|
||||
@rs_max_distance_km 800.0
|
||||
@rs_light_dbz 25.0
|
||||
@rs_heavy_dbz 35.0
|
||||
@rs_heavy_pixel_floor 3
|
||||
@rs_min_coverage_pct 25.0
|
||||
|
||||
@typedoc "Inputs expected by `classify/1`"
|
||||
@type inputs :: %{
|
||||
required(:band_mhz) => integer(),
|
||||
required(:distance_km) => number(),
|
||||
required(:qso_timestamp) => DateTime.t(),
|
||||
required(:pos1) => %{String.t() => number()},
|
||||
required(:pos2) => %{String.t() => number()},
|
||||
optional(:user_declared_prop_mode) => String.t() | nil,
|
||||
optional(:radar) => map() | nil,
|
||||
optional(:duct_either_endpoint) => boolean(),
|
||||
optional(:native_best_duct_ghz) => float() | nil,
|
||||
optional(:kp_index) => integer() | nil,
|
||||
optional(:foes_mhz) => float() | nil,
|
||||
optional(:active_meteor_shower) => String.t() | nil
|
||||
}
|
||||
|
||||
@type mechanism ::
|
||||
:eme
|
||||
| :aurora
|
||||
| :sporadic_e
|
||||
| :f2
|
||||
| :meteor_scatter
|
||||
| :aircraft_scatter
|
||||
| :rain_scatter
|
||||
| :rain_scatter_possible
|
||||
| :tropo_duct
|
||||
| :line_of_sight
|
||||
| :troposcatter
|
||||
| :unknown
|
||||
|
||||
@type result :: %{mechanism: mechanism(), confidence: :high | :medium | :low}
|
||||
|
||||
@spec classify(inputs()) :: result()
|
||||
def classify(inputs) do
|
||||
inputs
|
||||
|> apply_defaults()
|
||||
|> evaluate()
|
||||
end
|
||||
|
||||
defp apply_defaults(inputs) do
|
||||
Map.merge(
|
||||
%{
|
||||
user_declared_prop_mode: nil,
|
||||
radar: nil,
|
||||
duct_either_endpoint: false,
|
||||
native_best_duct_ghz: nil,
|
||||
kp_index: nil,
|
||||
foes_mhz: nil,
|
||||
active_meteor_shower: nil
|
||||
},
|
||||
inputs
|
||||
)
|
||||
end
|
||||
|
||||
# Priority: user-declared → EME → aurora → Es → meteor_scatter →
|
||||
# aircraft_scatter (we don't have flight data yet, so this only fires
|
||||
# when user declared) → rain_scatter → tropo_duct → line_of_sight →
|
||||
# troposcatter.
|
||||
defp evaluate(inputs) do
|
||||
with :no_match <- try_user_declared(inputs),
|
||||
:no_match <- try_eme(inputs),
|
||||
:no_match <- try_aurora(inputs),
|
||||
:no_match <- try_sporadic_e(inputs),
|
||||
:no_match <- try_meteor_scatter(inputs),
|
||||
:no_match <- try_rain_scatter(inputs),
|
||||
:no_match <- try_tropo_duct(inputs),
|
||||
:no_match <- try_line_of_sight(inputs) do
|
||||
%{mechanism: :troposcatter, confidence: :low}
|
||||
else
|
||||
%{mechanism: _, confidence: _} = result -> result
|
||||
end
|
||||
end
|
||||
|
||||
# — User-declared ADIF PROP_MODE — always wins when present —
|
||||
|
||||
defp try_user_declared(%{user_declared_prop_mode: nil}), do: :no_match
|
||||
|
||||
defp try_user_declared(%{user_declared_prop_mode: raw}) when is_binary(raw) do
|
||||
key = raw |> String.trim() |> String.upcase()
|
||||
|
||||
case Map.fetch(@user_declared_map, key) do
|
||||
{:ok, mechanism} -> %{mechanism: mechanism, confidence: :high}
|
||||
:error -> :no_match
|
||||
end
|
||||
end
|
||||
|
||||
# — EME (moon bounce) — calc-only, no new data source —
|
||||
|
||||
defp try_eme(%{band_mhz: band}) when band < @eme_min_band_mhz, do: :no_match
|
||||
defp try_eme(%{distance_km: dist}) when dist < @eme_min_distance_km, do: :no_match
|
||||
|
||||
defp try_eme(inputs) do
|
||||
lat1 = get_lat(inputs.pos1)
|
||||
lon1 = get_lon(inputs.pos1)
|
||||
lat2 = get_lat(inputs.pos2)
|
||||
lon2 = get_lon(inputs.pos2)
|
||||
|
||||
if MoonEphemeris.above_horizon?(lat1, lon1, inputs.qso_timestamp) and
|
||||
MoonEphemeris.above_horizon?(lat2, lon2, inputs.qso_timestamp) do
|
||||
%{mechanism: :eme, confidence: :high}
|
||||
else
|
||||
# Very long VHF+ path but moon not mutually visible — physically
|
||||
# implausible as EME, fall through.
|
||||
:no_match
|
||||
end
|
||||
end
|
||||
|
||||
# — Aurora (Kp + high-lat N-S path) —
|
||||
|
||||
defp try_aurora(%{band_mhz: band}) when band > @aurora_band_max_mhz, do: :no_match
|
||||
defp try_aurora(%{band_mhz: band}) when band < @aurora_band_min_mhz, do: :no_match
|
||||
defp try_aurora(%{kp_index: nil}), do: :no_match
|
||||
defp try_aurora(%{kp_index: kp}) when kp < @aurora_kp_floor, do: :no_match
|
||||
|
||||
defp try_aurora(inputs) do
|
||||
lat1 = get_lat(inputs.pos1)
|
||||
lat2 = get_lat(inputs.pos2)
|
||||
midpoint_lat = abs((lat1 + lat2) / 2)
|
||||
|
||||
if midpoint_lat >= @aurora_min_latitude do
|
||||
# Kp ≥ 7 is "likely" aurora; Kp 5-6 is "possible" aurora (medium).
|
||||
confidence = if inputs.kp_index >= 7, do: :high, else: :medium
|
||||
%{mechanism: :aurora, confidence: confidence}
|
||||
else
|
||||
:no_match
|
||||
end
|
||||
end
|
||||
|
||||
# — Sporadic-E (foEs strong enough for the band) —
|
||||
|
||||
defp try_sporadic_e(%{band_mhz: band}) when band > @es_band_max_mhz, do: :no_match
|
||||
defp try_sporadic_e(%{band_mhz: band}) when band < @es_band_min_mhz, do: :no_match
|
||||
|
||||
defp try_sporadic_e(%{distance_km: dist}) when dist < @es_min_distance_km or dist > @es_max_distance_km, do: :no_match
|
||||
|
||||
defp try_sporadic_e(%{foes_mhz: nil}), do: :no_match
|
||||
|
||||
defp try_sporadic_e(%{foes_mhz: foes, band_mhz: band}) do
|
||||
# Es MUF at 1500-2000 km obliquity is roughly 5 × foEs. If that
|
||||
# exceeds the working band frequency, Es can support the path.
|
||||
muf_mhz = 5.0 * foes
|
||||
|
||||
if muf_mhz >= band do
|
||||
confidence = if muf_mhz >= 1.5 * band, do: :high, else: :medium
|
||||
%{mechanism: :sporadic_e, confidence: confidence}
|
||||
else
|
||||
:no_match
|
||||
end
|
||||
end
|
||||
|
||||
# — Meteor scatter (during active shower, on VHF/UHF) —
|
||||
|
||||
defp try_meteor_scatter(%{active_meteor_shower: nil}), do: :no_match
|
||||
defp try_meteor_scatter(%{band_mhz: band}) when band > @ms_band_max_mhz, do: :no_match
|
||||
defp try_meteor_scatter(%{band_mhz: band}) when band < @ms_band_min_mhz, do: :no_match
|
||||
|
||||
defp try_meteor_scatter(%{distance_km: dist}) when dist < @ms_min_distance_km or dist > @ms_max_distance_km,
|
||||
do: :no_match
|
||||
|
||||
defp try_meteor_scatter(_inputs), do: %{mechanism: :meteor_scatter, confidence: :medium}
|
||||
|
||||
# — Rain scatter (5–11 GHz, path ≤ 800 km, heavy rain in CV) —
|
||||
|
||||
defp try_rain_scatter(%{radar: nil}), do: :no_match
|
||||
|
||||
defp try_rain_scatter(%{band_mhz: band}) when band < @rs_band_min_mhz or band > @rs_band_max_mhz, do: :no_match
|
||||
|
||||
defp try_rain_scatter(%{distance_km: dist}) when dist > @rs_max_distance_km, do: :no_match
|
||||
|
||||
# Duct signature dominates — covered by try_tropo_duct below.
|
||||
defp try_rain_scatter(%{duct_either_endpoint: true}), do: :no_match
|
||||
|
||||
defp try_rain_scatter(%{radar: radar}) do
|
||||
coverage = radar[:coverage_pct] || 0.0
|
||||
max_dbz = radar[:max_dbz] || 0.0
|
||||
heavy = radar[:heavy_rain_pixel_count] || 0
|
||||
|
||||
cond do
|
||||
coverage < @rs_min_coverage_pct ->
|
||||
:no_match
|
||||
|
||||
max_dbz >= @rs_heavy_dbz and heavy >= @rs_heavy_pixel_floor ->
|
||||
%{mechanism: :rain_scatter, confidence: :high}
|
||||
|
||||
max_dbz >= @rs_light_dbz ->
|
||||
%{mechanism: :rain_scatter_possible, confidence: :medium}
|
||||
|
||||
true ->
|
||||
:no_match
|
||||
end
|
||||
end
|
||||
|
||||
# — Tropospheric duct (HRRR native best_duct supports the band, or legacy flag) —
|
||||
|
||||
defp try_tropo_duct(%{duct_either_endpoint: true}), do: %{mechanism: :tropo_duct, confidence: :medium}
|
||||
|
||||
defp try_tropo_duct(%{native_best_duct_ghz: ghz, band_mhz: band}) when is_number(ghz) and ghz * 1_000 >= band,
|
||||
do: %{mechanism: :tropo_duct, confidence: :high}
|
||||
|
||||
defp try_tropo_duct(_), do: :no_match
|
||||
|
||||
# — Line of sight — conservative 50 km radio-horizon approximation.
|
||||
|
||||
defp try_line_of_sight(%{distance_km: dist}) when dist <= 50.0, do: %{mechanism: :line_of_sight, confidence: :high}
|
||||
|
||||
defp try_line_of_sight(_), do: :no_match
|
||||
|
||||
defp get_lat(%{"lat" => lat}), do: lat / 1.0
|
||||
defp get_lat(%{lat: lat}), do: lat / 1.0
|
||||
defp get_lon(%{"lon" => lon}), do: lon / 1.0
|
||||
defp get_lon(%{lon: lon}), do: lon / 1.0
|
||||
end
|
||||
130
lib/microwaveprop/propagation/moon_ephemeris.ex
Normal file
130
lib/microwaveprop/propagation/moon_ephemeris.ex
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
defmodule Microwaveprop.Propagation.MoonEphemeris do
|
||||
@moduledoc """
|
||||
Approximate moon-position calculation for EME classification.
|
||||
|
||||
Accuracy target: ±1° — enough to decide "is the moon above the horizon
|
||||
for both stations right now?" which is the EME feasibility test. We
|
||||
use Jean Meeus's low-precision algorithm (Astronomical Algorithms,
|
||||
chapter 47), good to ~10' over historical amateur contact timestamps.
|
||||
|
||||
No external ephemeris table — pure math so it works for any date.
|
||||
"""
|
||||
|
||||
@deg_to_rad :math.pi() / 180.0
|
||||
@rad_to_deg 180.0 / :math.pi()
|
||||
|
||||
@doc """
|
||||
Returns the moon's altitude above the horizon in degrees for a given
|
||||
observer lat/lon and UTC timestamp. Negative means the moon is below
|
||||
the horizon (not reachable).
|
||||
"""
|
||||
@spec altitude_deg(number(), number(), DateTime.t()) :: float()
|
||||
def altitude_deg(lat, lon, %DateTime{} = ts) do
|
||||
{ra, dec} = moon_ra_dec(ts)
|
||||
gst = greenwich_sidereal_hours(ts)
|
||||
# Local hour angle in degrees
|
||||
lst_deg = :math.fmod(gst * 15.0 + lon, 360.0)
|
||||
hour_angle = lst_deg - ra * 15.0
|
||||
|
||||
lat_r = lat * @deg_to_rad
|
||||
dec_r = dec * @deg_to_rad
|
||||
ha_r = hour_angle * @deg_to_rad
|
||||
|
||||
sin_alt =
|
||||
:math.sin(lat_r) * :math.sin(dec_r) +
|
||||
:math.cos(lat_r) * :math.cos(dec_r) * :math.cos(ha_r)
|
||||
|
||||
sin_alt |> max(-1.0) |> min(1.0) |> :math.asin() |> Kernel.*(@rad_to_deg)
|
||||
end
|
||||
|
||||
@doc """
|
||||
`true` if the moon is above the horizon at the given observer + time.
|
||||
"""
|
||||
@spec above_horizon?(number(), number(), DateTime.t()) :: boolean()
|
||||
def above_horizon?(lat, lon, %DateTime{} = ts), do: altitude_deg(lat, lon, ts) > 0.0
|
||||
|
||||
# — private math helpers below —
|
||||
|
||||
# Returns {right ascension (hours), declination (deg)} of the moon.
|
||||
defp moon_ra_dec(%DateTime{} = ts) do
|
||||
jd = julian_day(ts)
|
||||
t = (jd - 2_451_545.0) / 36_525.0
|
||||
|
||||
# Mean orbital elements (Meeus §47, degrees).
|
||||
l_prime = 218.3164477 + 481_267.88123421 * t
|
||||
d = 297.8501921 + 445_267.1114034 * t
|
||||
m = 357.5291092 + 35_999.0502909 * t
|
||||
m_prime = 134.9633964 + 477_198.8675055 * t
|
||||
f = 93.2720950 + 483_202.0175233 * t
|
||||
|
||||
# Longitude correction (leading term only — 6.289°).
|
||||
lambda =
|
||||
l_prime +
|
||||
6.289 * sin_deg(m_prime) -
|
||||
1.274 * sin_deg(2 * d - m_prime) +
|
||||
0.658 * sin_deg(2 * d) -
|
||||
0.186 * sin_deg(m)
|
||||
|
||||
# Latitude (β) leading term.
|
||||
beta = 5.128 * sin_deg(f)
|
||||
|
||||
# Obliquity of the ecliptic.
|
||||
eps = 23.439 - 0.0000004 * (jd - 2_451_545.0)
|
||||
|
||||
# Convert ecliptic (λ, β) → equatorial (α, δ).
|
||||
lam_r = lambda * @deg_to_rad
|
||||
beta_r = beta * @deg_to_rad
|
||||
eps_r = eps * @deg_to_rad
|
||||
|
||||
sin_alpha = :math.sin(lam_r) * :math.cos(eps_r) - :math.tan(beta_r) * :math.sin(eps_r)
|
||||
cos_alpha = :math.cos(lam_r)
|
||||
alpha_r = :math.atan2(sin_alpha, cos_alpha)
|
||||
|
||||
sin_delta =
|
||||
:math.sin(beta_r) * :math.cos(eps_r) +
|
||||
:math.cos(beta_r) * :math.sin(eps_r) * :math.sin(lam_r)
|
||||
|
||||
delta_r = :math.asin(sin_delta)
|
||||
|
||||
ra_hours = :math.fmod(alpha_r * @rad_to_deg / 15.0 + 48.0, 24.0)
|
||||
dec_deg = delta_r * @rad_to_deg
|
||||
{ra_hours, dec_deg}
|
||||
end
|
||||
|
||||
defp julian_day(%DateTime{year: y, month: m, day: d, hour: h, minute: mi, second: s}) do
|
||||
{y2, m2} =
|
||||
if m <= 2 do
|
||||
{y - 1, m + 12}
|
||||
else
|
||||
{y, m}
|
||||
end
|
||||
|
||||
a = div(y2, 100)
|
||||
b = 2 - a + div(a, 4)
|
||||
|
||||
jd_day =
|
||||
Float.floor(365.25 * (y2 + 4716)) +
|
||||
Float.floor(30.6001 * (m2 + 1)) +
|
||||
d + b - 1524.5
|
||||
|
||||
frac = (h + mi / 60 + s / 3600) / 24.0
|
||||
jd_day + frac
|
||||
end
|
||||
|
||||
defp greenwich_sidereal_hours(%DateTime{} = ts) do
|
||||
jd = julian_day(ts)
|
||||
t = (jd - 2_451_545.0) / 36_525.0
|
||||
|
||||
# Mean sidereal time at Greenwich, in hours.
|
||||
gmst_sec =
|
||||
67_310.54841 +
|
||||
(876_600 * 3600 + 8_640_184.812866) * t +
|
||||
0.093104 * t * t -
|
||||
6.2e-6 * t * t * t
|
||||
|
||||
hours = gmst_sec / 3600.0
|
||||
:math.fmod(:math.fmod(hours, 24.0) + 24.0, 24.0)
|
||||
end
|
||||
|
||||
defp sin_deg(deg), do: :math.sin(deg * @deg_to_rad)
|
||||
end
|
||||
|
|
@ -203,6 +203,15 @@ defmodule Microwaveprop.Radio.AdifImport do
|
|||
mode = normalize_mode(fields["MODE"], fields["SUBMODE"])
|
||||
band = resolve_band(fields)
|
||||
timestamp = parse_adif_datetime(fields["QSO_DATE"], fields["TIME_ON"])
|
||||
# ADIF PROP_MODE — trusted as ground truth by MechanismClassifier when
|
||||
# present. Stored verbatim (uppercased, whitespace-trimmed) so unusual
|
||||
# values survive the round-trip for human inspection.
|
||||
prop_mode =
|
||||
case fields["PROP_MODE"] do
|
||||
nil -> nil
|
||||
"" -> nil
|
||||
raw when is_binary(raw) -> raw |> String.trim() |> String.upcase()
|
||||
end
|
||||
|
||||
case {band, timestamp} do
|
||||
{nil, _} ->
|
||||
|
|
@ -220,7 +229,8 @@ defmodule Microwaveprop.Radio.AdifImport do
|
|||
"band" => band_str,
|
||||
"mode" => mode,
|
||||
"qso_timestamp" => DateTime.to_iso8601(dt),
|
||||
"submitter_email" => submitter_email
|
||||
"submitter_email" => submitter_email,
|
||||
"user_declared_prop_mode" => prop_mode
|
||||
}
|
||||
|
||||
changeset = Contact.submission_changeset(%Contact{}, attrs)
|
||||
|
|
|
|||
|
|
@ -42,6 +42,20 @@ defmodule Microwaveprop.Radio.Contact do
|
|||
values: [:pending, :queued, :processing, :complete, :failed, :unavailable],
|
||||
default: :pending
|
||||
|
||||
# ADIF PROP_MODE string as submitted by the operator. Trusted as
|
||||
# ground truth by MechanismClassifier when present.
|
||||
field :user_declared_prop_mode, :string
|
||||
|
||||
# Output of Microwaveprop.Propagation.MechanismClassifier. Values
|
||||
# align with the classifier's result type — kept as a string for
|
||||
# schema-free extension (new mechanisms don't require a migration).
|
||||
field :propagation_mechanism, :string
|
||||
field :propagation_mechanism_confidence, Ecto.Enum, values: [:high, :medium, :low]
|
||||
|
||||
field :propagation_mechanism_status, Ecto.Enum,
|
||||
values: [:pending, :queued, :processing, :complete, :failed, :unavailable],
|
||||
default: :pending
|
||||
|
||||
field :user_submitted, :boolean, default: false
|
||||
field :submitter_email, :string
|
||||
field :flagged_invalid, :boolean, default: false
|
||||
|
|
@ -54,7 +68,7 @@ defmodule Microwaveprop.Radio.Contact do
|
|||
@type t :: %__MODULE__{}
|
||||
|
||||
@required_fields ~w(station1 station2 qso_timestamp band)a
|
||||
@optional_fields ~w(grid1 grid2 pos1 pos2 distance_km user_id mode)a
|
||||
@optional_fields ~w(grid1 grid2 pos1 pos2 distance_km user_id mode user_declared_prop_mode)a
|
||||
|
||||
@spec changeset(t() | Ecto.Changeset.t(), map()) :: Ecto.Changeset.t()
|
||||
def changeset(contact, attrs) do
|
||||
|
|
@ -63,7 +77,7 @@ defmodule Microwaveprop.Radio.Contact do
|
|||
|> validate_required(@required_fields)
|
||||
end
|
||||
|
||||
@submission_fields ~w(station1 station2 qso_timestamp mode band grid1 grid2 submitter_email)a
|
||||
@submission_fields ~w(station1 station2 qso_timestamp mode band grid1 grid2 submitter_email user_declared_prop_mode)a
|
||||
@submission_required ~w(station1 station2 qso_timestamp band grid1 grid2)a
|
||||
@allowed_modes ~w(CW SSB FM FT8 FT4 Q65)
|
||||
# Keep in sync with Microwaveprop.Propagation.BandConfig.all_bands/0 and
|
||||
|
|
|
|||
|
|
@ -15,13 +15,19 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
require Logger
|
||||
|
||||
@enrichable [:pending, :queued, :failed]
|
||||
@valid_types ~w(hrrr weather terrain iemre narr)
|
||||
@valid_types ~w(hrrr weather terrain iemre narr radar mechanism)
|
||||
|
||||
# Virtual types have no <type>_status column on contacts; they're synthesized
|
||||
# from other status fields (e.g. :narr candidates are `hrrr_status = :unavailable`).
|
||||
# Skip these in any reconcile / ordering pass that reads a <type>_status field.
|
||||
@virtual_types [:narr]
|
||||
|
||||
# `radar` and `mechanism` use differently-named *_status columns than
|
||||
# `<type>_status`, so we remap them when building queries and update_alls.
|
||||
@status_column_overrides %{
|
||||
mechanism: :propagation_mechanism_status
|
||||
}
|
||||
|
||||
# A :queued contact whose updated_at is older than this is one the cron has
|
||||
# already tried ~144 times over 3 days without the underlying worker landing
|
||||
# data. At that point the data is effectively unreachable (dead ASOS station,
|
||||
|
|
@ -77,7 +83,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
types |> Enum.filter(&(&1 in @valid_types)) |> Enum.map(&String.to_existing_atom/1)
|
||||
end
|
||||
|
||||
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :narr]
|
||||
defp parse_types(_), do: [:hrrr, :weather, :terrain, :iemre, :narr, :radar, :mechanism]
|
||||
|
||||
defp status_priority_order(types) do
|
||||
# Prioritize pending/failed over queued so real work happens first.
|
||||
|
|
@ -88,7 +94,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
[desc: dynamic([c], c.qso_timestamp)]
|
||||
|
||||
type ->
|
||||
field = :"#{type}_status"
|
||||
field = status_column(type)
|
||||
|
||||
[
|
||||
asc:
|
||||
|
|
@ -101,6 +107,8 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
end
|
||||
end
|
||||
|
||||
defp status_column(type), do: Map.get(@status_column_overrides, type, :"#{type}_status")
|
||||
|
||||
# Flip contacts that have been stuck in :queued for longer than
|
||||
# @stale_queued_cutoff_seconds to :unavailable. One Repo.update_all per type,
|
||||
# scoped to only the types passed in args so a partial cron (e.g. hrrr-only)
|
||||
|
|
@ -116,7 +124,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
types
|
||||
|> Enum.reject(&(&1 in @virtual_types))
|
||||
|> Enum.reduce(0, fn type, acc ->
|
||||
field = :"#{type}_status"
|
||||
field = status_column(type)
|
||||
|
||||
{n, _} =
|
||||
Contact
|
||||
|
|
@ -160,7 +168,7 @@ defmodule Microwaveprop.Workers.BackfillEnqueueWorker do
|
|||
dynamic([c], ^acc or (c.hrrr_status == :unavailable and c.qso_timestamp < ^coverage_end))
|
||||
|
||||
type, acc ->
|
||||
field = :"#{type}_status"
|
||||
field = status_column(type)
|
||||
dynamic([c], ^acc or field(c, ^field) in ^@enrichable)
|
||||
end)
|
||||
end
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
alias Microwaveprop.Workers.CommonVolumeRadarWorker
|
||||
alias Microwaveprop.Workers.HrrrFetchWorker
|
||||
alias Microwaveprop.Workers.IemreFetchWorker
|
||||
alias Microwaveprop.Workers.MechanismClassifyWorker
|
||||
alias Microwaveprop.Workers.NarrFetchWorker
|
||||
alias Microwaveprop.Workers.TerrainProfileWorker
|
||||
alias Microwaveprop.Workers.WeatherFetchWorker
|
||||
|
|
@ -23,14 +24,17 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
Called directly from submission flow — no Oban indirection.
|
||||
Optionally pass a list of types to limit which enrichments run.
|
||||
"""
|
||||
def enqueue_for_contact(contact, types \\ [:hrrr, :weather, :terrain, :iemre, :radar]) do
|
||||
def enqueue_for_contact(contact, types \\ [:hrrr, :weather, :terrain, :iemre, :radar, :mechanism]) do
|
||||
contact = Radio.ensure_positions!(contact)
|
||||
jobs_by_type = build_jobs_by_type(contact, types)
|
||||
|
||||
bulk_jobs =
|
||||
jobs_by_type[:weather] ++
|
||||
jobs_by_type[:hrrr] ++
|
||||
jobs_by_type[:terrain] ++ jobs_by_type[:iemre] ++ jobs_by_type[:radar]
|
||||
jobs_by_type[:terrain] ++
|
||||
jobs_by_type[:iemre] ++
|
||||
jobs_by_type[:radar] ++
|
||||
jobs_by_type[:mechanism]
|
||||
|
||||
if bulk_jobs != [] do
|
||||
insert_all_chunked(bulk_jobs)
|
||||
|
|
@ -52,7 +56,8 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
terrain: if(:terrain in types, do: build_terrain_jobs([contact]), else: []),
|
||||
iemre: if(:iemre in types, do: build_iemre_jobs([contact]), else: []),
|
||||
narr: if(:narr in types, do: build_narr_jobs([contact]), else: []),
|
||||
radar: if(:radar in types, do: build_radar_jobs([contact]), else: [])
|
||||
radar: if(:radar in types, do: build_radar_jobs([contact]), else: []),
|
||||
mechanism: if(:mechanism in types, do: build_mechanism_jobs([contact]), else: [])
|
||||
}
|
||||
end
|
||||
|
||||
|
|
@ -60,15 +65,26 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
ids = [contact.id]
|
||||
|
||||
if contact.pos1 do
|
||||
mark_pos1_statuses(contact, ids, types, jobs_by_type)
|
||||
end
|
||||
|
||||
if contact.pos1 && contact.pos2 do
|
||||
mark_path_statuses(contact, ids, types, jobs_by_type)
|
||||
end
|
||||
end
|
||||
|
||||
defp mark_pos1_statuses(contact, ids, types, jobs_by_type) do
|
||||
if :weather in types, do: mark_status!(ids, :weather_status, jobs_by_type[:weather])
|
||||
if :hrrr in types, do: mark_hrrr_status!(contact, ids, jobs_by_type[:hrrr])
|
||||
if :iemre in types, do: mark_status!(ids, :iemre_status, jobs_by_type[:iemre])
|
||||
end
|
||||
|
||||
if contact.pos1 && contact.pos2 do
|
||||
defp mark_path_statuses(contact, ids, types, jobs_by_type) do
|
||||
if :terrain in types, do: mark_status!(ids, :terrain_status, jobs_by_type[:terrain])
|
||||
if :radar in types, do: mark_radar_status!(contact, ids, jobs_by_type[:radar])
|
||||
end
|
||||
|
||||
if :mechanism in types,
|
||||
do: mark_status!(ids, :propagation_mechanism_status, jobs_by_type[:mechanism])
|
||||
end
|
||||
|
||||
# IEM n0q composite reflectivity archive only has meaningful CONUS
|
||||
|
|
@ -188,6 +204,15 @@ defmodule Microwaveprop.Workers.ContactWeatherEnqueueWorker do
|
|||
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
||||
end
|
||||
|
||||
def build_mechanism_jobs(contacts) do
|
||||
contacts
|
||||
|> Enum.filter(fn contact -> contact.pos1 && contact.pos2 end)
|
||||
|> Enum.map(fn contact ->
|
||||
MechanismClassifyWorker.new(%{"contact_id" => contact.id})
|
||||
end)
|
||||
|> Enum.uniq_by(fn changeset -> changeset.changes.args end)
|
||||
end
|
||||
|
||||
def build_weather_jobs(contacts) do
|
||||
contacts
|
||||
|> Enum.flat_map(&jobs_for_contact/1)
|
||||
|
|
|
|||
225
lib/microwaveprop/workers/mechanism_classify_worker.ex
Normal file
225
lib/microwaveprop/workers/mechanism_classify_worker.ex
Normal file
|
|
@ -0,0 +1,225 @@
|
|||
defmodule Microwaveprop.Workers.MechanismClassifyWorker do
|
||||
@moduledoc """
|
||||
Per-contact propagation-mechanism classification.
|
||||
|
||||
Assembles the full input map for `MechanismClassifier`:
|
||||
|
||||
* `user_declared_prop_mode` — from the contact's ADIF PROP_MODE
|
||||
* `duct_either_endpoint` — from the HRRR profile at either endpoint
|
||||
* `native_best_duct_ghz` — from the nearest `hrrr_native_profiles` row
|
||||
* `radar` — from the pre-computed `contact_common_volume_radar` row
|
||||
* `kp_index` — from the daily `solar_indices` row
|
||||
* `foes_mhz` — from the nearest `ionosonde_observations` row
|
||||
* `active_meteor_shower` — looked up from the static shower calendar
|
||||
|
||||
Results land on the contact as `propagation_mechanism` (string),
|
||||
`propagation_mechanism_confidence` (enum), and
|
||||
`propagation_mechanism_status`.
|
||||
|
||||
Unique on `contact_id` so the submit-time enqueue path and the
|
||||
backfill cron collapse to a single job per contact.
|
||||
"""
|
||||
|
||||
use Oban.Worker,
|
||||
queue: :mechanism,
|
||||
max_attempts: 3,
|
||||
unique: [
|
||||
period: :infinity,
|
||||
states: [:available, :scheduled, :executing, :retryable],
|
||||
keys: [:contact_id]
|
||||
]
|
||||
|
||||
import Ecto.Query
|
||||
|
||||
alias Microwaveprop.Ionosphere.Observation, as: IonosondeObservation
|
||||
alias Microwaveprop.Propagation.MechanismClassifier
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Radio.ContactCommonVolumeRadar
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Weather.HrrrProfile
|
||||
|
||||
require Logger
|
||||
|
||||
@impl Oban.Worker
|
||||
def perform(%Oban.Job{args: %{"contact_id" => contact_id}}) do
|
||||
case Repo.get(Contact, contact_id) do
|
||||
nil ->
|
||||
:ok
|
||||
|
||||
%Contact{pos1: p1, pos2: p2} = contact when is_map(p1) and is_map(p2) ->
|
||||
classify_and_persist(contact)
|
||||
|
||||
contact ->
|
||||
mark_status(contact, :unavailable, nil, nil)
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
defp classify_and_persist(%Contact{} = contact) do
|
||||
inputs = build_inputs(contact)
|
||||
%{mechanism: mechanism, confidence: confidence} = MechanismClassifier.classify(inputs)
|
||||
|
||||
mark_status(contact, :complete, Atom.to_string(mechanism), confidence)
|
||||
:ok
|
||||
rescue
|
||||
e ->
|
||||
Logger.warning("MechanismClassifyWorker: failed for contact #{contact.id}: #{inspect(e)}")
|
||||
|
||||
mark_status(contact, :failed, nil, nil)
|
||||
:ok
|
||||
end
|
||||
|
||||
defp build_inputs(%Contact{} = contact) do
|
||||
%{
|
||||
band_mhz: Decimal.to_integer(contact.band),
|
||||
distance_km: distance_km(contact),
|
||||
qso_timestamp: contact.qso_timestamp,
|
||||
pos1: contact.pos1,
|
||||
pos2: contact.pos2,
|
||||
user_declared_prop_mode: contact.user_declared_prop_mode,
|
||||
radar: radar_for(contact),
|
||||
duct_either_endpoint: duct_at_either_endpoint?(contact),
|
||||
native_best_duct_ghz: native_duct_at_either_endpoint(contact),
|
||||
kp_index: kp_for(contact),
|
||||
foes_mhz: foes_for(contact),
|
||||
active_meteor_shower: active_shower_at(contact.qso_timestamp)
|
||||
}
|
||||
end
|
||||
|
||||
defp distance_km(%Contact{distance_km: nil}), do: 0.0
|
||||
defp distance_km(%Contact{distance_km: d}), do: Decimal.to_float(d)
|
||||
|
||||
defp radar_for(%Contact{id: id}) do
|
||||
case Repo.get_by(ContactCommonVolumeRadar, contact_id: id) do
|
||||
nil ->
|
||||
nil
|
||||
|
||||
row ->
|
||||
%{
|
||||
max_dbz: row.max_dbz,
|
||||
heavy_rain_pixel_count: row.heavy_rain_pixel_count,
|
||||
coverage_pct: row.coverage_pct
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
defp duct_at_either_endpoint?(%Contact{pos1: p1, pos2: p2, qso_timestamp: ts}) do
|
||||
Enum.any?([p1, p2], fn pos ->
|
||||
case nearest_hrrr(pos, ts) do
|
||||
nil -> false
|
||||
%HrrrProfile{ducting_detected: true} -> true
|
||||
_ -> false
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp native_duct_at_either_endpoint(%Contact{pos1: p1, pos2: p2, qso_timestamp: ts}) do
|
||||
[p1, p2]
|
||||
|> Enum.map(&nearest_native_duct(&1, ts))
|
||||
|> Enum.reject(&is_nil/1)
|
||||
|> case do
|
||||
[] -> nil
|
||||
values -> Enum.max(values)
|
||||
end
|
||||
end
|
||||
|
||||
defp nearest_hrrr(pos, ts) do
|
||||
lat = pos["lat"]
|
||||
lon = pos["lon"]
|
||||
|
||||
if lat && lon do
|
||||
Repo.one(
|
||||
from(h in HrrrProfile,
|
||||
where:
|
||||
h.lat >= ^(lat - 0.07) and h.lat <= ^(lat + 0.07) and h.lon >= ^(lon - 0.07) and h.lon <= ^(lon + 0.07) and
|
||||
h.valid_time >= ^DateTime.add(ts, -3600, :second) and h.valid_time <= ^DateTime.add(ts, 3600, :second),
|
||||
order_by: fragment("ABS(EXTRACT(EPOCH FROM ? - ?))", h.valid_time, ^ts),
|
||||
limit: 1
|
||||
)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defp nearest_native_duct(pos, ts) do
|
||||
lat = pos["lat"]
|
||||
lon = pos["lon"]
|
||||
|
||||
if lat && lon do
|
||||
Repo.one(
|
||||
from(h in "hrrr_native_profiles",
|
||||
where:
|
||||
h.lat >= ^(lat - 0.07) and h.lat <= ^(lat + 0.07) and h.lon >= ^(lon - 0.07) and h.lon <= ^(lon + 0.07) and
|
||||
h.valid_time >= ^DateTime.add(ts, -3600, :second) and h.valid_time <= ^DateTime.add(ts, 3600, :second),
|
||||
order_by: fragment("ABS(EXTRACT(EPOCH FROM ? - ?))", h.valid_time, ^ts),
|
||||
limit: 1,
|
||||
select: h.best_duct_band_ghz
|
||||
)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
defp kp_for(%Contact{qso_timestamp: ts}) do
|
||||
date = DateTime.to_date(ts)
|
||||
|
||||
# solar_indices.kp_values is the daily array of 3-hour Kp readings.
|
||||
# Take the day's peak — that's the signal that matters for aurora.
|
||||
from(s in "solar_indices", where: s.date == ^date, select: s.kp_values, limit: 1)
|
||||
|> Repo.one()
|
||||
|> case do
|
||||
nil -> nil
|
||||
[] -> nil
|
||||
values when is_list(values) -> values |> Enum.reject(&is_nil/1) |> peak_or_nil()
|
||||
end
|
||||
end
|
||||
|
||||
defp peak_or_nil([]), do: nil
|
||||
defp peak_or_nil(values), do: values |> Enum.max() |> round()
|
||||
|
||||
# Nearest ionosonde observation within ±1h of QSO timestamp. We don't
|
||||
# restrict by location — foEs is spatially noisy but at least the
|
||||
# "nearest station in the Americas" will catch US-continental Es events.
|
||||
defp foes_for(%Contact{qso_timestamp: ts}) do
|
||||
Repo.one(
|
||||
from(io in IonosondeObservation,
|
||||
where: io.valid_time >= ^DateTime.add(ts, -3600, :second) and io.valid_time <= ^DateTime.add(ts, 3600, :second),
|
||||
where: not is_nil(io.fo_es_mhz),
|
||||
order_by: fragment("ABS(EXTRACT(EPOCH FROM ? - ?))", io.valid_time, ^ts),
|
||||
limit: 1,
|
||||
select: io.fo_es_mhz
|
||||
)
|
||||
)
|
||||
end
|
||||
|
||||
# Static meteor-shower calendar (peak dates, approximate). Any contact
|
||||
# within ±3 days of a peak counts as "during the shower."
|
||||
@showers [
|
||||
{~D[2024-01-04], "Quadrantids"},
|
||||
{~D[2024-04-22], "Lyrids"},
|
||||
{~D[2024-05-06], "Eta Aquariids"},
|
||||
{~D[2024-07-30], "Southern Delta Aquariids"},
|
||||
{~D[2024-08-12], "Perseids"},
|
||||
{~D[2024-10-21], "Orionids"},
|
||||
{~D[2024-11-17], "Leonids"},
|
||||
{~D[2024-12-14], "Geminids"},
|
||||
{~D[2024-12-22], "Ursids"}
|
||||
]
|
||||
|
||||
defp active_shower_at(%DateTime{} = ts) do
|
||||
date = DateTime.to_date(ts)
|
||||
|
||||
Enum.find_value(@showers, fn {peak_mmdd, name} ->
|
||||
diff = abs(Date.diff(peak_mmdd, %{date | year: peak_mmdd.year}))
|
||||
if diff <= 3, do: name
|
||||
end)
|
||||
end
|
||||
|
||||
defp mark_status(%Contact{} = contact, status, mechanism, confidence) do
|
||||
contact
|
||||
|> Ecto.Changeset.change(%{
|
||||
propagation_mechanism_status: status,
|
||||
propagation_mechanism: mechanism,
|
||||
propagation_mechanism_confidence: confidence
|
||||
})
|
||||
|> Repo.update!()
|
||||
end
|
||||
end
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
defmodule Microwaveprop.Repo.Migrations.AddPropagationMechanismToContacts do
|
||||
use Ecto.Migration
|
||||
|
||||
def change do
|
||||
alter table(:contacts) do
|
||||
# Classifier output. `user_declared_prop_mode` is the raw ADIF
|
||||
# PROP_MODE string the operator submitted (if any); it's trusted
|
||||
# as ground truth by the classifier when present. The two
|
||||
# `propagation_mechanism*` columns are the classifier's output
|
||||
# and its enrichment status, following the same pattern as
|
||||
# hrrr_status / weather_status / terrain_status / radar_status.
|
||||
add :user_declared_prop_mode, :string
|
||||
add :propagation_mechanism, :string
|
||||
add :propagation_mechanism_confidence, :string
|
||||
add :propagation_mechanism_status, :string, null: false, default: "pending"
|
||||
end
|
||||
|
||||
# Partial index for the backfill cron to scan only contacts that
|
||||
# still need classification. Matches the pattern on other *_status
|
||||
# columns.
|
||||
create index(:contacts, [:propagation_mechanism_status],
|
||||
where: "propagation_mechanism_status <> 'complete'",
|
||||
name: :contacts_propagation_mechanism_status_pending_index
|
||||
)
|
||||
|
||||
# Lets algo.md / map / analysis queries filter by mechanism without
|
||||
# a full table scan.
|
||||
create index(:contacts, [:propagation_mechanism])
|
||||
end
|
||||
end
|
||||
292
test/microwaveprop/propagation/mechanism_classifier_test.exs
Normal file
292
test/microwaveprop/propagation/mechanism_classifier_test.exs
Normal file
|
|
@ -0,0 +1,292 @@
|
|||
defmodule Microwaveprop.Propagation.MechanismClassifierTest do
|
||||
use ExUnit.Case, async: true
|
||||
|
||||
alias Microwaveprop.Propagation.MechanismClassifier
|
||||
|
||||
defp inputs(overrides \\ %{}) do
|
||||
# Base: a plain 10 GHz tropo-scatter contact with everything nil.
|
||||
base = %{
|
||||
band_mhz: 10_000,
|
||||
distance_km: 250.0,
|
||||
qso_timestamp: ~U[2024-08-15 19:00:00Z],
|
||||
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
||||
pos2: %{"lat" => 30.3, "lon" => -97.7},
|
||||
user_declared_prop_mode: nil,
|
||||
radar: nil,
|
||||
duct_either_endpoint: false,
|
||||
native_best_duct_ghz: nil,
|
||||
kp_index: nil,
|
||||
foes_mhz: nil,
|
||||
active_meteor_shower: nil
|
||||
}
|
||||
|
||||
Map.merge(base, overrides)
|
||||
end
|
||||
|
||||
describe "user-declared mechanism (ADIF PROP_MODE) — highest priority" do
|
||||
test "trusts ADIF 'EME' even on 10 GHz short path" do
|
||||
result = MechanismClassifier.classify(inputs(%{user_declared_prop_mode: "EME"}))
|
||||
assert result.mechanism == :eme
|
||||
assert result.confidence == :high
|
||||
end
|
||||
|
||||
test "trusts ADIF 'ES' even when ionosonde data absent" do
|
||||
result = MechanismClassifier.classify(inputs(%{band_mhz: 144, user_declared_prop_mode: "ES"}))
|
||||
assert result.mechanism == :sporadic_e
|
||||
assert result.confidence == :high
|
||||
end
|
||||
|
||||
test "trusts ADIF 'MS' for meteor scatter" do
|
||||
result = MechanismClassifier.classify(inputs(%{band_mhz: 144, user_declared_prop_mode: "MS"}))
|
||||
assert result.mechanism == :meteor_scatter
|
||||
end
|
||||
|
||||
test "trusts ADIF 'RS' for rain scatter" do
|
||||
result = MechanismClassifier.classify(inputs(%{user_declared_prop_mode: "RS"}))
|
||||
assert result.mechanism == :rain_scatter
|
||||
end
|
||||
|
||||
test "trusts ADIF 'AS' for aircraft scatter" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{band_mhz: 1_296, user_declared_prop_mode: "AS"}))
|
||||
|
||||
assert result.mechanism == :aircraft_scatter
|
||||
end
|
||||
|
||||
test "unknown user-declared mode falls through to physics classification" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{user_declared_prop_mode: "GIBBERISH"}))
|
||||
|
||||
# No rain, no duct, no user hint → default to troposcatter
|
||||
assert result.mechanism == :troposcatter
|
||||
end
|
||||
end
|
||||
|
||||
describe "EME detection (calc-only)" do
|
||||
test "2m 5000 km path with moon visible at both ends -> :eme" do
|
||||
# A 5000 km path on 2m at a time when the moon can be visible at both
|
||||
# ends is basically always EME.
|
||||
result =
|
||||
MechanismClassifier.classify(
|
||||
inputs(%{
|
||||
band_mhz: 144,
|
||||
distance_km: 5_000.0,
|
||||
# A time chosen so the moon is above both stations — handled by
|
||||
# MoonEphemeris inside the classifier; test just asserts the
|
||||
# classification flows through when distance/band say "EME"
|
||||
qso_timestamp: ~U[2024-08-15 04:00:00Z]
|
||||
})
|
||||
)
|
||||
|
||||
assert result.mechanism == :eme
|
||||
end
|
||||
|
||||
test "short 300 km 2m path -> NOT EME" do
|
||||
result = MechanismClassifier.classify(inputs(%{band_mhz: 144, distance_km: 300.0}))
|
||||
refute result.mechanism == :eme
|
||||
end
|
||||
|
||||
test "6 m (50 MHz) is below EME's practical band range -> NOT EME" do
|
||||
result = MechanismClassifier.classify(inputs(%{band_mhz: 50, distance_km: 5_000.0}))
|
||||
refute result.mechanism == :eme
|
||||
end
|
||||
end
|
||||
|
||||
describe "sporadic-E (ionosonde-driven)" do
|
||||
test "6 m 1500 km path, foEs=12 MHz (MUF=60) -> :sporadic_e" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{band_mhz: 50, distance_km: 1_500.0, foes_mhz: 12.0}))
|
||||
|
||||
assert result.mechanism == :sporadic_e
|
||||
end
|
||||
|
||||
test "2 m 1500 km path, foEs=30 MHz (MUF=150) -> :sporadic_e (extraordinary event)" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{band_mhz: 144, distance_km: 1_500.0, foes_mhz: 30.0}))
|
||||
|
||||
assert result.mechanism == :sporadic_e
|
||||
end
|
||||
|
||||
test "6 m 1500 km with foEs=8 MHz (MUF=40 < 50) -> NOT Es (MUF too low)" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{band_mhz: 50, distance_km: 1_500.0, foes_mhz: 8.0}))
|
||||
|
||||
refute result.mechanism == :sporadic_e
|
||||
end
|
||||
|
||||
test "6 m short 200 km path -> NOT Es (below single-hop Es minimum)" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{band_mhz: 50, distance_km: 200.0, foes_mhz: 8.0}))
|
||||
|
||||
refute result.mechanism == :sporadic_e
|
||||
end
|
||||
|
||||
test "6 m 1500 km with weak foEs=3 MHz -> NOT Es (MUF too low for 6m)" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{band_mhz: 50, distance_km: 1_500.0, foes_mhz: 3.0}))
|
||||
|
||||
refute result.mechanism == :sporadic_e
|
||||
end
|
||||
end
|
||||
|
||||
describe "aurora (Kp-driven + high-lat path)" do
|
||||
test "2 m 500 km N-S path, Kp=7 -> :aurora" do
|
||||
result =
|
||||
MechanismClassifier.classify(
|
||||
inputs(%{
|
||||
band_mhz: 144,
|
||||
distance_km: 500.0,
|
||||
pos1: %{"lat" => 45.0, "lon" => -95.0},
|
||||
pos2: %{"lat" => 49.0, "lon" => -95.0},
|
||||
kp_index: 7
|
||||
})
|
||||
)
|
||||
|
||||
assert result.mechanism == :aurora
|
||||
end
|
||||
|
||||
test "10 GHz microwave is far above typical aurora band range -> NOT aurora" do
|
||||
result =
|
||||
MechanismClassifier.classify(
|
||||
inputs(%{
|
||||
band_mhz: 10_000,
|
||||
kp_index: 8,
|
||||
pos1: %{"lat" => 45.0, "lon" => -95.0},
|
||||
pos2: %{"lat" => 49.0, "lon" => -95.0}
|
||||
})
|
||||
)
|
||||
|
||||
refute result.mechanism == :aurora
|
||||
end
|
||||
|
||||
test "quiet Kp=2 on VHF -> NOT aurora even on a polar path" do
|
||||
result =
|
||||
MechanismClassifier.classify(
|
||||
inputs(%{
|
||||
band_mhz: 144,
|
||||
kp_index: 2,
|
||||
pos1: %{"lat" => 45.0, "lon" => -95.0},
|
||||
pos2: %{"lat" => 49.0, "lon" => -95.0}
|
||||
})
|
||||
)
|
||||
|
||||
refute result.mechanism == :aurora
|
||||
end
|
||||
end
|
||||
|
||||
describe "meteor scatter" do
|
||||
test "6 m 900 km path during Perseids -> :meteor_scatter (no other mechanism fires)" do
|
||||
result =
|
||||
MechanismClassifier.classify(
|
||||
inputs(%{
|
||||
band_mhz: 50,
|
||||
distance_km: 900.0,
|
||||
qso_timestamp: ~U[2024-08-12 10:00:00Z],
|
||||
active_meteor_shower: "Perseids"
|
||||
})
|
||||
)
|
||||
|
||||
assert result.mechanism == :meteor_scatter
|
||||
end
|
||||
|
||||
test "no active shower -> NOT meteor_scatter" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{band_mhz: 50, distance_km: 900.0, active_meteor_shower: nil}))
|
||||
|
||||
refute result.mechanism == :meteor_scatter
|
||||
end
|
||||
|
||||
test "10 GHz is far above practical meteor scatter -> NOT meteor_scatter" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{band_mhz: 10_000, active_meteor_shower: "Geminids"}))
|
||||
|
||||
refute result.mechanism == :meteor_scatter
|
||||
end
|
||||
end
|
||||
|
||||
describe "rain scatter (delegates to existing logic)" do
|
||||
test "10 GHz 250 km, heavy rain in CV, no duct -> :rain_scatter" do
|
||||
result =
|
||||
MechanismClassifier.classify(
|
||||
inputs(%{
|
||||
radar: %{max_dbz: 42.0, heavy_rain_pixel_count: 12, coverage_pct: 80.0}
|
||||
})
|
||||
)
|
||||
|
||||
assert result.mechanism == :rain_scatter
|
||||
end
|
||||
|
||||
test "light rain (25-35 dBZ) -> :rain_scatter_possible" do
|
||||
result =
|
||||
MechanismClassifier.classify(
|
||||
inputs(%{
|
||||
radar: %{max_dbz: 28.0, heavy_rain_pixel_count: 0, coverage_pct: 80.0}
|
||||
})
|
||||
)
|
||||
|
||||
assert result.mechanism == :rain_scatter_possible
|
||||
end
|
||||
end
|
||||
|
||||
describe "tropo duct (HRRR native-profile best duct ≥ target band)" do
|
||||
test "10 GHz with native_best_duct_ghz=24 -> :tropo_duct (duct supports target)" do
|
||||
result =
|
||||
MechanismClassifier.classify(inputs(%{native_best_duct_ghz: 24.0}))
|
||||
|
||||
assert result.mechanism == :tropo_duct
|
||||
end
|
||||
|
||||
test "10 GHz with native_best_duct_ghz=5 -> does NOT classify as duct at 10 GHz" do
|
||||
# The sub-10-GHz duct doesn't support 10 GHz propagation.
|
||||
result = MechanismClassifier.classify(inputs(%{native_best_duct_ghz: 5.0}))
|
||||
refute result.mechanism == :tropo_duct
|
||||
end
|
||||
|
||||
test "legacy flag duct_either_endpoint=true still classifies as duct" do
|
||||
result = MechanismClassifier.classify(inputs(%{duct_either_endpoint: true}))
|
||||
assert result.mechanism == :tropo_duct
|
||||
end
|
||||
end
|
||||
|
||||
describe "line of sight" do
|
||||
test "very short 5 km path -> :line_of_sight" do
|
||||
result = MechanismClassifier.classify(inputs(%{distance_km: 5.0}))
|
||||
assert result.mechanism == :line_of_sight
|
||||
end
|
||||
|
||||
test "normal 250 km path -> NOT line_of_sight" do
|
||||
result = MechanismClassifier.classify(inputs())
|
||||
refute result.mechanism == :line_of_sight
|
||||
end
|
||||
end
|
||||
|
||||
describe "troposcatter (default / fallback)" do
|
||||
test "10 GHz 250 km, no rain, no duct, no hint -> :troposcatter" do
|
||||
result = MechanismClassifier.classify(inputs())
|
||||
assert result.mechanism == :troposcatter
|
||||
end
|
||||
end
|
||||
|
||||
describe "confidence levels" do
|
||||
test "user-declared -> :high" do
|
||||
result = MechanismClassifier.classify(inputs(%{user_declared_prop_mode: "RS"}))
|
||||
assert result.confidence == :high
|
||||
end
|
||||
|
||||
test "strong physics signal -> :medium" do
|
||||
result =
|
||||
MechanismClassifier.classify(
|
||||
inputs(%{
|
||||
radar: %{max_dbz: 48.0, heavy_rain_pixel_count: 20, coverage_pct: 85.0}
|
||||
})
|
||||
)
|
||||
|
||||
assert result.confidence in [:high, :medium]
|
||||
end
|
||||
|
||||
test "default troposcatter fallback -> :low" do
|
||||
result = MechanismClassifier.classify(inputs())
|
||||
assert result.confidence == :low
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
@ -50,6 +50,22 @@ defmodule Microwaveprop.Radio.AdifImportTest do
|
|||
assert row.attrs["mode"] == "CW"
|
||||
end
|
||||
|
||||
test "captures PROP_MODE into user_declared_prop_mode" do
|
||||
adif = """
|
||||
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<BAND:4>23cm<MODE:2>CW<PROP_MODE:2>ES<QSO_DATE:8>20260605<TIME_ON:6>180000<EOR>
|
||||
"""
|
||||
|
||||
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
||||
assert [row] = preview.valid
|
||||
assert row.attrs["user_declared_prop_mode"] == "ES"
|
||||
end
|
||||
|
||||
test "missing PROP_MODE keeps user_declared_prop_mode nil" do
|
||||
assert {:ok, preview} = AdifImport.preview(@valid_adif, @submitter)
|
||||
[row] = preview.valid
|
||||
assert row.attrs["user_declared_prop_mode"] == nil
|
||||
end
|
||||
|
||||
test "parses ADIF with BAND field instead of FREQ" do
|
||||
adif = """
|
||||
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<BAND:4>23cm<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
||||
|
|
|
|||
141
test/microwaveprop/workers/mechanism_classify_worker_test.exs
Normal file
141
test/microwaveprop/workers/mechanism_classify_worker_test.exs
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
defmodule Microwaveprop.Workers.MechanismClassifyWorkerTest do
|
||||
use Microwaveprop.DataCase, async: true
|
||||
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Radio.ContactCommonVolumeRadar
|
||||
alias Microwaveprop.Workers.MechanismClassifyWorker
|
||||
|
||||
defp create_contact(attrs \\ %{}) do
|
||||
default = %{
|
||||
station1: "W5AA",
|
||||
station2: "K5TR",
|
||||
qso_timestamp: ~U[2024-08-15 19:00:00Z],
|
||||
mode: "CW",
|
||||
band: Decimal.new("10000"),
|
||||
grid1: "EM12",
|
||||
grid2: "EM00",
|
||||
pos1: %{"lat" => 32.9, "lon" => -97.0},
|
||||
pos2: %{"lat" => 30.3, "lon" => -97.7},
|
||||
distance_km: Decimal.new("295")
|
||||
}
|
||||
|
||||
{:ok, contact} =
|
||||
%Contact{}
|
||||
|> Contact.changeset(Map.merge(default, attrs))
|
||||
|> Repo.insert()
|
||||
|
||||
contact
|
||||
end
|
||||
|
||||
defp create_radar(contact, attrs) do
|
||||
{:ok, row} =
|
||||
%ContactCommonVolumeRadar{}
|
||||
|> ContactCommonVolumeRadar.changeset(
|
||||
Map.merge(
|
||||
%{
|
||||
contact_id: contact.id,
|
||||
observed_at: DateTime.to_naive(contact.qso_timestamp),
|
||||
max_dbz: 42.0,
|
||||
mean_dbz: 28.0,
|
||||
pixel_count: 100,
|
||||
rain_pixel_count: 80,
|
||||
heavy_rain_pixel_count: 15,
|
||||
core_pixel_count: 3,
|
||||
coverage_pct: 85.0,
|
||||
common_volume_km2: 1_200.0
|
||||
},
|
||||
attrs
|
||||
)
|
||||
)
|
||||
|> Repo.insert()
|
||||
|
||||
row
|
||||
end
|
||||
|
||||
describe "perform/1" do
|
||||
test "classifies a 10 GHz contact with heavy rain in CV as :rain_scatter" do
|
||||
contact = create_contact()
|
||||
create_radar(contact, %{})
|
||||
|
||||
assert :ok =
|
||||
MechanismClassifyWorker.perform(%Oban.Job{
|
||||
args: %{"contact_id" => contact.id}
|
||||
})
|
||||
|
||||
reloaded = Repo.get!(Contact, contact.id)
|
||||
assert reloaded.propagation_mechanism == "rain_scatter"
|
||||
assert reloaded.propagation_mechanism_status == :complete
|
||||
assert reloaded.propagation_mechanism_confidence == :high
|
||||
end
|
||||
|
||||
test "classifies a plain 10 GHz short-path contact with no enrichment as :troposcatter" do
|
||||
contact = create_contact()
|
||||
|
||||
assert :ok =
|
||||
MechanismClassifyWorker.perform(%Oban.Job{
|
||||
args: %{"contact_id" => contact.id}
|
||||
})
|
||||
|
||||
reloaded = Repo.get!(Contact, contact.id)
|
||||
assert reloaded.propagation_mechanism == "troposcatter"
|
||||
assert reloaded.propagation_mechanism_status == :complete
|
||||
assert reloaded.propagation_mechanism_confidence == :low
|
||||
end
|
||||
|
||||
test "trusts user_declared_prop_mode='EME' over physics" do
|
||||
contact =
|
||||
create_contact(%{
|
||||
user_declared_prop_mode: "EME",
|
||||
band: Decimal.new("1296"),
|
||||
distance_km: Decimal.new("400")
|
||||
})
|
||||
|
||||
assert :ok =
|
||||
MechanismClassifyWorker.perform(%Oban.Job{
|
||||
args: %{"contact_id" => contact.id}
|
||||
})
|
||||
|
||||
reloaded = Repo.get!(Contact, contact.id)
|
||||
assert reloaded.propagation_mechanism == "eme"
|
||||
assert reloaded.propagation_mechanism_confidence == :high
|
||||
end
|
||||
|
||||
test "short 5 km path classifies as :line_of_sight" do
|
||||
contact =
|
||||
create_contact(%{
|
||||
pos1: %{"lat" => 32.90, "lon" => -97.0},
|
||||
pos2: %{"lat" => 32.93, "lon" => -97.0},
|
||||
distance_km: Decimal.new("5")
|
||||
})
|
||||
|
||||
assert :ok =
|
||||
MechanismClassifyWorker.perform(%Oban.Job{
|
||||
args: %{"contact_id" => contact.id}
|
||||
})
|
||||
|
||||
reloaded = Repo.get!(Contact, contact.id)
|
||||
assert reloaded.propagation_mechanism == "line_of_sight"
|
||||
end
|
||||
|
||||
test "contact without positions is marked unavailable, not failed" do
|
||||
contact = create_contact(%{pos1: nil, pos2: nil})
|
||||
|
||||
assert :ok =
|
||||
MechanismClassifyWorker.perform(%Oban.Job{
|
||||
args: %{"contact_id" => contact.id}
|
||||
})
|
||||
|
||||
reloaded = Repo.get!(Contact, contact.id)
|
||||
assert reloaded.propagation_mechanism_status == :unavailable
|
||||
end
|
||||
|
||||
test "missing contact (deleted) returns :ok gracefully" do
|
||||
fake_id = Ecto.UUID.generate()
|
||||
|
||||
assert :ok =
|
||||
MechanismClassifyWorker.perform(%Oban.Job{
|
||||
args: %{"contact_id" => fake_id}
|
||||
})
|
||||
end
|
||||
end
|
||||
end
|
||||
Loading…
Add table
Reference in a new issue