Graham McIntire
622edee180
Classifies every contact's likely non-LOS propagation mechanism and persists the result on contacts.propagation_mechanism. Mechanism is determined in priority order: 1. user_declared_prop_mode (ADIF PROP_MODE from the operator log) 2. EME — moon-ephemeris check, ≥2m band, >1800 km path 3. aurora — Kp≥5 + high-lat path, 50-432 MHz 4. sporadic-E — foEs × 5 ≥ band_mhz, 400-2500 km path 5. meteor_scatter — ±3 days of a shower peak, VHF/UHF 6. rain_scatter — common-volume radar heavy rain, 5-11 GHz ≤800 km 7. tropo_duct — HRRR native_best_duct ≥ band or ducting_detected 8. line_of_sight — ≤50 km path 9. troposcatter — default Persisted via MechanismClassifyWorker (queue: :mechanism, unique on contact_id). Submit-time enqueue path includes :mechanism by default; BackfillEnqueueWorker cron now handles :mechanism alongside existing types so prod continuously backfills any contact with propagation_mechanism_status in (:pending, :queued, :failed). Also added :radar to the cron's type list so common-volume radar backfill runs automatically rather than only via `mix radar_backfill`. New modules: - Microwaveprop.Propagation.MoonEphemeris — Meeus low-precision moon position, accuracy ±1° — enough for the mutual-visibility EME test - Microwaveprop.Propagation.MechanismClassifier — plug-in priority chain over the evidence map - Microwaveprop.Workers.MechanismClassifyWorker — assembles inputs from HRRR / native profiles / common-volume radar / solar_indices / ionosonde + calls the classifier ADIF importer now reads PROP_MODE into user_declared_prop_mode so operator-tagged mechanisms (EME/ES/MS/RS/AS/AUR) become ground truth.
387 lines
14 KiB
Elixir
387 lines
14 KiB
Elixir
defmodule Microwaveprop.Radio.AdifImportTest do
|
|
use Microwaveprop.DataCase, async: true
|
|
|
|
alias Microwaveprop.Radio.AdifImport
|
|
alias Microwaveprop.Terrain.ElevationClient
|
|
alias Microwaveprop.Weather.HrrrClient
|
|
alias Microwaveprop.Weather.IemClient
|
|
|
|
setup do
|
|
Req.Test.stub(IemClient, fn conn ->
|
|
case conn.request_path do
|
|
"/cgi-bin/request/asos.py" -> Req.Test.text(conn, "#DEBUG,\nstation,valid,tmpf\n")
|
|
_ -> Req.Test.json(conn, %{"profiles" => []})
|
|
end
|
|
end)
|
|
|
|
Req.Test.stub(HrrrClient, fn conn ->
|
|
Plug.Conn.send_resp(conn, 404, "not found")
|
|
end)
|
|
|
|
Req.Test.stub(ElevationClient, fn conn ->
|
|
params = Plug.Conn.fetch_query_params(conn).query_params
|
|
lat_count = params["latitude"] |> String.split(",") |> length()
|
|
Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
|
|
end)
|
|
|
|
:ok
|
|
end
|
|
|
|
@submitter "test@example.com"
|
|
|
|
@valid_adif """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
describe "preview/2" do
|
|
test "parses a valid ADIF record" do
|
|
assert {:ok, preview} = AdifImport.preview(@valid_adif, @submitter)
|
|
assert length(preview.valid) == 1
|
|
assert preview.invalid == []
|
|
assert preview.duplicates == []
|
|
assert preview.refinements == []
|
|
|
|
[row] = preview.valid
|
|
assert row.attrs["station1"] == "W5XD"
|
|
assert row.attrs["station2"] == "K5TR"
|
|
assert row.attrs["grid1"] == "EM12"
|
|
assert row.attrs["grid2"] == "EM00"
|
|
assert row.attrs["band"] == "10000"
|
|
assert row.attrs["mode"] == "CW"
|
|
end
|
|
|
|
test "captures PROP_MODE into user_declared_prop_mode" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<BAND:4>23cm<MODE:2>CW<PROP_MODE:2>ES<QSO_DATE:8>20260605<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["user_declared_prop_mode"] == "ES"
|
|
end
|
|
|
|
test "missing PROP_MODE keeps user_declared_prop_mode nil" do
|
|
assert {:ok, preview} = AdifImport.preview(@valid_adif, @submitter)
|
|
[row] = preview.valid
|
|
assert row.attrs["user_declared_prop_mode"] == nil
|
|
end
|
|
|
|
test "parses ADIF with BAND field instead of FREQ" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<BAND:4>23cm<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["band"] == "1296"
|
|
end
|
|
|
|
test "accepts 33cm ADIF band name and maps it to 902 MHz" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<BAND:4>33cm<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["band"] == "902"
|
|
end
|
|
|
|
test "maps a 902 MHz FREQ value onto the 902 band" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:7>903.100<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["band"] == "902"
|
|
end
|
|
|
|
test "maps ADIF frequency to correct band" do
|
|
for {freq, expected_band} <- [
|
|
{"1296.200", "1296"},
|
|
{"2304.000", "2304"},
|
|
{"5760.100", "5760"},
|
|
{"10368.000", "10000"},
|
|
{"24192.000", "24000"},
|
|
{"47088.000", "47000"},
|
|
{"76032.000", "75000"}
|
|
] do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:#{byte_size(freq)}>#{freq}<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
|
|
assert row.attrs["band"] == expected_band,
|
|
"FREQ #{freq} should map to band #{expected_band}, got #{row.attrs["band"]}"
|
|
end
|
|
end
|
|
|
|
test "handles multiple records" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
<STATION_CALLSIGN:4>W5XD<CALL:5>N5DUP<GRIDSQUARE:4>EM13<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:3>SSB<QSO_DATE:8>20260328<TIME_ON:6>190000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert length(preview.valid) == 2
|
|
end
|
|
|
|
test "skips ADIF header before <EOH>" do
|
|
adif = """
|
|
ADIF Export from Logger32
|
|
<ADIF_VER:5>3.1.4
|
|
<PROGRAMID:8>Logger32
|
|
<EOH>
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert length(preview.valid) == 1
|
|
end
|
|
|
|
test "returns error for empty ADIF" do
|
|
assert {:error, :no_records} = AdifImport.preview("", @submitter)
|
|
end
|
|
|
|
test "returns error for header-only ADIF" do
|
|
adif = """
|
|
ADIF Export
|
|
<ADIF_VER:5>3.1.4
|
|
<EOH>
|
|
"""
|
|
|
|
assert {:error, :no_records} = AdifImport.preview(adif, @submitter)
|
|
end
|
|
|
|
test "marks record invalid when missing required fields" do
|
|
adif = """
|
|
<CALL:4>K5TR<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert preview.valid == []
|
|
assert length(preview.invalid) == 1
|
|
end
|
|
|
|
test "skips HF contacts" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:6>14.200<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert preview.valid == []
|
|
assert length(preview.invalid) == 1
|
|
end
|
|
|
|
test "accepts 2m contacts" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:7>144.200<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [%{attrs: %{"band" => "144"}}] = preview.valid
|
|
end
|
|
|
|
test "deduplicates against existing DB contacts" do
|
|
# Insert one via CSV import path
|
|
{:ok, _} =
|
|
Microwaveprop.Radio.create_contact(%{
|
|
station1: "W5XD",
|
|
station2: "K5TR",
|
|
grid1: "EM12",
|
|
grid2: "EM00",
|
|
band: "10000",
|
|
mode: "CW",
|
|
qso_timestamp: ~U[2026-03-28 18:00:00Z],
|
|
submitter_email: @submitter
|
|
})
|
|
|
|
assert {:ok, preview} = AdifImport.preview(@valid_adif, @submitter)
|
|
assert preview.valid == []
|
|
assert preview.refinements == []
|
|
assert length(preview.duplicates) == 1
|
|
end
|
|
|
|
test "upload with longer grid than existing routes to refinements" do
|
|
{:ok, existing} =
|
|
Microwaveprop.Radio.create_contact(%{
|
|
station1: "W5XD",
|
|
station2: "K5TR",
|
|
grid1: "EM12",
|
|
grid2: "EM00",
|
|
band: "10000",
|
|
mode: "CW",
|
|
qso_timestamp: ~U[2026-03-28 18:00:00Z],
|
|
submitter_email: @submitter
|
|
})
|
|
|
|
adif =
|
|
"<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:8>EM12kp37<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180500<EOR>"
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert preview.valid == []
|
|
assert preview.duplicates == []
|
|
assert [ref] = preview.refinements
|
|
assert ref.existing_id == existing.id
|
|
assert ref.changes == %{grid1: "EM12KP37"}
|
|
end
|
|
|
|
test "is case-insensitive for field names" do
|
|
adif = """
|
|
<station_callsign:4>W5XD<call:4>K5TR<gridsquare:4>EM00<my_gridsquare:4>EM12<freq:9>10368.000<mode:2>CW<qso_date:8>20260328<time_on:6>180000<eor>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert length(preview.valid) == 1
|
|
end
|
|
|
|
test "handles TIME_ON with 4 digits (HHMM)" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:4>1800<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.timestamp == ~U[2026-03-28 18:00:00Z]
|
|
end
|
|
|
|
test "uses OPERATOR field when STATION_CALLSIGN is missing" do
|
|
adif = """
|
|
<OPERATOR:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["station1"] == "W5XD"
|
|
end
|
|
|
|
test "normalizes gridsquare case to uppercase" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>em00<MY_GRIDSQUARE:6>em12kp<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["grid1"] == "EM12KP"
|
|
assert row.attrs["grid2"] == "EM00"
|
|
end
|
|
|
|
test "parses file with header tags but no <EOH>" do
|
|
adif = """
|
|
<ADIF_VER:5>3.1.4
|
|
<PROGRAMID:8>Logger32
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert length(preview.valid) == 1
|
|
end
|
|
|
|
test "strips APP_ prefix from app-defined fields" do
|
|
adif = """
|
|
<APP_N1MM_STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<APP_MYLOG_GRIDSQUARE:4>EM00<APP_MYLOG_MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["station1"] == "W5XD"
|
|
assert row.attrs["grid1"] == "EM12"
|
|
assert row.attrs["grid2"] == "EM00"
|
|
end
|
|
|
|
test "standard fields take precedence over app-defined" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<APP_FOO_STATION_CALLSIGN:5>WRONG<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["station1"] == "W5XD"
|
|
end
|
|
|
|
test "normalizes USB / LSB / DSB to SSB" do
|
|
for sideband <- ~w(USB LSB DSB) do
|
|
adif =
|
|
"<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:#{byte_size(sideband)}>#{sideband}<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>"
|
|
|
|
assert {:ok, %{valid: [row]}} = AdifImport.preview(adif, @submitter)
|
|
assert row.attrs["mode"] == "SSB", "expected #{sideband} to map to SSB"
|
|
end
|
|
end
|
|
|
|
test "normalizes NBFM / WFM to FM" do
|
|
for variant <- ~w(NBFM WFM) do
|
|
adif =
|
|
"<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:#{byte_size(variant)}>#{variant}<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>"
|
|
|
|
assert {:ok, %{valid: [row]}} = AdifImport.preview(adif, @submitter)
|
|
assert row.attrs["mode"] == "FM"
|
|
end
|
|
end
|
|
|
|
test "uses SUBMODE when MODE is a generic digital mode" do
|
|
# ADIF convention: digital modes are reported as MODE=MFSK, SUBMODE=FT8
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:4>MFSK<SUBMODE:3>FT8<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, %{valid: [row]}} = AdifImport.preview(adif, @submitter)
|
|
assert row.attrs["mode"] == "FT8"
|
|
end
|
|
|
|
test "SUBMODE takes precedence when both are allowed values" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:3>MFSK<SUBMODE:3>FT4<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, %{valid: [row]}} = AdifImport.preview(adif, @submitter)
|
|
assert row.attrs["mode"] == "FT4"
|
|
end
|
|
|
|
test "unknown modes are stored as nil so the row still imports" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:4>RTTY<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, %{valid: [row]}} = AdifImport.preview(adif, @submitter)
|
|
assert row.attrs["mode"] == nil
|
|
end
|
|
|
|
test "missing MODE imports as nil" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, %{valid: [row]}} = AdifImport.preview(adif, @submitter)
|
|
assert row.attrs["mode"] == nil
|
|
end
|
|
|
|
test "parses file with no header at all" do
|
|
adif = """
|
|
<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert length(preview.valid) == 1
|
|
end
|
|
|
|
test "parses records containing UTF-8 characters in earlier fields" do
|
|
# Regression: Regex.scan with `return: :index` returns byte offsets,
|
|
# but String.slice/3 uses character offsets. A multi-byte UTF-8 char
|
|
# (é = 2 bytes) in a NOTES field earlier in the record used to shift
|
|
# every subsequent field's character-based slice one byte left,
|
|
# causing a crash when the size slice landed on a ">" character.
|
|
adif = """
|
|
<NOTES:6>caf\u00e9 QSO<STATION_CALLSIGN:4>W5XD<CALL:4>K5TR<GRIDSQUARE:4>EM00<MY_GRIDSQUARE:4>EM12<FREQ:9>10368.000<MODE:2>CW<QSO_DATE:8>20260328<TIME_ON:6>180000<EOR>
|
|
"""
|
|
|
|
assert {:ok, preview} = AdifImport.preview(adif, @submitter)
|
|
assert [row] = preview.valid
|
|
assert row.attrs["station1"] == "W5XD"
|
|
assert row.attrs["station2"] == "K5TR"
|
|
assert row.attrs["band"] == "10000"
|
|
end
|
|
end
|
|
end
|