diff --git a/assets/css/app.css b/assets/css/app.css index ffa836ec..0f3f2846 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -105,6 +105,14 @@ /* Make LiveView wrapper divs transparent for layout */ [data-phx-session], [data-phx-teleported-src] { display: contents } +/* Fix daisyUI select text alignment — when select class is on the . */ +select.select { + margin: 0; + padding-block: 0; +} + /* Markdown rendered content — uses daisyUI theme variables for auto dark/light */ .markdown-content { max-width: 64rem; diff --git a/lib/microwaveprop/radio/adif_import.ex b/lib/microwaveprop/radio/adif_import.ex new file mode 100644 index 00000000..2740ed4f --- /dev/null +++ b/lib/microwaveprop/radio/adif_import.ex @@ -0,0 +1,286 @@ +defmodule Microwaveprop.Radio.AdifImport do + @moduledoc """ + Parse an ADIF file into the same preview format used by CsvImport, so the + submit UI can reuse the preview/commit flow. + + ADIF tagged-data format: `VALUE` + Records end with ``. An optional header ends with ``. + """ + + import Ecto.Query, only: [from: 2] + + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Repo + + @dedup_window_seconds 3600 + + # Band plan: map ADIF band names to our MHz values + @band_name_to_mhz %{ + "23cm" => 1296, + "13cm" => 2304, + "9cm" => 3456, + "6cm" => 5760, + "3cm" => 10_000, + "1.25cm" => 24_000, + "6mm" => 47_000, + "4mm" => 75_000, + "2.5mm" => 122_000, + "2mm" => 134_000, + "1mm" => 241_000 + } + + # Our allowed band centers in MHz, sorted ascending for nearest-match + @allowed_bands [1296, 2304, 3456, 5760, 10_000, 24_000, 47_000, 68_000, 75_000, 122_000, 134_000, 241_000] + + @doc """ + Parse ADIF string, validate, and deduplicate. Returns the same preview + shape as `CsvImport.preview/2`. + """ + def preview(adif_string, submitter_email) do + records = parse_adif(adif_string) + + if records == [] do + {:error, :no_records} + else + rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email)) + {valid, invalid} = split_parsed(rows) + existing = load_existing_for_dedup(valid) + {unique, duplicates} = dedupe(valid, existing) + + {:ok, + %{ + valid: unique, + invalid: invalid, + duplicates: duplicates, + total_rows: length(records), + submitter_email: submitter_email + }} + end + end + + # -- ADIF parser ----------------------------------------------------------- + + defp parse_adif(string) do + # Strip header if present (everything before ) + body = + case Regex.run(~r//i, string, return: :index) do + [{pos, len}] -> String.slice(string, (pos + len)..-1//1) + nil -> string + end + + # Split on to get individual records + body + |> String.split(~r//i) + |> Enum.map(&parse_fields/1) + |> Enum.reject(&(&1 == %{})) + end + + defp parse_fields(record_string) do + ~r/<([^:>]+):(\d+)(?::[^>]*)?>/i + |> Regex.scan(record_string, return: :index) + |> Enum.reduce(%{}, fn indices, acc -> + [{tag_start, _tag_len}, {name_start, name_len}, {size_start, size_len} | _] = indices + name = record_string |> String.slice(name_start, name_len) |> String.upcase() + size = record_string |> String.slice(size_start, size_len) |> String.to_integer() + + # Value starts right after the closing > + # Find the > after the tag + tag_text = String.slice(record_string, tag_start, 999) + value_offset = tag_start + byte_size(String.split(tag_text, ">", parts: 2) |> List.first()) + 1 + value = String.slice(record_string, value_offset, size) + + Map.put(acc, name, value) + end) + end + + # -- row building ---------------------------------------------------------- + + defp build_row({fields, row_num}, submitter_email) do + station1 = fields["STATION_CALLSIGN"] || fields["OPERATOR"] + station2 = fields["CALL"] + grid1 = fields["MY_GRIDSQUARE"] + grid2 = fields["GRIDSQUARE"] + mode = fields["MODE"] + band = resolve_band(fields) + timestamp = parse_adif_datetime(fields["QSO_DATE"], fields["TIME_ON"]) + + case {band, timestamp} do + {nil, _} -> + {:invalid, %{row_num: row_num, messages: ["could not determine a valid microwave band"]}} + + {_, nil} -> + {:invalid, %{row_num: row_num, messages: ["could not parse QSO_DATE/TIME_ON"]}} + + {band_str, {:ok, dt}} -> + attrs = %{ + "station1" => station1, + "station2" => station2, + "grid1" => grid1, + "grid2" => grid2, + "band" => band_str, + "mode" => mode, + "qso_timestamp" => DateTime.to_iso8601(dt), + "submitter_email" => submitter_email + } + + changeset = Contact.submission_changeset(%Contact{}, attrs) + + if changeset.valid? do + {:parsed, %{row_num: row_num, attrs: attrs, timestamp: dt}} + else + messages = changeset_error_strings(changeset) + {:invalid, %{row_num: row_num, messages: messages}} + end + end + end + + # -- band resolution ------------------------------------------------------- + + defp resolve_band(fields) do + cond do + freq = fields["FREQ"] -> + freq_mhz = parse_freq_mhz(freq) + nearest_band(freq_mhz) + + band_name = fields["BAND"] -> + band_name = String.downcase(band_name) + mhz = @band_name_to_mhz[band_name] + if mhz, do: to_string(mhz) + + true -> + nil + end + end + + defp parse_freq_mhz(freq_str) do + {freq, _} = Float.parse(String.trim(freq_str)) + # ADIF FREQ is always in MHz (e.g., 10368.000 for 10 GHz band) + freq + end + + defp nearest_band(freq_mhz) when freq_mhz >= 900 do + @allowed_bands + |> Enum.min_by(&abs(&1 - freq_mhz)) + |> to_string() + end + + defp nearest_band(_freq_mhz), do: nil + + # -- timestamp parsing ----------------------------------------------------- + + defp parse_adif_datetime(nil, _time), do: nil + defp parse_adif_datetime(_date, nil), do: nil + + defp parse_adif_datetime(date_str, time_str) do + with <> <- date_str, + {hour, min, sec} <- parse_adif_time(time_str), + {:ok, naive} <- NaiveDateTime.new(si(y), si(m), si(d), hour, min, sec) do + {:ok, DateTime.from_naive!(naive, "Etc/UTC")} + else + _ -> nil + end + end + + defp parse_adif_time(<>), do: {si(h), si(m), si(s)} + defp parse_adif_time(<>), do: {si(h), si(m), 0} + defp parse_adif_time(_), do: nil + + defp si(s), do: String.to_integer(s) + + # -- split / dedup (same logic as CsvImport) -------------------------------- + + defp split_parsed(rows) do + rows + |> Enum.reduce({[], []}, fn + {:parsed, row}, {valid, invalid} -> {[row | valid], invalid} + {:invalid, row}, {valid, invalid} -> {valid, [row | invalid]} + end) + |> then(fn {valid, invalid} -> {Enum.reverse(valid), Enum.reverse(invalid)} end) + end + + defp load_existing_for_dedup([]), do: %{} + + defp load_existing_for_dedup(valid_rows) do + bands = valid_rows |> Enum.map(& &1.attrs["band"]) |> Enum.uniq() |> Enum.map(&Decimal.new/1) + timestamps = Enum.map(valid_rows, & &1.timestamp) + min_ts = timestamps |> Enum.min(DateTime) |> DateTime.add(-@dedup_window_seconds, :second) + max_ts = timestamps |> Enum.max(DateTime) |> DateTime.add(@dedup_window_seconds, :second) + + from(c in Contact, + where: + c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and + c.flagged_invalid == false, + select: %{ + station1: c.station1, + station2: c.station2, + grid1: c.grid1, + grid2: c.grid2, + band: c.band, + qso_timestamp: c.qso_timestamp + } + ) + |> Repo.all() + |> Enum.reduce(%{}, fn row, acc -> + key = dedup_key(row.station1, row.grid1, row.station2, row.grid2, row.band) + Map.update(acc, key, [row.qso_timestamp], &[row.qso_timestamp | &1]) + end) + end + + defp dedupe(rows, db_map) do + {accepted, _in_batch, duplicates} = + Enum.reduce(rows, {[], %{}, []}, fn row, {accepted, in_batch, dups} -> + key = dedup_key(row.attrs["station1"], row.attrs["grid1"], row.attrs["station2"], row.attrs["grid2"], row.attrs["band"]) + + cond do + conflict_in_map?(db_map, key, row.timestamp) -> + {accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups]} + + conflict_in_map?(in_batch, key, row.timestamp) -> + {accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups]} + + true -> + new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1]) + {[row | accepted], new_batch, dups} + end + end) + + {Enum.reverse(accepted), Enum.reverse(duplicates)} + end + + defp conflict_in_map?(map, key, timestamp) do + case Map.fetch(map, key) do + :error -> false + {:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp)) + end + end + + defp within_window?(t1, t2) do + abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds + end + + defp dedup_key(s1, g1, s2, g2, band) do + a = {upcase(s1), upcase(g1)} + b = {upcase(s2), upcase(g2)} + [c1, c2] = Enum.sort([a, b]) + {c1, c2, normalize_band(band)} + end + + defp upcase(nil), do: "" + defp upcase(v), do: v |> to_string() |> String.trim() |> String.upcase() + + defp normalize_band(%Decimal{} = b), do: Decimal.to_integer(b) + defp normalize_band(b) when is_integer(b), do: b + defp normalize_band(b) when is_binary(b), do: b |> String.trim() |> String.to_integer() + + defp changeset_error_strings(changeset) do + changeset + |> Ecto.Changeset.traverse_errors(fn {message, opts} -> + Regex.replace(~r"%{(\w+)}", message, fn _, key -> + opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string() + end) + end) + |> Enum.flat_map(fn {field, messages} -> + Enum.map(messages, &"#{field} #{&1}") + end) + end +end diff --git a/lib/microwaveprop_web/live/submit_live.ex b/lib/microwaveprop_web/live/submit_live.ex index 1cb5efde..7dd3d548 100644 --- a/lib/microwaveprop_web/live/submit_live.ex +++ b/lib/microwaveprop_web/live/submit_live.ex @@ -6,6 +6,7 @@ defmodule MicrowavepropWeb.SubmitLive do alias Microwaveprop.Propagation.BandConfig alias Microwaveprop.Radio alias Microwaveprop.Radio.Contact + alias Microwaveprop.Radio.AdifImport alias Microwaveprop.Radio.CsvImport alias Microwaveprop.Workers.ContactWeatherEnqueueWorker @@ -15,7 +16,10 @@ defmodule MicrowavepropWeb.SubmitLive do @impl true def mount(_params, _session, socket) do - socket = allow_upload(socket, :csv, accept: ~w(.csv), max_entries: 1, max_file_size: 10_000_000) + socket = + socket + |> allow_upload(:csv, accept: ~w(.csv), max_entries: 1, max_file_size: 10_000_000) + |> allow_upload(:adif, accept: :any, max_entries: 1, max_file_size: 10_000_000) changeset = Radio.change_contact(%Contact{}) @@ -60,6 +64,10 @@ defmodule MicrowavepropWeb.SubmitLive do {:noreply, socket} end + def handle_event("validate_adif", _params, socket) do + {:noreply, socket} + end + # Minimum milliseconds between submissions per LiveView session. # For broader IP-based rate limiting, use a reverse proxy (nginx limit_req, Cloudflare). @submission_cooldown_ms 30_000 @@ -124,6 +132,35 @@ defmodule MicrowavepropWeb.SubmitLive do {:noreply, cancel_upload(socket, :csv, ref)} end + def handle_event("upload_adif", params, socket) do + email = + case current_user(socket.assigns) do + %{email: user_email} -> user_email + _ -> params |> Map.get("submitter_email", "") |> String.trim() + end + + if email == "" do + {:noreply, put_flash(socket, :error, "Email is required for ADIF upload")} + else + uploaded_contents = + consume_uploaded_entries(socket, :adif, fn %{path: path}, _entry -> + {:ok, File.read!(path)} + end) + + case uploaded_contents do + [content] -> + handle_adif_preview(content, email, socket) + + [] -> + {:noreply, put_flash(socket, :error, "Please select an ADIF file")} + end + end + end + + def handle_event("cancel_adif_upload", %{"ref" => ref}, socket) do + {:noreply, cancel_upload(socket, :adif, ref)} + end + def handle_event("confirm_csv", _params, socket) do case socket.assigns.csv_preview do %{valid: valid_rows} when valid_rows != [] -> @@ -147,6 +184,16 @@ defmodule MicrowavepropWeb.SubmitLive do {:noreply, assign(socket, csv_preview: nil, csv_result: nil)} end + defp handle_adif_preview(content, email, socket) do + case AdifImport.preview(content, email) do + {:ok, preview} -> + {:noreply, assign(socket, csv_preview: preview, csv_result: nil, csv_email: email)} + + {:error, :no_records} -> + {:noreply, put_flash(socket, :error, "ADIF file contains no records")} + end + end + defp handle_csv_preview(content, email, socket) do case CsvImport.preview(content, email) do {:ok, preview} -> @@ -181,7 +228,7 @@ defmodule MicrowavepropWeb.SubmitLive do defp error_to_string(:too_large), do: "File is too large" defp error_to_string(:too_many_files), do: "Only one file allowed" - defp error_to_string(:not_accepted), do: "Only .csv files are accepted" + defp error_to_string(:not_accepted), do: "File type not accepted" defp error_to_string(other), do: to_string(other) @impl true @@ -236,24 +283,44 @@ defmodule MicrowavepropWeb.SubmitLive do > Upload CSV + - <%= if @active_tab == :single do %> - <.single_contact_form - form={@form} - band_options={@band_options} - mode_options={@mode_options} - current_user={current_user(assigns)} - /> - <% else %> - <%= cond do %> - <% @csv_result -> %> - <.csv_results result={@csv_result} /> - <% @csv_preview -> %> - <.csv_preview preview={@csv_preview} /> - <% true -> %> - <.csv_upload_form uploads={@uploads} current_user={current_user(assigns)} /> - <% end %> + <%= case @active_tab do %> + <% :single -> %> + <.single_contact_form + form={@form} + band_options={@band_options} + mode_options={@mode_options} + current_user={current_user(assigns)} + /> + <% :csv -> %> + <%= cond do %> + <% @csv_result -> %> + <.csv_results result={@csv_result} /> + <% @csv_preview -> %> + <.csv_preview preview={@csv_preview} /> + <% true -> %> + <.csv_upload_form uploads={@uploads} current_user={current_user(assigns)} /> + <% end %> + <% :adif -> %> + <%= cond do %> + <% @csv_result -> %> + <.csv_results result={@csv_result} /> + <% @csv_preview -> %> + <.csv_preview preview={@csv_preview} /> + <% true -> %> + <.adif_upload_form uploads={@uploads} current_user={current_user(assigns)} /> + <% end %> <% end %> """ @@ -426,6 +493,91 @@ defmodule MicrowavepropWeb.SubmitLive do """ end + defp adif_upload_form(assigns) do + ~H""" +
+
+

+ Upload an ADIF (.adi / .adif) file exported from your logging program. +

+
    +
  • + Required fields: CALL, STATION_CALLSIGN (or OPERATOR), + GRIDSQUARE, MY_GRIDSQUARE, QSO_DATE, TIME_ON, + and FREQ or BAND +
  • +
  • Only microwave contacts (902 MHz and up) will be imported
  • +
  • Sub-microwave contacts are silently skipped
  • +
  • Frequencies are fuzzy-matched to the nearest microwave band
  • +
+
+ +
+
+ +
+ +
+
+ {entry.client_name} + + {entry.progress}% + + +
+ + +
+ {error_to_string(err)} +
+
+ +
+ {error_to_string(err)} +
+ + <%= if @current_user do %> + + <% else %> +
+ +
+ <% end %> + +
+ +
+
+
+ """ + end + defp csv_preview(assigns) do assigns = assign(assigns, diff --git a/test/microwaveprop/radio/adif_import_test.exs b/test/microwaveprop/radio/adif_import_test.exs new file mode 100644 index 00000000..9a6f3fb1 --- /dev/null +++ b/test/microwaveprop/radio/adif_import_test.exs @@ -0,0 +1,187 @@ +defmodule Microwaveprop.Radio.AdifImportTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.Radio.AdifImport + alias Microwaveprop.Terrain.ElevationClient + alias Microwaveprop.Weather.HrrrClient + alias Microwaveprop.Weather.IemClient + + setup do + Req.Test.stub(IemClient, fn conn -> + case conn.request_path do + "/cgi-bin/request/asos.py" -> Req.Test.text(conn, "#DEBUG,\nstation,valid,tmpf\n") + _ -> Req.Test.json(conn, %{"profiles" => []}) + end + end) + + Req.Test.stub(HrrrClient, fn conn -> + Plug.Conn.send_resp(conn, 404, "not found") + end) + + Req.Test.stub(ElevationClient, fn conn -> + params = Plug.Conn.fetch_query_params(conn).query_params + lat_count = params["latitude"] |> String.split(",") |> length() + Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)}) + end) + + :ok + end + + @submitter "test@example.com" + + @valid_adif """ + W5XDK5TREM00EM1210368.000CW20260328180000 + """ + + describe "preview/2" do + test "parses a valid ADIF record" do + assert {:ok, preview} = AdifImport.preview(@valid_adif, @submitter) + assert length(preview.valid) == 1 + assert preview.invalid == [] + assert preview.duplicates == [] + + [row] = preview.valid + assert row.attrs["station1"] == "W5XD" + assert row.attrs["station2"] == "K5TR" + assert row.attrs["grid1"] == "EM12" + assert row.attrs["grid2"] == "EM00" + assert row.attrs["band"] == "10000" + assert row.attrs["mode"] == "CW" + end + + test "parses ADIF with BAND field instead of FREQ" do + adif = """ + W5XDK5TREM00EM1223cmCW20260328180000 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert [row] = preview.valid + assert row.attrs["band"] == "1296" + end + + test "maps ADIF frequency to correct band" do + for {freq, expected_band} <- [ + {"1296.200", "1296"}, + {"2304.000", "2304"}, + {"5760.100", "5760"}, + {"10368.000", "10000"}, + {"24192.000", "24000"}, + {"47088.000", "47000"}, + {"76032.000", "75000"} + ] do + adif = """ + W5XDK5TREM00EM12#{freq}CW20260328180000 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert [row] = preview.valid + assert row.attrs["band"] == expected_band, "FREQ #{freq} should map to band #{expected_band}, got #{row.attrs["band"]}" + end + end + + test "handles multiple records" do + adif = """ + W5XDK5TREM00EM1210368.000CW20260328180000 + W5XDN5DUPEM13EM1210368.000SSB20260328190000 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert length(preview.valid) == 2 + end + + test "skips ADIF header before " do + adif = """ + ADIF Export from Logger32 + 3.1.4 + Logger32 + + W5XDK5TREM00EM1210368.000CW20260328180000 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert length(preview.valid) == 1 + end + + test "returns error for empty ADIF" do + assert {:error, :no_records} = AdifImport.preview("", @submitter) + end + + test "returns error for header-only ADIF" do + adif = """ + ADIF Export + 3.1.4 + + """ + + assert {:error, :no_records} = AdifImport.preview(adif, @submitter) + end + + test "marks record invalid when missing required fields" do + adif = """ + K5TRCW20260328180000 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert preview.valid == [] + assert length(preview.invalid) == 1 + end + + test "skips non-microwave contacts" do + adif = """ + W5XDK5TREM00EM12144.200CW20260328180000 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert preview.valid == [] + assert length(preview.invalid) == 1 + end + + test "deduplicates against existing DB contacts" do + # Insert one via CSV import path + {:ok, _} = + Microwaveprop.Radio.create_contact(%{ + station1: "W5XD", + station2: "K5TR", + grid1: "EM12", + grid2: "EM00", + band: "10000", + mode: "CW", + qso_timestamp: ~U[2026-03-28 18:00:00Z], + submitter_email: @submitter + }) + + assert {:ok, preview} = AdifImport.preview(@valid_adif, @submitter) + assert preview.valid == [] + assert length(preview.duplicates) == 1 + end + + test "is case-insensitive for field names" do + adif = """ + W5XDK5TREM00EM1210368.000CW20260328180000 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert length(preview.valid) == 1 + end + + test "handles TIME_ON with 4 digits (HHMM)" do + adif = """ + W5XDK5TREM00EM1210368.000CW202603281800 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert [row] = preview.valid + assert row.timestamp == ~U[2026-03-28 18:00:00Z] + end + + test "uses OPERATOR field when STATION_CALLSIGN is missing" do + adif = """ + W5XDK5TREM00EM1210368.000CW20260328180000 + """ + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert [row] = preview.valid + assert row.attrs["station1"] == "W5XD" + end + end +end