defmodule Microwaveprop.Radio.AdifImport do @moduledoc """ Parse an ADIF file into the same preview format used by CsvImport, so the submit UI can reuse the preview/commit flow. ADIF tagged-data format: `VALUE` Records end with ``. An optional header ends with ``. """ import Ecto.Query, only: [from: 2] alias Microwaveprop.Radio.Contact alias Microwaveprop.Repo @dedup_window_seconds 3600 # Band plan: map ADIF band names to our MHz values @band_name_to_mhz %{ "23cm" => 1296, "13cm" => 2304, "9cm" => 3456, "6cm" => 5760, "3cm" => 10_000, "1.25cm" => 24_000, "6mm" => 47_000, "4mm" => 75_000, "2.5mm" => 122_000, "2mm" => 134_000, "1mm" => 241_000 } # Our allowed band centers in MHz, sorted ascending for nearest-match @allowed_bands [1296, 2304, 3456, 5760, 10_000, 24_000, 47_000, 68_000, 75_000, 122_000, 134_000, 241_000] @doc """ Parse ADIF string, validate, and deduplicate. Returns the same preview shape as `CsvImport.preview/2`. """ @type preview_result :: %{ valid: [map()], invalid: [map()], duplicates: [map()], total_rows: non_neg_integer(), submitter_email: String.t() } @spec preview(String.t(), String.t()) :: {:ok, preview_result()} | {:error, :no_records} def preview(adif_string, submitter_email) do records = parse_adif(adif_string) if records == [] do {:error, :no_records} else rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email)) {valid, invalid} = split_parsed(rows) existing = load_existing_for_dedup(valid) {unique, duplicates} = dedupe(valid, existing) {:ok, %{ valid: unique, invalid: invalid, duplicates: duplicates, total_rows: length(records), submitter_email: submitter_email }} end end # -- ADIF parser ----------------------------------------------------------- defp parse_adif(string) do # Strip header if present (everything before ) body = case Regex.run(~r//i, string, return: :index) do [{pos, len}] -> String.slice(string, (pos + len)..-1//1) nil -> string end # Split on to get individual records body |> String.split(~r//i) |> Enum.map(&parse_fields/1) |> Enum.reject(&(&1 == %{})) end defp parse_fields(record_string) do ~r/<([^:>]+):(\d+)(?::[^>]*)?>/i |> Regex.scan(record_string, return: :index) |> Enum.reduce(%{}, fn indices, acc -> [{tag_start, _tag_len}, {name_start, name_len}, {size_start, size_len} | _] = indices name = record_string |> String.slice(name_start, name_len) |> String.upcase() size = record_string |> String.slice(size_start, size_len) |> String.to_integer() # Value starts right after the closing > # Find the > after the tag tag_text = String.slice(record_string, tag_start, 999) value_offset = tag_start + (tag_text |> String.split(">", parts: 2) |> List.first() |> byte_size()) + 1 value = String.slice(record_string, value_offset, size) Map.put(acc, name, value) end) end # -- row building ---------------------------------------------------------- defp build_row({fields, row_num}, submitter_email) do station1 = fields["STATION_CALLSIGN"] || fields["OPERATOR"] station2 = fields["CALL"] grid1 = fields["MY_GRIDSQUARE"] grid2 = fields["GRIDSQUARE"] mode = fields["MODE"] band = resolve_band(fields) timestamp = parse_adif_datetime(fields["QSO_DATE"], fields["TIME_ON"]) case {band, timestamp} do {nil, _} -> {:invalid, %{row_num: row_num, messages: ["could not determine a valid microwave band"]}} {_, nil} -> {:invalid, %{row_num: row_num, messages: ["could not parse QSO_DATE/TIME_ON"]}} {band_str, {:ok, dt}} -> attrs = %{ "station1" => station1, "station2" => station2, "grid1" => grid1, "grid2" => grid2, "band" => band_str, "mode" => mode, "qso_timestamp" => DateTime.to_iso8601(dt), "submitter_email" => submitter_email } changeset = Contact.submission_changeset(%Contact{}, attrs) if changeset.valid? do {:parsed, %{row_num: row_num, attrs: attrs, timestamp: dt}} else messages = changeset_error_strings(changeset) {:invalid, %{row_num: row_num, messages: messages}} end end end # -- band resolution ------------------------------------------------------- defp resolve_band(fields) do cond do freq = fields["FREQ"] -> freq_mhz = parse_freq_mhz(freq) nearest_band(freq_mhz) band_name = fields["BAND"] -> band_name = String.downcase(band_name) mhz = @band_name_to_mhz[band_name] if mhz, do: to_string(mhz) true -> nil end end defp parse_freq_mhz(freq_str) do {freq, _} = Float.parse(String.trim(freq_str)) # ADIF FREQ is always in MHz (e.g., 10368.000 for 10 GHz band) freq end defp nearest_band(freq_mhz) when freq_mhz >= 900 do @allowed_bands |> Enum.min_by(&abs(&1 - freq_mhz)) |> to_string() end defp nearest_band(_freq_mhz), do: nil # -- timestamp parsing ----------------------------------------------------- defp parse_adif_datetime(nil, _time), do: nil defp parse_adif_datetime(_date, nil), do: nil defp parse_adif_datetime(date_str, time_str) do with <> <- date_str, {hour, min, sec} <- parse_adif_time(time_str), {:ok, naive} <- NaiveDateTime.new(si(y), si(m), si(d), hour, min, sec) do {:ok, DateTime.from_naive!(naive, "Etc/UTC")} else _ -> nil end end defp parse_adif_time(<>), do: {si(h), si(m), si(s)} defp parse_adif_time(<>), do: {si(h), si(m), 0} defp parse_adif_time(_), do: nil defp si(s), do: String.to_integer(s) # -- split / dedup (same logic as CsvImport) -------------------------------- defp split_parsed(rows) do rows |> Enum.reduce({[], []}, fn {:parsed, row}, {valid, invalid} -> {[row | valid], invalid} {:invalid, row}, {valid, invalid} -> {valid, [row | invalid]} end) |> then(fn {valid, invalid} -> {Enum.reverse(valid), Enum.reverse(invalid)} end) end defp load_existing_for_dedup([]), do: %{} defp load_existing_for_dedup(valid_rows) do bands = valid_rows |> Enum.map(& &1.attrs["band"]) |> Enum.uniq() |> Enum.map(&Decimal.new/1) timestamps = Enum.map(valid_rows, & &1.timestamp) min_ts = timestamps |> Enum.min(DateTime) |> DateTime.add(-@dedup_window_seconds, :second) max_ts = timestamps |> Enum.max(DateTime) |> DateTime.add(@dedup_window_seconds, :second) from(c in Contact, where: c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and c.flagged_invalid == false, select: %{ station1: c.station1, station2: c.station2, grid1: c.grid1, grid2: c.grid2, band: c.band, qso_timestamp: c.qso_timestamp } ) |> Repo.all() |> Enum.reduce(%{}, fn row, acc -> key = dedup_key(row.station1, row.grid1, row.station2, row.grid2, row.band) Map.update(acc, key, [row.qso_timestamp], &[row.qso_timestamp | &1]) end) end defp dedupe(rows, db_map) do {accepted, _in_batch, duplicates} = Enum.reduce(rows, {[], %{}, []}, fn row, {accepted, in_batch, dups} -> key = dedup_key( row.attrs["station1"], row.attrs["grid1"], row.attrs["station2"], row.attrs["grid2"], row.attrs["band"] ) cond do conflict_in_map?(db_map, key, row.timestamp) -> {accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups]} conflict_in_map?(in_batch, key, row.timestamp) -> {accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups]} true -> new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1]) {[row | accepted], new_batch, dups} end end) {Enum.reverse(accepted), Enum.reverse(duplicates)} end defp conflict_in_map?(map, key, timestamp) do case Map.fetch(map, key) do :error -> false {:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp)) end end defp within_window?(t1, t2) do abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds end defp dedup_key(s1, g1, s2, g2, band) do a = {upcase(s1), upcase(g1)} b = {upcase(s2), upcase(g2)} [c1, c2] = Enum.sort([a, b]) {c1, c2, normalize_band(band)} end defp upcase(nil), do: "" defp upcase(v), do: v |> to_string() |> String.trim() |> String.upcase() defp normalize_band(%Decimal{} = b), do: Decimal.to_integer(b) defp normalize_band(b) when is_integer(b), do: b defp normalize_band(b) when is_binary(b), do: b |> String.trim() |> String.to_integer() defp changeset_error_strings(changeset) do changeset |> Ecto.Changeset.traverse_errors(fn {message, opts} -> Regex.replace(~r"%{(\w+)}", message, fn _, key -> opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string() end) end) |> Enum.flat_map(fn {field, messages} -> Enum.map(messages, &"#{field} #{&1}") end) end end