defmodule Microwaveprop.Radio.AdifImport do @moduledoc """ Parse an ADIF file into the same preview format used by CsvImport, so the submit UI can reuse the preview/commit flow. ADIF tagged-data format: `VALUE` Records end with ``. An optional header ends with ``. """ import Ecto.Query, only: [from: 2] alias Microwaveprop.Radio.BandResolver alias Microwaveprop.Radio.Contact alias Microwaveprop.Radio.ImportMatcher alias Microwaveprop.Radio.TextSanitizer alias Microwaveprop.Repo @dedup_window_seconds 3600 # Map ADIF MODE (and MODE+SUBMODE combinations) onto the fixed set of modes # the site accepts: CW / SSB / FM / FT8 / FT4 / Q65. Unknown values map to # nil — mode is optional on submission so unrecognized contacts still # import, just without a mode label. @allowed_modes ~w(CW SSB FM FT8 FT4 Q65) @mode_map %{ "CW" => "CW", "PCW" => "CW", "SSB" => "SSB", "USB" => "SSB", "LSB" => "SSB", "DSB" => "SSB", "FM" => "FM", "NBFM" => "FM", "WFM" => "FM", "FT8" => "FT8", "FT4" => "FT4", "Q65" => "Q65" } @doc """ Every ADIF MODE / SUBMODE combination we recognize, in display order. Used by the upload page to show users what will happen to their data before they commit the import. """ @spec mode_mapping_reference() :: [{String.t(), String.t()}] def mode_mapping_reference do [ {"CW / PCW", "CW"}, {"SSB / USB / LSB / DSB", "SSB"}, {"FM / NBFM / WFM", "FM"}, {"FT8 (or MFSK+SUBMODE=FT8)", "FT8"}, {"FT4 (or MFSK+SUBMODE=FT4)", "FT4"}, {"Q65 (or MFSK+SUBMODE=Q65)", "Q65"}, {"anything else", "(blank — row still imports)"} ] end @doc """ Normalize an ADIF MODE / SUBMODE pair to one of our allowed modes or nil. ADIF convention: digital modes are often reported as `MODE=MFSK` with a specific `SUBMODE` (FT8, FT4, Q65, JT65, …), so we prefer SUBMODE when it matches one of our allowed values. For everything else we look up the raw MODE in `@mode_map`; unknown values return nil. """ @spec normalize_mode(String.t() | nil, String.t() | nil) :: String.t() | nil def normalize_mode(mode, submode) do cond do allowed = lookup_allowed(submode) -> allowed allowed = lookup_allowed(mode) -> allowed true -> nil end end defp lookup_allowed(nil), do: nil defp lookup_allowed(raw) do normalized = raw |> to_string() |> String.trim() |> String.upcase() cond do normalized == "" -> nil normalized in @allowed_modes -> normalized true -> Map.get(@mode_map, normalized) end end @doc """ Parse ADIF string, validate, and deduplicate. Returns the same preview shape as `CsvImport.preview/2`. """ @type preview_result :: %{ valid: [map()], invalid: [map()], duplicates: [map()], refinements: [map()], total_rows: non_neg_integer(), submitter_email: String.t() } @spec preview(String.t(), String.t()) :: {:ok, preview_result()} | {:error, :no_records} def preview(adif_string, submitter_email) do records = parse_adif(adif_string) if records == [] do {:error, :no_records} else rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email)) {valid, invalid} = split_parsed(rows) existing = load_existing_for_dedup(valid) {unique, duplicates, refinements} = dedupe(valid, existing) {:ok, %{ valid: unique, invalid: invalid, duplicates: duplicates, refinements: refinements, total_rows: length(records), submitter_email: submitter_email }} end end # -- ADIF parser ----------------------------------------------------------- # Header-only ADIF tags that never appear inside QSO records @header_only_tags ~w(ADIF_VER PROGRAMID PROGRAMVERSION CREATED_TIMESTAMP) defp parse_adif(string) do # Strip header if present (everything before ) body = case Regex.run(~r//i, string, return: :index) do [{pos, len}] -> String.slice(string, (pos + len)..-1//1) nil -> # No — strip header-only tags (ADIF_VER, PROGRAMID, etc.) # and any preceding plain text so they don't leak into the first record. strip_header_tags(string) end # Split on to get individual records body |> String.split(~r//i) |> Enum.map(&parse_fields/1) |> Enum.reject(&(&1 == %{})) end # Sequential byte-level ADIF parser. # ADIF is a sequential format: each `` tag declares an exact # byte-length L for its value, and the next tag starts after those L # bytes. A global regex match would re-find ``-like text inside # a previous field's value, overwriting legitimate fields. Instead we # advance the cursor past each consumed value before looking for the # next tag, and `Map.put_new/3` ensures the first (genuine) occurrence # of a field always wins. defp parse_fields(record_string) do parse_fields_loop(record_string, 0, %{}) end defp parse_fields_loop(string, offset, acc) do case Regex.run(~r/<([^:>]+):(\d+)(?::[^>]*)?>/i, string, return: :index, offset: offset) do nil -> acc [{tag_start, tag_len}, {name_start, name_len}, {size_start, size_len}] -> raw_name = string |> :binary.part(name_start, name_len) |> String.upcase() name = strip_app_prefix(raw_name) size = string |> :binary.part(size_start, size_len) |> String.to_integer() value_offset = tag_start + tag_len value = string |> :binary.part(value_offset, size) |> TextSanitizer.sanitize() new_offset = value_offset + size parse_fields_loop(string, new_offset, Map.put_new(acc, name, value)) end end # Strip APP_PROGRAMNAME_ prefix so app-defined fields resolve to standard names. # e.g. APP_N1MM_MYGRID → MYGRID, APP_LOGGER32_CALL → CALL defp strip_app_prefix("APP_" <> rest) do case String.split(rest, "_", parts: 2) do [_program, field] -> field _ -> "APP_" <> rest end end defp strip_app_prefix(name), do: name # Strip known header-only tags and any plain text before the first record tag. defp strip_header_tags(string) do Enum.reduce(@header_only_tags, string, fn tag, acc -> Regex.replace(~r/<#{tag}:\d+(?::[^>]*)?>.*?(?=<)/is, acc, "") end) end defp normalize_grid(nil), do: nil defp normalize_grid(g), do: g |> String.trim() |> String.upcase() # -- row building ---------------------------------------------------------- defp build_row({fields, row_num}, submitter_email) do station1 = fields["STATION_CALLSIGN"] || fields["OPERATOR"] station2 = fields["CALL"] grid1 = normalize_grid(fields["MY_GRIDSQUARE"]) grid2 = normalize_grid(fields["GRIDSQUARE"]) mode = normalize_mode(fields["MODE"], fields["SUBMODE"]) band = resolve_band(fields) timestamp = parse_adif_datetime(fields["QSO_DATE"], fields["TIME_ON"]) # ADIF PROP_MODE — trusted as ground truth by MechanismClassifier when # present. Stored verbatim (uppercased, whitespace-trimmed) so unusual # values survive the round-trip for human inspection. prop_mode = case fields["PROP_MODE"] do nil -> nil "" -> nil raw when is_binary(raw) -> raw |> String.trim() |> String.upcase() end # Operator commentary: prefer ADIF (multi-line detailed # field) over (short one-liner). Whitespace-only values # collapse to nil so the column reflects "no notes" instead of an # empty string — mirrors the normalizer on the submission changeset. notes = adif_notes(fields) case {band, timestamp} do {nil, _} -> {:invalid, %{row_num: row_num, messages: ["could not determine a valid microwave band"]}} {_, nil} -> {:invalid, %{row_num: row_num, messages: ["could not parse QSO_DATE/TIME_ON"]}} {band_str, {:ok, dt}} -> attrs = %{ "station1" => station1, "station2" => station2, "grid1" => grid1, "grid2" => grid2, "band" => band_str, "mode" => mode, "qso_timestamp" => DateTime.to_iso8601(dt), "submitter_email" => submitter_email, "user_declared_prop_mode" => prop_mode, "notes" => notes } changeset = Contact.submission_changeset(%Contact{}, attrs) if changeset.valid? do {:parsed, %{row_num: row_num, attrs: attrs, timestamp: dt}} else messages = changeset_error_strings(changeset) {:invalid, %{row_num: row_num, messages: messages}} end end end defp adif_notes(fields) do Enum.find_value([fields["NOTES"], fields["COMMENT"]], fn nil -> nil value when is_binary(value) -> non_blank_or_nil(value) end) end defp non_blank_or_nil(value) do case String.trim(value) do "" -> nil _ -> value end end # -- band resolution ------------------------------------------------------- defp resolve_band(fields) do cond do freq = fields["FREQ"] -> BandResolver.resolve_as_string(freq) band_name = fields["BAND"] -> BandResolver.resolve_as_string(band_name) true -> nil end end # -- timestamp parsing ----------------------------------------------------- defp parse_adif_datetime(nil, _time), do: nil defp parse_adif_datetime(_date, nil), do: nil defp parse_adif_datetime(date_str, time_str) do with <> <- date_str, {hour, min, sec} <- parse_adif_time(time_str), {:ok, naive} <- NaiveDateTime.new(si(y), si(m), si(d), hour, min, sec) do {:ok, DateTime.from_naive!(naive, "Etc/UTC")} else _ -> nil end end defp parse_adif_time(<>), do: {si(h), si(m), si(s)} defp parse_adif_time(<>), do: {si(h), si(m), 0} defp parse_adif_time(_), do: nil defp si(s), do: String.to_integer(s) # -- split / dedup (same logic as CsvImport) -------------------------------- defp split_parsed(rows) do rows |> Enum.reduce({[], []}, fn {:parsed, row}, {valid, invalid} -> {[row | valid], invalid} {:invalid, row}, {valid, invalid} -> {valid, [row | invalid]} end) |> then(fn {valid, invalid} -> {Enum.reverse(valid), Enum.reverse(invalid)} end) end defp load_existing_for_dedup([]), do: %{} defp load_existing_for_dedup(valid_rows) do bands = valid_rows |> Enum.map(& &1.attrs["band"]) |> Enum.uniq() |> Enum.map(&Decimal.new/1) timestamps = Enum.map(valid_rows, & &1.timestamp) min_ts = timestamps |> Enum.min(DateTime) |> DateTime.add(-@dedup_window_seconds, :second) max_ts = timestamps |> Enum.max(DateTime) |> DateTime.add(@dedup_window_seconds, :second) from(c in Contact, where: c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and c.flagged_invalid == false ) |> Repo.all() |> Enum.reduce(%{}, fn contact, acc -> case ImportMatcher.prefix_key(contact.station1, contact.grid1, contact.station2, contact.grid2, contact.band) do :invalid -> acc key -> Map.update(acc, key, [contact], &[contact | &1]) end end) end defp dedupe(rows, db_map) do {accepted, _in_batch, duplicates, refinements} = Enum.reduce(rows, {[], %{}, [], []}, &classify_row(&1, &2, db_map)) {Enum.reverse(accepted), Enum.reverse(duplicates), Enum.reverse(refinements)} end defp classify_row(row, {accepted, in_batch, dups, refs}, db_map) do key = prefix_key_from_attrs(row.attrs) cond do key == :invalid -> {[row | accepted], in_batch, dups, refs} candidate = db_candidate(db_map, key, row.timestamp) -> classify_db_candidate(row, candidate, accepted, in_batch, dups, refs) batch_conflict?(in_batch, key, row.timestamp) -> {accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups], refs} true -> new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1]) {[row | accepted], new_batch, dups, refs} end end defp classify_db_candidate(row, candidate, accepted, in_batch, dups, refs) do case ImportMatcher.classify_against_existing(row.attrs, candidate) do {:refinement, changes} -> refinement = %{ row_num: row.row_num, attrs: row.attrs, timestamp: row.timestamp, existing_id: candidate.id, changes: changes } {accepted, in_batch, dups, [refinement | refs]} _ -> {accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups], refs} end end defp db_candidate(db_map, key, timestamp) do case Map.fetch(db_map, key) do :error -> nil {:ok, contacts} -> Enum.find(contacts, &within_window?(&1.qso_timestamp, timestamp)) end end defp batch_conflict?(map, key, timestamp) do case Map.fetch(map, key) do :error -> false {:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp)) end end defp within_window?(t1, t2) do abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds end defp prefix_key_from_attrs(attrs) do ImportMatcher.prefix_key( attrs["station1"], attrs["grid1"], attrs["station2"], attrs["grid2"], attrs["band"] ) end defp changeset_error_strings(changeset) do changeset |> Ecto.Changeset.traverse_errors(fn {message, opts} -> Regex.replace(~r"%{(\w+)}", message, fn _, key -> opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string() end) end) |> Enum.flat_map(fn {field, messages} -> Enum.map(messages, &"#{field} #{&1}") end) end end