419 lines
14 KiB
Elixir
419 lines
14 KiB
Elixir
defmodule Microwaveprop.Radio.AdifImport do
|
|
@moduledoc """
|
|
Parse an ADIF file into the same preview format used by CsvImport, so the
|
|
submit UI can reuse the preview/commit flow.
|
|
|
|
ADIF tagged-data format: `<FIELD_NAME:LENGTH>VALUE`
|
|
Records end with `<EOR>`. An optional header ends with `<EOH>`.
|
|
"""
|
|
|
|
import Ecto.Query, only: [from: 2]
|
|
|
|
alias Microwaveprop.Radio.BandResolver
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Radio.ImportMatcher
|
|
alias Microwaveprop.Radio.TextSanitizer
|
|
alias Microwaveprop.Repo
|
|
|
|
@dedup_window_seconds 3600
|
|
|
|
# Map ADIF MODE (and MODE+SUBMODE combinations) onto the fixed set of modes
|
|
# the site accepts: CW / SSB / FM / FT8 / FT4 / Q65. Unknown values map to
|
|
# nil — mode is optional on submission so unrecognized contacts still
|
|
# import, just without a mode label.
|
|
@allowed_modes ~w(CW SSB FM FT8 FT4 Q65)
|
|
@mode_map %{
|
|
"CW" => "CW",
|
|
"PCW" => "CW",
|
|
"SSB" => "SSB",
|
|
"USB" => "SSB",
|
|
"LSB" => "SSB",
|
|
"DSB" => "SSB",
|
|
"FM" => "FM",
|
|
"NBFM" => "FM",
|
|
"WFM" => "FM",
|
|
"FT8" => "FT8",
|
|
"FT4" => "FT4",
|
|
"Q65" => "Q65"
|
|
}
|
|
|
|
@doc """
|
|
Every ADIF MODE / SUBMODE combination we recognize, in display order.
|
|
Used by the upload page to show users what will happen to their data
|
|
before they commit the import.
|
|
"""
|
|
@spec mode_mapping_reference() :: [{String.t(), String.t()}]
|
|
def mode_mapping_reference do
|
|
[
|
|
{"CW / PCW", "CW"},
|
|
{"SSB / USB / LSB / DSB", "SSB"},
|
|
{"FM / NBFM / WFM", "FM"},
|
|
{"FT8 (or MFSK+SUBMODE=FT8)", "FT8"},
|
|
{"FT4 (or MFSK+SUBMODE=FT4)", "FT4"},
|
|
{"Q65 (or MFSK+SUBMODE=Q65)", "Q65"},
|
|
{"anything else", "(blank — row still imports)"}
|
|
]
|
|
end
|
|
|
|
@doc """
|
|
Normalize an ADIF MODE / SUBMODE pair to one of our allowed modes or nil.
|
|
|
|
ADIF convention: digital modes are often reported as `MODE=MFSK` with a
|
|
specific `SUBMODE` (FT8, FT4, Q65, JT65, …), so we prefer SUBMODE when it
|
|
matches one of our allowed values. For everything else we look up the raw
|
|
MODE in `@mode_map`; unknown values return nil.
|
|
"""
|
|
@spec normalize_mode(String.t() | nil, String.t() | nil) :: String.t() | nil
|
|
def normalize_mode(mode, submode) do
|
|
cond do
|
|
allowed = lookup_allowed(submode) -> allowed
|
|
allowed = lookup_allowed(mode) -> allowed
|
|
true -> nil
|
|
end
|
|
end
|
|
|
|
defp lookup_allowed(nil), do: nil
|
|
|
|
defp lookup_allowed(raw) do
|
|
normalized = raw |> to_string() |> String.trim() |> String.upcase()
|
|
|
|
cond do
|
|
normalized == "" -> nil
|
|
normalized in @allowed_modes -> normalized
|
|
true -> Map.get(@mode_map, normalized)
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Parse ADIF string, validate, and deduplicate. Returns the same preview
|
|
shape as `CsvImport.preview/2`.
|
|
"""
|
|
@type preview_result :: %{
|
|
valid: [map()],
|
|
invalid: [map()],
|
|
duplicates: [map()],
|
|
refinements: [map()],
|
|
total_rows: non_neg_integer(),
|
|
submitter_email: String.t()
|
|
}
|
|
|
|
@spec preview(String.t(), String.t()) :: {:ok, preview_result()} | {:error, :no_records}
|
|
def preview(adif_string, submitter_email) do
|
|
records = parse_adif(adif_string)
|
|
|
|
if records == [] do
|
|
{:error, :no_records}
|
|
else
|
|
rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email))
|
|
{valid, invalid} = split_parsed(rows)
|
|
existing = load_existing_for_dedup(valid)
|
|
{unique, duplicates, refinements} = dedupe(valid, existing)
|
|
|
|
{:ok,
|
|
%{
|
|
valid: unique,
|
|
invalid: invalid,
|
|
duplicates: duplicates,
|
|
refinements: refinements,
|
|
total_rows: length(records),
|
|
submitter_email: submitter_email
|
|
}}
|
|
end
|
|
end
|
|
|
|
# -- ADIF parser -----------------------------------------------------------
|
|
|
|
# Header-only ADIF tags that never appear inside QSO records
|
|
@header_only_tags ~w(ADIF_VER PROGRAMID PROGRAMVERSION CREATED_TIMESTAMP)
|
|
|
|
defp parse_adif(string) do
|
|
# Strip header if present (everything before <EOH>)
|
|
body =
|
|
case Regex.run(~r/<EOH>/i, string, return: :index) do
|
|
[{pos, len}] ->
|
|
String.slice(string, (pos + len)..-1//1)
|
|
|
|
nil ->
|
|
# No <EOH> — strip header-only tags (ADIF_VER, PROGRAMID, etc.)
|
|
# and any preceding plain text so they don't leak into the first record.
|
|
strip_header_tags(string)
|
|
end
|
|
|
|
# Split on <EOR> to get individual records
|
|
body
|
|
|> String.split(~r/<EOR>/i)
|
|
|> Enum.map(&parse_fields/1)
|
|
|> Enum.reject(&(&1 == %{}))
|
|
end
|
|
|
|
# Sequential byte-level ADIF parser.
|
|
# ADIF is a sequential format: each `<NAME:L>` tag declares an exact
|
|
# byte-length L for its value, and the next tag starts after those L
|
|
# bytes. A global regex match would re-find `<tag>`-like text inside
|
|
# a previous field's value, overwriting legitimate fields. Instead we
|
|
# advance the cursor past each consumed value before looking for the
|
|
# next tag, and `Map.put_new/3` ensures the first (genuine) occurrence
|
|
# of a field always wins.
|
|
defp parse_fields(record_string) do
|
|
parse_fields_loop(record_string, 0, %{})
|
|
end
|
|
|
|
defp parse_fields_loop(string, offset, acc) do
|
|
case Regex.run(~r/<([^:>]+):(\d+)(?::[^>]*)?>/i, string, return: :index, offset: offset) do
|
|
nil ->
|
|
acc
|
|
|
|
[{tag_start, tag_len}, {name_start, name_len}, {size_start, size_len}] ->
|
|
raw_name = string |> :binary.part(name_start, name_len) |> String.upcase()
|
|
name = strip_app_prefix(raw_name)
|
|
size = string |> :binary.part(size_start, size_len) |> String.to_integer()
|
|
|
|
value_offset = tag_start + tag_len
|
|
value = string |> :binary.part(value_offset, size) |> TextSanitizer.sanitize()
|
|
|
|
new_offset = value_offset + size
|
|
parse_fields_loop(string, new_offset, Map.put_new(acc, name, value))
|
|
end
|
|
end
|
|
|
|
# Strip APP_PROGRAMNAME_ prefix so app-defined fields resolve to standard names.
|
|
# e.g. APP_N1MM_MYGRID → MYGRID, APP_LOGGER32_CALL → CALL
|
|
defp strip_app_prefix("APP_" <> rest) do
|
|
case String.split(rest, "_", parts: 2) do
|
|
[_program, field] -> field
|
|
_ -> "APP_" <> rest
|
|
end
|
|
end
|
|
|
|
defp strip_app_prefix(name), do: name
|
|
|
|
# Strip known header-only tags and any plain text before the first record tag.
|
|
defp strip_header_tags(string) do
|
|
Enum.reduce(@header_only_tags, string, fn tag, acc ->
|
|
Regex.replace(~r/<#{tag}:\d+(?::[^>]*)?>.*?(?=<)/is, acc, "")
|
|
end)
|
|
end
|
|
|
|
defp normalize_grid(nil), do: nil
|
|
defp normalize_grid(g), do: g |> String.trim() |> String.upcase()
|
|
|
|
# -- row building ----------------------------------------------------------
|
|
|
|
defp build_row({fields, row_num}, submitter_email) do
|
|
station1 = fields["STATION_CALLSIGN"] || fields["OPERATOR"]
|
|
station2 = fields["CALL"]
|
|
grid1 = normalize_grid(fields["MY_GRIDSQUARE"])
|
|
grid2 = normalize_grid(fields["GRIDSQUARE"])
|
|
mode = normalize_mode(fields["MODE"], fields["SUBMODE"])
|
|
band = resolve_band(fields)
|
|
timestamp = parse_adif_datetime(fields["QSO_DATE"], fields["TIME_ON"])
|
|
# ADIF PROP_MODE — trusted as ground truth by MechanismClassifier when
|
|
# present. Stored verbatim (uppercased, whitespace-trimmed) so unusual
|
|
# values survive the round-trip for human inspection.
|
|
prop_mode =
|
|
case fields["PROP_MODE"] do
|
|
nil -> nil
|
|
"" -> nil
|
|
raw when is_binary(raw) -> raw |> String.trim() |> String.upcase()
|
|
end
|
|
|
|
# Operator commentary: prefer ADIF <NOTES> (multi-line detailed
|
|
# field) over <COMMENT> (short one-liner). Whitespace-only values
|
|
# collapse to nil so the column reflects "no notes" instead of an
|
|
# empty string — mirrors the normalizer on the submission changeset.
|
|
notes = adif_notes(fields)
|
|
|
|
case {band, timestamp} do
|
|
{nil, _} ->
|
|
{:invalid, %{row_num: row_num, messages: ["could not determine a valid microwave band"]}}
|
|
|
|
{_, nil} ->
|
|
{:invalid, %{row_num: row_num, messages: ["could not parse QSO_DATE/TIME_ON"]}}
|
|
|
|
{band_str, {:ok, dt}} ->
|
|
attrs = %{
|
|
"station1" => station1,
|
|
"station2" => station2,
|
|
"grid1" => grid1,
|
|
"grid2" => grid2,
|
|
"band" => band_str,
|
|
"mode" => mode,
|
|
"qso_timestamp" => DateTime.to_iso8601(dt),
|
|
"submitter_email" => submitter_email,
|
|
"user_declared_prop_mode" => prop_mode,
|
|
"notes" => notes
|
|
}
|
|
|
|
changeset = Contact.submission_changeset(%Contact{}, attrs)
|
|
|
|
if changeset.valid? do
|
|
{:parsed, %{row_num: row_num, attrs: attrs, timestamp: dt}}
|
|
else
|
|
messages = changeset_error_strings(changeset)
|
|
{:invalid, %{row_num: row_num, messages: messages}}
|
|
end
|
|
end
|
|
end
|
|
|
|
defp adif_notes(fields) do
|
|
Enum.find_value([fields["NOTES"], fields["COMMENT"]], fn
|
|
nil -> nil
|
|
value when is_binary(value) -> non_blank_or_nil(value)
|
|
end)
|
|
end
|
|
|
|
defp non_blank_or_nil(value) do
|
|
case String.trim(value) do
|
|
"" -> nil
|
|
_ -> value
|
|
end
|
|
end
|
|
|
|
# -- band resolution -------------------------------------------------------
|
|
|
|
defp resolve_band(fields) do
|
|
cond do
|
|
freq = fields["FREQ"] -> BandResolver.resolve_as_string(freq)
|
|
band_name = fields["BAND"] -> BandResolver.resolve_as_string(band_name)
|
|
true -> nil
|
|
end
|
|
end
|
|
|
|
# -- timestamp parsing -----------------------------------------------------
|
|
|
|
defp parse_adif_datetime(nil, _time), do: nil
|
|
defp parse_adif_datetime(_date, nil), do: nil
|
|
|
|
defp parse_adif_datetime(date_str, time_str) do
|
|
with <<y::binary-4, m::binary-2, d::binary-2>> <- date_str,
|
|
{hour, min, sec} <- parse_adif_time(time_str),
|
|
{:ok, naive} <- NaiveDateTime.new(si(y), si(m), si(d), hour, min, sec) do
|
|
{:ok, DateTime.from_naive!(naive, "Etc/UTC")}
|
|
else
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp parse_adif_time(<<h::binary-2, m::binary-2, s::binary-2>>), do: {si(h), si(m), si(s)}
|
|
defp parse_adif_time(<<h::binary-2, m::binary-2>>), do: {si(h), si(m), 0}
|
|
defp parse_adif_time(_), do: nil
|
|
|
|
defp si(s), do: String.to_integer(s)
|
|
|
|
# -- split / dedup (same logic as CsvImport) --------------------------------
|
|
|
|
defp split_parsed(rows) do
|
|
rows
|
|
|> Enum.reduce({[], []}, fn
|
|
{:parsed, row}, {valid, invalid} -> {[row | valid], invalid}
|
|
{:invalid, row}, {valid, invalid} -> {valid, [row | invalid]}
|
|
end)
|
|
|> then(fn {valid, invalid} -> {Enum.reverse(valid), Enum.reverse(invalid)} end)
|
|
end
|
|
|
|
defp load_existing_for_dedup([]), do: %{}
|
|
|
|
defp load_existing_for_dedup(valid_rows) do
|
|
bands = valid_rows |> Enum.map(& &1.attrs["band"]) |> Enum.uniq() |> Enum.map(&Decimal.new/1)
|
|
timestamps = Enum.map(valid_rows, & &1.timestamp)
|
|
min_ts = timestamps |> Enum.min(DateTime) |> DateTime.add(-@dedup_window_seconds, :second)
|
|
max_ts = timestamps |> Enum.max(DateTime) |> DateTime.add(@dedup_window_seconds, :second)
|
|
|
|
from(c in Contact,
|
|
where:
|
|
c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and
|
|
c.flagged_invalid == false
|
|
)
|
|
|> Repo.all()
|
|
|> Enum.reduce(%{}, fn contact, acc ->
|
|
case ImportMatcher.prefix_key(contact.station1, contact.grid1, contact.station2, contact.grid2, contact.band) do
|
|
:invalid -> acc
|
|
key -> Map.update(acc, key, [contact], &[contact | &1])
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp dedupe(rows, db_map) do
|
|
{accepted, _in_batch, duplicates, refinements} =
|
|
Enum.reduce(rows, {[], %{}, [], []}, &classify_row(&1, &2, db_map))
|
|
|
|
{Enum.reverse(accepted), Enum.reverse(duplicates), Enum.reverse(refinements)}
|
|
end
|
|
|
|
defp classify_row(row, {accepted, in_batch, dups, refs}, db_map) do
|
|
key = prefix_key_from_attrs(row.attrs)
|
|
|
|
cond do
|
|
key == :invalid ->
|
|
{[row | accepted], in_batch, dups, refs}
|
|
|
|
candidate = db_candidate(db_map, key, row.timestamp) ->
|
|
classify_db_candidate(row, candidate, accepted, in_batch, dups, refs)
|
|
|
|
batch_conflict?(in_batch, key, row.timestamp) ->
|
|
{accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups], refs}
|
|
|
|
true ->
|
|
new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1])
|
|
{[row | accepted], new_batch, dups, refs}
|
|
end
|
|
end
|
|
|
|
defp classify_db_candidate(row, candidate, accepted, in_batch, dups, refs) do
|
|
case ImportMatcher.classify_against_existing(row.attrs, candidate) do
|
|
{:refinement, changes} ->
|
|
refinement = %{
|
|
row_num: row.row_num,
|
|
attrs: row.attrs,
|
|
timestamp: row.timestamp,
|
|
existing_id: candidate.id,
|
|
changes: changes
|
|
}
|
|
|
|
{accepted, in_batch, dups, [refinement | refs]}
|
|
|
|
_ ->
|
|
{accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups], refs}
|
|
end
|
|
end
|
|
|
|
defp db_candidate(db_map, key, timestamp) do
|
|
case Map.fetch(db_map, key) do
|
|
:error -> nil
|
|
{:ok, contacts} -> Enum.find(contacts, &within_window?(&1.qso_timestamp, timestamp))
|
|
end
|
|
end
|
|
|
|
defp batch_conflict?(map, key, timestamp) do
|
|
case Map.fetch(map, key) do
|
|
:error -> false
|
|
{:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp))
|
|
end
|
|
end
|
|
|
|
defp within_window?(t1, t2) do
|
|
abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds
|
|
end
|
|
|
|
defp prefix_key_from_attrs(attrs) do
|
|
ImportMatcher.prefix_key(
|
|
attrs["station1"],
|
|
attrs["grid1"],
|
|
attrs["station2"],
|
|
attrs["grid2"],
|
|
attrs["band"]
|
|
)
|
|
end
|
|
|
|
defp changeset_error_strings(changeset) do
|
|
changeset
|
|
|> Ecto.Changeset.traverse_errors(fn {message, opts} ->
|
|
Regex.replace(~r"%{(\w+)}", message, fn _, key ->
|
|
opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
|
|
end)
|
|
end)
|
|
|> Enum.flat_map(fn {field, messages} ->
|
|
Enum.map(messages, &"#{field} #{&1}")
|
|
end)
|
|
end
|
|
end
|