Graham McIntire
265669fc3a
Regex.scan with `return: :index` reports BYTE offsets, but String.slice/3
uses CHARACTER offsets. When a record contained a multi-byte UTF-8
character in an earlier field (e.g. "café" in NOTES), every subsequent
field's slice shifted one byte left, eventually landing on a literal
">" that crashed String.to_integer/1 with ArgumentError.
Switch to :binary.part/3 which is byte-based and matches what Regex.scan
reports. Also simplify the value offset calculation to use the scanned
tag_len directly instead of re-splitting the tag text. Reproduced in a
new regression test.
Prod stack trace: AdifImport.parse_fields/1 crashed on upload with
binary_to_integer(">") -- a FreeDV/N1MM logger that embedded UTF-8
characters in a NOTES field triggered it.
407 lines
13 KiB
Elixir
407 lines
13 KiB
Elixir
defmodule Microwaveprop.Radio.AdifImport do
|
|
@moduledoc """
|
|
Parse an ADIF file into the same preview format used by CsvImport, so the
|
|
submit UI can reuse the preview/commit flow.
|
|
|
|
ADIF tagged-data format: `<FIELD_NAME:LENGTH>VALUE`
|
|
Records end with `<EOR>`. An optional header ends with `<EOH>`.
|
|
"""
|
|
|
|
import Ecto.Query, only: [from: 2]
|
|
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
|
|
@dedup_window_seconds 3600
|
|
|
|
# Band plan: map ADIF band names to our MHz values
|
|
@band_name_to_mhz %{
|
|
"23cm" => 1296,
|
|
"13cm" => 2304,
|
|
"9cm" => 3456,
|
|
"6cm" => 5760,
|
|
"3cm" => 10_000,
|
|
"1.25cm" => 24_000,
|
|
"6mm" => 47_000,
|
|
"4mm" => 75_000,
|
|
"2.5mm" => 122_000,
|
|
"2mm" => 134_000,
|
|
"1mm" => 241_000
|
|
}
|
|
|
|
# Our allowed band centers in MHz, sorted ascending for nearest-match
|
|
@allowed_bands [1296, 2304, 3456, 5760, 10_000, 24_000, 47_000, 68_000, 75_000, 122_000, 134_000, 241_000]
|
|
|
|
# Map ADIF MODE (and MODE+SUBMODE combinations) onto the fixed set of modes
|
|
# the site accepts: CW / SSB / FM / FT8 / FT4 / Q65. Unknown values map to
|
|
# nil — mode is optional on submission so unrecognized contacts still
|
|
# import, just without a mode label.
|
|
@allowed_modes ~w(CW SSB FM FT8 FT4 Q65)
|
|
@mode_map %{
|
|
"CW" => "CW",
|
|
"PCW" => "CW",
|
|
"SSB" => "SSB",
|
|
"USB" => "SSB",
|
|
"LSB" => "SSB",
|
|
"DSB" => "SSB",
|
|
"FM" => "FM",
|
|
"NBFM" => "FM",
|
|
"WFM" => "FM",
|
|
"FT8" => "FT8",
|
|
"FT4" => "FT4",
|
|
"Q65" => "Q65"
|
|
}
|
|
|
|
@doc """
|
|
Every ADIF MODE / SUBMODE combination we recognize, in display order.
|
|
Used by the upload page to show users what will happen to their data
|
|
before they commit the import.
|
|
"""
|
|
@spec mode_mapping_reference() :: [{String.t(), String.t()}]
|
|
def mode_mapping_reference do
|
|
[
|
|
{"CW / PCW", "CW"},
|
|
{"SSB / USB / LSB / DSB", "SSB"},
|
|
{"FM / NBFM / WFM", "FM"},
|
|
{"FT8 (or MFSK+SUBMODE=FT8)", "FT8"},
|
|
{"FT4 (or MFSK+SUBMODE=FT4)", "FT4"},
|
|
{"Q65 (or MFSK+SUBMODE=Q65)", "Q65"},
|
|
{"anything else", "(blank — row still imports)"}
|
|
]
|
|
end
|
|
|
|
@doc """
|
|
Normalize an ADIF MODE / SUBMODE pair to one of our allowed modes or nil.
|
|
|
|
ADIF convention: digital modes are often reported as `MODE=MFSK` with a
|
|
specific `SUBMODE` (FT8, FT4, Q65, JT65, …), so we prefer SUBMODE when it
|
|
matches one of our allowed values. For everything else we look up the raw
|
|
MODE in `@mode_map`; unknown values return nil.
|
|
"""
|
|
@spec normalize_mode(String.t() | nil, String.t() | nil) :: String.t() | nil
|
|
def normalize_mode(mode, submode) do
|
|
cond do
|
|
allowed = lookup_allowed(submode) -> allowed
|
|
allowed = lookup_allowed(mode) -> allowed
|
|
true -> nil
|
|
end
|
|
end
|
|
|
|
defp lookup_allowed(nil), do: nil
|
|
|
|
defp lookup_allowed(raw) do
|
|
normalized = raw |> to_string() |> String.trim() |> String.upcase()
|
|
|
|
cond do
|
|
normalized == "" -> nil
|
|
normalized in @allowed_modes -> normalized
|
|
true -> Map.get(@mode_map, normalized)
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Parse ADIF string, validate, and deduplicate. Returns the same preview
|
|
shape as `CsvImport.preview/2`.
|
|
"""
|
|
@type preview_result :: %{
|
|
valid: [map()],
|
|
invalid: [map()],
|
|
duplicates: [map()],
|
|
total_rows: non_neg_integer(),
|
|
submitter_email: String.t()
|
|
}
|
|
|
|
@spec preview(String.t(), String.t()) :: {:ok, preview_result()} | {:error, :no_records}
|
|
def preview(adif_string, submitter_email) do
|
|
records = parse_adif(adif_string)
|
|
|
|
if records == [] do
|
|
{:error, :no_records}
|
|
else
|
|
rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email))
|
|
{valid, invalid} = split_parsed(rows)
|
|
existing = load_existing_for_dedup(valid)
|
|
{unique, duplicates} = dedupe(valid, existing)
|
|
|
|
{:ok,
|
|
%{
|
|
valid: unique,
|
|
invalid: invalid,
|
|
duplicates: duplicates,
|
|
total_rows: length(records),
|
|
submitter_email: submitter_email
|
|
}}
|
|
end
|
|
end
|
|
|
|
# -- ADIF parser -----------------------------------------------------------
|
|
|
|
# Header-only ADIF tags that never appear inside QSO records
|
|
@header_only_tags ~w(ADIF_VER PROGRAMID PROGRAMVERSION CREATED_TIMESTAMP)
|
|
|
|
defp parse_adif(string) do
|
|
# Strip header if present (everything before <EOH>)
|
|
body =
|
|
case Regex.run(~r/<EOH>/i, string, return: :index) do
|
|
[{pos, len}] ->
|
|
String.slice(string, (pos + len)..-1//1)
|
|
|
|
nil ->
|
|
# No <EOH> — strip header-only tags (ADIF_VER, PROGRAMID, etc.)
|
|
# and any preceding plain text so they don't leak into the first record.
|
|
strip_header_tags(string)
|
|
end
|
|
|
|
# Split on <EOR> to get individual records
|
|
body
|
|
|> String.split(~r/<EOR>/i)
|
|
|> Enum.map(&parse_fields/1)
|
|
|> Enum.reject(&(&1 == %{}))
|
|
end
|
|
|
|
# Regex.scan with `return: :index` returns BYTE offsets. We must use
|
|
# `:binary.part/3` (byte-based) rather than `String.slice/3` (character-
|
|
# based); otherwise multi-byte UTF-8 characters earlier in the record
|
|
# (e.g. an accented NOTES field) shift every subsequent field and we
|
|
# wind up slicing the wrong substring.
|
|
defp parse_fields(record_string) do
|
|
~r/<([^:>]+):(\d+)(?::[^>]*)?>/i
|
|
|> Regex.scan(record_string, return: :index)
|
|
|> Enum.reduce(%{}, fn indices, acc ->
|
|
[{tag_start, tag_len}, {name_start, name_len}, {size_start, size_len} | _] = indices
|
|
raw_name = record_string |> :binary.part(name_start, name_len) |> String.upcase()
|
|
name = strip_app_prefix(raw_name)
|
|
size = record_string |> :binary.part(size_start, size_len) |> String.to_integer()
|
|
|
|
# Value starts right after the closing > of the tag.
|
|
value_offset = tag_start + tag_len
|
|
value = :binary.part(record_string, value_offset, size)
|
|
|
|
# Standard ADIF fields take precedence over app-defined ones
|
|
if raw_name != name and Map.has_key?(acc, name) do
|
|
acc
|
|
else
|
|
Map.put(acc, name, value)
|
|
end
|
|
end)
|
|
end
|
|
|
|
# Strip APP_PROGRAMNAME_ prefix so app-defined fields resolve to standard names.
|
|
# e.g. APP_N1MM_MYGRID → MYGRID, APP_LOGGER32_CALL → CALL
|
|
defp strip_app_prefix("APP_" <> rest) do
|
|
case String.split(rest, "_", parts: 2) do
|
|
[_program, field] -> field
|
|
_ -> "APP_" <> rest
|
|
end
|
|
end
|
|
|
|
defp strip_app_prefix(name), do: name
|
|
|
|
# Strip known header-only tags and any plain text before the first record tag.
|
|
defp strip_header_tags(string) do
|
|
Enum.reduce(@header_only_tags, string, fn tag, acc ->
|
|
Regex.replace(~r/<#{tag}:\d+(?::[^>]*)?>.*?(?=<)/is, acc, "")
|
|
end)
|
|
end
|
|
|
|
defp normalize_grid(nil), do: nil
|
|
defp normalize_grid(g), do: g |> String.trim() |> String.upcase()
|
|
|
|
# -- row building ----------------------------------------------------------
|
|
|
|
defp build_row({fields, row_num}, submitter_email) do
|
|
station1 = fields["STATION_CALLSIGN"] || fields["OPERATOR"]
|
|
station2 = fields["CALL"]
|
|
grid1 = normalize_grid(fields["MY_GRIDSQUARE"])
|
|
grid2 = normalize_grid(fields["GRIDSQUARE"])
|
|
mode = normalize_mode(fields["MODE"], fields["SUBMODE"])
|
|
band = resolve_band(fields)
|
|
timestamp = parse_adif_datetime(fields["QSO_DATE"], fields["TIME_ON"])
|
|
|
|
case {band, timestamp} do
|
|
{nil, _} ->
|
|
{:invalid, %{row_num: row_num, messages: ["could not determine a valid microwave band"]}}
|
|
|
|
{_, nil} ->
|
|
{:invalid, %{row_num: row_num, messages: ["could not parse QSO_DATE/TIME_ON"]}}
|
|
|
|
{band_str, {:ok, dt}} ->
|
|
attrs = %{
|
|
"station1" => station1,
|
|
"station2" => station2,
|
|
"grid1" => grid1,
|
|
"grid2" => grid2,
|
|
"band" => band_str,
|
|
"mode" => mode,
|
|
"qso_timestamp" => DateTime.to_iso8601(dt),
|
|
"submitter_email" => submitter_email
|
|
}
|
|
|
|
changeset = Contact.submission_changeset(%Contact{}, attrs)
|
|
|
|
if changeset.valid? do
|
|
{:parsed, %{row_num: row_num, attrs: attrs, timestamp: dt}}
|
|
else
|
|
messages = changeset_error_strings(changeset)
|
|
{:invalid, %{row_num: row_num, messages: messages}}
|
|
end
|
|
end
|
|
end
|
|
|
|
# -- band resolution -------------------------------------------------------
|
|
|
|
defp resolve_band(fields) do
|
|
cond do
|
|
freq = fields["FREQ"] ->
|
|
freq_mhz = parse_freq_mhz(freq)
|
|
nearest_band(freq_mhz)
|
|
|
|
band_name = fields["BAND"] ->
|
|
band_name = String.downcase(band_name)
|
|
mhz = @band_name_to_mhz[band_name]
|
|
if mhz, do: to_string(mhz)
|
|
|
|
true ->
|
|
nil
|
|
end
|
|
end
|
|
|
|
defp parse_freq_mhz(freq_str) do
|
|
{freq, _} = Float.parse(String.trim(freq_str))
|
|
# ADIF FREQ is always in MHz (e.g., 10368.000 for 10 GHz band)
|
|
freq
|
|
end
|
|
|
|
defp nearest_band(freq_mhz) when freq_mhz >= 900 do
|
|
@allowed_bands
|
|
|> Enum.min_by(&abs(&1 - freq_mhz))
|
|
|> to_string()
|
|
end
|
|
|
|
defp nearest_band(_freq_mhz), do: nil
|
|
|
|
# -- timestamp parsing -----------------------------------------------------
|
|
|
|
defp parse_adif_datetime(nil, _time), do: nil
|
|
defp parse_adif_datetime(_date, nil), do: nil
|
|
|
|
defp parse_adif_datetime(date_str, time_str) do
|
|
with <<y::binary-4, m::binary-2, d::binary-2>> <- date_str,
|
|
{hour, min, sec} <- parse_adif_time(time_str),
|
|
{:ok, naive} <- NaiveDateTime.new(si(y), si(m), si(d), hour, min, sec) do
|
|
{:ok, DateTime.from_naive!(naive, "Etc/UTC")}
|
|
else
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp parse_adif_time(<<h::binary-2, m::binary-2, s::binary-2>>), do: {si(h), si(m), si(s)}
|
|
defp parse_adif_time(<<h::binary-2, m::binary-2>>), do: {si(h), si(m), 0}
|
|
defp parse_adif_time(_), do: nil
|
|
|
|
defp si(s), do: String.to_integer(s)
|
|
|
|
# -- split / dedup (same logic as CsvImport) --------------------------------
|
|
|
|
defp split_parsed(rows) do
|
|
rows
|
|
|> Enum.reduce({[], []}, fn
|
|
{:parsed, row}, {valid, invalid} -> {[row | valid], invalid}
|
|
{:invalid, row}, {valid, invalid} -> {valid, [row | invalid]}
|
|
end)
|
|
|> then(fn {valid, invalid} -> {Enum.reverse(valid), Enum.reverse(invalid)} end)
|
|
end
|
|
|
|
defp load_existing_for_dedup([]), do: %{}
|
|
|
|
defp load_existing_for_dedup(valid_rows) do
|
|
bands = valid_rows |> Enum.map(& &1.attrs["band"]) |> Enum.uniq() |> Enum.map(&Decimal.new/1)
|
|
timestamps = Enum.map(valid_rows, & &1.timestamp)
|
|
min_ts = timestamps |> Enum.min(DateTime) |> DateTime.add(-@dedup_window_seconds, :second)
|
|
max_ts = timestamps |> Enum.max(DateTime) |> DateTime.add(@dedup_window_seconds, :second)
|
|
|
|
from(c in Contact,
|
|
where:
|
|
c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and
|
|
c.flagged_invalid == false,
|
|
select: %{
|
|
station1: c.station1,
|
|
station2: c.station2,
|
|
grid1: c.grid1,
|
|
grid2: c.grid2,
|
|
band: c.band,
|
|
qso_timestamp: c.qso_timestamp
|
|
}
|
|
)
|
|
|> Repo.all()
|
|
|> Enum.reduce(%{}, fn row, acc ->
|
|
key = dedup_key(row.station1, row.grid1, row.station2, row.grid2, row.band)
|
|
Map.update(acc, key, [row.qso_timestamp], &[row.qso_timestamp | &1])
|
|
end)
|
|
end
|
|
|
|
defp dedupe(rows, db_map) do
|
|
{accepted, _in_batch, duplicates} =
|
|
Enum.reduce(rows, {[], %{}, []}, fn row, {accepted, in_batch, dups} ->
|
|
key =
|
|
dedup_key(
|
|
row.attrs["station1"],
|
|
row.attrs["grid1"],
|
|
row.attrs["station2"],
|
|
row.attrs["grid2"],
|
|
row.attrs["band"]
|
|
)
|
|
|
|
cond do
|
|
conflict_in_map?(db_map, key, row.timestamp) ->
|
|
{accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups]}
|
|
|
|
conflict_in_map?(in_batch, key, row.timestamp) ->
|
|
{accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups]}
|
|
|
|
true ->
|
|
new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1])
|
|
{[row | accepted], new_batch, dups}
|
|
end
|
|
end)
|
|
|
|
{Enum.reverse(accepted), Enum.reverse(duplicates)}
|
|
end
|
|
|
|
defp conflict_in_map?(map, key, timestamp) do
|
|
case Map.fetch(map, key) do
|
|
:error -> false
|
|
{:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp))
|
|
end
|
|
end
|
|
|
|
defp within_window?(t1, t2) do
|
|
abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds
|
|
end
|
|
|
|
defp dedup_key(s1, g1, s2, g2, band) do
|
|
a = {upcase(s1), upcase(g1)}
|
|
b = {upcase(s2), upcase(g2)}
|
|
[c1, c2] = Enum.sort([a, b])
|
|
{c1, c2, normalize_band(band)}
|
|
end
|
|
|
|
defp upcase(nil), do: ""
|
|
defp upcase(v), do: v |> to_string() |> String.trim() |> String.upcase()
|
|
|
|
defp normalize_band(%Decimal{} = b), do: Decimal.to_integer(b)
|
|
defp normalize_band(b) when is_integer(b), do: b
|
|
defp normalize_band(b) when is_binary(b), do: b |> String.trim() |> String.to_integer()
|
|
|
|
defp changeset_error_strings(changeset) do
|
|
changeset
|
|
|> Ecto.Changeset.traverse_errors(fn {message, opts} ->
|
|
Regex.replace(~r"%{(\w+)}", message, fn _, key ->
|
|
opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
|
|
end)
|
|
end)
|
|
|> Enum.flat_map(fn {field, messages} ->
|
|
Enum.map(messages, &"#{field} #{&1}")
|
|
end)
|
|
end
|
|
end
|