Graham McIntire
c3becdac9f
- Add belongs_to :user on Beacon schema, preload in get_beacon!/1
- Show "Submitted by {callsign}" on beacon detail page
- Add NTMS donation link on the about page
- Change grid preference text to 8 characters (e.g. EM12kp37)
293 lines
9.1 KiB
Elixir
293 lines
9.1 KiB
Elixir
defmodule Microwaveprop.Radio.AdifImport do
|
|
@moduledoc """
|
|
Parse an ADIF file into the same preview format used by CsvImport, so the
|
|
submit UI can reuse the preview/commit flow.
|
|
|
|
ADIF tagged-data format: `<FIELD_NAME:LENGTH>VALUE`
|
|
Records end with `<EOR>`. An optional header ends with `<EOH>`.
|
|
"""
|
|
|
|
import Ecto.Query, only: [from: 2]
|
|
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Repo
|
|
|
|
@dedup_window_seconds 3600
|
|
|
|
# Band plan: map ADIF band names to our MHz values
|
|
@band_name_to_mhz %{
|
|
"23cm" => 1296,
|
|
"13cm" => 2304,
|
|
"9cm" => 3456,
|
|
"6cm" => 5760,
|
|
"3cm" => 10_000,
|
|
"1.25cm" => 24_000,
|
|
"6mm" => 47_000,
|
|
"4mm" => 75_000,
|
|
"2.5mm" => 122_000,
|
|
"2mm" => 134_000,
|
|
"1mm" => 241_000
|
|
}
|
|
|
|
# Our allowed band centers in MHz, sorted ascending for nearest-match
|
|
@allowed_bands [1296, 2304, 3456, 5760, 10_000, 24_000, 47_000, 68_000, 75_000, 122_000, 134_000, 241_000]
|
|
|
|
@doc """
|
|
Parse ADIF string, validate, and deduplicate. Returns the same preview
|
|
shape as `CsvImport.preview/2`.
|
|
"""
|
|
def preview(adif_string, submitter_email) do
|
|
records = parse_adif(adif_string)
|
|
|
|
if records == [] do
|
|
{:error, :no_records}
|
|
else
|
|
rows = records |> Enum.with_index(1) |> Enum.map(&build_row(&1, submitter_email))
|
|
{valid, invalid} = split_parsed(rows)
|
|
existing = load_existing_for_dedup(valid)
|
|
{unique, duplicates} = dedupe(valid, existing)
|
|
|
|
{:ok,
|
|
%{
|
|
valid: unique,
|
|
invalid: invalid,
|
|
duplicates: duplicates,
|
|
total_rows: length(records),
|
|
submitter_email: submitter_email
|
|
}}
|
|
end
|
|
end
|
|
|
|
# -- ADIF parser -----------------------------------------------------------
|
|
|
|
defp parse_adif(string) do
|
|
# Strip header if present (everything before <EOH>)
|
|
body =
|
|
case Regex.run(~r/<EOH>/i, string, return: :index) do
|
|
[{pos, len}] -> String.slice(string, (pos + len)..-1//1)
|
|
nil -> string
|
|
end
|
|
|
|
# Split on <EOR> to get individual records
|
|
body
|
|
|> String.split(~r/<EOR>/i)
|
|
|> Enum.map(&parse_fields/1)
|
|
|> Enum.reject(&(&1 == %{}))
|
|
end
|
|
|
|
defp parse_fields(record_string) do
|
|
~r/<([^:>]+):(\d+)(?::[^>]*)?>/i
|
|
|> Regex.scan(record_string, return: :index)
|
|
|> Enum.reduce(%{}, fn indices, acc ->
|
|
[{tag_start, _tag_len}, {name_start, name_len}, {size_start, size_len} | _] = indices
|
|
name = record_string |> String.slice(name_start, name_len) |> String.upcase()
|
|
size = record_string |> String.slice(size_start, size_len) |> String.to_integer()
|
|
|
|
# Value starts right after the closing >
|
|
# Find the > after the tag
|
|
tag_text = String.slice(record_string, tag_start, 999)
|
|
value_offset = tag_start + (tag_text |> String.split(">", parts: 2) |> List.first() |> byte_size()) + 1
|
|
value = String.slice(record_string, value_offset, size)
|
|
|
|
Map.put(acc, name, value)
|
|
end)
|
|
end
|
|
|
|
# -- row building ----------------------------------------------------------
|
|
|
|
defp build_row({fields, row_num}, submitter_email) do
|
|
station1 = fields["STATION_CALLSIGN"] || fields["OPERATOR"]
|
|
station2 = fields["CALL"]
|
|
grid1 = fields["MY_GRIDSQUARE"]
|
|
grid2 = fields["GRIDSQUARE"]
|
|
mode = fields["MODE"]
|
|
band = resolve_band(fields)
|
|
timestamp = parse_adif_datetime(fields["QSO_DATE"], fields["TIME_ON"])
|
|
|
|
case {band, timestamp} do
|
|
{nil, _} ->
|
|
{:invalid, %{row_num: row_num, messages: ["could not determine a valid microwave band"]}}
|
|
|
|
{_, nil} ->
|
|
{:invalid, %{row_num: row_num, messages: ["could not parse QSO_DATE/TIME_ON"]}}
|
|
|
|
{band_str, {:ok, dt}} ->
|
|
attrs = %{
|
|
"station1" => station1,
|
|
"station2" => station2,
|
|
"grid1" => grid1,
|
|
"grid2" => grid2,
|
|
"band" => band_str,
|
|
"mode" => mode,
|
|
"qso_timestamp" => DateTime.to_iso8601(dt),
|
|
"submitter_email" => submitter_email
|
|
}
|
|
|
|
changeset = Contact.submission_changeset(%Contact{}, attrs)
|
|
|
|
if changeset.valid? do
|
|
{:parsed, %{row_num: row_num, attrs: attrs, timestamp: dt}}
|
|
else
|
|
messages = changeset_error_strings(changeset)
|
|
{:invalid, %{row_num: row_num, messages: messages}}
|
|
end
|
|
end
|
|
end
|
|
|
|
# -- band resolution -------------------------------------------------------
|
|
|
|
defp resolve_band(fields) do
|
|
cond do
|
|
freq = fields["FREQ"] ->
|
|
freq_mhz = parse_freq_mhz(freq)
|
|
nearest_band(freq_mhz)
|
|
|
|
band_name = fields["BAND"] ->
|
|
band_name = String.downcase(band_name)
|
|
mhz = @band_name_to_mhz[band_name]
|
|
if mhz, do: to_string(mhz)
|
|
|
|
true ->
|
|
nil
|
|
end
|
|
end
|
|
|
|
defp parse_freq_mhz(freq_str) do
|
|
{freq, _} = Float.parse(String.trim(freq_str))
|
|
# ADIF FREQ is always in MHz (e.g., 10368.000 for 10 GHz band)
|
|
freq
|
|
end
|
|
|
|
defp nearest_band(freq_mhz) when freq_mhz >= 900 do
|
|
@allowed_bands
|
|
|> Enum.min_by(&abs(&1 - freq_mhz))
|
|
|> to_string()
|
|
end
|
|
|
|
defp nearest_band(_freq_mhz), do: nil
|
|
|
|
# -- timestamp parsing -----------------------------------------------------
|
|
|
|
defp parse_adif_datetime(nil, _time), do: nil
|
|
defp parse_adif_datetime(_date, nil), do: nil
|
|
|
|
defp parse_adif_datetime(date_str, time_str) do
|
|
with <<y::binary-4, m::binary-2, d::binary-2>> <- date_str,
|
|
{hour, min, sec} <- parse_adif_time(time_str),
|
|
{:ok, naive} <- NaiveDateTime.new(si(y), si(m), si(d), hour, min, sec) do
|
|
{:ok, DateTime.from_naive!(naive, "Etc/UTC")}
|
|
else
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
defp parse_adif_time(<<h::binary-2, m::binary-2, s::binary-2>>), do: {si(h), si(m), si(s)}
|
|
defp parse_adif_time(<<h::binary-2, m::binary-2>>), do: {si(h), si(m), 0}
|
|
defp parse_adif_time(_), do: nil
|
|
|
|
defp si(s), do: String.to_integer(s)
|
|
|
|
# -- split / dedup (same logic as CsvImport) --------------------------------
|
|
|
|
defp split_parsed(rows) do
|
|
rows
|
|
|> Enum.reduce({[], []}, fn
|
|
{:parsed, row}, {valid, invalid} -> {[row | valid], invalid}
|
|
{:invalid, row}, {valid, invalid} -> {valid, [row | invalid]}
|
|
end)
|
|
|> then(fn {valid, invalid} -> {Enum.reverse(valid), Enum.reverse(invalid)} end)
|
|
end
|
|
|
|
defp load_existing_for_dedup([]), do: %{}
|
|
|
|
defp load_existing_for_dedup(valid_rows) do
|
|
bands = valid_rows |> Enum.map(& &1.attrs["band"]) |> Enum.uniq() |> Enum.map(&Decimal.new/1)
|
|
timestamps = Enum.map(valid_rows, & &1.timestamp)
|
|
min_ts = timestamps |> Enum.min(DateTime) |> DateTime.add(-@dedup_window_seconds, :second)
|
|
max_ts = timestamps |> Enum.max(DateTime) |> DateTime.add(@dedup_window_seconds, :second)
|
|
|
|
from(c in Contact,
|
|
where:
|
|
c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and
|
|
c.flagged_invalid == false,
|
|
select: %{
|
|
station1: c.station1,
|
|
station2: c.station2,
|
|
grid1: c.grid1,
|
|
grid2: c.grid2,
|
|
band: c.band,
|
|
qso_timestamp: c.qso_timestamp
|
|
}
|
|
)
|
|
|> Repo.all()
|
|
|> Enum.reduce(%{}, fn row, acc ->
|
|
key = dedup_key(row.station1, row.grid1, row.station2, row.grid2, row.band)
|
|
Map.update(acc, key, [row.qso_timestamp], &[row.qso_timestamp | &1])
|
|
end)
|
|
end
|
|
|
|
defp dedupe(rows, db_map) do
|
|
{accepted, _in_batch, duplicates} =
|
|
Enum.reduce(rows, {[], %{}, []}, fn row, {accepted, in_batch, dups} ->
|
|
key =
|
|
dedup_key(
|
|
row.attrs["station1"],
|
|
row.attrs["grid1"],
|
|
row.attrs["station2"],
|
|
row.attrs["grid2"],
|
|
row.attrs["band"]
|
|
)
|
|
|
|
cond do
|
|
conflict_in_map?(db_map, key, row.timestamp) ->
|
|
{accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups]}
|
|
|
|
conflict_in_map?(in_batch, key, row.timestamp) ->
|
|
{accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups]}
|
|
|
|
true ->
|
|
new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1])
|
|
{[row | accepted], new_batch, dups}
|
|
end
|
|
end)
|
|
|
|
{Enum.reverse(accepted), Enum.reverse(duplicates)}
|
|
end
|
|
|
|
defp conflict_in_map?(map, key, timestamp) do
|
|
case Map.fetch(map, key) do
|
|
:error -> false
|
|
{:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp))
|
|
end
|
|
end
|
|
|
|
defp within_window?(t1, t2) do
|
|
abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds
|
|
end
|
|
|
|
defp dedup_key(s1, g1, s2, g2, band) do
|
|
a = {upcase(s1), upcase(g1)}
|
|
b = {upcase(s2), upcase(g2)}
|
|
[c1, c2] = Enum.sort([a, b])
|
|
{c1, c2, normalize_band(band)}
|
|
end
|
|
|
|
defp upcase(nil), do: ""
|
|
defp upcase(v), do: v |> to_string() |> String.trim() |> String.upcase()
|
|
|
|
defp normalize_band(%Decimal{} = b), do: Decimal.to_integer(b)
|
|
defp normalize_band(b) when is_integer(b), do: b
|
|
defp normalize_band(b) when is_binary(b), do: b |> String.trim() |> String.to_integer()
|
|
|
|
defp changeset_error_strings(changeset) do
|
|
changeset
|
|
|> Ecto.Changeset.traverse_errors(fn {message, opts} ->
|
|
Regex.replace(~r"%{(\w+)}", message, fn _, key ->
|
|
opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
|
|
end)
|
|
end)
|
|
|> Enum.flat_map(fn {field, messages} ->
|
|
Enum.map(messages, &"#{field} #{&1}")
|
|
end)
|
|
end
|
|
end
|