Graham McIntire
cd3950f366
Operators want a place to jot observations that aren't captured by
the structured fields — weather anecdotes, propagation mode
commentary, equipment details, band conditions. Add a text column
on contacts plus the two ingest paths users submit through:
* /submit single-contact form gets a 3-row textarea under the
other fields. Optional, max 2000 chars, with a live length cap
via maxlength. Whitespace-only input collapses to NULL so the
column reflects "no notes" rather than an empty string.
* CSV importer recognises a `notes` (or `note`) column via the
existing header_aliases table and flows values straight through
submission_changeset. Sample template gets a matching example
row with embedded commas so the quoted-field round-trip is
exercised on download.
ADIF import and the refinement/refinement-notify paths are out of
scope — users specifically asked for CSV + single-contact. Tests
cover the changeset (accept/blank/over-length), CSV header parsing
(plain + RFC-4180 quoted), and LiveView form submit end-to-end.
773 lines
25 KiB
Elixir
773 lines
25 KiB
Elixir
defmodule Microwaveprop.Radio.CsvImport do
|
|
@moduledoc """
|
|
Parse a CSV of contacts into a preview (validated + de-duplicated) and then
|
|
commit those rows into the database.
|
|
|
|
Upload flow:
|
|
|
|
{:ok, preview} = CsvImport.preview(csv_string, submitter_email)
|
|
# show preview.valid / preview.invalid / preview.duplicates to the user
|
|
{:ok, result} = CsvImport.commit(preview.valid)
|
|
|
|
Two rows count as duplicates of each other if they share:
|
|
|
|
* the same pair of callsigns (direction-agnostic),
|
|
* the same grids for those callsigns,
|
|
* the same band, and
|
|
* timestamps within 60 minutes of each other.
|
|
|
|
Dedup runs both against existing DB contacts and against earlier rows in
|
|
the same CSV (first row wins, subsequent matches are flagged).
|
|
"""
|
|
|
|
import Ecto.Query, only: [from: 2]
|
|
|
|
alias Microwaveprop.Radio
|
|
alias Microwaveprop.Radio.BandResolver
|
|
alias Microwaveprop.Radio.Contact
|
|
alias Microwaveprop.Radio.ImportMatcher
|
|
alias Microwaveprop.Radio.ImportRun
|
|
alias Microwaveprop.Repo
|
|
alias Microwaveprop.Workers.ContactImportWorker
|
|
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
|
|
|
@dedup_window_seconds 3600
|
|
@chunk_size 100
|
|
|
|
# Header parsing is case-insensitive and whitespace-insensitive. Accepts
|
|
# both the raw field names (station1, grid1, ...) and the human labels the
|
|
# /contacts live_table export produces (Station 1, Grid 1, QSO (UTC), ...)
|
|
# so export → import round-trips. Unknown columns are ignored.
|
|
@header_aliases %{
|
|
"station1" => "station1",
|
|
"station 1" => "station1",
|
|
"station2" => "station2",
|
|
"station 2" => "station2",
|
|
"grid1" => "grid1",
|
|
"grid 1" => "grid1",
|
|
"grid2" => "grid2",
|
|
"grid 2" => "grid2",
|
|
"band" => "band",
|
|
"mode" => "mode",
|
|
"qso_timestamp" => "qso_timestamp",
|
|
"qso timestamp" => "qso_timestamp",
|
|
"qso (utc)" => "qso_timestamp",
|
|
# Operator free-form notes. Optional on import; blank cells
|
|
# collapse to NULL via Contact.submission_changeset's notes
|
|
# normalizer.
|
|
"notes" => "notes",
|
|
"note" => "notes"
|
|
}
|
|
|
|
@required_columns ~w(station1 station2 grid1 grid2 band qso_timestamp)
|
|
|
|
@doc """
|
|
Parse-and-insert in one shot. Kept for backward compatibility — does NOT
|
|
perform duplicate detection. New code should use `preview/2` + `commit/1`.
|
|
"""
|
|
@type preview_result :: %{
|
|
valid: [map()],
|
|
invalid: [map()],
|
|
duplicates: [map()],
|
|
refinements: [map()],
|
|
total_rows: non_neg_integer(),
|
|
submitter_email: String.t()
|
|
}
|
|
|
|
@type import_result :: %{
|
|
imported: [Contact.t()],
|
|
errors: [{pos_integer(), [String.t()]}]
|
|
}
|
|
|
|
@type commit_result :: %{
|
|
imported: [Contact.t()],
|
|
refined: [Contact.t()],
|
|
errors: [{pos_integer(), [String.t()]}]
|
|
}
|
|
|
|
@spec import(String.t(), String.t()) ::
|
|
{:ok, import_result()}
|
|
| {:error, :empty_csv}
|
|
| {:error, :no_data_rows}
|
|
| {:error, {:missing_required_columns, [String.t()]}}
|
|
def import(csv_string, submitter_email) do
|
|
with {:ok, preview} <- preview(csv_string, submitter_email) do
|
|
{:ok, commit_result} = commit(%{valid: preview.valid, refinements: preview.refinements})
|
|
invalid_errors = Enum.map(preview.invalid, fn %{row_num: n, messages: m} -> {n, m} end)
|
|
|
|
{:ok,
|
|
%{
|
|
imported: commit_result.imported,
|
|
errors: invalid_errors ++ commit_result.errors
|
|
}}
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Parses the CSV, validates each row with the submission changeset, and
|
|
classifies every row as valid / invalid / duplicate without inserting.
|
|
|
|
Returns:
|
|
|
|
* `{:ok, preview}` where `preview` is a map with keys
|
|
`:valid`, `:invalid`, `:duplicates`, `:total_rows`, `:submitter_email`.
|
|
* `{:error, :empty_csv}` if there is nothing to process.
|
|
* `{:error, :no_data_rows}` if only a header is present.
|
|
|
|
Valid rows are maps of the form `%{row_num, attrs, timestamp}`. Duplicate
|
|
rows add a `:reason` field indicating whether the conflict was found in
|
|
the database (`:existing_contact`) or earlier in the uploaded file
|
|
(`:earlier_in_upload`).
|
|
"""
|
|
@spec preview(String.t(), String.t()) ::
|
|
{:ok, preview_result()}
|
|
| {:error, :empty_csv}
|
|
| {:error, :no_data_rows}
|
|
| {:error, {:missing_required_columns, [String.t()]}}
|
|
def preview(csv_string, submitter_email) do
|
|
with {:ok, rows} <- parse_csv(csv_string, submitter_email) do
|
|
{valid, invalid} = split_parsed(rows)
|
|
existing = load_existing_for_dedup(valid)
|
|
{unique, duplicates, refinements} = dedupe(valid, existing)
|
|
|
|
{:ok,
|
|
%{
|
|
valid: unique,
|
|
invalid: invalid,
|
|
duplicates: duplicates,
|
|
refinements: refinements,
|
|
total_rows: length(rows),
|
|
submitter_email: submitter_email
|
|
}}
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Commit a preview. Accepts either a list of valid rows (legacy) or a map with
|
|
`:valid` and `:refinements` keys. Valid rows are inserted via
|
|
`Radio.create_contact/1`. Refinements are applied to their matching existing
|
|
contacts via `Radio.apply_contact_refinement/2`; grid-changing refinements
|
|
re-enqueue enrichment, mode-only refinements do not.
|
|
|
|
Returns `{:ok, %{imported: [...], refined: [...], errors: [{row_num, messages}, ...]}}`.
|
|
"""
|
|
@spec commit([map()] | %{valid: [map()], refinements: [map()]}) :: {:ok, commit_result()}
|
|
def commit(valid_rows) when is_list(valid_rows) do
|
|
commit(%{valid: valid_rows, refinements: []})
|
|
end
|
|
|
|
def commit(%{valid: valid_rows, refinements: refinements}) do
|
|
{:ok, commit_rows(%{valid: valid_rows, refinements: refinements})}
|
|
end
|
|
|
|
@doc """
|
|
Synchronously run a batch of CSV rows through the insert / refinement
|
|
pipeline without wrapping the result in an `{:ok, _}` tuple. This is
|
|
the worker-facing variant of `commit/1` and is called per-chunk by
|
|
`Microwaveprop.Workers.ContactImportWorker`.
|
|
|
|
Accepts the same shape as `commit/1`: either a list of valid rows
|
|
(legacy) or a `%{valid: [...], refinements: [...]}` map.
|
|
"""
|
|
@spec commit_rows([map()] | %{valid: [map()], refinements: [map()]}) :: commit_result()
|
|
def commit_rows(valid_rows) when is_list(valid_rows) do
|
|
commit_rows(%{valid: valid_rows, refinements: []})
|
|
end
|
|
|
|
def commit_rows(%{valid: valid_rows, refinements: refinements}) do
|
|
init = %{imported: [], refined: [], errors: []}
|
|
|
|
result =
|
|
init
|
|
|> apply_valid_rows(valid_rows)
|
|
|> apply_refinements(refinements)
|
|
|
|
%{
|
|
imported: Enum.reverse(result.imported),
|
|
refined: Enum.reverse(result.refined),
|
|
errors: Enum.reverse(result.errors)
|
|
}
|
|
end
|
|
|
|
@doc """
|
|
Persist the preview result as an `ImportRun` and enqueue one
|
|
`ContactImportWorker` job per chunk of rows. Returns
|
|
`{:ok, import_run_id}`. The caller can then subscribe to
|
|
`"csv_import:<id>"` on `Microwaveprop.PubSub` to track progress.
|
|
|
|
The submitted rows are stored once on the `ImportRun` row as a JSONB
|
|
blob `%{"rows" => [...], "refinement_ids" => [...]}`. Valid-row entries
|
|
come first and refinements are appended after; `refinement_ids` holds
|
|
the `row_num` values of the refinements so the worker can route each
|
|
row to the correct pipeline without re-classifying.
|
|
"""
|
|
@spec enqueue(preview_result() | %{valid: [map()], refinements: [map()]}, String.t(), keyword()) ::
|
|
{:ok, Ecto.UUID.t()}
|
|
def enqueue(preview, submitter_email, opts \\ []) do
|
|
private = Keyword.get(opts, :private, false)
|
|
valid = preview |> Map.get(:valid, []) |> Enum.map(&stamp_private(&1, private))
|
|
refinements = preview |> Map.get(:refinements, []) |> Enum.map(&stamp_private(&1, private))
|
|
|
|
serialized_rows = Enum.map(valid, &serialize_valid_row/1) ++ Enum.map(refinements, &serialize_refinement_row/1)
|
|
refinement_ids = Enum.map(refinements, & &1.row_num)
|
|
total_rows = length(serialized_rows)
|
|
|
|
attrs = %{
|
|
submitter_email: submitter_email,
|
|
rows: %{"rows" => serialized_rows, "refinement_ids" => refinement_ids},
|
|
total_rows: total_rows
|
|
}
|
|
|
|
{:ok, run} =
|
|
%ImportRun{}
|
|
|> ImportRun.changeset(attrs)
|
|
|> Repo.insert()
|
|
|
|
enqueue_chunks(run.id, total_rows)
|
|
|
|
{:ok, run.id}
|
|
end
|
|
|
|
defp enqueue_chunks(_run_id, 0), do: :ok
|
|
|
|
defp enqueue_chunks(run_id, total_rows) do
|
|
0
|
|
|> Stream.iterate(&(&1 + @chunk_size))
|
|
|> Stream.take_while(&(&1 < total_rows))
|
|
|> Enum.each(fn offset ->
|
|
limit = min(@chunk_size, total_rows - offset)
|
|
|
|
%{"import_run_id" => run_id, "offset" => offset, "limit" => limit}
|
|
|> ContactImportWorker.new()
|
|
|> Oban.insert!()
|
|
end)
|
|
end
|
|
|
|
defp stamp_private(%{attrs: attrs} = row, true), do: %{row | attrs: Map.put(attrs, "private", true)}
|
|
defp stamp_private(row, _), do: row
|
|
|
|
defp serialize_valid_row(row) do
|
|
%{
|
|
"kind" => "insert",
|
|
"row_num" => row.row_num,
|
|
"attrs" => row.attrs,
|
|
"timestamp" => DateTime.to_iso8601(row.timestamp)
|
|
}
|
|
end
|
|
|
|
defp serialize_refinement_row(row) do
|
|
%{
|
|
"kind" => "refinement",
|
|
"row_num" => row.row_num,
|
|
"attrs" => row.attrs,
|
|
"timestamp" => DateTime.to_iso8601(row.timestamp),
|
|
"existing_id" => row.existing_id,
|
|
"changes" => stringify_changes(row.changes)
|
|
}
|
|
end
|
|
|
|
# The refinement map comes out of ImportMatcher with atom keys
|
|
# (:grid1/:grid2/:mode). JSONB serialization only round-trips string
|
|
# keys so normalize here before handing the row to the worker.
|
|
defp stringify_changes(changes) when is_map(changes) do
|
|
Map.new(changes, fn {k, v} -> {to_string(k), v} end)
|
|
end
|
|
|
|
@doc """
|
|
Decode a row out of the JSONB blob stored on `ImportRun.rows` back
|
|
into the shape `commit_rows/1` expects.
|
|
|
|
Returns `{:valid, row}` for insert rows and `{:refinement, row}` for
|
|
refinements so the worker can route each row to the right pipeline.
|
|
"""
|
|
@spec decode_row(map()) :: {:valid, map()} | {:refinement, map()}
|
|
def decode_row(%{"kind" => "insert"} = row) do
|
|
{:valid,
|
|
%{
|
|
row_num: row["row_num"],
|
|
attrs: row["attrs"],
|
|
timestamp: decode_timestamp(row["timestamp"])
|
|
}}
|
|
end
|
|
|
|
def decode_row(%{"kind" => "refinement"} = row) do
|
|
{:refinement,
|
|
%{
|
|
row_num: row["row_num"],
|
|
attrs: row["attrs"],
|
|
timestamp: decode_timestamp(row["timestamp"]),
|
|
existing_id: row["existing_id"],
|
|
changes: atomize_changes(row["changes"])
|
|
}}
|
|
end
|
|
|
|
defp decode_timestamp(nil), do: nil
|
|
|
|
defp decode_timestamp(iso) when is_binary(iso) do
|
|
case DateTime.from_iso8601(iso) do
|
|
{:ok, dt, _} -> dt
|
|
_ -> nil
|
|
end
|
|
end
|
|
|
|
@refinement_change_keys ~w(grid1 grid2 mode band station1 station2 qso_timestamp)
|
|
defp atomize_changes(changes) when is_map(changes) do
|
|
Map.new(changes, fn {k, v} ->
|
|
key_str = to_string(k)
|
|
|
|
if key_str in @refinement_change_keys do
|
|
{String.to_existing_atom(key_str), v}
|
|
else
|
|
{key_str, v}
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp atomize_changes(_), do: %{}
|
|
|
|
@spec chunk_size() :: pos_integer()
|
|
def chunk_size, do: @chunk_size
|
|
|
|
defp apply_valid_rows(acc, valid_rows) do
|
|
Enum.reduce(valid_rows, acc, fn row, acc ->
|
|
case Radio.create_contact(row.attrs) do
|
|
{:ok, contact} ->
|
|
ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
|
|
%{acc | imported: [contact | acc.imported]}
|
|
|
|
{:error, :duplicate, _existing} ->
|
|
acc
|
|
|
|
{:error, %Ecto.Changeset{} = changeset} ->
|
|
%{acc | errors: [{row.row_num, changeset_error_strings(changeset)} | acc.errors]}
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp apply_refinements(acc, refinements) do
|
|
Enum.reduce(refinements, acc, &apply_refinement_row/2)
|
|
end
|
|
|
|
defp apply_refinement_row(ref, acc) do
|
|
case Repo.get(Contact, ref.existing_id) do
|
|
nil ->
|
|
%{acc | errors: [{ref.row_num, ["existing contact no longer exists"]} | acc.errors]}
|
|
|
|
%Contact{} = contact ->
|
|
apply_refinement_to_contact(contact, ref, acc)
|
|
end
|
|
end
|
|
|
|
defp apply_refinement_to_contact(contact, ref, acc) do
|
|
case Radio.apply_contact_refinement(contact, ref.changes) do
|
|
{:ok, refined} ->
|
|
maybe_reenqueue_after_refinement(refined, ref.changes)
|
|
%{acc | refined: [refined | acc.refined]}
|
|
|
|
{:error, %Ecto.Changeset{} = changeset} ->
|
|
%{acc | errors: [{ref.row_num, changeset_error_strings(changeset)} | acc.errors]}
|
|
end
|
|
end
|
|
|
|
defp maybe_reenqueue_after_refinement(contact, changes) do
|
|
if Map.has_key?(changes, :grid1) or Map.has_key?(changes, :grid2) do
|
|
ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
|
|
end
|
|
|
|
:ok
|
|
end
|
|
|
|
# -- parsing ---------------------------------------------------------------
|
|
|
|
defp parse_csv(csv_string, submitter_email) do
|
|
numbered_lines =
|
|
csv_string
|
|
|> String.replace("\r\n", "\n")
|
|
|> String.split("\n")
|
|
|> Enum.with_index(1)
|
|
|
|
non_blank = Enum.reject(numbered_lines, fn {line, _num} -> blank?(line) end)
|
|
|
|
case non_blank do
|
|
[] ->
|
|
{:error, :empty_csv}
|
|
|
|
[{_header, _}] ->
|
|
{:error, :no_data_rows}
|
|
|
|
[{header_line, _} | data_lines] ->
|
|
with {:ok, column_map} <- parse_header(header_line) do
|
|
{:ok, parse_data_rows(data_lines, submitter_email, column_map)}
|
|
end
|
|
end
|
|
end
|
|
|
|
defp parse_data_rows(data_lines, submitter_email, column_map) do
|
|
Enum.map(data_lines, fn {line, row_num} ->
|
|
parse_row(line, row_num, submitter_email, column_map)
|
|
end)
|
|
end
|
|
|
|
defp blank?(line), do: String.trim(line) == ""
|
|
|
|
# Builds an index → field-name map from the header line. Unknown headers
|
|
# are silently skipped (that's how we accept the richer /contacts export).
|
|
defp parse_header(header_line) do
|
|
column_map =
|
|
header_line
|
|
|> parse_csv_fields()
|
|
|> Enum.with_index()
|
|
|> Enum.reduce(%{}, fn {raw, index}, acc ->
|
|
case Map.fetch(@header_aliases, raw |> String.downcase() |> String.trim()) do
|
|
{:ok, field} -> Map.put(acc, index, field)
|
|
:error -> acc
|
|
end
|
|
end)
|
|
|
|
present = column_map |> Map.values() |> MapSet.new()
|
|
missing = Enum.reject(@required_columns, &MapSet.member?(present, &1))
|
|
|
|
if missing == [] do
|
|
{:ok, column_map}
|
|
else
|
|
{:error, {:missing_required_columns, missing}}
|
|
end
|
|
end
|
|
|
|
defp parse_row(line, row_num, submitter_email, column_map) do
|
|
fields = parse_csv_fields(line)
|
|
|
|
attrs =
|
|
column_map
|
|
|> Enum.reduce(%{}, fn {index, field}, acc ->
|
|
case Enum.at(fields, index) do
|
|
nil -> acc
|
|
value -> Map.put(acc, field, value)
|
|
end
|
|
end)
|
|
|> Map.put("submitter_email", submitter_email)
|
|
|
|
if required_present?(attrs) do
|
|
parse_row_with_timestamp(attrs, row_num)
|
|
else
|
|
missing_on_row = Enum.reject(@required_columns, &Map.has_key?(attrs, &1))
|
|
|
|
{:invalid,
|
|
%{
|
|
row_num: row_num,
|
|
messages: ["row is missing required columns: #{Enum.join(missing_on_row, ", ")}"]
|
|
}}
|
|
end
|
|
end
|
|
|
|
defp required_present?(attrs), do: Enum.all?(@required_columns, &Map.has_key?(attrs, &1))
|
|
|
|
defp parse_row_with_timestamp(attrs, row_num) do
|
|
case normalize_timestamp(attrs["qso_timestamp"]) do
|
|
{:ok, iso_timestamp} ->
|
|
validate_parsed_row(attrs, row_num, iso_timestamp)
|
|
|
|
{:error, msg} ->
|
|
{:invalid, %{row_num: row_num, messages: [msg]}}
|
|
end
|
|
end
|
|
|
|
defp validate_parsed_row(attrs, row_num, iso_timestamp) do
|
|
attrs =
|
|
attrs
|
|
|> Map.put("qso_timestamp", iso_timestamp)
|
|
|> normalize_band_attr()
|
|
|
|
changeset = Contact.submission_changeset(%Contact{}, attrs)
|
|
|
|
if changeset.valid? do
|
|
{:ok, datetime} = parse_iso_datetime(iso_timestamp)
|
|
{:parsed, %{row_num: row_num, attrs: attrs, timestamp: datetime}}
|
|
else
|
|
{:invalid, %{row_num: row_num, messages: changeset_error_strings(changeset)}}
|
|
end
|
|
end
|
|
|
|
# CSV users may type ADIF wavelength labels ("33cm"), bare frequencies
|
|
# ("903.100"), or canonical band numbers ("902"). Run the supplied value
|
|
# through BandResolver so any of those forms end up as an integer MHz
|
|
# string that Contact.submission_changeset's validate_inclusion accepts.
|
|
# If the input is unresolvable we leave it alone so the user gets a
|
|
# specific "band is invalid" changeset error instead of a silent drop.
|
|
defp normalize_band_attr(attrs) do
|
|
case attrs["band"] do
|
|
nil ->
|
|
attrs
|
|
|
|
raw ->
|
|
case BandResolver.resolve_as_string(raw) do
|
|
nil -> attrs
|
|
canonical -> Map.put(attrs, "band", canonical)
|
|
end
|
|
end
|
|
end
|
|
|
|
defp split_parsed(rows) do
|
|
rows
|
|
|> Enum.reduce({[], []}, fn
|
|
{:parsed, row}, {valid, invalid} -> {[row | valid], invalid}
|
|
{:invalid, row}, {valid, invalid} -> {valid, [row | invalid]}
|
|
end)
|
|
|> then(fn {valid, invalid} -> {Enum.reverse(valid), Enum.reverse(invalid)} end)
|
|
end
|
|
|
|
# -- dedup -----------------------------------------------------------------
|
|
|
|
defp load_existing_for_dedup([]), do: %{}
|
|
|
|
defp load_existing_for_dedup(valid_rows) do
|
|
bands = valid_rows |> Enum.map(& &1.attrs["band"]) |> Enum.uniq() |> Enum.map(&to_decimal/1)
|
|
timestamps = Enum.map(valid_rows, & &1.timestamp)
|
|
min_ts = timestamps |> Enum.min(DateTime) |> DateTime.add(-@dedup_window_seconds, :second)
|
|
max_ts = timestamps |> Enum.max(DateTime) |> DateTime.add(@dedup_window_seconds, :second)
|
|
|
|
from(c in Contact,
|
|
where:
|
|
c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and
|
|
c.flagged_invalid == false
|
|
)
|
|
|> Repo.all()
|
|
|> Enum.reduce(%{}, fn contact, acc ->
|
|
case ImportMatcher.prefix_key(contact.station1, contact.grid1, contact.station2, contact.grid2, contact.band) do
|
|
:invalid ->
|
|
acc
|
|
|
|
key ->
|
|
Map.update(acc, key, [contact], &[contact | &1])
|
|
end
|
|
end)
|
|
end
|
|
|
|
defp dedupe(rows, db_map) do
|
|
{accepted, _in_batch, duplicates, refinements} =
|
|
Enum.reduce(rows, {[], %{}, [], []}, &classify_row(&1, &2, db_map))
|
|
|
|
{Enum.reverse(accepted), Enum.reverse(duplicates), Enum.reverse(refinements)}
|
|
end
|
|
|
|
defp classify_row(row, {accepted, in_batch, dups, refs}, db_map) do
|
|
key = prefix_key_from_attrs(row.attrs)
|
|
|
|
cond do
|
|
key == :invalid ->
|
|
{[row | accepted], in_batch, dups, refs}
|
|
|
|
candidate = db_candidate(db_map, key, row.timestamp) ->
|
|
classify_db_candidate(row, candidate, accepted, in_batch, dups, refs)
|
|
|
|
batch_conflict?(in_batch, key, row.timestamp) ->
|
|
{accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups], refs}
|
|
|
|
true ->
|
|
new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1])
|
|
{[row | accepted], new_batch, dups, refs}
|
|
end
|
|
end
|
|
|
|
defp classify_db_candidate(row, candidate, accepted, in_batch, dups, refs) do
|
|
case ImportMatcher.classify_against_existing(row.attrs, candidate) do
|
|
{:refinement, changes} ->
|
|
refinement = %{
|
|
row_num: row.row_num,
|
|
attrs: row.attrs,
|
|
timestamp: row.timestamp,
|
|
existing_id: candidate.id,
|
|
changes: changes
|
|
}
|
|
|
|
{accepted, in_batch, dups, [refinement | refs]}
|
|
|
|
_ ->
|
|
{accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups], refs}
|
|
end
|
|
end
|
|
|
|
defp db_candidate(db_map, key, timestamp) do
|
|
case Map.fetch(db_map, key) do
|
|
:error -> nil
|
|
{:ok, contacts} -> Enum.find(contacts, &within_window?(&1.qso_timestamp, timestamp))
|
|
end
|
|
end
|
|
|
|
defp batch_conflict?(map, key, timestamp) do
|
|
case Map.fetch(map, key) do
|
|
:error -> false
|
|
{:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp))
|
|
end
|
|
end
|
|
|
|
defp within_window?(t1, t2) do
|
|
abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds
|
|
end
|
|
|
|
defp prefix_key_from_attrs(attrs) do
|
|
ImportMatcher.prefix_key(
|
|
attrs["station1"],
|
|
attrs["grid1"],
|
|
attrs["station2"],
|
|
attrs["grid2"],
|
|
attrs["band"]
|
|
)
|
|
end
|
|
|
|
defp to_decimal(%Decimal{} = d), do: d
|
|
defp to_decimal(v) when is_integer(v), do: Decimal.new(v)
|
|
defp to_decimal(v) when is_binary(v), do: v |> String.trim() |> Decimal.new()
|
|
|
|
defp parse_iso_datetime(iso) do
|
|
case DateTime.from_iso8601(iso) do
|
|
{:ok, dt, _} -> {:ok, dt}
|
|
_ -> :error
|
|
end
|
|
end
|
|
|
|
# -- timestamps ------------------------------------------------------------
|
|
|
|
# Regex patterns for timestamp parsing, ordered most-specific first.
|
|
# Accepts many hand-entered formats. All times assumed UTC.
|
|
# Each returns {year, month, day, hour, minute, second} or nil.
|
|
defp timestamp_parsers do
|
|
[
|
|
# ISO-ish: 2024-06-15T14:30:00Z, 2024-06-15 14:30, 2024/06/15 14:30:00, etc.
|
|
# Accepts T or space separator, optional seconds, optional timezone suffix
|
|
{~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})[T\s]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$"i,
|
|
fn
|
|
[_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
|
|
[_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"}
|
|
end},
|
|
# Date only: 2024-06-15, 2024/06/15
|
|
{~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$", fn [_, y, m, d] -> {y, m, d, "00", "00", "00"} end},
|
|
# US with AM/PM: 6/15/2024 2:30 PM, 6-15-2024 2:30PM, 06.15.2024 02:30 am
|
|
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM)$"i,
|
|
fn
|
|
[_, m, d, y, h, mi, s, ampm] when byte_size(s) > 0 ->
|
|
{y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, s}
|
|
|
|
[_, m, d, y, h, mi, _, ampm] ->
|
|
{y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, "00"}
|
|
end},
|
|
# US 24h: 6/15/2024 14:30, 06-15-2024 14:30:00, 6.15.2024 14:30
|
|
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?$",
|
|
fn
|
|
[_, m, d, y, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
|
|
[_, m, d, y, h, mi | _] -> {y, m, d, h, mi, "00"}
|
|
end},
|
|
# US date only: 6/15/2024, 06-15-2024
|
|
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})$", fn [_, m, d, y] -> {y, m, d, "00", "00", "00"} end},
|
|
# Compact: 20240615T143000, 20240615 1430, 20240615T1430
|
|
{~r"^(\d{4})(\d{2})(\d{2})[T\s]?(\d{2})(\d{2})(\d{2})?$",
|
|
fn
|
|
[_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
|
|
[_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"}
|
|
end}
|
|
]
|
|
end
|
|
|
|
@doc """
|
|
Parses various date/time formats into ISO 8601 UTC strings.
|
|
All times are assumed UTC.
|
|
"""
|
|
@spec normalize_timestamp(String.t()) :: {:ok, String.t()} | {:error, String.t()}
|
|
def normalize_timestamp(raw) do
|
|
trimmed = String.trim(raw)
|
|
|
|
# Try ISO 8601 with Elixir's built-in parser first
|
|
case DateTime.from_iso8601(trimmed) do
|
|
{:ok, dt, _} ->
|
|
{:ok, DateTime.to_iso8601(dt)}
|
|
|
|
_ ->
|
|
try_regex_parsers(trimmed)
|
|
end
|
|
end
|
|
|
|
defp try_regex_parsers(trimmed) do
|
|
timestamp_parsers()
|
|
|> Enum.find_value(fn {regex, extractor} ->
|
|
case Regex.run(regex, trimmed) do
|
|
nil -> nil
|
|
match -> format_iso(extractor.(match))
|
|
end
|
|
end)
|
|
|> case do
|
|
{:ok, _} = ok -> ok
|
|
nil -> {:error, "qso_timestamp could not be parsed: #{trimmed}"}
|
|
end
|
|
end
|
|
|
|
defp parse_12h(12, "AM"), do: 0
|
|
defp parse_12h(12, "PM"), do: 12
|
|
defp parse_12h(h, "PM"), do: h + 12
|
|
defp parse_12h(h, "AM"), do: h
|
|
|
|
defp format_iso({y, m, d, h, mi, s}) do
|
|
y = String.pad_leading(to_string(y), 4, "0")
|
|
m = String.pad_leading(to_string(m), 2, "0")
|
|
d = String.pad_leading(to_string(d), 2, "0")
|
|
h = String.pad_leading(to_string(h), 2, "0")
|
|
mi = String.pad_leading(to_string(mi), 2, "0")
|
|
s = String.pad_leading(to_string(s), 2, "0")
|
|
{:ok, "#{y}-#{m}-#{d}T#{h}:#{mi}:#{s}Z"}
|
|
end
|
|
|
|
# -- csv field parser ------------------------------------------------------
|
|
|
|
# RFC 4180 CSV field parser. Handles quoted fields with commas and escaped quotes.
|
|
defp parse_csv_fields(line) do
|
|
line
|
|
|> String.trim()
|
|
|> do_parse_fields([], "")
|
|
|> Enum.reverse()
|
|
|> Enum.map(&String.trim/1)
|
|
end
|
|
|
|
defp do_parse_fields("", acc, current), do: [current | acc]
|
|
|
|
defp do_parse_fields(<<"\"", rest::binary>>, acc, "") do
|
|
parse_quoted_field(rest, acc, "")
|
|
end
|
|
|
|
defp do_parse_fields(<<",", rest::binary>>, acc, current) do
|
|
do_parse_fields(rest, [current | acc], "")
|
|
end
|
|
|
|
defp do_parse_fields(<<c, rest::binary>>, acc, current) do
|
|
do_parse_fields(rest, acc, current <> <<c>>)
|
|
end
|
|
|
|
defp parse_quoted_field(<<"\"\"", rest::binary>>, acc, current) do
|
|
parse_quoted_field(rest, acc, current <> "\"")
|
|
end
|
|
|
|
defp parse_quoted_field(<<"\"", rest::binary>>, acc, current) do
|
|
# End of quoted field — skip to next comma or end
|
|
case rest do
|
|
<<",", rest2::binary>> -> do_parse_fields(rest2, [current | acc], "")
|
|
"" -> [current | acc]
|
|
_ -> do_parse_fields(rest, [current | acc], "")
|
|
end
|
|
end
|
|
|
|
defp parse_quoted_field(<<c, rest::binary>>, acc, current) do
|
|
parse_quoted_field(rest, acc, current <> <<c>>)
|
|
end
|
|
|
|
defp parse_quoted_field("", acc, current), do: [current | acc]
|
|
|
|
defp changeset_error_strings(changeset) do
|
|
changeset
|
|
|> Ecto.Changeset.traverse_errors(fn {message, opts} ->
|
|
Regex.replace(~r"%{(\w+)}", message, fn _, key ->
|
|
opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
|
|
end)
|
|
end)
|
|
|> Enum.flat_map(fn {field, messages} ->
|
|
Enum.map(messages, &"#{field} #{&1}")
|
|
end)
|
|
end
|
|
end
|