CSV upload preview with duplicate detection before commit
Uploads now show a review page with three summary cards (valid, duplicates, invalid), tables for invalid and duplicate rows, and a sample of the first 20 valid rows. Nothing is written to the database until the user clicks "Looks good — submit N contacts". Duplicate detection treats two rows as the same contact if they share the same pair of (callsign, grid) tuples (direction-agnostic), the same band, and timestamps within 60 minutes. Dedup runs against both existing DB contacts (single band/time-scoped query) and earlier rows in the same upload, and the preview labels which side matched. CsvImport grows preview/2 + commit/1; the old import/2 is kept as a thin backward-compat wrapper that skips dedup. Added 11 new CsvImport tests and reworked the SubmitLive CSV tests for the two-step flow.
This commit is contained in:
parent
664f1353db
commit
44b169b234
4 changed files with 670 additions and 52 deletions
|
|
@ -1,23 +1,112 @@
|
|||
defmodule Microwaveprop.Radio.CsvImport do
|
||||
@moduledoc false
|
||||
@moduledoc """
|
||||
Parse a CSV of contacts into a preview (validated + de-duplicated) and then
|
||||
commit those rows into the database.
|
||||
|
||||
Upload flow:
|
||||
|
||||
{:ok, preview} = CsvImport.preview(csv_string, submitter_email)
|
||||
# show preview.valid / preview.invalid / preview.duplicates to the user
|
||||
{:ok, result} = CsvImport.commit(preview.valid)
|
||||
|
||||
Two rows count as duplicates of each other if they share:
|
||||
|
||||
* the same pair of callsigns (direction-agnostic),
|
||||
* the same grids for those callsigns,
|
||||
* the same band, and
|
||||
* timestamps within 60 minutes of each other.
|
||||
|
||||
Dedup runs both against existing DB contacts and against earlier rows in
|
||||
the same CSV (first row wins, subsequent matches are flagged).
|
||||
"""
|
||||
|
||||
import Ecto.Query, only: [from: 2]
|
||||
|
||||
alias Microwaveprop.Radio
|
||||
alias Microwaveprop.Radio.Contact
|
||||
alias Microwaveprop.Repo
|
||||
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
||||
|
||||
@dedup_window_seconds 3600
|
||||
|
||||
@expected_columns 7
|
||||
@columns ~w(station1 station2 grid1 grid2 band mode qso_timestamp)
|
||||
|
||||
@doc """
|
||||
Imports contacts from a CSV string. The first line is treated as a header.
|
||||
Each valid row is inserted via `Radio.create_contact/1` and enrichment is enqueued.
|
||||
The submitter_email is added to each row's attributes.
|
||||
|
||||
Returns:
|
||||
- `{:ok, %{imported: [%Contact{}, ...], errors: [{row_num, [error_string]}]}}`
|
||||
- `{:error, :empty_csv}` if the input is empty or whitespace-only
|
||||
- `{:error, :no_data_rows}` if there are no data rows after the header
|
||||
Parse-and-insert in one shot. Kept for backward compatibility — does NOT
|
||||
perform duplicate detection. New code should use `preview/2` + `commit/1`.
|
||||
"""
|
||||
def import(csv_string, submitter_email) do
|
||||
with {:ok, rows} <- parse_csv(csv_string, submitter_email) do
|
||||
{valid, invalid} = split_parsed(rows)
|
||||
{:ok, commit_result} = commit(valid)
|
||||
invalid_errors = Enum.map(invalid, fn %{row_num: n, messages: m} -> {n, m} end)
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
imported: commit_result.imported,
|
||||
errors: invalid_errors ++ commit_result.errors
|
||||
}}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Parses the CSV, validates each row with the submission changeset, and
|
||||
classifies every row as valid / invalid / duplicate without inserting.
|
||||
|
||||
Returns:
|
||||
|
||||
* `{:ok, preview}` where `preview` is a map with keys
|
||||
`:valid`, `:invalid`, `:duplicates`, `:total_rows`, `:submitter_email`.
|
||||
* `{:error, :empty_csv}` if there is nothing to process.
|
||||
* `{:error, :no_data_rows}` if only a header is present.
|
||||
|
||||
Valid rows are maps of the form `%{row_num, attrs, timestamp}`. Duplicate
|
||||
rows add a `:reason` field indicating whether the conflict was found in
|
||||
the database (`:existing_contact`) or earlier in the uploaded file
|
||||
(`:earlier_in_upload`).
|
||||
"""
|
||||
def preview(csv_string, submitter_email) do
|
||||
with {:ok, rows} <- parse_csv(csv_string, submitter_email) do
|
||||
{valid, invalid} = split_parsed(rows)
|
||||
existing = load_existing_for_dedup(valid)
|
||||
{unique, duplicates} = dedupe(valid, existing)
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
valid: unique,
|
||||
invalid: invalid,
|
||||
duplicates: duplicates,
|
||||
total_rows: length(rows),
|
||||
submitter_email: submitter_email
|
||||
}}
|
||||
end
|
||||
end
|
||||
|
||||
@doc """
|
||||
Inserts the rows returned by `preview/2` in `:valid`. Each is passed to
|
||||
`Radio.create_contact/1` and its enrichment is enqueued on success.
|
||||
Returns `{:ok, %{imported: [...], errors: [{row_num, messages}, ...]}}`.
|
||||
"""
|
||||
def commit(valid_rows) when is_list(valid_rows) do
|
||||
result =
|
||||
Enum.reduce(valid_rows, %{imported: [], errors: []}, fn row, acc ->
|
||||
case Radio.create_contact(row.attrs) do
|
||||
{:ok, contact} ->
|
||||
ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
|
||||
%{acc | imported: [contact | acc.imported]}
|
||||
|
||||
{:error, %Ecto.Changeset{} = changeset} ->
|
||||
%{acc | errors: [{row.row_num, changeset_error_strings(changeset)} | acc.errors]}
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}}
|
||||
end
|
||||
|
||||
# -- parsing ---------------------------------------------------------------
|
||||
|
||||
defp parse_csv(csv_string, submitter_email) do
|
||||
numbered_lines =
|
||||
csv_string
|
||||
|> String.replace("\r\n", "\n")
|
||||
|
|
@ -34,28 +123,14 @@ defmodule Microwaveprop.Radio.CsvImport do
|
|||
{:error, :no_data_rows}
|
||||
|
||||
[{_header, _} | data_lines] ->
|
||||
import_data_lines(data_lines, submitter_email)
|
||||
rows = Enum.map(data_lines, fn {line, row_num} -> parse_row(line, row_num, submitter_email) end)
|
||||
{:ok, rows}
|
||||
end
|
||||
end
|
||||
|
||||
defp blank?(line), do: String.trim(line) == ""
|
||||
|
||||
defp import_data_lines(data_lines, submitter_email) do
|
||||
result =
|
||||
Enum.reduce(data_lines, %{imported: [], errors: []}, fn {line, row_num}, acc ->
|
||||
case parse_and_import_row(line, submitter_email) do
|
||||
{:ok, contact} ->
|
||||
%{acc | imported: [contact | acc.imported]}
|
||||
|
||||
{:error, error_messages} ->
|
||||
%{acc | errors: [{row_num, error_messages} | acc.errors]}
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}}
|
||||
end
|
||||
|
||||
defp parse_and_import_row(line, submitter_email) do
|
||||
defp parse_row(line, row_num, submitter_email) do
|
||||
fields = parse_csv_fields(line)
|
||||
|
||||
if length(fields) == @expected_columns do
|
||||
|
|
@ -68,24 +143,132 @@ defmodule Microwaveprop.Radio.CsvImport do
|
|||
case normalize_timestamp(attrs["qso_timestamp"]) do
|
||||
{:ok, iso_timestamp} ->
|
||||
attrs = Map.put(attrs, "qso_timestamp", iso_timestamp)
|
||||
changeset = Contact.submission_changeset(%Contact{}, attrs)
|
||||
|
||||
case Radio.create_contact(attrs) do
|
||||
{:ok, contact} ->
|
||||
ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
|
||||
{:ok, contact}
|
||||
|
||||
{:error, changeset} ->
|
||||
{:error, changeset_error_strings(changeset)}
|
||||
if changeset.valid? do
|
||||
{:ok, datetime} = parse_iso_datetime(iso_timestamp)
|
||||
{:parsed, %{row_num: row_num, attrs: attrs, timestamp: datetime}}
|
||||
else
|
||||
{:invalid, %{row_num: row_num, messages: changeset_error_strings(changeset)}}
|
||||
end
|
||||
|
||||
{:error, msg} ->
|
||||
{:error, [msg]}
|
||||
{:invalid, %{row_num: row_num, messages: [msg]}}
|
||||
end
|
||||
else
|
||||
{:error, ["expected #{@expected_columns} columns, got #{length(fields)}"]}
|
||||
{:invalid, %{row_num: row_num, messages: ["expected #{@expected_columns} columns, got #{length(fields)}"]}}
|
||||
end
|
||||
end
|
||||
|
||||
defp split_parsed(rows) do
|
||||
rows
|
||||
|> Enum.reduce({[], []}, fn
|
||||
{:parsed, row}, {valid, invalid} -> {[row | valid], invalid}
|
||||
{:invalid, row}, {valid, invalid} -> {valid, [row | invalid]}
|
||||
end)
|
||||
|> then(fn {valid, invalid} -> {Enum.reverse(valid), Enum.reverse(invalid)} end)
|
||||
end
|
||||
|
||||
# -- dedup -----------------------------------------------------------------
|
||||
|
||||
defp load_existing_for_dedup([]), do: %{}
|
||||
|
||||
defp load_existing_for_dedup(valid_rows) do
|
||||
bands = valid_rows |> Enum.map(& &1.attrs["band"]) |> Enum.uniq() |> Enum.map(&to_decimal/1)
|
||||
timestamps = Enum.map(valid_rows, & &1.timestamp)
|
||||
min_ts = timestamps |> Enum.min(DateTime) |> DateTime.add(-@dedup_window_seconds, :second)
|
||||
max_ts = timestamps |> Enum.max(DateTime) |> DateTime.add(@dedup_window_seconds, :second)
|
||||
|
||||
from(c in Contact,
|
||||
where:
|
||||
c.band in ^bands and c.qso_timestamp >= ^min_ts and c.qso_timestamp <= ^max_ts and
|
||||
c.flagged_invalid == false,
|
||||
select: %{
|
||||
station1: c.station1,
|
||||
station2: c.station2,
|
||||
grid1: c.grid1,
|
||||
grid2: c.grid2,
|
||||
band: c.band,
|
||||
qso_timestamp: c.qso_timestamp
|
||||
}
|
||||
)
|
||||
|> Repo.all()
|
||||
|> Enum.reduce(%{}, fn row, acc ->
|
||||
key = dedup_key(row.station1, row.grid1, row.station2, row.grid2, row.band)
|
||||
Map.update(acc, key, [row.qso_timestamp], &[row.qso_timestamp | &1])
|
||||
end)
|
||||
end
|
||||
|
||||
defp dedupe(rows, db_map) do
|
||||
# in_batch_map accumulates keys from rows already accepted in this CSV so
|
||||
# we can distinguish DB duplicates from earlier-row duplicates when we
|
||||
# report them.
|
||||
{accepted, _in_batch, duplicates} =
|
||||
Enum.reduce(rows, {[], %{}, []}, fn row, {accepted, in_batch, dups} ->
|
||||
key = dedup_key_from_attrs(row.attrs)
|
||||
|
||||
cond do
|
||||
conflict_in_map?(db_map, key, row.timestamp) ->
|
||||
{accepted, in_batch, [Map.put(row, :reason, :existing_contact) | dups]}
|
||||
|
||||
conflict_in_map?(in_batch, key, row.timestamp) ->
|
||||
{accepted, in_batch, [Map.put(row, :reason, :earlier_in_upload) | dups]}
|
||||
|
||||
true ->
|
||||
new_batch = Map.update(in_batch, key, [row.timestamp], &[row.timestamp | &1])
|
||||
{[row | accepted], new_batch, dups}
|
||||
end
|
||||
end)
|
||||
|
||||
{Enum.reverse(accepted), Enum.reverse(duplicates)}
|
||||
end
|
||||
|
||||
defp conflict_in_map?(map, key, timestamp) do
|
||||
case Map.fetch(map, key) do
|
||||
:error -> false
|
||||
{:ok, timestamps} -> Enum.any?(timestamps, &within_window?(&1, timestamp))
|
||||
end
|
||||
end
|
||||
|
||||
defp within_window?(t1, t2) do
|
||||
abs(DateTime.diff(t1, t2, :second)) <= @dedup_window_seconds
|
||||
end
|
||||
|
||||
defp dedup_key_from_attrs(attrs) do
|
||||
dedup_key(attrs["station1"], attrs["grid1"], attrs["station2"], attrs["grid2"], attrs["band"])
|
||||
end
|
||||
|
||||
defp dedup_key(s1, g1, s2, g2, band) do
|
||||
a = normalize_callsite(s1, g1)
|
||||
b = normalize_callsite(s2, g2)
|
||||
[c1, c2] = Enum.sort([a, b])
|
||||
{c1, c2, normalize_band(band)}
|
||||
end
|
||||
|
||||
defp normalize_callsite(call, grid) do
|
||||
{upcase_trim(call), upcase_trim(grid)}
|
||||
end
|
||||
|
||||
defp upcase_trim(nil), do: ""
|
||||
defp upcase_trim(value), do: value |> to_string() |> String.trim() |> String.upcase()
|
||||
|
||||
defp normalize_band(%Decimal{} = b), do: Decimal.to_integer(b)
|
||||
defp normalize_band(b) when is_integer(b), do: b
|
||||
defp normalize_band(b) when is_binary(b), do: b |> String.trim() |> String.to_integer()
|
||||
|
||||
defp to_decimal(%Decimal{} = d), do: d
|
||||
defp to_decimal(v) when is_integer(v), do: Decimal.new(v)
|
||||
defp to_decimal(v) when is_binary(v), do: v |> String.trim() |> Decimal.new()
|
||||
|
||||
defp parse_iso_datetime(iso) do
|
||||
case DateTime.from_iso8601(iso) do
|
||||
{:ok, dt, _} -> {:ok, dt}
|
||||
_ -> :error
|
||||
end
|
||||
end
|
||||
|
||||
# -- timestamps ------------------------------------------------------------
|
||||
|
||||
# Regex patterns for timestamp parsing, ordered most-specific first.
|
||||
# Accepts many hand-entered formats. All times assumed UTC.
|
||||
# Each returns {year, month, day, hour, minute, second} or nil.
|
||||
|
|
@ -172,6 +355,8 @@ defmodule Microwaveprop.Radio.CsvImport do
|
|||
{:ok, "#{y}-#{m}-#{d}T#{h}:#{mi}:#{s}Z"}
|
||||
end
|
||||
|
||||
# -- csv field parser ------------------------------------------------------
|
||||
|
||||
# RFC 4180 CSV field parser. Handles quoted fields with commas and escaped quotes.
|
||||
defp parse_csv_fields(line) do
|
||||
line
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ defmodule MicrowavepropWeb.SubmitLive do
|
|||
mode_options: @mode_options,
|
||||
submitted_at: nil,
|
||||
active_tab: :single,
|
||||
csv_preview: nil,
|
||||
csv_result: nil,
|
||||
csv_email: ""
|
||||
)}
|
||||
|
|
@ -92,7 +93,7 @@ defmodule MicrowavepropWeb.SubmitLive do
|
|||
|
||||
case uploaded_contents do
|
||||
[content] ->
|
||||
handle_csv_import(content, email, socket)
|
||||
handle_csv_preview(content, email, socket)
|
||||
|
||||
[] ->
|
||||
{:noreply, put_flash(socket, :error, "Please select a CSV file")}
|
||||
|
|
@ -100,14 +101,33 @@ defmodule MicrowavepropWeb.SubmitLive do
|
|||
end
|
||||
end
|
||||
|
||||
def handle_event("reset_csv", _params, socket) do
|
||||
{:noreply, assign(socket, csv_result: nil)}
|
||||
def handle_event("confirm_csv", _params, socket) do
|
||||
case socket.assigns.csv_preview do
|
||||
%{valid: valid_rows} when valid_rows != [] ->
|
||||
{:ok, commit_result} = CsvImport.commit(valid_rows)
|
||||
|
||||
{:noreply,
|
||||
socket
|
||||
|> assign(csv_preview: nil, csv_result: commit_result)
|
||||
|> put_flash(:info, "Imported #{length(commit_result.imported)} contacts.")}
|
||||
|
||||
_ ->
|
||||
{:noreply, put_flash(socket, :error, "Nothing to import.")}
|
||||
end
|
||||
end
|
||||
|
||||
defp handle_csv_import(content, email, socket) do
|
||||
case CsvImport.import(content, email) do
|
||||
{:ok, result} ->
|
||||
{:noreply, assign(socket, csv_result: result, csv_email: email)}
|
||||
def handle_event("cancel_csv", _params, socket) do
|
||||
{:noreply, assign(socket, csv_preview: nil, csv_result: nil)}
|
||||
end
|
||||
|
||||
def handle_event("reset_csv", _params, socket) do
|
||||
{:noreply, assign(socket, csv_preview: nil, csv_result: nil)}
|
||||
end
|
||||
|
||||
defp handle_csv_preview(content, email, socket) do
|
||||
case CsvImport.preview(content, email) do
|
||||
{:ok, preview} ->
|
||||
{:noreply, assign(socket, csv_preview: preview, csv_result: nil, csv_email: email)}
|
||||
|
||||
{:error, :empty_csv} ->
|
||||
{:noreply, put_flash(socket, :error, "CSV file is empty")}
|
||||
|
|
@ -181,10 +201,13 @@ defmodule MicrowavepropWeb.SubmitLive do
|
|||
<%= if @active_tab == :single do %>
|
||||
<.single_contact_form form={@form} band_options={@band_options} mode_options={@mode_options} />
|
||||
<% else %>
|
||||
<%= if @csv_result do %>
|
||||
<.csv_results result={@csv_result} />
|
||||
<% else %>
|
||||
<.csv_upload_form uploads={@uploads} />
|
||||
<%= cond do %>
|
||||
<% @csv_result -> %>
|
||||
<.csv_results result={@csv_result} />
|
||||
<% @csv_preview -> %>
|
||||
<.csv_preview preview={@csv_preview} />
|
||||
<% true -> %>
|
||||
<.csv_upload_form uploads={@uploads} />
|
||||
<% end %>
|
||||
<% end %>
|
||||
</Layouts.app>
|
||||
|
|
@ -311,6 +334,189 @@ defmodule MicrowavepropWeb.SubmitLive do
|
|||
"""
|
||||
end
|
||||
|
||||
defp csv_preview(assigns) do
|
||||
assigns =
|
||||
assign(assigns,
|
||||
valid_count: length(assigns.preview.valid),
|
||||
invalid_count: length(assigns.preview.invalid),
|
||||
duplicate_count: length(assigns.preview.duplicates),
|
||||
valid_sample: Enum.take(assigns.preview.valid, 20)
|
||||
)
|
||||
|
||||
~H"""
|
||||
<div class="space-y-6">
|
||||
<div>
|
||||
<h2 class="text-xl font-bold mb-2">Review before submitting</h2>
|
||||
<p class="text-sm opacity-70">
|
||||
Processed {@preview.total_rows} {if @preview.total_rows == 1, do: "row", else: "rows"}.
|
||||
Nothing has been inserted yet — confirm below to import.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="grid grid-cols-1 sm:grid-cols-3 gap-3">
|
||||
<.summary_card
|
||||
tone="success"
|
||||
label="Valid"
|
||||
value={@valid_count}
|
||||
hint="Will be inserted"
|
||||
/>
|
||||
<.summary_card
|
||||
tone="warning"
|
||||
label="Duplicates"
|
||||
value={@duplicate_count}
|
||||
hint="Match existing or earlier rows"
|
||||
/>
|
||||
<.summary_card
|
||||
tone="error"
|
||||
label="Invalid"
|
||||
value={@invalid_count}
|
||||
hint="Skipped — see errors"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div :if={@invalid_count > 0}>
|
||||
<h3 class="font-semibold mb-2">Invalid rows ({@invalid_count})</h3>
|
||||
<div class="overflow-x-auto rounded-box border border-base-300">
|
||||
<table class="table table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="w-24">Row</th>
|
||||
<th>Errors</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr :for={row <- Enum.take(@preview.invalid, 100)}>
|
||||
<td class="font-mono">Row {row.row_num}</td>
|
||||
<td>
|
||||
<div :for={msg <- row.messages}>{msg}</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p :if={@invalid_count > 100} class="text-xs opacity-60 mt-1">
|
||||
Showing first 100 of {@invalid_count} invalid rows.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div :if={@duplicate_count > 0}>
|
||||
<h3 class="font-semibold mb-2">Duplicates ({@duplicate_count})</h3>
|
||||
<p class="text-xs opacity-60 mb-2">
|
||||
Same two callsigns at the same grids on the same band within an hour.
|
||||
</p>
|
||||
<div class="overflow-x-auto rounded-box border border-base-300">
|
||||
<table class="table table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="w-20">Row</th>
|
||||
<th>Station 1</th>
|
||||
<th>Station 2</th>
|
||||
<th>Band</th>
|
||||
<th>Timestamp</th>
|
||||
<th>Source</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr :for={row <- Enum.take(@preview.duplicates, 100)}>
|
||||
<td class="font-mono">Row {row.row_num}</td>
|
||||
<td>{row.attrs["station1"]} <span class="opacity-60">{row.attrs["grid1"]}</span></td>
|
||||
<td>{row.attrs["station2"]} <span class="opacity-60">{row.attrs["grid2"]}</span></td>
|
||||
<td class="tabular-nums">{row.attrs["band"]}</td>
|
||||
<td class="font-mono text-xs">
|
||||
{Calendar.strftime(row.timestamp, "%Y-%m-%d %H:%M UTC")}
|
||||
</td>
|
||||
<td class="text-xs opacity-70">{duplicate_source(row.reason)}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p :if={@duplicate_count > 100} class="text-xs opacity-60 mt-1">
|
||||
Showing first 100 of {@duplicate_count} duplicates.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div :if={@valid_count > 0}>
|
||||
<h3 class="font-semibold mb-2">Valid rows ready to import ({@valid_count})</h3>
|
||||
<div class="overflow-x-auto rounded-box border border-base-300">
|
||||
<table class="table table-sm">
|
||||
<thead>
|
||||
<tr>
|
||||
<th class="w-20">Row</th>
|
||||
<th>Station 1</th>
|
||||
<th>Station 2</th>
|
||||
<th>Band</th>
|
||||
<th>Mode</th>
|
||||
<th>Timestamp</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr :for={row <- @valid_sample}>
|
||||
<td class="font-mono">Row {row.row_num}</td>
|
||||
<td>{row.attrs["station1"]} <span class="opacity-60">{row.attrs["grid1"]}</span></td>
|
||||
<td>{row.attrs["station2"]} <span class="opacity-60">{row.attrs["grid2"]}</span></td>
|
||||
<td class="tabular-nums">{row.attrs["band"]}</td>
|
||||
<td>{row.attrs["mode"]}</td>
|
||||
<td class="font-mono text-xs">
|
||||
{Calendar.strftime(row.timestamp, "%Y-%m-%d %H:%M UTC")}
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<p :if={@valid_count > length(@valid_sample)} class="text-xs opacity-60 mt-1">
|
||||
Showing first {length(@valid_sample)} of {@valid_count} valid rows.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="flex flex-wrap gap-2 pt-4 border-t border-base-300">
|
||||
<button
|
||||
:if={@valid_count > 0}
|
||||
type="button"
|
||||
class="btn btn-primary btn-lg"
|
||||
phx-click="confirm_csv"
|
||||
phx-disable-with="Importing..."
|
||||
data-confirm={"Import #{@valid_count} contacts?"}
|
||||
>
|
||||
<.icon name="hero-check" class="w-5 h-5" />
|
||||
Looks good — submit {@valid_count} {if @valid_count == 1,
|
||||
do: "contact",
|
||||
else: "contacts"}
|
||||
</button>
|
||||
<button type="button" class="btn btn-ghost" phx-click="cancel_csv">
|
||||
Cancel
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
||||
attr :tone, :string, required: true
|
||||
attr :label, :string, required: true
|
||||
attr :value, :integer, required: true
|
||||
attr :hint, :string, required: true
|
||||
|
||||
defp summary_card(assigns) do
|
||||
~H"""
|
||||
<div class={[
|
||||
"rounded-box border p-4",
|
||||
case @tone do
|
||||
"success" -> "border-success/30 bg-success/10"
|
||||
"warning" -> "border-warning/30 bg-warning/10"
|
||||
"error" -> "border-error/30 bg-error/10"
|
||||
_ -> "border-base-300 bg-base-200"
|
||||
end
|
||||
]}>
|
||||
<div class="text-xs uppercase tracking-wider opacity-70">{@label}</div>
|
||||
<div class="text-3xl font-bold tabular-nums">{@value}</div>
|
||||
<div class="text-xs opacity-60">{@hint}</div>
|
||||
</div>
|
||||
"""
|
||||
end
|
||||
|
||||
defp duplicate_source(:existing_contact), do: "Already in database"
|
||||
defp duplicate_source(:earlier_in_upload), do: "Earlier row in this upload"
|
||||
defp duplicate_source(_), do: "Duplicate"
|
||||
|
||||
defp csv_results(assigns) do
|
||||
~H"""
|
||||
<div class="space-y-4">
|
||||
|
|
@ -329,7 +535,7 @@ defmodule MicrowavepropWeb.SubmitLive do
|
|||
<div class="alert alert-error">
|
||||
<.icon name="hero-exclamation-triangle" class="w-5 h-5" />
|
||||
<span>
|
||||
{length(@result.errors)} {if length(@result.errors) == 1, do: "row", else: "rows"} had errors
|
||||
{length(@result.errors)} {if length(@result.errors) == 1, do: "row", else: "rows"} had errors at insert time
|
||||
</span>
|
||||
</div>
|
||||
|
||||
|
|
|
|||
|
|
@ -425,4 +425,149 @@ defmodule Microwaveprop.Radio.CsvImportTest do
|
|||
end
|
||||
end
|
||||
end
|
||||
|
||||
describe "preview/2" do
|
||||
test "returns empty/no-data errors like import/2" do
|
||||
assert {:error, :empty_csv} = CsvImport.preview("", @submitter)
|
||||
assert {:error, :no_data_rows} = CsvImport.preview(@valid_header, @submitter)
|
||||
end
|
||||
|
||||
test "classifies a valid row as valid and nothing is inserted" do
|
||||
csv = @valid_header <> "\n" <> @valid_row
|
||||
|
||||
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
|
||||
assert [%{row_num: 2, attrs: attrs, timestamp: %DateTime{}}] = preview.valid
|
||||
assert attrs["station1"] == "W5XD"
|
||||
assert preview.invalid == []
|
||||
assert preview.duplicates == []
|
||||
assert preview.total_rows == 1
|
||||
assert preview.submitter_email == @submitter
|
||||
assert Repo.aggregate(Contact, :count) == 0
|
||||
end
|
||||
|
||||
test "classifies bad rows as invalid" do
|
||||
csv =
|
||||
Enum.join(
|
||||
[
|
||||
@valid_header,
|
||||
"W5XD,K5TR,ZZZZ,EM00,10000,CW,2026-03-28T18:00:00Z",
|
||||
"W5XD,K5TR,EM12,EM00,99999,CW,2026-03-28T18:00:00Z"
|
||||
],
|
||||
"\n"
|
||||
)
|
||||
|
||||
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
|
||||
assert preview.valid == []
|
||||
assert length(preview.invalid) == 2
|
||||
assert Enum.all?(preview.invalid, &(&1.messages != []))
|
||||
end
|
||||
|
||||
test "flags an existing DB contact as a duplicate" do
|
||||
csv = @valid_header <> "\n" <> @valid_row
|
||||
{:ok, preview} = CsvImport.preview(csv, @submitter)
|
||||
{:ok, %{imported: [_]}} = CsvImport.commit(preview.valid)
|
||||
|
||||
assert {:ok, second} = CsvImport.preview(csv, @submitter)
|
||||
assert second.valid == []
|
||||
assert [%{row_num: 2, reason: :existing_contact}] = second.duplicates
|
||||
end
|
||||
|
||||
test "treats direction swap as the same contact" do
|
||||
csv1 = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z"
|
||||
{:ok, p1} = CsvImport.preview(csv1, @submitter)
|
||||
{:ok, _} = CsvImport.commit(p1.valid)
|
||||
|
||||
# Same contact from K5TR's side — stations and grids swapped.
|
||||
csv2 = @valid_header <> "\n" <> "K5TR,W5XD,EM00,EM12,10000,CW,2026-03-28T18:20:00Z"
|
||||
{:ok, p2} = CsvImport.preview(csv2, @submitter)
|
||||
assert p2.valid == []
|
||||
assert [%{reason: :existing_contact}] = p2.duplicates
|
||||
end
|
||||
|
||||
test "flags an earlier row in the same upload as a duplicate" do
|
||||
csv =
|
||||
Enum.join(
|
||||
[
|
||||
@valid_header,
|
||||
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
|
||||
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:30:00Z"
|
||||
],
|
||||
"\n"
|
||||
)
|
||||
|
||||
{:ok, preview} = CsvImport.preview(csv, @submitter)
|
||||
assert [%{row_num: 2}] = preview.valid
|
||||
assert [%{row_num: 3, reason: :earlier_in_upload}] = preview.duplicates
|
||||
end
|
||||
|
||||
test "rows more than an hour apart are not duplicates" do
|
||||
csv =
|
||||
Enum.join(
|
||||
[
|
||||
@valid_header,
|
||||
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
|
||||
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T19:30:00Z"
|
||||
],
|
||||
"\n"
|
||||
)
|
||||
|
||||
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
|
||||
assert length(preview.valid) == 2
|
||||
assert preview.duplicates == []
|
||||
end
|
||||
|
||||
test "different bands are never duplicates" do
|
||||
csv =
|
||||
Enum.join(
|
||||
[
|
||||
@valid_header,
|
||||
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
|
||||
"W5XD,K5TR,EM12,EM00,24000,CW,2026-03-28T18:00:00Z"
|
||||
],
|
||||
"\n"
|
||||
)
|
||||
|
||||
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
|
||||
assert length(preview.valid) == 2
|
||||
end
|
||||
|
||||
test "different grids are not duplicates (rover moved)" do
|
||||
csv =
|
||||
Enum.join(
|
||||
[
|
||||
@valid_header,
|
||||
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
|
||||
"W5XD,K5TR,EM13,EM00,10000,CW,2026-03-28T18:15:00Z"
|
||||
],
|
||||
"\n"
|
||||
)
|
||||
|
||||
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
|
||||
assert length(preview.valid) == 2
|
||||
end
|
||||
end
|
||||
|
||||
describe "commit/1" do
|
||||
test "inserts each valid row and returns the created contacts" do
|
||||
csv =
|
||||
Enum.join(
|
||||
[
|
||||
@valid_header,
|
||||
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
|
||||
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
|
||||
],
|
||||
"\n"
|
||||
)
|
||||
|
||||
{:ok, preview} = CsvImport.preview(csv, @submitter)
|
||||
assert {:ok, %{imported: [a, b], errors: []}} = CsvImport.commit(preview.valid)
|
||||
assert %Contact{station1: "W5XD"} = a
|
||||
assert %Contact{station1: "N5AC"} = b
|
||||
assert Repo.aggregate(Contact, :count) == 2
|
||||
end
|
||||
|
||||
test "commit of [] is a no-op" do
|
||||
assert {:ok, %{imported: [], errors: []}} = CsvImport.commit([])
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ defmodule MicrowavepropWeb.SubmitLiveTest do
|
|||
assert html =~ "Download sample CSV"
|
||||
end
|
||||
|
||||
test "uploads valid CSV and shows import count", %{conn: conn} do
|
||||
test "uploads valid CSV and shows preview with confirm button", %{conn: conn} do
|
||||
{:ok, lv, _html} = live(conn, ~p"/submit")
|
||||
|
||||
lv
|
||||
|
|
@ -92,10 +92,62 @@ defmodule MicrowavepropWeb.SubmitLiveTest do
|
|||
|> form("#csv-upload-form", %{submitter_email: "test@example.com"})
|
||||
|> render_submit()
|
||||
|
||||
assert html =~ "1 contact imported"
|
||||
assert html =~ "Review before submitting"
|
||||
assert html =~ "Looks good"
|
||||
# Nothing was inserted yet
|
||||
assert Repo.aggregate(Contact, :count) == 0
|
||||
|
||||
confirm_html =
|
||||
lv
|
||||
|> element("button[phx-click=confirm_csv]")
|
||||
|> render_click()
|
||||
|
||||
assert confirm_html =~ "1 contact imported"
|
||||
assert Repo.aggregate(Contact, :count) == 1
|
||||
end
|
||||
|
||||
test "shows errors for invalid rows alongside successful imports", %{conn: conn} do
|
||||
test "preview flags duplicates before confirmation", %{conn: conn} do
|
||||
# Seed an existing contact so the next upload is a duplicate of it.
|
||||
{:ok, _} =
|
||||
Microwaveprop.Radio.create_contact(%{
|
||||
"station1" => "W5XD",
|
||||
"station2" => "K5TR",
|
||||
"grid1" => "EM12",
|
||||
"grid2" => "EM00",
|
||||
"band" => "10000",
|
||||
"mode" => "CW",
|
||||
"qso_timestamp" => "2026-03-28T18:00:00Z",
|
||||
"submitter_email" => "seed@example.com"
|
||||
})
|
||||
|
||||
{:ok, lv, _html} = live(conn, ~p"/submit")
|
||||
|
||||
lv
|
||||
|> element("[phx-value-tab=csv]")
|
||||
|> render_click()
|
||||
|
||||
csv_content =
|
||||
"station1,station2,grid1,grid2,band,mode,qso_timestamp\n" <>
|
||||
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:15:00Z\n"
|
||||
|
||||
csv_input =
|
||||
file_input(lv, "#csv-upload-form", :csv, [
|
||||
%{name: "contacts.csv", content: csv_content, type: "text/csv"}
|
||||
])
|
||||
|
||||
render_upload(csv_input, "contacts.csv")
|
||||
|
||||
html =
|
||||
lv
|
||||
|> form("#csv-upload-form", %{submitter_email: "test@example.com"})
|
||||
|> render_submit()
|
||||
|
||||
assert html =~ "Duplicates"
|
||||
assert html =~ "Already in database"
|
||||
refute html =~ "Looks good"
|
||||
end
|
||||
|
||||
test "shows errors for invalid rows and still offers the confirm button", %{conn: conn} do
|
||||
{:ok, lv, _html} = live(conn, ~p"/submit")
|
||||
|
||||
lv
|
||||
|
|
@ -119,9 +171,39 @@ defmodule MicrowavepropWeb.SubmitLiveTest do
|
|||
|> form("#csv-upload-form", %{submitter_email: "test@example.com"})
|
||||
|> render_submit()
|
||||
|
||||
assert html =~ "1 contact imported"
|
||||
assert html =~ "1 row had errors"
|
||||
assert html =~ "Invalid rows"
|
||||
assert html =~ "Row 3"
|
||||
assert html =~ "Looks good"
|
||||
end
|
||||
|
||||
test "cancel_csv clears the preview", %{conn: conn} do
|
||||
{:ok, lv, _html} = live(conn, ~p"/submit")
|
||||
|
||||
lv
|
||||
|> element("[phx-value-tab=csv]")
|
||||
|> render_click()
|
||||
|
||||
csv_content =
|
||||
"station1,station2,grid1,grid2,band,mode,qso_timestamp\nW5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z\n"
|
||||
|
||||
csv_input =
|
||||
file_input(lv, "#csv-upload-form", :csv, [
|
||||
%{name: "contacts.csv", content: csv_content, type: "text/csv"}
|
||||
])
|
||||
|
||||
render_upload(csv_input, "contacts.csv")
|
||||
|
||||
lv
|
||||
|> form("#csv-upload-form", %{submitter_email: "test@example.com"})
|
||||
|> render_submit()
|
||||
|
||||
html =
|
||||
lv
|
||||
|> element("button[phx-click=cancel_csv]")
|
||||
|> render_click()
|
||||
|
||||
assert html =~ "Upload CSV"
|
||||
assert Repo.aggregate(Contact, :count) == 0
|
||||
end
|
||||
|
||||
test "requires email for CSV upload", %{conn: conn} do
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue