prop/test/microwaveprop/radio/csv_import_test.exs
Graham McIntire 44b169b234 CSV upload preview with duplicate detection before commit
Uploads now show a review page with three summary cards (valid,
duplicates, invalid), tables for invalid and duplicate rows, and a
sample of the first 20 valid rows. Nothing is written to the database
until the user clicks "Looks good — submit N contacts".

Duplicate detection treats two rows as the same contact if they share
the same pair of (callsign, grid) tuples (direction-agnostic), the
same band, and timestamps within 60 minutes. Dedup runs against both
existing DB contacts (single band/time-scoped query) and earlier rows
in the same upload, and the preview labels which side matched.

CsvImport grows preview/2 + commit/1; the old import/2 is kept as a
thin backward-compat wrapper that skips dedup. Added 11 new CsvImport
tests and reworked the SubmitLive CSV tests for the two-step flow.
2026-04-09 12:30:10 -05:00

573 lines
19 KiB
Elixir

defmodule Microwaveprop.Radio.CsvImportTest do
use Microwaveprop.DataCase, async: true
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Radio.CsvImport
alias Microwaveprop.Terrain.ElevationClient
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.IemClient
setup do
Req.Test.stub(IemClient, fn conn ->
case conn.request_path do
"/cgi-bin/request/asos.py" -> Req.Test.text(conn, "#DEBUG,\nstation,valid,tmpf\n")
_ -> Req.Test.json(conn, %{"profiles" => []})
end
end)
Req.Test.stub(HrrrClient, fn conn ->
Plug.Conn.send_resp(conn, 404, "not found")
end)
Req.Test.stub(ElevationClient, fn conn ->
params = Plug.Conn.fetch_query_params(conn).query_params
lat_count = params["latitude"] |> String.split(",") |> length()
Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
end)
:ok
end
@valid_header "station1,station2,grid1,grid2,band,mode,qso_timestamp"
@valid_row "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z"
@submitter "test@example.com"
describe "import/2 with empty or missing data" do
test "returns error for empty string" do
assert {:error, :empty_csv} = CsvImport.import("", @submitter)
end
test "returns error for whitespace-only string" do
assert {:error, :empty_csv} = CsvImport.import(" \n \n ", @submitter)
end
test "returns error for header-only CSV" do
assert {:error, :no_data_rows} = CsvImport.import(@valid_header, @submitter)
end
test "returns error for header plus blank lines only" do
csv = @valid_header <> "\n\n\n"
assert {:error, :no_data_rows} = CsvImport.import(csv, @submitter)
end
end
describe "import/2 with valid data" do
test "imports a single valid row" do
csv = @valid_header <> "\n" <> @valid_row
assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter)
assert %Contact{} = contact
assert contact.station1 == "W5XD"
assert contact.station2 == "K5TR"
assert contact.grid1 == "EM12"
assert contact.grid2 == "EM00"
assert contact.mode == "CW"
assert contact.user_submitted == true
assert contact.submitter_email == @submitter
assert contact.pos1
assert contact.pos2
assert contact.distance_km
end
test "imports multiple valid rows" do
csv =
Enum.join(
[
@valid_header,
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
assert {:ok, %{imported: imported, errors: []}} = CsvImport.import(csv, @submitter)
assert length(imported) == 2
end
test "trims whitespace from fields" do
csv = @valid_header <> "\n" <> " W5XD , K5TR , EM12 , EM00 , 10000 , CW , 2026-03-28T18:00:00Z "
assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter)
assert contact.station1 == "W5XD"
assert contact.station2 == "K5TR"
end
test "handles Windows line endings" do
csv = @valid_header <> "\r\n" <> @valid_row <> "\r\n"
assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter)
assert contact.station1 == "W5XD"
end
test "skips blank lines between data rows" do
csv =
Enum.join(
[
@valid_header,
@valid_row,
"",
"",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
assert {:ok, %{imported: imported, errors: []}} = CsvImport.import(csv, @submitter)
assert length(imported) == 2
end
test "contacts are persisted to the database" do
csv = @valid_header <> "\n" <> @valid_row
assert {:ok, _result} = CsvImport.import(csv, @submitter)
assert Repo.aggregate(Contact, :count) == 1
end
end
describe "import/2 with invalid data" do
test "reports wrong column count with row number" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert Enum.any?(errors, &String.contains?(&1, "column"))
end
test "reports invalid band" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,99999,CW,2026-03-28T18:00:00Z"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
test "reports invalid grid square" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,ZZZZ,EM00,10000,CW,2026-03-28T18:00:00Z"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
test "reports invalid mode" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,RTTY,2026-03-28T18:00:00Z"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
test "reports missing required fields" do
csv = @valid_header <> "\n" <> ",K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
test "reports invalid timestamp" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,CW,not-a-date"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
end
describe "import/2 with mixed valid and invalid rows" do
test "imports valid rows and reports invalid rows" do
csv =
Enum.join(
[
@valid_header,
@valid_row,
"W5XD,K5TR,EM12,EM00,99999,CW,2026-03-28T18:00:00Z",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
assert {:ok, %{imported: imported, errors: errors}} = CsvImport.import(csv, @submitter)
assert length(imported) == 2
assert [{3, _error_msgs}] = errors
end
test "imports row with US date format" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,CW,6/15/2024 2:30 PM"
assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter)
assert contact.qso_timestamp == ~U[2024-06-15 14:30:00Z]
end
test "imports row with space-separated datetime" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,CW,2024-06-15 14:30"
assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter)
assert contact.qso_timestamp == ~U[2024-06-15 14:30:00Z]
end
test "row numbers account for header and blank lines" do
csv =
Enum.join(
[
@valid_header,
@valid_row,
"",
"bad row with wrong columns",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
assert {:ok, %{imported: imported, errors: errors}} = CsvImport.import(csv, @submitter)
assert length(imported) == 2
# Row 4 is the bad row (header=1, data=2, blank=3, bad=4, good=5)
assert [{4, _error_msgs}] = errors
end
end
describe "normalize_timestamp/1" do
test "parses ISO 8601 with Z" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("2024-06-15T14:30:00Z")
end
test "parses ISO 8601 with offset" do
assert {:ok, _} = CsvImport.normalize_timestamp("2024-06-15T14:30:00+00:00")
end
test "parses ISO without timezone" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("2024-06-15T14:30:00")
end
test "parses space-separated datetime" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("2024-06-15 14:30:00")
end
test "parses space-separated without seconds" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("2024-06-15 14:30")
end
test "parses date only" do
assert {:ok, "2024-06-15T00:00:00Z"} = CsvImport.normalize_timestamp("2024-06-15")
end
test "parses US date with AM/PM" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("6/15/2024 2:30 PM")
end
test "parses US date with AM" do
assert {:ok, "2024-06-15T09:30:00Z"} = CsvImport.normalize_timestamp("6/15/2024 9:30 AM")
end
test "parses 12:00 AM as midnight" do
assert {:ok, "2024-06-15T00:00:00Z"} = CsvImport.normalize_timestamp("6/15/2024 12:00 AM")
end
test "parses 12:00 PM as noon" do
assert {:ok, "2024-06-15T12:00:00Z"} = CsvImport.normalize_timestamp("6/15/2024 12:00 PM")
end
test "parses zero-padded US date with AM/PM" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("06/15/2024 02:30 PM")
end
test "parses US date 24h" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("06/15/2024 14:30")
end
test "parses US date 24h unpadded" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("6/15/2024 14:30")
end
test "parses US date with dashes" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("06-15-2024 14:30")
end
test "parses compact with T" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("20240615T143000")
end
test "parses compact with space" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("20240615 1430")
end
test "trims whitespace" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp(" 2024-06-15T14:30:00Z ")
end
test "returns error for garbage" do
assert {:error, "qso_timestamp could not be parsed: garbage"} =
CsvImport.normalize_timestamp("garbage")
end
test "returns error for empty string" do
assert {:error, _} = CsvImport.normalize_timestamp("")
end
test "parses US date with dots" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("06.15.2024 14:30")
end
test "parses ISO with slashes" do
assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("2024/06/15 14:30")
end
test "parses US date only" do
assert {:ok, "2024-06-15T00:00:00Z"} = CsvImport.normalize_timestamp("6/15/2024")
end
end
describe "normalize_timestamp/1 property tests" do
use ExUnitProperties
property "ISO 8601 format always parses successfully" do
check all(
year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
second <- integer(0..59)
) do
y = String.pad_leading(to_string(year), 4, "0")
m = String.pad_leading(to_string(month), 2, "0")
d = String.pad_leading(to_string(day), 2, "0")
h = String.pad_leading(to_string(hour), 2, "0")
mi = String.pad_leading(to_string(minute), 2, "0")
s = String.pad_leading(to_string(second), 2, "0")
iso = "#{y}-#{m}-#{d}T#{h}:#{mi}:#{s}Z"
assert {:ok, ^iso} = CsvImport.normalize_timestamp(iso)
end
end
property "US date format with 24h time always parses" do
check all(
year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
sep <- member_of(["/", "-", "."])
) do
input = "#{month}#{sep}#{day}#{sep}#{year} #{hour}:#{String.pad_leading(to_string(minute), 2, "0")}"
assert {:ok, result} = CsvImport.normalize_timestamp(input)
assert String.ends_with?(result, "Z")
{:ok, dt, _} = DateTime.from_iso8601(result)
assert dt.year == year
assert dt.month == month
assert dt.day == day
end
end
property "US date with AM/PM parses correctly" do
check all(
year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour_12 <- integer(1..12),
minute <- integer(0..59),
ampm <- member_of(["AM", "PM", "am", "pm"])
) do
input = "#{month}/#{day}/#{year} #{hour_12}:#{String.pad_leading(to_string(minute), 2, "0")} #{ampm}"
assert {:ok, result} = CsvImport.normalize_timestamp(input)
{:ok, dt, _} = DateTime.from_iso8601(result)
expected_hour =
case {hour_12, String.upcase(ampm)} do
{12, "AM"} -> 0
{12, "PM"} -> 12
{h, "PM"} -> h + 12
{h, "AM"} -> h
end
assert dt.hour == expected_hour
assert dt.minute == minute
end
end
property "all parsed results are valid ISO 8601 UTC datetimes" do
formats =
StreamData.member_of([
fn y, m, d, h, mi -> "#{y}-#{m}-#{d}T#{h}:#{mi}:00Z" end,
fn y, m, d, h, mi -> "#{y}-#{m}-#{d} #{h}:#{mi}" end,
fn y, m, d, _h, _mi -> "#{y}-#{m}-#{d}" end,
fn y, m, d, h, mi -> "#{m}/#{d}/#{y} #{h}:#{mi}" end
])
check all(
year <- integer(2000..2030),
month <- integer(1..12),
day <- integer(1..28),
hour <- integer(0..23),
minute <- integer(0..59),
fmt <- formats
) do
y = String.pad_leading(to_string(year), 4, "0")
m = String.pad_leading(to_string(month), 2, "0")
d = String.pad_leading(to_string(day), 2, "0")
h = String.pad_leading(to_string(hour), 2, "0")
mi = String.pad_leading(to_string(minute), 2, "0")
input = fmt.(y, m, d, h, mi)
case CsvImport.normalize_timestamp(input) do
{:ok, result} ->
assert String.ends_with?(result, "Z")
assert {:ok, _, _} = DateTime.from_iso8601(result)
{:error, _} ->
:ok
end
end
end
property "garbage input never returns ok" do
check all(garbage <- string(:alphanumeric, min_length: 1, max_length: 5)) do
case CsvImport.normalize_timestamp(garbage) do
{:ok, _} -> flunk("Parsed garbage: #{inspect(garbage)}")
{:error, _} -> :ok
end
end
end
end
describe "preview/2" do
test "returns empty/no-data errors like import/2" do
assert {:error, :empty_csv} = CsvImport.preview("", @submitter)
assert {:error, :no_data_rows} = CsvImport.preview(@valid_header, @submitter)
end
test "classifies a valid row as valid and nothing is inserted" do
csv = @valid_header <> "\n" <> @valid_row
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
assert [%{row_num: 2, attrs: attrs, timestamp: %DateTime{}}] = preview.valid
assert attrs["station1"] == "W5XD"
assert preview.invalid == []
assert preview.duplicates == []
assert preview.total_rows == 1
assert preview.submitter_email == @submitter
assert Repo.aggregate(Contact, :count) == 0
end
test "classifies bad rows as invalid" do
csv =
Enum.join(
[
@valid_header,
"W5XD,K5TR,ZZZZ,EM00,10000,CW,2026-03-28T18:00:00Z",
"W5XD,K5TR,EM12,EM00,99999,CW,2026-03-28T18:00:00Z"
],
"\n"
)
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
assert preview.valid == []
assert length(preview.invalid) == 2
assert Enum.all?(preview.invalid, &(&1.messages != []))
end
test "flags an existing DB contact as a duplicate" do
csv = @valid_header <> "\n" <> @valid_row
{:ok, preview} = CsvImport.preview(csv, @submitter)
{:ok, %{imported: [_]}} = CsvImport.commit(preview.valid)
assert {:ok, second} = CsvImport.preview(csv, @submitter)
assert second.valid == []
assert [%{row_num: 2, reason: :existing_contact}] = second.duplicates
end
test "treats direction swap as the same contact" do
csv1 = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z"
{:ok, p1} = CsvImport.preview(csv1, @submitter)
{:ok, _} = CsvImport.commit(p1.valid)
# Same contact from K5TR's side — stations and grids swapped.
csv2 = @valid_header <> "\n" <> "K5TR,W5XD,EM00,EM12,10000,CW,2026-03-28T18:20:00Z"
{:ok, p2} = CsvImport.preview(csv2, @submitter)
assert p2.valid == []
assert [%{reason: :existing_contact}] = p2.duplicates
end
test "flags an earlier row in the same upload as a duplicate" do
csv =
Enum.join(
[
@valid_header,
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:30:00Z"
],
"\n"
)
{:ok, preview} = CsvImport.preview(csv, @submitter)
assert [%{row_num: 2}] = preview.valid
assert [%{row_num: 3, reason: :earlier_in_upload}] = preview.duplicates
end
test "rows more than an hour apart are not duplicates" do
csv =
Enum.join(
[
@valid_header,
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T19:30:00Z"
],
"\n"
)
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
assert length(preview.valid) == 2
assert preview.duplicates == []
end
test "different bands are never duplicates" do
csv =
Enum.join(
[
@valid_header,
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
"W5XD,K5TR,EM12,EM00,24000,CW,2026-03-28T18:00:00Z"
],
"\n"
)
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
assert length(preview.valid) == 2
end
test "different grids are not duplicates (rover moved)" do
csv =
Enum.join(
[
@valid_header,
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
"W5XD,K5TR,EM13,EM00,10000,CW,2026-03-28T18:15:00Z"
],
"\n"
)
assert {:ok, preview} = CsvImport.preview(csv, @submitter)
assert length(preview.valid) == 2
end
end
describe "commit/1" do
test "inserts each valid row and returns the created contacts" do
csv =
Enum.join(
[
@valid_header,
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
{:ok, preview} = CsvImport.preview(csv, @submitter)
assert {:ok, %{imported: [a, b], errors: []}} = CsvImport.commit(preview.valid)
assert %Contact{station1: "W5XD"} = a
assert %Contact{station1: "N5AC"} = b
assert Repo.aggregate(Contact, :count) == 2
end
test "commit of [] is a no-op" do
assert {:ok, %{imported: [], errors: []}} = CsvImport.commit([])
end
end
end