Add CSV bulk upload to /submit page

Adds a tabbed UI with single-contact form and CSV upload. Users can
download a sample CSV, upload their file, and get partial import with
per-row error reporting. Enrichment jobs enqueue for each imported contact.
This commit is contained in:
Graham McIntire 2026-04-01 15:05:51 -05:00
parent 86b6c0db31
commit 18a291555c
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59
5 changed files with 665 additions and 57 deletions

View file

@ -0,0 +1,92 @@
defmodule Microwaveprop.Radio.CsvImport do
@moduledoc false
alias Microwaveprop.Radio
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
@expected_columns 7
@columns ~w(station1 station2 grid1 grid2 band mode qso_timestamp)
@doc """
Imports contacts from a CSV string. The first line is treated as a header.
Each valid row is inserted via `Radio.create_contact/1` and enrichment is enqueued.
The submitter_email is added to each row's attributes.
Returns:
- `{:ok, %{imported: [%Contact{}, ...], errors: [{row_num, [error_string]}]}}`
- `{:error, :empty_csv}` if the input is empty or whitespace-only
- `{:error, :no_data_rows}` if there are no data rows after the header
"""
def import(csv_string, submitter_email) do
numbered_lines =
csv_string
|> String.replace("\r\n", "\n")
|> String.split("\n")
|> Enum.with_index(1)
non_blank = Enum.reject(numbered_lines, fn {line, _num} -> blank?(line) end)
case non_blank do
[] ->
{:error, :empty_csv}
[{_header, _}] ->
{:error, :no_data_rows}
[{_header, _} | data_lines] ->
import_data_lines(data_lines, submitter_email)
end
end
defp blank?(line), do: String.trim(line) == ""
defp import_data_lines(data_lines, submitter_email) do
result =
Enum.reduce(data_lines, %{imported: [], errors: []}, fn {line, row_num}, acc ->
case parse_and_import_row(line, submitter_email) do
{:ok, contact} ->
%{acc | imported: [contact | acc.imported]}
{:error, error_messages} ->
%{acc | errors: [{row_num, error_messages} | acc.errors]}
end
end)
{:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}}
end
defp parse_and_import_row(line, submitter_email) do
fields = line |> String.split(",") |> Enum.map(&String.trim/1)
if length(fields) == @expected_columns do
attrs =
@columns
|> Enum.zip(fields)
|> Map.new()
|> Map.put("submitter_email", submitter_email)
case Radio.create_contact(attrs) do
{:ok, contact} ->
ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
{:ok, contact}
{:error, changeset} ->
{:error, changeset_error_strings(changeset)}
end
else
{:error, ["expected #{@expected_columns} columns, got #{length(fields)}"]}
end
end
defp changeset_error_strings(changeset) do
changeset
|> Ecto.Changeset.traverse_errors(fn {message, opts} ->
Regex.replace(~r"%{(\w+)}", message, fn _, key ->
opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
end)
end)
|> Enum.flat_map(fn {field, messages} ->
Enum.map(messages, &"#{field} #{&1}")
end)
end
end

View file

@ -4,6 +4,7 @@ defmodule MicrowavepropWeb.SubmitLive do
alias Microwaveprop.Radio
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Radio.CsvImport
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
@band_options [
@ -28,16 +29,25 @@ defmodule MicrowavepropWeb.SubmitLive do
changeset = Radio.change_contact(%Contact{})
{:ok,
assign(socket,
socket
|> assign(
page_title: "Submit Contact",
form: to_form(changeset),
band_options: @band_options,
mode_options: @mode_options,
submitted_at: nil
)}
submitted_at: nil,
active_tab: :single,
csv_result: nil,
csv_email: ""
)
|> allow_upload(:csv, accept: ~w(.csv), max_entries: 1, max_file_size: 10_000_000)}
end
@impl true
def handle_event("switch_tab", %{"tab" => tab}, socket) do
{:noreply, assign(socket, active_tab: String.to_existing_atom(tab))}
end
def handle_event("validate", %{"contact" => contact_params}, socket) do
changeset =
%Contact{}
@ -47,6 +57,10 @@ defmodule MicrowavepropWeb.SubmitLive do
{:noreply, assign(socket, form: to_form(changeset))}
end
def handle_event("validate_csv", _params, socket) do
{:noreply, socket}
end
# Minimum milliseconds between submissions per LiveView session.
# For broader IP-based rate limiting, use a reverse proxy (nginx limit_req, Cloudflare).
@submission_cooldown_ms 30_000
@ -71,6 +85,42 @@ defmodule MicrowavepropWeb.SubmitLive do
end
end
def handle_event("upload_csv", %{"submitter_email" => email}, socket) do
if String.trim(email) == "" do
{:noreply, put_flash(socket, :error, "Email is required for CSV upload")}
else
uploaded_contents =
consume_uploaded_entries(socket, :csv, fn %{path: path}, _entry ->
{:ok, File.read!(path)}
end)
case uploaded_contents do
[content] ->
handle_csv_import(content, email, socket)
[] ->
{:noreply, put_flash(socket, :error, "Please select a CSV file")}
end
end
end
def handle_event("reset_csv", _params, socket) do
{:noreply, assign(socket, csv_result: nil)}
end
defp handle_csv_import(content, email, socket) do
case CsvImport.import(content, email) do
{:ok, result} ->
{:noreply, assign(socket, csv_result: result, csv_email: email)}
{:error, :empty_csv} ->
{:noreply, put_flash(socket, :error, "CSV file is empty")}
{:error, :no_data_rows} ->
{:noreply, put_flash(socket, :error, "CSV file has no data rows")}
end
end
defp recently_submitted?(socket) do
case socket.assigns.submitted_at do
nil -> false
@ -97,62 +147,198 @@ defmodule MicrowavepropWeb.SubmitLive do
</p>
</div>
<.form for={@form} id="contact-form" phx-change="validate" phx-submit="save" class="space-y-4">
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<.input field={@form[:station1]} type="text" label="Station 1" placeholder="W5XD" />
<.input
field={@form[:grid1]}
type="text"
label="Grid 1"
placeholder="EM12kp"
phx-debounce="blur"
/>
<.input field={@form[:station2]} type="text" label="Station 2" placeholder="K5TR" />
<.input
field={@form[:grid2]}
type="text"
label="Grid 2"
placeholder="EM00cd"
phx-debounce="blur"
/>
</div>
<div role="tablist" class="tabs tabs-bordered mb-6">
<button
role="tab"
class={["tab", @active_tab == :single && "tab-active"]}
phx-click="switch_tab"
phx-value-tab="single"
>
Single Contact
</button>
<button
role="tab"
class={["tab", @active_tab == :csv && "tab-active"]}
phx-click="switch_tab"
phx-value-tab="csv"
>
Upload CSV
</button>
</div>
<p class="text-sm text-base-content/60 -mt-2">
Be as specific as possible with grid squares (6 characters preferred, e.g. EM12kp).
</p>
<div class="grid grid-cols-1 md:grid-cols-3 gap-4">
<.input
field={@form[:band]}
type="select"
label="Band"
prompt="Select band"
options={@band_options}
/>
<.input
field={@form[:mode]}
type="select"
label="Mode"
prompt="Select mode"
options={@mode_options}
/>
<.input field={@form[:qso_timestamp]} type="datetime-local" label="Timestamp (UTC)" />
</div>
<.input
field={@form[:submitter_email]}
type="email"
label="Your Email"
placeholder="you@example.com"
/>
<div class="mt-6">
<.button phx-disable-with="Submitting..." class="btn btn-primary btn-lg w-full sm:w-auto">
<.icon name="hero-arrow-up-tray" class="w-5 h-5" /> Submit Contact
</.button>
</div>
</.form>
<%= if @active_tab == :single do %>
<.single_contact_form form={@form} band_options={@band_options} mode_options={@mode_options} />
<% else %>
<%= if @csv_result do %>
<.csv_results result={@csv_result} />
<% else %>
<.csv_upload_form uploads={@uploads} />
<% end %>
<% end %>
</Layouts.app>
"""
end
defp single_contact_form(assigns) do
~H"""
<.form for={@form} id="contact-form" phx-change="validate" phx-submit="save" class="space-y-4">
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
<.input field={@form[:station1]} type="text" label="Station 1" placeholder="W5XD" />
<.input
field={@form[:grid1]}
type="text"
label="Grid 1"
placeholder="EM12kp"
phx-debounce="blur"
/>
<.input field={@form[:station2]} type="text" label="Station 2" placeholder="K5TR" />
<.input
field={@form[:grid2]}
type="text"
label="Grid 2"
placeholder="EM00cd"
phx-debounce="blur"
/>
</div>
<p class="text-sm text-base-content/60 -mt-2">
Be as specific as possible with grid squares (6 characters preferred, e.g. EM12kp).
</p>
<div class="grid grid-cols-1 md:grid-cols-3 gap-4">
<.input
field={@form[:band]}
type="select"
label="Band"
prompt="Select band"
options={@band_options}
/>
<.input
field={@form[:mode]}
type="select"
label="Mode"
prompt="Select mode"
options={@mode_options}
/>
<.input field={@form[:qso_timestamp]} type="datetime-local" label="Timestamp (UTC)" />
</div>
<.input
field={@form[:submitter_email]}
type="email"
label="Your Email"
placeholder="you@example.com"
/>
<div class="mt-6">
<.button phx-disable-with="Submitting..." class="btn btn-primary btn-lg w-full sm:w-auto">
<.icon name="hero-arrow-up-tray" class="w-5 h-5" /> Submit Contact
</.button>
</div>
</.form>
"""
end
defp csv_upload_form(assigns) do
~H"""
<div class="space-y-6">
<div class="bg-base-200 rounded-box p-4 text-sm">
<p class="mb-2">
Upload a CSV file with multiple contacts. The file should have these columns:
</p>
<code class="text-xs">station1, station2, grid1, grid2, band, mode, qso_timestamp</code>
<p class="mt-2 text-base-content/60">
Timestamps must be ISO 8601 UTC, e.g. <code>2024-06-15T14:30:00Z</code>.
Grid squares should be as detailed as possible (6 characters preferred, e.g. EM12kp).
</p>
<p class="mt-2">
<a href="/downloads/sample_contacts.csv" download class="link link-primary">
<.icon name="hero-arrow-down-tray" class="w-4 h-4" /> Download sample CSV
</a>
</p>
</div>
<form id="csv-upload-form" phx-submit="upload_csv" phx-change="validate_csv" class="space-y-4">
<div class="fieldset mb-2">
<label>
<span class="label mb-1">CSV File</span>
<.live_file_input upload={@uploads.csv} class="file-input file-input-bordered w-full" />
</label>
</div>
<div class="fieldset mb-2">
<label>
<span class="label mb-1">Your Email</span>
<input
type="email"
name="submitter_email"
class="w-full input"
placeholder="you@example.com"
value=""
/>
</label>
</div>
<div class="mt-6">
<button type="submit" class="btn btn-primary btn-lg w-full sm:w-auto">
<.icon name="hero-arrow-up-tray" class="w-5 h-5" /> Upload Contacts
</button>
</div>
</form>
</div>
"""
end
defp csv_results(assigns) do
~H"""
<div class="space-y-4">
<%= if length(@result.imported) > 0 do %>
<div class="alert alert-success">
<.icon name="hero-check-circle" class="w-5 h-5" />
<span>
{length(@result.imported)} {if length(@result.imported) == 1,
do: "contact",
else: "contacts"} imported successfully
</span>
</div>
<% end %>
<%= if length(@result.errors) > 0 do %>
<div class="alert alert-error">
<.icon name="hero-exclamation-triangle" class="w-5 h-5" />
<span>
{length(@result.errors)} {if length(@result.errors) == 1, do: "row", else: "rows"} had errors
</span>
</div>
<div class="overflow-x-auto">
<table class="table table-sm">
<thead>
<tr>
<th>Row</th>
<th>Errors</th>
</tr>
</thead>
<tbody>
<%= for {row_num, messages} <- @result.errors do %>
<tr>
<td class="font-mono">Row {row_num}</td>
<td>
<%= for msg <- messages do %>
<div>{msg}</div>
<% end %>
</td>
</tr>
<% end %>
</tbody>
</table>
</div>
<% end %>
<button type="button" class="btn btn-outline" phx-click="reset_csv">
Upload another
</button>
</div>
"""
end
end

View file

@ -0,0 +1,4 @@
station1,station2,grid1,grid2,band,mode,qso_timestamp
W5XD,K5TR,EM12kp,EM00cd,10000,CW,2024-06-15T14:30:00Z
N5AC,WA5LNL,EM12mr,EM13qa,24000,SSB,2024-06-15T15:00:00Z
KG5CCI,W5LUA,EM13md,EL29fp,47000,CW,2024-07-04T12:00:00Z
1 station1 station2 grid1 grid2 band mode qso_timestamp
2 W5XD K5TR EM12kp EM00cd 10000 CW 2024-06-15T14:30:00Z
3 N5AC WA5LNL EM12mr EM13qa 24000 SSB 2024-06-15T15:00:00Z
4 KG5CCI W5LUA EM13md EL29fp 47000 CW 2024-07-04T12:00:00Z

View file

@ -0,0 +1,208 @@
defmodule Microwaveprop.Radio.CsvImportTest do
use Microwaveprop.DataCase, async: true
alias Microwaveprop.Radio.Contact
alias Microwaveprop.Radio.CsvImport
alias Microwaveprop.Terrain.ElevationClient
alias Microwaveprop.Weather.HrrrClient
alias Microwaveprop.Weather.IemClient
setup do
Req.Test.stub(IemClient, fn conn ->
case conn.request_path do
"/cgi-bin/request/asos.py" -> Req.Test.text(conn, "#DEBUG,\nstation,valid,tmpf\n")
_ -> Req.Test.json(conn, %{"profiles" => []})
end
end)
Req.Test.stub(HrrrClient, fn conn ->
Plug.Conn.send_resp(conn, 404, "not found")
end)
Req.Test.stub(ElevationClient, fn conn ->
params = Plug.Conn.fetch_query_params(conn).query_params
lat_count = params["latitude"] |> String.split(",") |> length()
Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)})
end)
:ok
end
@valid_header "station1,station2,grid1,grid2,band,mode,qso_timestamp"
@valid_row "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z"
@submitter "test@example.com"
describe "import/2 with empty or missing data" do
test "returns error for empty string" do
assert {:error, :empty_csv} = CsvImport.import("", @submitter)
end
test "returns error for whitespace-only string" do
assert {:error, :empty_csv} = CsvImport.import(" \n \n ", @submitter)
end
test "returns error for header-only CSV" do
assert {:error, :no_data_rows} = CsvImport.import(@valid_header, @submitter)
end
test "returns error for header plus blank lines only" do
csv = @valid_header <> "\n\n\n"
assert {:error, :no_data_rows} = CsvImport.import(csv, @submitter)
end
end
describe "import/2 with valid data" do
test "imports a single valid row" do
csv = @valid_header <> "\n" <> @valid_row
assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter)
assert %Contact{} = contact
assert contact.station1 == "W5XD"
assert contact.station2 == "K5TR"
assert contact.grid1 == "EM12"
assert contact.grid2 == "EM00"
assert contact.mode == "CW"
assert contact.user_submitted == true
assert contact.submitter_email == @submitter
assert contact.pos1
assert contact.pos2
assert contact.distance_km
end
test "imports multiple valid rows" do
csv =
Enum.join(
[
@valid_header,
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
assert {:ok, %{imported: imported, errors: []}} = CsvImport.import(csv, @submitter)
assert length(imported) == 2
end
test "trims whitespace from fields" do
csv = @valid_header <> "\n" <> " W5XD , K5TR , EM12 , EM00 , 10000 , CW , 2026-03-28T18:00:00Z "
assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter)
assert contact.station1 == "W5XD"
assert contact.station2 == "K5TR"
end
test "handles Windows line endings" do
csv = @valid_header <> "\r\n" <> @valid_row <> "\r\n"
assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter)
assert contact.station1 == "W5XD"
end
test "skips blank lines between data rows" do
csv =
Enum.join(
[
@valid_header,
@valid_row,
"",
"",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
assert {:ok, %{imported: imported, errors: []}} = CsvImport.import(csv, @submitter)
assert length(imported) == 2
end
test "contacts are persisted to the database" do
csv = @valid_header <> "\n" <> @valid_row
assert {:ok, _result} = CsvImport.import(csv, @submitter)
assert Repo.aggregate(Contact, :count) == 1
end
end
describe "import/2 with invalid data" do
test "reports wrong column count with row number" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert Enum.any?(errors, &String.contains?(&1, "column"))
end
test "reports invalid band" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,99999,CW,2026-03-28T18:00:00Z"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
test "reports invalid grid square" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,ZZZZ,EM00,10000,CW,2026-03-28T18:00:00Z"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
test "reports invalid mode" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,RTTY,2026-03-28T18:00:00Z"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
test "reports missing required fields" do
csv = @valid_header <> "\n" <> ",K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
test "reports invalid timestamp" do
csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,CW,not-a-date"
assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter)
assert length(errors) > 0
end
end
describe "import/2 with mixed valid and invalid rows" do
test "imports valid rows and reports invalid rows" do
csv =
Enum.join(
[
@valid_header,
@valid_row,
"W5XD,K5TR,EM12,EM00,99999,CW,2026-03-28T18:00:00Z",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
assert {:ok, %{imported: imported, errors: errors}} = CsvImport.import(csv, @submitter)
assert length(imported) == 2
assert [{3, _error_msgs}] = errors
end
test "row numbers account for header and blank lines" do
csv =
Enum.join(
[
@valid_header,
@valid_row,
"",
"bad row with wrong columns",
"N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z"
],
"\n"
)
assert {:ok, %{imported: imported, errors: errors}} = CsvImport.import(csv, @submitter)
assert length(imported) == 2
# Row 4 is the bad row (header=1, data=2, blank=3, bad=4, good=5)
assert [{4, _error_msgs}] = errors
end
end
end

View file

@ -56,6 +56,124 @@ defmodule MicrowavepropWeb.SubmitLiveTest do
end
end
describe "csv upload" do
test "renders CSV upload tab and sample download link", %{conn: conn} do
{:ok, lv, _html} = live(conn, ~p"/submit")
html =
lv
|> element("[phx-value-tab=csv]")
|> render_click()
assert html =~ "Upload CSV"
assert html =~ "/downloads/sample_contacts.csv"
assert html =~ "Download sample CSV"
end
test "uploads valid CSV and shows import count", %{conn: conn} do
{:ok, lv, _html} = live(conn, ~p"/submit")
lv
|> element("[phx-value-tab=csv]")
|> render_click()
csv_content =
"station1,station2,grid1,grid2,band,mode,qso_timestamp\nW5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z\n"
csv_input =
file_input(lv, "#csv-upload-form", :csv, [
%{name: "contacts.csv", content: csv_content, type: "text/csv"}
])
render_upload(csv_input, "contacts.csv")
html =
lv
|> form("#csv-upload-form", %{submitter_email: "test@example.com"})
|> render_submit()
assert html =~ "1 contact imported"
end
test "shows errors for invalid rows alongside successful imports", %{conn: conn} do
{:ok, lv, _html} = live(conn, ~p"/submit")
lv
|> element("[phx-value-tab=csv]")
|> render_click()
csv_content =
"station1,station2,grid1,grid2,band,mode,qso_timestamp\n" <>
"W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z\n" <>
",,,,,,\n"
csv_input =
file_input(lv, "#csv-upload-form", :csv, [
%{name: "contacts.csv", content: csv_content, type: "text/csv"}
])
render_upload(csv_input, "contacts.csv")
html =
lv
|> form("#csv-upload-form", %{submitter_email: "test@example.com"})
|> render_submit()
assert html =~ "1 contact imported"
assert html =~ "1 row had errors"
assert html =~ "Row 3"
end
test "requires email for CSV upload", %{conn: conn} do
{:ok, lv, _html} = live(conn, ~p"/submit")
lv
|> element("[phx-value-tab=csv]")
|> render_click()
csv_content =
"station1,station2,grid1,grid2,band,mode,qso_timestamp\nW5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z\n"
csv_input =
file_input(lv, "#csv-upload-form", :csv, [
%{name: "contacts.csv", content: csv_content, type: "text/csv"}
])
render_upload(csv_input, "contacts.csv")
html =
lv
|> form("#csv-upload-form", %{submitter_email: ""})
|> render_submit()
assert html =~ "Email is required for CSV upload"
end
test "handles CSV with no data rows", %{conn: conn} do
{:ok, lv, _html} = live(conn, ~p"/submit")
lv
|> element("[phx-value-tab=csv]")
|> render_click()
csv_content = "station1,station2,grid1,grid2,band,mode,qso_timestamp\n"
csv_input =
file_input(lv, "#csv-upload-form", :csv, [
%{name: "header_only.csv", content: csv_content, type: "text/csv"}
])
render_upload(csv_input, "header_only.csv")
html =
lv
|> form("#csv-upload-form", %{submitter_email: "test@example.com"})
|> render_submit()
assert html =~ "CSV file has no data rows"
end
end
describe "save" do
test "creates QSO and redirects on valid submit", %{conn: conn} do
{:ok, lv, _html} = live(conn, ~p"/submit")