diff --git a/lib/microwaveprop/radio/csv_import.ex b/lib/microwaveprop/radio/csv_import.ex new file mode 100644 index 00000000..7344c5f1 --- /dev/null +++ b/lib/microwaveprop/radio/csv_import.ex @@ -0,0 +1,92 @@ +defmodule Microwaveprop.Radio.CsvImport do + @moduledoc false + + alias Microwaveprop.Radio + alias Microwaveprop.Workers.ContactWeatherEnqueueWorker + + @expected_columns 7 + @columns ~w(station1 station2 grid1 grid2 band mode qso_timestamp) + + @doc """ + Imports contacts from a CSV string. The first line is treated as a header. + Each valid row is inserted via `Radio.create_contact/1` and enrichment is enqueued. + The submitter_email is added to each row's attributes. + + Returns: + - `{:ok, %{imported: [%Contact{}, ...], errors: [{row_num, [error_string]}]}}` + - `{:error, :empty_csv}` if the input is empty or whitespace-only + - `{:error, :no_data_rows}` if there are no data rows after the header + """ + def import(csv_string, submitter_email) do + numbered_lines = + csv_string + |> String.replace("\r\n", "\n") + |> String.split("\n") + |> Enum.with_index(1) + + non_blank = Enum.reject(numbered_lines, fn {line, _num} -> blank?(line) end) + + case non_blank do + [] -> + {:error, :empty_csv} + + [{_header, _}] -> + {:error, :no_data_rows} + + [{_header, _} | data_lines] -> + import_data_lines(data_lines, submitter_email) + end + end + + defp blank?(line), do: String.trim(line) == "" + + defp import_data_lines(data_lines, submitter_email) do + result = + Enum.reduce(data_lines, %{imported: [], errors: []}, fn {line, row_num}, acc -> + case parse_and_import_row(line, submitter_email) do + {:ok, contact} -> + %{acc | imported: [contact | acc.imported]} + + {:error, error_messages} -> + %{acc | errors: [{row_num, error_messages} | acc.errors]} + end + end) + + {:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}} + end + + defp parse_and_import_row(line, submitter_email) do + fields = line |> String.split(",") |> Enum.map(&String.trim/1) + + if length(fields) == @expected_columns do + attrs = + @columns + |> Enum.zip(fields) + |> Map.new() + |> Map.put("submitter_email", submitter_email) + + case Radio.create_contact(attrs) do + {:ok, contact} -> + ContactWeatherEnqueueWorker.enqueue_for_contact(contact) + {:ok, contact} + + {:error, changeset} -> + {:error, changeset_error_strings(changeset)} + end + else + {:error, ["expected #{@expected_columns} columns, got #{length(fields)}"]} + end + end + + defp changeset_error_strings(changeset) do + changeset + |> Ecto.Changeset.traverse_errors(fn {message, opts} -> + Regex.replace(~r"%{(\w+)}", message, fn _, key -> + opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string() + end) + end) + |> Enum.flat_map(fn {field, messages} -> + Enum.map(messages, &"#{field} #{&1}") + end) + end +end diff --git a/lib/microwaveprop_web/live/submit_live.ex b/lib/microwaveprop_web/live/submit_live.ex index 1c6253ce..7b9239b0 100644 --- a/lib/microwaveprop_web/live/submit_live.ex +++ b/lib/microwaveprop_web/live/submit_live.ex @@ -4,6 +4,7 @@ defmodule MicrowavepropWeb.SubmitLive do alias Microwaveprop.Radio alias Microwaveprop.Radio.Contact + alias Microwaveprop.Radio.CsvImport alias Microwaveprop.Workers.ContactWeatherEnqueueWorker @band_options [ @@ -28,16 +29,25 @@ defmodule MicrowavepropWeb.SubmitLive do changeset = Radio.change_contact(%Contact{}) {:ok, - assign(socket, + socket + |> assign( page_title: "Submit Contact", form: to_form(changeset), band_options: @band_options, mode_options: @mode_options, - submitted_at: nil - )} + submitted_at: nil, + active_tab: :single, + csv_result: nil, + csv_email: "" + ) + |> allow_upload(:csv, accept: ~w(.csv), max_entries: 1, max_file_size: 10_000_000)} end @impl true + def handle_event("switch_tab", %{"tab" => tab}, socket) do + {:noreply, assign(socket, active_tab: String.to_existing_atom(tab))} + end + def handle_event("validate", %{"contact" => contact_params}, socket) do changeset = %Contact{} @@ -47,6 +57,10 @@ defmodule MicrowavepropWeb.SubmitLive do {:noreply, assign(socket, form: to_form(changeset))} end + def handle_event("validate_csv", _params, socket) do + {:noreply, socket} + end + # Minimum milliseconds between submissions per LiveView session. # For broader IP-based rate limiting, use a reverse proxy (nginx limit_req, Cloudflare). @submission_cooldown_ms 30_000 @@ -71,6 +85,42 @@ defmodule MicrowavepropWeb.SubmitLive do end end + def handle_event("upload_csv", %{"submitter_email" => email}, socket) do + if String.trim(email) == "" do + {:noreply, put_flash(socket, :error, "Email is required for CSV upload")} + else + uploaded_contents = + consume_uploaded_entries(socket, :csv, fn %{path: path}, _entry -> + {:ok, File.read!(path)} + end) + + case uploaded_contents do + [content] -> + handle_csv_import(content, email, socket) + + [] -> + {:noreply, put_flash(socket, :error, "Please select a CSV file")} + end + end + end + + def handle_event("reset_csv", _params, socket) do + {:noreply, assign(socket, csv_result: nil)} + end + + defp handle_csv_import(content, email, socket) do + case CsvImport.import(content, email) do + {:ok, result} -> + {:noreply, assign(socket, csv_result: result, csv_email: email)} + + {:error, :empty_csv} -> + {:noreply, put_flash(socket, :error, "CSV file is empty")} + + {:error, :no_data_rows} -> + {:noreply, put_flash(socket, :error, "CSV file has no data rows")} + end + end + defp recently_submitted?(socket) do case socket.assigns.submitted_at do nil -> false @@ -97,62 +147,198 @@ defmodule MicrowavepropWeb.SubmitLive do

- <.form for={@form} id="contact-form" phx-change="validate" phx-submit="save" class="space-y-4"> -
- <.input field={@form[:station1]} type="text" label="Station 1" placeholder="W5XD" /> - <.input - field={@form[:grid1]} - type="text" - label="Grid 1" - placeholder="EM12kp" - phx-debounce="blur" - /> - <.input field={@form[:station2]} type="text" label="Station 2" placeholder="K5TR" /> - <.input - field={@form[:grid2]} - type="text" - label="Grid 2" - placeholder="EM00cd" - phx-debounce="blur" - /> -
+
+ + +
-

- Be as specific as possible with grid squares (6 characters preferred, e.g. EM12kp). -

- -
- <.input - field={@form[:band]} - type="select" - label="Band" - prompt="Select band" - options={@band_options} - /> - <.input - field={@form[:mode]} - type="select" - label="Mode" - prompt="Select mode" - options={@mode_options} - /> - <.input field={@form[:qso_timestamp]} type="datetime-local" label="Timestamp (UTC)" /> -
- - <.input - field={@form[:submitter_email]} - type="email" - label="Your Email" - placeholder="you@example.com" - /> - -
- <.button phx-disable-with="Submitting..." class="btn btn-primary btn-lg w-full sm:w-auto"> - <.icon name="hero-arrow-up-tray" class="w-5 h-5" /> Submit Contact - -
- + <%= if @active_tab == :single do %> + <.single_contact_form form={@form} band_options={@band_options} mode_options={@mode_options} /> + <% else %> + <%= if @csv_result do %> + <.csv_results result={@csv_result} /> + <% else %> + <.csv_upload_form uploads={@uploads} /> + <% end %> + <% end %> """ end + + defp single_contact_form(assigns) do + ~H""" + <.form for={@form} id="contact-form" phx-change="validate" phx-submit="save" class="space-y-4"> +
+ <.input field={@form[:station1]} type="text" label="Station 1" placeholder="W5XD" /> + <.input + field={@form[:grid1]} + type="text" + label="Grid 1" + placeholder="EM12kp" + phx-debounce="blur" + /> + <.input field={@form[:station2]} type="text" label="Station 2" placeholder="K5TR" /> + <.input + field={@form[:grid2]} + type="text" + label="Grid 2" + placeholder="EM00cd" + phx-debounce="blur" + /> +
+ +

+ Be as specific as possible with grid squares (6 characters preferred, e.g. EM12kp). +

+ +
+ <.input + field={@form[:band]} + type="select" + label="Band" + prompt="Select band" + options={@band_options} + /> + <.input + field={@form[:mode]} + type="select" + label="Mode" + prompt="Select mode" + options={@mode_options} + /> + <.input field={@form[:qso_timestamp]} type="datetime-local" label="Timestamp (UTC)" /> +
+ + <.input + field={@form[:submitter_email]} + type="email" + label="Your Email" + placeholder="you@example.com" + /> + +
+ <.button phx-disable-with="Submitting..." class="btn btn-primary btn-lg w-full sm:w-auto"> + <.icon name="hero-arrow-up-tray" class="w-5 h-5" /> Submit Contact + +
+ + """ + end + + defp csv_upload_form(assigns) do + ~H""" +
+
+

+ Upload a CSV file with multiple contacts. The file should have these columns: +

+ station1, station2, grid1, grid2, band, mode, qso_timestamp +

+ Timestamps must be ISO 8601 UTC, e.g. 2024-06-15T14:30:00Z. + Grid squares should be as detailed as possible (6 characters preferred, e.g. EM12kp). +

+

+ + <.icon name="hero-arrow-down-tray" class="w-4 h-4" /> Download sample CSV + +

+
+ +
+
+ +
+ +
+ +
+ +
+ +
+
+
+ """ + end + + defp csv_results(assigns) do + ~H""" +
+ <%= if length(@result.imported) > 0 do %> +
+ <.icon name="hero-check-circle" class="w-5 h-5" /> + + {length(@result.imported)} {if length(@result.imported) == 1, + do: "contact", + else: "contacts"} imported successfully + +
+ <% end %> + + <%= if length(@result.errors) > 0 do %> +
+ <.icon name="hero-exclamation-triangle" class="w-5 h-5" /> + + {length(@result.errors)} {if length(@result.errors) == 1, do: "row", else: "rows"} had errors + +
+ +
+ + + + + + + + + <%= for {row_num, messages} <- @result.errors do %> + + + + + <% end %> + +
RowErrors
Row {row_num} + <%= for msg <- messages do %> +
{msg}
+ <% end %> +
+
+ <% end %> + + +
+ """ + end end diff --git a/priv/static/downloads/sample_contacts.csv b/priv/static/downloads/sample_contacts.csv new file mode 100644 index 00000000..fed35d7e --- /dev/null +++ b/priv/static/downloads/sample_contacts.csv @@ -0,0 +1,4 @@ +station1,station2,grid1,grid2,band,mode,qso_timestamp +W5XD,K5TR,EM12kp,EM00cd,10000,CW,2024-06-15T14:30:00Z +N5AC,WA5LNL,EM12mr,EM13qa,24000,SSB,2024-06-15T15:00:00Z +KG5CCI,W5LUA,EM13md,EL29fp,47000,CW,2024-07-04T12:00:00Z diff --git a/test/microwaveprop/radio/csv_import_test.exs b/test/microwaveprop/radio/csv_import_test.exs new file mode 100644 index 00000000..021298c3 --- /dev/null +++ b/test/microwaveprop/radio/csv_import_test.exs @@ -0,0 +1,208 @@ +defmodule Microwaveprop.Radio.CsvImportTest do + use Microwaveprop.DataCase, async: true + + alias Microwaveprop.Radio.Contact + alias Microwaveprop.Radio.CsvImport + alias Microwaveprop.Terrain.ElevationClient + alias Microwaveprop.Weather.HrrrClient + alias Microwaveprop.Weather.IemClient + + setup do + Req.Test.stub(IemClient, fn conn -> + case conn.request_path do + "/cgi-bin/request/asos.py" -> Req.Test.text(conn, "#DEBUG,\nstation,valid,tmpf\n") + _ -> Req.Test.json(conn, %{"profiles" => []}) + end + end) + + Req.Test.stub(HrrrClient, fn conn -> + Plug.Conn.send_resp(conn, 404, "not found") + end) + + Req.Test.stub(ElevationClient, fn conn -> + params = Plug.Conn.fetch_query_params(conn).query_params + lat_count = params["latitude"] |> String.split(",") |> length() + Req.Test.json(conn, %{"elevation" => List.duplicate(200.0, lat_count)}) + end) + + :ok + end + + @valid_header "station1,station2,grid1,grid2,band,mode,qso_timestamp" + @valid_row "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z" + @submitter "test@example.com" + + describe "import/2 with empty or missing data" do + test "returns error for empty string" do + assert {:error, :empty_csv} = CsvImport.import("", @submitter) + end + + test "returns error for whitespace-only string" do + assert {:error, :empty_csv} = CsvImport.import(" \n \n ", @submitter) + end + + test "returns error for header-only CSV" do + assert {:error, :no_data_rows} = CsvImport.import(@valid_header, @submitter) + end + + test "returns error for header plus blank lines only" do + csv = @valid_header <> "\n\n\n" + assert {:error, :no_data_rows} = CsvImport.import(csv, @submitter) + end + end + + describe "import/2 with valid data" do + test "imports a single valid row" do + csv = @valid_header <> "\n" <> @valid_row + + assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter) + assert %Contact{} = contact + assert contact.station1 == "W5XD" + assert contact.station2 == "K5TR" + assert contact.grid1 == "EM12" + assert contact.grid2 == "EM00" + assert contact.mode == "CW" + assert contact.user_submitted == true + assert contact.submitter_email == @submitter + assert contact.pos1 + assert contact.pos2 + assert contact.distance_km + end + + test "imports multiple valid rows" do + csv = + Enum.join( + [ + @valid_header, + "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z", + "N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z" + ], + "\n" + ) + + assert {:ok, %{imported: imported, errors: []}} = CsvImport.import(csv, @submitter) + assert length(imported) == 2 + end + + test "trims whitespace from fields" do + csv = @valid_header <> "\n" <> " W5XD , K5TR , EM12 , EM00 , 10000 , CW , 2026-03-28T18:00:00Z " + + assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter) + assert contact.station1 == "W5XD" + assert contact.station2 == "K5TR" + end + + test "handles Windows line endings" do + csv = @valid_header <> "\r\n" <> @valid_row <> "\r\n" + + assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter) + assert contact.station1 == "W5XD" + end + + test "skips blank lines between data rows" do + csv = + Enum.join( + [ + @valid_header, + @valid_row, + "", + "", + "N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z" + ], + "\n" + ) + + assert {:ok, %{imported: imported, errors: []}} = CsvImport.import(csv, @submitter) + assert length(imported) == 2 + end + + test "contacts are persisted to the database" do + csv = @valid_header <> "\n" <> @valid_row + + assert {:ok, _result} = CsvImport.import(csv, @submitter) + assert Repo.aggregate(Contact, :count) == 1 + end + end + + describe "import/2 with invalid data" do + test "reports wrong column count with row number" do + csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12" + + assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter) + assert Enum.any?(errors, &String.contains?(&1, "column")) + end + + test "reports invalid band" do + csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,99999,CW,2026-03-28T18:00:00Z" + + assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter) + assert length(errors) > 0 + end + + test "reports invalid grid square" do + csv = @valid_header <> "\n" <> "W5XD,K5TR,ZZZZ,EM00,10000,CW,2026-03-28T18:00:00Z" + + assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter) + assert length(errors) > 0 + end + + test "reports invalid mode" do + csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,RTTY,2026-03-28T18:00:00Z" + + assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter) + assert length(errors) > 0 + end + + test "reports missing required fields" do + csv = @valid_header <> "\n" <> ",K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z" + + assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter) + assert length(errors) > 0 + end + + test "reports invalid timestamp" do + csv = @valid_header <> "\n" <> "W5XD,K5TR,EM12,EM00,10000,CW,not-a-date" + + assert {:ok, %{imported: [], errors: [{2, errors}]}} = CsvImport.import(csv, @submitter) + assert length(errors) > 0 + end + end + + describe "import/2 with mixed valid and invalid rows" do + test "imports valid rows and reports invalid rows" do + csv = + Enum.join( + [ + @valid_header, + @valid_row, + "W5XD,K5TR,EM12,EM00,99999,CW,2026-03-28T18:00:00Z", + "N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z" + ], + "\n" + ) + + assert {:ok, %{imported: imported, errors: errors}} = CsvImport.import(csv, @submitter) + assert length(imported) == 2 + assert [{3, _error_msgs}] = errors + end + + test "row numbers account for header and blank lines" do + csv = + Enum.join( + [ + @valid_header, + @valid_row, + "", + "bad row with wrong columns", + "N5AC,W5LUA,EM13,EM20,24000,SSB,2026-03-28T19:00:00Z" + ], + "\n" + ) + + assert {:ok, %{imported: imported, errors: errors}} = CsvImport.import(csv, @submitter) + assert length(imported) == 2 + # Row 4 is the bad row (header=1, data=2, blank=3, bad=4, good=5) + assert [{4, _error_msgs}] = errors + end + end +end diff --git a/test/microwaveprop_web/live/submit_live_test.exs b/test/microwaveprop_web/live/submit_live_test.exs index edb4acc1..f4fea863 100644 --- a/test/microwaveprop_web/live/submit_live_test.exs +++ b/test/microwaveprop_web/live/submit_live_test.exs @@ -56,6 +56,124 @@ defmodule MicrowavepropWeb.SubmitLiveTest do end end + describe "csv upload" do + test "renders CSV upload tab and sample download link", %{conn: conn} do + {:ok, lv, _html} = live(conn, ~p"/submit") + + html = + lv + |> element("[phx-value-tab=csv]") + |> render_click() + + assert html =~ "Upload CSV" + assert html =~ "/downloads/sample_contacts.csv" + assert html =~ "Download sample CSV" + end + + test "uploads valid CSV and shows import count", %{conn: conn} do + {:ok, lv, _html} = live(conn, ~p"/submit") + + lv + |> element("[phx-value-tab=csv]") + |> render_click() + + csv_content = + "station1,station2,grid1,grid2,band,mode,qso_timestamp\nW5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z\n" + + csv_input = + file_input(lv, "#csv-upload-form", :csv, [ + %{name: "contacts.csv", content: csv_content, type: "text/csv"} + ]) + + render_upload(csv_input, "contacts.csv") + + html = + lv + |> form("#csv-upload-form", %{submitter_email: "test@example.com"}) + |> render_submit() + + assert html =~ "1 contact imported" + end + + test "shows errors for invalid rows alongside successful imports", %{conn: conn} do + {:ok, lv, _html} = live(conn, ~p"/submit") + + lv + |> element("[phx-value-tab=csv]") + |> render_click() + + csv_content = + "station1,station2,grid1,grid2,band,mode,qso_timestamp\n" <> + "W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z\n" <> + ",,,,,,\n" + + csv_input = + file_input(lv, "#csv-upload-form", :csv, [ + %{name: "contacts.csv", content: csv_content, type: "text/csv"} + ]) + + render_upload(csv_input, "contacts.csv") + + html = + lv + |> form("#csv-upload-form", %{submitter_email: "test@example.com"}) + |> render_submit() + + assert html =~ "1 contact imported" + assert html =~ "1 row had errors" + assert html =~ "Row 3" + end + + test "requires email for CSV upload", %{conn: conn} do + {:ok, lv, _html} = live(conn, ~p"/submit") + + lv + |> element("[phx-value-tab=csv]") + |> render_click() + + csv_content = + "station1,station2,grid1,grid2,band,mode,qso_timestamp\nW5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z\n" + + csv_input = + file_input(lv, "#csv-upload-form", :csv, [ + %{name: "contacts.csv", content: csv_content, type: "text/csv"} + ]) + + render_upload(csv_input, "contacts.csv") + + html = + lv + |> form("#csv-upload-form", %{submitter_email: ""}) + |> render_submit() + + assert html =~ "Email is required for CSV upload" + end + + test "handles CSV with no data rows", %{conn: conn} do + {:ok, lv, _html} = live(conn, ~p"/submit") + + lv + |> element("[phx-value-tab=csv]") + |> render_click() + + csv_content = "station1,station2,grid1,grid2,band,mode,qso_timestamp\n" + + csv_input = + file_input(lv, "#csv-upload-form", :csv, [ + %{name: "header_only.csv", content: csv_content, type: "text/csv"} + ]) + + render_upload(csv_input, "header_only.csv") + + html = + lv + |> form("#csv-upload-form", %{submitter_email: "test@example.com"}) + |> render_submit() + + assert html =~ "CSV file has no data rows" + end + end + describe "save" do test "creates QSO and redirects on valid submit", %{conn: conn} do {:ok, lv, _html} = live(conn, ~p"/submit")