prop/lib/microwaveprop/radio/csv_import.ex
Graham McIntire ec388e4bf5
Use Timex for flexible CSV timestamp parsing with tests
- Add timex ~> 3.7, downgrade gettext to ~> 0.26 for compatibility
- Parse ISO 8601, US dates (M/D/YYYY), AM/PM, 24h, compact formats
- 22 tests covering all accepted timestamp formats and edge cases
- Integration tests for CSV import with US date and space-separated formats
2026-04-01 15:39:02 -05:00

145 lines
4.2 KiB
Elixir

defmodule Microwaveprop.Radio.CsvImport do
@moduledoc false
alias Microwaveprop.Radio
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
@expected_columns 7
@columns ~w(station1 station2 grid1 grid2 band mode qso_timestamp)
@doc """
Imports contacts from a CSV string. The first line is treated as a header.
Each valid row is inserted via `Radio.create_contact/1` and enrichment is enqueued.
The submitter_email is added to each row's attributes.
Returns:
- `{:ok, %{imported: [%Contact{}, ...], errors: [{row_num, [error_string]}]}}`
- `{:error, :empty_csv}` if the input is empty or whitespace-only
- `{:error, :no_data_rows}` if there are no data rows after the header
"""
def import(csv_string, submitter_email) do
numbered_lines =
csv_string
|> String.replace("\r\n", "\n")
|> String.split("\n")
|> Enum.with_index(1)
non_blank = Enum.reject(numbered_lines, fn {line, _num} -> blank?(line) end)
case non_blank do
[] ->
{:error, :empty_csv}
[{_header, _}] ->
{:error, :no_data_rows}
[{_header, _} | data_lines] ->
import_data_lines(data_lines, submitter_email)
end
end
defp blank?(line), do: String.trim(line) == ""
defp import_data_lines(data_lines, submitter_email) do
result =
Enum.reduce(data_lines, %{imported: [], errors: []}, fn {line, row_num}, acc ->
case parse_and_import_row(line, submitter_email) do
{:ok, contact} ->
%{acc | imported: [contact | acc.imported]}
{:error, error_messages} ->
%{acc | errors: [{row_num, error_messages} | acc.errors]}
end
end)
{:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}}
end
defp parse_and_import_row(line, submitter_email) do
fields = line |> String.split(",") |> Enum.map(&String.trim/1)
if length(fields) == @expected_columns do
attrs =
@columns
|> Enum.zip(fields)
|> Map.new()
|> Map.put("submitter_email", submitter_email)
case normalize_timestamp(attrs["qso_timestamp"]) do
{:ok, iso_timestamp} ->
attrs = Map.put(attrs, "qso_timestamp", iso_timestamp)
case Radio.create_contact(attrs) do
{:ok, contact} ->
ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
{:ok, contact}
{:error, changeset} ->
{:error, changeset_error_strings(changeset)}
end
{:error, msg} ->
{:error, [msg]}
end
else
{:error, ["expected #{@expected_columns} columns, got #{length(fields)}"]}
end
end
@timestamp_formats [
"{ISO:Extended:Z}",
"{ISO:Extended}",
"{YYYY}-{0M}-{0D}T{h24}:{0m}:{0s}",
"{YYYY}-{0M}-{0D}T{h24}:{0m}",
"{YYYY}-{0M}-{0D} {h24}:{0m}:{0s}",
"{YYYY}-{0M}-{0D} {h24}:{0m}",
"{YYYY}-{0M}-{0D}",
"{M}/{D}/{YYYY} {h12}:{m} {AM}",
"{0M}/{0D}/{YYYY} {h12}:{0m} {AM}",
"{M}/{D}/{YYYY} {h24}:{m}:{s}",
"{M}/{D}/{YYYY} {h24}:{m}",
"{0M}/{0D}/{YYYY} {h24}:{0m}:{0s}",
"{0M}/{0D}/{YYYY} {h24}:{0m}",
"{M}-{D}-{YYYY} {h24}:{m}",
"{0M}-{0D}-{YYYY} {h24}:{0m}:{0s}",
"{0M}-{0D}-{YYYY} {h24}:{0m}",
"{YYYY}{0M}{0D}T{h24}{0m}{0s}",
"{YYYY}{0M}{0D} {h24}{0m}"
]
@doc """
Parses various date/time formats into ISO 8601 UTC strings using Timex.
All times are assumed UTC.
"""
def normalize_timestamp(raw) do
trimmed = String.trim(raw)
@timestamp_formats
|> Enum.find_value(fn fmt ->
case Timex.parse(trimmed, fmt) do
{:ok, parsed} ->
dt = Timex.to_datetime(parsed, "Etc/UTC")
{:ok, DateTime.to_iso8601(dt)}
_ ->
nil
end
end)
|> case do
{:ok, _} = ok -> ok
nil -> {:error, "qso_timestamp could not be parsed: #{trimmed}"}
end
end
defp changeset_error_strings(changeset) do
changeset
|> Ecto.Changeset.traverse_errors(fn {message, opts} ->
Regex.replace(~r"%{(\w+)}", message, fn _, key ->
opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
end)
end)
|> Enum.flat_map(fn {field, messages} ->
Enum.map(messages, &"#{field} #{&1}")
end)
end
end