defmodule Microwaveprop.Radio.CsvImport do @moduledoc false alias Microwaveprop.Radio alias Microwaveprop.Workers.ContactWeatherEnqueueWorker @expected_columns 7 @columns ~w(station1 station2 grid1 grid2 band mode qso_timestamp) @doc """ Imports contacts from a CSV string. The first line is treated as a header. Each valid row is inserted via `Radio.create_contact/1` and enrichment is enqueued. The submitter_email is added to each row's attributes. Returns: - `{:ok, %{imported: [%Contact{}, ...], errors: [{row_num, [error_string]}]}}` - `{:error, :empty_csv}` if the input is empty or whitespace-only - `{:error, :no_data_rows}` if there are no data rows after the header """ def import(csv_string, submitter_email) do numbered_lines = csv_string |> String.replace("\r\n", "\n") |> String.split("\n") |> Enum.with_index(1) non_blank = Enum.reject(numbered_lines, fn {line, _num} -> blank?(line) end) case non_blank do [] -> {:error, :empty_csv} [{_header, _}] -> {:error, :no_data_rows} [{_header, _} | data_lines] -> import_data_lines(data_lines, submitter_email) end end defp blank?(line), do: String.trim(line) == "" defp import_data_lines(data_lines, submitter_email) do result = Enum.reduce(data_lines, %{imported: [], errors: []}, fn {line, row_num}, acc -> case parse_and_import_row(line, submitter_email) do {:ok, contact} -> %{acc | imported: [contact | acc.imported]} {:error, error_messages} -> %{acc | errors: [{row_num, error_messages} | acc.errors]} end end) {:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}} end defp parse_and_import_row(line, submitter_email) do fields = parse_csv_fields(line) if length(fields) == @expected_columns do attrs = @columns |> Enum.zip(fields) |> Map.new() |> Map.put("submitter_email", submitter_email) case normalize_timestamp(attrs["qso_timestamp"]) do {:ok, iso_timestamp} -> attrs = Map.put(attrs, "qso_timestamp", iso_timestamp) case Radio.create_contact(attrs) do {:ok, contact} -> ContactWeatherEnqueueWorker.enqueue_for_contact(contact) {:ok, contact} {:error, changeset} -> {:error, changeset_error_strings(changeset)} end {:error, msg} -> {:error, [msg]} end else {:error, ["expected #{@expected_columns} columns, got #{length(fields)}"]} end end # Regex patterns for timestamp parsing, ordered most-specific first. # Accepts many hand-entered formats. All times assumed UTC. # Each returns {year, month, day, hour, minute, second} or nil. defp timestamp_parsers do [ # ISO-ish: 2024-06-15T14:30:00Z, 2024-06-15 14:30, 2024/06/15 14:30:00, etc. # Accepts T or space separator, optional seconds, optional timezone suffix {~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})[T\s]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$"i, fn [_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} [_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"} end}, # Date only: 2024-06-15, 2024/06/15 {~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$", fn [_, y, m, d] -> {y, m, d, "00", "00", "00"} end}, # US with AM/PM: 6/15/2024 2:30 PM, 6-15-2024 2:30PM, 06.15.2024 02:30 am {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM)$"i, fn [_, m, d, y, h, mi, s, ampm] when byte_size(s) > 0 -> {y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, s} [_, m, d, y, h, mi, _, ampm] -> {y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, "00"} end}, # US 24h: 6/15/2024 14:30, 06-15-2024 14:30:00, 6.15.2024 14:30 {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?$", fn [_, m, d, y, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} [_, m, d, y, h, mi | _] -> {y, m, d, h, mi, "00"} end}, # US date only: 6/15/2024, 06-15-2024 {~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})$", fn [_, m, d, y] -> {y, m, d, "00", "00", "00"} end}, # Compact: 20240615T143000, 20240615 1430, 20240615T1430 {~r"^(\d{4})(\d{2})(\d{2})[T\s]?(\d{2})(\d{2})(\d{2})?$", fn [_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s} [_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"} end} ] end @doc """ Parses various date/time formats into ISO 8601 UTC strings. All times are assumed UTC. """ def normalize_timestamp(raw) do trimmed = String.trim(raw) # Try ISO 8601 with Elixir's built-in parser first case DateTime.from_iso8601(trimmed) do {:ok, dt, _} -> {:ok, DateTime.to_iso8601(dt)} _ -> try_regex_parsers(trimmed) end end defp try_regex_parsers(trimmed) do timestamp_parsers() |> Enum.find_value(fn {regex, extractor} -> case Regex.run(regex, trimmed) do nil -> nil match -> format_iso(extractor.(match)) end end) |> case do {:ok, _} = ok -> ok nil -> {:error, "qso_timestamp could not be parsed: #{trimmed}"} end end defp parse_12h(12, "AM"), do: 0 defp parse_12h(12, "PM"), do: 12 defp parse_12h(h, "PM"), do: h + 12 defp parse_12h(h, "AM"), do: h defp format_iso({y, m, d, h, mi, s}) do y = String.pad_leading(to_string(y), 4, "0") m = String.pad_leading(to_string(m), 2, "0") d = String.pad_leading(to_string(d), 2, "0") h = String.pad_leading(to_string(h), 2, "0") mi = String.pad_leading(to_string(mi), 2, "0") s = String.pad_leading(to_string(s), 2, "0") {:ok, "#{y}-#{m}-#{d}T#{h}:#{mi}:#{s}Z"} end # RFC 4180 CSV field parser. Handles quoted fields with commas and escaped quotes. defp parse_csv_fields(line) do line |> String.trim() |> do_parse_fields([], "") |> Enum.reverse() |> Enum.map(&String.trim/1) end defp do_parse_fields("", acc, current), do: [current | acc] defp do_parse_fields(<<"\"", rest::binary>>, acc, "") do parse_quoted_field(rest, acc, "") end defp do_parse_fields(<<",", rest::binary>>, acc, current) do do_parse_fields(rest, [current | acc], "") end defp do_parse_fields(<>, acc, current) do do_parse_fields(rest, acc, current <> <>) end defp parse_quoted_field(<<"\"\"", rest::binary>>, acc, current) do parse_quoted_field(rest, acc, current <> "\"") end defp parse_quoted_field(<<"\"", rest::binary>>, acc, current) do # End of quoted field — skip to next comma or end case rest do <<",", rest2::binary>> -> do_parse_fields(rest2, [current | acc], "") "" -> [current | acc] _ -> do_parse_fields(rest, [current | acc], "") end end defp parse_quoted_field(<>, acc, current) do parse_quoted_field(rest, acc, current <> <>) end defp parse_quoted_field("", acc, current), do: [current | acc] defp changeset_error_strings(changeset) do changeset |> Ecto.Changeset.traverse_errors(fn {message, opts} -> Regex.replace(~r"%{(\w+)}", message, fn _, key -> opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string() end) end) |> Enum.flat_map(fn {field, messages} -> Enum.map(messages, &"#{field} #{&1}") end) end end