228 lines
7.5 KiB
Elixir
228 lines
7.5 KiB
Elixir
defmodule Microwaveprop.Radio.CsvImport do
|
|
@moduledoc false
|
|
|
|
alias Microwaveprop.Radio
|
|
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
|
|
|
@expected_columns 7
|
|
@columns ~w(station1 station2 grid1 grid2 band mode qso_timestamp)
|
|
|
|
@doc """
|
|
Imports contacts from a CSV string. The first line is treated as a header.
|
|
Each valid row is inserted via `Radio.create_contact/1` and enrichment is enqueued.
|
|
The submitter_email is added to each row's attributes.
|
|
|
|
Returns:
|
|
- `{:ok, %{imported: [%Contact{}, ...], errors: [{row_num, [error_string]}]}}`
|
|
- `{:error, :empty_csv}` if the input is empty or whitespace-only
|
|
- `{:error, :no_data_rows}` if there are no data rows after the header
|
|
"""
|
|
def import(csv_string, submitter_email) do
|
|
numbered_lines =
|
|
csv_string
|
|
|> String.replace("\r\n", "\n")
|
|
|> String.split("\n")
|
|
|> Enum.with_index(1)
|
|
|
|
non_blank = Enum.reject(numbered_lines, fn {line, _num} -> blank?(line) end)
|
|
|
|
case non_blank do
|
|
[] ->
|
|
{:error, :empty_csv}
|
|
|
|
[{_header, _}] ->
|
|
{:error, :no_data_rows}
|
|
|
|
[{_header, _} | data_lines] ->
|
|
import_data_lines(data_lines, submitter_email)
|
|
end
|
|
end
|
|
|
|
defp blank?(line), do: String.trim(line) == ""
|
|
|
|
defp import_data_lines(data_lines, submitter_email) do
|
|
result =
|
|
Enum.reduce(data_lines, %{imported: [], errors: []}, fn {line, row_num}, acc ->
|
|
case parse_and_import_row(line, submitter_email) do
|
|
{:ok, contact} ->
|
|
%{acc | imported: [contact | acc.imported]}
|
|
|
|
{:error, error_messages} ->
|
|
%{acc | errors: [{row_num, error_messages} | acc.errors]}
|
|
end
|
|
end)
|
|
|
|
{:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}}
|
|
end
|
|
|
|
defp parse_and_import_row(line, submitter_email) do
|
|
fields = parse_csv_fields(line)
|
|
|
|
if length(fields) == @expected_columns do
|
|
attrs =
|
|
@columns
|
|
|> Enum.zip(fields)
|
|
|> Map.new()
|
|
|> Map.put("submitter_email", submitter_email)
|
|
|
|
case normalize_timestamp(attrs["qso_timestamp"]) do
|
|
{:ok, iso_timestamp} ->
|
|
attrs = Map.put(attrs, "qso_timestamp", iso_timestamp)
|
|
|
|
case Radio.create_contact(attrs) do
|
|
{:ok, contact} ->
|
|
ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
|
|
{:ok, contact}
|
|
|
|
{:error, changeset} ->
|
|
{:error, changeset_error_strings(changeset)}
|
|
end
|
|
|
|
{:error, msg} ->
|
|
{:error, [msg]}
|
|
end
|
|
else
|
|
{:error, ["expected #{@expected_columns} columns, got #{length(fields)}"]}
|
|
end
|
|
end
|
|
|
|
# Regex patterns for timestamp parsing, ordered most-specific first.
|
|
# Accepts many hand-entered formats. All times assumed UTC.
|
|
# Each returns {year, month, day, hour, minute, second} or nil.
|
|
defp timestamp_parsers do
|
|
[
|
|
# ISO-ish: 2024-06-15T14:30:00Z, 2024-06-15 14:30, 2024/06/15 14:30:00, etc.
|
|
# Accepts T or space separator, optional seconds, optional timezone suffix
|
|
{~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})[T\s]+(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?$"i,
|
|
fn
|
|
[_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
|
|
[_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"}
|
|
end},
|
|
# Date only: 2024-06-15, 2024/06/15
|
|
{~r"^(\d{4})[-/](\d{1,2})[-/](\d{1,2})$", fn [_, y, m, d] -> {y, m, d, "00", "00", "00"} end},
|
|
# US with AM/PM: 6/15/2024 2:30 PM, 6-15-2024 2:30PM, 06.15.2024 02:30 am
|
|
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?\s*(AM|PM)$"i,
|
|
fn
|
|
[_, m, d, y, h, mi, s, ampm] when byte_size(s) > 0 ->
|
|
{y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, s}
|
|
|
|
[_, m, d, y, h, mi, _, ampm] ->
|
|
{y, m, d, to_string(parse_12h(String.to_integer(h), String.upcase(ampm))), mi, "00"}
|
|
end},
|
|
# US 24h: 6/15/2024 14:30, 06-15-2024 14:30:00, 6.15.2024 14:30
|
|
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})\s+(\d{1,2}):(\d{2})(?::(\d{2}))?$",
|
|
fn
|
|
[_, m, d, y, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
|
|
[_, m, d, y, h, mi | _] -> {y, m, d, h, mi, "00"}
|
|
end},
|
|
# US date only: 6/15/2024, 06-15-2024
|
|
{~r"^(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})$", fn [_, m, d, y] -> {y, m, d, "00", "00", "00"} end},
|
|
# Compact: 20240615T143000, 20240615 1430, 20240615T1430
|
|
{~r"^(\d{4})(\d{2})(\d{2})[T\s]?(\d{2})(\d{2})(\d{2})?$",
|
|
fn
|
|
[_, y, m, d, h, mi, s] when byte_size(s) > 0 -> {y, m, d, h, mi, s}
|
|
[_, y, m, d, h, mi | _] -> {y, m, d, h, mi, "00"}
|
|
end}
|
|
]
|
|
end
|
|
|
|
@doc """
|
|
Parses various date/time formats into ISO 8601 UTC strings.
|
|
All times are assumed UTC.
|
|
"""
|
|
def normalize_timestamp(raw) do
|
|
trimmed = String.trim(raw)
|
|
|
|
# Try ISO 8601 with Elixir's built-in parser first
|
|
case DateTime.from_iso8601(trimmed) do
|
|
{:ok, dt, _} ->
|
|
{:ok, DateTime.to_iso8601(dt)}
|
|
|
|
_ ->
|
|
try_regex_parsers(trimmed)
|
|
end
|
|
end
|
|
|
|
defp try_regex_parsers(trimmed) do
|
|
timestamp_parsers()
|
|
|> Enum.find_value(fn {regex, extractor} ->
|
|
case Regex.run(regex, trimmed) do
|
|
nil -> nil
|
|
match -> format_iso(extractor.(match))
|
|
end
|
|
end)
|
|
|> case do
|
|
{:ok, _} = ok -> ok
|
|
nil -> {:error, "qso_timestamp could not be parsed: #{trimmed}"}
|
|
end
|
|
end
|
|
|
|
defp parse_12h(12, "AM"), do: 0
|
|
defp parse_12h(12, "PM"), do: 12
|
|
defp parse_12h(h, "PM"), do: h + 12
|
|
defp parse_12h(h, "AM"), do: h
|
|
|
|
defp format_iso({y, m, d, h, mi, s}) do
|
|
y = String.pad_leading(to_string(y), 4, "0")
|
|
m = String.pad_leading(to_string(m), 2, "0")
|
|
d = String.pad_leading(to_string(d), 2, "0")
|
|
h = String.pad_leading(to_string(h), 2, "0")
|
|
mi = String.pad_leading(to_string(mi), 2, "0")
|
|
s = String.pad_leading(to_string(s), 2, "0")
|
|
{:ok, "#{y}-#{m}-#{d}T#{h}:#{mi}:#{s}Z"}
|
|
end
|
|
|
|
# RFC 4180 CSV field parser. Handles quoted fields with commas and escaped quotes.
|
|
defp parse_csv_fields(line) do
|
|
line
|
|
|> String.trim()
|
|
|> do_parse_fields([], "")
|
|
|> Enum.reverse()
|
|
|> Enum.map(&String.trim/1)
|
|
end
|
|
|
|
defp do_parse_fields("", acc, current), do: [current | acc]
|
|
|
|
defp do_parse_fields(<<"\"", rest::binary>>, acc, "") do
|
|
parse_quoted_field(rest, acc, "")
|
|
end
|
|
|
|
defp do_parse_fields(<<",", rest::binary>>, acc, current) do
|
|
do_parse_fields(rest, [current | acc], "")
|
|
end
|
|
|
|
defp do_parse_fields(<<c, rest::binary>>, acc, current) do
|
|
do_parse_fields(rest, acc, current <> <<c>>)
|
|
end
|
|
|
|
defp parse_quoted_field(<<"\"\"", rest::binary>>, acc, current) do
|
|
parse_quoted_field(rest, acc, current <> "\"")
|
|
end
|
|
|
|
defp parse_quoted_field(<<"\"", rest::binary>>, acc, current) do
|
|
# End of quoted field — skip to next comma or end
|
|
case rest do
|
|
<<",", rest2::binary>> -> do_parse_fields(rest2, [current | acc], "")
|
|
"" -> [current | acc]
|
|
_ -> do_parse_fields(rest, [current | acc], "")
|
|
end
|
|
end
|
|
|
|
defp parse_quoted_field(<<c, rest::binary>>, acc, current) do
|
|
parse_quoted_field(rest, acc, current <> <<c>>)
|
|
end
|
|
|
|
defp parse_quoted_field("", acc, current), do: [current | acc]
|
|
|
|
defp changeset_error_strings(changeset) do
|
|
changeset
|
|
|> Ecto.Changeset.traverse_errors(fn {message, opts} ->
|
|
Regex.replace(~r"%{(\w+)}", message, fn _, key ->
|
|
opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
|
|
end)
|
|
end)
|
|
|> Enum.flat_map(fn {field, messages} ->
|
|
Enum.map(messages, &"#{field} #{&1}")
|
|
end)
|
|
end
|
|
end
|