Parse US dates (6/15/2024 2:30 PM), 24h (06/15/2024 14:30), dashes (06-15-2024 14:30), compact (20240615 1430), date-only (2024-06-15), and ISO 8601 — all normalized to UTC before insert.
167 lines
5.5 KiB
Elixir
167 lines
5.5 KiB
Elixir
defmodule Microwaveprop.Radio.CsvImport do
|
|
@moduledoc false
|
|
|
|
alias Microwaveprop.Radio
|
|
alias Microwaveprop.Workers.ContactWeatherEnqueueWorker
|
|
|
|
@expected_columns 7
|
|
@columns ~w(station1 station2 grid1 grid2 band mode qso_timestamp)
|
|
|
|
@doc """
|
|
Imports contacts from a CSV string. The first line is treated as a header.
|
|
Each valid row is inserted via `Radio.create_contact/1` and enrichment is enqueued.
|
|
The submitter_email is added to each row's attributes.
|
|
|
|
Returns:
|
|
- `{:ok, %{imported: [%Contact{}, ...], errors: [{row_num, [error_string]}]}}`
|
|
- `{:error, :empty_csv}` if the input is empty or whitespace-only
|
|
- `{:error, :no_data_rows}` if there are no data rows after the header
|
|
"""
|
|
def import(csv_string, submitter_email) do
|
|
numbered_lines =
|
|
csv_string
|
|
|> String.replace("\r\n", "\n")
|
|
|> String.split("\n")
|
|
|> Enum.with_index(1)
|
|
|
|
non_blank = Enum.reject(numbered_lines, fn {line, _num} -> blank?(line) end)
|
|
|
|
case non_blank do
|
|
[] ->
|
|
{:error, :empty_csv}
|
|
|
|
[{_header, _}] ->
|
|
{:error, :no_data_rows}
|
|
|
|
[{_header, _} | data_lines] ->
|
|
import_data_lines(data_lines, submitter_email)
|
|
end
|
|
end
|
|
|
|
defp blank?(line), do: String.trim(line) == ""
|
|
|
|
defp import_data_lines(data_lines, submitter_email) do
|
|
result =
|
|
Enum.reduce(data_lines, %{imported: [], errors: []}, fn {line, row_num}, acc ->
|
|
case parse_and_import_row(line, submitter_email) do
|
|
{:ok, contact} ->
|
|
%{acc | imported: [contact | acc.imported]}
|
|
|
|
{:error, error_messages} ->
|
|
%{acc | errors: [{row_num, error_messages} | acc.errors]}
|
|
end
|
|
end)
|
|
|
|
{:ok, %{imported: Enum.reverse(result.imported), errors: Enum.reverse(result.errors)}}
|
|
end
|
|
|
|
defp parse_and_import_row(line, submitter_email) do
|
|
fields = line |> String.split(",") |> Enum.map(&String.trim/1)
|
|
|
|
if length(fields) == @expected_columns do
|
|
attrs =
|
|
@columns
|
|
|> Enum.zip(fields)
|
|
|> Map.new()
|
|
|> Map.put("submitter_email", submitter_email)
|
|
|
|
case normalize_timestamp(attrs["qso_timestamp"]) do
|
|
{:ok, iso_timestamp} ->
|
|
attrs = Map.put(attrs, "qso_timestamp", iso_timestamp)
|
|
|
|
case Radio.create_contact(attrs) do
|
|
{:ok, contact} ->
|
|
ContactWeatherEnqueueWorker.enqueue_for_contact(contact)
|
|
{:ok, contact}
|
|
|
|
{:error, changeset} ->
|
|
{:error, changeset_error_strings(changeset)}
|
|
end
|
|
|
|
{:error, msg} ->
|
|
{:error, [msg]}
|
|
end
|
|
else
|
|
{:error, ["expected #{@expected_columns} columns, got #{length(fields)}"]}
|
|
end
|
|
end
|
|
|
|
@doc """
|
|
Parses various date/time formats into ISO 8601 UTC strings.
|
|
|
|
Accepted formats:
|
|
- ISO 8601: 2024-06-15T14:30:00Z, 2024-06-15 14:30:00Z, 2024-06-15T14:30:00
|
|
- US date: 06/15/2024 14:30, 6/15/2024 2:30 PM
|
|
- US date: 06-15-2024 14:30
|
|
- EU date: 15/06/2024 14:30
|
|
- Date only: 2024-06-15 (assumes 00:00 UTC)
|
|
- Compact: 20240615 1430, 20240615T143000
|
|
"""
|
|
def normalize_timestamp(raw) do
|
|
trimmed = String.trim(raw)
|
|
|
|
cond do
|
|
# Already ISO 8601 with timezone
|
|
match?({:ok, _, _}, DateTime.from_iso8601(trimmed)) ->
|
|
{:ok, trimmed}
|
|
|
|
# ISO-ish without Z: 2024-06-15T14:30:00 or 2024-06-15 14:30:00
|
|
Regex.match?(~r/^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2})?$/, trimmed) ->
|
|
{:ok, trimmed <> "Z"}
|
|
|
|
# Date only: 2024-06-15
|
|
Regex.match?(~r/^\d{4}-\d{2}-\d{2}$/, trimmed) ->
|
|
{:ok, trimmed <> "T00:00:00Z"}
|
|
|
|
# US format with AM/PM: 6/15/2024 2:30 PM or 06/15/2024 02:30 PM
|
|
match = Regex.run(~r"^(\d{1,2})/(\d{1,2})/(\d{4})\s+(\d{1,2}):(\d{2})\s*(AM|PM)$"i, trimmed) ->
|
|
[_, m, d, y, h, min, ampm] = match
|
|
hour = parse_12h(String.to_integer(h), String.upcase(ampm))
|
|
format_iso(y, m, d, hour, min)
|
|
|
|
# US format 24h: 6/15/2024 14:30 or 06/15/2024 14:30:00
|
|
match = Regex.run(~r"^(\d{1,2})/(\d{1,2})/(\d{4})\s+(\d{1,2}):(\d{2})(:\d{2})?$", trimmed) ->
|
|
[_, m, d, y, h, min | _] = match
|
|
format_iso(y, m, d, String.to_integer(h), min)
|
|
|
|
# US format with dashes: 06-15-2024 14:30
|
|
match = Regex.run(~r"^(\d{1,2})-(\d{1,2})-(\d{4})\s+(\d{1,2}):(\d{2})(:\d{2})?$", trimmed) ->
|
|
[_, m, d, y, h, min | _] = match
|
|
format_iso(y, m, d, String.to_integer(h), min)
|
|
|
|
# Compact: 20240615 1430 or 20240615T143000
|
|
match = Regex.run(~r"^(\d{4})(\d{2})(\d{2})[T ]?(\d{2})(\d{2})(\d{2})?$", trimmed) ->
|
|
[_, y, m, d, h, min | _] = match
|
|
format_iso(y, m, d, String.to_integer(h), min)
|
|
|
|
true ->
|
|
{:error, "qso_timestamp could not be parsed: #{trimmed}"}
|
|
end
|
|
end
|
|
|
|
defp parse_12h(12, "AM"), do: 0
|
|
defp parse_12h(12, "PM"), do: 12
|
|
defp parse_12h(h, "PM"), do: h + 12
|
|
defp parse_12h(h, "AM"), do: h
|
|
|
|
defp format_iso(y, m, d, hour, min) do
|
|
y = String.pad_leading(to_string(y), 4, "0")
|
|
m = String.pad_leading(to_string(m), 2, "0")
|
|
d = String.pad_leading(to_string(d), 2, "0")
|
|
h = String.pad_leading(to_string(hour), 2, "0")
|
|
min = String.pad_leading(to_string(min), 2, "0")
|
|
{:ok, "#{y}-#{m}-#{d}T#{h}:#{min}:00Z"}
|
|
end
|
|
|
|
defp changeset_error_strings(changeset) do
|
|
changeset
|
|
|> Ecto.Changeset.traverse_errors(fn {message, opts} ->
|
|
Regex.replace(~r"%{(\w+)}", message, fn _, key ->
|
|
opts |> Keyword.get(String.to_existing_atom(key), key) |> to_string()
|
|
end)
|
|
end)
|
|
|> Enum.flat_map(fn {field, messages} ->
|
|
Enum.map(messages, &"#{field} #{&1}")
|
|
end)
|
|
end
|
|
end
|