From be6f1a1d2819f4aa5d5eb519652b7592ae4ab3cc Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 16 Apr 2026 09:53:14 -0500 Subject: [PATCH] CsvImport: header-driven column mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /contacts LiveTable export produces a 12-column CSV with label-style headers (Station 1 / Grid 1 / QSO (UTC) / ...) in a different column order plus extras (id, distance_km, hrrr_status, flagged_invalid, inserted_at). The old positional import rejected it with "expected 6 or 7 columns, got 12". Replace the positional parser with header-driven mapping: - Parse the header row, normalize each cell (downcase + trim), and look it up in an alias map that covers both the raw field names and the export labels. - Required columns: station1, station2, grid1, grid2, band, qso_timestamp (mode stays optional). If any are missing the header, return {:error, {:missing_required_columns, [...]}}. - Unknown columns are silently ignored so the export's extras don't break round-trip. - Per-row validation stays the same (changeset errors still reported by row number). Existing 6- or 7-column positional CSVs that already have a header still work — the header parser recognizes `station1`, `station2`, etc. directly. --- lib/microwaveprop/radio/csv_import.ex | 112 ++++++++++++++----- test/microwaveprop/radio/csv_import_test.exs | 68 +++++++++++ 2 files changed, 153 insertions(+), 27 deletions(-) diff --git a/lib/microwaveprop/radio/csv_import.ex b/lib/microwaveprop/radio/csv_import.ex index 124f6494..73e963b3 100644 --- a/lib/microwaveprop/radio/csv_import.ex +++ b/lib/microwaveprop/radio/csv_import.ex @@ -30,8 +30,27 @@ defmodule Microwaveprop.Radio.CsvImport do @dedup_window_seconds 3600 - @columns_with_mode ~w(station1 station2 grid1 grid2 band mode qso_timestamp) - @columns_without_mode ~w(station1 station2 grid1 grid2 band qso_timestamp) + # Header parsing is case-insensitive and whitespace-insensitive. Accepts + # both the raw field names (station1, grid1, ...) and the human labels the + # /contacts live_table export produces (Station 1, Grid 1, QSO (UTC), ...) + # so export → import round-trips. Unknown columns are ignored. + @header_aliases %{ + "station1" => "station1", + "station 1" => "station1", + "station2" => "station2", + "station 2" => "station2", + "grid1" => "grid1", + "grid 1" => "grid1", + "grid2" => "grid2", + "grid 2" => "grid2", + "band" => "band", + "mode" => "mode", + "qso_timestamp" => "qso_timestamp", + "qso timestamp" => "qso_timestamp", + "qso (utc)" => "qso_timestamp" + } + + @required_columns ~w(station1 station2 grid1 grid2 band qso_timestamp) @doc """ Parse-and-insert in one shot. Kept for backward compatibility — does NOT @@ -56,7 +75,10 @@ defmodule Microwaveprop.Radio.CsvImport do } @spec import(String.t(), String.t()) :: - {:ok, import_result()} | {:error, :empty_csv} | {:error, :no_data_rows} + {:ok, import_result()} + | {:error, :empty_csv} + | {:error, :no_data_rows} + | {:error, {:missing_required_columns, [String.t()]}} def import(csv_string, submitter_email) do with {:ok, rows} <- parse_csv(csv_string, submitter_email) do {valid, invalid} = split_parsed(rows) @@ -88,7 +110,10 @@ defmodule Microwaveprop.Radio.CsvImport do (`:earlier_in_upload`). """ @spec preview(String.t(), String.t()) :: - {:ok, preview_result()} | {:error, :empty_csv} | {:error, :no_data_rows} + {:ok, preview_result()} + | {:error, :empty_csv} + | {:error, :no_data_rows} + | {:error, {:missing_required_columns, [String.t()]}} def preview(csv_string, submitter_email) do with {:ok, rows} <- parse_csv(csv_string, submitter_email) do {valid, invalid} = split_parsed(rows) @@ -146,39 +171,72 @@ defmodule Microwaveprop.Radio.CsvImport do [{_header, _}] -> {:error, :no_data_rows} - [{_header, _} | data_lines] -> - rows = Enum.map(data_lines, fn {line, row_num} -> parse_row(line, row_num, submitter_email) end) - {:ok, rows} + [{header_line, _} | data_lines] -> + with {:ok, column_map} <- parse_header(header_line) do + {:ok, parse_data_rows(data_lines, submitter_email, column_map)} + end end end + defp parse_data_rows(data_lines, submitter_email, column_map) do + Enum.map(data_lines, fn {line, row_num} -> + parse_row(line, row_num, submitter_email, column_map) + end) + end + defp blank?(line), do: String.trim(line) == "" - defp parse_row(line, row_num, submitter_email) do - fields = parse_csv_fields(line) + # Builds an index → field-name map from the header line. Unknown headers + # are silently skipped (that's how we accept the richer /contacts export). + defp parse_header(header_line) do + column_map = + header_line + |> parse_csv_fields() + |> Enum.with_index() + |> Enum.reduce(%{}, fn {raw, index}, acc -> + case Map.fetch(@header_aliases, raw |> String.downcase() |> String.trim()) do + {:ok, field} -> Map.put(acc, index, field) + :error -> acc + end + end) - case columns_for(length(fields)) do - nil -> - {:invalid, - %{ - row_num: row_num, - messages: ["expected 6 columns (no mode) or 7 columns (with mode), got #{length(fields)}"] - }} + present = column_map |> Map.values() |> MapSet.new() + missing = Enum.reject(@required_columns, &MapSet.member?(present, &1)) - columns -> - attrs = - columns - |> Enum.zip(fields) - |> Map.new() - |> Map.put("submitter_email", submitter_email) - - parse_row_with_timestamp(attrs, row_num) + if missing == [] do + {:ok, column_map} + else + {:error, {:missing_required_columns, missing}} end end - defp columns_for(7), do: @columns_with_mode - defp columns_for(6), do: @columns_without_mode - defp columns_for(_), do: nil + defp parse_row(line, row_num, submitter_email, column_map) do + fields = parse_csv_fields(line) + + attrs = + column_map + |> Enum.reduce(%{}, fn {index, field}, acc -> + case Enum.at(fields, index) do + nil -> acc + value -> Map.put(acc, field, value) + end + end) + |> Map.put("submitter_email", submitter_email) + + if required_present?(attrs) do + parse_row_with_timestamp(attrs, row_num) + else + missing_on_row = Enum.reject(@required_columns, &Map.has_key?(attrs, &1)) + + {:invalid, + %{ + row_num: row_num, + messages: ["row is missing required columns: #{Enum.join(missing_on_row, ", ")}"] + }} + end + end + + defp required_present?(attrs), do: Enum.all?(@required_columns, &Map.has_key?(attrs, &1)) defp parse_row_with_timestamp(attrs, row_num) do case normalize_timestamp(attrs["qso_timestamp"]) do diff --git a/test/microwaveprop/radio/csv_import_test.exs b/test/microwaveprop/radio/csv_import_test.exs index 9ed7bbe0..9634ba48 100644 --- a/test/microwaveprop/radio/csv_import_test.exs +++ b/test/microwaveprop/radio/csv_import_test.exs @@ -239,6 +239,74 @@ defmodule Microwaveprop.Radio.CsvImportTest do end end + describe "import/2 with header-driven column mapping" do + # The /contacts export produces a rich CSV with label-style headers and + # extra columns (id, distance_km, status fields, inserted_at). The import + # must read the header row, match by field name / label, and ignore + # anything it doesn't recognize so export → import round-trips. + + test "accepts /contacts export format (labels + extra columns + different order)" do + # Mirrors the live_table export: 12 columns, labels like "Station 1", + # grids interleaved (station1, grid1, station2, grid2…), and extra + # columns the importer should ignore. + csv = """ + ID,Station 1,Grid 1,Station 2,Grid 2,Band,Mode,Distance (km),Enriched,Flag,QSO (UTC),Added (UTC) + abc-123,W5XD,EM12,K5TR,EM00,10000,CW,295,Complete,,2026-03-28 18:00,2026-03-28 18:05 + """ + + assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter) + assert contact.station1 == "W5XD" + assert contact.station2 == "K5TR" + assert contact.grid1 == "EM12" + assert contact.grid2 == "EM00" + assert contact.mode == "CW" + end + + test "accepts arbitrary column order" do + csv = """ + qso_timestamp,band,mode,grid2,grid1,station2,station1 + 2026-03-28T18:00:00Z,10000,CW,EM00,EM12,K5TR,W5XD + """ + + assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter) + assert contact.station1 == "W5XD" + assert contact.grid1 == "EM12" + end + + test "is case-insensitive on header names" do + csv = """ + STATION1,STATION2,GRID1,GRID2,BAND,MODE,QSO_TIMESTAMP + W5XD,K5TR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z + """ + + assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter) + assert contact.station1 == "W5XD" + end + + test "ignores unknown columns" do + csv = """ + station1,notes,station2,grid1,ignored_col,grid2,band,mode,qso_timestamp + W5XD,some note,K5TR,EM12,xyz,EM00,10000,CW,2026-03-28T18:00:00Z + """ + + assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter) + assert contact.station1 == "W5XD" + end + + test "rejects when a required column header is missing" do + # No qso_timestamp column at all. + csv = """ + station1,station2,grid1,grid2,band,mode + W5XD,K5TR,EM12,EM00,10000,CW + """ + + assert {:error, {:missing_required_columns, missing}} = + CsvImport.import(csv, @submitter) + + assert "qso_timestamp" in missing + end + end + describe "normalize_timestamp/1" do test "parses ISO 8601 with Z" do assert {:ok, "2024-06-15T14:30:00Z"} = CsvImport.normalize_timestamp("2024-06-15T14:30:00Z")