diff --git a/lib/microwaveprop/radio/adif_import.ex b/lib/microwaveprop/radio/adif_import.ex index dbba9e87..027b6e54 100644 --- a/lib/microwaveprop/radio/adif_import.ex +++ b/lib/microwaveprop/radio/adif_import.ex @@ -12,6 +12,7 @@ defmodule Microwaveprop.Radio.AdifImport do alias Microwaveprop.Radio.BandResolver alias Microwaveprop.Radio.Contact alias Microwaveprop.Radio.ImportMatcher + alias Microwaveprop.Radio.TextSanitizer alias Microwaveprop.Repo @dedup_window_seconds 3600 @@ -161,7 +162,7 @@ defmodule Microwaveprop.Radio.AdifImport do # Value starts right after the closing > of the tag. value_offset = tag_start + tag_len - value = :binary.part(record_string, value_offset, size) + value = record_string |> :binary.part(value_offset, size) |> TextSanitizer.sanitize() # Standard ADIF fields take precedence over app-defined ones if raw_name != name and Map.has_key?(acc, name) do diff --git a/lib/microwaveprop/radio/csv_import.ex b/lib/microwaveprop/radio/csv_import.ex index ccc04828..beb0bd11 100644 --- a/lib/microwaveprop/radio/csv_import.ex +++ b/lib/microwaveprop/radio/csv_import.ex @@ -27,6 +27,7 @@ defmodule Microwaveprop.Radio.CsvImport do alias Microwaveprop.Radio.Contact alias Microwaveprop.Radio.ImportMatcher alias Microwaveprop.Radio.ImportRun + alias Microwaveprop.Radio.TextSanitizer alias Microwaveprop.Repo alias Microwaveprop.Workers.ContactImportWorker alias Microwaveprop.Workers.ContactWeatherEnqueueWorker @@ -125,7 +126,7 @@ defmodule Microwaveprop.Radio.CsvImport do | {:error, :no_data_rows} | {:error, {:missing_required_columns, [String.t()]}} def preview(csv_string, submitter_email) do - with {:ok, rows} <- parse_csv(csv_string, submitter_email) do + with {:ok, rows} <- parse_csv(TextSanitizer.sanitize(csv_string), submitter_email) do {valid, invalid} = split_parsed(rows) existing = load_existing_for_dedup(valid) {unique, duplicates, refinements} = dedupe(valid, existing) diff --git a/lib/microwaveprop/radio/text_sanitizer.ex b/lib/microwaveprop/radio/text_sanitizer.ex new file mode 100644 index 00000000..2cb4b1a9 --- /dev/null +++ b/lib/microwaveprop/radio/text_sanitizer.ex @@ -0,0 +1,52 @@ +defmodule Microwaveprop.Radio.TextSanitizer do + @moduledoc """ + Strip non-printable characters from user-uploaded log text. Removes BOM, + zero-width spaces/joiners, C0 control bytes (except CR/LF/TAB), and DEL; + collapses non-breaking and other unicode whitespace into a regular space. + + Used by CSV and ADIF importers because field values frequently arrive + with stray NBSPs from website copy-paste, BOM markers from Windows + exports, etc. + """ + + @spec sanitize(nil) :: nil + @spec sanitize(String.t()) :: String.t() + def sanitize(nil), do: nil + def sanitize(""), do: "" + + def sanitize(string) when is_binary(string) do + string + |> :unicode.characters_to_list() + |> Enum.reduce([], fn cp, acc -> + case sanitize_codepoint(cp) do + :strip -> acc + replacement -> [replacement | acc] + end + end) + |> Enum.reverse() + |> :unicode.characters_to_binary() + end + + defp sanitize_codepoint(?\t), do: ?\t + defp sanitize_codepoint(?\n), do: ?\n + defp sanitize_codepoint(?\r), do: ?\r + + # C0 control characters and DEL — strip. + defp sanitize_codepoint(cp) when cp < 0x20 or cp == 0x7F, do: :strip + + # NBSP and other unicode whitespace → regular space. + defp sanitize_codepoint(0x00A0), do: ?\s + defp sanitize_codepoint(cp) when cp >= 0x2000 and cp <= 0x200A, do: ?\s + defp sanitize_codepoint(0x202F), do: ?\s + defp sanitize_codepoint(0x205F), do: ?\s + defp sanitize_codepoint(0x3000), do: ?\s + + # Zero-width characters (ZWSP, ZWNJ, ZWJ, word-joiner, BOM) — strip. + defp sanitize_codepoint(0x200B), do: :strip + defp sanitize_codepoint(0x200C), do: :strip + defp sanitize_codepoint(0x200D), do: :strip + defp sanitize_codepoint(0x2060), do: :strip + defp sanitize_codepoint(0xFEFF), do: :strip + + defp sanitize_codepoint(cp), do: cp +end diff --git a/test/microwaveprop/radio/adif_import_test.exs b/test/microwaveprop/radio/adif_import_test.exs index 2936ed4c..55b4471e 100644 --- a/test/microwaveprop/radio/adif_import_test.exs +++ b/test/microwaveprop/radio/adif_import_test.exs @@ -66,6 +66,24 @@ defmodule Microwaveprop.Radio.AdifImportTest do assert row.attrs["user_declared_prop_mode"] == nil end + test "strips non-printables (zero-width, BOM, control) from ADIF field values" do + # Zero-width space (U+200B = 0xE2 0x80 0x8B, 3 bytes) inside CALL and + # GRIDSQUARE. ADIF length tags count BYTES so the tag widths include + # the multi-byte payload — sanitization strips the zero-width chars + # and leaves the underlying ASCII intact. + adif = + "W5XD" <> + "K5\u200BTR" <> + "EM\u200B00" <> + "EM12" <> + "10368.000CW20260328180000\n" + + assert {:ok, preview} = AdifImport.preview(adif, @submitter) + assert [row] = preview.valid + assert row.attrs["station2"] == "K5TR" + assert row.attrs["grid2"] == "EM00" + end + test "captures ADIF NOTES into notes" do adif = """ W5XDK5TREM00EM1210368.000CW20260328180000Rare Es along TX/OK front diff --git a/test/microwaveprop/radio/csv_import_test.exs b/test/microwaveprop/radio/csv_import_test.exs index cdb379f2..fb79ac9f 100644 --- a/test/microwaveprop/radio/csv_import_test.exs +++ b/test/microwaveprop/radio/csv_import_test.exs @@ -104,6 +104,18 @@ defmodule Microwaveprop.Radio.CsvImportTest do assert contact.station1 == "W5XD" end + test "strips non-printables (BOM, NBSP, zero-width) from uploaded CSV" do + # \uFEFF BOM at file start, NBSP between fields, zero-width space inside callsign. + csv = + "\uFEFF" <> + @valid_header <> + "\nW5XD,K5\u200BTR,EM12,EM00,10000,CW,2026-03-28T18:00:00Z\u00A0" + + assert {:ok, %{imported: [contact], errors: []}} = CsvImport.import(csv, @submitter) + assert contact.station1 == "W5XD" + assert contact.station2 == "K5TR" + end + test "skips blank lines between data rows" do csv = Enum.join( diff --git a/test/microwaveprop/radio/text_sanitizer_test.exs b/test/microwaveprop/radio/text_sanitizer_test.exs new file mode 100644 index 00000000..1698fc74 --- /dev/null +++ b/test/microwaveprop/radio/text_sanitizer_test.exs @@ -0,0 +1,56 @@ +defmodule Microwaveprop.Radio.TextSanitizerTest do + use ExUnit.Case, async: true + + alias Microwaveprop.Radio.TextSanitizer + + describe "sanitize/1" do + test "passes through plain ASCII unchanged" do + assert TextSanitizer.sanitize("K5ABC EM12 1296") == "K5ABC EM12 1296" + end + + test "preserves CR, LF, and TAB" do + assert TextSanitizer.sanitize("a\nb\r\nc\td") == "a\nb\r\nc\td" + end + + test "replaces NBSP (U+00A0) with regular space" do + assert TextSanitizer.sanitize("K5ABC\u00A0EM12") == "K5ABC EM12" + end + + test "replaces narrow no-break space (U+202F) and figure space (U+2007) with space" do + assert TextSanitizer.sanitize("a\u202Fb\u2007c") == "a b c" + end + + test "replaces all unicode general-punctuation spaces (U+2000–U+200A) with space" do + assert TextSanitizer.sanitize("a\u2000b\u2003c\u200Ad") == "a b c d" + end + + test "strips BOM (U+FEFF) anywhere in the string" do + assert TextSanitizer.sanitize("\uFEFFK5ABC") == "K5ABC" + assert TextSanitizer.sanitize("K5\uFEFFABC") == "K5ABC" + end + + test "strips zero-width joiners and non-joiners" do + assert TextSanitizer.sanitize("K5\u200BABC\u200C\u200D") == "K5ABC" + end + + test "strips C0 control characters except TAB/CR/LF" do + assert TextSanitizer.sanitize("a\x00b\x07c\x1Bd") == "abcd" + end + + test "strips DEL (U+007F)" do + assert TextSanitizer.sanitize("a\x7Fb") == "ab" + end + + test "handles nil" do + assert TextSanitizer.sanitize(nil) == nil + end + + test "handles empty string" do + assert TextSanitizer.sanitize("") == "" + end + + test "preserves UTF-8 letters with diacritics" do + assert TextSanitizer.sanitize("café Müller") == "café Müller" + end + end +end