Graham McIntire
fe2ecf2618
Adds Microwaveprop.Radio.TextSanitizer to remove BOM, zero-width spaces/joiners, C0 control bytes (except CR/LF/TAB), and DEL, and to collapse NBSP and other unicode whitespace into a regular space. CSV: applied to the whole upload at the top of preview/2. ADIF: applied per-field-value after extraction so the byte-counted length tags still slice the right substring.
52 lines
1.8 KiB
Elixir
52 lines
1.8 KiB
Elixir
defmodule Microwaveprop.Radio.TextSanitizer do
|
|
@moduledoc """
|
|
Strip non-printable characters from user-uploaded log text. Removes BOM,
|
|
zero-width spaces/joiners, C0 control bytes (except CR/LF/TAB), and DEL;
|
|
collapses non-breaking and other unicode whitespace into a regular space.
|
|
|
|
Used by CSV and ADIF importers because field values frequently arrive
|
|
with stray NBSPs from website copy-paste, BOM markers from Windows
|
|
exports, etc.
|
|
"""
|
|
|
|
@spec sanitize(nil) :: nil
|
|
@spec sanitize(String.t()) :: String.t()
|
|
def sanitize(nil), do: nil
|
|
def sanitize(""), do: ""
|
|
|
|
def sanitize(string) when is_binary(string) do
|
|
string
|
|
|> :unicode.characters_to_list()
|
|
|> Enum.reduce([], fn cp, acc ->
|
|
case sanitize_codepoint(cp) do
|
|
:strip -> acc
|
|
replacement -> [replacement | acc]
|
|
end
|
|
end)
|
|
|> Enum.reverse()
|
|
|> :unicode.characters_to_binary()
|
|
end
|
|
|
|
defp sanitize_codepoint(?\t), do: ?\t
|
|
defp sanitize_codepoint(?\n), do: ?\n
|
|
defp sanitize_codepoint(?\r), do: ?\r
|
|
|
|
# C0 control characters and DEL — strip.
|
|
defp sanitize_codepoint(cp) when cp < 0x20 or cp == 0x7F, do: :strip
|
|
|
|
# NBSP and other unicode whitespace → regular space.
|
|
defp sanitize_codepoint(0x00A0), do: ?\s
|
|
defp sanitize_codepoint(cp) when cp >= 0x2000 and cp <= 0x200A, do: ?\s
|
|
defp sanitize_codepoint(0x202F), do: ?\s
|
|
defp sanitize_codepoint(0x205F), do: ?\s
|
|
defp sanitize_codepoint(0x3000), do: ?\s
|
|
|
|
# Zero-width characters (ZWSP, ZWNJ, ZWJ, word-joiner, BOM) — strip.
|
|
defp sanitize_codepoint(0x200B), do: :strip
|
|
defp sanitize_codepoint(0x200C), do: :strip
|
|
defp sanitize_codepoint(0x200D), do: :strip
|
|
defp sanitize_codepoint(0x2060), do: :strip
|
|
defp sanitize_codepoint(0xFEFF), do: :strip
|
|
|
|
defp sanitize_codepoint(cp), do: cp
|
|
end
|