Graham McIntire
fe2ecf2618
Adds Microwaveprop.Radio.TextSanitizer to remove BOM, zero-width spaces/joiners, C0 control bytes (except CR/LF/TAB), and DEL, and to collapse NBSP and other unicode whitespace into a regular space. CSV: applied to the whole upload at the top of preview/2. ADIF: applied per-field-value after extraction so the byte-counted length tags still slice the right substring.
56 lines
1.7 KiB
Elixir
56 lines
1.7 KiB
Elixir
defmodule Microwaveprop.Radio.TextSanitizerTest do
|
||
use ExUnit.Case, async: true
|
||
|
||
alias Microwaveprop.Radio.TextSanitizer
|
||
|
||
describe "sanitize/1" do
|
||
test "passes through plain ASCII unchanged" do
|
||
assert TextSanitizer.sanitize("K5ABC EM12 1296") == "K5ABC EM12 1296"
|
||
end
|
||
|
||
test "preserves CR, LF, and TAB" do
|
||
assert TextSanitizer.sanitize("a\nb\r\nc\td") == "a\nb\r\nc\td"
|
||
end
|
||
|
||
test "replaces NBSP (U+00A0) with regular space" do
|
||
assert TextSanitizer.sanitize("K5ABC\u00A0EM12") == "K5ABC EM12"
|
||
end
|
||
|
||
test "replaces narrow no-break space (U+202F) and figure space (U+2007) with space" do
|
||
assert TextSanitizer.sanitize("a\u202Fb\u2007c") == "a b c"
|
||
end
|
||
|
||
test "replaces all unicode general-punctuation spaces (U+2000–U+200A) with space" do
|
||
assert TextSanitizer.sanitize("a\u2000b\u2003c\u200Ad") == "a b c d"
|
||
end
|
||
|
||
test "strips BOM (U+FEFF) anywhere in the string" do
|
||
assert TextSanitizer.sanitize("\uFEFFK5ABC") == "K5ABC"
|
||
assert TextSanitizer.sanitize("K5\uFEFFABC") == "K5ABC"
|
||
end
|
||
|
||
test "strips zero-width joiners and non-joiners" do
|
||
assert TextSanitizer.sanitize("K5\u200BABC\u200C\u200D") == "K5ABC"
|
||
end
|
||
|
||
test "strips C0 control characters except TAB/CR/LF" do
|
||
assert TextSanitizer.sanitize("a\x00b\x07c\x1Bd") == "abcd"
|
||
end
|
||
|
||
test "strips DEL (U+007F)" do
|
||
assert TextSanitizer.sanitize("a\x7Fb") == "ab"
|
||
end
|
||
|
||
test "handles nil" do
|
||
assert TextSanitizer.sanitize(nil) == nil
|
||
end
|
||
|
||
test "handles empty string" do
|
||
assert TextSanitizer.sanitize("") == ""
|
||
end
|
||
|
||
test "preserves UTF-8 letters with diacritics" do
|
||
assert TextSanitizer.sanitize("café Müller") == "café Müller"
|
||
end
|
||
end
|
||
end
|