From b809ffdd4af91a6c73c2937b4d566206dd4b72b0 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 13 Jun 2025 10:17:35 -0500 Subject: [PATCH] fix web showing packets --- lib/aprs/encoding_utils.ex | 141 +++++++++++++++++ lib/aprs_web/live/packets_live/index.ex | 6 +- test/aprs/encoding_utils_test.exs | 192 ++++++++++++++++++++++++ 3 files changed, 338 insertions(+), 1 deletion(-) create mode 100644 lib/aprs/encoding_utils.ex create mode 100644 test/aprs/encoding_utils_test.exs diff --git a/lib/aprs/encoding_utils.ex b/lib/aprs/encoding_utils.ex new file mode 100644 index 0000000..e238068 --- /dev/null +++ b/lib/aprs/encoding_utils.ex @@ -0,0 +1,141 @@ +defmodule Aprs.EncodingUtils do + @moduledoc """ + Utilities for handling encoding issues in APRS packet data. + + APRS packets can contain arbitrary bytes that may not be valid UTF-8, + which causes issues when trying to JSON encode the data for transmission + to web clients. + """ + + @doc """ + Sanitizes a binary to ensure it can be safely JSON encoded. + + If the binary is valid UTF-8, returns it as-is. + If it contains invalid UTF-8 sequences, replaces them with the Unicode + replacement character (�) or removes them entirely. + + ## Examples + + iex> Aprs.EncodingUtils.sanitize_string("Hello World") + "Hello World" + + iex> Aprs.EncodingUtils.sanitize_string(<<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>) + "HelloWorld" + """ + def sanitize_string(binary) when is_binary(binary) do + if String.valid?(binary) do + binary + else + # Replace invalid UTF-8 sequences with replacement character or remove them + scrub_invalid_utf8(binary) + end + end + + def sanitize_string(nil), do: nil + def sanitize_string(other), do: other + + @doc """ + Sanitizes all string fields in an APRS packet to ensure safe JSON encoding. + """ + def sanitize_packet(%Aprs.Packet{} = packet) do + %{ + packet + | information_field: sanitize_string(packet.information_field), + data_extended: sanitize_data_extended(packet.data_extended) + } + end + + @doc """ + Sanitizes string fields in the data_extended structure. + """ + def sanitize_data_extended(nil), do: nil + + def sanitize_data_extended(%{comment: comment} = data_extended) when is_map(data_extended) do + %{data_extended | comment: sanitize_string(comment)} + end + + def sanitize_data_extended(%Parser.Types.MicE{message: message} = mic_e) do + %{mic_e | message: sanitize_string(message)} + end + + def sanitize_data_extended(data_extended), do: data_extended + + # Private helper functions + + defp scrub_invalid_utf8(binary) do + binary + |> :binary.bin_to_list() + |> Enum.filter(&valid_byte?/1) + |> Enum.map(&scrub_byte/1) + |> :binary.list_to_bin() + |> String.trim() + end + + # Check if byte should be kept + defp valid_byte?(byte) when byte >= 32 and byte <= 126, do: true + defp valid_byte?(byte) when byte in [9, 10, 13], do: true + defp valid_byte?(_byte), do: false + + # Keep valid bytes as-is + defp scrub_byte(byte), do: byte + + @doc """ + Converts a binary to a hex string representation for debugging. + + ## Examples + + iex> Aprs.EncodingUtils.to_hex(<<72, 101, 108, 108, 111>>) + "48656C6C6F" + """ + def to_hex(binary) when is_binary(binary) do + binary + |> :binary.bin_to_list() + |> Enum.map(&Integer.to_string(&1, 16)) + |> Enum.map_join("", &String.pad_leading(&1, 2, "0")) + end + + @doc """ + Returns information about a binary's encoding validity. + + ## Examples + + iex> Aprs.EncodingUtils.encoding_info("Hello") + %{valid_utf8: true, byte_count: 5, char_count: 5} + + iex> Aprs.EncodingUtils.encoding_info(<<72, 101, 211, 108, 111>>) + %{valid_utf8: false, byte_count: 5, invalid_at: 2} + """ + def encoding_info(binary) when is_binary(binary) do + valid = String.valid?(binary) + byte_count = byte_size(binary) + + base_info = %{ + valid_utf8: valid, + byte_count: byte_count + } + + if valid do + Map.put(base_info, :char_count, String.length(binary)) + else + # Try to find where the invalid sequence starts + invalid_at = find_invalid_byte_position(binary, 0) + Map.put(base_info, :invalid_at, invalid_at) + end + end + + defp find_invalid_byte_position(<<>>, _pos), do: nil + + defp find_invalid_byte_position(binary, pos) do + case binary do + <> -> + if String.valid?(head) do + find_invalid_byte_position(tail, pos + 1) + else + pos + end + + _ -> + pos + end + end +end diff --git a/lib/aprs_web/live/packets_live/index.ex b/lib/aprs_web/live/packets_live/index.ex index 17c1512..f3b7195 100644 --- a/lib/aprs_web/live/packets_live/index.ex +++ b/lib/aprs_web/live/packets_live/index.ex @@ -2,6 +2,7 @@ defmodule AprsWeb.PacketsLive.Index do @moduledoc false use AprsWeb, :live_view + alias Aprs.EncodingUtils alias AprsWeb.Endpoint @impl true @@ -15,7 +16,10 @@ defmodule AprsWeb.PacketsLive.Index do @impl true def handle_info(%{event: "packet", payload: payload}, socket) do - socket = assign(socket, :packets, [payload | socket.assigns.packets]) + # Sanitize the packet to prevent JSON encoding errors + sanitized_payload = EncodingUtils.sanitize_packet(payload) + packets = Enum.take([sanitized_payload | socket.assigns.packets], 100) + socket = assign(socket, :packets, packets) {:noreply, socket} end end diff --git a/test/aprs/encoding_utils_test.exs b/test/aprs/encoding_utils_test.exs new file mode 100644 index 0000000..65a4411 --- /dev/null +++ b/test/aprs/encoding_utils_test.exs @@ -0,0 +1,192 @@ +defmodule Aprs.EncodingUtilsTest do + use ExUnit.Case + + alias Aprs.EncodingUtils + alias Aprs.Packet + + doctest Aprs.EncodingUtils + + describe "sanitize_string/1" do + test "returns valid UTF-8 strings unchanged" do + assert EncodingUtils.sanitize_string("Hello World") == "Hello World" + assert EncodingUtils.sanitize_string("Café") == "Café" + assert EncodingUtils.sanitize_string("你好") == "你好" + end + + test "handles nil input" do + assert EncodingUtils.sanitize_string(nil) == nil + end + + test "handles non-binary input" do + assert EncodingUtils.sanitize_string(123) == 123 + assert EncodingUtils.sanitize_string(:atom) == :atom + end + + test "sanitizes invalid UTF-8 sequences" do + # Binary with invalid UTF-8 bytes (0xD3 and 0xD1 from the error) + invalid_binary = <<85, 78, 73, 211, 78, 32, 80, 65, 78, 65, 77, 69, 209, 65>> + result = EncodingUtils.sanitize_string(invalid_binary) + + # Should be a valid UTF-8 string + assert String.valid?(result) + + # Should contain some recognizable text + assert String.contains?(result, "UNI") + assert String.contains?(result, "PANAME") + end + + test "handles mixed valid and invalid UTF-8" do + # "Hello" + invalid byte + "World" + mixed_binary = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>> + result = EncodingUtils.sanitize_string(mixed_binary) + + assert String.valid?(result) + assert String.starts_with?(result, "Hello") + assert String.ends_with?(result, "World") + end + end + + describe "sanitize_packet/1" do + test "sanitizes information_field" do + invalid_info = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>> + + packet = %Packet{ + id: "test-id", + sender: "TEST-1", + path: "TCPIP*", + destination: "APX100", + information_field: invalid_info, + data_type: :position, + base_callsign: "TEST", + ssid: "1", + data_extended: %{comment: "Valid comment"} + } + + sanitized = EncodingUtils.sanitize_packet(packet) + + assert String.valid?(sanitized.information_field) + assert String.contains?(sanitized.information_field, "Hello") + assert String.contains?(sanitized.information_field, "World") + end + + test "sanitizes comment in data_extended" do + invalid_comment = <<85, 78, 73, 211, 78, 32, 80, 65, 78, 65, 77, 69, 209, 65>> + + packet = %Packet{ + id: "test-id", + sender: "TEST-1", + path: "TCPIP*", + destination: "APX100", + information_field: "Valid info", + data_type: :position, + base_callsign: "TEST", + ssid: "1", + data_extended: %{comment: invalid_comment} + } + + sanitized = EncodingUtils.sanitize_packet(packet) + + assert String.valid?(sanitized.data_extended.comment) + assert String.contains?(sanitized.data_extended.comment, "UNI") + end + + test "handles packet with nil data_extended" do + packet = %Packet{ + id: "test-id", + sender: "TEST-1", + path: "TCPIP*", + destination: "APX100", + information_field: "Valid info", + data_type: :position, + base_callsign: "TEST", + ssid: "1", + data_extended: nil + } + + sanitized = EncodingUtils.sanitize_packet(packet) + + assert sanitized.data_extended == nil + assert sanitized.information_field == "Valid info" + end + end + + describe "sanitize_data_extended/1" do + test "handles nil input" do + assert EncodingUtils.sanitize_data_extended(nil) == nil + end + + test "sanitizes comment field in map" do + invalid_comment = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>> + data_extended = %{comment: invalid_comment, latitude: 12.34, longitude: -56.78} + + sanitized = EncodingUtils.sanitize_data_extended(data_extended) + + assert String.valid?(sanitized.comment) + assert sanitized.latitude == 12.34 + assert sanitized.longitude == -56.78 + end + + test "sanitizes message field in MicE struct" do + invalid_message = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>> + mic_e = %Parser.Types.MicE{message: invalid_message, lat_degrees: 40} + + sanitized = EncodingUtils.sanitize_data_extended(mic_e) + + assert String.valid?(sanitized.message) + assert sanitized.lat_degrees == 40 + end + + test "returns other data unchanged" do + data = %{some_field: "value", number: 42} + assert EncodingUtils.sanitize_data_extended(data) == data + end + end + + describe "to_hex/1" do + test "converts binary to hex string" do + assert EncodingUtils.to_hex(<<72, 101, 108, 108, 111>>) == "48656C6C6F" + assert EncodingUtils.to_hex(<<0, 255>>) == "00FF" + assert EncodingUtils.to_hex(<<>>) == "" + end + + test "handles invalid UTF-8 bytes" do + invalid_binary = <<85, 78, 73, 211, 78>> + result = EncodingUtils.to_hex(invalid_binary) + assert result == "554E49D34E" + end + end + + describe "encoding_info/1" do + test "returns info for valid UTF-8 string" do + info = EncodingUtils.encoding_info("Hello") + assert info.valid_utf8 == true + assert info.byte_count == 5 + assert info.char_count == 5 + end + + test "returns info for invalid UTF-8 binary" do + invalid_binary = <<72, 101, 211, 108, 111>> + info = EncodingUtils.encoding_info(invalid_binary) + + assert info.valid_utf8 == false + assert info.byte_count == 5 + assert info.invalid_at == 2 + end + + test "handles empty binary" do + info = EncodingUtils.encoding_info("") + assert info.valid_utf8 == true + assert info.byte_count == 0 + assert info.char_count == 0 + end + + test "handles multi-byte UTF-8 characters" do + info = EncodingUtils.encoding_info("Café") + assert info.valid_utf8 == true + # 'é' takes 2 bytes + assert info.byte_count == 5 + # but counts as 1 character + assert info.char_count == 4 + end + end +end