fix web showing packets

lib/aprs/encoding_utils.ex

@@ -0,0 +1,141 @@
+defmodule Aprs.EncodingUtils do
+  @moduledoc """
+  Utilities for handling encoding issues in APRS packet data.
+
+  APRS packets can contain arbitrary bytes that may not be valid UTF-8,
+  which causes issues when trying to JSON encode the data for transmission
+  to web clients.
+  """
+
+  @doc """
+  Sanitizes a binary to ensure it can be safely JSON encoded.
+
+  If the binary is valid UTF-8, returns it as-is.
+  If it contains invalid UTF-8 sequences, replaces them with the Unicode
+  replacement character (<EFBFBD>) or removes them entirely.
+
+  ## Examples
+
+      iex> Aprs.EncodingUtils.sanitize_string("Hello World")
+      "Hello World"
+
+      iex> Aprs.EncodingUtils.sanitize_string(<<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>)
+      "HelloWorld"
+  """
+  def sanitize_string(binary) when is_binary(binary) do
+    if String.valid?(binary) do
+      binary
+    else
+      # Replace invalid UTF-8 sequences with replacement character or remove them
+      scrub_invalid_utf8(binary)
+    end
+  end
+
+  def sanitize_string(nil), do: nil
+  def sanitize_string(other), do: other
+
+  @doc """
+  Sanitizes all string fields in an APRS packet to ensure safe JSON encoding.
+  """
+  def sanitize_packet(%Aprs.Packet{} = packet) do
+    %{
+      packet
+      | information_field: sanitize_string(packet.information_field),
+        data_extended: sanitize_data_extended(packet.data_extended)
+    }
+  end
+
+  @doc """
+  Sanitizes string fields in the data_extended structure.
+  """
+  def sanitize_data_extended(nil), do: nil
+
+  def sanitize_data_extended(%{comment: comment} = data_extended) when is_map(data_extended) do
+    %{data_extended | comment: sanitize_string(comment)}
+  end
+
+  def sanitize_data_extended(%Parser.Types.MicE{message: message} = mic_e) do
+    %{mic_e | message: sanitize_string(message)}
+  end
+
+  def sanitize_data_extended(data_extended), do: data_extended
+
+  # Private helper functions
+
+  defp scrub_invalid_utf8(binary) do
+    binary
+    |> :binary.bin_to_list()
+    |> Enum.filter(&valid_byte?/1)
+    |> Enum.map(&scrub_byte/1)
+    |> :binary.list_to_bin()
+    |> String.trim()
+  end
+
+  # Check if byte should be kept
+  defp valid_byte?(byte) when byte >= 32 and byte <= 126, do: true
+  defp valid_byte?(byte) when byte in [9, 10, 13], do: true
+  defp valid_byte?(_byte), do: false
+
+  # Keep valid bytes as-is
+  defp scrub_byte(byte), do: byte
+
+  @doc """
+  Converts a binary to a hex string representation for debugging.
+
+  ## Examples
+
+      iex> Aprs.EncodingUtils.to_hex(<<72, 101, 108, 108, 111>>)
+      "48656C6C6F"
+  """
+  def to_hex(binary) when is_binary(binary) do
+    binary
+    |> :binary.bin_to_list()
+    |> Enum.map(&Integer.to_string(&1, 16))
+    |> Enum.map_join("", &String.pad_leading(&1, 2, "0"))
+  end
+
+  @doc """
+  Returns information about a binary's encoding validity.
+
+  ## Examples
+
+      iex> Aprs.EncodingUtils.encoding_info("Hello")
+      %{valid_utf8: true, byte_count: 5, char_count: 5}
+
+      iex> Aprs.EncodingUtils.encoding_info(<<72, 101, 211, 108, 111>>)
+      %{valid_utf8: false, byte_count: 5, invalid_at: 2}
+  """
+  def encoding_info(binary) when is_binary(binary) do
+    valid = String.valid?(binary)
+    byte_count = byte_size(binary)
+
+    base_info = %{
+      valid_utf8: valid,
+      byte_count: byte_count
+    }
+
+    if valid do
+      Map.put(base_info, :char_count, String.length(binary))
+    else
+      # Try to find where the invalid sequence starts
+      invalid_at = find_invalid_byte_position(binary, 0)
+      Map.put(base_info, :invalid_at, invalid_at)
+    end
+  end
+
+  defp find_invalid_byte_position(<<>>, _pos), do: nil
+
+  defp find_invalid_byte_position(binary, pos) do
+    case binary do
+      <<head::binary-size(1), tail::binary>> ->
+        if String.valid?(head) do
+          find_invalid_byte_position(tail, pos + 1)
+        else
+          pos
+        end
+
+      _ ->
+        pos
+    end
+  end
+end

lib/aprs_web/live/packets_live/index.ex

@@ -2,6 +2,7 @@ defmodule AprsWeb.PacketsLive.Index do
   @moduledoc false
   use AprsWeb, :live_view
 
+  alias Aprs.EncodingUtils
   alias AprsWeb.Endpoint
 
   @impl true
@@ -15,7 +16,10 @@ defmodule AprsWeb.PacketsLive.Index do
 
   @impl true
   def handle_info(%{event: "packet", payload: payload}, socket) do
-    socket = assign(socket, :packets, [payload | socket.assigns.packets])
+    # Sanitize the packet to prevent JSON encoding errors
+    sanitized_payload = EncodingUtils.sanitize_packet(payload)
+    packets = Enum.take([sanitized_payload | socket.assigns.packets], 100)
+    socket = assign(socket, :packets, packets)
     {:noreply, socket}
   end
 end

test/aprs/encoding_utils_test.exs

@@ -0,0 +1,192 @@
+defmodule Aprs.EncodingUtilsTest do
+  use ExUnit.Case
+
+  alias Aprs.EncodingUtils
+  alias Aprs.Packet
+
+  doctest Aprs.EncodingUtils
+
+  describe "sanitize_string/1" do
+    test "returns valid UTF-8 strings unchanged" do
+      assert EncodingUtils.sanitize_string("Hello World") == "Hello World"
+      assert EncodingUtils.sanitize_string("Café") == "Café"
+      assert EncodingUtils.sanitize_string("你好") == "你好"
+    end
+
+    test "handles nil input" do
+      assert EncodingUtils.sanitize_string(nil) == nil
+    end
+
+    test "handles non-binary input" do
+      assert EncodingUtils.sanitize_string(123) == 123
+      assert EncodingUtils.sanitize_string(:atom) == :atom
+    end
+
+    test "sanitizes invalid UTF-8 sequences" do
+      # Binary with invalid UTF-8 bytes (0xD3 and 0xD1 from the error)
+      invalid_binary = <<85, 78, 73, 211, 78, 32, 80, 65, 78, 65, 77, 69, 209, 65>>
+      result = EncodingUtils.sanitize_string(invalid_binary)
+
+      # Should be a valid UTF-8 string
+      assert String.valid?(result)
+
+      # Should contain some recognizable text
+      assert String.contains?(result, "UNI")
+      assert String.contains?(result, "PANAME")
+    end
+
+    test "handles mixed valid and invalid UTF-8" do
+      # "Hello" + invalid byte + "World"
+      mixed_binary = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>
+      result = EncodingUtils.sanitize_string(mixed_binary)
+
+      assert String.valid?(result)
+      assert String.starts_with?(result, "Hello")
+      assert String.ends_with?(result, "World")
+    end
+  end
+
+  describe "sanitize_packet/1" do
+    test "sanitizes information_field" do
+      invalid_info = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>
+
+      packet = %Packet{
+        id: "test-id",
+        sender: "TEST-1",
+        path: "TCPIP*",
+        destination: "APX100",
+        information_field: invalid_info,
+        data_type: :position,
+        base_callsign: "TEST",
+        ssid: "1",
+        data_extended: %{comment: "Valid comment"}
+      }
+
+      sanitized = EncodingUtils.sanitize_packet(packet)
+
+      assert String.valid?(sanitized.information_field)
+      assert String.contains?(sanitized.information_field, "Hello")
+      assert String.contains?(sanitized.information_field, "World")
+    end
+
+    test "sanitizes comment in data_extended" do
+      invalid_comment = <<85, 78, 73, 211, 78, 32, 80, 65, 78, 65, 77, 69, 209, 65>>
+
+      packet = %Packet{
+        id: "test-id",
+        sender: "TEST-1",
+        path: "TCPIP*",
+        destination: "APX100",
+        information_field: "Valid info",
+        data_type: :position,
+        base_callsign: "TEST",
+        ssid: "1",
+        data_extended: %{comment: invalid_comment}
+      }
+
+      sanitized = EncodingUtils.sanitize_packet(packet)
+
+      assert String.valid?(sanitized.data_extended.comment)
+      assert String.contains?(sanitized.data_extended.comment, "UNI")
+    end
+
+    test "handles packet with nil data_extended" do
+      packet = %Packet{
+        id: "test-id",
+        sender: "TEST-1",
+        path: "TCPIP*",
+        destination: "APX100",
+        information_field: "Valid info",
+        data_type: :position,
+        base_callsign: "TEST",
+        ssid: "1",
+        data_extended: nil
+      }
+
+      sanitized = EncodingUtils.sanitize_packet(packet)
+
+      assert sanitized.data_extended == nil
+      assert sanitized.information_field == "Valid info"
+    end
+  end
+
+  describe "sanitize_data_extended/1" do
+    test "handles nil input" do
+      assert EncodingUtils.sanitize_data_extended(nil) == nil
+    end
+
+    test "sanitizes comment field in map" do
+      invalid_comment = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>
+      data_extended = %{comment: invalid_comment, latitude: 12.34, longitude: -56.78}
+
+      sanitized = EncodingUtils.sanitize_data_extended(data_extended)
+
+      assert String.valid?(sanitized.comment)
+      assert sanitized.latitude == 12.34
+      assert sanitized.longitude == -56.78
+    end
+
+    test "sanitizes message field in MicE struct" do
+      invalid_message = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>
+      mic_e = %Parser.Types.MicE{message: invalid_message, lat_degrees: 40}
+
+      sanitized = EncodingUtils.sanitize_data_extended(mic_e)
+
+      assert String.valid?(sanitized.message)
+      assert sanitized.lat_degrees == 40
+    end
+
+    test "returns other data unchanged" do
+      data = %{some_field: "value", number: 42}
+      assert EncodingUtils.sanitize_data_extended(data) == data
+    end
+  end
+
+  describe "to_hex/1" do
+    test "converts binary to hex string" do
+      assert EncodingUtils.to_hex(<<72, 101, 108, 108, 111>>) == "48656C6C6F"
+      assert EncodingUtils.to_hex(<<0, 255>>) == "00FF"
+      assert EncodingUtils.to_hex(<<>>) == ""
+    end
+
+    test "handles invalid UTF-8 bytes" do
+      invalid_binary = <<85, 78, 73, 211, 78>>
+      result = EncodingUtils.to_hex(invalid_binary)
+      assert result == "554E49D34E"
+    end
+  end
+
+  describe "encoding_info/1" do
+    test "returns info for valid UTF-8 string" do
+      info = EncodingUtils.encoding_info("Hello")
+      assert info.valid_utf8 == true
+      assert info.byte_count == 5
+      assert info.char_count == 5
+    end
+
+    test "returns info for invalid UTF-8 binary" do
+      invalid_binary = <<72, 101, 211, 108, 111>>
+      info = EncodingUtils.encoding_info(invalid_binary)
+
+      assert info.valid_utf8 == false
+      assert info.byte_count == 5
+      assert info.invalid_at == 2
+    end
+
+    test "handles empty binary" do
+      info = EncodingUtils.encoding_info("")
+      assert info.valid_utf8 == true
+      assert info.byte_count == 0
+      assert info.char_count == 0
+    end
+
+    test "handles multi-byte UTF-8 characters" do
+      info = EncodingUtils.encoding_info("Café")
+      assert info.valid_utf8 == true
+      # 'é' takes 2 bytes
+      assert info.byte_count == 5
+      # but counts as 1 character
+      assert info.char_count == 4
+    end
+  end
+end