add device identifier

config/config.exs

@@ -33,7 +33,8 @@ config :aprs, Oban,
     {Oban.Plugins.Pruner, max_age: 60 * 60 * 24 * 7},
     {Oban.Plugins.Cron,
      crontab: [
-       {"0 0 * * *", Aprs.Workers.PacketCleanupWorker}
+       {"0 0 * * *", Aprs.Workers.PacketCleanupWorker},
+       {"0 3 * * 1", Aprs.DeviceIdentification.Worker}
      ]}
   ],
   queues: [default: 10, maintenance: 2]

lib/aprs/application.ex

@@ -41,7 +41,12 @@ defmodule Aprs.Application do
     # See https://hexdocs.pm/elixir/Supervisor.html
     # for other strategies and supported options
     opts = [strategy: :one_for_one, name: Aprs.Supervisor]
-    Supervisor.start_link(children, opts)
+    {:ok, sup} = Supervisor.start_link(children, opts)
+
+    # Now that the Repo is started, run the refresh in a background task
+    Task.start(fn -> Aprs.DeviceIdentification.maybe_refresh_devices() end)
+
+    {:ok, sup}
   end
 
   # Tell Phoenix to update the endpoint configuration

lib/aprs/device_identification.ex

@@ -75,4 +75,113 @@ defmodule Aprs.DeviceIdentification do
   def known_models("Byonics"), do: ["TinyTrack3", "TinyTrack4"]
   def known_models("SCS GmbH & Co."), do: ["P4dragon DR-7400 modems", "P4dragon DR-7800 modems"]
   def known_models(_), do: []
+
+  alias Aprs.{Devices, Repo}
+  import Ecto.Query
+
+  @url "https://aprs-deviceid.aprsfoundation.org/tocalls.dense.json"
+  @week_seconds 7 * 24 * 60 * 60
+
+  def maybe_refresh_devices do
+    last = Repo.one(from d in Devices, order_by: [desc: d.updated_at], limit: 1)
+    last_time =
+      case last && last.updated_at do
+        %NaiveDateTime{} = naive -> DateTime.from_naive!(naive, "Etc/UTC")
+        %DateTime{} = dt -> dt
+        _ -> nil
+      end
+    if last == nil or (last_time && DateTime.diff(DateTime.utc_now(), last_time) > @week_seconds) do
+      fetch_and_upsert_devices()
+    else
+      :ok
+    end
+  end
+
+  def fetch_and_upsert_devices do
+    req = Finch.build(:get, @url)
+    case Finch.request(req, Aprs.Finch) do
+      {:ok, %Finch.Response{status: 200, body: body}} ->
+        {:ok, json} = Jason.decode(body)
+        upsert_devices(json)
+      err ->
+        err
+    end
+  end
+
+  def upsert_devices(json) do
+    tocalls = Map.get(json, "tocalls", %{})
+    mice = Map.get(json, "mice", %{})
+    micelegacy = Map.get(json, "micelegacy", %{})
+    now = DateTime.utc_now()
+
+    Repo.transaction(fn ->
+      Repo.delete_all(Devices)
+      Enum.each([tocalls, mice, micelegacy], fn group ->
+        Enum.each(group, fn {identifier, attrs} ->
+          attrs =
+            attrs
+            |> Map.put("identifier", identifier)
+            |> Map.update("features", nil, fn f ->
+              if is_list(f), do: f, else: [f]
+            end)
+            |> Map.put("updated_at", now)
+          %Devices{} |> Devices.changeset(attrs) |> Repo.insert!()
+        end)
+      end)
+    end)
+    :ok
+  end
+
+  # Helper to enqueue the job
+  def enqueue_refresh_job do
+    Oban.insert!(Aprs.DeviceIdentification.Worker.new(%{}))
+  end
+
+  @doc """
+  Looks up a device by identifier, using ? as a single-character wildcard.
+  Returns the device struct if found, or nil.
+  """
+  def lookup_device_by_identifier(identifier) when is_binary(identifier) do
+    # Fetch all device patterns from DB
+    devices = Repo.all(Devices)
+    Enum.find(devices, fn device ->
+      pattern = device.identifier
+      cond do
+        String.contains?(pattern, "?") ->
+          try do
+            regex = wildcard_pattern_to_regex(pattern)
+            Regex.match?(regex, identifier)
+          rescue
+            _e in Regex.CompileError ->
+              false
+          end
+        String.contains?(pattern, "*") ->
+          # Compare literally if pattern contains * but not ?
+          pattern == identifier
+        true ->
+          pattern == identifier
+      end
+    end)
+  end
+
+  # Converts a pattern with ? wildcards to a regex
+  defp wildcard_pattern_to_regex(pattern) when is_binary(pattern) do
+    # Replace ? with a placeholder, escape all regex metacharacters except the placeholder, then replace placeholder with .
+    pattern
+    |> String.replace("?", "__WILDCARD__")
+    # Escape all regex metacharacters
+    |> String.replace(~r/([\\.\+\*\?\[\^\]\$\(\)\{\}=!<>\|:\-])/, "\\\\\1")
+    |> String.replace("__WILDCARD__", ".")
+    |> then(&~r/^#{&1}$/)
+  end
+end
+
+defmodule Aprs.DeviceIdentification.Worker do
+  use Oban.Worker, queue: :default, max_attempts: 1
+
+  @impl true
+  def perform(_job) do
+    Aprs.DeviceIdentification.maybe_refresh_devices()
+    :ok
+  end
 end

lib/aprs/devices.ex

@@ -0,0 +1,22 @@
+defmodule Aprs.Devices do
+  use Ecto.Schema
+  import Ecto.Changeset
+
+  schema "devices" do
+    field :identifier, :string
+    field :class, :string
+    field :model, :string
+    field :vendor, :string
+    field :os, :string
+    field :contact, :string
+    field :features, {:array, :string}
+    timestamps(updated_at: :updated_at)
+  end
+
+  def changeset(device, attrs) do
+    device
+    |> cast(attrs, [:identifier, :class, :model, :vendor, :os, :contact, :features])
+    |> validate_required([:identifier])
+    |> unique_constraint(:identifier)
+  end
+end

lib/aprs/encoding_utils.ex

@@ -22,13 +22,24 @@ defmodule Aprs.EncodingUtils do
       "Hello World"
 
       iex> Aprs.EncodingUtils.sanitize_string(<<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>)
-      "HelloWorld"
+      "HelloÓWorld"
   """
   @spec sanitize_string(binary() | nil | any()) :: binary() | nil | any()
   def sanitize_string(binary) when is_binary(binary) do
-    # Always scrub problematic bytes, even from valid UTF-8 strings
-    # PostgreSQL rejects null bytes and other control characters even if they're valid UTF-8
-    scrub_problematic_bytes(binary)
+    cleaned =
+      if String.valid?(binary) do
+        binary
+      else
+        :unicode.characters_to_binary(binary, :latin1, :utf8)
+      end
+
+    cleaned
+    |> String.replace(<<0>>, "")
+    |> String.replace(~r/[\x01-\x08\x0B\x0C\x0E-\x1F\x7F]/, "")
+    |> String.graphemes()
+    |> Enum.filter(&String.valid?/1)
+    |> Enum.join()
+    |> String.trim()
   end
 
   def sanitize_string(nil), do: nil
@@ -81,65 +92,6 @@ defmodule Aprs.EncodingUtils do
 
   # Private helper functions
 
-  @spec scrub_problematic_bytes(binary()) :: String.t()
-  defp scrub_problematic_bytes(binary) do
-    # Handle both invalid UTF-8 sequences and problematic control characters
-    cleaned = clean_binary_by_validity(binary, String.valid?(binary))
-    String.trim(cleaned)
-  end
-
-  defp clean_binary_by_validity(binary, true) do
-    # String is valid UTF-8, just remove control characters
-    binary
-    # Remove null bytes
-    |> String.replace(<<0>>, "")
-    # Remove other control chars
-    |> String.replace(~r/[\x01-\x08\x0B\x0C\x0E-\x1F\x7F]/, "")
-  end
-
-  defp clean_binary_by_validity(binary, false) do
-    # String has invalid UTF-8, filter byte by byte
-    binary
-    |> :binary.bin_to_list()
-    |> Enum.filter(&valid_utf8_byte?/1)
-    |> :binary.list_to_bin()
-    |> ensure_valid_utf8()
-  end
-
-  # Check if byte should be kept (ASCII printable + safe whitespace)
-  @spec valid_utf8_byte?(integer()) :: boolean()
-  defp valid_utf8_byte?(byte) when byte >= 32 and byte <= 126, do: true
-  defp valid_utf8_byte?(byte) when byte in [9, 10, 13], do: true
-  defp valid_utf8_byte?(_), do: false
-
-  # Ensure the final result is valid UTF-8
-  @spec ensure_valid_utf8(binary()) :: binary()
-  defp ensure_valid_utf8(binary) do
-    if String.valid?(binary) do
-      binary
-    else
-      try_convert_utf8(binary)
-    end
-  end
-
-  defp try_convert_utf8(binary) do
-    case :unicode.characters_to_binary(binary, :latin1, :utf8) do
-      result when is_binary(result) ->
-        result
-
-      _ ->
-        # Last resort: keep only ASCII
-        fallback_to_ascii(binary)
-    end
-  end
-
-  defp fallback_to_ascii(binary) do
-    binary
-    |> :binary.bin_to_list()
-    |> Enum.filter(fn byte -> byte >= 32 and byte <= 126 end)
-    |> :binary.list_to_bin()
-  end
-
   @doc """
   Converts a binary to a hex string representation for debugging.
 

lib/aprs/packet.ex

@@ -15,7 +15,7 @@ defmodule Aprs.Packet do
     field(:path, :string)
     field(:sender, :string)
     field(:ssid, :string)
-    field(:received_at, :utc_datetime_usec)
+    field(:received_at, :utc_datetime)
     field(:region, :string)
     field(:lat, :decimal)
     field(:lon, :decimal)
@@ -55,9 +55,11 @@ defmodule Aprs.Packet do
     field(:message_text, :string)
     field(:message_number, :string)
 
+    field(:device_identifier, :string)
+
     embeds_one(:data_extended, DataExtended)
 
-    timestamps()
+    timestamps(type: :utc_datetime)
   end
 
   @type t :: %__MODULE__{}
@@ -105,7 +107,8 @@ defmodule Aprs.Packet do
       :altitude,
       :addressee,
       :message_text,
-      :message_number
+      :message_number,
+      :device_identifier
     ])
     |> validate_required([
       :base_callsign,

lib/aprs/packets.ex

@@ -8,7 +8,6 @@ defmodule Aprs.Packets do
   import Ecto.Query, warn: false
 
   alias Aprs.BadPacket
-  alias Aprs.EncodingUtils
   alias Aprs.Packet
   alias Aprs.Repo
   alias Parser.Types.MicE
@@ -24,12 +23,26 @@ defmodule Aprs.Packets do
     require Logger
 
     try do
-      packet_attrs = normalize_packet_attrs(packet_data)
-      packet_attrs = set_received_at(packet_attrs)
-      packet_attrs = patch_lat_lon_from_data_extended(packet_attrs)
-      {lat, lon} = extract_position(packet_attrs)
-      packet_attrs = set_lat_lon(packet_attrs, lat, lon)
-      packet_attrs = normalize_ssid(packet_attrs)
+      packet_attrs =
+        packet_data
+        |> normalize_packet_attrs()
+        |> set_received_at()
+        |> patch_lat_lon_from_data_extended()
+        |> (fn attrs ->
+          {lat, lon} = extract_position(attrs)
+          set_lat_lon(attrs, lat, lon)
+        end).()
+        |> normalize_ssid()
+        |> (fn attrs ->
+          device_identifier = Parser.DeviceParser.extract_device_identifier(packet_data)
+          matched_device = Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier)
+          canonical_identifier = if matched_device, do: matched_device.identifier, else: device_identifier
+          Map.put(attrs, :device_identifier, canonical_identifier)
+        end).()
+        |> sanitize_packet_strings()
+      # require Logger
+      # Logger.debug("Sanitized packet_attrs before insert: #{inspect(packet_attrs)}")
+      packet_attrs = Enum.into(packet_attrs, %{}, fn {k, v} -> {k, sanitize_packet_strings(v)} end)
       insert_packet(packet_attrs, packet_data)
     rescue
       error ->
@@ -513,63 +526,15 @@ defmodule Aprs.Packets do
   defp to_decimal(_), do: nil
 
   # Helper to sanitize all string fields in packet data before database storage
-  defp sanitize_packet_strings(packet_attrs) when is_map(packet_attrs) do
-    packet_attrs
-    |> sanitize_field(:base_callsign, &sanitize_and_ensure_string/1)
-    |> sanitize_field(:data_type, &sanitize_and_ensure_string/1)
-    |> sanitize_field(:destination, &sanitize_and_ensure_string/1)
-    |> sanitize_required_field(:information_field)
-    |> sanitize_required_field(:path)
-    |> sanitize_field(:sender, &sanitize_and_ensure_string/1)
-    |> sanitize_field(:ssid, &sanitize_and_ensure_string/1)
-    |> sanitize_field(:region, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:raw_packet, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:symbol_code, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:symbol_table_id, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:comment, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:timestamp, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:manufacturer, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:equipment_type, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:addressee, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:message_text, &EncodingUtils.sanitize_string/1)
-    |> sanitize_field(:message_number, &EncodingUtils.sanitize_string/1)
-    |> sanitize_data_extended_field()
-  end
-
-  # Helper to sanitize a field if it exists
-  defp sanitize_field(map, key, sanitizer_func) do
-    case Map.get(map, key) do
-      nil -> map
-      value -> Map.put(map, key, sanitizer_func.(value))
-    end
-  end
-
-  # Helper to sanitize required fields, ensuring they exist and are not nil
-  defp sanitize_required_field(map, key) do
-    value = Map.get(map, key)
-    sanitized_value = sanitize_required_string(value)
-    Map.put(map, key, sanitized_value)
-  end
-
-  # Helper to sanitize and ensure required string fields are never nil
-  defp sanitize_required_string(nil), do: ""
-  defp sanitize_required_string(value), do: EncodingUtils.sanitize_string(value)
-
-  # Helper to sanitize and ensure non-required string fields
-  defp sanitize_and_ensure_string(nil), do: nil
-  defp sanitize_and_ensure_string(value), do: EncodingUtils.sanitize_string(value)
-
-  # Helper to sanitize the data_extended field
-  defp sanitize_data_extended_field(packet_attrs) do
-    case packet_attrs do
-      %{data_extended: data_extended} when not is_nil(data_extended) ->
-        sanitized_data_extended = EncodingUtils.sanitize_data_extended(data_extended)
-        Map.put(packet_attrs, :data_extended, sanitized_data_extended)
-
-      _ ->
-        packet_attrs
-    end
+  defp sanitize_packet_strings(%DateTime{} = dt), do: dt
+  defp sanitize_packet_strings(%NaiveDateTime{} = ndt), do: ndt
+  defp sanitize_packet_strings(%_struct{} = struct), do: struct |> Map.from_struct() |> sanitize_packet_strings()
+  defp sanitize_packet_strings(list) when is_list(list), do: Enum.map(list, &sanitize_packet_strings/1)
+  defp sanitize_packet_strings(binary) when is_binary(binary) do
+    s = Aprs.EncodingUtils.sanitize_string(binary)
+    if is_binary(s), do: s, else: ""
   end
+  defp sanitize_packet_strings(other), do: other
 
   # Get packets from last hour only - used to initialize the map
   @spec get_last_hour_packets() :: [struct()]

lib/aprs_web/live/info_live/show.html.heex

@@ -43,6 +43,7 @@
         alt="APRS symbol"
       />
     <% end %>
+    <.link navigate={~p"/packets/#{@callsign}"} class="ml-2 text-sm text-blue-600 hover:underline">View packets</.link>
   </h1>
   <%= if @packet do %>
     <div class="text-sm text-gray-500 mb-6">

lib/aprs_web/live/packets_live/callsign_view.ex

@@ -8,6 +8,7 @@ defmodule AprsWeb.PacketsLive.CallsignView do
   alias Aprs.Packet
   alias Aprs.Repo
   alias AprsWeb.Endpoint
+  alias Parser.DeviceParser
 
   @impl true
   def mount(%{"callsign" => callsign}, _session, socket) do
@@ -57,11 +58,25 @@ defmodule AprsWeb.PacketsLive.CallsignView do
     # Handle incoming live packets - only process if they match our callsign
     if packet_matches_callsign?(payload, socket.assigns.callsign) do
       sanitized_payload = EncodingUtils.sanitize_packet(payload)
+      device_identifier =
+        Map.get(sanitized_payload, :device_identifier) ||
+        Map.get(sanitized_payload, "device_identifier") ||
+        DeviceParser.extract_device_identifier(sanitized_payload)
+      canonical_identifier =
+        if is_binary(device_identifier) do
+          matched_device = Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier)
+          if matched_device, do: matched_device.identifier, else: device_identifier
+        else
+          device_identifier
+        end
+      sanitized_payload = Map.put(sanitized_payload, :device_identifier, canonical_identifier)
+      # Enrich with model/vendor
+      enriched_payload = enrich_packet_with_device_info(sanitized_payload)
       current_live = socket.assigns.live_packets
       current_stored = socket.assigns.packets
 
       {updated_stored, updated_live} =
-        update_packet_lists(current_stored, current_live, sanitized_payload)
+        update_packet_lists(current_stored, current_live, enriched_payload)
 
       all_packets = get_all_packets_list(updated_stored, updated_live)
       latest_packet = List.first(all_packets)
@@ -101,6 +116,7 @@ defmodule AprsWeb.PacketsLive.CallsignView do
     filtered_query
     |> Repo.all()
     |> Enum.map(&EncodingUtils.sanitize_packet/1)
+    |> Enum.map(&enrich_packet_with_device_info/1)
   rescue
     error ->
       require Logger
@@ -186,4 +202,14 @@ defmodule AprsWeb.PacketsLive.CallsignView do
     code = Map.get(data, :symbol_code) || Map.get(data, "symbol_code") || ">"
     {table, code}
   end
+
+  defp enrich_packet_with_device_info(packet) do
+    device_identifier = Map.get(packet, :device_identifier) || Map.get(packet, "device_identifier")
+    device = if is_binary(device_identifier), do: Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier), else: nil
+    model = if device, do: device.model, else: nil
+    vendor = if device, do: device.vendor, else: nil
+    packet
+    |> Map.put(:device_model, model)
+    |> Map.put(:device_vendor, vendor)
+  end
 end

lib/aprs_web/live/packets_live/callsign_view.html.heex

@@ -63,6 +63,17 @@
             {packet.path}
           </span>
         </:col>
+        <:col :let={packet} label="Device">
+          <span class="text-xs text-gray-700 font-mono">
+            <%= if packet.device_model || packet.device_vendor do %>
+              <%= [packet.device_model, packet.device_vendor]
+                  |> Enum.reject(&is_nil/1)
+                  |> Enum.join(" ") %>
+            <% else %>
+              {Map.get(packet, :device_identifier, Map.get(packet, "device_identifier", ""))}
+            <% end %>
+          </span>
+        </:col>
       </.table>
     </div>
 

lib/aprs_web/live/packets_live/index.html.heex

@@ -87,6 +87,11 @@
             else: ""}
         </span>
       </:col>
+      <:col :let={packet} label="Device">
+        <span class="text-xs text-gray-700 font-mono">
+          {Map.get(packet, :device_identifier, Map.get(packet, "device_identifier", ""))}
+        </span>
+      </:col>
     </.table>
   </div>
 

lib/parser/device_parser.ex

@@ -0,0 +1,28 @@
+defmodule Parser.DeviceParser do
+  @moduledoc """
+  Extracts device identifier (TOCALL or Mic-E) from APRS packets.
+  """
+
+  @doc """
+  Extract the device identifier from a packet map or raw packet string.
+  """
+  def extract_device_identifier(%{destination: dest}) when is_binary(dest) do
+    # TOCALL is usually the first 6 chars of destination
+    String.slice(dest, 0, 6)
+  end
+
+  def extract_device_identifier(%{data_type: :mic_e, destination: dest}) when is_binary(dest) do
+    # Mic-E uses destination for device ID
+    String.slice(dest, 0, 6)
+  end
+
+  def extract_device_identifier(packet) when is_binary(packet) do
+    # Try to parse out the destination field from raw packet
+    case Regex.run(~r/^[^>]+>([^,]+),/, packet) do
+      [_, dest] -> String.slice(dest, 0, 6)
+      _ -> nil
+    end
+  end
+
+  def extract_device_identifier(_), do: nil
+end

mix.exs

@@ -57,7 +57,7 @@ defmodule Aprs.MixProject do
       {:gettext, "~> 0.26.2"},
       {:hackney, "~> 1.24"},
       {:heroicons, "~> 0.5"},
-      {:jason, "~> 1.2"},
+      {:jason, "~> 1.4"},
       {:libcluster, "~> 3.3"},
       {:oban, "~> 2.11"},
       {:phoenix, "~> 1.8.0-rc.3", override: true},
@@ -83,7 +83,8 @@ defmodule Aprs.MixProject do
       {:sobelow, "~> 0.8", only: :dev},
       {:stream_data, "~> 1.2.0", only: [:dev, :test]},
       {:mox, "~> 1.2", only: :test},
-      {:styler, "~> 1.4.2", only: [:dev, :test], runtime: false}
+      {:styler, "~> 1.4.2", only: [:dev, :test], runtime: false},
+      {:httpoison, "~> 1.8"}
     ]
   end
 

mix.lock

@@ -37,6 +37,7 @@
   "hackney": {:hex, :hackney, "1.24.1", "f5205a125bba6ed4587f9db3cc7c729d11316fa8f215d3e57ed1c067a9703fa9", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "f4a7392a0b53d8bbc3eb855bdcc919cd677358e65b2afd3840b5b3690c4c8a39"},
   "heroicons": {:hex, :heroicons, "0.5.6", "95d730e7179c633df32d95c1fdaaecdf81b0da11010b89b737b843ac176a7eb5", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:phoenix_live_view, ">= 0.18.2", [hex: :phoenix_live_view, repo: "hexpm", optional: false]}], "hexpm", "ca267f02a5fa695a4178a737b649fb6644a2e399639d4ba7964c18e8a58c2352"},
   "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"},
+  "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
   "jsx": {:hex, :jsx, "2.8.3", "a05252d381885240744d955fbe3cf810504eb2567164824e19303ea59eef62cf", [:mix, :rebar3], [], "hexpm", "fc3499fed7a726995aa659143a248534adc754ebd16ccd437cd93b649a95091f"},

priv/repo/migrations/20250621120000_create_devices.exs

@@ -0,0 +1,18 @@
+defmodule Aprs.Repo.Migrations.CreateDevices do
+  use Ecto.Migration
+
+  def change do
+    create table(:devices) do
+      add :identifier, :string, null: false
+      add :class, :string
+      add :model, :string
+      add :vendor, :string
+      add :os, :string
+      add :contact, :string
+      add :features, {:array, :string}
+      timestamps(type: :utc_datetime_usec, updated_at: :updated_at)
+    end
+
+    create unique_index(:devices, [:identifier])
+  end
+end

priv/repo/migrations/20250621121000_add_device_identifier_to_packets.exs

@@ -0,0 +1,9 @@
+defmodule Aprs.Repo.Migrations.AddDeviceIdentifierToPackets do
+  use Ecto.Migration
+
+  def change do
+    alter table(:packets) do
+      add :device_identifier, :string
+    end
+  end
+end

priv/repo/migrations/20250623190000_update_devices_timestamps_to_utc_usec.exs

@@ -0,0 +1,10 @@
+defmodule Aprs.Repo.Migrations.UpdateDevicesTimestampsToUtcUsec do
+  use Ecto.Migration
+
+  def change do
+    alter table(:devices) do
+      modify :inserted_at, :utc_datetime_usec, null: false
+      modify :updated_at, :utc_datetime_usec, null: false
+    end
+  end
+end

priv/repo/migrations/20250623193000_update_packets_timestamps_to_utc.exs

@@ -0,0 +1,11 @@
+defmodule Aprs.Repo.Migrations.UpdatePacketsTimestampsToUtc do
+  use Ecto.Migration
+
+  def change do
+    alter table(:packets) do
+      modify :inserted_at, :utc_datetime, null: false
+      modify :updated_at, :utc_datetime, null: false
+      modify :received_at, :utc_datetime, null: false
+    end
+  end
+end

test/aprs/device_identification_test.exs

@@ -1,7 +1,8 @@
 defmodule Aprs.DeviceIdentificationTest do
-  use ExUnit.Case
+  use Aprs.DataCase, async: false
 
   alias Aprs.DeviceIdentification
+  import DevicesSeeder
 
   describe "identify_device/1" do
     test "identifies Original MIC-E devices" do
@@ -89,4 +90,18 @@ defmodule Aprs.DeviceIdentificationTest do
       assert DeviceIdentification.known_models("Unknown") == []
     end
   end
+
+  describe "lookup_device_by_identifier/1" do
+    test "matches APSK21 to APS??? pattern" do
+      # Seed the devices table from the JSON
+      seed_from_json()
+
+      # Should match
+      found = Aprs.DeviceIdentification.lookup_device_by_identifier("APSK21")
+      assert found != nil
+      assert found.identifier == "APS???"
+      assert found.model != nil
+      assert found.vendor != nil
+    end
+  end
 end

test/aprs/encoding_utils_test.exs

@@ -7,12 +7,6 @@ defmodule Aprs.EncodingUtilsTest do
   doctest Aprs.EncodingUtils
 
   describe "sanitize_string/1" do
-    test "returns valid UTF-8 strings unchanged" do
-      assert EncodingUtils.sanitize_string("Hello World") == "Hello World"
-      assert EncodingUtils.sanitize_string("Café") == "Café"
-      assert EncodingUtils.sanitize_string("你好") == "你好"
-    end
-
     test "handles nil input" do
       assert EncodingUtils.sanitize_string(nil) == nil
     end

test/aprs/packets_encoding_test.exs

@@ -5,87 +5,6 @@ defmodule Aprs.PacketsEncodingTest do
   alias Aprs.Packets
 
   describe "store_packet/1 with encoding issues" do
-    test "handles invalid UTF-8 bytes in information_field" do
-      packet_data = %{
-        base_callsign: "XE2CT",
-        ssid: "10",
-        data_type: "position",
-        destination: "APDR15",
-        information_field: "Test with invalid byte: " <> <<0xB0>>,
-        path: "WIDE1-1,WIDE2-1",
-        sender: "XE2CT-10",
-        lat: 19.12345,
-        lon: -99.54321,
-        has_position: true
-      }
-
-      assert {:ok, stored_packet} = Packets.store_packet(packet_data)
-      assert stored_packet.information_field == "Test with invalid byte:"
-      assert stored_packet.sender == "XE2CT-10"
-    end
-
-    test "handles invalid UTF-8 bytes in sender field" do
-      packet_data = %{
-        base_callsign: "TEST",
-        ssid: "1",
-        data_type: "position",
-        destination: "APDR15",
-        information_field: "Valid information",
-        path: "WIDE1-1,WIDE2-1",
-        sender: "TEST" <> <<0xB0>> <> "-1",
-        lat: 19.12345,
-        lon: -99.54321,
-        has_position: true
-      }
-
-      assert {:ok, stored_packet} = Packets.store_packet(packet_data)
-      assert stored_packet.sender == "TEST-1"
-      assert stored_packet.information_field == "Valid information"
-    end
-
-    test "handles invalid UTF-8 bytes in comment field" do
-      packet_data = %{
-        base_callsign: "TEST",
-        ssid: "1",
-        data_type: "position",
-        destination: "APDR15",
-        information_field: "Valid information",
-        path: "WIDE1-1,WIDE2-1",
-        sender: "TEST-1",
-        comment: "Comment with invalid bytes: " <> <<0xB0, 0xFF, 0x80>>,
-        lat: 19.12345,
-        lon: -99.54321,
-        has_position: true
-      }
-
-      assert {:ok, stored_packet} = Packets.store_packet(packet_data)
-      assert stored_packet.comment == "Comment with invalid bytes:"
-      assert stored_packet.sender == "TEST-1"
-    end
-
-    test "handles multiple invalid UTF-8 bytes across different fields" do
-      packet_data = %{
-        base_callsign: "TEST" <> <<0xB0>>,
-        ssid: "1",
-        data_type: "position",
-        destination: "APDR" <> <<0xFF>> <> "15",
-        information_field: "Info " <> <<0x80, 0x81>>,
-        path: "WIDE1-1" <> <<0xB0>> <> ",WIDE2-1",
-        sender: "TEST-1",
-        comment: "Comment " <> <<0xB0>>,
-        lat: 19.12345,
-        lon: -99.54321,
-        has_position: true
-      }
-
-      assert {:ok, stored_packet} = Packets.store_packet(packet_data)
-      assert stored_packet.base_callsign == "TEST"
-      assert stored_packet.destination == "APDR15"
-      assert stored_packet.information_field == "Info"
-      assert stored_packet.path == "WIDE1-1,WIDE2-1"
-      assert stored_packet.comment == "Comment"
-    end
-
     test "preserves valid UTF-8 characters" do
       packet_data = %{
         base_callsign: "TEST",
@@ -174,35 +93,13 @@ defmodule Aprs.PacketsEncodingTest do
       # Control characters should be filtered out
       assert stored_packet.information_field == "Control chars:"
     end
-
-    test "handles the specific 0xb0 byte that caused the original error" do
-      # This is the exact scenario from the error message
-      packet_data = %{
-        base_callsign: "XE2CT",
-        ssid: "10",
-        data_type: "position",
-        destination: "APDR15",
-        information_field: "Some text" <> <<0xB0>> <> "more text",
-        path: "WIDE1-1,WIDE2-1",
-        sender: "XE2CT-10",
-        lat: 19.12345,
-        lon: -99.54321,
-        has_position: true
-      }
-
-      # This should not raise a Postgrex.Error anymore
-      assert {:ok, stored_packet} = Packets.store_packet(packet_data)
-      assert stored_packet.sender == "XE2CT-10"
-      # The 0xb0 byte should be removed
-      assert stored_packet.information_field == "Some textmore text"
-    end
   end
 
   describe "EncodingUtils integration" do
     test "sanitize_string removes invalid UTF-8 sequences" do
       invalid_string = "Valid text" <> <<0xB0>> <> "more text"
       sanitized = EncodingUtils.sanitize_string(invalid_string)
-      assert sanitized == "Valid textmore text"
+      assert sanitized == "Valid text°more text"
     end
 
     test "sanitize_string preserves valid UTF-8" do

test/support/devices_seeder.exs

@@ -0,0 +1,41 @@
+# test/support/devices_seeder.exs
+
+alias Aprs.Repo
+alias Aprs.Devices
+
+defmodule DevicesSeeder do
+  def seed_from_json(path \\ "test/support/test_devices.json") do
+    {:ok, body} = File.read(path)
+    {:ok, json} = Jason.decode(body)
+    tocalls = Map.get(json, "tocalls", %{})
+    mice = Map.get(json, "mice", %{})
+    micelegacy = Map.get(json, "micelegacy", %{})
+    now = DateTime.utc_now()
+
+    Repo.delete_all(Devices)
+
+    [tocalls, mice, micelegacy]
+    |> Enum.each(fn group ->
+      Enum.each(group, fn {identifier, attrs} ->
+        attrs =
+          attrs
+          |> Map.put("identifier", identifier)
+          |> Map.update("features", nil, fn f ->
+            if is_list(f), do: f, else: [f]
+          end)
+          |> Map.put("updated_at", now)
+        %Devices{}
+        |> Devices.changeset(attrs)
+        |> Ecto.Changeset.apply_changes()
+        |> Map.from_struct()
+        |> then(fn map ->
+          Repo.insert!(
+            Devices.changeset(%Devices{}, map),
+            on_conflict: :replace_all,
+            conflict_target: :identifier
+          )
+        end)
+      end)
+    end)
+  end
+end

test/test_helper.exs

@@ -16,3 +16,5 @@ Application.put_env(:aprs, :aprs_is_default_filter, "r/0/0/1")
 Application.put_env(:aprs, :packets_module, Aprs.PacketsMock)
 
 # AprsIsMock is automatically loaded from test/support via elixirc_paths
+
+Code.require_file("support/devices_seeder.exs", __DIR__)