better device parsing

This commit is contained in:
Graham McIntire 2025-06-23 14:25:19 -05:00
parent a48b2edcb3
commit 54b3bbf522
No known key found for this signature in database
8 changed files with 91 additions and 22 deletions

View file

@ -3,6 +3,11 @@ defmodule Aprs.DeviceIdentification do
Handles APRS device identification based on the APRS device identification database. Handles APRS device identification based on the APRS device identification database.
""" """
import Ecto.Query
alias Aprs.Devices
alias Aprs.Repo
@device_patterns [ @device_patterns [
{~r/^ \x00\x00$/, "Original MIC-E"}, {~r/^ \x00\x00$/, "Original MIC-E"},
{~r/^>\x00\^$/, "Kenwood TH-D74"}, {~r/^>\x00\^$/, "Kenwood TH-D74"},
@ -33,6 +38,17 @@ defmodule Aprs.DeviceIdentification do
@doc """ @doc """
Identifies the manufacturer and model of an APRS device based on its symbol pattern. Identifies the manufacturer and model of an APRS device based on its symbol pattern.
Returns a tuple of {manufacturer, model} or "Unknown" if the device cannot be identified. Returns a tuple of {manufacturer, model} or "Unknown" if the device cannot be identified.
## Examples
iex> Aprs.DeviceIdentification.identify_device(">" <> <<0>> <> "^")
"Kenwood TH-D74"
iex> Aprs.DeviceIdentification.identify_device("`_#")
"Yaesu VX-8G"
iex> Aprs.DeviceIdentification.identify_device("not-a-match")
"Unknown"
""" """
@spec identify_device(String.t()) :: String.t() @spec identify_device(String.t()) :: String.t()
def identify_device(symbols) do def identify_device(symbols) do
@ -45,6 +61,14 @@ defmodule Aprs.DeviceIdentification do
@doc """ @doc """
Returns a list of all known device manufacturers. Returns a list of all known device manufacturers.
## Examples
iex> "Kenwood" in Aprs.DeviceIdentification.known_manufacturers()
true
iex> Enum.member?(Aprs.DeviceIdentification.known_manufacturers(), "AP510")
true
""" """
@spec known_manufacturers() :: [String.t()] @spec known_manufacturers() :: [String.t()]
def known_manufacturers do def known_manufacturers do
@ -66,6 +90,14 @@ defmodule Aprs.DeviceIdentification do
@doc """ @doc """
Returns a list of all known device models for a given manufacturer. Returns a list of all known device models for a given manufacturer.
## Examples
iex> Aprs.DeviceIdentification.known_models("Kenwood")
["TH-D74", "TH-D74A", "DM-710", "DM-700"]
iex> Aprs.DeviceIdentification.known_models("Unknown")
[]
""" """
@spec known_models(String.t()) :: [String.t()] @spec known_models(String.t()) :: [String.t()]
def known_models("Kenwood"), do: ["TH-D74", "TH-D74A", "DM-710", "DM-700"] def known_models("Kenwood"), do: ["TH-D74", "TH-D74A", "DM-710", "DM-700"]
@ -76,20 +108,19 @@ defmodule Aprs.DeviceIdentification do
def known_models("SCS GmbH & Co."), do: ["P4dragon DR-7400 modems", "P4dragon DR-7800 modems"] def known_models("SCS GmbH & Co."), do: ["P4dragon DR-7400 modems", "P4dragon DR-7800 modems"]
def known_models(_), do: [] def known_models(_), do: []
alias Aprs.{Devices, Repo}
import Ecto.Query
@url "https://aprs-deviceid.aprsfoundation.org/tocalls.dense.json" @url "https://aprs-deviceid.aprsfoundation.org/tocalls.dense.json"
@week_seconds 7 * 24 * 60 * 60 @week_seconds 7 * 24 * 60 * 60
def maybe_refresh_devices do def maybe_refresh_devices do
last = Repo.one(from d in Devices, order_by: [desc: d.updated_at], limit: 1) last = Repo.one(from d in Devices, order_by: [desc: d.updated_at], limit: 1)
last_time = last_time =
case last && last.updated_at do case last && last.updated_at do
%NaiveDateTime{} = naive -> DateTime.from_naive!(naive, "Etc/UTC") %NaiveDateTime{} = naive -> DateTime.from_naive!(naive, "Etc/UTC")
%DateTime{} = dt -> dt %DateTime{} = dt -> dt
_ -> nil _ -> nil
end end
if last == nil or (last_time && DateTime.diff(DateTime.utc_now(), last_time) > @week_seconds) do if last == nil or (last_time && DateTime.diff(DateTime.utc_now(), last_time) > @week_seconds) do
fetch_and_upsert_devices() fetch_and_upsert_devices()
else else
@ -99,10 +130,12 @@ defmodule Aprs.DeviceIdentification do
def fetch_and_upsert_devices do def fetch_and_upsert_devices do
req = Finch.build(:get, @url) req = Finch.build(:get, @url)
case Finch.request(req, Aprs.Finch) do case Finch.request(req, Aprs.Finch) do
{:ok, %Finch.Response{status: 200, body: body}} -> {:ok, %Finch.Response{status: 200, body: body}} ->
{:ok, json} = Jason.decode(body) {:ok, json} = Jason.decode(body)
upsert_devices(json) upsert_devices(json)
err -> err ->
err err
end end
@ -116,6 +149,7 @@ defmodule Aprs.DeviceIdentification do
Repo.transaction(fn -> Repo.transaction(fn ->
Repo.delete_all(Devices) Repo.delete_all(Devices)
Enum.each([tocalls, mice, micelegacy], fn group -> Enum.each([tocalls, mice, micelegacy], fn group ->
Enum.each(group, fn {identifier, attrs} -> Enum.each(group, fn {identifier, attrs} ->
attrs = attrs =
@ -125,10 +159,12 @@ defmodule Aprs.DeviceIdentification do
if is_list(f), do: f, else: [f] if is_list(f), do: f, else: [f]
end) end)
|> Map.put("updated_at", now) |> Map.put("updated_at", now)
%Devices{} |> Devices.changeset(attrs) |> Repo.insert!() %Devices{} |> Devices.changeset(attrs) |> Repo.insert!()
end) end)
end) end)
end) end)
:ok :ok
end end
@ -140,12 +176,16 @@ defmodule Aprs.DeviceIdentification do
@doc """ @doc """
Looks up a device by identifier, using ? as a single-character wildcard. Looks up a device by identifier, using ? as a single-character wildcard.
Returns the device struct if found, or nil. Returns the device struct if found, or nil.
Note: This function requires the devices table to be seeded and a running Repo context.
""" """
def lookup_device_by_identifier(identifier) when is_binary(identifier) do def lookup_device_by_identifier(identifier) when is_binary(identifier) do
# Fetch all device patterns from DB # Fetch all device patterns from DB
devices = Repo.all(Devices) devices = Repo.all(Devices)
Enum.find(devices, fn device -> Enum.find(devices, fn device ->
pattern = device.identifier pattern = device.identifier
cond do cond do
String.contains?(pattern, "?") -> String.contains?(pattern, "?") ->
try do try do
@ -155,9 +195,11 @@ defmodule Aprs.DeviceIdentification do
_e in Regex.CompileError -> _e in Regex.CompileError ->
false false
end end
String.contains?(pattern, "*") -> String.contains?(pattern, "*") ->
# Compare literally if pattern contains * but not ? # Compare literally if pattern contains * but not ?
pattern == identifier pattern == identifier
true -> true ->
pattern == identifier pattern == identifier
end end
@ -177,6 +219,7 @@ defmodule Aprs.DeviceIdentification do
end end
defmodule Aprs.DeviceIdentification.Worker do defmodule Aprs.DeviceIdentification.Worker do
@moduledoc false
use Oban.Worker, queue: :default, max_attempts: 1 use Oban.Worker, queue: :default, max_attempts: 1
@impl true @impl true

View file

@ -1,5 +1,7 @@
defmodule Aprs.Devices do defmodule Aprs.Devices do
@moduledoc false
use Ecto.Schema use Ecto.Schema
import Ecto.Changeset import Ecto.Changeset
schema "devices" do schema "devices" do

View file

@ -28,21 +28,22 @@ defmodule Aprs.Packets do
|> normalize_packet_attrs() |> normalize_packet_attrs()
|> set_received_at() |> set_received_at()
|> patch_lat_lon_from_data_extended() |> patch_lat_lon_from_data_extended()
|> (fn attrs -> |> then(fn attrs ->
{lat, lon} = extract_position(attrs) {lat, lon} = extract_position(attrs)
set_lat_lon(attrs, lat, lon) set_lat_lon(attrs, lat, lon)
end).() end)
|> normalize_ssid() |> normalize_ssid()
|> (fn attrs -> |> then(fn attrs ->
device_identifier = Parser.DeviceParser.extract_device_identifier(packet_data) device_identifier = Parser.DeviceParser.extract_device_identifier(packet_data)
matched_device = Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier) matched_device = Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier)
canonical_identifier = if matched_device, do: matched_device.identifier, else: device_identifier canonical_identifier = if matched_device, do: matched_device.identifier, else: device_identifier
Map.put(attrs, :device_identifier, canonical_identifier) Map.put(attrs, :device_identifier, canonical_identifier)
end).() end)
|> sanitize_packet_strings() |> sanitize_packet_strings()
# require Logger # require Logger
# Logger.debug("Sanitized packet_attrs before insert: #{inspect(packet_attrs)}") # Logger.debug("Sanitized packet_attrs before insert: #{inspect(packet_attrs)}")
packet_attrs = Enum.into(packet_attrs, %{}, fn {k, v} -> {k, sanitize_packet_strings(v)} end) packet_attrs = Map.new(packet_attrs, fn {k, v} -> {k, sanitize_packet_strings(v)} end)
insert_packet(packet_attrs, packet_data) insert_packet(packet_attrs, packet_data)
rescue rescue
error -> error ->
@ -530,10 +531,12 @@ defmodule Aprs.Packets do
defp sanitize_packet_strings(%NaiveDateTime{} = ndt), do: ndt defp sanitize_packet_strings(%NaiveDateTime{} = ndt), do: ndt
defp sanitize_packet_strings(%_struct{} = struct), do: struct |> Map.from_struct() |> sanitize_packet_strings() defp sanitize_packet_strings(%_struct{} = struct), do: struct |> Map.from_struct() |> sanitize_packet_strings()
defp sanitize_packet_strings(list) when is_list(list), do: Enum.map(list, &sanitize_packet_strings/1) defp sanitize_packet_strings(list) when is_list(list), do: Enum.map(list, &sanitize_packet_strings/1)
defp sanitize_packet_strings(binary) when is_binary(binary) do defp sanitize_packet_strings(binary) when is_binary(binary) do
s = Aprs.EncodingUtils.sanitize_string(binary) s = Aprs.EncodingUtils.sanitize_string(binary)
if is_binary(s), do: s, else: "" if is_binary(s), do: s, else: ""
end end
defp sanitize_packet_strings(other), do: other defp sanitize_packet_strings(other), do: other
# Get packets from last hour only - used to initialize the map # Get packets from last hour only - used to initialize the map

View file

@ -43,7 +43,9 @@
alt="APRS symbol" alt="APRS symbol"
/> />
<% end %> <% end %>
<.link navigate={~p"/packets/#{@callsign}"} class="ml-2 text-sm text-blue-600 hover:underline">View packets</.link> <.link navigate={~p"/packets/#{@callsign}"} class="ml-2 text-sm text-blue-600 hover:underline">
View packets
</.link>
</h1> </h1>
<%= if @packet do %> <%= if @packet do %>
<div class="text-sm text-gray-500 mb-6"> <div class="text-sm text-gray-500 mb-6">

View file

@ -58,10 +58,12 @@ defmodule AprsWeb.PacketsLive.CallsignView do
# Handle incoming live packets - only process if they match our callsign # Handle incoming live packets - only process if they match our callsign
if packet_matches_callsign?(payload, socket.assigns.callsign) do if packet_matches_callsign?(payload, socket.assigns.callsign) do
sanitized_payload = EncodingUtils.sanitize_packet(payload) sanitized_payload = EncodingUtils.sanitize_packet(payload)
device_identifier = device_identifier =
Map.get(sanitized_payload, :device_identifier) || Map.get(sanitized_payload, :device_identifier) ||
Map.get(sanitized_payload, "device_identifier") || Map.get(sanitized_payload, "device_identifier") ||
DeviceParser.extract_device_identifier(sanitized_payload) DeviceParser.extract_device_identifier(sanitized_payload)
canonical_identifier = canonical_identifier =
if is_binary(device_identifier) do if is_binary(device_identifier) do
matched_device = Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier) matched_device = Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier)
@ -69,6 +71,7 @@ defmodule AprsWeb.PacketsLive.CallsignView do
else else
device_identifier device_identifier
end end
sanitized_payload = Map.put(sanitized_payload, :device_identifier, canonical_identifier) sanitized_payload = Map.put(sanitized_payload, :device_identifier, canonical_identifier)
# Enrich with model/vendor # Enrich with model/vendor
enriched_payload = enrich_packet_with_device_info(sanitized_payload) enriched_payload = enrich_packet_with_device_info(sanitized_payload)
@ -205,9 +208,10 @@ defmodule AprsWeb.PacketsLive.CallsignView do
defp enrich_packet_with_device_info(packet) do defp enrich_packet_with_device_info(packet) do
device_identifier = Map.get(packet, :device_identifier) || Map.get(packet, "device_identifier") device_identifier = Map.get(packet, :device_identifier) || Map.get(packet, "device_identifier")
device = if is_binary(device_identifier), do: Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier), else: nil device = if is_binary(device_identifier), do: Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier)
model = if device, do: device.model, else: nil model = if device, do: device.model
vendor = if device, do: device.vendor, else: nil vendor = if device, do: device.vendor
packet packet
|> Map.put(:device_model, model) |> Map.put(:device_model, model)
|> Map.put(:device_vendor, vendor) |> Map.put(:device_vendor, vendor)

View file

@ -66,9 +66,9 @@
<:col :let={packet} label="Device"> <:col :let={packet} label="Device">
<span class="text-xs text-gray-700 font-mono"> <span class="text-xs text-gray-700 font-mono">
<%= if packet.device_model || packet.device_vendor do %> <%= if packet.device_model || packet.device_vendor do %>
<%= [packet.device_model, packet.device_vendor] {[packet.device_model, packet.device_vendor]
|> Enum.reject(&is_nil/1) |> Enum.reject(&is_nil/1)
|> Enum.join(" ") %> |> Enum.join(" ")}
<% else %> <% else %>
{Map.get(packet, :device_identifier, Map.get(packet, "device_identifier", ""))} {Map.get(packet, :device_identifier, Map.get(packet, "device_identifier", ""))}
<% end %> <% end %>

View file

@ -1,9 +1,12 @@
defmodule Aprs.DeviceIdentificationTest do defmodule Aprs.DeviceIdentificationTest do
use Aprs.DataCase, async: false use Aprs.DataCase, async: false
alias Aprs.DeviceIdentification
import DevicesSeeder import DevicesSeeder
alias Aprs.DeviceIdentification
doctest Aprs.DeviceIdentification
describe "identify_device/1" do describe "identify_device/1" do
test "identifies Original MIC-E devices" do test "identifies Original MIC-E devices" do
assert DeviceIdentification.identify_device(" " <> <<0, 0>>) == "Original MIC-E" assert DeviceIdentification.identify_device(" " <> <<0, 0>>) == "Original MIC-E"
@ -103,5 +106,17 @@ defmodule Aprs.DeviceIdentificationTest do
assert found.model != nil assert found.model != nil
assert found.vendor != nil assert found.vendor != nil
end end
test "matches Mic-E device identifier from raw packet" do
# Seed the devices table from the JSON
seed_from_json()
# The device identifier extracted from the raw packet is "]="
found = Aprs.DeviceIdentification.lookup_device_by_identifier("]=")
assert found != nil
assert found.identifier == "]="
assert found.model != nil
assert found.vendor != nil
end
end end
end end

View file

@ -1,9 +1,9 @@
# test/support/devices_seeder.exs # test/support/devices_seeder.exs
alias Aprs.Repo
alias Aprs.Devices alias Aprs.Devices
alias Aprs.Repo
defmodule DevicesSeeder do defmodule DevicesSeeder do
@moduledoc false
def seed_from_json(path \\ "test/support/test_devices.json") do def seed_from_json(path \\ "test/support/test_devices.json") do
{:ok, body} = File.read(path) {:ok, body} = File.read(path)
{:ok, json} = Jason.decode(body) {:ok, json} = Jason.decode(body)
@ -14,8 +14,7 @@ defmodule DevicesSeeder do
Repo.delete_all(Devices) Repo.delete_all(Devices)
[tocalls, mice, micelegacy] Enum.each([tocalls, mice, micelegacy], fn group ->
|> Enum.each(fn group ->
Enum.each(group, fn {identifier, attrs} -> Enum.each(group, fn {identifier, attrs} ->
attrs = attrs =
attrs attrs
@ -24,6 +23,7 @@ defmodule DevicesSeeder do
if is_list(f), do: f, else: [f] if is_list(f), do: f, else: [f]
end) end)
|> Map.put("updated_at", now) |> Map.put("updated_at", now)
%Devices{} %Devices{}
|> Devices.changeset(attrs) |> Devices.changeset(attrs)
|> Ecto.Changeset.apply_changes() |> Ecto.Changeset.apply_changes()