better device parsing

This commit is contained in:
Graham McIntire 2025-06-23 14:25:19 -05:00
parent a48b2edcb3
commit 54b3bbf522
No known key found for this signature in database
8 changed files with 91 additions and 22 deletions

View file

@ -3,6 +3,11 @@ defmodule Aprs.DeviceIdentification do
Handles APRS device identification based on the APRS device identification database.
"""
import Ecto.Query
alias Aprs.Devices
alias Aprs.Repo
@device_patterns [
{~r/^ \x00\x00$/, "Original MIC-E"},
{~r/^>\x00\^$/, "Kenwood TH-D74"},
@ -33,6 +38,17 @@ defmodule Aprs.DeviceIdentification do
@doc """
Identifies the manufacturer and model of an APRS device based on its symbol pattern.
Returns a tuple of {manufacturer, model} or "Unknown" if the device cannot be identified.
## Examples
iex> Aprs.DeviceIdentification.identify_device(">" <> <<0>> <> "^")
"Kenwood TH-D74"
iex> Aprs.DeviceIdentification.identify_device("`_#")
"Yaesu VX-8G"
iex> Aprs.DeviceIdentification.identify_device("not-a-match")
"Unknown"
"""
@spec identify_device(String.t()) :: String.t()
def identify_device(symbols) do
@ -45,6 +61,14 @@ defmodule Aprs.DeviceIdentification do
@doc """
Returns a list of all known device manufacturers.
## Examples
iex> "Kenwood" in Aprs.DeviceIdentification.known_manufacturers()
true
iex> Enum.member?(Aprs.DeviceIdentification.known_manufacturers(), "AP510")
true
"""
@spec known_manufacturers() :: [String.t()]
def known_manufacturers do
@ -66,6 +90,14 @@ defmodule Aprs.DeviceIdentification do
@doc """
Returns a list of all known device models for a given manufacturer.
## Examples
iex> Aprs.DeviceIdentification.known_models("Kenwood")
["TH-D74", "TH-D74A", "DM-710", "DM-700"]
iex> Aprs.DeviceIdentification.known_models("Unknown")
[]
"""
@spec known_models(String.t()) :: [String.t()]
def known_models("Kenwood"), do: ["TH-D74", "TH-D74A", "DM-710", "DM-700"]
@ -76,20 +108,19 @@ defmodule Aprs.DeviceIdentification do
def known_models("SCS GmbH & Co."), do: ["P4dragon DR-7400 modems", "P4dragon DR-7800 modems"]
def known_models(_), do: []
alias Aprs.{Devices, Repo}
import Ecto.Query
@url "https://aprs-deviceid.aprsfoundation.org/tocalls.dense.json"
@week_seconds 7 * 24 * 60 * 60
def maybe_refresh_devices do
last = Repo.one(from d in Devices, order_by: [desc: d.updated_at], limit: 1)
last_time =
case last && last.updated_at do
%NaiveDateTime{} = naive -> DateTime.from_naive!(naive, "Etc/UTC")
%DateTime{} = dt -> dt
_ -> nil
end
if last == nil or (last_time && DateTime.diff(DateTime.utc_now(), last_time) > @week_seconds) do
fetch_and_upsert_devices()
else
@ -99,10 +130,12 @@ defmodule Aprs.DeviceIdentification do
def fetch_and_upsert_devices do
req = Finch.build(:get, @url)
case Finch.request(req, Aprs.Finch) do
{:ok, %Finch.Response{status: 200, body: body}} ->
{:ok, json} = Jason.decode(body)
upsert_devices(json)
err ->
err
end
@ -116,6 +149,7 @@ defmodule Aprs.DeviceIdentification do
Repo.transaction(fn ->
Repo.delete_all(Devices)
Enum.each([tocalls, mice, micelegacy], fn group ->
Enum.each(group, fn {identifier, attrs} ->
attrs =
@ -125,10 +159,12 @@ defmodule Aprs.DeviceIdentification do
if is_list(f), do: f, else: [f]
end)
|> Map.put("updated_at", now)
%Devices{} |> Devices.changeset(attrs) |> Repo.insert!()
end)
end)
end)
:ok
end
@ -140,12 +176,16 @@ defmodule Aprs.DeviceIdentification do
@doc """
Looks up a device by identifier, using ? as a single-character wildcard.
Returns the device struct if found, or nil.
Note: This function requires the devices table to be seeded and a running Repo context.
"""
def lookup_device_by_identifier(identifier) when is_binary(identifier) do
# Fetch all device patterns from DB
devices = Repo.all(Devices)
Enum.find(devices, fn device ->
pattern = device.identifier
cond do
String.contains?(pattern, "?") ->
try do
@ -155,9 +195,11 @@ defmodule Aprs.DeviceIdentification do
_e in Regex.CompileError ->
false
end
String.contains?(pattern, "*") ->
# Compare literally if pattern contains * but not ?
pattern == identifier
true ->
pattern == identifier
end
@ -177,6 +219,7 @@ defmodule Aprs.DeviceIdentification do
end
defmodule Aprs.DeviceIdentification.Worker do
@moduledoc false
use Oban.Worker, queue: :default, max_attempts: 1
@impl true

View file

@ -1,5 +1,7 @@
defmodule Aprs.Devices do
@moduledoc false
use Ecto.Schema
import Ecto.Changeset
schema "devices" do

View file

@ -28,21 +28,22 @@ defmodule Aprs.Packets do
|> normalize_packet_attrs()
|> set_received_at()
|> patch_lat_lon_from_data_extended()
|> (fn attrs ->
|> then(fn attrs ->
{lat, lon} = extract_position(attrs)
set_lat_lon(attrs, lat, lon)
end).()
end)
|> normalize_ssid()
|> (fn attrs ->
|> then(fn attrs ->
device_identifier = Parser.DeviceParser.extract_device_identifier(packet_data)
matched_device = Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier)
canonical_identifier = if matched_device, do: matched_device.identifier, else: device_identifier
Map.put(attrs, :device_identifier, canonical_identifier)
end).()
end)
|> sanitize_packet_strings()
# require Logger
# Logger.debug("Sanitized packet_attrs before insert: #{inspect(packet_attrs)}")
packet_attrs = Enum.into(packet_attrs, %{}, fn {k, v} -> {k, sanitize_packet_strings(v)} end)
packet_attrs = Map.new(packet_attrs, fn {k, v} -> {k, sanitize_packet_strings(v)} end)
insert_packet(packet_attrs, packet_data)
rescue
error ->
@ -530,10 +531,12 @@ defmodule Aprs.Packets do
defp sanitize_packet_strings(%NaiveDateTime{} = ndt), do: ndt
defp sanitize_packet_strings(%_struct{} = struct), do: struct |> Map.from_struct() |> sanitize_packet_strings()
defp sanitize_packet_strings(list) when is_list(list), do: Enum.map(list, &sanitize_packet_strings/1)
defp sanitize_packet_strings(binary) when is_binary(binary) do
s = Aprs.EncodingUtils.sanitize_string(binary)
if is_binary(s), do: s, else: ""
end
defp sanitize_packet_strings(other), do: other
# Get packets from last hour only - used to initialize the map

View file

@ -43,7 +43,9 @@
alt="APRS symbol"
/>
<% end %>
<.link navigate={~p"/packets/#{@callsign}"} class="ml-2 text-sm text-blue-600 hover:underline">View packets</.link>
<.link navigate={~p"/packets/#{@callsign}"} class="ml-2 text-sm text-blue-600 hover:underline">
View packets
</.link>
</h1>
<%= if @packet do %>
<div class="text-sm text-gray-500 mb-6">

View file

@ -58,10 +58,12 @@ defmodule AprsWeb.PacketsLive.CallsignView do
# Handle incoming live packets - only process if they match our callsign
if packet_matches_callsign?(payload, socket.assigns.callsign) do
sanitized_payload = EncodingUtils.sanitize_packet(payload)
device_identifier =
Map.get(sanitized_payload, :device_identifier) ||
Map.get(sanitized_payload, "device_identifier") ||
DeviceParser.extract_device_identifier(sanitized_payload)
Map.get(sanitized_payload, "device_identifier") ||
DeviceParser.extract_device_identifier(sanitized_payload)
canonical_identifier =
if is_binary(device_identifier) do
matched_device = Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier)
@ -69,6 +71,7 @@ defmodule AprsWeb.PacketsLive.CallsignView do
else
device_identifier
end
sanitized_payload = Map.put(sanitized_payload, :device_identifier, canonical_identifier)
# Enrich with model/vendor
enriched_payload = enrich_packet_with_device_info(sanitized_payload)
@ -205,9 +208,10 @@ defmodule AprsWeb.PacketsLive.CallsignView do
defp enrich_packet_with_device_info(packet) do
device_identifier = Map.get(packet, :device_identifier) || Map.get(packet, "device_identifier")
device = if is_binary(device_identifier), do: Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier), else: nil
model = if device, do: device.model, else: nil
vendor = if device, do: device.vendor, else: nil
device = if is_binary(device_identifier), do: Aprs.DeviceIdentification.lookup_device_by_identifier(device_identifier)
model = if device, do: device.model
vendor = if device, do: device.vendor
packet
|> Map.put(:device_model, model)
|> Map.put(:device_vendor, vendor)

View file

@ -66,9 +66,9 @@
<:col :let={packet} label="Device">
<span class="text-xs text-gray-700 font-mono">
<%= if packet.device_model || packet.device_vendor do %>
<%= [packet.device_model, packet.device_vendor]
|> Enum.reject(&is_nil/1)
|> Enum.join(" ") %>
{[packet.device_model, packet.device_vendor]
|> Enum.reject(&is_nil/1)
|> Enum.join(" ")}
<% else %>
{Map.get(packet, :device_identifier, Map.get(packet, "device_identifier", ""))}
<% end %>

View file

@ -1,9 +1,12 @@
defmodule Aprs.DeviceIdentificationTest do
use Aprs.DataCase, async: false
alias Aprs.DeviceIdentification
import DevicesSeeder
alias Aprs.DeviceIdentification
doctest Aprs.DeviceIdentification
describe "identify_device/1" do
test "identifies Original MIC-E devices" do
assert DeviceIdentification.identify_device(" " <> <<0, 0>>) == "Original MIC-E"
@ -103,5 +106,17 @@ defmodule Aprs.DeviceIdentificationTest do
assert found.model != nil
assert found.vendor != nil
end
test "matches Mic-E device identifier from raw packet" do
# Seed the devices table from the JSON
seed_from_json()
# The device identifier extracted from the raw packet is "]="
found = Aprs.DeviceIdentification.lookup_device_by_identifier("]=")
assert found != nil
assert found.identifier == "]="
assert found.model != nil
assert found.vendor != nil
end
end
end

View file

@ -1,9 +1,9 @@
# test/support/devices_seeder.exs
alias Aprs.Repo
alias Aprs.Devices
alias Aprs.Repo
defmodule DevicesSeeder do
@moduledoc false
def seed_from_json(path \\ "test/support/test_devices.json") do
{:ok, body} = File.read(path)
{:ok, json} = Jason.decode(body)
@ -14,8 +14,7 @@ defmodule DevicesSeeder do
Repo.delete_all(Devices)
[tocalls, mice, micelegacy]
|> Enum.each(fn group ->
Enum.each([tocalls, mice, micelegacy], fn group ->
Enum.each(group, fn {identifier, attrs} ->
attrs =
attrs
@ -24,6 +23,7 @@ defmodule DevicesSeeder do
if is_list(f), do: f, else: [f]
end)
|> Map.put("updated_at", now)
%Devices{}
|> Devices.changeset(attrs)
|> Ecto.Changeset.apply_changes()