From b1ee1f92f9d83cf098822decfeb7aeb59222acac Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Thu, 10 Jul 2025 15:16:01 -0500 Subject: [PATCH] dont parse device identifiers on each insert --- assets/js/map.ts | 22 +-- lib/aprsme/application.ex | 2 + lib/aprsme/device_cache.ex | 130 ++++++++++++++++++ lib/aprsme/packet_consumer.ex | 4 +- .../controllers/api/v1/json/callsign_json.ex | 14 ++ lib/aprsme_web/live/shared/packet_handler.ex | 30 +--- 6 files changed, 153 insertions(+), 49 deletions(-) create mode 100644 lib/aprsme/device_cache.ex diff --git a/assets/js/map.ts b/assets/js/map.ts index 91c3f8c..94a726f 100644 --- a/assets/js/map.ts +++ b/assets/js/map.ts @@ -1319,25 +1319,9 @@ let MapAPRSMap = { } // For historical packets that are not the most recent for their callsign, - // still show the proper APRS symbol but with reduced opacity + // show a simple dot instead of the full APRS symbol if (data.historical && !data.is_most_recent_for_callsign) { - // Use server-generated symbol HTML if available - if (data.symbol_html) { - // Add opacity to the symbol HTML for historical markers - const historicalHtml = data.symbol_html.replace( - /style="([^"]*)"/, - 'style="$1 opacity: 0.7;"' - ); - - return L.divIcon({ - html: historicalHtml, - className: "historical-aprs-marker", - iconSize: [120, 32], - iconAnchor: [16, 16], - }); - } - - // Fallback: red dot for historical positions without symbol data + // Always show a red dot for historical positions const iconHtml = `
`; + " title="Historical position for ${data.callsign}">`; return L.divIcon({ html: iconHtml, diff --git a/lib/aprsme/application.ex b/lib/aprsme/application.ex index d57433c..546b8ad 100644 --- a/lib/aprsme/application.ex +++ b/lib/aprsme/application.ex @@ -31,6 +31,8 @@ defmodule Aprsme.Application do %{id: :symbol_cache, start: {Cachex, :start_link, [:symbol_cache, [limit: 1_000]]}}, # Start circuit breaker Aprsme.CircuitBreaker, + # Start device cache manager + Aprsme.DeviceCache, # Start the Endpoint (http/https) AprsmeWeb.Endpoint, diff --git a/lib/aprsme/device_cache.ex b/lib/aprsme/device_cache.ex new file mode 100644 index 0000000..88d6459 --- /dev/null +++ b/lib/aprsme/device_cache.ex @@ -0,0 +1,130 @@ +defmodule Aprsme.DeviceCache do + @moduledoc """ + Caches device information and provides efficient lookup by identifier with wildcard matching. + """ + + use GenServer + + alias Aprsme.Devices + alias Aprsme.Repo + + @cache_name :device_cache + @refresh_interval to_timeout(day: 1) + + # Client API + + @doc """ + Starts the DeviceCache GenServer. + """ + def start_link(_opts) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @doc """ + Looks up a device by identifier using wildcard matching. + Returns the device struct if found, or nil. + """ + def lookup_device(nil), do: nil + + def lookup_device(identifier) when is_binary(identifier) do + case Cachex.get(@cache_name, :all_devices) do + {:ok, nil} -> + # Cache miss - load devices + GenServer.call(__MODULE__, :refresh_cache) + lookup_device(identifier) + + {:ok, devices} when is_list(devices) -> + find_matching_device(devices, identifier) + + _ -> + nil + end + end + + @doc """ + Refreshes the device cache from the database. + """ + def refresh_cache do + GenServer.call(__MODULE__, :refresh_cache) + end + + # Server callbacks + + @impl true + def init(_) do + # Load devices on startup + load_devices_into_cache() + + # Schedule periodic refresh + Process.send_after(self(), :refresh_cache, @refresh_interval) + + {:ok, %{}} + end + + @impl true + def handle_call(:refresh_cache, _from, state) do + result = load_devices_into_cache() + {:reply, result, state} + end + + @impl true + def handle_info(:refresh_cache, state) do + load_devices_into_cache() + + # Schedule next refresh + Process.send_after(self(), :refresh_cache, @refresh_interval) + + {:noreply, state} + end + + # Private functions + + defp load_devices_into_cache do + devices = Repo.all(Devices) + + # Store all devices in cache + case Cachex.put(@cache_name, :all_devices, devices) do + {:ok, true} -> :ok + error -> error + end + end + + defp find_matching_device(devices, identifier) do + Enum.find(devices, fn device -> + pattern = device.identifier + + cond do + String.contains?(pattern, "?") -> + try do + regex = wildcard_pattern_to_regex(pattern) + Regex.match?(regex, identifier) + rescue + _e in Regex.CompileError -> + false + end + + String.contains?(pattern, "*") -> + # Compare literally if pattern contains * but not ? + pattern == identifier + + true -> + pattern == identifier + end + end) + end + + # Converts a pattern with ? wildcards to a regex + defp wildcard_pattern_to_regex(pattern) when is_binary(pattern) do + # Replace ? with a placeholder, escape all regex metacharacters except the placeholder, + # then replace placeholder with . + pattern + |> String.replace("?", "__WILDCARD__") + # Escape all regex metacharacters + |> String.replace(~r/([\\.\+\*\?\[\^\]\$\(\)\{\}=!<>\|:\-])/, "\\\\\\1") + |> String.replace("__WILDCARD__", ".") + |> then(fn s -> + regex = "^" <> s <> "$" + ~r/#{regex}/ + end) + end +end diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex index 720aecf..857a13f 100644 --- a/lib/aprsme/packet_consumer.ex +++ b/lib/aprsme/packet_consumer.ex @@ -209,9 +209,7 @@ defmodule Aprsme.PacketConsumer do |> normalize_ssid() |> then(fn attrs -> device_identifier = Aprsme.DeviceParser.extract_device_identifier(packet_data) - matched_device = Aprsme.DeviceIdentification.lookup_device_by_identifier(device_identifier) - canonical_identifier = if matched_device, do: matched_device.identifier, else: device_identifier - Map.put(attrs, :device_identifier, canonical_identifier) + Map.put(attrs, :device_identifier, device_identifier) end) |> sanitize_packet_strings() |> Map.put(:inserted_at, current_time) diff --git a/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex b/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex index 1c2b13a..c4d5242 100644 --- a/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex +++ b/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex @@ -3,6 +3,7 @@ defmodule AprsmeWeb.Api.V1.CallsignJSON do Renders callsign and packet data for API v1. """ + alias Aprsme.DeviceCache alias Aprsme.Packet def render("show.json", %{packet: packet}) do @@ -97,10 +98,23 @@ defmodule AprsmeWeb.Api.V1.CallsignJSON do end defp equipment_json(%Packet{} = packet) do + # Look up device info based on device_identifier + device = + case packet.device_identifier do + nil -> nil + "" -> nil + identifier -> DeviceCache.lookup_device(identifier) + end + equipment_data = %{} + |> maybe_add(:device_identifier, packet.device_identifier) |> maybe_add(:manufacturer, packet.manufacturer) |> maybe_add(:equipment_type, packet.equipment_type) + |> maybe_add(:device_model, device && device.model) + |> maybe_add(:device_vendor, device && device.vendor) + |> maybe_add(:device_contact, device && device.contact) + |> maybe_add(:device_class, device && device.class) case equipment_data do empty when empty == %{} -> nil diff --git a/lib/aprsme_web/live/shared/packet_handler.ex b/lib/aprsme_web/live/shared/packet_handler.ex index c010674..a492b8c 100644 --- a/lib/aprsme_web/live/shared/packet_handler.ex +++ b/lib/aprsme_web/live/shared/packet_handler.ex @@ -5,8 +5,7 @@ defmodule AprsmeWeb.Live.SharedPacketHandler do """ alias Aprsme.Callsign - alias Aprsme.DeviceIdentification - alias Aprsme.DeviceParser + alias Aprsme.DeviceCache alias Aprsme.EncodingUtils @doc """ @@ -67,13 +66,13 @@ defmodule AprsmeWeb.Live.SharedPacketHandler do Enriches packet with device information. """ def enrich_with_device_info(packet) do - device_identifier = extract_device_identifier(packet) + device_identifier = Map.get(packet, :device_identifier) || Map.get(packet, "device_identifier") device = case device_identifier do nil -> nil "" -> nil - identifier -> DeviceIdentification.lookup_device_by_identifier(identifier) + identifier -> DeviceCache.lookup_device(identifier) end packet @@ -100,27 +99,4 @@ defmodule AprsmeWeb.Live.SharedPacketHandler do packet_matches_callsign?(packet, callsign) end end - - # Private functions - - defp extract_device_identifier(packet) do - comment = Map.get(packet, :comment) || Map.get(packet, "comment") || "" - - normalized_comment = - case comment do - comment when is_binary(comment) -> - comment - |> String.trim() - |> String.replace(~r/[^\x20-\x7E]+/, "") - |> String.trim() - - _ -> - "" - end - - case DeviceParser.extract_device_identifier(%{comment: normalized_comment}) do - nil -> nil - identifier -> String.trim(identifier) - end - end end