From ecee5fd6775e4b495f135a406f651ac1c64bc2ca Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Fri, 20 Jun 2025 14:09:43 -0500 Subject: [PATCH] major refactor and fixes with streaming --- lib/aprs/aprs_is_connection.ex | 5 + lib/aprs/data_extended.ex | 12 +- lib/aprs/is/is.ex | 5 + lib/aprs/packet.ex | 17 +- lib/aprs/packets.ex | 142 +++++-- lib/aprs_web/live/map_live/callsign_view.ex | 143 +++---- lib/aprs_web/live/map_live/index.ex | 57 +-- lib/aprs_web/live/packets_live/index.ex | 9 + .../live/packets_live/index.html.heex | 70 +++- lib/parser.ex | 363 +++++++++++++----- lib/parser/helpers.ex | 31 +- lib/parser/item.ex | 14 +- lib/parser/position.ex | 18 +- lib/parser/types.ex | 14 +- ...4_change_lat_lon_to_decimal_in_packets.exs | 10 + test/aprs/geometry_test.exs | 4 +- test/aprs/packets_mic_e_test.exs | 12 +- .../live/map_live/callsign_view_test.exs | 39 -- test/parser/helpers_test.exs | 4 +- test/parser/parser_test.exs | 56 ++- test/parser/position_test.exs | 4 +- test/parser/property_test.exs | 8 + test/parser/types_test.exs | 15 +- test/types/position_test.exs | 8 +- 24 files changed, 713 insertions(+), 347 deletions(-) create mode 100644 priv/repo/migrations/20250620182604_change_lat_lon_to_decimal_in_packets.exs diff --git a/lib/aprs/aprs_is_connection.ex b/lib/aprs/aprs_is_connection.ex index cbfe4b1..6ecf3ce 100644 --- a/lib/aprs/aprs_is_connection.ex +++ b/lib/aprs/aprs_is_connection.ex @@ -120,4 +120,9 @@ defmodule Aprs.AprsIsConnection do error end end + + @impl true + def code_change(_old_vsn, state, _extra) do + {:ok, state} + end end diff --git a/lib/aprs/data_extended.ex b/lib/aprs/data_extended.ex index d37246e..01ad076 100644 --- a/lib/aprs/data_extended.ex +++ b/lib/aprs/data_extended.ex @@ -35,10 +35,18 @@ defmodule Aprs.DataExtended do :aprs_messaging, :comment, :data_type, - :latitude, - :longitude, :symbol_code, :symbol_table_id ]) + |> validate_required_if_present(:latitude) + |> validate_required_if_present(:longitude) + end + + defp validate_required_if_present(changeset, field) do + if Map.has_key?(changeset.changes, field) do + validate_required(changeset, [field]) + else + changeset + end end end diff --git a/lib/aprs/is/is.ex b/lib/aprs/is/is.ex index e6e33be..bd4cde3 100644 --- a/lib/aprs/is/is.ex +++ b/lib/aprs/is/is.ex @@ -369,6 +369,11 @@ defmodule Aprs.Is do :normal end + @impl true + def code_change(_old_vsn, state, _extra) do + {:ok, state} + end + @spec dispatch(binary) :: nil | :ok def dispatch("#" <> comment_text) do Logger.debug("COMMENT: " <> String.trim(comment_text)) diff --git a/lib/aprs/packet.ex b/lib/aprs/packet.ex index 6c52810..b6a7f8e 100644 --- a/lib/aprs/packet.ex +++ b/lib/aprs/packet.ex @@ -17,8 +17,8 @@ defmodule Aprs.Packet do field(:ssid, :string) field(:received_at, :utc_datetime_usec) field(:region, :string) - field(:lat, :float, virtual: true) - field(:lon, :float, virtual: true) + field(:lat, :decimal) + field(:lon, :decimal) field(:location, Geo.PostGIS.Geometry) field(:has_position, :boolean, default: false) @@ -185,6 +185,9 @@ defmodule Aprs.Packet do end defp valid_coordinates?(lat, lon) do + lat = if is_struct(lat, Decimal), do: Decimal.to_float(lat), else: lat + lon = if is_struct(lon, Decimal), do: Decimal.to_float(lon), else: lon + is_number(lat) && is_number(lon) && lat >= -90 && lat <= 90 && lon >= -180 && lon <= 180 @@ -309,6 +312,8 @@ defmodule Aprs.Packet do # Extract data from MicE packets defp extract_from_mic_e(mic_e) do %{} + |> maybe_put(:lat, mic_e[:latitude]) + |> maybe_put(:lon, mic_e[:longitude]) |> maybe_put(:comment, mic_e.message) |> maybe_put(:manufacturer, mic_e.manufacturer) |> maybe_put(:course, mic_e.heading) @@ -320,6 +325,8 @@ defmodule Aprs.Packet do # Extract data from converted MicE map (from struct_to_map conversion) defp extract_from_mic_e_map(mic_e_map) do %{} + |> maybe_put(:lat, mic_e_map[:latitude]) + |> maybe_put(:lon, mic_e_map[:longitude]) |> maybe_put(:comment, mic_e_map[:message]) |> maybe_put(:manufacturer, mic_e_map[:manufacturer]) |> maybe_put(:course, mic_e_map[:heading]) @@ -476,7 +483,11 @@ defmodule Aprs.Packet do Create a geometry point from lat/lon coordinates. """ @spec create_point(number() | nil, number() | nil) :: Geo.Point.t() | nil - def create_point(lat, lon) when is_number(lat) and is_number(lon) do + def create_point(lat, lon) + when (is_number(lat) or is_struct(lat, Decimal)) and (is_number(lon) or is_struct(lon, Decimal)) do + lat = if is_struct(lat, Decimal), do: Decimal.to_float(lat), else: lat + lon = if is_struct(lon, Decimal), do: Decimal.to_float(lon), else: lon + if valid_coordinates?(lat, lon) do %Geo.Point{coordinates: {lon, lat}, srid: 4326} end diff --git a/lib/aprs/packets.ex b/lib/aprs/packets.ex index e833b2b..cde5c43 100644 --- a/lib/aprs/packets.ex +++ b/lib/aprs/packets.ex @@ -9,6 +9,7 @@ defmodule Aprs.Packets do alias Aprs.EncodingUtils alias Aprs.Packet alias Aprs.Repo + alias Parser.Types.MicE @doc """ Stores a packet in the database. @@ -50,38 +51,80 @@ defmodule Aprs.Packets do current_time = DateTime.truncate(DateTime.utc_now(), :microsecond) packet_attrs = Map.put(packet_attrs, :received_at, current_time) + # Patch: If data_extended has latitude/longitude, set them as lat/lon as Decimal + packet_attrs = + case packet_attrs[:data_extended] do + %{} = ext -> + # Convert struct to map if needed + ext_map = if Map.has_key?(ext, :__struct__), do: Map.from_struct(ext), else: ext + + lat = + ext_map[:latitude] || ext_map["latitude"] || + (Map.has_key?(ext_map, :position) && + (ext_map[:position][:latitude] || ext_map[:position]["latitude"])) || + (Map.has_key?(ext_map, "position") && + (ext_map["position"][:latitude] || ext_map["position"]["latitude"])) + + lon = + ext_map[:longitude] || ext_map["longitude"] || + (Map.has_key?(ext_map, :position) && + (ext_map[:position][:longitude] || ext_map[:position]["longitude"])) || + (Map.has_key?(ext_map, "position") && + (ext_map["position"][:longitude] || ext_map["position"]["longitude"])) + + latd = to_decimal(lat) + lond = to_decimal(lon) + + if not is_nil(latd) and not is_nil(lond) do + packet_attrs + |> Map.put(:lat, latd) + |> Map.put(:lon, lond) + else + packet_attrs + end + + _ -> + packet_attrs + end + # Extract position data with error handling {lat, lon} = extract_position(packet_attrs) - # Set position fields if found - packet_attrs = - if lat && lon do - # Validate coordinates before creating geometry - if are_valid_coordinates?(lat, lon) do - packet_attrs - |> Map.put(:lat, lat) - |> Map.put(:lon, lon) - |> Map.put(:has_position, true) - |> Map.put(:region, "#{Float.round(lat, 1)},#{Float.round(lon, 1)}") - else - Logger.debug("Invalid coordinates for packet from #{packet_attrs[:sender]}: lat=#{lat}, lon=#{lon}") + # Helper to round to 6 decimal places + round6 = fn + nil -> + nil - # Set region based on callsign if coordinates are invalid - sender_region = - if packet_attrs[:sender], - do: String.slice(packet_attrs.sender || "", 0, 3), - else: "unknown" + %Decimal{} = d -> + Decimal.round(d, 6) - Map.put(packet_attrs, :region, "call:#{sender_region}") + n when is_float(n) -> + Float.round(n, 6) + + n when is_integer(n) -> + n * 1.0 + + n when is_binary(n) -> + case Float.parse(n) do + {f, _} -> Float.round(f, 6) + :error -> nil end - else - # Set region based on callsign if no position - sender_region = - if packet_attrs[:sender], - do: String.slice(packet_attrs.sender || "", 0, 3), - else: "unknown" - Map.put(packet_attrs, :region, "call:#{sender_region}") + _ -> + nil + end + + # Set position fields if found (this will overwrite above if extract_position finds something different) + packet_attrs = + packet_attrs + |> Map.put(:lat, round6.(lat)) + |> Map.put(:lon, round6.(lon)) + + # Normalize ssid to string if present and not nil + packet_attrs = + case Map.get(packet_attrs, :ssid) do + nil -> packet_attrs + ssid -> Map.put(packet_attrs, :ssid, to_string(ssid)) end # Insert the packet @@ -89,16 +132,6 @@ defmodule Aprs.Packets do |> Packet.changeset(packet_attrs) |> Repo.insert() do {:ok, packet} -> - symbol_table = - Map.get(packet_attrs, :symbol_table_id) || - get_in(packet_attrs, [:data_extended, :symbol_table_id]) || "/" - - symbol_code = - Map.get(packet_attrs, :symbol_code) || - get_in(packet_attrs, [:data_extended, :symbol_code]) || ">" - - Logger.debug("SYMBOL TABLE: #{inspect(symbol_table)}") - Logger.debug("SYMBOL CODE: #{inspect(symbol_code)}") {:ok, packet} {:error, changeset} -> @@ -170,13 +203,26 @@ defmodule Aprs.Packets do {to_float(data_extended[:latitude]), to_float(data_extended[:longitude])} # MicE packet format with components else - extract_position_from_mic_e(data_extended) + case data_extended do + %{__struct__: MicE} = mic_e -> + lat = mic_e[:latitude] + lon = mic_e[:longitude] + + if is_number(lat) and is_number(lon) do + {lat, lon} + else + {nil, nil} + end + + _ -> + extract_position_from_mic_e(data_extended) + end end end defp extract_position_from_data_extended(_), do: {nil, nil} - defp extract_position_from_mic_e(%{__struct__: Parser.Types.MicE} = mic_e) do + defp extract_position_from_mic_e(%{__struct__: MicE} = mic_e) do if is_number(mic_e.lat_degrees) and is_number(mic_e.lat_minutes) and is_number(mic_e.lon_degrees) and is_number(mic_e.lon_minutes) do lat = mic_e.lat_degrees + mic_e.lat_minutes / 60.0 @@ -431,12 +477,26 @@ defmodule Aprs.Packets do defp to_float(_), do: nil - # Helper to validate coordinate values - defp are_valid_coordinates?(lat, lon) do - is_number(lat) and is_number(lon) and - lat >= -90 and lat <= 90 and lon >= -180 and lon <= 180 + # Helper to convert various types to Decimal + defp to_decimal(%Decimal{} = d), do: d + defp to_decimal(f) when is_float(f), do: Decimal.from_float(f) + defp to_decimal(i) when is_integer(i), do: Decimal.new(i) + + defp to_decimal(s) when is_binary(s) do + case Decimal.parse(s) do + {d, _} -> + d + + :error -> + case Float.parse(s) do + {f, _} -> Decimal.from_float(f) + :error -> nil + end + end end + defp to_decimal(_), do: nil + # Helper to sanitize all string fields in packet data before database storage defp sanitize_packet_strings(packet_attrs) when is_map(packet_attrs) do packet_attrs diff --git a/lib/aprs_web/live/map_live/callsign_view.ex b/lib/aprs_web/live/map_live/callsign_view.ex index c1320dc..735590f 100644 --- a/lib/aprs_web/live/map_live/callsign_view.ex +++ b/lib/aprs_web/live/map_live/callsign_view.ex @@ -271,40 +271,42 @@ defmodule AprsWeb.MapLive.CallsignView do "#{sanitized_packet.base_callsign}#{if sanitized_packet.ssid, do: "-#{sanitized_packet.ssid}", else: ""}" if has_position_data?(sanitized_packet) and - Map.has_key?(socket.assigns.visible_packets, callsign_key) and - not within_bounds?(%{lat: lat, lon: lng}, socket.assigns.map_bounds) do - socket = push_event(socket, "remove_marker", %{id: callsign_key}) - new_visible_packets = Map.delete(socket.assigns.visible_packets, callsign_key) - {:noreply, assign(socket, visible_packets: new_visible_packets)} - else - if has_position_data?(sanitized_packet) and - within_bounds?(%{lat: lat, lon: lng}, socket.assigns.map_bounds) and - packet_matches_callsign?(sanitized_packet, socket.assigns.callsign) and - packet_within_time_threshold?( - sanitized_packet, - socket.assigns.packet_age_threshold - ) do - packet_data = build_packet_data(sanitized_packet) + packet_matches_callsign?(sanitized_packet, socket.assigns.callsign) and + within_bounds?(%{lat: lat, lon: lng}, socket.assigns.map_bounds) and + packet_within_time_threshold?( + sanitized_packet, + socket.assigns.packet_age_threshold + ) do + packet_data = build_packet_data(sanitized_packet) - if packet_data do - visible_packets = - Map.put(socket.assigns.visible_packets, callsign_key, sanitized_packet) - - last_known_position = - if lat && lng, do: %{lat: lat, lng: lng}, else: socket.assigns.last_known_position - - socket = - socket - |> push_event("new_packet", packet_data) - |> assign( - visible_packets: visible_packets, - last_known_position: last_known_position - ) - - {:noreply, socket} + # Remove any previous marker for this callsign + socket = + if Map.has_key?(socket.assigns.visible_packets, callsign_key) do + push_event(socket, "remove_marker", %{id: callsign_key}) else - {:noreply, socket} + socket end + + visible_packets = %{callsign_key => sanitized_packet} + + last_known_position = + if lat && lng, do: %{lat: lat, lng: lng}, else: socket.assigns.last_known_position + + socket = + socket + |> push_event("new_packet", packet_data) + |> assign( + visible_packets: visible_packets, + last_known_position: last_known_position + ) + + {:noreply, socket} + else + # Remove marker if it exists and is now out of bounds or expired + if Map.has_key?(socket.assigns.visible_packets, callsign_key) do + socket = push_event(socket, "remove_marker", %{id: callsign_key}) + visible_packets = %{} + {:noreply, assign(socket, visible_packets: visible_packets)} else {:noreply, socket} end @@ -633,31 +635,6 @@ defmodule AprsWeb.MapLive.CallsignView do - <%= if @replay_active do %> -
- - -
- - x -
-
- <% end %> - - - <%= if map_size(@visible_packets) == 0 and not @replay_active do %>

Loading Historical Data

@@ -707,7 +684,9 @@ defmodule AprsWeb.MapLive.CallsignView do # Use Map.drop/2 for better performance updated_visible_packets = Map.drop(socket.assigns.visible_packets, expired_keys) - assign(socket, visible_packets: updated_visible_packets) + socket = assign(socket, visible_packets: updated_visible_packets) + + {:noreply, socket} end defp packet_within_time_threshold?(packet, threshold) do @@ -917,12 +896,10 @@ defmodule AprsWeb.MapLive.CallsignView do end defp load_callsign_packets(socket, callsign) do - # Load recent packets (last hour) for this specific callsign - recent_packets = Packets.get_recent_packets(%{callsign: callsign}) - - # Find the most recent packet with position data for auto-zoom and symbol info + # Load only the latest packet for this specific callsign latest_packet = - recent_packets + %{callsign: callsign} + |> Packets.get_recent_packets() |> Enum.filter(&has_position_data?/1) |> Enum.sort_by(& &1.received_at, {:desc, DateTime}) |> List.first() @@ -949,29 +926,31 @@ defmodule AprsWeb.MapLive.CallsignView do _ -> ">" end - # Convert packets to client-friendly format and send to map - packet_data_list = - recent_packets - |> Stream.filter(&has_position_data?/1) - |> Stream.map(&build_packet_data/1) - |> Stream.filter(&(&1 != nil)) - |> Enum.to_list() - - # Build visible_packets map from the loaded packets + # Build visible_packets map with only the latest packet visible_packets = - recent_packets - |> Stream.filter(&has_position_data?/1) - |> Map.new(fn packet -> - callsign_key = "#{packet.base_callsign}#{if packet.ssid, do: "-#{packet.ssid}", else: ""}" - {callsign_key, packet} - end) + case latest_packet do + nil -> + %{} - # Send packets to map if any exist + packet -> + callsign_key = + "#{packet.base_callsign}#{if packet.ssid, do: "-#{packet.ssid}", else: ""}" + + %{callsign_key => packet} + end + + # Send only the latest marker to the map if it exists socket = - if length(packet_data_list) > 0 do - push_event(socket, "add_markers", %{markers: packet_data_list}) - else - socket + case latest_packet do + nil -> + socket + + packet -> + packet_data = build_packet_data(packet) + + if packet_data, + do: push_event(socket, "add_markers", %{markers: [packet_data]}), + else: socket end assign(socket, diff --git a/lib/aprs_web/live/map_live/index.ex b/lib/aprs_web/live/map_live/index.ex index 440ba02..84994b6 100644 --- a/lib/aprs_web/live/map_live/index.ex +++ b/lib/aprs_web/live/map_live/index.ex @@ -13,8 +13,6 @@ defmodule AprsWeb.MapLive.Index do @default_zoom 5 @ip_api_url "https://ip-api.com/json/" @finch_name Aprs.Finch - @default_replay_speed 1000 - @debounce_interval 200 @impl true def mount(_params, _session, socket) do @@ -107,11 +105,6 @@ defmodule AprsWeb.MapLive.Index do end) end - defp schedule_timers do - Process.send_after(self(), :cleanup_old_packets, 60_000) - Process.send_after(self(), :initialize_replay, 2000) - end - @impl true def handle_event("bounds_changed", %{"bounds" => bounds}, socket) do Logger.debug("handle_event bounds_changed: #{inspect(bounds)} vs current #{inspect(socket.assigns.map_bounds)}") @@ -285,8 +278,6 @@ defmodule AprsWeb.MapLive.Index do if compare_bounds(map_bounds, socket.assigns.map_bounds) do {:noreply, socket} # Cancel any pending bounds update timer - - # Set a debounced update timer to prevent excessive processing else if socket.assigns[:bounds_update_timer] do Process.cancel_timer(socket.assigns.bounds_update_timer) @@ -348,10 +339,6 @@ defmodule AprsWeb.MapLive.Index do # Private handler functions for each message type defp handle_info_process_bounds_update(map_bounds, socket) do - Logger.debug( - "handle_info_process_bounds_update: #{inspect(map_bounds)} vs current #{inspect(socket.assigns.map_bounds)}" - ) - if !socket.assigns.initial_bounds_loaded or not compare_bounds(map_bounds, socket.assigns.map_bounds) do socket = process_bounds_update(map_bounds, socket) @@ -411,6 +398,10 @@ defmodule AprsWeb.MapLive.Index do do: to_string(packet["id"]), else: System.unique_integer([:positive]) + Logger.debug( + "[MAP] Incoming packet: id=#{inspect(callsign_key)} lat=#{inspect(lat)} lon=#{inspect(lon)} bounds=#{inspect(socket.assigns.map_bounds)} within_bounds?=#{inspect(within_bounds?(%{lat: lat, lon: lon}, socket.assigns.map_bounds))}" + ) + all_packets = Map.put(socket.assigns.all_packets, callsign_key, packet) socket = assign(socket, all_packets: all_packets) @@ -435,7 +426,7 @@ defmodule AprsWeb.MapLive.Index do end end - defp handle_valid_postgres_packet(packet, lat, lon, socket) do + defp handle_valid_postgres_packet(packet, _lat, _lon, socket) do # Add the packet to visible_packets and push a marker immediately callsign_key = if Map.has_key?(packet, "id"), @@ -804,14 +795,31 @@ defmodule AprsWeb.MapLive.Index do (is_map(data_extended) && (Map.get(data_extended, :longitude) || Map.get(data_extended, "longitude"))) + lat = + cond do + is_struct(lat, Decimal) -> Decimal.to_float(lat) + is_float(lat) -> lat + is_integer(lat) -> lat * 1.0 + true -> lat + end + + lon = + cond do + is_struct(lon, Decimal) -> Decimal.to_float(lon) + is_float(lon) -> lon + is_integer(lon) -> lon * 1.0 + true -> lon + end + {lat, lon, data_extended} end @spec build_packet_data(map() | struct()) :: map() | nil defp build_packet_data(packet) do {lat, lon, data_extended} = get_coordinates(packet) - # Only include packets with valid position data - if lat && lon do + callsign = Map.get(packet, :base_callsign, Map.get(packet, "base_callsign", "")) + # Only include packets with valid position data and a non-empty callsign + if lat && lon && callsign != "" && callsign != nil do build_packet_map(packet, lat, lon, data_extended) end end @@ -858,12 +866,12 @@ defmodule AprsWeb.MapLive.Index do %{ "id" => callsign, "callsign" => callsign, - "base_callsign" => packet.base_callsign || "", - "ssid" => packet.ssid || "", + "base_callsign" => Map.get(packet, :base_callsign, Map.get(packet, "base_callsign", "")), + "ssid" => Map.get(packet, :ssid, Map.get(packet, "ssid", 0)), "lat" => lat, "lng" => lon, - "data_type" => to_string(packet.data_type || "unknown"), - "path" => packet.path || "", + "data_type" => to_string(Map.get(packet, :data_type, Map.get(packet, "data_type", "unknown"))), + "path" => Map.get(packet, :path, Map.get(packet, "path", "")), "comment" => comment, "data_extended" => data_extended || %{}, "symbol_table_id" => symbol_table_id, @@ -876,10 +884,13 @@ defmodule AprsWeb.MapLive.Index do @spec generate_callsign(map() | struct()) :: String.t() defp generate_callsign(packet) do - if packet.ssid && packet.ssid != "" do - "#{packet.base_callsign}-#{packet.ssid}" + base_callsign = Map.get(packet, :base_callsign, Map.get(packet, "base_callsign", "")) + ssid = Map.get(packet, :ssid, Map.get(packet, "ssid", 0)) + + if ssid != 0 and ssid != "" and ssid != nil do + "#{base_callsign}-#{ssid}" else - packet.base_callsign || "" + base_callsign end end diff --git a/lib/aprs_web/live/packets_live/index.ex b/lib/aprs_web/live/packets_live/index.ex index f3b7195..05c88f0 100644 --- a/lib/aprs_web/live/packets_live/index.ex +++ b/lib/aprs_web/live/packets_live/index.ex @@ -9,6 +9,7 @@ defmodule AprsWeb.PacketsLive.Index do def mount(_params, _session, socket) do if connected?(socket) do Endpoint.subscribe("aprs_messages") + Phoenix.PubSub.subscribe(Aprs.PubSub, "postgres:aprs_packets") end {:ok, assign(socket, :packets, [])} @@ -22,4 +23,12 @@ defmodule AprsWeb.PacketsLive.Index do socket = assign(socket, :packets, packets) {:noreply, socket} end + + @impl true + def handle_info({:postgres_packet, payload}, socket) do + sanitized_payload = Aprs.EncodingUtils.sanitize_packet(payload) + packets = Enum.take([sanitized_payload | socket.assigns.packets], 100) + socket = assign(socket, :packets, packets) + {:noreply, socket} + end end diff --git a/lib/aprs_web/live/packets_live/index.html.heex b/lib/aprs_web/live/packets_live/index.html.heex index 4647ae1..0f4bfeb 100644 --- a/lib/aprs_web/live/packets_live/index.html.heex +++ b/lib/aprs_web/live/packets_live/index.html.heex @@ -22,26 +22,32 @@ <.table id="packets" rows={@packets}> <:col :let={packet} label="Sender"> <.link - navigate={~p"/packets/#{packet.base_callsign}"} + navigate={ + ~p"/packets/#{Map.get(packet, :base_callsign, Map.get(packet, "base_callsign", ""))}" + } class="text-blue-600 hover:text-blue-800 font-medium" > - {packet.sender} + {Map.get(packet, :sender, Map.get(packet, "sender", ""))} <:col :let={packet} label="SSID"> - {packet.ssid} + + {Map.get(packet, :ssid, Map.get(packet, "ssid", ""))} + <:col :let={packet} label="Base Callsign"> <.link - navigate={~p"/packets/#{packet.base_callsign}"} + navigate={ + ~p"/packets/#{Map.get(packet, :base_callsign, Map.get(packet, "base_callsign", ""))}" + } class="text-blue-600 hover:text-blue-800 font-medium" > - {packet.base_callsign} + {Map.get(packet, :base_callsign, Map.get(packet, "base_callsign", ""))} <:col :let={packet} label="Data Type"> - {packet.data_type} + {Map.get(packet, :data_type, Map.get(packet, "data_type", ""))} <:col :let={packet} label="Symbol"> @@ -53,14 +59,52 @@ {table}{code} - <:col :let={packet} label="Destination"> - {packet.destination} - - <:col :let={packet} label="Information"> - {packet.information_field} - <:col :let={packet} label="Path"> - {packet.path} + + {Map.get(packet, :path, Map.get(packet, "path", ""))} + + + <:col :let={packet} label="Latitude"> + + <% lat = + Map.get(packet, :lat) || + if( + Map.has_key?(packet, :location) and not is_nil(packet.location) and + function_exported?(Aprs.Packet, :lat, 1), + do: Aprs.Packet.lat(packet), + else: nil + ) || + case Map.get(packet, :data_extended) do + %{} = data_ext -> Map.get(data_ext, :latitude) || Map.get(data_ext, "latitude") + _ -> nil + end %> + {if lat, + do: + (is_float(lat) and :io_lib.format("~.6f", [lat]) |> List.to_string()) or + (is_binary(lat) and Regex.replace(~r/(\d+\.\d{1,6})\d*/, lat, "\\1")) or lat, + else: ""} + + + <:col :let={packet} label="Longitude"> + + <% lon = + Map.get(packet, :lon) || + if( + Map.has_key?(packet, :location) and not is_nil(packet.location) and + function_exported?(Aprs.Packet, :lon, 1), + do: Aprs.Packet.lon(packet), + else: nil + ) || + case Map.get(packet, :data_extended) do + %{} = data_ext -> Map.get(data_ext, :longitude) || Map.get(data_ext, "longitude") + _ -> nil + end %> + {if lon, + do: + (is_float(lon) and :io_lib.format("~.6f", [lon]) |> List.to_string()) or + (is_binary(lon) and Regex.replace(~r/(\d+\.\d{1,6})\d*/, lon, "\\1")) or lon, + else: ""} +
diff --git a/lib/parser.ex b/lib/parser.ex index 878a75a..e8d9b68 100644 --- a/lib/parser.ex +++ b/lib/parser.ex @@ -6,24 +6,31 @@ defmodule Parser do """ # import Bitwise alias Aprs.Convert - alias Parser.Types.MicE require Logger # Simple APRS position parsing to replace parse_aprs_position - defp parse_aprs_position( - <>, - <> - ) - when lat_dir in ["N", "S"] and lon_dir in ["E", "W"] do - lat_val = String.to_integer(lat_deg) + String.to_float(lat_min) / 60 - lon_val = String.to_integer(lon_deg) + String.to_float(lon_min) / 60 - lat = if lat_dir == "S", do: -lat_val, else: lat_val - lon = if lon_dir == "W", do: -lon_val, else: lon_val - %{latitude: lat, longitude: lon} - end + defp parse_aprs_position(lat, lon) do + # Regex for latitude: 2 deg, 2+ min, 1 dir (N/S) + # Regex for longitude: 3 deg, 2+ min, 1 dir (E/W) + lat_re = ~r/^(\d{2})(\d{2}\.\d+)([NS])$/ + lon_re = ~r/^(\d{3})(\d{2}\.\d+)([EW])$/ - defp parse_aprs_position(_, _), do: %{latitude: nil, longitude: nil} + with [_, lat_deg, lat_min, lat_dir] <- Regex.run(lat_re, lat), + [_, lon_deg, lon_min, lon_dir] <- Regex.run(lon_re, lon) do + lat_val = + Decimal.add(Decimal.new(lat_deg), Decimal.div(Decimal.new(lat_min), Decimal.new("60"))) + + lon_val = + Decimal.add(Decimal.new(lon_deg), Decimal.div(Decimal.new(lon_min), Decimal.new("60"))) + + lat = if lat_dir == "S", do: Decimal.negate(lat_val), else: lat_val + lon = if lon_dir == "W", do: Decimal.negate(lon_val), else: lon_val + %{latitude: lat, longitude: lon} + else + _ -> %{latitude: nil, longitude: nil} + end + end @type packet :: %{ id: String.t(), @@ -55,7 +62,7 @@ defmodule Parser do defp do_parse(message) do with {:ok, [sender, path, data]} <- split_packet(message), - {:ok, [base_callsign, ssid]} <- parse_callsign(sender), + {:ok, callsign_parts} <- parse_callsign(sender), {:ok, data_type} <- parse_datatype_safe(data), {:ok, [destination, path]} <- split_path(path), :ok <- validate_callsign(sender, :src), @@ -65,6 +72,26 @@ defmodule Parser do data_without_type = String.slice(data_trimmed, 1..-1//1) data_extended = parse_data(data_type, destination, data_without_type) + base_callsign = List.first(callsign_parts) + + ssid = + case List.last(callsign_parts) do + nil -> + 0 + + s when is_binary(s) -> + case Integer.parse(s) do + {i, _} -> i + :error -> 0 + end + + i when is_integer(i) -> + i + + _ -> + 0 + end + {:ok, %{ id: 16 |> :crypto.strong_rand_bytes() |> Base.encode16(case: :lower), @@ -170,32 +197,32 @@ defmodule Parser do @spec parse_datatype(String.t()) :: atom() def parse_datatype(<<":", _::binary>>), do: :message def parse_datatype(<<">", _::binary>>), do: :status - def parse_datatype(<<"!", _::binary>>), do: :position - def parse_datatype(<<"/", _::binary>>), do: :timestamped_position - def parse_datatype(<<"=", _::binary>>), do: :position_with_message - def parse_datatype(<<"@", _::binary>>), do: :timestamped_position_with_message - def parse_datatype(<<";", _::binary>>), do: :object - def parse_datatype(<<"`", _::binary>>), do: :mic_e - def parse_datatype(<<"'", _::binary>>), do: :mic_e_old - def parse_datatype(<<"_", _::binary>>), do: :weather - def parse_datatype(<<"T", _::binary>>), do: :telemetry - def parse_datatype(<<"$", _::binary>>), do: :raw_gps_ultimeter - def parse_datatype(<<"<", _::binary>>), do: :station_capabilities - def parse_datatype(<<"?", _::binary>>), do: :query - def parse_datatype(<<"{", _::binary>>), do: :user_defined - def parse_datatype(<<"}", _::binary>>), do: :third_party_traffic - def parse_datatype(<<"%", _::binary>>), do: :item - def parse_datatype(<<")", _::binary>>), do: :item - def parse_datatype(<<"*", _::binary>>), do: :peet_logging - def parse_datatype(<<",", _::binary>>), do: :invalid_test_data - def parse_datatype(<<"#DFS", _::binary>>), do: :df_report - def parse_datatype(<<"#PHG", _::binary>>), do: :phg_data - def parse_datatype(<<"#", _::binary>>), do: :phg_data + def parse_datatype("!" <> _), do: :position + def parse_datatype("/" <> _), do: :timestamped_position + def parse_datatype("=" <> _), do: :position_with_message + def parse_datatype("@" <> _), do: :timestamped_position_with_message + def parse_datatype(";" <> _), do: :object + def parse_datatype("`" <> _), do: :mic_e_old + def parse_datatype("'" <> _), do: :mic_e_old + def parse_datatype("_" <> _), do: :weather + def parse_datatype("T" <> _), do: :telemetry + def parse_datatype("$" <> _), do: :raw_gps_ultimeter + def parse_datatype("<" <> _), do: :station_capabilities + def parse_datatype("?" <> _), do: :query + def parse_datatype("{" <> _), do: :user_defined + def parse_datatype("}" <> _), do: :third_party_traffic + def parse_datatype("%" <> _), do: :item + def parse_datatype(")" <> _), do: :item + def parse_datatype("*" <> _), do: :peet_logging + def parse_datatype("," <> _), do: :invalid_test_data + def parse_datatype("#DFS" <> _), do: :df_report + def parse_datatype("#PHG" <> _), do: :phg_data + def parse_datatype("#" <> _), do: :phg_data def parse_datatype(_), do: :unknown_datatype @spec parse_data(atom(), String.t(), String.t()) :: map() | nil - def parse_data(:mic_e, _destination, data), do: parse_mic_e(data) - def parse_data(:mic_e_old, _destination, data), do: parse_mic_e(data) + def parse_data(:mic_e, destination, data), do: parse_mic_e(data, destination) + def parse_data(:mic_e_old, destination, data), do: parse_mic_e(data, destination) def parse_data(:position, destination, <<"!", rest::binary>>) do parse_data(:position, destination, rest) @@ -216,27 +243,55 @@ defmodule Parser do %{result | data_type: :position} end - def parse_data(:timestamped_position, _destination, <<"/", _::binary>>) do - %{ - data_type: :timestamped_position_error, - error: "Compressed position not supported in timestamped position" - } - end - def parse_data(:timestamped_position, _destination, data) do parse_position_with_timestamp(false, data, :timestamped_position) end def parse_data(:timestamped_position_with_message, _destination, data) do + # Correct binary pattern match for APRS position with weather case data do - <<"/", _::binary>> -> - %{ - data_type: :timestamped_position_error, - error: "Compressed position not supported in timestamped position" - } + <> -> + weather_start = String.starts_with?(rest, "_") + + if weather_start do + result = + Parser.parse_position_with_datetime_and_weather( + true, + time, + latitude, + sym_table_id, + longitude, + symbol_code, + rest + ) + + Map.put( + result, + :has_location, + (is_number(result[:latitude]) or is_struct(result[:latitude], Decimal)) and + (is_number(result[:longitude]) or is_struct(result[:longitude], Decimal)) + ) + else + result = parse_position_with_timestamp(true, data, :timestamped_position_with_message) + + Map.put( + result, + :has_location, + (is_number(result[:latitude]) or is_struct(result[:latitude], Decimal)) and + (is_number(result[:longitude]) or is_struct(result[:longitude], Decimal)) + ) + end _ -> - parse_position_with_timestamp(true, data) + result = parse_position_with_timestamp(true, data, :timestamped_position_with_message) + + Map.put( + result, + :has_location, + (is_number(result[:latitude]) or is_struct(result[:latitude], Decimal)) and + (is_number(result[:longitude]) or is_struct(result[:longitude], Decimal)) + ) end end @@ -324,38 +379,37 @@ defmodule Parser do def parse_data(_type, _destination, _data), do: nil - @spec parse_position_with_datetime_and_weather(boolean(), String.t(), String.t()) :: map() - def parse_position_with_datetime_and_weather(aprs_messaging?, date_time_position_data, weather_report) do - case date_time_position_data do - <> -> - %{latitude: lat, longitude: lon} = parse_aprs_position(latitude, longitude) - weather_data = parse_weather_data(weather_report) + @spec parse_position_with_datetime_and_weather( + boolean(), + binary(), + binary(), + binary(), + binary(), + binary(), + binary() + ) :: map() + def parse_position_with_datetime_and_weather( + aprs_messaging?, + time, + latitude, + sym_table_id, + longitude, + symbol_code, + weather_report + ) do + pos = parse_aprs_position(latitude, longitude) + weather_data = parse_weather_data(weather_report) - %{ - latitude: lat, - longitude: lon, - timestamp: time, - symbol_table_id: sym_table_id, - symbol_code: "_", - weather: weather_data, - data_type: :position_with_datetime_and_weather, - aprs_messaging?: aprs_messaging? - } - - _ -> - weather_data = parse_weather_data(weather_report) - - %{ - latitude: nil, - longitude: nil, - timestamp: nil, - symbol_table_id: nil, - symbol_code: nil, - weather: weather_data, - data_type: :position_with_datetime_and_weather, - aprs_messaging?: aprs_messaging? - } - end + %{ + latitude: pos.latitude, + longitude: pos.longitude, + timestamp: time, + symbol_table_id: sym_table_id, + symbol_code: symbol_code, + weather: weather_data, + data_type: :position_with_datetime_and_weather, + aprs_messaging?: aprs_messaging? + } end @spec decode_compressed_position(binary()) :: map() @@ -400,6 +454,10 @@ defmodule Parser do dao_data = parse_dao_extension(comment) {course, speed} = extract_course_and_speed(comment) + has_position = + (is_number(lat) or is_struct(lat, Decimal)) and + (is_number(lon) or is_struct(lon, Decimal)) + %{ latitude: lat, longitude: lon, @@ -413,13 +471,18 @@ defmodule Parser do position_ambiguity: ambiguity, dao: dao_data, course: course, - speed: speed + speed: speed, + has_position: has_position } <> -> %{latitude: lat, longitude: lon} = parse_aprs_position(latitude, longitude) ambiguity = Parser.Helpers.calculate_position_ambiguity(latitude, longitude) + has_position = + (is_number(lat) or is_struct(lat, Decimal)) and + (is_number(lon) or is_struct(lon, Decimal)) + %{ latitude: lat, longitude: lon, @@ -432,7 +495,8 @@ defmodule Parser do position_ambiguity: ambiguity, dao: nil, course: nil, - speed: nil + speed: nil, + has_position: has_position } <<"/", latitude_compressed::binary-size(4), longitude_compressed::binary-size(4), symbol_code::binary-size(1), @@ -443,6 +507,10 @@ defmodule Parser do compressed_cs = Parser.Helpers.convert_compressed_cs(cs) ambiguity = Parser.Helpers.calculate_compressed_ambiguity(compression_type) + has_position = + (is_number(converted_lat) or is_struct(converted_lat, Decimal)) and + (is_number(converted_lon) or is_struct(converted_lon, Decimal)) + base_data = %{ latitude: converted_lat, longitude: converted_lon, @@ -454,7 +522,8 @@ defmodule Parser do data_type: :position, compressed?: true, position_ambiguity: ambiguity, - dao: nil + dao: nil, + has_position: has_position } # Merge in course/speed from compressed_cs @@ -474,7 +543,8 @@ defmodule Parser do position_ambiguity: Parser.Helpers.calculate_compressed_ambiguity(compression_type), dao: nil, course: nil, - speed: nil + speed: nil, + has_position: false } end @@ -491,7 +561,8 @@ defmodule Parser do comment: String.trim(position_data), dao: nil, course: nil, - speed: nil + speed: nil, + has_position: false } end end @@ -530,35 +601,97 @@ defmodule Parser do course: course, speed: speed } + + _ -> + # Fallback: try to extract lat/lon using regex if binary pattern match fails + regex = + ~r/^(?