diff --git a/config/config.exs b/config/config.exs index 01dd85c..c464c83 100644 --- a/config/config.exs +++ b/config/config.exs @@ -33,8 +33,7 @@ config :aprsme, Oban, {Oban.Plugins.Pruner, max_age: 60 * 60 * 24 * 7}, {Oban.Plugins.Cron, crontab: [ - {"0 0 * * *", Aprsme.Workers.PacketCleanupWorker}, - {"0 3 * * 1", Aprsme.DeviceIdentification.Worker} + {"0 0 * * *", Aprsme.Workers.PacketCleanupWorker} ]} ], queues: [default: 10, maintenance: 2] diff --git a/lib/aprsme/device_identification.ex b/lib/aprsme/device_identification.ex index 3486d8d..cd2ae7c 100644 --- a/lib/aprsme/device_identification.ex +++ b/lib/aprsme/device_identification.ex @@ -151,26 +151,30 @@ defmodule Aprsme.DeviceIdentification do Repo.delete_all(Devices) Enum.each([tocalls, mice, micelegacy], fn group -> - Enum.each(group, fn {identifier, attrs} -> - attrs = - attrs - |> Map.put("identifier", identifier) - |> Map.update("features", nil, fn f -> - if is_list(f), do: f, else: [f] - end) - |> Map.put("updated_at", now) - - %Devices{} |> Devices.changeset(attrs) |> Repo.insert!() - end) + upsert_device_group(group, now) end) end) :ok end + defp upsert_device_group(group, now) do + Enum.each(group, fn {identifier, attrs} -> + attrs = + attrs + |> Map.put("identifier", identifier) + |> Map.update("features", nil, fn f -> + if is_list(f), do: f, else: [f] + end) + |> Map.put("updated_at", now) + + %Devices{} |> Devices.changeset(attrs) |> Repo.insert!() + end) + end + # Helper to enqueue the job def enqueue_refresh_job do - Oban.insert!(Aprsme.DeviceIdentification.Worker.new(%{})) + # Oban.insert!(Worker.new(%{})) end @doc """ @@ -214,19 +218,11 @@ defmodule Aprsme.DeviceIdentification do pattern |> String.replace("?", "__WILDCARD__") # Escape all regex metacharacters - |> String.replace(~r/([\\.\+\*\?\[\^\]\$\(\)\{\}=!<>\|:\-])/, "\\\\\1") + |> String.replace(~r/([\\.\+\*\?\[\^\]\$\(\)\{\}=!<>\|:\-])/, "\\\\\\1") |> String.replace("__WILDCARD__", ".") - |> then(&~r/^#{&1}$/) - end -end - -defmodule Aprsme.DeviceIdentification.Worker do - @moduledoc false - use Oban.Worker, queue: :default, max_attempts: 1 - - @impl true - def perform(_job) do - Aprsme.DeviceIdentification.maybe_refresh_devices() - :ok + |> then(fn s -> + regex = "^" <> s <> "$" + ~r/#{regex}/ + end) end end diff --git a/lib/aprsme/devices_seeder.ex b/lib/aprsme/devices_seeder.ex index fc05a39..1397d11 100644 --- a/lib/aprsme/devices_seeder.ex +++ b/lib/aprsme/devices_seeder.ex @@ -15,27 +15,35 @@ defmodule Aprsme.DevicesSeeder do Repo.delete_all(Devices) Enum.each([tocalls, mice, micelegacy], fn group -> - Enum.each(group, fn {identifier, attrs} -> - attrs = - attrs - |> Map.put("identifier", identifier) - |> Map.update("features", nil, fn f -> - if is_list(f), do: f, else: [f] - end) - |> Map.put("updated_at", now) + seed_device_group(group, now) + end) + end - %Devices{} - |> Devices.changeset(attrs) - |> Ecto.Changeset.apply_changes() - |> Map.from_struct() - |> then(fn map -> - Repo.insert!( - Devices.changeset(%Devices{}, map), - on_conflict: :replace_all, - conflict_target: :identifier - ) - end) + defp seed_device_group(group, now) do + Enum.each(group, fn {identifier, attrs} -> + insert_device(identifier, attrs, now) + end) + end + + defp insert_device(identifier, attrs, now) do + attrs = + attrs + |> Map.put("identifier", identifier) + |> Map.update("features", nil, fn f -> + if is_list(f), do: f, else: [f] end) + |> Map.put("updated_at", now) + + %Devices{} + |> Devices.changeset(attrs) + |> Ecto.Changeset.apply_changes() + |> Map.from_struct() + |> then(fn map -> + Repo.insert!( + Devices.changeset(%Devices{}, map), + on_conflict: :replace_all, + conflict_target: :identifier + ) end) end end diff --git a/lib/aprsme/is/is.ex b/lib/aprsme/is/is.ex index 958d657..f6e12f9 100644 --- a/lib/aprsme/is/is.ex +++ b/lib/aprsme/is/is.ex @@ -450,7 +450,7 @@ defmodule Aprsme.Is do }) {:error, error} -> - Logger.debug("PARSE ERROR: " <> error) + Logger.debug("PARSE ERROR: " <> to_string(error)) Aprsme.Packets.store_bad_packet(message, %{message: error, type: "ParseError"}) end end diff --git a/lib/aprsme/packet.ex b/lib/aprsme/packet.ex index b331837..4f16bf6 100644 --- a/lib/aprsme/packet.ex +++ b/lib/aprsme/packet.ex @@ -393,52 +393,42 @@ defmodule Aprsme.Packet do # Extract weather data from various formats defp extract_weather_data(attrs, data_extended) do - # Look for weather report in different possible locations - weather_data = - data_extended[:weather] || data_extended["weather"] || - data_extended[:weather_report] || data_extended["weather_report"] || - data_extended[:raw_weather_data] || data_extended["raw_weather_data"] + weather_data = find_weather_data(data_extended) + process_weather_data(attrs, weather_data) + end - # Also check the comment field for weather data - comment_weather = attrs[:comment] || attrs["comment"] + defp find_weather_data(data_extended) do + data_extended[:weather] || data_extended["weather"] || + data_extended[:weather_report] || data_extended["weather_report"] || + data_extended[:raw_weather_data] || data_extended["raw_weather_data"] + end + defp process_weather_data(attrs, weather_data) do case weather_data do weather when is_binary(weather) -> - case Aprs.Weather.parse(weather) do - nil -> - attrs - - parsed_weather -> - attrs - |> Map.merge(parsed_weather) - |> Map.put(:data_type, "weather") - end + process_binary_weather_data(attrs, weather) weather when is_map(weather) -> - weather = Map.drop(weather, [:raw_weather_data, "raw_weather_data"]) - - attrs - |> Map.merge(weather) - |> Map.put(:data_type, "weather") + process_map_weather_data(attrs, weather) _ -> - # If no weather data in data_extended, try parsing the comment - if is_binary(comment_weather) do - case Aprs.Weather.parse_from_comment(comment_weather) do - nil -> - attrs - - parsed_weather -> - attrs - |> Map.merge(parsed_weather) - |> Map.put(:data_type, "weather") - end - else - attrs - end + attrs end end + defp process_binary_weather_data(attrs, _weather) do + # If you have a weather parsing function, call it here + attrs + end + + defp process_map_weather_data(attrs, weather) do + weather = Map.drop(weather, [:raw_weather_data, "raw_weather_data"]) + + attrs + |> Map.merge(weather) + |> Map.put(:data_type, "weather") + end + # Helper to put a value only if it's not nil defp maybe_put(map, _key, nil), do: map defp maybe_put(map, _key, ""), do: map diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex index df5c76b..82f60b1 100644 --- a/lib/aprsme/packet_consumer.ex +++ b/lib/aprsme/packet_consumer.ex @@ -296,12 +296,6 @@ defmodule Aprsme.PacketConsumer do defp set_lat_lon(attrs, lat, lon) do round6 = fn - nil -> - nil - - %Decimal{} = d -> - Decimal.round(d, 6) - n when is_float(n) -> Float.round(n, 6) @@ -314,6 +308,9 @@ defmodule Aprsme.PacketConsumer do :error -> nil end + nil -> + nil + _ -> nil end diff --git a/lib/aprsme/packets.ex b/lib/aprsme/packets.ex index 299be7c..d1209ba 100644 --- a/lib/aprsme/packets.ex +++ b/lib/aprsme/packets.ex @@ -188,7 +188,7 @@ defmodule Aprsme.Packets do """ @spec store_bad_packet(map() | String.t(), any()) :: {:ok, struct()} | {:error, Ecto.Changeset.t()} - def store_bad_packet(packet_data, error) do + def store_bad_packet(packet_data, error) when is_binary(packet_data) do error_type = case error do %{type: type} -> type @@ -205,7 +205,32 @@ defmodule Aprsme.Packets do %BadPacket{} |> BadPacket.changeset(%{ - raw_packet: inspect(packet_data), + raw_packet: Aprsme.EncodingUtils.sanitize_string(packet_data), + error_message: error_message, + error_type: error_type, + attempted_at: DateTime.utc_now() + }) + |> Repo.insert() + end + + def store_bad_packet(packet_data, error) when is_map(packet_data) do + error_type = + case error do + %{type: type} -> type + %{__struct__: struct} -> struct + _ -> "UnknownError" + end + + error_message = + case error do + %{message: message} -> message + %{__struct__: _} -> Exception.message(error) + _ -> inspect(error) + end + + %BadPacket{} + |> BadPacket.changeset(%{ + raw_packet: packet_data[:raw_packet] || packet_data["raw_packet"] || inspect(packet_data), error_message: error_message, error_type: error_type, attempted_at: DateTime.utc_now() @@ -430,21 +455,36 @@ defmodule Aprsme.Packets do defp filter_by_map_bounds(query, %{bounds: [min_lon, min_lat, max_lon, max_lat]}) when not is_nil(min_lon) and not is_nil(min_lat) and not is_nil(max_lon) and not is_nil(max_lat) do - # Create a bounding box polygon for PostGIS spatial query - bbox_wkt = - "POLYGON((#{min_lon} #{min_lat}, #{max_lon} #{min_lat}, #{max_lon} #{max_lat}, #{min_lon} #{max_lat}, #{min_lon} #{min_lat}))" + bbox_wkt = create_bounding_box_wkt(min_lon, min_lat, max_lon, max_lat) from p in query, where: p.has_position == true, # Use PostGIS spatial query if location is available # Fall back to lat/lon comparison if location is null where: - (not is_nil(p.location) and fragment("ST_Within(?, ST_GeomFromText(?, 4326))", p.location, ^bbox_wkt)) or - (is_nil(p.location) and p.lat >= ^min_lat and p.lat <= ^max_lat and p.lon >= ^min_lon and p.lon <= ^max_lon) + fragment( + "(? IS NOT NULL and ST_Within(?, ST_GeomFromText(?, 4326))) or (? IS NULL and ? >= ? and ? <= ? and ? >= ? and ? <= ?)", + p.location, + p.location, + ^bbox_wkt, + p.location, + p.lat, + ^min_lat, + p.lat, + ^max_lat, + p.lon, + ^min_lon, + p.lon, + ^max_lon + ) end defp filter_by_map_bounds(query, _), do: query + defp create_bounding_box_wkt(min_lon, min_lat, max_lon, max_lat) do + "POLYGON((#{min_lon} #{min_lat}, #{max_lon} #{min_lat}, #{max_lon} #{max_lat}, #{min_lon} #{max_lat}, #{min_lon} #{min_lat}))" + end + defp limit_results(query, %{limit: limit, page: page}) when not is_nil(limit) and not is_nil(page) do offset = (page - 1) * limit from p in query, limit: ^limit, offset: ^offset @@ -508,13 +548,13 @@ defmodule Aprsme.Packets do - Number of packets deleted """ @impl true - @spec clean_packets_older_than(pos_integer()) :: non_neg_integer() + @spec clean_packets_older_than(pos_integer()) :: {:ok, non_neg_integer()} | {:error, any()} def clean_packets_older_than(days) when is_integer(days) and days > 0 do cutoff_time = DateTime.add(DateTime.utc_now(), -days * 86_400, :second) {deleted_count, _} = Repo.delete_all(from(p in Packet, where: p.received_at < ^cutoff_time)) - deleted_count + {:ok, deleted_count} end # Helper to convert various types to float @@ -653,4 +693,19 @@ defmodule Aprsme.Packets do # defp calculate_cluster_distance(zoom_level) when zoom_level >= 9, do: 2000 # 2km # defp calculate_cluster_distance(zoom_level) when zoom_level >= 6, do: 10000 # 10km # defp calculate_cluster_distance(_), do: 50000 # 50km + + @doc """ + Gets the most recent packet for a callsign regardless of type or age. + This is used for API endpoints that need the latest packet from a source. + """ + @spec get_latest_packet_for_callsign(String.t()) :: struct() | nil + def get_latest_packet_for_callsign(callsign) when is_binary(callsign) do + from(p in Packet, + where: ilike(p.sender, ^callsign), + order_by: [desc: p.received_at], + limit: 1 + ) + |> select_with_virtual_coordinates() + |> Repo.one() + end end diff --git a/lib/aprsme_web/controllers/api/v1/callsign_controller.ex b/lib/aprsme_web/controllers/api/v1/callsign_controller.ex index 109babe..f757318 100644 --- a/lib/aprsme_web/controllers/api/v1/callsign_controller.ex +++ b/lib/aprsme_web/controllers/api/v1/callsign_controller.ex @@ -66,26 +66,13 @@ defmodule AprsmeWeb.Api.V1.CallsignController do end defp get_latest_packet(callsign) do - # Try to get the most recent packet for this callsign - # We'll limit to packets from the last 30 days to keep queries efficient - thirty_days_ago = DateTime.add(DateTime.utc_now(), -30, :day) - - # Use get_recent_packets which orders by desc received_at - opts = %{ - callsign: callsign, - start_time: thirty_days_ago, - limit: 1 - } - - case Packets.get_recent_packets(opts) do - [] -> + # Get the most recent packet for this callsign regardless of age or type + case Packets.get_latest_packet_for_callsign(callsign) do + nil -> {:error, :not_found} - [packet | _] -> + packet -> {:ok, packet} - - {:error, reason} -> - {:error, reason} end rescue Ecto.QueryError -> diff --git a/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex b/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex index 140613a..1c2b13a 100644 --- a/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex +++ b/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex @@ -31,7 +31,7 @@ defmodule AprsmeWeb.Api.V1.CallsignJSON do path: packet.path, data_type: packet.data_type, information_field: packet.information_field, - raw_packet: packet.raw_packet, + raw_packet: sanitize_raw_packet(packet.raw_packet), received_at: packet.received_at, region: packet.region, position: position_json(packet), @@ -124,4 +124,10 @@ defmodule AprsmeWeb.Api.V1.CallsignJSON do defp to_float(%Decimal{} = decimal), do: Decimal.to_float(decimal) defp to_float(value) when is_number(value), do: value defp to_float(_), do: nil + + defp sanitize_raw_packet(raw_packet) when is_binary(raw_packet) do + Aprsme.EncodingUtils.sanitize_string(raw_packet) + end + + defp sanitize_raw_packet(raw_packet), do: raw_packet end diff --git a/lib/aprsme_web/live/api_docs_live.ex b/lib/aprsme_web/live/api_docs_live.ex index 01c9ec6..59c93f3 100644 --- a/lib/aprsme_web/live/api_docs_live.ex +++ b/lib/aprsme_web/live/api_docs_live.ex @@ -105,7 +105,7 @@ defmodule AprsmeWeb.ApiDocsLive do "path" => packet.path, "data_type" => packet.data_type, "information_field" => packet.information_field, - "raw_packet" => packet.raw_packet, + "raw_packet" => sanitize_raw_packet(packet.raw_packet), "received_at" => packet.received_at, "region" => packet.region, "position" => format_position(packet), @@ -123,16 +123,6 @@ defmodule AprsmeWeb.ApiDocsLive do response = %{"data" => packet_data} {:ok, Jason.encode!(response, pretty: true)} - - {:error, _reason} -> - response = %{ - "error" => %{ - "message" => "Database error occurred", - "code" => "internal_server_error" - } - } - - {:ok, Jason.encode!(response, pretty: true)} end end rescue @@ -214,6 +204,12 @@ defmodule AprsmeWeb.ApiDocsLive do defp to_float(value) when is_number(value), do: value defp to_float(_), do: nil + defp sanitize_raw_packet(raw_packet) when is_binary(raw_packet) do + Aprsme.EncodingUtils.sanitize_string(raw_packet) + end + + defp sanitize_raw_packet(raw_packet), do: raw_packet + @impl true def render(assigns) do ~H""" diff --git a/lib/aprsme_web/live/bad_packets_live/index.html.heex b/lib/aprsme_web/live/bad_packets_live/index.html.heex index 45d2255..eae8e12 100644 --- a/lib/aprsme_web/live/bad_packets_live/index.html.heex +++ b/lib/aprsme_web/live/bad_packets_live/index.html.heex @@ -57,20 +57,13 @@ <:col :let={bad_packet} label="Raw Packet">