graham/aprs.me
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor and performance improvements

Browse source
This commit is contained in:
Graham McIntire 2025-07-10 10:53:34 -05:00
parent 79c0a0a43a
commit 48447ab42e
No known key found for this signature in database
26 changed files with 1323 additions and 656 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
lib/aprsme/cached_queries.ex
View file

@ -3,7 +3,9 @@ defmodule Aprsme.CachedQueries do
Caching layer for database queries to improve performance
"""
alias Aprsme.Packet
alias Aprsme.Packets
alias Aprsme.Repo
# 2 minutes for frequently changing data
@cache_ttl_short to_timeout(minute: 2)
@ -79,6 +81,54 @@ defmodule Aprsme.CachedQueries do
end
end
@doc """
Get latest weather packet for callsign with caching.
Uses the optimized query that checks recent data first.
"""
def get_latest_weather_packet_cached(callsign) do
cache_key = generate_cache_key("latest_weather_packet", callsign)
case Cachex.get(:query_cache, cache_key) do
{:ok, result} when not is_nil(result) ->
result
_ ->
result = Packets.get_latest_weather_packet_optimized(callsign)
# Cache for 5 minutes since weather updates are less frequent
Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_short)
result
end
end
@doc """
Check if a callsign has weather packets with caching.
Uses exact match for performance.
"""
def has_weather_packets_cached?(callsign) do
cache_key = generate_cache_key("has_weather_packets", callsign)
case Cachex.get(:query_cache, cache_key) do
{:ok, result} when not is_nil(result) ->
result
_ ->
# Use exact match with proper index instead of ilike
import Ecto.Query
query =
from p in Packet,
where: p.sender == ^callsign,
where: p.data_type == "weather" or (p.symbol_table_id == "/" and p.symbol_code == "_"),
limit: 1,
select: true
result = Repo.exists?(query)
# Cache for 15 minutes
Cachex.put(:query_cache, cache_key, result, ttl: @cache_ttl_medium)
result
end
end
@doc """
Invalidate cache entries for a specific callsign
"""
@ -86,6 +136,8 @@ defmodule Aprsme.CachedQueries do
# Pattern-based cache invalidation
patterns = [
"latest_packet:#{callsign}",
"latest_weather_packet:#{callsign}",
"has_weather_packets:#{callsign}",
"weather:#{callsign}:*"
]

133
lib/aprsme/callsign.ex Normal file
View file

@ -0,0 +1,133 @@
defmodule Aprsme.Callsign do
@moduledoc """
Utilities for callsign normalization, validation, and manipulation.
"""
@doc """
Normalizes a callsign by trimming whitespace and converting to uppercase.
"""
@spec normalize(String.t() | nil) :: String.t()
def normalize(nil), do: ""
def normalize(callsign) when is_binary(callsign) do
callsign
|> String.trim()
|> String.upcase()
end
def normalize(_), do: ""
@doc """
Validates if a callsign format is reasonable for amateur radio use.
## Examples
iex> Aprsme.Callsign.valid?("W5ABC")
true
iex> Aprsme.Callsign.valid?("W5ABC-15")
true
iex> Aprsme.Callsign.valid?("")
false
iex> Aprsme.Callsign.valid?("A")
false
"""
@spec valid?(String.t() | nil) :: boolean()
def valid?(nil), do: false
def valid?(callsign) when is_binary(callsign) do
trimmed = String.trim(callsign)
cond do
trimmed == "" -> false
byte_size(trimmed) < 3 -> false
byte_size(trimmed) > 15 -> false
true -> Regex.match?(~r/^[A-Z0-9]+(-[A-Z0-9]{1,2})?$/i, trimmed)
end
end
def valid?(_), do: false
@doc """
Checks if a packet's sender matches the target callsign.
Both callsigns are normalized before comparison.
"""
@spec matches?(String.t() | nil, String.t() | nil) :: boolean()
def matches?(packet_callsign, target_callsign) do
normalize(packet_callsign) == normalize(target_callsign)
end
@doc """
Extracts the base callsign from a full callsign (removes SSID if present).
## Examples
iex> Aprsme.Callsign.extract_base("W5ABC-15")
"W5ABC"
iex> Aprsme.Callsign.extract_base("W5ABC")
"W5ABC"
"""
@spec extract_base(String.t() | nil) :: String.t()
def extract_base(nil), do: ""
def extract_base(callsign) when is_binary(callsign) do
case String.split(callsign, "-") do
[base, _ssid] -> base
[base] -> base
_ -> ""
end
end
def extract_base(_), do: ""
@doc """
Extracts the SSID from a callsign, returning "0" if no SSID is present.
## Examples
iex> Aprsme.Callsign.extract_ssid("W5ABC-15")
"15"
iex> Aprsme.Callsign.extract_ssid("W5ABC")
"0"
"""
@spec extract_ssid(String.t() | nil) :: String.t()
def extract_ssid(nil), do: "0"
def extract_ssid(callsign) when is_binary(callsign) do
case String.split(callsign, "-") do
[_base, ssid] -> ssid
[_base] -> "0"
_ -> "0"
end
end
def extract_ssid(_), do: "0"
@doc """
Extracts both base callsign and SSID as a tuple.
## Examples
iex> Aprsme.Callsign.extract_parts("W5ABC-15")
{"W5ABC", "15"}
iex> Aprsme.Callsign.extract_parts("W5ABC")
{"W5ABC", "0"}
"""
@spec extract_parts(String.t() | nil) :: {String.t(), String.t()}
def extract_parts(nil), do: {"", "0"}
def extract_parts(callsign) when is_binary(callsign) do
case String.split(callsign, "-") do
[base, ssid] -> {base, ssid}
[base] -> {base, "0"}
_ -> {"", "0"}
end
end
def extract_parts(_), do: {"", "0"}
end

115
lib/aprsme/encoding_utils.ex
View file

@ -26,18 +26,56 @@ defmodule Aprsme.EncodingUtils do
"""
@spec sanitize_string(binary() | nil | any()) :: binary() | nil | any()
def sanitize_string(binary) when is_binary(binary) do
# First, handle the encoding conversion
cleaned =
if String.valid?(binary) do
binary
else
:unicode.characters_to_binary(binary, :latin1, :utf8)
case :unicode.characters_to_binary(binary, :latin1, :utf8) do
{:error, _, _} ->
# If conversion fails, try to extract valid parts
binary
|> :binary.bin_to_list()
|> Enum.filter(fn byte -> byte >= 32 and byte <= 126 end)
|> :binary.list_to_bin()
{:incomplete, partial, _} ->
partial
result when is_binary(result) ->
result
end
end
# Remove control characters including null bytes
# We filter at the codepoint level to handle all Unicode control characters
cleaned
|> String.replace(<<0>>, "")
|> String.replace(~r/[\x01-\x08\x0B\x0C\x0E-\x1F\x7F]/, "")
|> String.graphemes()
|> Enum.filter(&String.valid?/1)
|> String.codepoints()
|> Enum.filter(fn cp ->
case :unicode.characters_to_list(cp) do
[codepoint] ->
# Allow printable characters and common whitespace
# Remove C0 controls (0x00-0x1F except tab, newline, carriage return)
# Remove C1 controls (0x80-0x9F)
# Remove DEL (0x7F)
cond do
# Tab
codepoint == 0x09 -> true
# Newline
codepoint == 0x0A -> true
# Carriage return
codepoint == 0x0D -> true
codepoint >= 0x00 and codepoint <= 0x1F -> false
codepoint == 0x7F -> false
codepoint >= 0x80 and codepoint <= 0x9F -> false
true -> true
end
_ ->
# Multi-codepoint grapheme, keep it
true
end
end)
|> Enum.join()
|> String.trim()
end
@ -119,7 +157,7 @@ defmodule Aprsme.EncodingUtils do
iex> Aprsme.EncodingUtils.sanitize_packet_strings(["abc", <<255>>])
["abc", "ÿ"]
iex> Aprsme.EncodingUtils.sanitize_packet_strings(%{"foo" => <<0, 65, 66, 67>>})
%{"foo" => "\0ABC"}
%{"foo" => "ABC"}
iex> Aprsme.EncodingUtils.sanitize_packet_strings(nil)
nil
"""
@ -129,6 +167,12 @@ defmodule Aprsme.EncodingUtils do
def sanitize_packet_strings(%_struct{} = struct), do: struct |> Map.from_struct() |> sanitize_packet_strings()
def sanitize_packet_strings(list) when is_list(list), do: Enum.map(list, &sanitize_packet_strings/1)
def sanitize_packet_strings(map) when is_map(map) do
Enum.reduce(map, %{}, fn {key, value}, acc ->
Map.put(acc, key, sanitize_packet_strings(value))
end)
end
def sanitize_packet_strings(binary) when is_binary(binary) do
s = sanitize_string(binary)
if is_binary(s), do: s, else: ""
@ -206,9 +250,9 @@ defmodule Aprsme.EncodingUtils do
## Examples
iex> result = Aprsme.EncodingUtils.sanitize_packet(%{"information_field" => <<0, 65, 66, 67>>, "data_extended" => %{"comment" => <<0, 68, 69, 70>>}})
iex> result["information_field"] == "\0ABC"
iex> result["information_field"] == "ABC"
true
iex> result["data_extended"]["comment"] == "\0DEF"
iex> result["data_extended"]["comment"] == "DEF"
true
"""
@spec sanitize_packet(struct() | map()) :: struct() | map()
@ -221,9 +265,56 @@ defmodule Aprsme.EncodingUtils do
end
def sanitize_packet(packet) when is_map(packet) do
packet
|> Map.update(:information_field, nil, &sanitize_string/1)
|> Map.update(:data_extended, nil, &sanitize_data_extended/1)
# Handle all known string fields, checking for both atom and string keys
string_fields = [
:information_field,
:comment,
:path,
:raw_packet,
:destination,
:sender,
:base_callsign,
:ssid,
:manufacturer,
:equipment_type,
:message_text,
:addressee,
:symbol_code,
:symbol_table_id,
:dao,
:timestamp,
:device_identifier
]
# Sanitize all string fields
sanitized =
Enum.reduce(string_fields, packet, fn field, acc ->
atom_key = field
string_key = to_string(field)
cond do
Map.has_key?(acc, atom_key) ->
Map.update(acc, atom_key, nil, &sanitize_string/1)
Map.has_key?(acc, string_key) ->
Map.update(acc, string_key, nil, &sanitize_string/1)
true ->
acc
end
end)
# Handle data_extended separately
cond do
Map.has_key?(sanitized, :data_extended) ->
Map.update(sanitized, :data_extended, nil, &sanitize_data_extended/1)
Map.has_key?(sanitized, "data_extended") ->
Map.update(sanitized, "data_extended", nil, &sanitize_data_extended/1)
true ->
sanitized
end
end
@doc """
@ -247,7 +338,7 @@ defmodule Aprsme.EncodingUtils do
%{mic_e | message: sanitize_string(message)}
end
def sanitize_data_extended(data_extended) when is_map(data_extended) do
def sanitize_data_extended(data_extended) when is_map(data_extended) and not is_struct(data_extended) do
# Handle generic maps by sanitizing all string values
Enum.reduce(data_extended, %{}, fn {key, value}, acc ->
sanitized_value = sanitize_map_value(value)

7
lib/aprsme/packet_consumer.ex
View file

@ -221,8 +221,6 @@ defmodule Aprsme.PacketConsumer do
# Remove embedded field for batch insert
|> Map.delete(:data_extended)
|> normalize_numeric_types()
|> sanitize_raw_packet()
|> then(fn attrs -> Map.new(attrs, fn {k, v} -> {k, sanitize_packet_strings(v)} end) end)
|> truncate_datetimes_to_second()
# Explicitly remove raw_weather_data to prevent insert_all errors
|> Map.delete(:raw_weather_data)
@ -442,9 +440,4 @@ defmodule Aprsme.PacketConsumer do
end
end)
end
defp sanitize_raw_packet(attrs) do
# Use the exact same sanitization approach as the original working code
sanitize_packet_strings(attrs)
end
end

363
lib/aprsme/packets.ex
View file

@ -10,7 +10,9 @@ defmodule Aprsme.Packets do
alias Aprs.Types.MicE
alias Aprsme.BadPacket
alias Aprsme.Packet
alias Aprsme.Packets.QueryBuilder
alias Aprsme.Repo
alias AprsmeWeb.TimeUtils
@doc """
Stores a packet in the database.
@ -23,9 +25,14 @@ defmodule Aprsme.Packets do
require Logger
try do
# First sanitize the input data
sanitized_packet_data = Aprsme.EncodingUtils.sanitize_packet(packet_data)
packet_attrs =
packet_data
|> Aprsme.Packet.extract_additional_data(packet_data[:raw_packet] || packet_data["raw_packet"] || "")
sanitized_packet_data
|> Aprsme.Packet.extract_additional_data(
sanitized_packet_data[:raw_packet] || sanitized_packet_data["raw_packet"] || ""
)
|> normalize_packet_attrs()
|> set_received_at()
|> patch_lat_lon_from_data_extended()
@ -40,11 +47,9 @@ defmodule Aprsme.Packets do
canonical_identifier = if matched_device, do: matched_device.identifier, else: device_identifier
Map.put(attrs, :device_identifier, canonical_identifier)
end)
|> sanitize_packet_strings()
# require Logger
# Logger.debug("Sanitized packet_attrs before insert: #{inspect(packet_attrs)}")
packet_attrs = Map.new(packet_attrs, fn {k, v} -> {k, sanitize_packet_strings(v)} end)
# Set device_identifier to parsed value, fallback to destination if nil
parsed_device_id = Aprsme.DeviceParser.extract_device_identifier(packet_data)
device_id = parsed_device_id || Map.get(packet_attrs, :destination)
@ -74,7 +79,7 @@ defmodule Aprsme.Packets do
end
case_result
|> sanitize_packet_strings()
|> Aprsme.EncodingUtils.sanitize_packet()
|> normalize_data_type()
end
@ -166,6 +171,34 @@ defmodule Aprsme.Packets do
end
defp insert_packet(attrs, packet_data) do
# Ensure data_extended is properly sanitized before insertion
attrs =
if attrs[:data_extended] do
sanitized_extended = Aprsme.EncodingUtils.sanitize_data_extended(attrs[:data_extended])
# Double-check all values are sanitized
# Only do additional sanitization for plain maps, not structs
sanitized_extended =
if is_struct(sanitized_extended) do
sanitized_extended
else
Enum.reduce(sanitized_extended, %{}, fn {k, v}, acc ->
sanitized_value = if is_binary(v), do: Aprsme.EncodingUtils.sanitize_string(v), else: v
Map.put(acc, k, sanitized_value)
end)
end
Map.put(attrs, :data_extended, sanitized_extended)
else
attrs
end
# Debug log to see what we're trying to insert
if attrs[:data_extended] do
require Logger
Logger.debug("Final data_extended before insert: #{inspect(attrs[:data_extended], binaries: :as_binaries)}")
end
case %Packet{} |> Packet.changeset(attrs) |> Repo.insert() do
{:ok, packet} ->
# Invalidate cache for this packet's callsign
@ -335,64 +368,55 @@ defmodule Aprsme.Packets do
"""
@impl true
def get_packets_for_replay(opts \\ %{}) do
base_query = from(p in Packet, order_by: [asc: p.received_at], where: p.has_position == true)
limit = Map.get(opts, :limit, 1000)
bounds = Map.get(opts, :bounds)
query =
base_query
|> filter_by_time(opts)
|> filter_by_region(opts)
|> filter_by_callsign(opts)
|> filter_by_map_bounds(opts)
|> limit_results(opts)
|> select_with_virtual_coordinates()
Packet
|> QueryBuilder.with_position()
|> QueryBuilder.with_time_range(opts)
|> QueryBuilder.maybe_filter_region(opts)
|> maybe_filter_by_callsign(opts)
|> maybe_filter_by_bounds(bounds)
|> QueryBuilder.chronological()
|> QueryBuilder.paginate(limit)
|> QueryBuilder.with_coordinates()
Repo.all(query)
end
defp maybe_filter_by_callsign(query, %{callsign: callsign}) when not is_nil(callsign) do
QueryBuilder.for_callsign(query, callsign)
end
defp maybe_filter_by_callsign(query, _), do: query
defp maybe_filter_by_bounds(query, bounds) when is_list(bounds) and length(bounds) == 4 do
QueryBuilder.within_bounds(query, bounds)
end
defp maybe_filter_by_bounds(query, _), do: query
@doc """
Gets historical packet count for a map area.
"""
@impl true
@spec get_historical_packet_count(map()) :: non_neg_integer()
def get_historical_packet_count(opts \\ %{}) do
base_query = from(p in Packet, select: count(p.id), where: p.has_position == true)
bounds = Map.get(opts, :bounds)
query =
base_query
|> filter_by_time(opts)
|> filter_by_region(opts)
|> filter_by_callsign(opts)
|> filter_by_map_bounds(opts)
Packet
|> QueryBuilder.with_position()
|> QueryBuilder.with_time_range(opts)
|> QueryBuilder.maybe_filter_region(opts)
|> maybe_filter_by_callsign(opts)
|> maybe_filter_by_bounds(bounds)
|> select(count())
Repo.one(query)
Repo.one(query) || 0
end
# Adds a select clause to populate virtual lat/lon fields from PostGIS geometry.
defp select_with_virtual_coordinates(query) do
from p in query,
select: %{p | lat: fragment("ST_Y(?)", p.location), lon: fragment("ST_X(?)", p.location)}
end
# Query building helpers
# Handle both start_time and end_time
defp filter_by_time(query, %{start_time: start_time, end_time: end_time}) do
from p in query,
where: p.received_at >= ^start_time and p.received_at <= ^end_time
end
# Handle only start_time
defp filter_by_time(query, %{start_time: start_time}) do
from p in query, where: p.received_at >= ^start_time
end
# Handle only end_time
defp filter_by_time(query, %{end_time: end_time}) do
from p in query, where: p.received_at <= ^end_time
end
# Default case
defp filter_by_time(query, _), do: query
@doc """
Gets recent packets for the map view.
This is used for initial map loading to show only recent packets.
@ -401,22 +425,22 @@ defmodule Aprsme.Packets do
@spec get_recent_packets(map()) :: [struct()]
def get_recent_packets(opts \\ %{}) do
# Always limit to the last 24 hours for more symbol variety
one_hour_ago = DateTime.add(DateTime.utc_now(), -24 * 3600, :second)
opts_with_time = Map.put(opts, :hours_back, 24)
# Merge the one-hour limit with any other filters
opts_with_time = Map.put(opts, :start_time, one_hour_ago)
get_packets_for_replay(opts_with_time)
opts_with_time
|> QueryBuilder.recent_position_packets()
|> Repo.all()
end
@doc """
Gets recent packets optimized for initial map load.
This uses a more efficient query pattern for the most common use case.
"""
@impl true
@spec get_recent_packets_optimized(map()) :: [struct()]
def get_recent_packets_optimized(opts \\ %{}) do
# Always limit to the last 24 hours for more symbol variety
one_hour_ago = DateTime.add(DateTime.utc_now(), -24 * 3600, :second)
one_hour_ago = TimeUtils.one_day_ago()
limit = Map.get(opts, :limit, 200)
offset = Map.get(opts, :offset, 0)
@ -429,19 +453,68 @@ defmodule Aprsme.Packets do
# Apply spatial and other filters BEFORE limiting
# This ensures we get the most recent packets within the specified bounds
bounds = Map.get(opts, :bounds)
query =
base_query
|> filter_by_region(opts)
|> filter_by_callsign(opts)
|> filter_by_map_bounds(opts)
|> QueryBuilder.maybe_filter_region(opts)
|> maybe_filter_by_callsign(opts)
|> maybe_filter_by_bounds(bounds)
|> order_by(desc: :received_at)
|> limit(^limit)
|> offset(^offset)
|> select_with_virtual_coordinates()
|> QueryBuilder.with_coordinates()
Repo.all(query)
end
@doc """
Gets the closest stations to a given point.
Uses PostGIS spatial indexes for efficient querying.
Returns only the most recent packet per callsign, ordered by distance.
"""
@impl true
@spec get_nearby_stations(float(), float(), String.t() | nil, map()) :: [struct()]
def get_nearby_stations(lat, lon, exclude_callsign \\ nil, opts \\ %{}) do
limit = Map.get(opts, :limit, 10)
hours_back = Map.get(opts, :hours_back, 1)
cutoff_time = TimeUtils.hours_ago(hours_back)
# Use a CTE with window function to get the most recent packet per callsign
# This approach maintains proper distance ordering while deduplicating by callsign
query = """
WITH ranked_packets AS (
SELECT p.*,
ST_Distance(p.location::geography, ST_SetSRID(ST_MakePoint($2, $1), 4326)::geography) as distance,
ROW_NUMBER() OVER (PARTITION BY p.base_callsign ORDER BY p.received_at DESC) as row_num
FROM packets p
WHERE p.has_position = true
AND p.received_at >= $3
#{if exclude_callsign, do: "AND p.sender != $4", else: ""}
)
SELECT *
FROM ranked_packets
WHERE row_num = 1
ORDER BY distance ASC
LIMIT #{if exclude_callsign, do: "$5", else: "$4"}
"""
params =
if exclude_callsign do
[lat, lon, cutoff_time, exclude_callsign, limit]
else
[lat, lon, cutoff_time, limit]
end
case Repo.query(query, params) do
{:ok, result} ->
Enum.map(result.rows, &Repo.load(Packet, {result.columns, &1}))
{:error, _} ->
[]
end
end
@doc """
Gets weather packets for a specific callsign within a time range.
This is optimized for weather queries by filtering at the database level.
@ -449,26 +522,21 @@ defmodule Aprsme.Packets do
@impl true
@spec get_weather_packets(String.t(), DateTime.t(), DateTime.t(), map()) :: [struct()]
def get_weather_packets(callsign, start_time, end_time, opts \\ %{}) do
limit = Map.get(opts, :limit, 500)
opts =
Map.merge(opts, %{
callsign: callsign,
start_time: start_time,
end_time: end_time,
limit: Map.get(opts, :limit, 500)
})
base_query = from(p in Packet, order_by: [desc: p.received_at])
query =
base_query
|> filter_by_time(%{start_time: start_time, end_time: end_time})
|> filter_by_callsign(%{callsign: callsign})
|> filter_weather_packets()
|> limit_results(%{limit: limit})
|> select_with_virtual_coordinates()
Repo.all(query)
opts
|> QueryBuilder.weather_packets()
|> QueryBuilder.with_coordinates()
|> Repo.all()
end
# Filter for weather packets at the database level
defp filter_weather_packets(query) do
from p in query,
where: p.data_type == "weather" or (p.symbol_table_id == "/" and p.symbol_code == "_")
end
@doc """
Retrieves a continuous stream of stored packets for replay in chronological order.
@ -504,72 +572,6 @@ defmodule Aprsme.Packets do
end)
end
defp filter_by_region(query, %{region: region}) do
from p in query, where: p.region == ^region
end
defp filter_by_region(query, _), do: query
defp filter_by_callsign(query, %{callsign: callsign}) do
# Use sender field for exact callsign matching
# Sanitize callsign to prevent SQL injection
sanitized_callsign = sanitize_callsign(callsign)
from p in query, where: ilike(p.sender, ^sanitized_callsign)
end
defp filter_by_callsign(query, _), do: query
defp filter_by_map_bounds(query, %{bounds: [min_lon, min_lat, max_lon, max_lat]})
when not is_nil(min_lon) and not is_nil(min_lat) and not is_nil(max_lon) and not is_nil(max_lat) do
bbox_wkt = create_bounding_box_wkt(min_lon, min_lat, max_lon, max_lat)
from p in query,
where: p.has_position == true,
# Use PostGIS spatial query if location is available
# Fall back to lat/lon comparison if location is null
where:
fragment(
"(? IS NOT NULL and ST_Within(?, ST_GeomFromText(?, 4326))) or (? IS NULL and ? >= ? and ? <= ? and ? >= ? and ? <= ?)",
p.location,
p.location,
^bbox_wkt,
p.location,
p.lat,
^min_lat,
p.lat,
^max_lat,
p.lon,
^min_lon,
p.lon,
^max_lon
)
end
defp filter_by_map_bounds(query, _), do: query
defp create_bounding_box_wkt(min_lon, min_lat, max_lon, max_lat) do
# Validate and sanitize coordinates to prevent SQL injection
with {:ok, min_lon} <- validate_coordinate(min_lon),
{:ok, min_lat} <- validate_coordinate(min_lat),
{:ok, max_lon} <- validate_coordinate(max_lon),
{:ok, max_lat} <- validate_coordinate(max_lat) do
"POLYGON((#{min_lon} #{min_lat}, #{max_lon} #{min_lat}, #{max_lon} #{max_lat}, #{min_lon} #{max_lat}, #{min_lon} #{min_lat}))"
else
_ -> raise ArgumentError, "Invalid coordinates provided"
end
end
defp limit_results(query, %{limit: limit, page: page}) when not is_nil(limit) and not is_nil(page) do
offset = (page - 1) * limit
from p in query, limit: ^limit, offset: ^offset
end
defp limit_results(query, %{limit: limit}) when not is_nil(limit) do
from p in query, limit: ^limit
end
defp limit_results(query, _), do: query
@doc """
Gets the total count of stored packets in the database.
"""
@ -636,20 +638,12 @@ defmodule Aprsme.Packets do
# Helper to convert various types to Decimal
defp to_decimal(value), do: Aprsme.EncodingUtils.to_decimal(value)
# Helper to sanitize all string fields in packet data before database storage
defp sanitize_packet_strings(value), do: Aprsme.EncodingUtils.sanitize_packet_strings(value)
# Get packets from last hour only - used to initialize the map
@spec get_last_hour_packets() :: [struct()]
def get_last_hour_packets do
one_hour_ago = DateTime.add(DateTime.utc_now(), -3600, :second)
Packet
|> where([p], p.has_position == true)
|> where([p], p.received_at >= ^one_hour_ago)
|> order_by([p], asc: p.received_at)
|> limit(500)
|> select_with_virtual_coordinates()
%{hours_back: 1, limit: 500}
|> QueryBuilder.recent_position_packets()
|> QueryBuilder.chronological()
|> Repo.all()
end
@ -735,44 +729,49 @@ defmodule Aprsme.Packets do
"""
@spec get_latest_packet_for_callsign(String.t()) :: struct() | nil
def get_latest_packet_for_callsign(callsign) when is_binary(callsign) do
sanitized_callsign = sanitize_callsign(callsign)
from(p in Packet,
where: ilike(p.sender, ^sanitized_callsign),
order_by: [desc: p.received_at],
limit: 1
)
|> select_with_virtual_coordinates()
callsign
|> QueryBuilder.callsign_history(%{limit: 1})
|> QueryBuilder.with_coordinates()
|> Repo.one()
end
# Sanitization functions to prevent SQL injection
defp sanitize_callsign(callsign) when is_binary(callsign) do
# APRS callsigns are alphanumeric with hyphens and up to 9 characters
# Remove any characters that could be used for SQL injection
callsign
|> String.upcase()
|> String.replace(~r/[^A-Z0-9\-]/, "")
|> String.slice(0, 9)
end
@doc """
Gets the most recent weather packet for a callsign.
Optimized to check only recent data first before expanding the search window.
"""
@spec get_latest_weather_packet_optimized(String.t()) :: struct() | nil
def get_latest_weather_packet_optimized(callsign) when is_binary(callsign) do
# First try last 24 hours
one_day_ago = TimeUtils.one_day_ago()
defp sanitize_callsign(_), do: ""
recent_packet =
Repo.one(
from(p in Packet,
where: p.sender == ^callsign,
where: p.data_type == "weather",
where: p.received_at >= ^one_day_ago,
order_by: [desc: p.received_at],
limit: 1
)
)
defp validate_coordinate(coord) when is_number(coord) do
# Validate coordinate ranges
if coord >= -180 and coord <= 180 do
{:ok, coord}
else
{:error, :invalid_coordinate}
case recent_packet do
nil ->
# If no recent packet, expand to 7 days
one_week_ago = TimeUtils.one_week_ago()
Repo.one(
from(p in Packet,
where: p.sender == ^callsign,
where: p.data_type == "weather",
where: p.received_at >= ^one_week_ago,
order_by: [desc: p.received_at],
limit: 1
)
)
packet ->
packet
end
end
defp validate_coordinate(coord) when is_binary(coord) do
case Float.parse(coord) do
{float_val, ""} -> validate_coordinate(float_val)
_ -> {:error, :invalid_coordinate}
end
end
defp validate_coordinate(_), do: {:error, :invalid_coordinate}
end

220
lib/aprsme/packets/query_builder.ex Normal file
View file

@ -0,0 +1,220 @@
defmodule Aprsme.Packets.QueryBuilder do
@moduledoc """
Query builder functions for composing Packet queries.
Provides reusable, composable functions for common query patterns.
"""
import Ecto.Query
alias Aprsme.Packet
@doc """
Filters query by time range. Supports start_time, end_time, and hours_back options.
## Options
* `:start_time` - DateTime to filter packets after
* `:end_time` - DateTime to filter packets before
* `:hours_back` - Alternative to start_time, filters packets from X hours ago
"""
@spec with_time_range(Ecto.Query.t(), map() | keyword()) :: Ecto.Query.t()
def with_time_range(query, opts) when is_list(opts) do
with_time_range(query, Map.new(opts))
end
def with_time_range(query, opts) when is_map(opts) do
query
|> maybe_filter_start_time(opts[:start_time] || opts["start_time"])
|> maybe_filter_end_time(opts[:end_time] || opts["end_time"])
|> maybe_filter_hours_back(opts[:hours_back] || opts["hours_back"])
end
@doc """
Filters query to only include packets with position data.
"""
@spec with_position(Ecto.Query.t()) :: Ecto.Query.t()
def with_position(query) do
from p in query, where: p.has_position == true
end
@doc """
Filters query by exact callsign match (case-insensitive).
"""
@spec for_callsign(Ecto.Query.t(), String.t()) :: Ecto.Query.t()
def for_callsign(query, callsign) when is_binary(callsign) do
normalized = String.upcase(String.trim(callsign))
from p in query, where: ilike(p.sender, ^normalized)
end
@doc """
Filters query by base callsign (without SSID).
"""
@spec for_base_callsign(Ecto.Query.t(), String.t()) :: Ecto.Query.t()
def for_base_callsign(query, base_callsign) when is_binary(base_callsign) do
from p in query, where: p.base_callsign == ^base_callsign
end
@doc """
Orders query by received_at descending (most recent first).
"""
@spec recent_first(Ecto.Query.t()) :: Ecto.Query.t()
def recent_first(query) do
from p in query, order_by: [desc: p.received_at]
end
@doc """
Orders query by received_at ascending (oldest first).
"""
@spec chronological(Ecto.Query.t()) :: Ecto.Query.t()
def chronological(query) do
from p in query, order_by: [asc: p.received_at]
end
@doc """
Filters query to only weather packets.
"""
@spec weather_only(Ecto.Query.t()) :: Ecto.Query.t()
def weather_only(query) do
from p in query,
where: p.data_type == "weather" or (p.symbol_table_id == "/" and p.symbol_code == "_")
end
@doc """
Applies pagination with limit and optional offset.
"""
@spec paginate(Ecto.Query.t(), integer(), integer()) :: Ecto.Query.t()
def paginate(query, limit, offset \\ 0) when is_integer(limit) and is_integer(offset) do
from p in query, limit: ^limit, offset: ^offset
end
@doc """
Adds PostGIS coordinate extraction to select clause.
"""
@spec with_coordinates(Ecto.Query.t()) :: Ecto.Query.t()
def with_coordinates(query) do
from p in query,
select: %{p | lat: fragment("ST_Y(?)", p.location), lon: fragment("ST_X(?)", p.location)}
end
@doc """
Filters by region if provided in options.
"""
@spec maybe_filter_region(Ecto.Query.t(), map() | keyword()) :: Ecto.Query.t()
def maybe_filter_region(query, opts) when is_list(opts) do
maybe_filter_region(query, Map.new(opts))
end
def maybe_filter_region(query, %{region: region}) when not is_nil(region) do
from p in query, where: p.region == ^region
end
def maybe_filter_region(query, _), do: query
@doc """
Filters by map bounds using bounding box coordinates.
## Example
within_bounds(query, [-74.0, 40.0, -73.0, 41.0])
# [west, south, east, north]
"""
@spec within_bounds(Ecto.Query.t(), list(number())) :: Ecto.Query.t()
def within_bounds(query, [west, south, east, north])
when is_number(west) and is_number(south) and is_number(east) and is_number(north) do
from p in query,
where: p.has_position == true,
where: p.lat >= ^south and p.lat <= ^north,
where: p.lon >= ^west and p.lon <= ^east
end
def within_bounds(query, _), do: query
@doc """
Common query composition for recent position packets.
## Options
* `:limit` - Number of packets to return (default: 100)
* `:start_time` - Filter packets after this time
* `:end_time` - Filter packets before this time
* `:region` - Filter by region
"""
@spec recent_position_packets(map() | keyword()) :: Ecto.Query.t()
def recent_position_packets(opts \\ %{}) do
limit = opts[:limit] || opts["limit"] || 100
Packet
|> with_position()
|> with_time_range(opts)
|> maybe_filter_region(opts)
|> recent_first()
|> paginate(limit)
|> with_coordinates()
end
@doc """
Common query composition for callsign packet history.
## Options
* `:limit` - Number of packets to return (default: 100)
* `:start_time` - Filter packets after this time
* `:end_time` - Filter packets before this time
"""
@spec callsign_history(String.t(), map() | keyword()) :: Ecto.Query.t()
def callsign_history(callsign, opts \\ %{}) do
limit = opts[:limit] || opts["limit"] || 100
Packet
|> for_callsign(callsign)
|> with_time_range(opts)
|> recent_first()
|> paginate(limit)
end
@doc """
Common query composition for weather packets.
## Options
* `:callsign` - Filter by specific callsign
* `:limit` - Number of packets to return (default: 100)
* `:start_time` - Filter packets after this time
* `:end_time` - Filter packets before this time
"""
@spec weather_packets(map() | keyword()) :: Ecto.Query.t()
def weather_packets(opts \\ %{}) do
limit = opts[:limit] || opts["limit"] || 100
callsign = opts[:callsign] || opts["callsign"]
query =
Packet
|> weather_only()
|> with_time_range(opts)
|> recent_first()
|> paginate(limit)
if callsign do
for_callsign(query, callsign)
else
query
end
end
# Private helper functions
defp maybe_filter_start_time(query, nil), do: query
defp maybe_filter_start_time(query, start_time) do
from p in query, where: p.received_at >= ^start_time
end
defp maybe_filter_end_time(query, nil), do: query
defp maybe_filter_end_time(query, end_time) do
from p in query, where: p.received_at <= ^end_time
end
defp maybe_filter_hours_back(query, nil), do: query
defp maybe_filter_hours_back(query, hours) when is_number(hours) do
start_time = DateTime.add(DateTime.utc_now(), -hours * 3600, :second)
from p in query, where: p.received_at >= ^start_time
end
end

2
lib/aprsme/packets_behaviour.ex
View file

@ -7,6 +7,8 @@ defmodule Aprsme.PacketsBehaviour do
@callback stream_packets_for_replay(map()) :: Enumerable.t()
@callback get_packets_for_replay(map()) :: list()
@callback get_recent_packets(map()) :: list()
@callback get_recent_packets_optimized(map()) :: list()
@callback get_nearby_stations(float(), float(), String.t() | nil, map()) :: list()
@callback get_weather_packets(String.t(), DateTime.t(), DateTime.t(), map()) :: list()
@callback clean_old_packets() :: {:ok, non_neg_integer()} | {:error, any()}
@callback clean_packets_older_than(pos_integer()) :: {:ok, non_neg_integer()} | {:error, any()}

3
lib/aprsme_web/live/api_docs_live.ex
View file

@ -61,6 +61,7 @@ defmodule AprsmeWeb.ApiDocsLive do
defp make_http_request(callsign) do
alias Aprsme.Packets
alias AprsmeWeb.TimeUtils
# Validate callsign format (same as controller)
callsign_regex = ~r/^[A-Z0-9]{1,3}[0-9][A-Z]{1,4}(-[0-9]{1,2})?$/
@ -76,7 +77,7 @@ defmodule AprsmeWeb.ApiDocsLive do
{:ok, Jason.encode!(response, pretty: true)}
else
# Get packet data (same logic as controller)
thirty_days_ago = DateTime.add(DateTime.utc_now(), -30, :day)
thirty_days_ago = TimeUtils.days_ago(30)
opts = %{
callsign: callsign,

210
lib/aprsme_web/live/info_live/show.ex
View file

@ -3,16 +3,19 @@ defmodule AprsmeWeb.InfoLive.Show do
use AprsmeWeb, :live_view
use Gettext, backend: AprsmeWeb.Gettext
alias Aprsme.Callsign
alias Aprsme.EncodingUtils
alias Aprsme.Packets
alias AprsmeWeb.AprsSymbol
alias AprsmeWeb.Live.SharedPacketHandler
alias AprsmeWeb.MapLive.PacketUtils
alias AprsmeWeb.TimeUtils
@neighbor_radius_km 10
@neighbor_limit 10
@impl true
def mount(%{"callsign" => callsign}, _session, socket) do
normalized_callsign = String.upcase(String.trim(callsign))
normalized_callsign = Callsign.normalize(callsign)
# Subscribe to Postgres notifications for live updates
if connected?(socket) do
@ -20,7 +23,7 @@ defmodule AprsmeWeb.InfoLive.Show do
end
packet = get_latest_packet(normalized_callsign)
packet = enrich_packet_with_device_info(packet)
packet = if packet, do: SharedPacketHandler.enrich_with_device_info(packet)
# Get locale from socket assigns (set by LocaleHook)
locale = Map.get(socket.assigns, :locale, "en")
neighbors = get_neighbors(packet, normalized_callsign, locale)
@ -41,61 +44,40 @@ defmodule AprsmeWeb.InfoLive.Show do
@impl true
def handle_info({:postgres_packet, packet}, socket) do
# Only update if the packet is for our callsign
if packet_matches_callsign?(packet, socket.assigns.callsign) do
# Refresh data when new packet arrives
packet = get_latest_packet(socket.assigns.callsign)
packet = enrich_packet_with_device_info(packet)
# Get locale from socket assigns
locale = Map.get(socket.assigns, :locale, "en")
neighbors = get_neighbors(packet, socket.assigns.callsign, locale)
has_weather_packets = PacketUtils.has_weather_packets?(socket.assigns.callsign)
other_ssids = get_other_ssids(socket.assigns.callsign)
socket =
socket
|> assign(:packet, packet)
|> assign(:neighbors, neighbors)
|> assign(:has_weather_packets, has_weather_packets)
|> assign(:other_ssids, other_ssids)
{:noreply, socket}
else
{:noreply, socket}
end
SharedPacketHandler.handle_packet_update(packet, socket,
filter_fn: SharedPacketHandler.callsign_filter(socket.assigns.callsign),
process_fn: &process_packet_update/2,
enrich_packet: false
)
end
def handle_info(_message, socket), do: {:noreply, socket}
defp packet_matches_callsign?(packet, callsign) do
packet_sender = Map.get(packet, "sender") || Map.get(packet, :sender, "")
String.upcase(packet_sender) == String.upcase(callsign)
defp process_packet_update(_packet, socket) do
# Refresh data when new packet arrives
packet = get_latest_packet(socket.assigns.callsign)
packet = if packet, do: SharedPacketHandler.enrich_with_device_info(packet)
# Get locale from socket assigns
locale = Map.get(socket.assigns, :locale, "en")
neighbors = get_neighbors(packet, socket.assigns.callsign, locale)
has_weather_packets = PacketUtils.has_weather_packets?(socket.assigns.callsign)
other_ssids = get_other_ssids(socket.assigns.callsign)
socket =
socket
|> assign(:packet, packet)
|> assign(:neighbors, neighbors)
|> assign(:has_weather_packets, has_weather_packets)
|> assign(:other_ssids, other_ssids)
{:noreply, socket}
end
defp get_latest_packet(callsign) do
# Get the most recent packet for this callsign, regardless of type
# This ensures we show the most recent activity, not just position packets
Packets.get_latest_packet_for_callsign(callsign)
end
defp enrich_packet_with_device_info(nil), do: nil
defp enrich_packet_with_device_info(packet) do
device_identifier = Map.get(packet, :device_identifier) || Map.get(packet, "device_identifier")
device =
if is_binary(device_identifier), do: Aprsme.DeviceIdentification.lookup_device_by_identifier(device_identifier)
model = if device, do: device.model
vendor = if device, do: device.vendor
contact = if device, do: device.contact
class = if device, do: device.class
packet
|> Map.put(:device_model, model)
|> Map.put(:device_vendor, vendor)
|> Map.put(:device_contact, contact)
|> Map.put(:device_class, class)
# Use cached version for better performance
Aprsme.CachedQueries.get_latest_packet_for_callsign_cached(callsign)
end
defp get_neighbors(nil, _callsign, _locale), do: []
@ -107,52 +89,38 @@ defmodule AprsmeWeb.InfoLive.Show do
if is_nil(lat) or is_nil(lon) do
[]
else
# Simple bounding box for ~10km radius
delta = @neighbor_radius_km / 111.0
min_lat = lat - delta
max_lat = lat + delta
min_lon = lon - delta
max_lon = lon + delta
opts = %{bounds: [min_lon, min_lat, max_lon, max_lat], limit: 50}
# Convert Decimal to float if needed
lat_float = to_float(lat)
lon_float = to_float(lon)
opts
|> Packets.get_recent_packets()
|> Enum.filter(fn p ->
(p.sender != callsign and p.lat) && p.lon
end)
|> uniq_by(& &1.sender)
|> Enum.map(fn p ->
# Use the spatial query to get the closest stations
# The query already returns them sorted by distance
nearby_packets =
Packets.get_nearby_stations(
lat_float,
lon_float,
callsign,
%{limit: @neighbor_limit}
)
# Calculate distance and course for each neighbor
# The packets are already sorted by distance from the database
Enum.map(nearby_packets, fn p ->
dist = haversine(lat, lon, p.lat, p.lon)
course = calculate_course(lat, lon, p.lat, p.lon)
%{
callsign: p.sender,
distance: format_distance(dist, locale),
distance_km: dist,
course: course,
last_heard: format_timestamp_for_display(p),
packet: p
}
end)
|> Enum.sort_by(& &1.distance)
|> Enum.take(@neighbor_limit)
end
end
defp uniq_by(list, fun) do
list
|> Enum.reduce({MapSet.new(), []}, fn item, {set, acc} ->
key = fun.(item)
if MapSet.member?(set, key) do
{set, acc}
else
{MapSet.put(set, key), [item | acc]}
end
end)
|> elem(1)
|> Enum.reverse()
end
def haversine(lat1, lon1, lat2, lon2) do
# Returns distance in km
# Convert Decimal to float if needed
@ -174,18 +142,7 @@ defmodule AprsmeWeb.InfoLive.Show do
r * c
end
defp to_float(%Decimal{} = decimal), do: Decimal.to_float(decimal)
defp to_float(value) when is_float(value), do: value
defp to_float(value) when is_integer(value), do: value * 1.0
defp to_float(value) when is_binary(value) do
case Float.parse(value) do
{f, _} -> f
:error -> 0.0
end
end
defp to_float(_), do: 0.0
defp to_float(value), do: EncodingUtils.to_float(value) || 0.0
defp format_timestamp_for_display(packet) do
received_at = get_received_at(packet)
@ -265,46 +222,57 @@ defmodule AprsmeWeb.InfoLive.Show do
end
defp get_other_ssids(callsign) do
import Ecto.Query
alias Aprsme.Packet
alias Aprsme.Repo
# Extract base callsign from the full callsign (remove SSID if present)
base_callsign = extract_base_callsign(callsign)
# Query directly for packets with the same base_callsign
# Get recent packets for the base callsign to find other SSIDs
one_hour_ago = DateTime.add(DateTime.utc_now(), -3600, :second)
one_hour_ago = TimeUtils.one_hour_ago()
query =
from p in Packet,
where: p.base_callsign == ^base_callsign,
where: p.received_at >= ^one_hour_ago,
order_by: [desc: p.received_at],
limit: 100
# Use a window function to get the most recent packet per sender
# This is much more efficient than fetching 100 packets and filtering in Elixir
query = """
WITH recent_ssids AS (
SELECT DISTINCT ON (sender)
sender, ssid, received_at, id
FROM packets
WHERE base_callsign = $1
AND received_at >= $2
AND sender != $3
ORDER BY sender, received_at DESC
)
SELECT * FROM recent_ssids
ORDER BY received_at DESC
LIMIT 10
"""
query
|> Repo.all()
|> Enum.map(fn p ->
%{
callsign: p.sender,
ssid: p.ssid,
last_heard: format_timestamp_for_display(p),
packet: p
}
end)
|> uniq_by(& &1.callsign)
|> Enum.filter(fn ssid_info -> ssid_info.callsign != callsign end)
|> Enum.sort_by(& &1.last_heard, :desc)
|> Enum.take(10)
case Repo.query(query, [base_callsign, one_hour_ago, callsign]) do
{:ok, result} ->
Enum.map(result.rows, fn [sender, ssid, received_at, id] ->
# Create a minimal packet struct for display
packet = %Packet{
id: id,
sender: sender,
ssid: ssid,
received_at: received_at
}
%{
callsign: sender,
ssid: ssid,
last_heard: format_timestamp_for_display(packet),
packet: packet
}
end)
{:error, _} ->
[]
end
end
defp extract_base_callsign(callsign) do
case String.split(callsign, "-") do
[base, _ssid] -> base
[base] -> base
end
Callsign.extract_base(callsign)
end
@doc """

29
lib/aprsme_web/live/map_live/callsign_view.ex
View file

@ -4,10 +4,13 @@ defmodule AprsmeWeb.MapLive.CallsignView do
import Phoenix.LiveView, only: [connected?: 1, push_event: 3]
alias Aprsme.Callsign
alias Aprsme.EncodingUtils
alias Aprsme.Packets
alias AprsmeWeb.Endpoint
alias AprsmeWeb.MapLive.MapHelpers
alias AprsmeWeb.MapLive.PacketUtils
alias AprsmeWeb.TimeUtils
@default_center %{lat: 39.0, lng: -98.0}
@default_zoom 4
@ -16,10 +19,10 @@ defmodule AprsmeWeb.MapLive.CallsignView do
@impl true
def mount(%{"callsign" => callsign}, _session, socket) do
# Normalize callsign to uppercase
normalized_callsign = String.upcase(callsign)
normalized_callsign = Callsign.normalize(callsign)
# Calculate one hour ago for packet age filtering
one_hour_ago = DateTime.add(DateTime.utc_now(), -3600, :second)
one_hour_ago = TimeUtils.one_hour_ago()
socket =
assign(socket,
@ -142,7 +145,7 @@ defmodule AprsmeWeb.MapLive.CallsignView do
end
def handle_event("adjust_replay_speed", %{"speed" => speed}, socket) do
speed_float = to_float(speed)
speed_float = EncodingUtils.to_float(speed) || 1.0
{:noreply, assign(socket, replay_speed: speed_float)}
end
@ -207,12 +210,7 @@ defmodule AprsmeWeb.MapLive.CallsignView do
defp handle_bounds_update(bounds, socket) do
# Convert string keys to atom keys and parse values
normalized_bounds = %{
north: to_float(bounds["north"]),
south: to_float(bounds["south"]),
east: to_float(bounds["east"]),
west: to_float(bounds["west"])
}
normalized_bounds = MapHelpers.normalize_bounds(bounds)
# Clean up old packets first
socket = cleanup_old_packets(socket)
@ -252,17 +250,6 @@ defmodule AprsmeWeb.MapLive.CallsignView do
{:noreply, socket}
end
# Helper function to convert string or float to float
defp to_float(value) when is_float(value), do: value
defp to_float(value) when is_integer(value), do: value * 1.0
defp to_float(value) when is_binary(value) do
case Float.parse(value) do
{float_val, _} -> float_val
:error -> raise ArgumentError, "Invalid float string: #{value}"
end
end
@impl true
def handle_info({:zoom_to_location, lat, lng, zoom}, socket) do
socket = push_event(socket, "zoom_to_location", %{lat: lat, lng: lng, zoom: zoom})
@ -732,7 +719,7 @@ defmodule AprsmeWeb.MapLive.CallsignView do
defp start_historical_replay(socket) do
# Fetch historical packets for the specific callsign from the last hour
now = DateTime.utc_now()
one_hour_ago = DateTime.add(now, -3600, :second)
one_hour_ago = TimeUtils.one_hour_ago()
packets =
fetch_historical_packets_for_callsign(

10
lib/aprsme_web/live/map_live/index.ex
View file

@ -12,6 +12,7 @@ defmodule AprsmeWeb.MapLive.Index do
alias AprsmeWeb.MapLive.MapHelpers
alias AprsmeWeb.MapLive.PacketUtils
alias AprsmeWeb.MapLive.PopupComponent
alias AprsmeWeb.TimeUtils
alias Phoenix.HTML.Safe
alias Phoenix.LiveView.Socket
@ -79,7 +80,7 @@ defmodule AprsmeWeb.MapLive.Index do
deployed_at = Aprsme.Release.deployed_at()
# Show 24 hours for more symbol variety
one_hour_ago = DateTime.add(DateTime.utc_now(), -24 * 3600, :second)
one_hour_ago = TimeUtils.one_day_ago()
# Parse map state from URL parameters
{map_center, map_zoom} = parse_map_params(params)
@ -1655,12 +1656,7 @@ defmodule AprsmeWeb.MapLive.Index do
@spec handle_bounds_update(map(), Socket.t()) :: {:noreply, Socket.t()}
defp handle_bounds_update(bounds, socket) do
# Update the map bounds from the client, converting to atom keys
map_bounds = %{
north: bounds["north"],
south: bounds["south"],
east: bounds["east"],
west: bounds["west"]
}
map_bounds = MapHelpers.normalize_bounds(bounds)
# Validate bounds to prevent invalid coordinates
if valid_bounds?(map_bounds) do

39
lib/aprsme_web/live/map_live/map_helpers.ex
View file

@ -106,16 +106,35 @@ defmodule AprsmeWeb.MapLive.MapHelpers do
defp extract_lat_lon_string(packet), do: {packet["lat"], packet["lon"]}
defp extract_lat_lon_atom_alt(packet), do: {packet.latitude, packet.longitude}
defp to_float(n) when is_float(n), do: n
defp to_float(n) when is_integer(n), do: n * 1.0
defp to_float(%Decimal{} = d), do: Decimal.to_float(d)
defp to_float(n) when is_binary(n) do
case Float.parse(n) do
{f, _} -> f
:error -> 0.0
end
defp to_float(value) do
Aprsme.EncodingUtils.to_float(value) || 0.0
end
defp to_float(_), do: 0.0
@doc """
Normalizes map bounds from string keys to atom keys and converts values to floats.
## Examples
iex> normalize_bounds(%{"north" => "40.5", "south" => "40.0", "east" => "-73.5", "west" => "-74.0"})
%{north: 40.5, south: 40.0, east: -73.5, west: -74.0}
iex> normalize_bounds(%{"north" => 40.5, "south" => 40.0, "east" => -73.5, "west" => -74.0})
%{north: 40.5, south: 40.0, east: -73.5, west: -74.0}
"""
@spec normalize_bounds(map()) :: map()
def normalize_bounds(%{"north" => n, "south" => s, "east" => e, "west" => w}) do
%{
north: to_float(n),
south: to_float(s),
east: to_float(e),
west: to_float(w)
}
end
def normalize_bounds(%{north: _, south: _, east: _, west: _} = bounds) do
# Already normalized with atom keys
bounds
end
def normalize_bounds(_), do: nil
end

13
lib/aprsme_web/live/map_live/packet_utils.ex
View file

@ -88,17 +88,8 @@ defmodule AprsmeWeb.MapLive.PacketUtils do
@spec has_weather_packets?(String.t()) :: boolean()
# Get recent packets for this callsign and check if any are weather packets
def has_weather_packets?(callsign) when is_binary(callsign) do
# Use a more efficient query that only checks for existence
import Ecto.Query
query =
from p in Aprsme.Packet,
where: ilike(p.sender, ^callsign),
where: p.data_type == "weather" or (p.symbol_table_id == "/" and p.symbol_code == "_"),
limit: 1,
select: true
Aprsme.Repo.exists?(query)
# Use cached query for better performance
Aprsme.CachedQueries.has_weather_packets_cached?(callsign)
rescue
_ -> false
end

62
lib/aprsme_web/live/packets_live/callsign_view.ex
View file

@ -4,18 +4,20 @@ defmodule AprsmeWeb.PacketsLive.CallsignView do
import Ecto.Query
alias Aprsme.Callsign
alias Aprsme.DeviceParser
alias Aprsme.EncodingUtils
alias Aprsme.Packet
alias Aprsme.Repo
alias AprsmeWeb.Endpoint
alias AprsmeWeb.Live.SharedPacketHandler
@impl true
def mount(%{"callsign" => callsign}, _session, socket) do
# Validate and normalize callsign
normalized_callsign = String.upcase(String.trim(callsign))
normalized_callsign = Callsign.normalize(callsign)
if valid_callsign?(normalized_callsign) do
if Callsign.valid?(normalized_callsign) do
# Subscribe to live packet updates if connected
if connected?(socket) do
Endpoint.subscribe("aprs_messages")
@ -55,20 +57,17 @@ defmodule AprsmeWeb.PacketsLive.CallsignView do
@impl true
def handle_info(%{event: "packet", payload: payload}, socket) do
# Handle incoming live packets - only process if they match our callsign
if packet_matches_callsign?(payload, socket.assigns.callsign) do
process_matching_packet(payload, socket)
else
{:noreply, socket}
end
SharedPacketHandler.handle_packet_update(payload, socket,
filter_fn: SharedPacketHandler.callsign_filter(socket.assigns.callsign),
process_fn: &process_matching_packet/2
)
end
def handle_info(_message, socket), do: {:noreply, socket}
defp process_matching_packet(payload, socket) do
sanitized_payload = EncodingUtils.sanitize_packet(payload)
enriched_payload = enrich_packet_with_device_identifier(sanitized_payload)
enriched_payload = enrich_packet_with_device_info(enriched_payload)
defp process_matching_packet(enriched_payload, socket) do
# Device identifier enrichment specific to this module
enriched_payload = enrich_packet_with_device_identifier(enriched_payload)
current_live = socket.assigns.live_packets
current_stored = socket.assigns.packets
@ -123,7 +122,7 @@ defmodule AprsmeWeb.PacketsLive.CallsignView do
filtered_query
|> Repo.all()
|> Enum.map(&EncodingUtils.sanitize_packet/1)
|> Enum.map(&enrich_packet_with_device_info/1)
|> Enum.map(&SharedPacketHandler.enrich_with_device_info/1)
rescue
error ->
require Logger
@ -132,18 +131,6 @@ defmodule AprsmeWeb.PacketsLive.CallsignView do
[]
end
defp packet_matches_callsign?(packet, target_callsign) do
# Check exact match for sender field only
sender = packet.sender || ""
# Convert to uppercase for case-insensitive comparison
sender_upper = String.upcase(sender)
target_upper = String.upcase(target_callsign)
# Exact match only
sender_upper == target_upper
end
# Helper to get all packets (stored + live) in chronological order
# Combines stored and live packets, sorts by timestamp (newest first), and limits to 100
defp get_all_packets_list(stored, live) do
@ -188,17 +175,6 @@ defmodule AprsmeWeb.PacketsLive.CallsignView do
end
end
# Validates if the callsign format is reasonable
defp valid_callsign?(callsign) do
# Basic validation for amateur radio callsign format
# Should be 3-8 characters, can contain letters, numbers, and one hyphen for SSID
case String.trim(callsign) do
"" -> false
cs when byte_size(cs) < 3 or byte_size(cs) > 15 -> false
cs -> Regex.match?(~r/^[A-Z0-9]+(-[A-Z0-9]{1,2})?$/i, cs)
end
end
# Helper to extract symbol table and code from a packet
defp extract_symbol_info(nil), do: {"/", ">"}
@ -208,18 +184,4 @@ defmodule AprsmeWeb.PacketsLive.CallsignView do
code = Map.get(data, :symbol_code) || Map.get(data, "symbol_code") || ">"
{table, code}
end
defp enrich_packet_with_device_info(packet) do
device_identifier = Map.get(packet, :device_identifier) || Map.get(packet, "device_identifier")
device =
if is_binary(device_identifier), do: Aprsme.DeviceIdentification.lookup_device_by_identifier(device_identifier)
model = if device, do: device.model
vendor = if device, do: device.vendor
packet
|> Map.put(:device_model, model)
|> Map.put(:device_vendor, vendor)
end
end

126
lib/aprsme_web/live/shared/packet_handler.ex Normal file
View file

@ -0,0 +1,126 @@
defmodule AprsmeWeb.Live.SharedPacketHandler do
@moduledoc """
Shared functionality for handling packet updates in LiveView modules.
Provides common patterns for packet filtering, enrichment, and processing.
"""
alias Aprsme.Callsign
alias Aprsme.DeviceIdentification
alias Aprsme.DeviceParser
alias Aprsme.EncodingUtils
@doc """
Handles incoming packet updates with filtering and processing.
## Options
* `:filter_fn` - Function to determine if packet should be processed (required)
* `:process_fn` - Function to process matching packets and update socket (required)
* `:enrich_packet` - Whether to enrich packet with device info (default: true)
"""
def handle_packet_update(packet, socket, opts) do
filter_fn = Keyword.fetch!(opts, :filter_fn)
process_fn = Keyword.fetch!(opts, :process_fn)
enrich_packet? = Keyword.get(opts, :enrich_packet, true)
if filter_fn.(packet, socket) do
enriched_packet =
if enrich_packet? do
packet
|> sanitize_packet()
|> enrich_with_device_info()
else
packet
end
process_fn.(enriched_packet, socket)
else
{:noreply, socket}
end
end
@doc """
Checks if packet sender matches the given callsign.
"""
def packet_matches_callsign?(packet, callsign) do
packet_sender = Map.get(packet, "sender") || Map.get(packet, :sender, "")
Callsign.matches?(packet_sender, callsign)
end
@doc """
Checks if packet contains weather data.
"""
def has_weather_data?(packet) do
Enum.any?(EncodingUtils.weather_fields(), fn field ->
value = Map.get(packet, field) || Map.get(packet, to_string(field))
not is_nil(value)
end)
end
@doc """
Sanitizes packet strings to handle encoding issues.
"""
def sanitize_packet(packet) do
EncodingUtils.sanitize_packet(packet)
end
@doc """
Enriches packet with device information.
"""
def enrich_with_device_info(packet) do
device_identifier = extract_device_identifier(packet)
device =
case device_identifier do
nil -> nil
"" -> nil
identifier -> DeviceIdentification.lookup_device_by_identifier(identifier)
end
packet
|> Map.put(:device_model, device && device.model)
|> Map.put(:device_vendor, device && device.vendor)
|> Map.put(:device_contact, device && device.contact)
|> Map.put(:device_class, device && device.class)
end
@doc """
Creates a filter function that checks both callsign and weather data.
"""
def callsign_and_weather_filter(callsign) do
fn packet, _socket ->
packet_matches_callsign?(packet, callsign) and has_weather_data?(packet)
end
end
@doc """
Creates a filter function that only checks callsign.
"""
def callsign_filter(callsign) do
fn packet, _socket ->
packet_matches_callsign?(packet, callsign)
end
end
# Private functions
defp extract_device_identifier(packet) do
comment = Map.get(packet, :comment) || Map.get(packet, "comment") || ""
normalized_comment =
case comment do
comment when is_binary(comment) ->
comment
|> String.trim()
|> String.replace(~r/[^\x20-\x7E]+/, "")
|> String.trim()
_ ->
""
end
case DeviceParser.extract_device_identifier(%{comment: normalized_comment}) do
nil -> nil
identifier -> String.trim(identifier)
end
end
end

311
lib/aprsme_web/live/weather_live/callsign_view.ex
View file

@ -5,13 +5,14 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
import Phoenix.LiveView, only: [push_event: 3, connected?: 1]
alias Aprsme.Packets
alias Aprsme.Callsign
alias AprsmeWeb.MapLive.PacketUtils
alias AprsmeWeb.TimeUtils
alias AprsmeWeb.WeatherUnits
@impl true
def mount(%{"callsign" => callsign}, _session, socket) do
normalized_callsign = String.upcase(String.trim(callsign))
normalized_callsign = Callsign.normalize(callsign)
# Subscribe to weather-specific topic for targeted updates
if connected?(socket) do
@ -27,37 +28,7 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
locale = Map.get(socket.assigns, :locale, "en")
unit_labels = WeatherUnits.unit_labels(locale)
weather_history_json =
weather_history
|> Enum.map(fn pkt ->
dew_point =
if is_number(pkt.temperature) and is_number(pkt.humidity) do
calc_dew_point(pkt.temperature, pkt.humidity)
end
# Convert units based on locale
{temp_value, _} = WeatherUnits.format_temperature(pkt.temperature, locale)
{dew_value, _} = if dew_point, do: WeatherUnits.format_temperature(dew_point, locale), else: {nil, nil}
{wind_speed_value, _} = WeatherUnits.format_wind_speed(pkt.wind_speed, locale)
{rain_1h_value, _} = WeatherUnits.format_rain(pkt.rain_1h, locale)
{rain_24h_value, _} = WeatherUnits.format_rain(pkt.rain_24h, locale)
{rain_since_midnight_value, _} = WeatherUnits.format_rain(pkt.rain_since_midnight, locale)
%{
timestamp: pkt.received_at,
temperature: temp_value,
dew_point: dew_value,
humidity: pkt.humidity,
pressure: pkt.pressure,
wind_direction: pkt.wind_direction,
wind_speed: wind_speed_value,
rain_1h: rain_1h_value,
rain_24h: rain_24h_value,
rain_since_midnight: rain_since_midnight_value,
luminosity: pkt.luminosity
}
end)
|> Jason.encode!()
weather_history_json = convert_weather_history_to_json(weather_history, locale)
# Add chart labels for translation with locale-aware units
chart_labels = %{
@ -90,8 +61,38 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
end
@impl true
def handle_info({:weather_packet, _packet}, socket) do
# Update weather data when new weather packet arrives for this callsign
def handle_info({:weather_packet, packet}, socket) do
# Check if this packet is actually newer than what we have
current_packet = socket.assigns.weather_packet
if should_update_weather?(current_packet, packet) do
update_weather_data(socket)
else
{:noreply, socket}
end
end
# Keep the old handler for backward compatibility
def handle_info({:postgres_packet, packet}, socket) do
# Only process if it's a weather packet for this callsign
if packet.sender == socket.assigns.callsign && packet.data_type == "weather" do
handle_info({:weather_packet, packet}, socket)
else
{:noreply, socket}
end
end
def handle_info(_message, socket), do: {:noreply, socket}
defp should_update_weather?(nil, _new_packet), do: true
defp should_update_weather?(current_packet, new_packet) do
# Only update if the new packet is actually newer
DateTime.after?(new_packet.received_at, current_packet.received_at)
end
defp update_weather_data(socket) do
# Fetch updated data only once
weather_packet = get_latest_weather_packet(socket.assigns.callsign)
{start_time, end_time} = default_time_range()
weather_history = get_weather_history(socket.assigns.callsign, start_time, end_time)
@ -99,37 +100,7 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
# Get locale from socket assigns
locale = Map.get(socket.assigns, :locale, "en")
weather_history_json =
weather_history
|> Enum.map(fn pkt ->
dew_point =
if is_number(pkt.temperature) and is_number(pkt.humidity) do
calc_dew_point(pkt.temperature, pkt.humidity)
end
# Convert units based on locale
{temp_value, _} = WeatherUnits.format_temperature(pkt.temperature, locale)
{dew_value, _} = if dew_point, do: WeatherUnits.format_temperature(dew_point, locale), else: {nil, nil}
{wind_speed_value, _} = WeatherUnits.format_wind_speed(pkt.wind_speed, locale)
{rain_1h_value, _} = WeatherUnits.format_rain(pkt.rain_1h, locale)
{rain_24h_value, _} = WeatherUnits.format_rain(pkt.rain_24h, locale)
{rain_since_midnight_value, _} = WeatherUnits.format_rain(pkt.rain_since_midnight, locale)
%{
timestamp: pkt.received_at,
temperature: temp_value,
dew_point: dew_value,
humidity: pkt.humidity,
pressure: pkt.pressure,
wind_direction: pkt.wind_direction,
wind_speed: wind_speed_value,
rain_1h: rain_1h_value,
rain_24h: rain_24h_value,
rain_since_midnight: rain_since_midnight_value,
luminosity: pkt.luminosity
}
end)
|> Jason.encode!()
weather_history_json = convert_weather_history_to_json(weather_history, locale)
socket =
socket
@ -143,97 +114,18 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
{:noreply, socket}
end
# Keep the old handler for backward compatibility
def handle_info({:postgres_packet, packet}, socket) do
# Only update if the packet is for our callsign and contains weather data
if packet_matches_callsign?(packet, socket.assigns.callsign) and has_weather_data?(packet) do
# Refresh weather data when new weather packet arrives
weather_packet = get_latest_weather_packet(socket.assigns.callsign)
{start_time, end_time} = default_time_range()
weather_history = get_weather_history(socket.assigns.callsign, start_time, end_time)
# Get locale from socket assigns
locale = Map.get(socket.assigns, :locale, "en")
weather_history_json =
weather_history
|> Enum.map(fn pkt ->
dew_point =
if is_number(pkt.temperature) and is_number(pkt.humidity) do
calc_dew_point(pkt.temperature, pkt.humidity)
end
# Convert units based on locale
{temp_value, _} = WeatherUnits.format_temperature(pkt.temperature, locale)
{dew_value, _} = if dew_point, do: WeatherUnits.format_temperature(dew_point, locale), else: {nil, nil}
{wind_speed_value, _} = WeatherUnits.format_wind_speed(pkt.wind_speed, locale)
{rain_1h_value, _} = WeatherUnits.format_rain(pkt.rain_1h, locale)
{rain_24h_value, _} = WeatherUnits.format_rain(pkt.rain_24h, locale)
{rain_since_midnight_value, _} = WeatherUnits.format_rain(pkt.rain_since_midnight, locale)
%{
timestamp: pkt.received_at,
temperature: temp_value,
dew_point: dew_value,
humidity: pkt.humidity,
pressure: pkt.pressure,
wind_direction: pkt.wind_direction,
wind_speed: wind_speed_value,
rain_1h: rain_1h_value,
rain_24h: rain_24h_value,
rain_since_midnight: rain_since_midnight_value,
luminosity: pkt.luminosity
}
end)
|> Jason.encode!()
socket =
socket
|> assign(:weather_packet, weather_packet)
|> assign(:weather_history, weather_history)
|> assign(:weather_history_json, weather_history_json)
# Push event to update all charts with new data
socket = push_event(socket, "update_weather_charts", %{weather_history: weather_history_json})
{:noreply, socket}
else
{:noreply, socket}
end
end
def handle_info(_message, socket), do: {:noreply, socket}
defp packet_matches_callsign?(packet, callsign) do
packet_sender = Map.get(packet, "sender") || Map.get(packet, :sender, "")
String.upcase(packet_sender) == String.upcase(callsign)
end
defp has_weather_data?(packet) do
# Check if packet contains any weather-related fields
Enum.any?(Aprsme.EncodingUtils.weather_fields(), fn field ->
value = Map.get(packet, field) || Map.get(packet, to_string(field))
not is_nil(value)
end)
end
defp get_latest_weather_packet(callsign) do
# Get weather packets from the last 7 days to find the most recent one
end_time = DateTime.utc_now()
start_time = DateTime.add(end_time, -7 * 24 * 3600, :second)
callsign
|> Packets.get_weather_packets(start_time, end_time, %{limit: 1})
|> List.first()
# Use optimized cached query that checks recent data first
Aprsme.CachedQueries.get_latest_weather_packet_cached(callsign)
end
defp get_weather_history(callsign, start_time, end_time) do
Packets.get_weather_packets(callsign, start_time, end_time, %{limit: 500})
# Use cached queries to avoid repeated database hits
Aprsme.CachedQueries.get_weather_packets_cached(callsign, start_time, end_time, %{limit: 500})
end
defp default_time_range do
now = DateTime.utc_now()
{DateTime.add(now, -48 * 3600, :second), now}
TimeUtils.time_range(:last_two_days)
end
defp calc_dew_point(temp, humidity) when is_number(temp) and is_number(humidity) do
@ -242,6 +134,39 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
defp calc_dew_point(_, _), do: nil
defp convert_weather_history_to_json(weather_history, locale) do
weather_history
|> Enum.map(fn pkt ->
dew_point =
if is_number(pkt.temperature) and is_number(pkt.humidity) do
calc_dew_point(pkt.temperature, pkt.humidity)
end
# Convert units based on locale
{temp_value, _} = WeatherUnits.format_temperature(pkt.temperature, locale)
{dew_value, _} = if dew_point, do: WeatherUnits.format_temperature(dew_point, locale), else: {nil, nil}
{wind_speed_value, _} = WeatherUnits.format_wind_speed(pkt.wind_speed, locale)
{rain_1h_value, _} = WeatherUnits.format_rain(pkt.rain_1h, locale)
{rain_24h_value, _} = WeatherUnits.format_rain(pkt.rain_24h, locale)
{rain_since_midnight_value, _} = WeatherUnits.format_rain(pkt.rain_since_midnight, locale)
%{
timestamp: pkt.received_at,
temperature: temp_value,
dew_point: dew_value,
humidity: pkt.humidity,
pressure: pkt.pressure,
wind_direction: pkt.wind_direction,
wind_speed: wind_speed_value,
rain_1h: rain_1h_value,
rain_24h: rain_24h_value,
rain_since_midnight: rain_since_midnight_value,
luminosity: pkt.luminosity
}
end)
|> Jason.encode!()
end
@doc """
Gets weather field value, returning "0" instead of "N/A" for missing numeric data.
"""
@ -255,78 +180,44 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
end
end
# Weather formatters mapping
@weather_formatters %{
temperature: {&WeatherUnits.format_temperature/2, ""},
wind_speed: {&WeatherUnits.format_wind_speed/2, " "},
wind_gust: {&WeatherUnits.format_wind_speed/2, " "},
rain_1h: {&WeatherUnits.format_rain/2, " "},
rain_24h: {&WeatherUnits.format_rain/2, " "},
rain_since_midnight: {&WeatherUnits.format_rain/2, " "}
}
@doc """
Formats weather values with appropriate units based on locale.
"""
def format_weather_value(packet, key, locale) do
value = PacketUtils.get_weather_field(packet, key)
case {key, value} do
{:temperature, value} when is_binary(value) and value != "N/A" ->
case Float.parse(value) do
{num_value, _} ->
{converted_value, unit} = WeatherUnits.format_temperature(num_value, locale)
"#{converted_value}#{unit}"
case value do
"N/A" ->
"N/A"
:error ->
value
end
value when is_binary(value) ->
case @weather_formatters[key] do
{formatter, separator} ->
case Float.parse(value) do
{num_value, _} ->
{converted_value, unit} = formatter.(num_value, locale)
"#{converted_value}#{separator}#{unit}"
{:wind_speed, value} when is_binary(value) and value != "N/A" ->
case Float.parse(value) do
{num_value, _} ->
{converted_value, unit} = WeatherUnits.format_wind_speed(num_value, locale)
"#{converted_value} #{unit}"
:error ->
value
end
:error ->
value
end
{:wind_gust, value} when is_binary(value) and value != "N/A" ->
case Float.parse(value) do
{num_value, _} ->
{converted_value, unit} = WeatherUnits.format_wind_speed(num_value, locale)
"#{converted_value} #{unit}"
:error ->
value
end
{:rain_1h, value} when is_binary(value) and value != "N/A" ->
case Float.parse(value) do
{num_value, _} ->
{converted_value, unit} = WeatherUnits.format_rain(num_value, locale)
"#{converted_value} #{unit}"
:error ->
value
end
{:rain_24h, value} when is_binary(value) and value != "N/A" ->
case Float.parse(value) do
{num_value, _} ->
{converted_value, unit} = WeatherUnits.format_rain(num_value, locale)
"#{converted_value} #{unit}"
:error ->
value
end
{:rain_since_midnight, value} when is_binary(value) and value != "N/A" ->
case Float.parse(value) do
{num_value, _} ->
{converted_value, unit} = WeatherUnits.format_rain(num_value, locale)
"#{converted_value} #{unit}"
:error ->
value
nil ->
"#{value}"
end
_ ->
case value do
"N/A" -> "N/A"
_ -> "#{value}"
end
"#{value}"
end
end
end

52
lib/aprsme_web/time_utils.ex Normal file
View file

@ -0,0 +1,52 @@
defmodule AprsmeWeb.TimeUtils do
@moduledoc """
Common time calculation utilities to reduce duplication across the codebase.
"""
@doc """
Returns a DateTime that is the specified number of hours before now.
"""
def hours_ago(hours) when is_number(hours) do
DateTime.add(DateTime.utc_now(), -hours * 3600, :second)
end
@doc """
Returns a DateTime that is one hour before now.
"""
def one_hour_ago, do: hours_ago(1)
@doc """
Returns a DateTime that is 24 hours (one day) before now.
"""
def one_day_ago, do: hours_ago(24)
@doc """
Returns a DateTime that is 48 hours (two days) before now.
"""
def two_days_ago, do: hours_ago(48)
@doc """
Returns a DateTime that is 7 days (one week) before now.
"""
def one_week_ago, do: hours_ago(24 * 7)
@doc """
Returns a DateTime that is the specified number of days before now.
"""
def days_ago(days) when is_number(days) do
DateTime.add(DateTime.utc_now(), -days, :day)
end
@doc """
Returns a tuple of {start_time, end_time} for common time ranges.
"""
def time_range(:last_hour), do: {one_hour_ago(), DateTime.utc_now()}
def time_range(:last_day), do: {one_day_ago(), DateTime.utc_now()}
def time_range(:last_two_days), do: {two_days_ago(), DateTime.utc_now()}
def time_range(:last_week), do: {one_week_ago(), DateTime.utc_now()}
@doc """
Returns a default time range of 48 hours.
"""
def default_time_range, do: time_range(:last_two_days)
end

1
mix.exs
View file

@ -10,7 +10,6 @@ defmodule Aprsme.MixProject do
start_permanent: Mix.env() == :prod,
aliases: aliases(),
deps: deps(),
test_coverage: [tool: ExCoveralls],
listeners: [Phoenix.CodeReloader],
dialyzer: [
ignore_warnings: ".dialyzer_ignore.exs",

63
priv/repo/migrations/20250710150258_optimize_info_weather_pages.exs Normal file
View file

@ -0,0 +1,63 @@
defmodule Aprsme.Repo.Migrations.OptimizeInfoWeatherPages do
use Ecto.Migration
@disable_ddl_transaction true
@disable_migration_lock true
def up do
# Critical index for get_other_ssids function on info page
create_if_not_exists(
index(:packets, [:base_callsign, :received_at],
name: :packets_base_callsign_received_at_idx,
concurrently: true,
comment: "Index for finding other SSIDs of same base callsign"
)
)
# Add spatial index for get_nearby_stations if not exists
# This is crucial for the ST_Distance calculations
execute """
CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_location_spatial_idx
ON packets USING GIST (location)
WHERE has_position = true;
""",
""
# Optimize the get_nearby_stations query with a specialized index
# This combines spatial and temporal filtering
execute """
CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_spatial_temporal_idx
ON packets (received_at DESC, base_callsign)
WHERE has_position = true;
""",
""
# Add index for weather page callsign history queries
# This optimizes the initial load of weather data
create_if_not_exists(
index(:packets, [:sender, :data_type, :received_at],
name: :packets_weather_history_idx,
where: "data_type = 'weather'",
concurrently: true,
comment: "Optimized index for weather history queries"
)
)
# Analyze tables to update statistics for query planner
execute "ANALYZE packets;", ""
end
def down do
drop_if_exists(
index(:packets, [:base_callsign, :received_at],
name: :packets_base_callsign_received_at_idx
)
)
execute "DROP INDEX CONCURRENTLY IF EXISTS packets_location_spatial_idx;", ""
execute "DROP INDEX CONCURRENTLY IF EXISTS packets_spatial_temporal_idx;", ""
drop_if_exists(
index(:packets, [:sender, :data_type, :received_at], name: :packets_weather_history_idx)
)
end
end

32
priv/repo/migrations/20250710155050_add_sender_lower_index.exs Normal file
View file

@ -0,0 +1,32 @@
defmodule Aprsme.Repo.Migrations.AddSenderLowerIndex do
use Ecto.Migration
@disable_ddl_transaction true
@disable_migration_lock true
def up do
# Add functional index for case-insensitive sender searches
# This helps if we ever need to do ILIKE queries
execute """
CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_sender_lower_idx
ON packets (LOWER(sender))
WHERE sender IS NOT NULL;
""",
""
# Also add a trigram index for more flexible pattern matching if needed
# Note: This requires pg_trgm extension
execute "CREATE EXTENSION IF NOT EXISTS pg_trgm;", ""
execute """
CREATE INDEX CONCURRENTLY IF NOT EXISTS packets_sender_trgm_idx
ON packets USING gin (sender gin_trgm_ops)
WHERE sender IS NOT NULL;
""",
""
end
def down do
execute "DROP INDEX CONCURRENTLY IF EXISTS packets_sender_lower_idx;", ""
execute "DROP INDEX CONCURRENTLY IF EXISTS packets_sender_trgm_idx;", ""
end
end

9
test/aprsme_web/integration/aprs_status_test.exs
View file

@ -96,6 +96,12 @@ defmodule AprsmeWeb.Integration.AprsStatusTest do
describe "LiveView event handling without APRS" do
test "map events work without APRS connection", %{conn: conn} do
# Set the mock to allow calls from any process
Mox.set_mox_global()
# Stub the function that will be called during bounds changes
Mox.stub(Aprsme.PacketsMock, :get_recent_packets_optimized, fn _opts -> [] end)
{:ok, view, _html} = live(conn, "/")
# Test map bounds change event
@ -111,6 +117,9 @@ defmodule AprsmeWeb.Integration.AprsStatusTest do
# Should not crash when handling events without APRS connection
assert render_hook(view, "bounds_changed", bounds_params)
# Wait a bit for any async processes
Process.sleep(50)
# Test map ready event
assert render_hook(view, "map_ready", %{})

4
test/aprsme_web/live/map_live/overlay_rendering_test.exs
View file

@ -28,9 +28,9 @@ defmodule AprsmeWeb.MapLive.OverlayRenderingTest do
symbol_html = result["symbol_html"]
# Verify the overlay symbol is rendered correctly with overlay character on top
# The overlay character (D) should be first in the background-image list
# The overlay character (D) from table 2, base symbol (&) from table 1
assert symbol_html =~
"background-image: url(/aprs-symbols/aprs-symbols-128-2@2x.png), url(/aprs-symbols/aprs-symbols-128-2@2x.png)"
"background-image: url(/aprs-symbols/aprs-symbols-128-2@2x.png), url(/aprs-symbols/aprs-symbols-128-1@2x.png)"
# Verify the positions (overlay character D first at -96.0px -64.0px, then base & at -160.0px 0.0px)
assert symbol_html =~ "background-position: -96.0px -64.0px, -160.0px 0.0px"

57
test/aprsme_web/live/map_live/performance_test.exs
View file

@ -4,12 +4,31 @@ defmodule AprsmeWeb.MapLive.PerformanceTest do
import Aprsme.PacketsFixtures
import Phoenix.LiveViewTest
alias Aprsme.Repo
describe "historical packet loading performance" do
setup do
# Set up query counter using process dictionary
Process.put(:query_count, 0)
# Set up telemetry handler to count queries
:telemetry.attach(
"test-query-counter",
[:aprsme, :repo, :query],
fn _event, _measurements, _metadata, _config ->
Process.put(:query_count, (Process.get(:query_count) || 0) + 1)
end,
nil
)
on_exit(fn ->
:telemetry.detach("test-query-counter")
end)
:ok
end
test "efficiently loads historical packets without N+1 queries", %{conn: conn} do
# Create test packets with different callsigns
callsigns = ["TEST1", "TEST2", "TEST3", "WEATHER1", "WEATHER2"]
_callsigns = ["TEST1", "TEST2", "TEST3", "WEATHER1", "WEATHER2"]
# Create regular packets
for callsign <- ["TEST1", "TEST2", "TEST3"] do
@ -35,36 +54,32 @@ defmodule AprsmeWeb.MapLive.PerformanceTest do
})
end
# Reset query counter before the test
Process.put(:query_count, 0)
# Load the live view
{:ok, lv, _html} = live(conn, "/")
# Trigger map ready which loads historical packets
# Count queries to ensure we're not doing N+1 queries
query_count_before = get_query_count()
# Reset counter again to measure only map loading queries
Process.put(:query_count, 0)
# Trigger map ready which loads historical packets
lv
|> element("#aprs-map")
|> render_hook("map_ready", %{})
# Wait for historical packets to load
:timer.sleep(100)
:timer.sleep(200)
query_count_after = get_query_count()
queries_executed = query_count_after - query_count_before
queries_executed = Process.get(:query_count) || 0
# Should execute only a few queries:
# 1. Main packet query
# 2. Batch weather callsign query
# Plus maybe a few system queries
# But definitely not one query per callsign (which would be 5+ queries)
assert queries_executed < 10, "Too many queries executed: #{queries_executed}"
# 1. Initial packet query
# 2. Batch weather callsign queries (if any)
# 3. Maybe some additional optimized queries
# But definitely not one query per packet/callsign (which would be 5+ queries)
# Given the batched loading, we expect more queries but they should be efficient
assert queries_executed < 20, "Too many queries executed: #{queries_executed}"
end
end
# Helper to get approximate query count from Repo stats
defp get_query_count do
# This is a simplified way to track queries
# In a real test, you might use Ecto telemetry or query logging
System.unique_integer([:positive])
end
end

54
test/support/fixtures/packets_fixtures.ex Normal file
View file

@ -0,0 +1,54 @@
defmodule Aprsme.PacketsFixtures do
@moduledoc """
This module defines test helpers for creating
entities via the `Aprsme.Packets` context.
"""
alias Aprsme.Callsign
alias Aprsme.Packet
alias Aprsme.Repo
@doc """
Generate a packet.
"""
def packet_fixture(attrs \\ %{}) do
base_attrs = %{
sender: "TEST-1",
base_callsign: "TEST",
ssid: "1",
destination: "APRS",
received_at: DateTime.utc_now(),
lat: Decimal.new("40.7128"),
lon: Decimal.new("-74.0060"),
has_position: true,
raw_packet: "TEST-1>APRS:=4042.77N/07400.36W>Test packet",
data_type: "position",
path: "APRS",
data_extended: %{
symbol_table_id: "/",
symbol_code: ">",
comment: "Test packet"
}
}
# Extract base_callsign and ssid from sender if provided
final_attrs =
case Map.get(attrs, :sender) do
nil ->
base_attrs
sender ->
{base, ssid} = Callsign.extract_parts(sender)
Map.merge(base_attrs, %{base_callsign: base, ssid: ssid})
end
{:ok, packet} =
attrs
|> Enum.into(final_attrs)
|> then(&Packet.changeset(%Packet{}, &1))
|> Repo.insert()
packet
end
end

8
test/support/mock_helpers.ex
View file

@ -16,6 +16,14 @@ defmodule Aprsme.MockHelpers do
Mox.stub(Aprsme.PacketsMock, :get_packets_for_callsign_with_date_range, fn _callsign, _start_date, _end_date ->
{:ok, []}
end)
Mox.stub(Aprsme.PacketsMock, :get_recent_packets_optimized, fn _opts ->
[]
end)
Mox.stub(Aprsme.PacketsMock, :get_nearby_stations, fn _lat, _lon, _exclude, _opts ->
[]
end)
end
def stub_badpackets_mock do

4
test/test_helper.exs
View file

@ -6,6 +6,10 @@ Mox.defmock(Aprsme.PacketsMock, for: Aprsme.PacketsBehaviour)
Mox.defmock(Aprsme.PacketReplayMock, for: Aprsme.PacketReplayBehaviour)
Mox.defmock(PacketsMock, for: Aprsme.PacketsBehaviour)
# Set up default stubs for commonly used functions
Mox.stub(Aprsme.PacketsMock, :get_recent_packets_optimized, fn _opts -> [] end)
Mox.stub(Aprsme.PacketsMock, :get_nearby_stations, fn _lat, _lon, _exclude, _opts -> [] end)
# Ensure no external APRS connections during tests
Application.put_env(:aprsme, :disable_aprs_connection, true)
Application.put_env(:aprsme, :aprs_is_server, "mock.aprs.test")