Graham McIntire
f225f57044
get_recent_packets/1 and get_recent_packets_for_map/1 now accept a
:cursor option (%{received_at: DateTime, id: uuid}) instead of :offset
for deep paging. On the 100M-row partitioned packets table, offset
scanning was linear in page depth; keyset is constant.
Ordering tightened to (received_at DESC, id DESC) so the cursor is
stable across ties. idx_packets_received_desc still covers the lookup;
a dedicated (received_at DESC, id DESC) index isn't worth its size
given microsecond timestamp granularity.
historical_loader migrated from batch_offset*batch_size scheduling to
a sequential cursor chain. The old loader fanned out N Process.send_after
batches up-front which could arrive out-of-order; the cursor version
waits for batch N before scheduling N+1, and also short-circuits the
remaining schedule when an under-full batch signals exhaustion.
mobile_channel load_historical_packets dropped its meaningless
`offset: 0`; callsign history already lacked offset.
No LiveView or WebSocket payload shapes changed. offset: 0 (or omitted)
still behaves as first page; non-zero offsets are silently ignored.
877 lines
28 KiB
Elixir
877 lines
28 KiB
Elixir
defmodule Aprsme.Packets do
|
|
@moduledoc """
|
|
The Packets context.
|
|
"""
|
|
|
|
@behaviour Aprsme.PacketsBehaviour
|
|
|
|
import Ecto.Query, warn: false
|
|
|
|
alias Aprs.Types.MicE
|
|
alias Aprsme.BadPacket
|
|
alias Aprsme.Packet
|
|
alias Aprsme.Packets.PreparedQueries
|
|
alias Aprsme.Packets.QueryBuilder
|
|
alias Aprsme.Repo
|
|
|
|
require Logger
|
|
|
|
@doc """
|
|
Stores a packet in the database.
|
|
|
|
## Parameters
|
|
* `packet_data` - Map containing packet data to be stored
|
|
"""
|
|
@spec store_packet(map()) :: {:ok, struct()} | {:error, :validation_error | :storage_exception}
|
|
def store_packet(packet_data) do
|
|
require Logger
|
|
|
|
with {:ok, sanitized_data} <- sanitize_packet_data(packet_data),
|
|
{:ok, packet_attrs} <- build_packet_attrs(sanitized_data),
|
|
{:ok, packet} <- insert_packet(packet_attrs, packet_data) do
|
|
{:ok, packet}
|
|
else
|
|
{:error, {:validation_error, message}} ->
|
|
Logger.error("Failed to store packet for #{inspect(packet_data[:sender])}: #{message}")
|
|
store_bad_packet(packet_data, %{message: message, type: "ValidationError"})
|
|
{:error, :validation_error}
|
|
|
|
{:error, {:storage_exception, exception}} ->
|
|
Logger.error("Storage exception for #{inspect(packet_data[:sender])}: #{inspect(exception)}")
|
|
store_bad_packet(packet_data, %{message: inspect(exception), type: "StorageException"})
|
|
{:error, :storage_exception}
|
|
|
|
{:error, reason} = error ->
|
|
Logger.error("Failed to store packet for #{inspect(packet_data[:sender])}: #{inspect(reason)}")
|
|
store_bad_packet(packet_data, reason)
|
|
error
|
|
end
|
|
end
|
|
|
|
# Pattern matching for sanitization
|
|
defp sanitize_packet_data(packet_data) do
|
|
{:ok, Aprsme.EncodingUtils.sanitize_packet(packet_data)}
|
|
rescue
|
|
error -> {:error, {:sanitization_failed, error}}
|
|
end
|
|
|
|
# Build packet attributes with pipeline
|
|
defp build_packet_attrs(sanitized_data) do
|
|
raw_packet = get_raw_packet(sanitized_data)
|
|
|
|
attrs =
|
|
sanitized_data
|
|
|> Packet.extract_additional_data(raw_packet)
|
|
|> normalize_packet_attrs()
|
|
|> set_received_at()
|
|
|> patch_lat_lon_from_data_extended()
|
|
|> apply_position_data()
|
|
|> normalize_ssid()
|
|
|> apply_device_identifier(sanitized_data)
|
|
|
|
{:ok, attrs}
|
|
rescue
|
|
error -> {:error, {:build_attrs_failed, error}}
|
|
end
|
|
|
|
# Pattern matching for raw packet extraction
|
|
defp get_raw_packet(%{raw_packet: raw}) when is_binary(raw), do: raw
|
|
defp get_raw_packet(%{"raw_packet" => raw}) when is_binary(raw), do: raw
|
|
defp get_raw_packet(_), do: ""
|
|
|
|
# Apply position data using pattern matching
|
|
defp apply_position_data(attrs) do
|
|
case extract_position(attrs) do
|
|
{nil, nil} -> attrs
|
|
{lat, lon} -> set_lat_lon(attrs, lat, lon)
|
|
end
|
|
end
|
|
|
|
# Apply device identifier with pattern matching
|
|
defp apply_device_identifier(attrs, original_data) do
|
|
case Aprsme.DeviceParser.extract_device_identifier(original_data) do
|
|
nil ->
|
|
# Fallback to destination
|
|
Map.put(attrs, :device_identifier, Map.get(attrs, :destination))
|
|
|
|
device_id ->
|
|
Map.put(attrs, :device_identifier, get_canonical_device_identifier(device_id))
|
|
end
|
|
end
|
|
|
|
defp normalize_packet_attrs(packet_data) do
|
|
case_result =
|
|
case packet_data do
|
|
%Packet{} = packet ->
|
|
packet
|
|
|> Map.from_struct()
|
|
|> Map.delete(:__meta__)
|
|
|
|
%{} ->
|
|
packet_data
|
|
end
|
|
|
|
case_result
|
|
|> Aprsme.EncodingUtils.sanitize_packet()
|
|
|> normalize_data_type()
|
|
end
|
|
|
|
defp normalize_data_type(attrs) do
|
|
Aprsme.EncodingUtils.normalize_data_type(attrs)
|
|
end
|
|
|
|
defp set_received_at(attrs) do
|
|
current_time = DateTime.truncate(DateTime.utc_now(), :microsecond)
|
|
Map.put(attrs, :received_at, current_time)
|
|
end
|
|
|
|
# Pattern matching with guards for cleaner code
|
|
defp patch_lat_lon_from_data_extended(%{data_extended: %{} = ext} = attrs) do
|
|
ext_map = normalize_to_map(ext)
|
|
|
|
with lat when not is_nil(lat) <- extract_lat_from_ext_map(ext_map),
|
|
lon when not is_nil(lon) <- extract_lon_from_ext_map(ext_map),
|
|
{:ok, lat_decimal} <- to_decimal_safe(lat),
|
|
{:ok, lon_decimal} <- to_decimal_safe(lon) do
|
|
attrs
|
|
|> Map.put(:lat, lat_decimal)
|
|
|> Map.put(:lon, lon_decimal)
|
|
else
|
|
_ -> attrs
|
|
end
|
|
end
|
|
|
|
defp patch_lat_lon_from_data_extended(attrs), do: attrs
|
|
|
|
# Helper to normalize struct or map
|
|
defp normalize_to_map(%{__struct__: _} = struct), do: Map.from_struct(struct)
|
|
defp normalize_to_map(map) when is_map(map), do: map
|
|
|
|
# Safe decimal conversion with pattern matching
|
|
defp to_decimal_safe(nil), do: {:error, :nil_value}
|
|
|
|
defp to_decimal_safe(value) do
|
|
case Aprsme.EncodingUtils.to_decimal(value) do
|
|
nil -> {:error, :conversion_failed}
|
|
decimal -> {:ok, decimal}
|
|
end
|
|
end
|
|
|
|
# Pattern matching for coordinate extraction
|
|
defp extract_lat_from_ext_map(ext_map) do
|
|
extract_coordinate(ext_map, :latitude)
|
|
end
|
|
|
|
defp extract_lon_from_ext_map(ext_map) do
|
|
extract_coordinate(ext_map, :longitude)
|
|
end
|
|
|
|
# Unified coordinate extraction with pattern matching
|
|
defp extract_coordinate(%{} = map, coord_type) when coord_type in [:latitude, :longitude] do
|
|
coord_string = Atom.to_string(coord_type)
|
|
|
|
find_coordinate_value(map, coord_type) || find_coordinate_value(map, coord_string)
|
|
end
|
|
|
|
defp extract_coordinate(_, _), do: nil
|
|
|
|
# Pattern matching for direct coordinate access
|
|
defp find_coordinate_value(%{latitude: lat}, :latitude) when not is_nil(lat), do: lat
|
|
defp find_coordinate_value(%{longitude: lon}, :longitude) when not is_nil(lon), do: lon
|
|
defp find_coordinate_value(%{"latitude" => lat}, "latitude") when not is_nil(lat), do: lat
|
|
defp find_coordinate_value(%{"longitude" => lon}, "longitude") when not is_nil(lon), do: lon
|
|
|
|
# Pattern matching for nested position access
|
|
defp find_coordinate_value(%{position: %{latitude: lat}}, :latitude) when not is_nil(lat), do: lat
|
|
defp find_coordinate_value(%{position: %{longitude: lon}}, :longitude) when not is_nil(lon), do: lon
|
|
defp find_coordinate_value(%{position: %{"latitude" => lat}}, :latitude) when not is_nil(lat), do: lat
|
|
defp find_coordinate_value(%{position: %{"longitude" => lon}}, :longitude) when not is_nil(lon), do: lon
|
|
defp find_coordinate_value(%{"position" => %{latitude: lat}}, "latitude") when not is_nil(lat), do: lat
|
|
defp find_coordinate_value(%{"position" => %{longitude: lon}}, "longitude") when not is_nil(lon), do: lon
|
|
defp find_coordinate_value(%{"position" => %{"latitude" => lat}}, "latitude") when not is_nil(lat), do: lat
|
|
defp find_coordinate_value(%{"position" => %{"longitude" => lon}}, "longitude") when not is_nil(lon), do: lon
|
|
|
|
defp find_coordinate_value(_, _), do: nil
|
|
|
|
defp set_lat_lon(attrs, lat, lon) do
|
|
# Optimize: avoid creating anonymous function on each call
|
|
attrs
|
|
|> Map.put(:lat, round_coordinate(lat))
|
|
|> Map.put(:lon, round_coordinate(lon))
|
|
end
|
|
|
|
# Moved out as a separate function to avoid recreating on each call
|
|
defp round_coordinate(nil), do: nil
|
|
defp round_coordinate(%Decimal{} = d), do: Decimal.round(d, 6)
|
|
defp round_coordinate(n) when is_float(n), do: Float.round(n, 6)
|
|
defp round_coordinate(n) when is_integer(n), do: n * 1.0
|
|
|
|
defp round_coordinate(n) when is_binary(n) do
|
|
case Float.parse(n) do
|
|
{f, _} -> Float.round(f, 6)
|
|
:error -> nil
|
|
end
|
|
end
|
|
|
|
defp round_coordinate(_), do: nil
|
|
|
|
# Pattern matching for SSID normalization
|
|
defp normalize_ssid(%{ssid: nil} = attrs), do: attrs
|
|
defp normalize_ssid(%{ssid: ssid} = attrs) when is_binary(ssid), do: attrs
|
|
defp normalize_ssid(%{ssid: ssid} = attrs), do: Map.put(attrs, :ssid, to_string(ssid))
|
|
defp normalize_ssid(attrs), do: attrs
|
|
|
|
defp insert_packet(attrs, _packet_data) do
|
|
# Ensure data_extended is properly sanitized before insertion
|
|
attrs = sanitize_data_extended_attr(attrs)
|
|
|
|
case %Packet{} |> Packet.changeset(attrs) |> Repo.insert() do
|
|
{:ok, packet} ->
|
|
{:ok, packet}
|
|
|
|
{:error, changeset} ->
|
|
error_message =
|
|
Enum.map_join(changeset.errors, ", ", fn {field, {msg, _}} -> "#{field}: #{msg}" end)
|
|
|
|
{:error, {:validation_error, error_message}}
|
|
end
|
|
rescue
|
|
exception ->
|
|
{:error, {:storage_exception, exception}}
|
|
end
|
|
|
|
defp sanitize_data_extended_attr(attrs) do
|
|
if attrs[:data_extended] do
|
|
sanitized_extended = Aprsme.EncodingUtils.sanitize_data_extended(attrs[:data_extended])
|
|
sanitized_extended = deep_sanitize_map(sanitized_extended)
|
|
Map.put(attrs, :data_extended, sanitized_extended)
|
|
else
|
|
attrs
|
|
end
|
|
end
|
|
|
|
defp deep_sanitize_map(data) when is_struct(data), do: data
|
|
|
|
defp deep_sanitize_map(data) when is_map(data) do
|
|
Enum.reduce(data, %{}, fn {k, v}, acc ->
|
|
sanitized_value = if is_binary(v), do: Aprsme.EncodingUtils.sanitize_string(v), else: v
|
|
Map.put(acc, k, sanitized_value)
|
|
end)
|
|
end
|
|
|
|
defp deep_sanitize_map(data), do: data
|
|
|
|
@doc """
|
|
Stores a bad packet in the database.
|
|
|
|
## Parameters
|
|
* `packet_data` - The original packet data that failed to parse
|
|
* `error` - The error that occurred during parsing
|
|
"""
|
|
@spec store_bad_packet(map() | String.t(), any()) ::
|
|
{:ok, struct()} | {:error, Ecto.Changeset.t()}
|
|
def store_bad_packet(packet_data, error) when is_binary(packet_data) do
|
|
error_type =
|
|
case error do
|
|
%{type: type} -> type
|
|
%{__struct__: struct} -> struct |> to_string() |> String.replace("Elixir.", "")
|
|
_ -> "UnknownError"
|
|
end
|
|
|
|
error_message =
|
|
case error do
|
|
%{message: message} -> message
|
|
%{__struct__: _} -> Exception.message(error)
|
|
_ -> inspect(error)
|
|
end
|
|
|
|
%BadPacket{}
|
|
|> BadPacket.changeset(%{
|
|
raw_packet: Aprsme.EncodingUtils.sanitize_string(packet_data),
|
|
error_message: error_message,
|
|
error_type: error_type,
|
|
attempted_at: DateTime.utc_now()
|
|
})
|
|
|> Repo.insert()
|
|
end
|
|
|
|
def store_bad_packet(packet_data, error) when is_map(packet_data) do
|
|
error_type =
|
|
case error do
|
|
%{type: type} -> type
|
|
%{__struct__: struct} -> struct |> to_string() |> String.replace("Elixir.", "")
|
|
_ -> "UnknownError"
|
|
end
|
|
|
|
error_message =
|
|
case error do
|
|
%{message: message} -> message
|
|
%{__struct__: _} -> Exception.message(error)
|
|
_ -> inspect(error)
|
|
end
|
|
|
|
%BadPacket{}
|
|
|> BadPacket.changeset(%{
|
|
raw_packet: packet_data[:raw_packet] || packet_data["raw_packet"] || inspect(packet_data),
|
|
error_message: error_message,
|
|
error_type: error_type,
|
|
attempted_at: DateTime.utc_now()
|
|
})
|
|
|> Repo.insert()
|
|
end
|
|
|
|
# Extracts position data from packet, checking various possible locations
|
|
defp extract_position(packet_data) do
|
|
# Check for lat/lon at top level
|
|
if not is_nil(packet_data[:lat]) and not is_nil(packet_data[:lon]) do
|
|
{Aprsme.EncodingUtils.to_float(packet_data.lat), Aprsme.EncodingUtils.to_float(packet_data.lon)}
|
|
else
|
|
extract_position_from_data_extended(packet_data[:data_extended])
|
|
end
|
|
end
|
|
|
|
defp extract_position_from_data_extended(nil), do: {nil, nil}
|
|
|
|
defp extract_position_from_data_extended(data_extended) when is_map(data_extended) do
|
|
if has_standard_position?(data_extended) do
|
|
extract_standard_position(data_extended)
|
|
else
|
|
extract_position_from_data_extended_case(data_extended)
|
|
end
|
|
end
|
|
|
|
defp extract_position_from_data_extended(_), do: {nil, nil}
|
|
|
|
defp has_standard_position?(data_extended) when is_map(data_extended) and not is_struct(data_extended) do
|
|
not is_nil(data_extended[:latitude]) and not is_nil(data_extended[:longitude])
|
|
end
|
|
|
|
defp has_standard_position?(_), do: false
|
|
|
|
defp extract_standard_position(data_extended) when is_map(data_extended) and not is_struct(data_extended) do
|
|
{Aprsme.EncodingUtils.to_float(data_extended[:latitude]), Aprsme.EncodingUtils.to_float(data_extended[:longitude])}
|
|
end
|
|
|
|
defp extract_standard_position(_), do: {nil, nil}
|
|
|
|
defp extract_position_from_data_extended_case(data_extended) do
|
|
case data_extended do
|
|
%{__struct__: MicE} = mic_e ->
|
|
extract_position_from_mic_e_struct(mic_e)
|
|
|
|
%{__struct__: Aprs.Types.ParseError} ->
|
|
# ParseError structs don't contain position data and don't implement Access behavior
|
|
{nil, nil}
|
|
|
|
_ ->
|
|
extract_position_from_mic_e(data_extended)
|
|
end
|
|
end
|
|
|
|
defp extract_position_from_mic_e_struct(%{__struct__: MicE} = mic_e) do
|
|
# Use Access behavior for MicE struct which implements it
|
|
lat = mic_e[:latitude]
|
|
lon = mic_e[:longitude]
|
|
|
|
if is_number(lat) and is_number(lon) do
|
|
{lat, lon}
|
|
else
|
|
{nil, nil}
|
|
end
|
|
end
|
|
|
|
defp extract_position_from_mic_e(%{
|
|
lat_degrees: lat_deg,
|
|
lat_minutes: lat_min,
|
|
lat_direction: lat_dir,
|
|
lon_degrees: lon_deg,
|
|
lon_minutes: lon_min,
|
|
lon_direction: lon_dir
|
|
})
|
|
when is_number(lat_deg) and is_number(lat_min) and is_number(lon_deg) and is_number(lon_min) do
|
|
lat = apply_direction(lat_deg + lat_min / 60.0, lat_dir, :south)
|
|
lon = apply_direction(lon_deg + lon_min / 60.0, lon_dir, :west)
|
|
{lat, lon}
|
|
end
|
|
|
|
defp extract_position_from_mic_e(_data_extended), do: {nil, nil}
|
|
|
|
defp apply_direction(value, direction, negative_direction) when direction == negative_direction, do: -value
|
|
defp apply_direction(value, _direction, _negative_direction), do: value
|
|
|
|
# Pattern matching for canonical device identifier lookup
|
|
defp get_canonical_device_identifier(device_identifier) when is_binary(device_identifier) do
|
|
case Aprsme.DeviceIdentification.lookup_device_by_identifier(device_identifier) do
|
|
%{identifier: canonical_id} when is_binary(canonical_id) -> canonical_id
|
|
_ -> device_identifier
|
|
end
|
|
end
|
|
|
|
defp get_canonical_device_identifier(device_identifier), do: to_string(device_identifier)
|
|
|
|
@doc """
|
|
Gets packets for replay.
|
|
|
|
## Parameters
|
|
* `opts` - Map of options for filtering and pagination:
|
|
* `:lat` - Latitude for center point filtering
|
|
* `:lon` - Longitude for center point filtering
|
|
* `:radius` - Radius in kilometers for filtering
|
|
* `:callsign` - Filter by callsign
|
|
* `:region` - Filter by region
|
|
* `:start_time` - Start time for replay (DateTime)
|
|
* `:end_time` - End time for replay (DateTime)
|
|
* `:limit` - Maximum number of packets to return
|
|
* `:page` - Page number for pagination
|
|
"""
|
|
@impl true
|
|
def get_packets_for_replay(opts \\ %{}) do
|
|
limit = Map.get(opts, :limit, 1000)
|
|
bounds = Map.get(opts, :bounds)
|
|
|
|
query =
|
|
from(p in Packet)
|
|
|> QueryBuilder.with_position()
|
|
|> QueryBuilder.with_time_range(opts)
|
|
|> QueryBuilder.maybe_filter_region(opts)
|
|
|> maybe_filter_by_callsign(opts)
|
|
|> maybe_filter_by_bounds(bounds)
|
|
|> QueryBuilder.chronological()
|
|
|> QueryBuilder.paginate(limit)
|
|
|> QueryBuilder.with_coordinates()
|
|
|
|
Repo.all(query)
|
|
end
|
|
|
|
defp maybe_filter_by_callsign(query, %{callsign: callsign}) when not is_nil(callsign) do
|
|
QueryBuilder.for_callsign(query, callsign)
|
|
end
|
|
|
|
defp maybe_filter_by_callsign(query, _), do: query
|
|
|
|
defp maybe_filter_by_bounds(query, bounds) when is_list(bounds) and length(bounds) == 4 do
|
|
QueryBuilder.within_bounds(query, bounds)
|
|
end
|
|
|
|
defp maybe_filter_by_bounds(query, _), do: query
|
|
|
|
@doc """
|
|
Gets historical packet count for a map area.
|
|
"""
|
|
@impl true
|
|
@spec get_historical_packet_count(map()) :: non_neg_integer()
|
|
def get_historical_packet_count(opts \\ %{}) do
|
|
bounds = Map.get(opts, :bounds)
|
|
|
|
query =
|
|
from(p in Packet)
|
|
|> QueryBuilder.with_position()
|
|
|> QueryBuilder.with_time_range(opts)
|
|
|> QueryBuilder.maybe_filter_region(opts)
|
|
|> maybe_filter_by_callsign(opts)
|
|
|> maybe_filter_by_bounds(bounds)
|
|
|> select(count())
|
|
|
|
case Repo.one(query) do
|
|
nil -> 0
|
|
count when is_integer(count) -> count
|
|
_ -> 0
|
|
end
|
|
rescue
|
|
_ -> 0
|
|
end
|
|
|
|
@doc """
|
|
Gets recent packets for initial map load.
|
|
This uses an efficient query pattern for the most common use case.
|
|
|
|
Ordering is `received_at DESC, id DESC`. Pagination is keyset/cursor-based:
|
|
|
|
* `:cursor` - Optional `%{received_at: DateTime.t(), id: Ecto.UUID.t()}` marking
|
|
the last row from the previous page. When present, only rows strictly older
|
|
than the cursor are returned. When absent, the first page is returned.
|
|
* `:limit` - Maximum number of packets to return (default 200).
|
|
|
|
Offset-based pagination is no longer supported; callers doing deep paging must
|
|
thread `{received_at, id}` from the last returned packet back in as `:cursor`.
|
|
Passing `offset: 0` (or omitting it) is still accepted and behaves as "first page".
|
|
"""
|
|
@impl true
|
|
@spec get_recent_packets(map()) :: [struct()]
|
|
def get_recent_packets(opts \\ %{}) do
|
|
opts
|
|
|> build_recent_packets_query()
|
|
|> QueryBuilder.with_coordinates()
|
|
|> Repo.all()
|
|
end
|
|
|
|
@doc """
|
|
Gets recent packets for map display with only the columns needed for rendering.
|
|
Returns maps (not full Packet structs) to avoid loading all 73 columns.
|
|
|
|
Pagination is cursor/keyset-based — see `get_recent_packets/1`.
|
|
"""
|
|
@spec get_recent_packets_for_map(map()) :: [map()]
|
|
def get_recent_packets_for_map(opts \\ %{}) do
|
|
opts
|
|
|> build_recent_packets_query()
|
|
|> QueryBuilder.select_map_fields()
|
|
|> Repo.all()
|
|
end
|
|
|
|
# Shared query construction for get_recent_packets/1 and get_recent_packets_for_map/1.
|
|
# Applies time filter, position/callsign/bounds/region filters, cursor keyset,
|
|
# and ordering/limit. The two public functions differ only in their select shape.
|
|
defp build_recent_packets_query(opts) do
|
|
hours_back = Map.get(opts, :hours_back, 24)
|
|
time_ago = DateTime.add(DateTime.utc_now(), -hours_back * 3600, :second)
|
|
limit = Map.get(opts, :limit, 200)
|
|
|
|
callsign = opts[:callsign]
|
|
normalized_callsign = if is_binary(callsign), do: String.trim(callsign), else: ""
|
|
|
|
base_query =
|
|
if normalized_callsign == "" do
|
|
# For general map view, only show packets with positions
|
|
from(p in Packet,
|
|
where: p.has_position == true,
|
|
where: p.received_at >= ^time_ago
|
|
)
|
|
else
|
|
# When tracking a callsign, show all their packets regardless of position
|
|
from(p in Packet,
|
|
where: p.received_at >= ^time_ago
|
|
)
|
|
end
|
|
|
|
bounds = Map.get(opts, :bounds)
|
|
|
|
base_query
|
|
|> QueryBuilder.maybe_filter_region(opts)
|
|
|> maybe_filter_by_callsign(opts)
|
|
|> maybe_filter_by_bounds(bounds)
|
|
|> maybe_apply_cursor(Map.get(opts, :cursor))
|
|
|> order_by([p], desc: p.received_at, desc: p.id)
|
|
|> limit(^limit)
|
|
end
|
|
|
|
# Keyset pagination: only rows strictly older than the cursor in the
|
|
# (received_at DESC, id DESC) ordering.
|
|
defp maybe_apply_cursor(query, %{received_at: %DateTime{} = cursor_received_at, id: cursor_id})
|
|
when is_binary(cursor_id) do
|
|
from p in query,
|
|
where:
|
|
p.received_at < ^cursor_received_at or
|
|
(p.received_at == ^cursor_received_at and p.id < ^cursor_id)
|
|
end
|
|
|
|
defp maybe_apply_cursor(query, _), do: query
|
|
|
|
@doc """
|
|
Gets the closest stations to a given point.
|
|
Uses PostGIS spatial indexes for efficient querying.
|
|
Returns only the most recent packet per callsign, ordered by distance.
|
|
"""
|
|
@impl true
|
|
@spec get_nearby_stations(float(), float(), String.t() | nil, map()) :: [struct()]
|
|
def get_nearby_stations(lat, lon, exclude_callsign \\ nil, opts \\ %{}) do
|
|
# Use KNN (K-nearest neighbors) query for better performance
|
|
results = PreparedQueries.get_nearby_stations_knn(lat, lon, exclude_callsign, opts)
|
|
|
|
# Convert results back to Packet structs if needed
|
|
Enum.map(results, fn result ->
|
|
# If result is already a Packet struct, return it
|
|
if is_struct(result, Packet) do
|
|
result
|
|
else
|
|
# Otherwise, create a minimal Packet struct from the map
|
|
%Packet{
|
|
sender: result.callsign,
|
|
base_callsign: result.base_callsign,
|
|
lat: result.lat,
|
|
lon: result.lon,
|
|
received_at: result.received_at,
|
|
symbol_table_id: result.symbol_table_id,
|
|
symbol_code: result.symbol_code,
|
|
comment: result.comment
|
|
}
|
|
end
|
|
end)
|
|
end
|
|
|
|
@doc """
|
|
Gets weather packets for a specific callsign within a time range.
|
|
This is optimized for weather queries by filtering at the database level.
|
|
"""
|
|
@impl true
|
|
@spec get_weather_packets(String.t(), DateTime.t(), DateTime.t(), map()) :: [struct()]
|
|
def get_weather_packets(callsign, start_time, end_time, opts \\ %{}) do
|
|
opts =
|
|
Map.merge(opts, %{
|
|
callsign: callsign,
|
|
start_time: start_time,
|
|
end_time: end_time,
|
|
limit: Map.get(opts, :limit, 500)
|
|
})
|
|
|
|
opts
|
|
|> QueryBuilder.weather_packets()
|
|
|> QueryBuilder.with_coordinates()
|
|
|> Repo.all()
|
|
end
|
|
|
|
# Filter for weather packets at the database level
|
|
|
|
@doc """
|
|
Retrieves a continuous stream of stored packets for replay in chronological order.
|
|
|
|
This function returns a Stream that can be used to process packets in chronological
|
|
order, preserving the timing between packets.
|
|
|
|
## Parameters
|
|
* `opts` - The same options as `get_packets_for_replay/1`
|
|
* `:playback_speed` - Speed multiplier (1.0 = real-time, 2.0 = 2x speed, etc.)
|
|
|
|
## Returns
|
|
* Stream of packets with timing information
|
|
"""
|
|
@impl true
|
|
def stream_packets_for_replay(opts \\ %{}) do
|
|
packets = get_packets_for_replay(opts)
|
|
playback_speed = Map.get(opts, :playback_speed, 1.0)
|
|
|
|
# Return a stream that emits packets with their original timing
|
|
Stream.unfold({packets, nil}, fn
|
|
{[], _} ->
|
|
nil
|
|
|
|
{[packet | rest], nil} ->
|
|
{{0, packet}, {rest, packet}}
|
|
|
|
{[next | rest], prev} ->
|
|
# Calculate delay between packets in milliseconds, then convert to seconds
|
|
delay_ms = DateTime.diff(next.received_at, prev.received_at, :millisecond)
|
|
adjusted_delay = delay_ms / (playback_speed * 1000)
|
|
{{adjusted_delay, next}, {rest, next}}
|
|
end)
|
|
end
|
|
|
|
@doc """
|
|
Gets the total count of stored packets in the database.
|
|
Uses the efficient packet_counters table with triggers for O(1) performance.
|
|
"""
|
|
@spec get_total_packet_count() :: non_neg_integer()
|
|
def get_total_packet_count do
|
|
# Use the PostgreSQL function for instant count retrieval
|
|
case Repo.query("SELECT get_packet_count()", []) do
|
|
{:ok, %{rows: [[count]]}} when is_integer(count) ->
|
|
count
|
|
|
|
{:ok, %{rows: [[nil]]}} ->
|
|
# Fallback to actual count if counter is not initialized
|
|
Repo.one(from p in Packet, select: count(p.id)) || 0
|
|
|
|
{:error, _} ->
|
|
# Try the faster counter table directly as a fallback
|
|
query = """
|
|
SELECT count FROM packet_counters
|
|
WHERE counter_type = 'total_packets'
|
|
LIMIT 1
|
|
"""
|
|
|
|
case Repo.query(query, []) do
|
|
{:ok, %{rows: [[count]]}} -> count
|
|
_ -> 0
|
|
end
|
|
end
|
|
rescue
|
|
DBConnection.ConnectionError ->
|
|
require Logger
|
|
|
|
Logger.warning("Database connection error in get_total_packet_count, returning 0")
|
|
0
|
|
|
|
error ->
|
|
require Logger
|
|
|
|
Logger.error("Error getting total packet count: #{inspect(error)}")
|
|
0
|
|
end
|
|
|
|
@doc """
|
|
Gets the timestamp of the oldest stored packet in the database.
|
|
Returns nil if no packets exist.
|
|
"""
|
|
@spec get_oldest_packet_timestamp() :: DateTime.t() | nil
|
|
def get_oldest_packet_timestamp do
|
|
Repo.one(
|
|
from p in Packet,
|
|
select: min(p.received_at)
|
|
)
|
|
rescue
|
|
_ -> nil
|
|
end
|
|
|
|
@doc """
|
|
Configure packet retention policy.
|
|
|
|
Packets are retained based on these rules:
|
|
- Default retention is 365 days (1 year) (configurable via :packet_retention_days)
|
|
- Returns the number of packets deleted
|
|
"""
|
|
@impl true
|
|
def clean_old_packets do
|
|
retention_days = Application.get_env(:aprsme, :packet_retention_days, 7)
|
|
cutoff_time = DateTime.add(DateTime.utc_now(), -retention_days * 86_400, :second)
|
|
|
|
{deleted_count, _} = Repo.delete_all(from(p in Packet, where: p.received_at < ^cutoff_time))
|
|
|
|
deleted_count
|
|
end
|
|
|
|
@doc """
|
|
Clean packets older than a specific number of days.
|
|
|
|
This function allows for more granular cleanup operations by specifying
|
|
the exact age threshold for packet deletion.
|
|
|
|
## Parameters
|
|
- `days` - Number of days to keep (packets older than this will be deleted)
|
|
|
|
## Returns
|
|
- Number of packets deleted
|
|
"""
|
|
@impl true
|
|
@spec clean_packets_older_than(pos_integer()) :: {:ok, non_neg_integer()} | {:error, any()}
|
|
def clean_packets_older_than(days) when is_integer(days) and days > 0 do
|
|
cutoff_time = DateTime.add(DateTime.utc_now(), -days * 86_400, :second)
|
|
|
|
{deleted_count, _} = Repo.delete_all(from(p in Packet, where: p.received_at < ^cutoff_time))
|
|
|
|
{:ok, deleted_count}
|
|
end
|
|
|
|
# Get packets from last hour only - used to initialize the map
|
|
@spec get_last_hour_packets() :: [struct()]
|
|
def get_last_hour_packets do
|
|
%{hours_back: 1, limit: 500}
|
|
|> QueryBuilder.recent_position_packets()
|
|
|> QueryBuilder.chronological()
|
|
|> Repo.all()
|
|
rescue
|
|
error ->
|
|
Logger.error("Failed to get last hour packets: #{inspect(error)}")
|
|
[]
|
|
end
|
|
|
|
@doc """
|
|
Gets the most recent packet for a callsign regardless of type or age.
|
|
This is used for API endpoints that need the latest packet from a source.
|
|
"""
|
|
@spec get_latest_packet_for_callsign(String.t()) :: struct() | nil
|
|
def get_latest_packet_for_callsign(callsign) when is_binary(callsign) do
|
|
# Use prepared statement for better performance
|
|
PreparedQueries.get_latest_packet_for_callsign(callsign)
|
|
end
|
|
|
|
@doc """
|
|
Gets latest positions for multiple callsigns in a single batch query.
|
|
Returns a list of `%{callsign: String.t(), lat: float(), lng: float()}` maps.
|
|
"""
|
|
@spec get_latest_positions_for_callsigns(list(String.t())) :: [map()]
|
|
def get_latest_positions_for_callsigns(callsigns) when is_list(callsigns) do
|
|
PreparedQueries.get_latest_positions_for_callsigns(callsigns)
|
|
end
|
|
|
|
@doc """
|
|
Gets the latest full packet for each callsign in a single batch query.
|
|
Returns a list of `Packet` structs with lat/lon populated.
|
|
"""
|
|
@spec get_latest_packets_for_callsigns(list(String.t())) :: [Packet.t()]
|
|
def get_latest_packets_for_callsigns(callsigns) when is_list(callsigns) do
|
|
PreparedQueries.get_latest_packets_for_callsigns(callsigns)
|
|
end
|
|
|
|
@doc """
|
|
Gets the most recent weather packet for a callsign.
|
|
Looks for any packet containing weather data fields.
|
|
"""
|
|
@spec get_latest_weather_packet(String.t()) :: struct() | nil
|
|
def get_latest_weather_packet(callsign) when is_binary(callsign) do
|
|
import Ecto.Query
|
|
|
|
query =
|
|
from p in Packet,
|
|
where: p.sender == ^callsign,
|
|
where: p.has_weather == true,
|
|
order_by: [desc: p.received_at],
|
|
limit: 1
|
|
|
|
Repo.one(query)
|
|
end
|
|
|
|
@doc """
|
|
Check if a callsign has any weather packets.
|
|
"""
|
|
def has_weather_packets?(callsign) when is_binary(callsign) do
|
|
# Use prepared statement for better performance
|
|
PreparedQueries.has_weather_packets?(callsign)
|
|
end
|
|
|
|
@doc """
|
|
Check which callsigns from a list have weather packets.
|
|
Returns a MapSet of uppercased callsigns with weather data.
|
|
"""
|
|
@spec weather_callsigns(list(String.t())) :: MapSet.t(String.t())
|
|
def weather_callsigns(callsigns) when is_list(callsigns) do
|
|
PreparedQueries.weather_callsigns(callsigns)
|
|
end
|
|
|
|
@doc """
|
|
Gets other SSIDs for a given callsign's base callsign.
|
|
Returns a list of maps with callsign, ssid, received_at, and packet info.
|
|
Only returns SSIDs active within the last hour.
|
|
"""
|
|
@spec get_other_ssids(String.t()) :: list(map())
|
|
def get_other_ssids(callsign) when is_binary(callsign) do
|
|
base_callsign = Aprsme.Callsign.extract_base(callsign)
|
|
one_hour_ago = DateTime.add(DateTime.utc_now(), -3600, :second)
|
|
|
|
query =
|
|
from p in Packet,
|
|
where: p.base_callsign == ^base_callsign,
|
|
where: p.received_at >= ^one_hour_ago,
|
|
where: p.sender != ^callsign,
|
|
distinct: p.sender,
|
|
order_by: [desc: p.received_at],
|
|
limit: 10,
|
|
select: %{
|
|
sender: p.sender,
|
|
ssid: p.ssid,
|
|
received_at: p.received_at,
|
|
id: p.id,
|
|
symbol_table_id: p.symbol_table_id,
|
|
symbol_code: p.symbol_code
|
|
}
|
|
|
|
query
|
|
|> Repo.all()
|
|
|> Enum.map(fn row ->
|
|
packet = %Packet{
|
|
id: row.id,
|
|
sender: row.sender,
|
|
ssid: row.ssid,
|
|
received_at: row.received_at,
|
|
symbol_table_id: row.symbol_table_id,
|
|
symbol_code: row.symbol_code
|
|
}
|
|
|
|
%{
|
|
callsign: row.sender,
|
|
ssid: row.ssid,
|
|
received_at: row.received_at,
|
|
packet: packet
|
|
}
|
|
end)
|
|
end
|
|
end
|