aprs.me/lib/aprsme/streaming_packets_pubsub.ex
Graham McIntire 234fc2010f
Implement critical performance optimizations and error handling
This commit introduces several major performance improvements and adds
robust error handling for malformed HTTP requests:

Performance Optimizations:
- Add database migration with BRIN indexes for time-series data and partial
  indexes for recent queries, significantly improving query performance
- Create global StreamingPacketsPubSub system for real-time packet distribution
  with geographic bounds filtering using ETS for fast lookups
- Refactor PacketConsumer to use Stream module for memory-efficient processing,
  preventing memory accumulation during batch operations
- Implement dedicated BroadcastTaskSupervisor pool for async broadcast operations,
  preventing GenServer blocking on I/O
- Increase database connection pool from 25 to 45 (production) and 15 to 30 (dev)
  for better concurrency support

Error Handling:
- Add SentryFilter to prevent Bandit.HTTPError from missing Host headers from
  cluttering Sentry (common with bots/scanners)
- Configure custom 400 Bad Request error handling
- Filter out common bot/scanner paths from error reporting

All changes include comprehensive test coverage following TDD practices.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
2025-07-18 18:19:49 -05:00

163 lines
4 KiB
Elixir

defmodule Aprsme.StreamingPacketsPubSub do
@moduledoc """
Global PubSub system for streaming APRS packets to subscribers based on geographic bounds.
This module provides efficient real-time packet distribution with geographic filtering,
allowing multiple GenServers to subscribe to packet streams filtered by lat/long bounds.
"""
use GenServer
require Logger
@table_name :streaming_packets_subscribers
# Client API
@doc """
Starts the StreamingPacketsPubSub GenServer.
"""
def start_link(opts \\ []) do
GenServer.start_link(__MODULE__, opts, name: __MODULE__)
end
@doc """
Subscribe to packets within the specified geographic bounds.
## Parameters
- pid: The process to receive packet notifications
- bounds: Map with :north, :south, :east, :west keys defining the geographic area
## Returns
- :ok
"""
def subscribe_to_bounds(pid, bounds) do
GenServer.call(__MODULE__, {:subscribe, pid, bounds})
end
@doc """
Unsubscribe from packet notifications.
"""
def unsubscribe(pid) do
GenServer.call(__MODULE__, {:unsubscribe, pid})
end
@doc """
Broadcast a packet to all subscribers whose bounds contain the packet's location.
"""
def broadcast_packet(packet) do
GenServer.cast(__MODULE__, {:broadcast, packet})
end
@doc """
List all active subscribers and their bounds.
"""
def list_subscribers do
GenServer.call(__MODULE__, :list_subscribers)
end
# Server Callbacks
@impl true
def init(_opts) do
# Create ETS table for fast lookups
:ets.new(@table_name, [:set, :protected, :named_table])
# Monitor subscribers for cleanup
Process.flag(:trap_exit, true)
{:ok, %{}}
end
@impl true
def handle_call({:subscribe, pid, bounds}, _from, state) do
# Validate bounds
if valid_bounds?(bounds) do
# Monitor the subscriber
Process.monitor(pid)
# Store in ETS for fast lookup
:ets.insert(@table_name, {pid, bounds})
{:reply, :ok, state}
else
{:reply, {:error, :invalid_bounds}, state}
end
end
@impl true
def handle_call({:unsubscribe, pid}, _from, state) do
:ets.delete(@table_name, pid)
{:reply, :ok, state}
end
@impl true
def handle_call(:list_subscribers, _from, state) do
subscribers = :ets.tab2list(@table_name)
{:reply, subscribers, state}
end
@impl true
def handle_cast({:broadcast, packet}, state) do
# Get packet coordinates
lat = packet[:latitude] || packet[:lat]
lon = packet[:longitude] || packet[:lon] || packet[:lng]
if lat && lon do
# Find all subscribers whose bounds contain this packet
subscribers =
:ets.select(@table_name, [
{
{:"$1", :"$2"},
[],
[{{:"$1", :"$2"}}]
}
])
# Send to matching subscribers using BroadcastTaskSupervisor
Aprsme.BroadcastTaskSupervisor.async_execute(fn ->
subscribers
|> Stream.filter(fn {_pid, bounds} -> packet_in_bounds?(lat, lon, bounds) end)
|> Enum.each(fn {pid, _bounds} -> send(pid, {:streaming_packet, packet}) end)
end)
end
{:noreply, state}
end
@impl true
def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do
# Clean up subscriber when process dies
:ets.delete(@table_name, pid)
{:noreply, state}
end
# Private functions
defp valid_bounds?(%{north: n, south: s, east: e, west: w}) do
is_number(n) and is_number(s) and is_number(e) and is_number(w) and
n >= s and
n >= -90 and n <= 90 and
s >= -90 and s <= 90 and
e >= -180 and e <= 180 and
w >= -180 and w <= 180
end
defp valid_bounds?(_), do: false
defp packet_in_bounds?(lat, lon, %{north: n, south: s, east: e, west: w}) do
lat_in_bounds = lat >= s and lat <= n
# Handle longitude wrap-around at international date line
lon_in_bounds =
if w > e do
# Bounds cross the date line
lon >= w or lon <= e
else
# Normal bounds
lon >= w and lon <= e
end
lat_in_bounds and lon_in_bounds
end
end