use postgis for testing

.github/workflows/elixir.yaml

@@ -29,7 +29,7 @@ jobs:
     # Additional services can be defined here if required.
     services:
       db:
-        image: postgres:16
+        image: postgis/postgis:17-3.5
         ports: ["5432:5432"]
         env:
           POSTGRES_PASSWORD: postgres

assets/js/app.js

@@ -27,6 +27,8 @@ let csrfToken = document.querySelector("meta[name='csrf-token']").getAttribute("
 
 // Import minimal APRS map hook
 import MapAPRSMap from "./map";
+// Import error boundary hook
+import ErrorBoundary from "./hooks/error_boundary";
 
 // Responsive Slideover Hook
 let ResponsiveSlideoverHook = {
@@ -98,6 +100,7 @@ let Hooks = {};
 Hooks.APRSMap = MapAPRSMap;
 Hooks.ResponsiveSlideoverHook = ResponsiveSlideoverHook;
 Hooks.BodyClassHook = BodyClassHook;
+Hooks.ErrorBoundary = ErrorBoundary;
 
 // Register weather chart hooks from TypeScript
 import { WeatherChartHooks } from "./features/weather_charts";

assets/js/hooks/error_boundary.js

@@ -0,0 +1,84 @@
+// Error Boundary Hook for Phoenix LiveView
+// Catches JavaScript errors and displays fallback content
+
+const ErrorBoundary = {
+  mounted() {
+    // Store original error handler
+    this.originalErrorHandler = window.onerror;
+    this.originalUnhandledRejection = window.onunhandledrejection;
+    
+    // Get content and fallback elements
+    this.content = this.el.querySelector('.error-boundary-content');
+    this.fallback = this.el.querySelector('.error-boundary-fallback');
+    
+    // Set up error handlers for this component
+    this.setupErrorHandlers();
+  },
+  
+  setupErrorHandlers() {
+    // Handle synchronous errors
+    window.onerror = (message, source, lineno, colno, error) => {
+      if (this.isErrorInComponent(error)) {
+        this.handleError(error || new Error(message));
+        return true; // Prevent default error handling
+      }
+      
+      // Call original handler if error is not in this component
+      if (this.originalErrorHandler) {
+        return this.originalErrorHandler(message, source, lineno, colno, error);
+      }
+      return false;
+    };
+    
+    // Handle async errors (unhandled promise rejections)
+    window.onunhandledrejection = (event) => {
+      if (this.isErrorInComponent(event.reason)) {
+        this.handleError(event.reason);
+        event.preventDefault();
+        return;
+      }
+      
+      // Call original handler if error is not in this component
+      if (this.originalUnhandledRejection) {
+        this.originalUnhandledRejection(event);
+      }
+    };
+  },
+  
+  isErrorInComponent(error) {
+    // Check if the error occurred within this component's DOM tree
+    // This is a simplified check - in production you might want more sophisticated logic
+    if (!error || !error.stack) return false;
+    
+    // Check if any element in the component has an error
+    const hasError = this.el.querySelector('[data-phx-error]');
+    return !!hasError;
+  },
+  
+  handleError(error) {
+    console.error('Error caught by boundary:', error);
+    
+    // Hide content and show fallback
+    if (this.content) {
+      this.content.style.display = 'none';
+    }
+    if (this.fallback) {
+      this.fallback.classList.remove('hidden');
+    }
+    
+    // Log error to server
+    this.pushEvent('error_boundary_triggered', {
+      message: error.message,
+      stack: error.stack,
+      component_id: this.el.id
+    });
+  },
+  
+  destroyed() {
+    // Restore original error handlers
+    window.onerror = this.originalErrorHandler;
+    window.onunhandledrejection = this.originalUnhandledRejection;
+  }
+};
+
+export default ErrorBoundary;

assets/js/map.ts

@@ -1,5 +1,16 @@
-// Declare Leaflet as a global variable
-declare const L: any;
+// Import type definitions
+import type { 
+  LiveViewHookContext, 
+  MarkerData, 
+  BoundsData, 
+  CenterData, 
+  MarkerState, 
+  MapEventData 
+} from './types/map';
+import type { Map as LeafletMap, Marker, TileLayer, LayerGroup, DivIcon, LatLngBounds } from 'leaflet';
+
+// Declare Leaflet as a global variable with proper typing
+declare const L: typeof import('leaflet');
 declare const OverlappingMarkerSpiderfier: any;
 
 // Import trail management functionality
@@ -10,85 +21,6 @@ import { parseTimestamp, getTrailId, saveMapState, safePushEvent, getLiveSocket
 // APRS Map Hook - handles only basic map interaction
 // All data logic handled by LiveView
 
-type LiveViewHookContext = {
-  el: HTMLElement & { _leaflet_id?: any };
-  pushEvent: (event: string, payload: any) => void;
-  handleEvent: (event: string, callback: Function) => void;
-  map?: any;
-  markers?: Map<string, any>;
-  markerStates?: Map<string, MarkerState>;
-  markerLayer?: any;
-  trailManager?: TrailManager;
-  boundsTimer?: ReturnType<typeof setTimeout>;
-  resizeHandler?: () => void;
-  errors?: string[];
-  initializationAttempts?: number;
-  maxInitializationAttempts?: number;
-  lastZoom?: number;
-  currentPopupMarkerId?: string | null;
-  oms?: any;
-  programmaticMoveId?: string;
-  programmaticMoveTimeout?: ReturnType<typeof setTimeout>;
-  cleanupInterval?: ReturnType<typeof setInterval>;
-  mapEventHandlers?: Map<string, Function>;
-  isDestroyed?: boolean;
-  popupNavigationHandler?: (e: Event) => void;
-  [key: string]: any;
-};
-
-interface MarkerData {
-  id: string;
-  lat: number;
-  lng: number;
-  callsign?: string;
-  comment?: string;
-  symbol_table_id?: string;
-  symbol_code?: string;
-  symbol_description?: string;
-  popup?: string;
-  historical?: boolean;
-  color?: string;
-  timestamp?: number;
-  is_most_recent_for_callsign?: boolean;
-  callsign_group?: string;
-  symbol_html?: string;
-}
-
-interface BoundsData {
-  north: number;
-  south: number;
-  east: number;
-  west: number;
-}
-
-interface CenterData {
-  lat: number;
-  lng: number;
-}
-
-interface MarkerState {
-  lat: number;
-  lng: number;
-  symbol_table: string;
-  symbol_code: string;
-  popup?: string;
-  historical?: boolean;
-  is_most_recent_for_callsign?: boolean;
-  callsign_group?: string;
-  callsign?: string;
-}
-
-interface MapEventData {
-  bounds?: BoundsData;
-  center?: CenterData;
-  zoom?: number;
-  id?: string;
-  callsign?: string;
-  lat?: number;
-  lng?: number;
-  markers?: MarkerData[];
-}
-
 
 let MapAPRSMap = {
   mounted() {

assets/js/types/map.d.ts

@@ -0,0 +1,102 @@
+// Type definitions for APRS Map application
+
+import type { Map as LeafletMap, Marker, LatLng, LatLngBounds, DivIcon, LayerGroup, Popup } from 'leaflet';
+
+export interface LiveViewHookContext {
+  el: HTMLElement & { _leaflet_id?: any };
+  pushEvent: (event: string, payload: any) => void;
+  handleEvent: (event: string, callback: (data: any) => void) => void;
+  map?: LeafletMap;
+  markers?: Map<string, Marker>;
+  markerStates?: Map<string, MarkerState>;
+  markerLayer?: LayerGroup;
+  trailManager?: any; // Import from trail_manager.ts when typed
+  boundsTimer?: ReturnType<typeof setTimeout>;
+  resizeHandler?: () => void;
+  errors?: string[];
+  initializationAttempts?: number;
+  maxInitializationAttempts?: number;
+  lastZoom?: number;
+  currentPopupMarkerId?: string | null;
+  oms?: any; // OverlappingMarkerSpiderfier type
+  programmaticMoveId?: string;
+  programmaticMoveTimeout?: ReturnType<typeof setTimeout>;
+  cleanupInterval?: ReturnType<typeof setInterval>;
+  mapEventHandlers?: Map<string, Function>;
+  isDestroyed?: boolean;
+  popupNavigationHandler?: (e: Event) => void;
+  moveEndHandler?: () => void;
+  zoomEndHandler?: () => void;
+  [key: string]: any;
+}
+
+export interface MarkerData {
+  id: string;
+  lat: number;
+  lng: number;
+  callsign?: string;
+  comment?: string;
+  symbol_table_id?: string;
+  symbol_code?: string;
+  symbol_description?: string;
+  popup?: string;
+  historical?: boolean;
+  color?: string;
+  timestamp?: number | string;
+  is_most_recent_for_callsign?: boolean;
+  callsign_group?: string;
+  symbol_html?: string;
+  openPopup?: boolean;
+}
+
+export interface BoundsData {
+  north: number;
+  south: number;
+  east: number;
+  west: number;
+}
+
+export interface CenterData {
+  lat: number;
+  lng: number;
+}
+
+export interface MarkerState {
+  lat: number;
+  lng: number;
+  symbol_table: string;
+  symbol_code: string;
+  popup?: string;
+  historical?: boolean;
+  is_most_recent_for_callsign?: boolean;
+  callsign_group?: string;
+  callsign?: string;
+}
+
+export interface MapEventData {
+  bounds?: BoundsData;
+  center?: CenterData;
+  zoom?: number;
+  id?: string;
+  callsign?: string;
+  lat?: number;
+  lng?: number;
+  markers?: MarkerData[];
+}
+
+export interface MapState {
+  lat: number;
+  lng: number;
+  zoom: number;
+}
+
+export interface LiveSocket {
+  connected: boolean;
+  pushHistoryPatch: (href: string, state: string, target: HTMLElement) => void;
+}
+
+declare global {
+  interface Window {
+    liveSocket?: LiveSocket;
+  }
+}

docs/virtual_marker_scrolling.md

@@ -0,0 +1,114 @@
+# Virtual Marker Scrolling for Large Datasets
+
+## Overview
+
+When dealing with thousands of APRS markers on the map, rendering all markers at once can cause performance issues. Virtual scrolling for map markers is a technique where only visible markers within the viewport (plus a buffer zone) are rendered.
+
+## Current Implementation
+
+The current implementation already has some optimizations:
+1. Viewport-based filtering in `get_recent_packets_optimized/3`
+2. Marker state tracking to prevent unnecessary updates
+3. Batch processing of historical packets
+
+## Proposed Virtual Scrolling Enhancement
+
+### 1. Marker Clustering at Low Zoom Levels
+```typescript
+// Use Leaflet.markercluster for automatic clustering
+// Already included in the project but not fully utilized
+if (self.map.getZoom() < 10) {
+  // Use marker clustering
+  self.clusterLayer = L.markerClusterGroup({
+    maxClusterRadius: 80,
+    spiderfyOnMaxZoom: true,
+    showCoverageOnHover: false,
+    zoomToBoundsOnClick: true
+  });
+}
+```
+
+### 2. Quadtree-based Spatial Indexing
+```elixir
+defmodule AprsmeWeb.MapLive.QuadTree do
+  @moduledoc """
+  Quadtree implementation for efficient spatial queries
+  """
+  
+  defstruct [:bounds, :markers, :children, :max_markers]
+  
+  def insert(tree, marker) do
+    # Insert marker into appropriate quadrant
+  end
+  
+  def query(tree, bounds) do
+    # Return only markers within bounds
+  end
+end
+```
+
+### 3. Progressive Rendering with RequestIdleCallback
+```typescript
+function renderMarkersProgressively(markers: MarkerData[]) {
+  let index = 0;
+  const batchSize = 50;
+  
+  function renderBatch() {
+    const batch = markers.slice(index, index + batchSize);
+    batch.forEach(marker => self.addMarker(marker));
+    index += batchSize;
+    
+    if (index < markers.length) {
+      requestIdleCallback(renderBatch);
+    }
+  }
+  
+  requestIdleCallback(renderBatch);
+}
+```
+
+### 4. Server-side Aggregation
+```elixir
+def aggregate_markers_for_zoom(zoom_level, bounds) do
+  case zoom_level do
+    z when z < 8 ->
+      # Return aggregated grid cells with counts
+      Packets.get_grid_aggregates(bounds, grid_size: 50)
+    
+    z when z < 12 ->
+      # Return simplified markers (no popups)
+      Packets.get_simplified_markers(bounds)
+    
+    _ ->
+      # Return full markers
+      Packets.get_recent_packets_optimized(bounds)
+  end
+end
+```
+
+### 5. Viewport Buffer Strategy
+```typescript
+// Render markers in expanded viewport to reduce re-renders during panning
+const bounds = self.map.getBounds();
+const bufferedBounds = bounds.pad(0.5); // 50% buffer
+```
+
+## Implementation Priority
+
+1. **Phase 1**: Implement marker clustering (easiest, biggest impact)
+2. **Phase 2**: Add server-side aggregation for low zoom levels
+3. **Phase 3**: Implement progressive rendering
+4. **Phase 4**: Add quadtree spatial indexing if still needed
+
+## Performance Metrics to Track
+
+- Time to first marker render
+- Frame rate during pan/zoom
+- Memory usage with 10k+ markers
+- Server query time by zoom level
+
+## Notes
+
+- The current implementation already handles viewport filtering well
+- Virtual scrolling is most beneficial when dealing with 5000+ markers
+- Consider implementing only if performance issues are reported by users

lib/aprsme_web/live/components/error_boundary.ex

@@ -0,0 +1,85 @@
+defmodule AprsmeWeb.Components.ErrorBoundary do
+  @moduledoc """
+  Error boundary component for gracefully handling errors in LiveView.
+
+  This component wraps other components and catches errors, displaying
+  a user-friendly error message instead of crashing the entire view.
+  """
+  use Phoenix.Component
+
+  import AprsmeWeb.CoreComponents
+
+  alias Phoenix.LiveView.Socket
+
+  @doc """
+  Wraps content in an error boundary that catches and displays errors gracefully.
+
+  ## Examples
+
+      <.error_boundary id="map-error-boundary">
+        <%= live_render(@socket, AprsmeWeb.MapLive.Index) %>
+      </.error_boundary>
+  """
+  attr :id, :string, required: true
+  attr :class, :string, default: nil
+  slot :inner_block, required: true
+
+  slot :fallback do
+    attr :error, :string
+  end
+
+  def error_boundary(assigns) do
+    ~H"""
+    <div id={@id} class={@class} phx-hook="ErrorBoundary">
+      <div class="error-boundary-content">
+        {render_slot(@inner_block)}
+      </div>
+      <div class="error-boundary-fallback hidden">
+        <%= if @fallback == [] do %>
+          <.default_error_message />
+        <% else %>
+          {render_slot(@fallback, %{})}
+        <% end %>
+      </div>
+    </div>
+    """
+  end
+
+  defp default_error_message(assigns) do
+    ~H"""
+    <div class="rounded-md bg-red-50 p-4 my-4">
+      <div class="flex">
+        <div class="flex-shrink-0">
+          <svg class="h-5 w-5 text-red-400" viewBox="0 0 20 20" fill="currentColor">
+            <path
+              fill-rule="evenodd"
+              d="M10 18a8 8 0 100-16 8 8 0 000 16zM8.707 7.293a1 1 0 00-1.414 1.414L8.586 10l-1.293 1.293a1 1 0 101.414 1.414L10 11.414l1.293 1.293a1 1 0 001.414-1.414L11.414 10l1.293-1.293a1 1 0 00-1.414-1.414L10 8.586 8.707 7.293z"
+              clip-rule="evenodd"
+            />
+          </svg>
+        </div>
+        <div class="ml-3">
+          <h3 class="text-sm font-medium text-red-800">
+            Something went wrong
+          </h3>
+          <div class="mt-2 text-sm text-red-700">
+            <p>
+              We're sorry, but something unexpected happened. The error has been logged
+              and we'll look into it. Please try refreshing the page.
+            </p>
+          </div>
+          <div class="mt-4">
+            <button
+              type="button"
+              onclick="window.location.reload()"
+              class="bg-red-50 px-2 py-1.5 rounded-md text-sm font-medium text-red-800 hover:bg-red-100 focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-offset-red-50 focus:ring-red-600"
+            >
+              Refresh page
+            </button>
+          </div>
+        </div>
+      </div>
+    </div>
+    """
+  end
+end

lib/aprsme_web/live/map_live/index.ex

@@ -4,6 +4,7 @@ defmodule AprsmeWeb.MapLive.Index do
   """
   use AprsmeWeb, :live_view
 
+  import AprsmeWeb.Components.ErrorBoundary
   import AprsmeWeb.TimeHelpers, only: [time_ago_in_words: 1]
   import Phoenix.LiveView, only: [connected?: 1, push_event: 3, push_navigate: 2, push_patch: 2]
 
@@ -408,6 +409,23 @@ defmodule AprsmeWeb.MapLive.Index do
     {:noreply, socket}
   end
 
+  @impl true
+  def handle_event(
+        "error_boundary_triggered",
+        %{"message" => message, "stack" => stack, "component_id" => component_id},
+        socket
+      ) do
+    # Log the error for monitoring
+    require Logger
+
+    Logger.error("Error boundary triggered in component #{component_id}: #{message}\n#{stack}")
+
+    # You could also send this to an error tracking service here
+    # ErrorTracker.report_error(message, stack, %{component: component_id, user_id: socket.assigns[:current_user_id]})
+
+    {:noreply, socket}
+  end
+
   @impl true
   def handle_params(params, _url, socket) do
     # Parse new map state from URL parameters
@@ -781,15 +799,17 @@ defmodule AprsmeWeb.MapLive.Index do
       }
     </style>
 
-    <div
-      id="aprs-map"
-      class={if @slideover_open, do: "slideover-open", else: "slideover-closed"}
-      phx-hook="APRSMap"
-      phx-update="ignore"
-      data-center={Jason.encode!(@map_center)}
-      data-zoom={@map_zoom}
-    >
-    </div>
+    <.error_boundary id="map-error-boundary">
+      <div
+        id="aprs-map"
+        class={if @slideover_open, do: "slideover-open", else: "slideover-closed"}
+        phx-hook="APRSMap"
+        phx-update="ignore"
+        data-center={Jason.encode!(@map_center)}
+        data-zoom={@map_zoom}
+      >
+      </div>
+    </.error_boundary>
 
     <button
       class="locate-button"

test/assets/js/map_helpers_test.ts

@@ -0,0 +1,186 @@
+import { describe, test, expect, beforeEach, afterEach, vi } from 'vitest';
+import { 
+  parseTimestamp, 
+  getTrailId, 
+  saveMapState, 
+  safePushEvent,
+  isLiveViewConnected,
+  getLiveSocket
+} from '../../../assets/js/map_helpers';
+
+describe('map_helpers', () => {
+  describe('parseTimestamp', () => {
+    test('returns current time for null/undefined', () => {
+      const before = Date.now();
+      const result = parseTimestamp(null);
+      const after = Date.now();
+      expect(result).toBeGreaterThanOrEqual(before);
+      expect(result).toBeLessThanOrEqual(after);
+    });
+
+    test('returns number timestamp as-is', () => {
+      const timestamp = 1234567890;
+      expect(parseTimestamp(timestamp)).toBe(timestamp);
+    });
+
+    test('parses string timestamp', () => {
+      const dateStr = '2024-01-01T00:00:00Z';
+      const expected = new Date(dateStr).getTime();
+      expect(parseTimestamp(dateStr)).toBe(expected);
+    });
+
+    test('returns current time for invalid input', () => {
+      const before = Date.now();
+      const result = parseTimestamp({});
+      const after = Date.now();
+      expect(result).toBeGreaterThanOrEqual(before);
+      expect(result).toBeLessThanOrEqual(after);
+    });
+  });
+
+  describe('getTrailId', () => {
+    test('prioritizes callsign_group', () => {
+      const data = {
+        callsign_group: 'GROUP-1',
+        callsign: 'CALL-1',
+        id: 'ID-1'
+      };
+      expect(getTrailId(data)).toBe('GROUP-1');
+    });
+
+    test('falls back to callsign when no callsign_group', () => {
+      const data = {
+        callsign: 'CALL-1',
+        id: 'ID-1'
+      };
+      expect(getTrailId(data)).toBe('CALL-1');
+    });
+
+    test('falls back to id when no callsign_group or callsign', () => {
+      const data = {
+        id: 'ID-1'
+      };
+      expect(getTrailId(data)).toBe('ID-1');
+    });
+  });
+
+  describe('saveMapState', () => {
+    let mockMap: any;
+    let mockPushEvent: any;
+
+    beforeEach(() => {
+      // Mock localStorage
+      const localStorageMock = {
+        setItem: vi.fn()
+      };
+      Object.defineProperty(window, 'localStorage', {
+        value: localStorageMock,
+        writable: true
+      });
+
+      // Mock map object
+      mockMap = {
+        getCenter: vi.fn().mockReturnValue({ lat: 40.7128, lng: -74.0060 }),
+        getZoom: vi.fn().mockReturnValue(10),
+        getBounds: vi.fn().mockReturnValue({
+          getNorth: vi.fn().mockReturnValue(41.0),
+          getSouth: vi.fn().mockReturnValue(40.0),
+          getEast: vi.fn().mockReturnValue(-73.0),
+          getWest: vi.fn().mockReturnValue(-75.0)
+        })
+      };
+
+      mockPushEvent = vi.fn();
+    });
+
+    test('saves truncated coordinates to localStorage', () => {
+      saveMapState(mockMap, mockPushEvent);
+      
+      expect(localStorage.setItem).toHaveBeenCalledWith(
+        'aprs_map_state',
+        JSON.stringify({ lat: 40.7128, lng: -74.006, zoom: 10 })
+      );
+    });
+
+    test('pushes event with map state and bounds', () => {
+      saveMapState(mockMap, mockPushEvent);
+      
+      expect(mockPushEvent).toHaveBeenCalledWith('update_map_state', {
+        center: { lat: 40.7128, lng: -74.006 },
+        zoom: 10,
+        bounds: {
+          north: 41.0,
+          south: 40.0,
+          east: -73.0,
+          west: -75.0
+        }
+      });
+    });
+
+    test('truncates coordinates to 5 decimal places', () => {
+      mockMap.getCenter.mockReturnValue({ lat: 40.71281234567, lng: -74.00601234567 });
+      
+      saveMapState(mockMap, mockPushEvent);
+      
+      const call = mockPushEvent.mock.calls[0][1];
+      expect(call.center.lat).toBe(40.71281);
+      expect(call.center.lng).toBe(-74.00601);
+    });
+  });
+
+  describe('safePushEvent', () => {
+    test('calls pushEvent and returns true on success', () => {
+      const mockPushEvent = vi.fn();
+      const result = safePushEvent(mockPushEvent, 'test_event', { data: 'test' });
+      
+      expect(mockPushEvent).toHaveBeenCalledWith('test_event', { data: 'test' });
+      expect(result).toBe(true);
+    });
+
+    test('returns false when pushEvent is undefined', () => {
+      const result = safePushEvent(undefined, 'test_event', { data: 'test' });
+      expect(result).toBe(false);
+    });
+
+    test('catches error and returns false', () => {
+      const mockPushEvent = vi.fn().mockImplementation(() => {
+        throw new Error('LiveView not connected');
+      });
+      
+      const consoleSpy = vi.spyOn(console, 'debug').mockImplementation(() => {});
+      const result = safePushEvent(mockPushEvent, 'test_event', { data: 'test' });
+      
+      expect(result).toBe(false);
+      expect(consoleSpy).toHaveBeenCalledWith('Unable to send test_event event - LiveView disconnected');
+      
+      consoleSpy.mockRestore();
+    });
+  });
+
+  describe('LiveView socket helpers', () => {
+    afterEach(() => {
+      // Clean up window.liveSocket
+      delete (window as any).liveSocket;
+    });
+
+    test('isLiveViewConnected returns true when socket exists', () => {
+      (window as any).liveSocket = { connected: true };
+      expect(isLiveViewConnected()).toBe(true);
+    });
+
+    test('isLiveViewConnected returns false when socket missing', () => {
+      expect(isLiveViewConnected()).toBe(false);
+    });
+
+    test('getLiveSocket returns the socket', () => {
+      const mockSocket = { connected: true, pushHistoryPatch: vi.fn() };
+      (window as any).liveSocket = mockSocket;
+      
+      expect(getLiveSocket()).toBe(mockSocket);
+    });
+
+    test('getLiveSocket returns undefined when missing', () => {
+      expect(getLiveSocket()).toBeUndefined();
+    });
+  });
+});