Implement progressive loading for historical packets to improve map performance

- Add progressive loading system that splits 200 packets into 4 batches of 50 each
- Implement client-side chunked rendering to prevent UI blocking
- Add caching layer using Aprsme.CachedQueries for better database performance
- Maintain proper chronological ordering for trail drawing
- Use LiveView's efficient update mechanisms for smooth user experience

Performance improvements:
- Faster initial load with immediate visual feedback
- Reduced memory pressure through smaller batch sizes
- Better caching with automatic cache invalidation
- Smooth, non-blocking UI updates using requestAnimationFrame

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

assets/js/map.ts

@@ -645,6 +645,71 @@ let MapAPRSMap = {
       }
     });
 
+    // Handle progressive loading of historical packets (batch processing)
+    self.handleEvent("add_historical_packets_batch", (data: { packets: MarkerData[], batch: number, is_final: boolean }) => {
+      if (data.packets && Array.isArray(data.packets)) {
+        // Use requestAnimationFrame to ensure smooth rendering
+        requestAnimationFrame(() => {
+          // Process packets in smaller chunks to prevent UI blocking
+          const chunkSize = 10;
+          let currentIndex = 0;
+          
+          const processChunk = () => {
+            const chunk = data.packets.slice(currentIndex, currentIndex + chunkSize);
+            
+            // Group packets by callsign to process them in chronological order for proper trail drawing
+            const packetsByCallsign = new Map<string, MarkerData[]>();
+
+            chunk.forEach((packet) => {
+              const callsign = packet.callsign_group || packet.callsign || packet.id;
+              if (!packetsByCallsign.has(callsign)) {
+                packetsByCallsign.set(callsign, []);
+              }
+              packetsByCallsign.get(callsign)!.push(packet);
+            });
+
+            // Process each callsign group in chronological order (oldest first)
+            packetsByCallsign.forEach((packets, callsign) => {
+              // Sort by timestamp (oldest first) to ensure proper trail line drawing
+              const sortedPackets = packets.sort((a, b) => {
+                const timeA = a.timestamp
+                  ? typeof a.timestamp === "number"
+                    ? a.timestamp
+                    : new Date(a.timestamp).getTime()
+                  : 0;
+                const timeB = b.timestamp
+                  ? typeof b.timestamp === "number"
+                    ? b.timestamp
+                    : new Date(b.timestamp).getTime()
+                  : 0;
+                return timeA - timeB;
+              });
+
+              // Add markers in chronological order
+              sortedPackets.forEach((packet) => {
+                self.addMarker({
+                  ...packet,
+                  historical: true,
+                  popup: packet.popup || self.buildPopupContent(packet),
+                });
+              });
+            });
+            
+            currentIndex += chunkSize;
+            
+            // Continue processing if there are more packets
+            if (currentIndex < data.packets.length) {
+              // Use setTimeout to yield control and prevent blocking
+              setTimeout(processChunk, 0);
+            }
+          };
+          
+          // Start processing
+          processChunk();
+        });
+      }
+    });
+
     // Handle refresh markers event
     self.handleEvent("refresh_markers", () => {
       // Remove markers that are outside current bounds

lib/aprsme/packets.ex

@@ -168,6 +168,11 @@ defmodule Aprsme.Packets do
   defp insert_packet(attrs, packet_data) do
     case %Packet{} |> Packet.changeset(attrs) |> Repo.insert() do
       {:ok, packet} ->
+        # Invalidate cache for this packet's callsign
+        if Map.has_key?(attrs, :sender) do
+          Aprsme.CachedQueries.invalidate_callsign_cache(attrs.sender)
+        end
+
         {:ok, packet}
 
       {:error, changeset} ->
@@ -413,6 +418,7 @@ defmodule Aprsme.Packets do
     # Always limit to the last hour for initial load
     one_hour_ago = DateTime.add(DateTime.utc_now(), -3600, :second)
     limit = Map.get(opts, :limit, 200)
+    offset = Map.get(opts, :offset, 0)
 
     # Use a more efficient query that leverages the partial indexes
     # Order by received_at DESC to get the most recent packets first
@@ -421,7 +427,8 @@ defmodule Aprsme.Packets do
         where: p.has_position == true,
         where: p.received_at >= ^one_hour_ago,
         order_by: [desc: p.received_at],
-        limit: ^limit
+        limit: ^limit,
+        offset: ^offset
       )
 
     query =

lib/aprsme_web/live/map_live/index.ex

@@ -283,6 +283,11 @@ defmodule AprsmeWeb.MapLive.Index do
 
   def handle_info(:start_geolocation, socket), do: handle_info_start_geolocation(socket)
 
+  def handle_info({:load_historical_batch, batch_offset}, socket) do
+    socket = load_historical_batch(socket, batch_offset)
+    {:noreply, socket}
+  end
+
   def handle_info(%Phoenix.Socket.Broadcast{topic: "aprs_messages", event: "packet", payload: packet}, socket),
     do: handle_info({:postgres_packet, packet}, socket)
 
@@ -970,8 +975,8 @@ defmodule AprsmeWeb.MapLive.Index do
       # Clear existing historical packets
       socket = push_event(socket, "clear_historical_packets", %{})
 
-      # Use optimized loading for better performance
+      # Start progressive loading using LiveView's efficient batching
-      socket = load_historical_packets_for_bounds_optimized(socket, socket.assigns.map_bounds)
+      socket = start_progressive_historical_loading(socket)
 
       {:noreply, socket}
     else
@@ -1166,11 +1171,21 @@ defmodule AprsmeWeb.MapLive.Index do
     packets_module = Application.get_env(:aprsme, :packets_module, Aprsme.Packets)
 
     historical_packets =
-      packets_module.get_recent_packets_optimized(%{
+      if packets_module == Aprsme.Packets do
-        bounds: bounds,
+        # Use cached queries for better performance
-        # Reduced limit for faster initial load
+        Aprsme.CachedQueries.get_recent_packets_cached(%{
-        limit: 200
+          bounds: bounds,
-      })
+          # Reduced limit for faster initial load
+          limit: 200
+        })
+      else
+        # Fallback for testing
+        packets_module.get_recent_packets_optimized(%{
+          bounds: bounds,
+          # Reduced limit for faster initial load
+          limit: 200
+        })
+      end
 
     if Enum.empty?(historical_packets) do
       assign(socket, historical_loaded: true)
@@ -1179,6 +1194,85 @@ defmodule AprsmeWeb.MapLive.Index do
     end
   end
 
+  # Progressive loading functions using LiveView's efficient update mechanisms
+  @spec start_progressive_historical_loading(Socket.t()) :: Socket.t()
+  defp start_progressive_historical_loading(socket) do
+    # Start with a small batch for immediate visual feedback
+    socket =
+      socket
+      |> assign(loading_batch: 0, total_batches: 4)
+      |> load_historical_batch(0)
+
+    # Schedule next batches using LiveView's async processing
+    Process.send_after(self(), {:load_historical_batch, 1}, 50)
+    Process.send_after(self(), {:load_historical_batch, 2}, 150)
+    Process.send_after(self(), {:load_historical_batch, 3}, 300)
+
+    socket
+  end
+
+  @spec load_historical_batch(Socket.t(), integer()) :: Socket.t()
+  defp load_historical_batch(socket, batch_offset) do
+    if socket.assigns.map_bounds do
+      bounds = [
+        socket.assigns.map_bounds.west,
+        socket.assigns.map_bounds.south,
+        socket.assigns.map_bounds.east,
+        socket.assigns.map_bounds.north
+      ]
+
+      # Load smaller batches with offset for progressive loading
+      batch_size = 50
+      offset = batch_offset * batch_size
+
+      packets_module = Application.get_env(:aprsme, :packets_module, Aprsme.Packets)
+
+      historical_packets =
+        if packets_module == Aprsme.Packets do
+          # Use cached queries for better performance
+          Aprsme.CachedQueries.get_recent_packets_cached(%{
+            bounds: bounds,
+            limit: batch_size,
+            offset: offset
+          })
+        else
+          # Fallback for testing
+          packets_module.get_recent_packets_optimized(%{
+            bounds: bounds,
+            limit: batch_size,
+            offset: offset
+          })
+        end
+
+      if Enum.any?(historical_packets) do
+        # Process this batch and send to frontend
+        packet_data_list = build_packet_data_list(historical_packets)
+
+        if Enum.any?(packet_data_list) do
+          # Use LiveView's efficient push_event for incremental updates
+          socket =
+            push_event(socket, "add_historical_packets_batch", %{
+              packets: packet_data_list,
+              batch: batch_offset,
+              is_final: batch_offset >= 3
+            })
+
+          # Update progress for user feedback
+          socket = assign(socket, loading_batch: batch_offset + 1)
+
+          socket
+        else
+          socket
+        end
+      else
+        # No more data in this batch
+        socket
+      end
+    else
+      socket
+    end
+  end
+
   @spec within_bounds?(map() | struct(), map()) :: boolean()
   defp within_bounds?(packet, bounds) do
     {lat, lon, _data_extended} = MapHelpers.get_coordinates(packet)