diff --git a/assets/js/map.ts b/assets/js/map.ts index 6bc9b36..eef7a57 100644 --- a/assets/js/map.ts +++ b/assets/js/map.ts @@ -645,6 +645,71 @@ let MapAPRSMap = { } }); + // Handle progressive loading of historical packets (batch processing) + self.handleEvent("add_historical_packets_batch", (data: { packets: MarkerData[], batch: number, is_final: boolean }) => { + if (data.packets && Array.isArray(data.packets)) { + // Use requestAnimationFrame to ensure smooth rendering + requestAnimationFrame(() => { + // Process packets in smaller chunks to prevent UI blocking + const chunkSize = 10; + let currentIndex = 0; + + const processChunk = () => { + const chunk = data.packets.slice(currentIndex, currentIndex + chunkSize); + + // Group packets by callsign to process them in chronological order for proper trail drawing + const packetsByCallsign = new Map(); + + chunk.forEach((packet) => { + const callsign = packet.callsign_group || packet.callsign || packet.id; + if (!packetsByCallsign.has(callsign)) { + packetsByCallsign.set(callsign, []); + } + packetsByCallsign.get(callsign)!.push(packet); + }); + + // Process each callsign group in chronological order (oldest first) + packetsByCallsign.forEach((packets, callsign) => { + // Sort by timestamp (oldest first) to ensure proper trail line drawing + const sortedPackets = packets.sort((a, b) => { + const timeA = a.timestamp + ? typeof a.timestamp === "number" + ? a.timestamp + : new Date(a.timestamp).getTime() + : 0; + const timeB = b.timestamp + ? typeof b.timestamp === "number" + ? b.timestamp + : new Date(b.timestamp).getTime() + : 0; + return timeA - timeB; + }); + + // Add markers in chronological order + sortedPackets.forEach((packet) => { + self.addMarker({ + ...packet, + historical: true, + popup: packet.popup || self.buildPopupContent(packet), + }); + }); + }); + + currentIndex += chunkSize; + + // Continue processing if there are more packets + if (currentIndex < data.packets.length) { + // Use setTimeout to yield control and prevent blocking + setTimeout(processChunk, 0); + } + }; + + // Start processing + processChunk(); + }); + } + }); + // Handle refresh markers event self.handleEvent("refresh_markers", () => { // Remove markers that are outside current bounds diff --git a/lib/aprsme/packets.ex b/lib/aprsme/packets.ex index 3782d49..71f6cfa 100644 --- a/lib/aprsme/packets.ex +++ b/lib/aprsme/packets.ex @@ -168,6 +168,11 @@ defmodule Aprsme.Packets do defp insert_packet(attrs, packet_data) do case %Packet{} |> Packet.changeset(attrs) |> Repo.insert() do {:ok, packet} -> + # Invalidate cache for this packet's callsign + if Map.has_key?(attrs, :sender) do + Aprsme.CachedQueries.invalidate_callsign_cache(attrs.sender) + end + {:ok, packet} {:error, changeset} -> @@ -413,6 +418,7 @@ defmodule Aprsme.Packets do # Always limit to the last hour for initial load one_hour_ago = DateTime.add(DateTime.utc_now(), -3600, :second) limit = Map.get(opts, :limit, 200) + offset = Map.get(opts, :offset, 0) # Use a more efficient query that leverages the partial indexes # Order by received_at DESC to get the most recent packets first @@ -421,7 +427,8 @@ defmodule Aprsme.Packets do where: p.has_position == true, where: p.received_at >= ^one_hour_ago, order_by: [desc: p.received_at], - limit: ^limit + limit: ^limit, + offset: ^offset ) query = diff --git a/lib/aprsme_web/live/map_live/index.ex b/lib/aprsme_web/live/map_live/index.ex index f4e2cae..196deca 100644 --- a/lib/aprsme_web/live/map_live/index.ex +++ b/lib/aprsme_web/live/map_live/index.ex @@ -283,6 +283,11 @@ defmodule AprsmeWeb.MapLive.Index do def handle_info(:start_geolocation, socket), do: handle_info_start_geolocation(socket) + def handle_info({:load_historical_batch, batch_offset}, socket) do + socket = load_historical_batch(socket, batch_offset) + {:noreply, socket} + end + def handle_info(%Phoenix.Socket.Broadcast{topic: "aprs_messages", event: "packet", payload: packet}, socket), do: handle_info({:postgres_packet, packet}, socket) @@ -970,8 +975,8 @@ defmodule AprsmeWeb.MapLive.Index do # Clear existing historical packets socket = push_event(socket, "clear_historical_packets", %{}) - # Use optimized loading for better performance - socket = load_historical_packets_for_bounds_optimized(socket, socket.assigns.map_bounds) + # Start progressive loading using LiveView's efficient batching + socket = start_progressive_historical_loading(socket) {:noreply, socket} else @@ -1166,11 +1171,21 @@ defmodule AprsmeWeb.MapLive.Index do packets_module = Application.get_env(:aprsme, :packets_module, Aprsme.Packets) historical_packets = - packets_module.get_recent_packets_optimized(%{ - bounds: bounds, - # Reduced limit for faster initial load - limit: 200 - }) + if packets_module == Aprsme.Packets do + # Use cached queries for better performance + Aprsme.CachedQueries.get_recent_packets_cached(%{ + bounds: bounds, + # Reduced limit for faster initial load + limit: 200 + }) + else + # Fallback for testing + packets_module.get_recent_packets_optimized(%{ + bounds: bounds, + # Reduced limit for faster initial load + limit: 200 + }) + end if Enum.empty?(historical_packets) do assign(socket, historical_loaded: true) @@ -1179,6 +1194,85 @@ defmodule AprsmeWeb.MapLive.Index do end end + # Progressive loading functions using LiveView's efficient update mechanisms + @spec start_progressive_historical_loading(Socket.t()) :: Socket.t() + defp start_progressive_historical_loading(socket) do + # Start with a small batch for immediate visual feedback + socket = + socket + |> assign(loading_batch: 0, total_batches: 4) + |> load_historical_batch(0) + + # Schedule next batches using LiveView's async processing + Process.send_after(self(), {:load_historical_batch, 1}, 50) + Process.send_after(self(), {:load_historical_batch, 2}, 150) + Process.send_after(self(), {:load_historical_batch, 3}, 300) + + socket + end + + @spec load_historical_batch(Socket.t(), integer()) :: Socket.t() + defp load_historical_batch(socket, batch_offset) do + if socket.assigns.map_bounds do + bounds = [ + socket.assigns.map_bounds.west, + socket.assigns.map_bounds.south, + socket.assigns.map_bounds.east, + socket.assigns.map_bounds.north + ] + + # Load smaller batches with offset for progressive loading + batch_size = 50 + offset = batch_offset * batch_size + + packets_module = Application.get_env(:aprsme, :packets_module, Aprsme.Packets) + + historical_packets = + if packets_module == Aprsme.Packets do + # Use cached queries for better performance + Aprsme.CachedQueries.get_recent_packets_cached(%{ + bounds: bounds, + limit: batch_size, + offset: offset + }) + else + # Fallback for testing + packets_module.get_recent_packets_optimized(%{ + bounds: bounds, + limit: batch_size, + offset: offset + }) + end + + if Enum.any?(historical_packets) do + # Process this batch and send to frontend + packet_data_list = build_packet_data_list(historical_packets) + + if Enum.any?(packet_data_list) do + # Use LiveView's efficient push_event for incremental updates + socket = + push_event(socket, "add_historical_packets_batch", %{ + packets: packet_data_list, + batch: batch_offset, + is_final: batch_offset >= 3 + }) + + # Update progress for user feedback + socket = assign(socket, loading_batch: batch_offset + 1) + + socket + else + socket + end + else + # No more data in this batch + socket + end + else + socket + end + end + @spec within_bounds?(map() | struct(), map()) :: boolean() defp within_bounds?(packet, bounds) do {lat, lon, _data_extended} = MapHelpers.get_coordinates(packet)