defmodule AprsmeWeb.MapLive.DataBuilder do @moduledoc """ Centralizes all packet data building logic for map display. This module consolidates data building functions that were previously scattered across multiple modules (index.ex, display_manager.ex, packet_utils.ex, and historical_loader.ex). The refactoring eliminates code duplication and provides a single source of truth for: - Building packet data for real-time display - Building minimal packet data for historical display - Creating popup content for both weather and standard packets - Converting packet data structures for frontend consumption All functions handle both real-time and historical packet data with proper coordinate validation, symbol rendering, and weather data processing. """ alias AprsmeWeb.Live.Shared.CoordinateUtils alias AprsmeWeb.Live.Shared.PacketUtils, as: SharedPacketUtils alias AprsmeWeb.Live.Shared.ParamUtils alias AprsmeWeb.MapLive.PopupComponent alias AprsmeWeb.TimeHelpers alias Phoenix.HTML.Safe alias Phoenix.LiveView.Socket @doc """ Build packet data list from a map of packets. Replaces the duplicated function in both index.ex and display_manager.ex. """ @spec build_packet_data_list_from_map(map(), boolean(), Socket.t()) :: list() def build_packet_data_list_from_map(packets_map, is_most_recent, socket) do locale = get_locale(socket) # Preload weather callsigns in one batch query instead of N+1 individual queries callsigns = packets_map |> Map.values() |> Enum.map(&display_name/1) |> Enum.uniq() weather_set = Aprsme.Packets.weather_callsigns(callsigns) Process.put(:weather_callsigns_cache, weather_set) # try/after so the cache is cleared even if packet building raises — # otherwise a leftover :weather_callsigns_cache entry would poison # subsequent calls on the same LiveView process. try do packets_map |> Enum.map(fn {_callsign, packet} -> build_packet_data(packet, is_most_recent, locale) end) |> Enum.filter(& &1) |> deduplicate_by_callsign_group() after Process.delete(:weather_callsigns_cache) end end @doc """ Build packet data for display on the map. Main function moved from packet_utils.ex. """ @spec build_packet_data(map(), boolean(), String.t()) :: map() | nil def build_packet_data(packet, is_most_recent_for_callsign, locale) when is_boolean(is_most_recent_for_callsign) do {lat, lon, data_extended} = CoordinateUtils.get_coordinates(packet) callsign = generate_callsign(packet) # Validate coordinates and callsign before building packet data if valid_coordinates?(lat, lon) && callsign != "" do packet_data = build_packet_map(packet, to_float(lat), to_float(lon), data_extended, locale) Map.put(packet_data, "is_most_recent_for_callsign", is_most_recent_for_callsign) else # Log invalid data for debugging if not valid_coordinates?(lat, lon) do require Logger Logger.debug("Invalid coordinates in packet #{get_packet_id(packet)}: lat=#{inspect(lat)}, lon=#{inspect(lon)}") end nil end end @doc """ Build packet data with default locale. """ @spec build_packet_data(map(), boolean()) :: nil | %{<<_::16, _::_*8>> => any()} def build_packet_data(packet, is_most_recent_for_callsign) when is_boolean(is_most_recent_for_callsign) do build_packet_data(packet, is_most_recent_for_callsign, "en") end @doc """ Build packet data with default parameters. """ @spec build_packet_data(map()) :: nil | %{<<_::16, _::_*8>> => any()} def build_packet_data(packet) when is_map(packet) do build_packet_data(packet, false, "en") end @doc """ Build minimal packet data for historical display. Moved from historical_loader.ex. """ @spec build_minimal_packet_data(map(), boolean(), boolean()) :: map() | nil def build_minimal_packet_data(packet, is_most_recent, has_weather) do # Build minimal packet data without calling expensive build_packet_data {lat, lon, _} = get_coordinates(packet) # Validate coordinates are numeric and within valid ranges if valid_coordinates?(lat, lon) do # Use get_packet_field to get symbol information properly (includes data_extended fallback) symbol_table_id = get_packet_field(packet, :symbol_table_id, "/") symbol_code = get_packet_field(packet, :symbol_code, ">") callsign = display_name(packet) label = map_label(packet) # For historical (non-most-recent) packets, use a simple red dot HTML # instead of the full APRS symbol — matches what the JS createMarkerIcon # would generate anyway, saving the client from icon re-creation symbol_html = if is_most_recent do AprsmeWeb.SymbolRenderer.render_marker_symbol( symbol_table_id, symbol_code, label, 32 ) else historical_dot_html(label) end raw_comment = get_packet_field(packet, :comment, "") clean_comment = Aprsme.EncodingUtils.sanitize_comment(raw_comment) %{ "id" => if(is_most_recent, do: "current_#{get_packet_id(packet)}", else: "hist_#{get_packet_id(packet)}"), "lat" => to_float(lat), "lng" => to_float(lon), "callsign" => label, "callsign_group" => callsign, "symbol_table_id" => symbol_table_id, "symbol_code" => symbol_code, "symbol_html" => symbol_html, "comment" => clean_comment, "timestamp" => get_packet_timestamp_unix(packet), "historical" => !is_most_recent, "is_most_recent_for_callsign" => is_most_recent, "path" => get_packet_field(packet, :path, ""), "popup" => build_simple_popup(packet, has_weather) } else # Log invalid coordinates for debugging require Logger Logger.debug("Invalid coordinates in packet #{get_packet_id(packet)}: lat=#{inspect(lat)}, lon=#{inspect(lon)}") nil end end # Validate coordinates are numeric and within valid ranges @spec valid_coordinates?(number() | nil, number() | nil) :: boolean() defp valid_coordinates?(lat, lon) when is_number(lat) and is_number(lon) do # Range checks ensure finite values; infinity would be outside these ranges lat >= -90 and lat <= 90 and lon >= -180 and lon <= 180 end defp valid_coordinates?(_, _), do: false @doc """ Build packet data list for historical display. Moved from historical_loader.ex. """ @spec build_packet_data_list(list()) :: list() def build_packet_data_list(historical_packets) do # Include weather data in initial grouping to avoid separate query # Group by display name so objects/items are grouped by their name, not sender grouped_packets = Enum.group_by(historical_packets, fn packet -> display_name(packet) end) # Build weather callsign set from packets themselves (no DB query needed) weather_callsigns = build_weather_callsign_set(historical_packets) # For each callsign group, find the most recent packet and mark it appropriately. # Output is grouped by callsign with chronological (oldest first) order within # each group, so the JS can iterate directly for trail drawing without re-sorting. grouped_packets |> Enum.flat_map(fn {callsign, packets} -> # Sort by received_at to find most recent sorted_packets = Enum.sort_by(packets, &get_packet_received_at(&1), {:desc, DateTime}) case sorted_packets do [] -> [] packets_list -> # Find the best packet to display as "current" - prioritize position over weather selected_packet = select_best_packet_for_display(packets_list) historical = Enum.reject(packets_list, &(get_packet_id(&1) == get_packet_id(selected_packet))) # Always include the selected packet has_weather = MapSet.member?(weather_callsigns, String.upcase(callsign)) most_recent_data = build_minimal_packet_data(selected_packet, true, has_weather) # Get coordinates of selected packet for distance filtering {most_recent_lat, most_recent_lon, _} = get_coordinates(selected_packet) # Filter historical packets that are too close to most recent position filtered_historical = filter_historical_by_distance(historical, most_recent_lat, most_recent_lon) # Build data for remaining historical packets historical_data = build_historical_packet_data(filtered_historical, has_weather) # Sort chronologically (oldest first) within the group for proper trail drawing group_packets = Enum.filter([most_recent_data | historical_data], & &1) Enum.sort_by(group_packets, & &1["timestamp"]) end end) |> Enum.filter(& &1) end @doc """ Build simple popup content for historical display. Moved from historical_loader.ex. """ @spec build_simple_popup(map(), boolean()) :: String.t() def build_simple_popup(packet, has_weather) do # Build popup HTML directly without database queries callsign = map_label(packet) timestamp_dt = get_packet_received_at(packet) cache_buster = System.system_time(:millisecond) # Check if this packet itself is a weather packet is_weather = weather_packet?(packet) if is_weather do # Build weather popup %{ callsign: callsign, comment: nil, timestamp_dt: timestamp_dt, cache_buster: cache_buster, weather: true, weather_link: true, temperature: get_weather_field(packet, :temperature), temp_unit: "°F", humidity: get_weather_field(packet, :humidity), wind_direction: get_weather_field(packet, :wind_direction), wind_speed: get_weather_field(packet, :wind_speed), wind_unit: "mph", wind_gust: get_weather_field(packet, :wind_gust), gust_unit: "mph", pressure: get_weather_field(packet, :pressure), rain_1h: get_weather_field(packet, :rain_1h), rain_24h: get_weather_field(packet, :rain_24h), rain_since_midnight: get_weather_field(packet, :rain_since_midnight), rain_1h_unit: "in", rain_24h_unit: "in", rain_since_midnight_unit: "in" } |> PopupComponent.popup() |> Safe.to_iodata() |> IO.iodata_to_binary() else # Build standard popup raw_comment = get_packet_field(packet, :comment, "") clean_comment = Aprsme.EncodingUtils.sanitize_comment(raw_comment) %{ callsign: callsign, comment: clean_comment, timestamp_dt: timestamp_dt, cache_buster: cache_buster, weather: false, # Use pre-fetched weather info weather_link: has_weather } |> PopupComponent.popup() |> Safe.to_iodata() |> IO.iodata_to_binary() end end @doc """ Select the best packet to display for a callsign. Moved from historical_loader.ex. """ @spec select_best_packet_for_display(list()) :: map() def select_best_packet_for_display(packets) do # Separate position and weather packets using the same logic as weather_packet? {position_packets, weather_packets} = Enum.split_with(packets, fn packet -> # A packet is a position packet if it's NOT a weather packet not weather_packet?(packet) end) # Prefer the most recent position packet, fall back to most recent weather packet case position_packets do [] -> # No position packets, use most recent weather packet hd(weather_packets) [single_position] -> # Only one position packet, use it single_position position_list -> # Multiple position packets, use most recent one Enum.max_by(position_list, &get_packet_received_at(&1), DateTime) end end @doc """ Build weather callsign set from packets. Moved from historical_loader.ex. """ @spec build_weather_callsign_set(list()) :: MapSet.t() def build_weather_callsign_set(packets) do packets |> Enum.filter(&weather_packet?/1) |> MapSet.new(fn packet -> String.upcase(display_name(packet)) end) end # Private functions moved from packet_utils.ex @spec build_packet_map(map(), float(), float(), map(), String.t()) :: map() defp build_packet_map(packet, lat, lon, data_extended, locale) do data_extended = data_extended || %{} packet_info = extract_packet_info(packet, data_extended) popup = build_popup_content(packet, packet_info, lat, lon, locale) build_packet_result(packet, packet_info, lat, lon, popup) end @spec extract_packet_info(map(), map()) :: map() defp extract_packet_info(packet, data_extended) do raw_comment = get_packet_field(packet, :comment, "") clean_comment = Aprsme.EncodingUtils.sanitize_comment(raw_comment) %{ callsign: map_label(packet), callsign_group: display_name(packet), symbol_table_id: get_packet_field(packet, :symbol_table_id, "/"), symbol_code: get_packet_field(packet, :symbol_code, ">"), timestamp: get_timestamp(packet), comment: clean_comment, safe_data_extended: convert_tuples_to_strings(data_extended), is_weather_packet: weather_packet?(packet) } end @spec build_popup_content(map(), map(), float(), float(), String.t()) :: String.t() defp build_popup_content(packet, packet_info, lat, lon, locale) do if packet_info.is_weather_packet do build_weather_popup_html(packet, packet_info.callsign, locale) else build_standard_popup_html(packet_info, lat, lon) end end @spec build_standard_popup_html( %{ :callsign => binary(), :callsign_group => binary(), :comment => any(), :is_weather_packet => boolean(), :safe_data_extended => any(), :symbol_code => any(), :symbol_table_id => any(), :timestamp => binary() }, float(), float() ) :: binary() defp build_standard_popup_html(packet_info, _lat, _lon) do timestamp_dt = TimeHelpers.to_datetime(packet_info.timestamp) cache_buster = System.system_time(:millisecond) weather_link = has_weather_packets?(packet_info.callsign) popup_data = %{ callsign: packet_info.callsign, comment: packet_info.comment, timestamp_dt: timestamp_dt, cache_buster: cache_buster, weather: false, weather_link: weather_link } # Optimize string conversion - avoid intermediate iodata step popup_data |> PopupComponent.popup() |> Safe.to_iodata() |> IO.iodata_to_binary() end @spec build_weather_popup_html(map(), String.t(), String.t()) :: String.t() defp build_weather_popup_html(packet, sender, locale) do received_at = get_received_at(packet) timestamp_dt = TimeHelpers.to_datetime(received_at) cache_buster = System.system_time(:millisecond) # Get weather data and convert units based on locale temperature_raw = get_weather_field(packet, :temperature) wind_speed_raw = get_weather_field(packet, :wind_speed) wind_gust_raw = get_weather_field(packet, :wind_gust) rain_1h_raw = get_weather_field(packet, :rain_1h) rain_24h_raw = get_weather_field(packet, :rain_24h) rain_since_midnight_raw = get_weather_field(packet, :rain_since_midnight) # Convert units if the values are numbers {temperature, temp_unit} = convert_temperature(temperature_raw, locale) {wind_speed, wind_unit} = convert_wind_speed(wind_speed_raw, locale) {wind_gust, gust_unit} = convert_wind_speed(wind_gust_raw, locale) {rain_1h, rain_1h_unit} = convert_rain(rain_1h_raw, locale) {rain_24h, rain_24h_unit} = convert_rain(rain_24h_raw, locale) {rain_since_midnight, rain_since_midnight_unit} = convert_rain(rain_since_midnight_raw, locale) %{ callsign: sender, comment: nil, timestamp_dt: timestamp_dt, cache_buster: cache_buster, weather: true, weather_link: true, temperature: temperature, temp_unit: temp_unit, humidity: get_weather_field(packet, :humidity), wind_direction: get_weather_field(packet, :wind_direction), wind_speed: wind_speed, wind_unit: wind_unit, wind_gust: wind_gust, gust_unit: gust_unit, pressure: get_weather_field(packet, :pressure), rain_1h: rain_1h, rain_24h: rain_24h, rain_since_midnight: rain_since_midnight, rain_1h_unit: rain_1h_unit, rain_24h_unit: rain_24h_unit, rain_since_midnight_unit: rain_since_midnight_unit } |> PopupComponent.popup() |> Safe.to_iodata() |> IO.iodata_to_binary() end @spec build_packet_result( map(), %{ :callsign => binary(), :callsign_group => binary(), :comment => any(), :is_weather_packet => boolean(), :safe_data_extended => any(), :symbol_code => nil | binary(), :symbol_table_id => nil | binary(), :timestamp => binary() }, float(), float(), binary() ) :: %{<<_::16, _::_*8>> => any()} defp build_packet_result(packet, packet_info, lat, lon, popup) do # Generate unique ID for live packets to prevent conflicts packet_id = if Map.has_key?(packet, :id) do "live_#{packet.id}_#{System.unique_integer([:positive])}" else "live_#{packet_info.callsign}_#{System.unique_integer([:positive])}" end # Generate symbol HTML using the server-side renderer symbol_html = AprsmeWeb.SymbolRenderer.render_marker_symbol( packet_info.symbol_table_id, packet_info.symbol_code, packet_info.callsign, 32 ) path_value = get_packet_field(packet, :path, "") %{ "id" => packet_id, "callsign" => packet_info.callsign, "callsign_group" => packet_info.callsign_group, "base_callsign" => get_packet_field(packet, :base_callsign, ""), "ssid" => get_packet_field(packet, :ssid, nil), "lat" => to_float(lat), "lng" => to_float(lon), "data_type" => to_string(get_packet_field(packet, :data_type, "unknown")), "path" => path_value, "comment" => packet_info.comment, "data_extended" => packet_info.safe_data_extended || %{}, "symbol_table_id" => packet_info.symbol_table_id, "symbol_code" => packet_info.symbol_code, "symbol_description" => "Symbol: #{packet_info.symbol_table_id}#{packet_info.symbol_code}", "timestamp" => packet_info.timestamp, "popup" => popup, "symbol_html" => symbol_html } end # Utility functions for packet field access @spec get_packet_field( map(), :base_callsign | :comment | :data_type | :path | :ssid | :symbol_code | :symbol_table_id, nil | binary() ) :: any() defp get_packet_field(packet, field, default) do SharedPacketUtils.get_packet_field(packet, field, default) end @spec get_timestamp(map()) :: String.t() defp get_timestamp(packet) do SharedPacketUtils.get_timestamp(packet) end @spec to_float(number() | nil) :: float() defp to_float(value) do ParamUtils.to_float(value) end @spec generate_callsign(map()) :: String.t() defp generate_callsign(packet) do SharedPacketUtils.generate_callsign(packet) end @spec display_name(map()) :: String.t() defp display_name(packet) do SharedPacketUtils.display_name(packet) end @spec map_label(map()) :: String.t() defp map_label(packet) do SharedPacketUtils.map_label(packet) end @spec weather_packet?(map()) :: boolean() defp weather_packet?(packet) do SharedPacketUtils.weather_packet?(packet) end @spec has_weather_packets?(String.t()) :: boolean() defp has_weather_packets?(callsign) when is_binary(callsign) do case Process.get(:weather_callsigns_cache) do nil -> Aprsme.Packets.has_weather_packets?(callsign) weather_set -> MapSet.member?(weather_set, String.upcase(String.trim(callsign))) end rescue _ -> false end @spec convert_tuples_to_strings(any()) :: any() defp convert_tuples_to_strings(map) when is_map(map) do # Avoid processing structs entirely case map do %{__struct__: _} -> map _ -> Map.new(map, &convert_tuple_entry/1) end end defp convert_tuples_to_strings(list) when is_list(list) do # Use Stream for memory efficiency on large lists list |> Stream.map(&convert_tuples_to_strings/1) |> Enum.to_list() end defp convert_tuples_to_strings(tuple) when is_tuple(tuple) do # Optimize tuple string conversion - avoid inspect for common cases case tuple_size(tuple) do 2 -> {a, b} = tuple "#{a}, #{b}" 3 -> {a, b, c} = tuple "#{a}, #{b}, #{c}" _ -> inspect(tuple) end end defp convert_tuples_to_strings(other), do: other defp convert_tuple_entry({k, v}), do: {k, convert_tuples_to_strings(v)} @spec get_weather_field(map(), atom()) :: String.t() defp get_weather_field(packet, key) do case SharedPacketUtils.get_weather_field(packet, key) do nil -> "N/A" value -> value end end @spec get_received_at(map()) :: DateTime.t() | nil defp get_received_at(packet) do cond do Map.has_key?(packet, :received_at) -> packet.received_at Map.has_key?(packet, "received_at") -> packet["received_at"] true -> nil end end # Helper to get locale from socket @spec get_locale(Socket.t()) :: String.t() defp get_locale(socket) do Map.get(socket.assigns, :locale, "en") end # Helper to get coordinates from packet @spec get_coordinates(map()) :: {float() | nil, float() | nil, map()} defp get_coordinates(packet) do CoordinateUtils.get_coordinates(packet) end # Helper to get packet ID @spec get_packet_id(map()) :: any() defp get_packet_id(packet) do Map.get(packet, :id) || Map.get(packet, "id") end # Helper to get packet received_at datetime @spec get_packet_received_at(map()) :: DateTime.t() defp get_packet_received_at(packet) do get_received_at(packet) || DateTime.utc_now() end # Helper to get packet timestamp as unix milliseconds @spec get_packet_timestamp_unix(map()) :: integer() defp get_packet_timestamp_unix(packet) do received_at = get_packet_received_at(packet) DateTime.to_unix(received_at, :millisecond) end # Weather unit conversion helpers defp convert_temperature(value, locale) when is_binary(value) and value != "N/A" do case Float.parse(value) do {num_value, _} -> AprsmeWeb.WeatherUnits.format_temperature(num_value, locale) :error -> {value, "°F"} end end defp convert_temperature(value, _locale), do: {value, "°F"} defp convert_wind_speed(value, locale) when is_binary(value) and value != "N/A" do case Float.parse(value) do {num_value, _} -> AprsmeWeb.WeatherUnits.format_wind_speed(num_value, locale) :error -> {value, "mph"} end end defp convert_wind_speed(value, _locale), do: {value, "mph"} defp convert_rain(value, locale) when is_binary(value) and value != "N/A" do case Float.parse(value) do {num_value, _} -> AprsmeWeb.WeatherUnits.format_rain(num_value, locale) :error -> {value, "in"} end end defp convert_rain(value, _locale), do: {value, "in"} defp filter_historical_by_distance(historical, most_recent_lat, most_recent_lon) do if most_recent_lat && most_recent_lon do Enum.filter(historical, fn packet -> packet_far_enough_from_most_recent?(packet, most_recent_lat, most_recent_lon) end) else # If most recent has no coordinates, include all historical historical end end defp packet_far_enough_from_most_recent?(packet, most_recent_lat, most_recent_lon) do {lat, lon, _} = get_coordinates(packet) if lat && lon do distance_meters = CoordinateUtils.calculate_distance_meters( most_recent_lat, most_recent_lon, lat, lon ) # Only show if 10+ meters away distance_meters >= 10.0 else # Skip packets without coordinates false end end defp historical_dot_html(callsign) do escaped = callsign |> Phoenix.HTML.html_escape() |> Phoenix.HTML.safe_to_string() "
" end defp build_historical_packet_data(filtered_historical, has_weather) do filtered_historical |> Enum.map(fn packet -> build_minimal_packet_data(packet, false, has_weather) end) |> Enum.filter(& &1) end # When multiple senders track the same object, keep only one "current" marker # per callsign_group to avoid duplicate labels on the map. defp deduplicate_by_callsign_group(marker_list) do marker_list |> Enum.reduce(%{}, fn marker, acc -> group = marker["callsign_group"] || marker["callsign"] Map.put(acc, group, marker) end) |> Map.values() end end