defmodule Aprsme.Packet do @moduledoc false use Aprsme.Schema import Ecto.Changeset alias Aprs.Types.MicE alias Aprsme.DataExtended alias AprsmeWeb.Live.Shared.CoordinateUtils schema "packets" do field(:base_callsign, :string) field(:data_type, :string) field(:destination, :string) field(:path, :string) field(:sender, :string) field(:ssid, :string) field(:received_at, :utc_datetime) field(:region, :string) field(:lat, :decimal) field(:lon, :decimal) field(:location, Geo.PostGIS.Geometry) field(:has_position, :boolean, default: false) # Original raw packet and symbol information field(:raw_packet, :string) field(:symbol_code, :string) field(:symbol_table_id, :string) # Additional packet data field(:comment, :string) field(:timestamp, :string) field(:aprs_messaging, :boolean, default: false) # Weather data field(:temperature, :float) field(:humidity, :float) field(:wind_speed, :float) field(:wind_direction, :integer) field(:wind_gust, :float) field(:pressure, :float) field(:rain_1h, :float) field(:rain_24h, :float) field(:rain_since_midnight, :float) field(:snow, :float) # Equipment/status information field(:manufacturer, :string) field(:equipment_type, :string) field(:course, :integer) field(:speed, :float) field(:altitude, :float) # Message-specific fields field(:addressee, :string) field(:message_text, :string) field(:message_number, :string) field(:has_weather, :boolean, default: false) # JSONB column for display-only fields (PHG, telemetry, radiorange, etc.) field(:data, :map, default: %{}) field(:device_identifier, :string) # APRS Items/Objects support field(:item_name, :string) field(:object_name, :string) field(:is_item, :boolean, default: false) field(:is_object, :boolean, default: false) # DAO (Datum As Offset) extension for extra position precision field(:dao, :map) embeds_one(:data_extended, DataExtended) timestamps(type: :utc_datetime) end @type t :: %__MODULE__{} @doc false @spec changeset(Aprsme.Packet.t(), map()) :: Ecto.Changeset.t() def changeset(packet, attrs) do # Convert atom data_type to string attrs = normalize_data_type(attrs) packet |> cast(attrs, [ :base_callsign, :data_type, :destination, :path, :sender, :ssid, :received_at, :region, :lat, :lon, :location, :has_position, :raw_packet, :symbol_code, :symbol_table_id, :comment, :timestamp, :aprs_messaging, :temperature, :humidity, :wind_speed, :wind_direction, :wind_gust, :pressure, :rain_1h, :rain_24h, :rain_since_midnight, :snow, :manufacturer, :equipment_type, :course, :speed, :altitude, :addressee, :message_text, :message_number, :has_weather, :data, :device_identifier, :item_name, :object_name, :is_item, :is_object, :dao ]) |> validate_required([ :base_callsign, :data_type, :destination, :sender, :ssid, :received_at ]) |> maybe_set_location_and_position() end defp maybe_set_location_and_position(changeset) do changeset |> maybe_create_geometry_from_lat_lon() |> maybe_set_has_position() |> maybe_set_has_weather() |> normalize_symbols() |> normalize_course() |> normalize_wind_direction() end # Mirrors PacketConsumer.set_has_weather/1 so direct changeset inserts # (tests, backfills, anything bypassing the GenStage pipeline) still get # has_weather populated — previously a DB trigger did this. defp maybe_set_has_weather(changeset) do has_weather? = Enum.any?(Aprsme.EncodingUtils.weather_fields(), fn field -> not is_nil(get_field(changeset, field)) end) put_change(changeset, :has_weather, has_weather?) end defp normalize_symbols(changeset) do # Only normalize if packet has position if get_field(changeset, :has_position) do symbol_code = get_field(changeset, :symbol_code) symbol_table_id = get_field(changeset, :symbol_table_id) changeset |> normalize_symbol_code(symbol_code) |> normalize_symbol_table_id(symbol_table_id) else changeset end end defp normalize_symbol_code(changeset, code) when is_binary(code) and byte_size(code) == 1 do changeset end defp normalize_symbol_code(changeset, _invalid_code) do # Default to '>' (car) if symbol_code is missing or invalid put_change(changeset, :symbol_code, ">") end defp normalize_symbol_table_id(changeset, table) when is_binary(table) and byte_size(table) == 1 do changeset end defp normalize_symbol_table_id(changeset, _invalid_table) do # Default to '/' (primary symbol table) if table_id is missing or invalid put_change(changeset, :symbol_table_id, "/") end defp normalize_course(changeset) do course = get_field(changeset, :course) || get_change(changeset, :course) case course do nil -> changeset c when is_integer(c) and c >= 0 and c <= 359 -> changeset c when is_integer(c) -> # Invalid course value (negative or >= 360), normalize to 0 put_change(changeset, :course, 0) _ -> changeset end end defp normalize_wind_direction(changeset) do wind_direction = get_field(changeset, :wind_direction) || get_change(changeset, :wind_direction) case wind_direction do nil -> changeset dir when is_integer(dir) and dir >= 0 and dir <= 359 -> changeset 360 -> # 360 degrees = 0 degrees (full circle) put_change(changeset, :wind_direction, 0) dir when is_integer(dir) -> # Invalid wind direction (negative or > 360), normalize to 0 put_change(changeset, :wind_direction, 0) _ -> changeset end end @spec maybe_create_geometry_from_lat_lon(Ecto.Changeset.t()) :: Ecto.Changeset.t() defp maybe_create_geometry_from_lat_lon(changeset) do lat = get_field(changeset, :lat) || get_change(changeset, :lat) lon = get_field(changeset, :lon) || get_change(changeset, :lon) # Also check data_extended for coordinates {lat, lon} = extract_coordinates_from_changeset(changeset, {lat, lon}) create_geometry_from_coordinates(changeset, lat, lon) end defp extract_coordinates_from_changeset(changeset, {nil, nil}) do data_extended = get_change(changeset, :data_extended) extract_coordinates_from_data_extended(data_extended) end defp extract_coordinates_from_changeset(_changeset, coords), do: coords defp extract_coordinates_from_data_extended(nil), do: {nil, nil} defp extract_coordinates_from_data_extended(data_extended) when is_map(data_extended) and not is_struct(data_extended) do {data_extended[:latitude], data_extended[:longitude]} end defp extract_coordinates_from_data_extended(_), do: {nil, nil} defp create_geometry_from_coordinates(changeset, lat, lon) do # Normalize coordinates to valid ranges normalized_lat = CoordinateUtils.normalize_latitude(lat) normalized_lon = CoordinateUtils.normalize_longitude(lon) if valid_coordinates?(normalized_lat, normalized_lon) do create_and_set_location(changeset, normalized_lat, normalized_lon) else changeset end end defp create_and_set_location(changeset, lat, lon) do case create_point(lat, lon) do nil -> changeset location -> put_change(changeset, :location, location) end rescue error -> require Logger Logger.error("Failed to create geometry for lat=#{lat}, lon=#{lon}: #{inspect(error)}") changeset end defp maybe_set_has_position(changeset) do location = get_field(changeset, :location) || get_change(changeset, :location) case location do nil -> check_legacy_coordinates(changeset) _location -> put_change(changeset, :has_position, true) end end defp check_legacy_coordinates(changeset) do lat = get_field(changeset, :lat) || get_change(changeset, :lat) lon = get_field(changeset, :lon) || get_change(changeset, :lon) if valid_coordinates?(lat, lon) do put_change(changeset, :has_position, true) else changeset end end defp valid_coordinates?(lat, lon) do CoordinateUtils.valid_coordinates_any_type?(lat, lon) end defp normalize_coordinate(coord) do CoordinateUtils.normalize_coordinate(coord) end defp normalize_data_type(attrs), do: Aprsme.EncodingUtils.normalize_data_type(attrs) @doc """ Extracts additional data from the raw packet and data_extended structure and merges it with the packet attributes for storage. """ @spec extract_additional_data(map(), String.t() | nil) :: map() def extract_additional_data(attrs, raw_packet \\ nil) do data_extended = attrs[:data_extended] || attrs["data_extended"] || %{} # Start with the base attributes and add the raw packet base_attrs = Map.put(attrs, :raw_packet, raw_packet) # Remove raw_weather_data from base attributes base_attrs = Map.delete(base_attrs, :raw_weather_data) base_attrs = Map.delete(base_attrs, "raw_weather_data") # Check if symbol data exists at the top level of attrs (from APRS parser) # and preserve it if not already in base_attrs base_attrs = add_symbol_data(base_attrs, attrs) # Extract data based on the type of data_extended additional_data = case data_extended do %MicE{} = mic_e -> extract_from_mic_e(mic_e) %{__original_struct__: MicE} = mic_e_map -> extract_from_mic_e_map(mic_e_map) # Handle ParseError structs gracefully %{__struct__: Aprs.Types.ParseError} -> %{} %{} when is_map(data_extended) -> extract_from_map(data_extended) _ -> %{} end # Merge all extracted data, then sweep display-only fields into data map # and strip dead parser compat fields base_attrs |> merge_extracted_data(additional_data, attrs) |> collect_into_data_map() |> strip_dead_fields() end defp add_symbol_data(base_attrs, attrs) do base_attrs |> maybe_put(:symbol_code, attrs[:symbol_code] || attrs["symbol_code"]) |> maybe_put(:symbol_table_id, attrs[:symbol_table_id] || attrs["symbol_table_id"]) end defp merge_extracted_data(base_attrs, additional_data, attrs) do # Process telemetry fields from top-level attrs if not already processed telemetry_data = if Map.has_key?(additional_data, :telemetry_vals) do # Already processed from data_extended %{} else # Process from top-level attrs put_telemetry_fields(%{}, attrs) end # Process radio range from top-level attrs radio_range_data = put_radio_range_field(%{}, attrs) base_attrs |> Map.merge(additional_data) |> Map.merge(telemetry_data) |> Map.merge(radio_range_data) end # Fields that get swept into the JSONB `data` column @data_fields ~w[telemetry_seq telemetry_vals telemetry_bits phg_power phg_height phg_gain phg_directivity radiorange information_field format posresolution position_ambiguity luminosity rain_midnight]a defp collect_into_data_map(attrs) do {data_values, remaining} = Enum.reduce(@data_fields, {%{}, attrs}, fn field, {data, rest} -> case Map.pop(rest, field) do {nil, rest} -> {data, rest} {value, rest} -> {Map.put(data, to_string(field), value), rest} end end) existing_data = Map.get(remaining, :data, %{}) || %{} Map.put(remaining, :data, Map.merge(existing_data, data_values)) end # Dead parser compat fields — never read, safe to strip @dead_fields ~w[srccallsign dstcallsign origpacket body header alive posambiguity symboltable symbolcode messaging]a defp strip_dead_fields(attrs) do Map.drop(attrs, @dead_fields) end # Extract data from standard map-based data_extended # Convert struct to map if possible, otherwise return empty map defp extract_from_map(data_extended) when is_struct(data_extended) do data_extended |> Map.from_struct() |> extract_from_map() rescue _ -> %{} end defp extract_from_map(data_extended) when is_map(data_extended) do # Remove raw_weather_data before processing data_extended = Map.delete(data_extended, :raw_weather_data) data_extended = Map.delete(data_extended, "raw_weather_data") # Handle nested data_extended structures nested_data_extended = data_extended[:data_extended] || data_extended["data_extended"] # Merge data from both levels combined_data = if nested_data_extended do Map.merge(data_extended, nested_data_extended) else data_extended end result = %{} |> put_symbol_fields(combined_data) |> extract_weather_data(combined_data) |> put_weather_fields(combined_data) |> put_equipment_fields(combined_data) |> put_message_fields(combined_data) |> put_telemetry_fields(combined_data) |> put_radio_range_field(combined_data) # Don't override data_type - trust the APRS parser's determination # The parser already correctly identifies weather packets by: # 1. Data type indicator (e.g., "_" for weather) # 2. Symbol (e.g., "/_" for weather station) # 3. Actual weather data patterns in the content result end defp put_symbol_fields(map, data_extended) do # First try to get symbol data from the data_extended map symbol_code = data_extended[:symbol_code] || data_extended["symbol_code"] symbol_table_id = data_extended[:symbol_table_id] || data_extended["symbol_table_id"] # Clean comment by removing altitude and PHG data comment = clean_comment(data_extended[:comment] || data_extended["comment"]) map |> maybe_put(:symbol_code, symbol_code) |> maybe_put(:symbol_table_id, symbol_table_id) |> maybe_put(:comment, comment) |> maybe_put(:timestamp, data_extended[:timestamp] || data_extended["timestamp"]) |> maybe_put( :aprs_messaging, data_extended[:aprs_messaging?] || data_extended["aprs_messaging?"] ) |> maybe_put(:position_ambiguity, data_extended[:position_ambiguity] || data_extended["position_ambiguity"]) |> maybe_put(:posresolution, data_extended[:posresolution] || data_extended["posresolution"]) |> maybe_put(:format, normalize_format_field(data_extended[:format] || data_extended["format"])) end defp put_weather_fields(map, data_extended) do Enum.reduce(Aprsme.EncodingUtils.weather_fields(), map, fn field, acc -> value = data_extended[field] || data_extended[to_string(field)] maybe_put(acc, field, value) end) end defp put_equipment_fields(map, data_extended) do # Extract altitude from comment if not already in data_extended altitude = get_field_value(data_extended, :altitude) || extract_altitude_from_comment(get_field_value(data_extended, :comment)) # Extract PHG from comment if not already in data_extended phg = get_field_value(data_extended, :phg) || extract_phg_from_comment(get_field_value(data_extended, :comment)) # Update data_extended with extracted values data_extended = data_extended |> Map.put(:altitude, altitude) |> Map.put(:phg, phg) map |> maybe_put(:manufacturer, get_field_value(data_extended, :manufacturer)) |> maybe_put(:equipment_type, get_field_value(data_extended, :equipment_type)) |> maybe_put(:course, get_field_value(data_extended, :course)) |> maybe_put(:speed, get_field_value(data_extended, :speed)) |> maybe_put(:altitude, altitude) # Don't add :phg to the map - it will be split into individual fields by put_phg_fields |> put_phg_fields(data_extended) end defp put_phg_fields(map, data_extended) do phg = get_field_value(data_extended, :phg) cond do phg && is_map(phg) -> map |> maybe_put(:phg_power, get_field_value(phg, :power)) |> maybe_put(:phg_height, get_field_value(phg, :height)) |> maybe_put(:phg_gain, get_field_value(phg, :gain)) |> maybe_put(:phg_directivity, get_field_value(phg, :directivity)) phg && is_binary(phg) && String.length(phg) == 4 -> # Handle new string format from improved parser (e.g., "1060") parse_phg_string(map, phg) true -> map end end # Parse PHG string format (e.g., "1060" -> power=1, height=0, gain=6, dir=0) defp parse_phg_string( map, <> ) do map |> maybe_put(:phg_power, calculate_phg_power(power)) |> maybe_put(:phg_height, calculate_phg_height(height)) |> maybe_put(:phg_gain, String.to_integer(gain)) |> maybe_put(:phg_directivity, calculate_phg_directivity(dir)) rescue _ -> map end defp parse_phg_string(map, _), do: map defp put_message_fields(map, data_extended) do map |> maybe_put(:addressee, data_extended[:addressee] || data_extended["addressee"]) |> maybe_put(:message_text, data_extended[:message_text] || data_extended["message_text"]) |> maybe_put( :message_number, data_extended[:message_number] || data_extended["message_number"] ) end defp put_telemetry_fields(map, data_extended) do telemetry = data_extended[:telemetry] || data_extended["telemetry"] || data_extended map |> maybe_put(:telemetry_seq, parse_telemetry_seq(telemetry)) |> maybe_put(:telemetry_vals, parse_telemetry_vals(telemetry)) |> maybe_put(:telemetry_bits, telemetry[:bits] || telemetry["bits"]) end defp parse_telemetry_seq(telemetry) do case telemetry[:seq] || telemetry["seq"] do nil -> nil seq when is_integer(seq) -> seq seq when is_binary(seq) -> parse_integer_string(seq) _ -> nil end end defp parse_telemetry_vals(telemetry) do case telemetry[:vals] || telemetry["vals"] do nil -> nil vals when is_list(vals) -> Enum.map(vals, &normalize_telemetry_value/1) _ -> nil end end defp parse_integer_string(str) do case Integer.parse(str) do {num, _} -> num _ -> nil end end defp normalize_telemetry_value(val) when is_integer(val), do: val defp normalize_telemetry_value(val) when is_float(val), do: round(val) defp normalize_telemetry_value(val) when is_binary(val) do case Float.parse(val) do {num, _} -> round(num) _ -> 0 end end defp normalize_telemetry_value(_), do: 0 # Extract data from MicE packets defp extract_from_mic_e(mic_e) do %{} |> maybe_put(:lat, mic_e[:latitude]) |> maybe_put(:lon, mic_e[:longitude]) |> maybe_put(:comment, mic_e[:message]) |> maybe_put(:manufacturer, mic_e[:manufacturer]) |> maybe_put(:course, mic_e[:heading]) |> maybe_put(:speed, mic_e[:speed]) |> maybe_put(:symbol_code, mic_e[:symbol_code]) |> maybe_put(:symbol_table_id, mic_e[:symbol_table_id]) end # Extract data from converted MicE map (from struct_to_map conversion) defp extract_from_mic_e_map(mic_e_map) do %{} |> maybe_put(:lat, mic_e_map[:latitude]) |> maybe_put(:lon, mic_e_map[:longitude]) |> maybe_put(:comment, mic_e_map[:message]) |> maybe_put(:manufacturer, mic_e_map[:manufacturer]) |> maybe_put(:course, mic_e_map[:heading]) |> maybe_put(:speed, mic_e_map[:speed]) # Use symbol data from MicE if available, otherwise use default car symbol |> maybe_put(:symbol_code, mic_e_map[:symbol_code] || mic_e_map["symbol_code"] || ">") |> maybe_put(:symbol_table_id, mic_e_map[:symbol_table_id] || mic_e_map["symbol_table_id"] || "/") end # Extract weather data from various formats, including new wx field defp extract_weather_data(attrs, data_extended) do weather_data = find_weather_data(data_extended) # Check for dedicated wx field from improved parser wx_data = data_extended[:wx] || data_extended["wx"] final_weather_data = wx_data || weather_data process_weather_data(attrs, final_weather_data) end defp find_weather_data(data_extended) do data_extended[:weather] || data_extended["weather"] || data_extended[:weather_report] || data_extended["weather_report"] || data_extended[:raw_weather_data] || data_extended["raw_weather_data"] end defp process_weather_data(attrs, weather_data) do case weather_data do weather when is_binary(weather) -> process_binary_weather_data(attrs, weather) weather when is_map(weather) -> process_map_weather_data(attrs, weather) _ -> attrs end end defp process_binary_weather_data(attrs, _weather) do # If you have a weather parsing function, call it here attrs end defp process_map_weather_data(attrs, weather) do weather = Map.drop(weather, [:raw_weather_data, "raw_weather_data"]) # Only merge weather data, don't override data_type Map.merge(attrs, weather) end # Helper to put a value only if it's not nil defp maybe_put(map, _key, nil), do: map defp maybe_put(map, _key, ""), do: map defp maybe_put(map, key, value), do: Map.put(map, key, value) @doc """ Create a geometry point from lat/lon coordinates. """ @spec create_point(number() | nil, number() | nil) :: Geo.Point.t() | nil def create_point(lat, lon) when (is_number(lat) or is_struct(lat, Decimal)) and (is_number(lon) or is_struct(lon, Decimal)) do lat = normalize_coordinate(lat) lon = normalize_coordinate(lon) if valid_coordinates?(lat, lon) do %Geo.Point{coordinates: {lon, lat}, srid: 4326} end end def create_point(_, _), do: nil @doc """ Extract lat/lon from a PostGIS geometry point. """ @spec extract_coordinates(Geo.Point.t() | any()) :: {number() | nil, number() | nil} def extract_coordinates(%Geo.Point{coordinates: {lon, lat}}), do: {lat, lon} def extract_coordinates(_), do: {nil, nil} @doc """ Get latitude from a packet's location geometry. """ @spec lat(Aprsme.Packet.t()) :: number() | nil def lat(%__MODULE__{location: %Geo.Point{coordinates: {_lon, lat}}}), do: lat def lat(_), do: nil @doc """ Get longitude from a packet's location geometry. """ @spec lon(Aprsme.Packet.t()) :: number() | nil def lon(%__MODULE__{location: %Geo.Point{coordinates: {lon, _lat}}}), do: lon def lon(_), do: nil # Clean comment by removing altitude and PHG data defp clean_comment(nil), do: nil defp clean_comment(comment) when is_binary(comment) do comment |> strip_weather_data() # Remove altitude |> String.replace(~r/\s*\/A=-?\d{5,6}/, "") # Remove PHG |> String.replace(~r/PHG\d{4}\s*/, "") # Remove RNG |> String.replace(~r/RNG\d{4}\s*/, "") |> String.trim() |> case do "" -> nil cleaned -> cleaned end end defp clean_comment(comment), do: comment defp strip_weather_data(comment) do # Matches position-included format (XXX/XXX) and positionless format (cXXXsXXX) case Regex.run(~r/^_?(?:[\d .]{3}\/[\d .]{3}|c[\d .]{3}s[\d .]{3})/, comment) do [match] -> rest = String.slice(comment, String.length(match)..-1//1) # Only strip if at least one weather field follows the preamble if match_wx_field(rest) do strip_wx_fields(rest) else comment end nil -> comment end end defp strip_wx_fields(str) do case match_wx_field(str) do nil -> str match -> str |> String.slice(String.length(match)..-1//1) |> strip_wx_fields() end end # APRS weather fields: each is a letter prefix + fixed number of digits/spaces defp match_wx_field(str) do Enum.find_value( [ ~r/^g[\d .]{3}/, ~r/^t[\d .-]{3}/, ~r/^r[\d .]{3}/, ~r/^p[\d .]{3}/, ~r/^P[\d .]{3}/, ~r/^h[\d .]{2}/, ~r/^b[\d .]{5}/, ~r/^s[\d .]{3}/, ~r/^[Ll][\d .]{3}/ ], fn pat -> case Regex.run(pat, str) do [match] -> match nil -> nil end end ) end # Extract altitude from APRS comment field (e.g., "/A=000680" means 680 feet) defp extract_altitude_from_comment(nil), do: nil defp extract_altitude_from_comment(comment) when is_binary(comment) do case Regex.run(~r/\/A=(\d{6})/, comment) do [_, altitude_str] -> # Convert string to integer (altitude in feet) case Integer.parse(altitude_str) do # Convert to float {altitude, _} -> altitude * 1.0 _ -> nil end _ -> nil end end defp extract_altitude_from_comment(_), do: nil # Extract PHG (Power-Height-Gain) from APRS comment field (e.g., "PHG5530") defp extract_phg_from_comment(nil), do: nil defp extract_phg_from_comment(comment) when is_binary(comment) do case Regex.run(~r/PHG(\d)(\d)(\d)(\d)/, comment) do [_, power, height, gain, dir] -> %{ power: calculate_phg_power(power), height: calculate_phg_height(height), gain: String.to_integer(gain), directivity: calculate_phg_directivity(dir) } _ -> nil end end defp extract_phg_from_comment(_), do: nil # PHG power calculation: power = n^2 watts defp calculate_phg_power(n) when is_binary(n) do case Integer.parse(n) do {num, _} -> num * num _ -> 0 end end # PHG height calculation: height = 10 * 2^n feet defp calculate_phg_height(n) when is_binary(n) do case Integer.parse(n) do {num, _} -> 10 * trunc(:math.pow(2, num)) _ -> 10 end end # PHG directivity: 0-8 = directional (n * 45 degrees), 9 = omni (360 degrees) defp calculate_phg_directivity(n) when is_binary(n) do case Integer.parse(n) do {9, _} -> 360 {num, _} when num >= 0 and num <= 8 -> num * 45 _ -> 0 end end # Normalize format field from atom to string defp normalize_format_field(nil), do: nil defp normalize_format_field(format) when is_atom(format), do: to_string(format) defp normalize_format_field(format) when is_binary(format), do: format defp normalize_format_field(_), do: nil # Extract radio range field defp put_radio_range_field(map, data) do maybe_put(map, :radiorange, get_field_value(data, :radiorange)) end # Helper to get field value from either atom or string key defp get_field_value(data, field) when is_map(data) do data[field] || data[to_string(field)] end defp get_field_value(_, _), do: nil end