diff --git a/lib/aprsme/encoding_utils.ex b/lib/aprsme/encoding_utils.ex index a49c768..36d1018 100644 --- a/lib/aprsme/encoding_utils.ex +++ b/lib/aprsme/encoding_utils.ex @@ -201,8 +201,7 @@ defmodule Aprsme.EncodingUtils do :rain_1h, :rain_24h, :rain_since_midnight, - :snow, - :luminosity + :snow ] end @@ -211,71 +210,83 @@ defmodule Aprsme.EncodingUtils do ## Examples - iex> result = Aprsme.EncodingUtils.sanitize_packet(%{"information_field" => <<0, 65, 66, 67>>, "data_extended" => %{"comment" => <<0, 68, 69, 70>>}}) - iex> result["information_field"] == "ABC" + iex> result = Aprsme.EncodingUtils.sanitize_packet(%{"comment" => <<0, 65, 66, 67>>, "data_extended" => %{"comment" => <<0, 68, 69, 70>>}}) + iex> result["comment"] == "ABC" true iex> result["data_extended"]["comment"] == "DEF" true """ @spec sanitize_packet(struct() | map()) :: struct() | map() def sanitize_packet(%Aprsme.Packet{} = packet) do + sanitized_data = + case packet.data do + nil -> nil + data when is_map(data) -> sanitize_packet_strings(data) + other -> other + end + %{ packet - | information_field: sanitize_string(packet.information_field), + | data: sanitized_data, data_extended: sanitize_data_extended(packet.data_extended) } end def sanitize_packet(packet) when is_map(packet) do - # Handle all known string fields, checking for both atom and string keys - string_fields = [ - :information_field, - :comment, - :path, - :raw_packet, - :destination, - :sender, - :base_callsign, - :ssid, - :manufacturer, - :equipment_type, - :message_text, - :addressee, - :symbol_code, - :symbol_table_id, - :dao, - :timestamp, - :device_identifier - ] + packet + |> sanitize_string_fields() + |> sanitize_nested_map(:data, &sanitize_packet_strings/1) + |> sanitize_nested_map(:data_extended, &sanitize_data_extended/1) + end - # Sanitize all string fields - sanitized = - Enum.reduce(string_fields, packet, fn field, acc -> - atom_key = field - string_key = to_string(field) + @string_fields [ + :comment, + :path, + :raw_packet, + :destination, + :sender, + :base_callsign, + :ssid, + :manufacturer, + :equipment_type, + :message_text, + :addressee, + :symbol_code, + :symbol_table_id, + :dao, + :timestamp, + :device_identifier + ] - cond do - Map.has_key?(acc, atom_key) -> - Map.update(acc, atom_key, nil, &sanitize_string/1) + defp sanitize_string_fields(packet) do + Enum.reduce(@string_fields, packet, fn field, acc -> + string_key = to_string(field) - Map.has_key?(acc, string_key) -> - Map.update(acc, string_key, nil, &sanitize_string/1) + cond do + Map.has_key?(acc, field) -> + Map.update(acc, field, nil, &sanitize_string/1) - true -> - acc - end - end) + Map.has_key?(acc, string_key) -> + Map.update(acc, string_key, nil, &sanitize_string/1) + + true -> + acc + end + end) + end + + defp sanitize_nested_map(packet, key, sanitizer) do + string_key = to_string(key) - # Handle data_extended separately cond do - Map.has_key?(sanitized, :data_extended) -> - Map.update(sanitized, :data_extended, nil, &sanitize_data_extended/1) + Map.has_key?(packet, key) -> + Map.update(packet, key, nil, &if(is_nil(&1), do: nil, else: sanitizer.(&1))) - Map.has_key?(sanitized, "data_extended") -> - Map.update(sanitized, "data_extended", nil, &sanitize_data_extended/1) + Map.has_key?(packet, string_key) -> + Map.update(packet, string_key, nil, &if(is_nil(&1), do: nil, else: sanitizer.(&1))) true -> - sanitized + packet end end diff --git a/lib/aprsme/packet.ex b/lib/aprsme/packet.ex index b07388d..00fa8da 100644 --- a/lib/aprsme/packet.ex +++ b/lib/aprsme/packet.ex @@ -11,7 +11,6 @@ defmodule Aprsme.Packet do field(:base_callsign, :string) field(:data_type, :string) field(:destination, :string) - field(:information_field, :string) field(:path, :string) field(:sender, :string) field(:ssid, :string) @@ -45,55 +44,22 @@ defmodule Aprsme.Packet do field(:snow, :float) # Equipment/status information - field(:luminosity, :integer) field(:manufacturer, :string) field(:equipment_type, :string) field(:course, :integer) field(:speed, :float) field(:altitude, :float) - # Position ambiguity level (0-4) - field(:position_ambiguity, :integer) - - # Position resolution and format - field(:posresolution, :float) - field(:format, :string) - - # PHG (Power-Height-Gain) fields - field(:phg_power, :integer) - field(:phg_height, :integer) - field(:phg_gain, :integer) - field(:phg_directivity, :integer) - # Message-specific fields field(:addressee, :string) field(:message_text, :string) field(:message_number, :string) - # Telemetry fields - field(:telemetry_seq, :integer) - field(:telemetry_vals, {:array, :integer}) - field(:telemetry_bits, :string) - - # Radio range field - field(:radiorange, :string) - - # Standard parser compatibility fields - field(:srccallsign, :string) - field(:dstcallsign, :string) - field(:body, :string) - field(:origpacket, :string) - field(:header, :string) - field(:alive, :integer, default: 1) - field(:posambiguity, :integer) - field(:symboltable, :string) - field(:symbolcode, :string) - field(:messaging, :integer) - - # Additional weather fields - field(:rain_midnight, :float) field(:has_weather, :boolean, default: false) + # JSONB column for display-only fields (PHG, telemetry, radiorange, etc.) + field(:data, :map, default: %{}) + field(:device_identifier, :string) # APRS Items/Objects support @@ -123,7 +89,6 @@ defmodule Aprsme.Packet do :base_callsign, :data_type, :destination, - :information_field, :path, :sender, :ssid, @@ -149,38 +114,16 @@ defmodule Aprsme.Packet do :rain_24h, :rain_since_midnight, :snow, - :luminosity, :manufacturer, :equipment_type, :course, :speed, :altitude, - :position_ambiguity, - :posresolution, - :format, - :phg_power, - :phg_height, - :phg_gain, - :phg_directivity, :addressee, :message_text, :message_number, - :telemetry_seq, - :telemetry_vals, - :telemetry_bits, - :radiorange, - :srccallsign, - :dstcallsign, - :body, - :origpacket, - :header, - :alive, - :posambiguity, - :symboltable, - :symbolcode, - :messaging, - :rain_midnight, :has_weather, + :data, :device_identifier, :item_name, :object_name, @@ -345,8 +288,12 @@ defmodule Aprsme.Packet do %{} end - # Merge all extracted data - merge_extracted_data(base_attrs, additional_data, attrs) + # Merge all extracted data, then sweep display-only fields into data map + # and strip dead parser compat fields + base_attrs + |> merge_extracted_data(additional_data, attrs) + |> collect_into_data_map() + |> strip_dead_fields() end defp add_symbol_data(base_attrs, attrs) do @@ -366,16 +313,39 @@ defmodule Aprsme.Packet do put_telemetry_fields(%{}, attrs) end - # Process standard parser fields and radio range from top-level attrs - parser_data = - %{} - |> put_standard_parser_fields(attrs) - |> put_radio_range_field(attrs) + # Process radio range from top-level attrs + radio_range_data = put_radio_range_field(%{}, attrs) base_attrs |> Map.merge(additional_data) |> Map.merge(telemetry_data) - |> Map.merge(parser_data) + |> Map.merge(radio_range_data) + end + + # Fields that get swept into the JSONB `data` column + @data_fields ~w[telemetry_seq telemetry_vals telemetry_bits phg_power phg_height + phg_gain phg_directivity radiorange information_field format posresolution + position_ambiguity luminosity rain_midnight]a + + defp collect_into_data_map(attrs) do + {data_values, remaining} = + Enum.reduce(@data_fields, {%{}, attrs}, fn field, {data, rest} -> + case Map.pop(rest, field) do + {nil, rest} -> {data, rest} + {value, rest} -> {Map.put(data, to_string(field), value), rest} + end + end) + + existing_data = Map.get(remaining, :data, %{}) || %{} + Map.put(remaining, :data, Map.merge(existing_data, data_values)) + end + + # Dead parser compat fields — never read, safe to strip + @dead_fields ~w[srccallsign dstcallsign origpacket body header alive + posambiguity symboltable symbolcode messaging]a + + defp strip_dead_fields(attrs) do + Map.drop(attrs, @dead_fields) end # Extract data from standard map-based data_extended @@ -407,7 +377,6 @@ defmodule Aprsme.Packet do result = %{} |> put_symbol_fields(combined_data) - |> put_standard_parser_fields(combined_data) |> extract_weather_data(combined_data) |> put_weather_fields(combined_data) |> put_equipment_fields(combined_data) @@ -861,21 +830,6 @@ defmodule Aprsme.Packet do defp normalize_format_field(format) when is_binary(format), do: format defp normalize_format_field(_), do: nil - # Extract standard parser compatibility fields - defp put_standard_parser_fields(map, data) do - map - |> maybe_put(:srccallsign, get_field_value(data, :srccallsign)) - |> maybe_put(:dstcallsign, get_field_value(data, :dstcallsign)) - |> maybe_put(:body, get_field_value(data, :body)) - |> maybe_put(:origpacket, get_field_value(data, :origpacket)) - |> maybe_put(:header, get_field_value(data, :header)) - |> maybe_put(:alive, get_field_value(data, :alive)) - |> maybe_put(:posambiguity, get_field_value(data, :posambiguity)) - |> maybe_put(:symboltable, get_field_value(data, :symboltable)) - |> maybe_put(:symbolcode, get_field_value(data, :symbolcode)) - |> maybe_put(:messaging, get_field_value(data, :messaging)) - end - # Extract radio range field defp put_radio_range_field(map, data) do maybe_put(map, :radiorange, get_field_value(data, :radiorange)) diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex index 1b21099..03862c3 100644 --- a/lib/aprsme/packet_consumer.ex +++ b/lib/aprsme/packet_consumer.ex @@ -486,9 +486,10 @@ defmodule Aprsme.PacketConsumer do end defp object_packet?(attrs) do + info_field = get_in(attrs, [:data, "information_field"]) + attrs[:data_type] == "object" or attrs["data_type"] == "object" or - (is_binary(attrs[:information_field]) and - String.starts_with?(attrs[:information_field], ";")) + (is_binary(info_field) and String.starts_with?(info_field, ";")) end defp item_packet?(attrs) do @@ -496,9 +497,11 @@ defmodule Aprsme.PacketConsumer do end defp apply_object_fields(attrs) do + info_field = get_in(attrs, [:data, "information_field"]) + object_name = extract_object_name(attrs) || - extract_object_name_from_info_field(attrs[:information_field]) + extract_object_name_from_info_field(info_field) attrs |> Map.put(:object_name, object_name) @@ -587,7 +590,6 @@ defmodule Aprsme.PacketConsumer do |> Map.put_new(:base_callsign, attrs[:sender]) |> Map.put_new(:data_type, "unknown") |> Map.put_new(:destination, "") - |> Map.put_new(:information_field, "") |> Map.put_new(:path, "") |> Map.put_new(:ssid, "") |> Map.put_new(:data_extended, %{}) @@ -741,9 +743,7 @@ defmodule Aprsme.PacketConsumer do :rain_since_midnight, :snow, :speed, - :altitude, - :posresolution, - :rain_midnight + :altitude ] Enum.reduce(float_fields, attrs, fn field, acc -> diff --git a/lib/aprsme/packet_field_whitelist.ex b/lib/aprsme/packet_field_whitelist.ex index 79b2cf8..8b9210e 100644 --- a/lib/aprsme/packet_field_whitelist.ex +++ b/lib/aprsme/packet_field_whitelist.ex @@ -9,51 +9,34 @@ defmodule Aprsme.PacketFieldWhitelist do # Using string list to handle both atom and string keys consistently @allowed_fields ~w[ addressee - alive altitude aprs_messaging base_callsign - body comment course dao + data data_type destination device_identifier - dstcallsign equipment_type - format has_position has_weather - header humidity - information_field is_item is_object item_name lat location lon - luminosity manufacturer message_number message_text - messaging object_name - origpacket path - phg_directivity - phg_gain - phg_height - phg_power - posambiguity - position_ambiguity - posresolution pressure - radiorange rain_1h rain_24h - rain_midnight rain_since_midnight raw_packet received_at @@ -61,15 +44,9 @@ defmodule Aprsme.PacketFieldWhitelist do sender snow speed - srccallsign ssid symbol_code symbol_table_id - symbolcode - symboltable - telemetry_bits - telemetry_seq - telemetry_vals temperature timestamp wind_direction diff --git a/lib/aprsme/packet_sanitizer.ex b/lib/aprsme/packet_sanitizer.ex index 085fdfc..5ddfa23 100644 --- a/lib/aprsme/packet_sanitizer.ex +++ b/lib/aprsme/packet_sanitizer.ex @@ -19,31 +19,27 @@ defmodule Aprsme.PacketSanitizer do timestamp: 50, message_number: 20, addressee: 50, - symboltable: 5, - symbolcode: 5, - srccallsign: 20, - dstcallsign: 20, # These fields can be longer but still have sanity limits path: 500, manufacturer: 100, equipment_type: 100, - format: 100, device_identifier: 255, item_name: 100, object_name: 100, - # Very long fields - these will be TEXT in DB but we still - # want to prevent abuse with extremely long data - information_field: 5000, + # Very long fields raw_packet: 5000, comment: 2000, - message_text: 2000, - body: 5000, - origpacket: 5000, - header: 1000, - radiorange: 1000, - telemetry_bits: 1000 + message_text: 2000 + } + + # Max lengths for string values inside the JSONB `data` map + @data_string_max_lengths %{ + "information_field" => 5000, + "radiorange" => 1000, + "telemetry_bits" => 1000, + "format" => 100 } @doc """ @@ -57,6 +53,10 @@ defmodule Aprsme.PacketSanitizer do end) end + defp sanitize_field(:data, value) when is_map(value) do + sanitize_data_map(value) + end + defp sanitize_field(key, value) when is_binary(value) do case Map.get(@max_lengths, key) do nil -> @@ -70,6 +70,24 @@ defmodule Aprsme.PacketSanitizer do defp sanitize_field(_key, value), do: value + defp sanitize_data_map(data) when is_map(data) do + Enum.reduce(data, %{}, fn {key, value}, acc -> + sanitized = + case {is_binary(value), Map.get(@data_string_max_lengths, key)} do + {true, nil} -> strip_null_bytes(value) + {true, max_length} -> value |> strip_null_bytes() |> truncate_string(max_length) + _ -> value + end + + Map.put(acc, key, sanitized) + end) + end + + # PostgreSQL JSONB does not support \u0000 (null bytes) + defp strip_null_bytes(string) when is_binary(string) do + String.replace(string, <<0x00>>, "") + end + defp truncate_string(string, max_length) when byte_size(string) <= max_length do string end diff --git a/lib/aprsme/packets/query_builder.ex b/lib/aprsme/packets/query_builder.ex index 6bb4686..ea8e7c5 100644 --- a/lib/aprsme/packets/query_builder.ex +++ b/lib/aprsme/packets/query_builder.ex @@ -125,7 +125,7 @@ defmodule Aprsme.Packets.QueryBuilder do rain_24h: p.rain_24h, rain_since_midnight: p.rain_since_midnight, snow: p.snow, - luminosity: p.luminosity + luminosity: fragment("(?->>'luminosity')::integer", p.data) } end diff --git a/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex b/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex index 8aa15a3..61061ad 100644 --- a/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex +++ b/lib/aprsme_web/controllers/api/v1/json/callsign_json.ex @@ -31,7 +31,7 @@ defmodule AprsmeWeb.Api.V1.CallsignJSON do destination: packet.destination, path: packet.path, data_type: packet.data_type, - information_field: packet.information_field, + information_field: get_in(packet.data || %{}, ["information_field"]), raw_packet: sanitize_raw_packet(packet.raw_packet), received_at: packet.received_at, region: packet.region, diff --git a/lib/aprsme_web/live/api_docs_live.ex b/lib/aprsme_web/live/api_docs_live.ex index d6dbb74..dc2510b 100644 --- a/lib/aprsme_web/live/api_docs_live.ex +++ b/lib/aprsme_web/live/api_docs_live.ex @@ -105,7 +105,7 @@ defmodule AprsmeWeb.ApiDocsLive do "destination" => packet.destination, "path" => packet.path, "data_type" => packet.data_type, - "information_field" => packet.information_field, + "information_field" => get_in(packet.data || %{}, ["information_field"]), "raw_packet" => sanitize_raw_packet(packet.raw_packet), "received_at" => packet.received_at, "region" => packet.region, diff --git a/lib/aprsme_web/live/info_live/show.html.heex b/lib/aprsme_web/live/info_live/show.html.heex index d0966e3..07ad272 100644 --- a/lib/aprsme_web/live/info_live/show.html.heex +++ b/lib/aprsme_web/live/info_live/show.html.heex @@ -143,38 +143,38 @@ <% end %> - <%= if @packet.phg_power || @packet.phg_height || @packet.phg_gain do %> + <%= if (@packet.data || %{})["phg_power"] || (@packet.data || %{})["phg_height"] || (@packet.data || %{})["phg_gain"] do %>
{gettext("PHG (Power-Height-Gain)")}
- <%= if @packet.phg_power do %> + <%= if (@packet.data || %{})["phg_power"] do %>
{gettext("Power:")} - {@packet.phg_power} W + {(@packet.data || %{})["phg_power"]} W
<% end %> - <%= if @packet.phg_height do %> + <%= if (@packet.data || %{})["phg_height"] do %>
{gettext("HAAT:")} - {@packet.phg_height} ft + {(@packet.data || %{})["phg_height"]} ft
<% end %> - <%= if @packet.phg_gain do %> + <%= if (@packet.data || %{})["phg_gain"] do %>
{gettext("Gain:")} - {@packet.phg_gain} dBi + {(@packet.data || %{})["phg_gain"]} dBi
<% end %> - <%= if @packet.phg_directivity do %> + <%= if (@packet.data || %{})["phg_directivity"] do %>
{gettext("Dir:")} - <%= if @packet.phg_directivity == 0 or @packet.phg_directivity == 360 do %> + <%= if (@packet.data || %{})["phg_directivity"] == 0 or (@packet.data || %{})["phg_directivity"] == 360 do %> {gettext("Omni")} <% else %> - {@packet.phg_directivity}° + {(@packet.data || %{})["phg_directivity"]}° <% end %>
diff --git a/lib/aprsme_web/live/map_live/packet_utils.ex b/lib/aprsme_web/live/map_live/packet_utils.ex index 5a51e8d..a375410 100644 --- a/lib/aprsme_web/live/map_live/packet_utils.ex +++ b/lib/aprsme_web/live/map_live/packet_utils.ex @@ -119,7 +119,7 @@ defmodule AprsmeWeb.MapLive.PacketUtils do where: p.sender == ^callsign, where: fragment( - "? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL", + "? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ?->>'luminosity' IS NOT NULL OR ?->>'rain_midnight' IS NOT NULL", p.temperature, p.humidity, p.pressure, @@ -128,9 +128,9 @@ defmodule AprsmeWeb.MapLive.PacketUtils do p.wind_gust, p.rain_1h, p.rain_24h, - p.rain_midnight, - p.luminosity, - p.snow_24h + p.snow, + p.data, + p.data ), select: fragment("1"), limit: 1 diff --git a/lib/aprsme_web/live/packets_live/callsign_view.html.heex b/lib/aprsme_web/live/packets_live/callsign_view.html.heex index 2ca1380..0b2fb43 100644 --- a/lib/aprsme_web/live/packets_live/callsign_view.html.heex +++ b/lib/aprsme_web/live/packets_live/callsign_view.html.heex @@ -60,12 +60,13 @@ - <%= if String.length(packet.information_field || "") > 50 do %> - - {String.slice(packet.information_field, 0, 50)}... + <% info_field = (packet.data || %{})["information_field"] || "" %> + <%= if String.length(info_field) > 50 do %> + + {String.slice(info_field, 0, 50)}... <% else %> - {packet.information_field} + {info_field} <% end %> diff --git a/lib/aprsme_web/live/shared/packet_utils.ex b/lib/aprsme_web/live/shared/packet_utils.ex index 6ea2505..3dbd082 100644 --- a/lib/aprsme_web/live/shared/packet_utils.ex +++ b/lib/aprsme_web/live/shared/packet_utils.ex @@ -103,8 +103,7 @@ defmodule AprsmeWeb.Live.Shared.PacketUtils do :rain_1h, :rain_24h, :rain_since_midnight, - :snow, - :luminosity + :snow ] Enum.any?(weather_fields, fn field -> diff --git a/lib/aprsme_web/live/weather_live/callsign_view.ex b/lib/aprsme_web/live/weather_live/callsign_view.ex index 4769ea5..85941c2 100644 --- a/lib/aprsme_web/live/weather_live/callsign_view.ex +++ b/lib/aprsme_web/live/weather_live/callsign_view.ex @@ -193,7 +193,7 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do rain_1h: rain_1h_value, rain_24h: rain_24h_value, rain_since_midnight: rain_since_midnight_value, - luminosity: pkt.luminosity + luminosity: get_in(pkt.data || %{}, ["luminosity"]) } end) |> Jason.encode!() diff --git a/priv/repo/migrations/20260220230000_consolidate_packets_columns.exs b/priv/repo/migrations/20260220230000_consolidate_packets_columns.exs new file mode 100644 index 0000000..a9bada4 --- /dev/null +++ b/priv/repo/migrations/20260220230000_consolidate_packets_columns.exs @@ -0,0 +1,468 @@ +defmodule Aprsme.Repo.Migrations.ConsolidatePacketsColumns do + use Ecto.Migration + + @moduledoc """ + Consolidates the packets table by: + - Dropping 10 dead parser-compat columns (never read) + - Moving 14 display-only columns into a single `data` JSONB column + - Fixing the notify_packets_insert() trigger to read from `data` JSONB + + The packets table is ephemeral (refills from APRS-IS stream) so data loss + from DROP/recreate is acceptable. + """ + + def up do + execute("SET LOCAL statement_timeout = '0'") + + # Drop existing partitioned table + execute("DROP TABLE IF EXISTS packets CASCADE") + + # Recreate without the 24 dropped/moved columns, plus new `data` JSONB column + execute(""" + CREATE TABLE packets ( + id uuid NOT NULL, + base_callsign varchar, + data_type varchar, + destination varchar, + path text, + sender varchar, + ssid varchar, + data_extended jsonb, + inserted_at timestamp NOT NULL, + updated_at timestamp NOT NULL, + received_at timestamp NOT NULL, + region varchar, + lat numeric, + lon numeric, + has_position boolean DEFAULT false, + item_name text, + object_name text, + is_item boolean DEFAULT false, + is_object boolean DEFAULT false, + dao jsonb, + raw_packet text, + symbol_code varchar, + symbol_table_id varchar, + comment text, + "timestamp" varchar, + aprs_messaging boolean DEFAULT false, + temperature double precision, + humidity double precision, + wind_speed double precision, + wind_direction integer, + wind_gust double precision, + pressure double precision, + rain_1h double precision, + rain_24h double precision, + rain_since_midnight double precision, + manufacturer text, + equipment_type text, + course integer, + speed double precision, + altitude double precision, + addressee text, + message_text text, + message_number varchar, + location geometry(Point,4326), + device_identifier text, + snow double precision, + has_weather boolean DEFAULT false, + data jsonb DEFAULT '{}'::jsonb, + PRIMARY KEY (id, received_at) + ) PARTITION BY RANGE (received_at) + """) + + # Default partition + execute("CREATE TABLE packets_default PARTITION OF packets DEFAULT") + + # Daily partitions for recent past + near future + today = Date.utc_today() + + for offset <- -8..2 do + date = Date.add(today, offset) + name = "packets_#{Calendar.strftime(date, "%Y%m%d")}" + next_date = Date.add(date, 1) + + execute( + "CREATE TABLE IF NOT EXISTS #{name} PARTITION OF packets FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next_date} 00:00:00')" + ) + end + + create_indexes() + update_notify_trigger() + create_triggers("packets") + end + + def down do + execute("SET LOCAL statement_timeout = '0'") + execute("DROP TABLE IF EXISTS packets CASCADE") + + # Recreate original table with all columns (non-partitioned for simplicity) + execute(""" + CREATE TABLE packets ( + id uuid NOT NULL PRIMARY KEY, + base_callsign varchar, + data_type varchar, + destination varchar, + information_field text, + path text, + sender varchar, + ssid varchar, + data_extended jsonb, + inserted_at timestamp NOT NULL, + updated_at timestamp NOT NULL, + received_at timestamp NOT NULL, + region varchar, + lat numeric, + lon numeric, + has_position boolean DEFAULT false, + item_name text, + object_name text, + is_item boolean DEFAULT false, + is_object boolean DEFAULT false, + dao jsonb, + raw_packet text, + symbol_code varchar, + symbol_table_id varchar, + comment text, + "timestamp" varchar, + aprs_messaging boolean DEFAULT false, + temperature double precision, + humidity double precision, + wind_speed double precision, + wind_direction integer, + wind_gust double precision, + pressure double precision, + rain_1h double precision, + rain_24h double precision, + rain_since_midnight double precision, + manufacturer text, + equipment_type text, + course integer, + speed double precision, + altitude double precision, + addressee text, + message_text text, + message_number varchar, + location geometry(Point,4326), + device_identifier text, + luminosity integer, + snow double precision, + phg_power integer, + phg_height integer, + phg_gain integer, + phg_directivity integer, + has_weather boolean DEFAULT false, + rain_midnight double precision, + position_ambiguity integer, + posresolution double precision, + format text, + telemetry_seq integer, + telemetry_vals integer[], + telemetry_bits text, + radiorange text, + srccallsign varchar, + dstcallsign varchar, + body text, + origpacket text, + header text, + alive integer DEFAULT 1, + posambiguity integer, + symboltable varchar, + symbolcode varchar, + messaging integer + ) + """) + + restore_notify_trigger() + create_triggers("packets") + end + + defp create_indexes do + # Time-series + execute("CREATE INDEX idx_packets_received_desc ON packets (received_at DESC)") + execute("CREATE INDEX idx_packets_received_brin ON packets USING brin (received_at)") + + # Sender/Callsign + execute("CREATE INDEX idx_packets_sender_received ON packets (sender, received_at DESC)") + + execute( + "CREATE INDEX idx_packets_upper_sender ON packets (upper(sender::text), received_at DESC)" + ) + + execute("CREATE INDEX idx_packets_sender_pattern ON packets (sender text_pattern_ops)") + + execute( + "CREATE INDEX idx_packets_base_callsign_pattern ON packets (base_callsign text_pattern_ops)" + ) + + execute( + "CREATE INDEX idx_packets_base_callsign_time ON packets (base_callsign, received_at DESC)" + ) + + execute("CREATE INDEX idx_packets_sender_id_desc ON packets (sender, id DESC)") + + execute( + "CREATE INDEX idx_packets_upper_base_callsign ON packets (upper(base_callsign::text))" + ) + + execute( + "CREATE INDEX idx_packets_sender_trgm ON packets USING gin (sender gin_trgm_ops) WHERE sender IS NOT NULL" + ) + + # Position/Geographic + execute( + "CREATE INDEX idx_packets_positioned ON packets (received_at DESC, lat, lon, sender) WHERE has_position = true" + ) + + execute( + "CREATE INDEX idx_packets_position_time ON packets (has_position, received_at DESC) WHERE has_position = true" + ) + + execute( + "CREATE INDEX idx_packets_location ON packets USING gist (location) WHERE has_position = true" + ) + + execute( + "CREATE INDEX idx_packets_location_geography ON packets USING gist ((location::geography))" + ) + + execute( + "CREATE INDEX idx_packets_spatial_temporal ON packets (received_at DESC, base_callsign) WHERE has_position = true" + ) + + execute( + "CREATE INDEX idx_packets_sender_position ON packets (sender, received_at DESC) WHERE lat IS NOT NULL AND lon IS NOT NULL" + ) + + execute("CREATE INDEX idx_packets_lat ON packets (lat)") + execute("CREATE INDEX idx_packets_lon ON packets (lon)") + + # Region + execute("CREATE INDEX idx_packets_region ON packets (region)") + execute("CREATE INDEX idx_packets_region_received ON packets (region, received_at)") + + execute( + "CREATE INDEX idx_packets_region_position_time ON packets (region, has_position, received_at DESC) WHERE has_position = true AND region IS NOT NULL" + ) + + execute( + "CREATE INDEX idx_packets_region_data_type ON packets (region, data_type, received_at)" + ) + + execute( + "CREATE INDEX idx_packets_region_position ON packets (region, has_position, received_at)" + ) + + # Weather + execute( + "CREATE INDEX idx_packets_weather ON packets (sender, received_at DESC) WHERE temperature IS NOT NULL OR humidity IS NOT NULL OR pressure IS NOT NULL OR wind_speed IS NOT NULL OR wind_direction IS NOT NULL OR rain_1h IS NOT NULL" + ) + + execute( + "CREATE INDEX idx_packets_has_weather ON packets (has_weather, sender, received_at DESC) WHERE has_weather = true" + ) + + execute( + "CREATE INDEX idx_packets_temperature ON packets (temperature) WHERE temperature IS NOT NULL" + ) + + execute("CREATE INDEX idx_packets_humidity ON packets (humidity) WHERE humidity IS NOT NULL") + execute("CREATE INDEX idx_packets_pressure ON packets (pressure) WHERE pressure IS NOT NULL") + + execute( + "CREATE INDEX idx_packets_wind_speed ON packets (wind_speed) WHERE wind_speed IS NOT NULL" + ) + + execute( + "CREATE INDEX idx_packets_received_temp ON packets (received_at, temperature) WHERE temperature IS NOT NULL" + ) + + execute( + "CREATE INDEX idx_packets_sender_temp ON packets (sender, temperature) WHERE temperature IS NOT NULL" + ) + + execute( + "CREATE INDEX idx_packets_weather_history ON packets (sender, data_type, received_at) WHERE data_type = 'weather'" + ) + + execute( + "CREATE INDEX idx_packets_weather_selective ON packets (received_at DESC) WHERE data_type IN ('weather', 'Weather', 'WX', 'wx')" + ) + + execute( + "CREATE INDEX idx_packets_weather_symbol ON packets (received_at DESC, lat, lon) WHERE (symbol_table_id = '/' AND symbol_code = '_') OR (symbol_table_id = E'\\\\' AND symbol_code = '_')" + ) + + execute( + "CREATE INDEX idx_packets_sender_weather_lookup ON packets (upper(sender::text), received_at DESC) INCLUDE (data_type, symbol_table_id, symbol_code) WHERE data_type = 'weather' OR (symbol_table_id = '/' AND symbol_code = '_') OR (symbol_table_id = E'\\\\' AND symbol_code = '_')" + ) + + # Data type & Device + execute( + "CREATE INDEX idx_packets_datatype_sender_time ON packets (data_type, sender, received_at DESC)" + ) + + execute( + "CREATE INDEX idx_packets_device_identifier ON packets (device_identifier) WHERE device_identifier IS NOT NULL" + ) + + execute( + "CREATE INDEX idx_packets_device_received ON packets (device_identifier, received_at) WHERE device_identifier IS NOT NULL" + ) + + # Path + execute( + "CREATE INDEX idx_packets_path_pattern ON packets (path text_pattern_ops) WHERE path IS NOT NULL AND path <> ''" + ) + + execute( + "CREATE INDEX idx_packets_path_trgm ON packets USING gin (path gin_trgm_ops) WHERE path IS NOT NULL AND path <> '' AND path !~ '^TCPIP' AND path !~ ',TCPIP'" + ) + + # Digipeater/stations heard + execute( + "CREATE INDEX idx_packets_digipeater ON packets (sender, received_at DESC) WHERE location IS NOT NULL" + ) + + execute( + "CREATE INDEX idx_packets_stations_heard ON packets (received_at DESC, sender, lat, lon) WHERE path IS NOT NULL AND path <> '' AND path !~ '^TCPIP' AND path !~ ',TCPIP' AND lat IS NOT NULL AND lon IS NOT NULL" + ) + + # Symbol + execute( + "CREATE INDEX idx_packets_symbol_time ON packets (symbol_table_id, symbol_code, received_at)" + ) + + # JSONB data column — GIN index for key lookups + execute( + "CREATE INDEX idx_packets_data_gin ON packets USING gin (data) WHERE data IS NOT NULL AND data <> '{}'::jsonb" + ) + end + + # Updated trigger reads PHG/luminosity/rain_midnight from data JSONB + defp update_notify_trigger do + execute("DROP FUNCTION IF EXISTS notify_packets_insert() CASCADE") + + execute(""" + CREATE OR REPLACE FUNCTION notify_packets_insert() RETURNS trigger AS $$ + DECLARE + payload TEXT; + BEGIN + payload := json_build_object( + 'id', NEW.id, + 'sender', NEW.sender, + 'ssid', NEW.ssid, + 'base_callsign', NEW.base_callsign, + 'lat', NEW.lat, + 'lon', NEW.lon, + 'altitude', NEW.altitude, + 'course', NEW.course, + 'speed', NEW.speed, + 'symbol_table_id', NEW.symbol_table_id, + 'symbol_code', NEW.symbol_code, + 'device_identifier', NEW.device_identifier, + 'path', NEW.path, + 'data_type', NEW.data_type, + 'received_at', NEW.received_at, + 'comment', NEW.comment, + 'raw_packet', NEW.raw_packet, + 'phg_power', NEW.data->>'phg_power', + 'phg_height', NEW.data->>'phg_height', + 'phg_gain', NEW.data->>'phg_gain', + 'phg_directivity', NEW.data->>'phg_directivity', + 'temperature', NEW.temperature, + 'humidity', NEW.humidity, + 'pressure', NEW.pressure, + 'wind_speed', NEW.wind_speed, + 'wind_direction', NEW.wind_direction, + 'wind_gust', NEW.wind_gust, + 'rain_1h', NEW.rain_1h, + 'rain_24h', NEW.rain_24h, + 'rain_midnight', NEW.data->>'rain_midnight', + 'luminosity', NEW.data->>'luminosity', + 'snow', NEW.snow, + 'has_weather', NEW.has_weather, + 'inserted_at', NEW.inserted_at + )::text; + + PERFORM pg_notify('aprs_packets', payload); + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """) + end + + # Restore original trigger for rollback + defp restore_notify_trigger do + execute("DROP FUNCTION IF EXISTS notify_packets_insert() CASCADE") + + execute(""" + CREATE OR REPLACE FUNCTION notify_packets_insert() RETURNS trigger AS $$ + DECLARE + payload TEXT; + BEGIN + payload := json_build_object( + 'id', NEW.id, + 'sender', NEW.sender, + 'ssid', NEW.ssid, + 'base_callsign', NEW.base_callsign, + 'lat', NEW.lat, + 'lon', NEW.lon, + 'altitude', NEW.altitude, + 'course', NEW.course, + 'speed', NEW.speed, + 'symbol_table_id', NEW.symbol_table_id, + 'symbol_code', NEW.symbol_code, + 'device_identifier', NEW.device_identifier, + 'path', NEW.path, + 'data_type', NEW.data_type, + 'received_at', NEW.received_at, + 'comment', NEW.comment, + 'raw_packet', NEW.raw_packet, + 'phg_power', NEW.phg_power, + 'phg_height', NEW.phg_height, + 'phg_gain', NEW.phg_gain, + 'phg_directivity', NEW.phg_directivity, + 'temperature', NEW.temperature, + 'humidity', NEW.humidity, + 'pressure', NEW.pressure, + 'wind_speed', NEW.wind_speed, + 'wind_direction', NEW.wind_direction, + 'wind_gust', NEW.wind_gust, + 'rain_1h', NEW.rain_1h, + 'rain_24h', NEW.rain_24h, + 'rain_midnight', NEW.rain_midnight, + 'luminosity', NEW.luminosity, + 'snow', NEW.snow, + 'has_weather', NEW.has_weather, + 'inserted_at', NEW.inserted_at + )::text; + + PERFORM pg_notify('aprs_packets', payload); + + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """) + end + + defp create_triggers(table) do + execute( + "CREATE TRIGGER update_has_weather_trigger BEFORE INSERT OR UPDATE ON #{table} FOR EACH ROW EXECUTE FUNCTION update_has_weather()" + ) + + execute( + "CREATE TRIGGER packets_notify_insert AFTER INSERT ON #{table} FOR EACH ROW EXECUTE FUNCTION notify_packets_insert()" + ) + + execute( + "CREATE TRIGGER packet_insert_sequence AFTER INSERT ON #{table} FOR EACH ROW EXECUTE FUNCTION increment_packet_sequence()" + ) + + execute( + "CREATE TRIGGER packet_delete_sequence AFTER DELETE ON #{table} FOR EACH ROW EXECUTE FUNCTION decrement_packet_sequence()" + ) + end +end diff --git a/test/aprsme/encoding_utils_test.exs b/test/aprsme/encoding_utils_test.exs index 1657109..3b980cd 100644 --- a/test/aprsme/encoding_utils_test.exs +++ b/test/aprsme/encoding_utils_test.exs @@ -41,7 +41,7 @@ defmodule Aprsme.EncodingUtilsTest do end describe "sanitize_packet/1" do - test "sanitizes information_field" do + test "sanitizes string values in data map" do invalid_info = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>> packet = %Packet{ @@ -49,18 +49,20 @@ defmodule Aprsme.EncodingUtilsTest do sender: "TEST-1", path: "TCPIP*", destination: "APX100", - information_field: invalid_info, data_type: :position, base_callsign: "TEST", ssid: "1", + data: %{"information_field" => invalid_info, "radiorange" => "0050"}, data_extended: %{comment: "Valid comment"} } sanitized = EncodingUtils.sanitize_packet(packet) - assert String.valid?(sanitized.information_field) - assert String.contains?(sanitized.information_field, "Hello") - assert String.contains?(sanitized.information_field, "World") + assert String.valid?(sanitized.data["information_field"]) + assert String.contains?(sanitized.data["information_field"], "Hello") + assert String.contains?(sanitized.data["information_field"], "World") + # Non-string values in data map are preserved + assert sanitized.data["radiorange"] == "0050" end test "sanitizes comment in data_extended" do @@ -71,7 +73,6 @@ defmodule Aprsme.EncodingUtilsTest do sender: "TEST-1", path: "TCPIP*", destination: "APX100", - information_field: "Valid info", data_type: :position, base_callsign: "TEST", ssid: "1", @@ -90,7 +91,6 @@ defmodule Aprsme.EncodingUtilsTest do sender: "TEST-1", path: "TCPIP*", destination: "APX100", - information_field: "Valid info", data_type: :position, base_callsign: "TEST", ssid: "1", @@ -100,7 +100,6 @@ defmodule Aprsme.EncodingUtilsTest do sanitized = EncodingUtils.sanitize_packet(packet) assert sanitized.data_extended == nil - assert sanitized.information_field == "Valid info" end end diff --git a/test/aprsme/enhanced_parser_test.exs b/test/aprsme/enhanced_parser_test.exs index ded16b3..2ce35a1 100644 --- a/test/aprsme/enhanced_parser_test.exs +++ b/test/aprsme/enhanced_parser_test.exs @@ -4,7 +4,7 @@ defmodule Aprsme.EnhancedParserTest do alias Aprsme.Packet describe "enhanced parser field extraction" do - test "extracts standard parser compatibility fields" do + test "dead parser compat fields are dropped (not preserved)" do attrs = %{ srccallsign: "W1AW", dstcallsign: "APRS", @@ -26,19 +26,20 @@ defmodule Aprsme.EnhancedParserTest do result = Packet.extract_additional_data(attrs, "W1AW>APRS:!1234.56N/12345.67W-Test") - assert result[:srccallsign] == "W1AW" - assert result[:dstcallsign] == "APRS" - assert result[:body] == "!1234.56N/12345.67W-Test" - assert result[:origpacket] == "W1AW>APRS:!1234.56N/12345.67W-Test" - assert result[:header] == "W1AW>APRS" - assert result[:alive] == 1 - assert result[:posambiguity] == 0 - assert result[:symboltable] == "/" - assert result[:symbolcode] == "-" - assert result[:messaging] == 0 + # Dead fields should not be in result + refute Map.has_key?(result, :srccallsign) + refute Map.has_key?(result, :dstcallsign) + refute Map.has_key?(result, :body) + refute Map.has_key?(result, :origpacket) + refute Map.has_key?(result, :header) + refute Map.has_key?(result, :alive) + refute Map.has_key?(result, :posambiguity) + refute Map.has_key?(result, :symboltable) + refute Map.has_key?(result, :symbolcode) + refute Map.has_key?(result, :messaging) end - test "extracts radio range field" do + test "extracts radio range into data map" do attrs = %{ radiorange: "0050", sender: "W1AW", @@ -51,7 +52,7 @@ defmodule Aprsme.EnhancedParserTest do result = Packet.extract_additional_data(attrs) - assert result[:radiorange] == "0050" + assert result[:data]["radiorange"] == "0050" end test "handles weather data from wx field" do @@ -82,7 +83,7 @@ defmodule Aprsme.EnhancedParserTest do assert result[:wind_direction] == 180 end - test "handles PHG data in string format" do + test "handles PHG data in string format and moves to data map" do attrs = %{ sender: "W1AW", base_callsign: "W1AW", @@ -98,13 +99,14 @@ defmodule Aprsme.EnhancedParserTest do result = Packet.extract_additional_data(attrs) # PHG 5430: power=5^2=25W, height=4->160ft, gain=3dB, dir=0->omni - assert result[:phg_power] == 25 - assert result[:phg_height] == 160 - assert result[:phg_gain] == 3 - assert result[:phg_directivity] == 0 + # PHG fields should be in the data map + assert result[:data]["phg_power"] == 25 + assert result[:data]["phg_height"] == 160 + assert result[:data]["phg_gain"] == 3 + assert result[:data]["phg_directivity"] == 0 end - test "handles PHG data in legacy map format" do + test "handles PHG data in legacy map format and moves to data map" do attrs = %{ sender: "W1AW", base_callsign: "W1AW", @@ -124,15 +126,103 @@ defmodule Aprsme.EnhancedParserTest do result = Packet.extract_additional_data(attrs) - assert result[:phg_power] == 25 - assert result[:phg_height] == 160 - assert result[:phg_gain] == 3 - assert result[:phg_directivity] == 0 + assert result[:data]["phg_power"] == 25 + assert result[:data]["phg_height"] == 160 + assert result[:data]["phg_gain"] == 3 + assert result[:data]["phg_directivity"] == 0 + end + + test "collect_into_data_map sweeps all 14 fields into data" do + attrs = %{ + sender: "W1AW", + base_callsign: "W1AW", + ssid: "0", + destination: "APRS", + data_type: "position", + received_at: DateTime.utc_now(), + data_extended: %{ + phg: "5430", + format: :uncompressed, + posresolution: 18.52, + position_ambiguity: 0, + comment: "Test PHG5430" + } + } + + result = Packet.extract_additional_data(attrs) + + # These fields should be in the data map, not at top level + assert is_map(result[:data]) + assert result[:data]["phg_power"] == 25 + assert result[:data]["format"] == "uncompressed" + assert result[:data]["posresolution"] == 18.52 + assert result[:data]["position_ambiguity"] == 0 + + # They should NOT be at top level + refute Map.has_key?(result, :phg_power) + refute Map.has_key?(result, :format) + refute Map.has_key?(result, :posresolution) + refute Map.has_key?(result, :position_ambiguity) + end + + test "telemetry fields are moved into data map" do + attrs = %{ + sender: "W1AW", + base_callsign: "W1AW", + ssid: "0", + destination: "APRS", + data_type: "telemetry", + received_at: DateTime.utc_now(), + data_extended: %{ + telemetry: %{ + seq: 123, + vals: [1, 2, 3, 4, 5], + bits: "11111111" + } + } + } + + result = Packet.extract_additional_data(attrs) + + assert result[:data]["telemetry_seq"] == 123 + assert result[:data]["telemetry_vals"] == [1, 2, 3, 4, 5] + assert result[:data]["telemetry_bits"] == "11111111" + + refute Map.has_key?(result, :telemetry_seq) + refute Map.has_key?(result, :telemetry_vals) + refute Map.has_key?(result, :telemetry_bits) + end + + test "luminosity and rain_midnight are moved into data map" do + attrs = %{ + sender: "WX1STN", + base_callsign: "WX1STN", + ssid: "0", + destination: "APRS", + data_type: "weather", + received_at: DateTime.utc_now(), + data_extended: %{ + wx: %{ + temperature: 72.5, + luminosity: 500, + rain_midnight: 0.5 + } + } + } + + result = Packet.extract_additional_data(attrs) + + # luminosity and rain_midnight should be in data map + assert result[:data]["luminosity"] == 500 + assert result[:data]["rain_midnight"] == 0.5 + + # temperature stays at top level + assert result[:temperature] == 72.5 end end describe "enhanced parser packet storage" do - test "stores packet with enhanced parser fields" do + test "stores packet with data map fields" do attrs = %{ sender: "W1AW-1", base_callsign: "W1AW", @@ -146,36 +236,17 @@ defmodule Aprsme.EnhancedParserTest do symbol_code: "-", symbol_table_id: "/", comment: "Test station", - # Enhanced parser fields - srccallsign: "W1AW-1", - dstcallsign: "APRS", - body: "!1234.56N/12345.67W-Test RNG0050", - origpacket: "W1AW-1>APRS:!1234.56N/12345.67W-Test RNG0050", - header: "W1AW-1>APRS", - alive: 1, - posambiguity: 0, - symboltable: "/", - symbolcode: "-", - messaging: 0, - radiorange: "0050" + radiorange: "0050", + information_field: "!1234.56N/12345.67W-Test" } processed_attrs = Packet.extract_additional_data(attrs) assert {:ok, packet} = Aprsme.Packets.store_packet(processed_attrs) - # Verify enhanced fields are stored - assert packet.srccallsign == "W1AW-1" - assert packet.dstcallsign == "APRS" - assert packet.body == "!1234.56N/12345.67W-Test RNG0050" - assert packet.origpacket == "W1AW-1>APRS:!1234.56N/12345.67W-Test RNG0050" - assert packet.header == "W1AW-1>APRS" - assert packet.alive == 1 - assert packet.posambiguity == 0 - assert packet.symboltable == "/" - assert packet.symbolcode == "-" - assert packet.messaging == 0 - assert packet.radiorange == "0050" + # radiorange and information_field should be in data map + assert packet.data["radiorange"] == "0050" + assert packet.data["information_field"] == "!1234.56N/12345.67W-Test" end end end diff --git a/test/aprsme/packet_parsing_test.exs b/test/aprsme/packet_parsing_test.exs index a9ed492..d193197 100644 --- a/test/aprsme/packet_parsing_test.exs +++ b/test/aprsme/packet_parsing_test.exs @@ -22,15 +22,15 @@ defmodule Aprsme.PacketParsingTest do # Verify altitude was extracted assert extracted_attrs[:altitude] == 680.0 - # Verify PHG data was extracted into individual fields + # Verify PHG data was extracted into the data map # PHG5 = 25W (5^2) - assert extracted_attrs[:phg_power] == 25 + assert extracted_attrs.data["phg_power"] == 25 # PHG x5x = 320 ft - assert extracted_attrs[:phg_height] == 320 + assert extracted_attrs.data["phg_height"] == 320 # PHG xx3 = 3 dBi - assert extracted_attrs[:phg_gain] == 3 + assert extracted_attrs.data["phg_gain"] == 3 # PHG xxx0 = 0 degrees (not 360 for omni in our implementation) - assert extracted_attrs[:phg_directivity] == 0 + assert extracted_attrs.data["phg_directivity"] == 0 # Verify comment has altitude and PHG removed assert extracted_attrs[:comment] == "Collin Cty Wide Digi" @@ -73,10 +73,10 @@ defmodule Aprsme.PacketParsingTest do # Check that fields were properly set changes = changeset.changes assert changes.altitude == 680.0 - assert changes.phg_power == 25 - assert changes.phg_height == 320 - assert changes.phg_gain == 3 - assert changes.phg_directivity == 0 + assert changes.data["phg_power"] == 25 + assert changes.data["phg_height"] == 320 + assert changes.data["phg_gain"] == 3 + assert changes.data["phg_directivity"] == 0 assert changes.comment == "Collin Cty Wide Digi" end end diff --git a/test/aprsme/packet_test.exs b/test/aprsme/packet_test.exs index 9d589a2..20030b1 100644 --- a/test/aprsme/packet_test.exs +++ b/test/aprsme/packet_test.exs @@ -19,10 +19,10 @@ defmodule Aprsme.PacketTest do result = Packet.extract_additional_data(attrs, raw_packet) - assert result.telemetry_seq == 5 + assert result.data["telemetry_seq"] == 5 # 12.80 rounds to 13 - assert result.telemetry_vals == [13, 0, 0, 0, 0] - assert result.telemetry_bits == "00000000" + assert result.data["telemetry_vals"] == [13, 0, 0, 0, 0] + assert result.data["telemetry_bits"] == "00000000" end test "handles mixed integer and string telemetry_vals" do @@ -38,10 +38,10 @@ defmodule Aprsme.PacketTest do result = Packet.extract_additional_data(attrs, "test_packet") - assert result.telemetry_seq == 10 + assert result.data["telemetry_seq"] == 10 # Mixed types converted properly - assert result.telemetry_vals == [180, 38, 0, 88, 165] - assert result.telemetry_bits == "10101010" + assert result.data["telemetry_vals"] == [180, 38, 0, 88, 165] + assert result.data["telemetry_bits"] == "10101010" end test "handles float telemetry_vals" do @@ -56,9 +56,9 @@ defmodule Aprsme.PacketTest do result = Packet.extract_additional_data(attrs, "test_packet") - assert result.telemetry_seq == 1 + assert result.data["telemetry_seq"] == 1 # Floats rounded to integers - assert result.telemetry_vals == [13, 37, 0, 89, 165] + assert result.data["telemetry_vals"] == [13, 37, 0, 89, 165] end test "handles invalid telemetry_vals gracefully" do @@ -73,9 +73,9 @@ defmodule Aprsme.PacketTest do result = Packet.extract_additional_data(attrs, "test_packet") - assert result.telemetry_seq == 2 + assert result.data["telemetry_seq"] == 2 # Invalid values become 0 - assert result.telemetry_vals == [0, 0, 0, 13, 0] + assert result.data["telemetry_vals"] == [0, 0, 0, 13, 0] end test "handles missing telemetry data" do @@ -86,9 +86,9 @@ defmodule Aprsme.PacketTest do result = Packet.extract_additional_data(attrs, "test_packet") - assert Map.get(result, :telemetry_seq) == nil - assert Map.get(result, :telemetry_vals) == nil - assert Map.get(result, :telemetry_bits) == nil + assert get_in(result, [:data, "telemetry_seq"]) == nil + assert get_in(result, [:data, "telemetry_vals"]) == nil + assert get_in(result, [:data, "telemetry_bits"]) == nil end end end diff --git a/test/aprsme/packets_encoding_test.exs b/test/aprsme/packets_encoding_test.exs index dceec0a..7d37453 100644 --- a/test/aprsme/packets_encoding_test.exs +++ b/test/aprsme/packets_encoding_test.exs @@ -21,7 +21,7 @@ defmodule Aprsme.PacketsEncodingTest do } assert {:ok, stored_packet} = Packets.store_packet(packet_data) - assert stored_packet.information_field == "Valid UTF-8: ñ, é, 中文, 🚀" + assert stored_packet.data["information_field"] == "Valid UTF-8: ñ, é, 中文, 🚀" assert stored_packet.comment == "More UTF-8: αβγ, русский" end @@ -68,8 +68,6 @@ defmodule Aprsme.PacketsEncodingTest do } assert {:ok, stored_packet} = Packets.store_packet(packet_data) - # Empty strings may be stored as nil or empty string depending on Ecto behavior - assert stored_packet.information_field in [nil, ""] assert stored_packet.path in [nil, ""] assert is_nil(stored_packet.comment) end @@ -90,8 +88,8 @@ defmodule Aprsme.PacketsEncodingTest do } assert {:ok, stored_packet} = Packets.store_packet(packet_data) - # Control characters should be filtered out - assert stored_packet.information_field == "Control chars:" + # Control characters should be filtered out, field is now in data map + assert stored_packet.data["information_field"] == "Control chars:" end end