Consolidate 24 packet columns into JSONB data field

Drop 10 dead parser-compat columns (srccallsign, dstcallsign,
origpacket, body, header, alive, posambiguity, symboltable,
symbolcode, messaging) and move 14 display-only columns into a
single `data` JSONB column (PHG, telemetry, radiorange,
information_field, format, posresolution, position_ambiguity,
luminosity, rain_midnight).

Reduces row width from ~70 to ~48 columns. Table is ephemeral
so migration uses DROP/recreate. Also fixes snow_24h -> snow bug
in weather check query and adds null byte stripping for JSONB
string sanitization.

lib/aprsme/encoding_utils.ex

@@ -201,8 +201,7 @@ defmodule Aprsme.EncodingUtils do
       :rain_1h,
       :rain_24h,
       :rain_since_midnight,
-      :snow,
+      :snow
-      :luminosity
     ]
   end
 
@@ -211,71 +210,83 @@ defmodule Aprsme.EncodingUtils do
 
   ## Examples
 
-      iex> result = Aprsme.EncodingUtils.sanitize_packet(%{"information_field" => <<0, 65, 66, 67>>, "data_extended" => %{"comment" => <<0, 68, 69, 70>>}})
+      iex> result = Aprsme.EncodingUtils.sanitize_packet(%{"comment" => <<0, 65, 66, 67>>, "data_extended" => %{"comment" => <<0, 68, 69, 70>>}})
-      iex> result["information_field"] == "ABC"
+      iex> result["comment"] == "ABC"
       true
       iex> result["data_extended"]["comment"] == "DEF"
       true
   """
   @spec sanitize_packet(struct() | map()) :: struct() | map()
   def sanitize_packet(%Aprsme.Packet{} = packet) do
+    sanitized_data =
+      case packet.data do
+        nil -> nil
+        data when is_map(data) -> sanitize_packet_strings(data)
+        other -> other
+      end
+
     %{
       packet
-      | information_field: sanitize_string(packet.information_field),
+      | data: sanitized_data,
         data_extended: sanitize_data_extended(packet.data_extended)
     }
   end
 
   def sanitize_packet(packet) when is_map(packet) do
-    # Handle all known string fields, checking for both atom and string keys
+    packet
-    string_fields = [
+    |> sanitize_string_fields()
-      :information_field,
+    |> sanitize_nested_map(:data, &sanitize_packet_strings/1)
-      :comment,
+    |> sanitize_nested_map(:data_extended, &sanitize_data_extended/1)
-      :path,
+  end
-      :raw_packet,
-      :destination,
-      :sender,
-      :base_callsign,
-      :ssid,
-      :manufacturer,
-      :equipment_type,
-      :message_text,
-      :addressee,
-      :symbol_code,
-      :symbol_table_id,
-      :dao,
-      :timestamp,
-      :device_identifier
-    ]
 
-    # Sanitize all string fields
+  @string_fields [
-    sanitized =
+    :comment,
-      Enum.reduce(string_fields, packet, fn field, acc ->
+    :path,
-        atom_key = field
+    :raw_packet,
-        string_key = to_string(field)
+    :destination,
+    :sender,
+    :base_callsign,
+    :ssid,
+    :manufacturer,
+    :equipment_type,
+    :message_text,
+    :addressee,
+    :symbol_code,
+    :symbol_table_id,
+    :dao,
+    :timestamp,
+    :device_identifier
+  ]
 
-        cond do
+  defp sanitize_string_fields(packet) do
-          Map.has_key?(acc, atom_key) ->
+    Enum.reduce(@string_fields, packet, fn field, acc ->
-            Map.update(acc, atom_key, nil, &sanitize_string/1)
+      string_key = to_string(field)
 
-          Map.has_key?(acc, string_key) ->
+      cond do
-            Map.update(acc, string_key, nil, &sanitize_string/1)
+        Map.has_key?(acc, field) ->
+          Map.update(acc, field, nil, &sanitize_string/1)
 
-          true ->
+        Map.has_key?(acc, string_key) ->
-            acc
+          Map.update(acc, string_key, nil, &sanitize_string/1)
-        end
+
-      end)
+        true ->
+          acc
+      end
+    end)
+  end
+
+  defp sanitize_nested_map(packet, key, sanitizer) do
+    string_key = to_string(key)
 
-    # Handle data_extended separately
     cond do
-      Map.has_key?(sanitized, :data_extended) ->
+      Map.has_key?(packet, key) ->
-        Map.update(sanitized, :data_extended, nil, &sanitize_data_extended/1)
+        Map.update(packet, key, nil, &if(is_nil(&1), do: nil, else: sanitizer.(&1)))
 
-      Map.has_key?(sanitized, "data_extended") ->
+      Map.has_key?(packet, string_key) ->
-        Map.update(sanitized, "data_extended", nil, &sanitize_data_extended/1)
+        Map.update(packet, string_key, nil, &if(is_nil(&1), do: nil, else: sanitizer.(&1)))
 
       true ->
-        sanitized
+        packet
     end
   end
 

lib/aprsme/packet.ex

@@ -11,7 +11,6 @@ defmodule Aprsme.Packet do
     field(:base_callsign, :string)
     field(:data_type, :string)
     field(:destination, :string)
-    field(:information_field, :string)
     field(:path, :string)
     field(:sender, :string)
     field(:ssid, :string)
@@ -45,55 +44,22 @@ defmodule Aprsme.Packet do
     field(:snow, :float)
 
     # Equipment/status information
-    field(:luminosity, :integer)
     field(:manufacturer, :string)
     field(:equipment_type, :string)
     field(:course, :integer)
     field(:speed, :float)
     field(:altitude, :float)
 
-    # Position ambiguity level (0-4)
-    field(:position_ambiguity, :integer)
-
-    # Position resolution and format
-    field(:posresolution, :float)
-    field(:format, :string)
-
-    # PHG (Power-Height-Gain) fields
-    field(:phg_power, :integer)
-    field(:phg_height, :integer)
-    field(:phg_gain, :integer)
-    field(:phg_directivity, :integer)
-
     # Message-specific fields
     field(:addressee, :string)
     field(:message_text, :string)
     field(:message_number, :string)
 
-    # Telemetry fields
-    field(:telemetry_seq, :integer)
-    field(:telemetry_vals, {:array, :integer})
-    field(:telemetry_bits, :string)
-
-    # Radio range field
-    field(:radiorange, :string)
-
-    # Standard parser compatibility fields
-    field(:srccallsign, :string)
-    field(:dstcallsign, :string)
-    field(:body, :string)
-    field(:origpacket, :string)
-    field(:header, :string)
-    field(:alive, :integer, default: 1)
-    field(:posambiguity, :integer)
-    field(:symboltable, :string)
-    field(:symbolcode, :string)
-    field(:messaging, :integer)
-
-    # Additional weather fields
-    field(:rain_midnight, :float)
     field(:has_weather, :boolean, default: false)
 
+    # JSONB column for display-only fields (PHG, telemetry, radiorange, etc.)
+    field(:data, :map, default: %{})
+
     field(:device_identifier, :string)
 
     # APRS Items/Objects support
@@ -123,7 +89,6 @@ defmodule Aprsme.Packet do
       :base_callsign,
       :data_type,
       :destination,
-      :information_field,
       :path,
       :sender,
       :ssid,
@@ -149,38 +114,16 @@ defmodule Aprsme.Packet do
       :rain_24h,
       :rain_since_midnight,
       :snow,
-      :luminosity,
       :manufacturer,
       :equipment_type,
       :course,
       :speed,
       :altitude,
-      :position_ambiguity,
-      :posresolution,
-      :format,
-      :phg_power,
-      :phg_height,
-      :phg_gain,
-      :phg_directivity,
       :addressee,
       :message_text,
       :message_number,
-      :telemetry_seq,
-      :telemetry_vals,
-      :telemetry_bits,
-      :radiorange,
-      :srccallsign,
-      :dstcallsign,
-      :body,
-      :origpacket,
-      :header,
-      :alive,
-      :posambiguity,
-      :symboltable,
-      :symbolcode,
-      :messaging,
-      :rain_midnight,
       :has_weather,
+      :data,
       :device_identifier,
       :item_name,
       :object_name,
@@ -345,8 +288,12 @@ defmodule Aprsme.Packet do
           %{}
       end
 
-    # Merge all extracted data
+    # Merge all extracted data, then sweep display-only fields into data map
-    merge_extracted_data(base_attrs, additional_data, attrs)
+    # and strip dead parser compat fields
+    base_attrs
+    |> merge_extracted_data(additional_data, attrs)
+    |> collect_into_data_map()
+    |> strip_dead_fields()
   end
 
   defp add_symbol_data(base_attrs, attrs) do
@@ -366,16 +313,39 @@ defmodule Aprsme.Packet do
         put_telemetry_fields(%{}, attrs)
       end
 
-    # Process standard parser fields and radio range from top-level attrs
+    # Process radio range from top-level attrs
-    parser_data =
+    radio_range_data = put_radio_range_field(%{}, attrs)
-      %{}
-      |> put_standard_parser_fields(attrs)
-      |> put_radio_range_field(attrs)
 
     base_attrs
     |> Map.merge(additional_data)
     |> Map.merge(telemetry_data)
-    |> Map.merge(parser_data)
+    |> Map.merge(radio_range_data)
+  end
+
+  # Fields that get swept into the JSONB `data` column
+  @data_fields ~w[telemetry_seq telemetry_vals telemetry_bits phg_power phg_height
+    phg_gain phg_directivity radiorange information_field format posresolution
+    position_ambiguity luminosity rain_midnight]a
+
+  defp collect_into_data_map(attrs) do
+    {data_values, remaining} =
+      Enum.reduce(@data_fields, {%{}, attrs}, fn field, {data, rest} ->
+        case Map.pop(rest, field) do
+          {nil, rest} -> {data, rest}
+          {value, rest} -> {Map.put(data, to_string(field), value), rest}
+        end
+      end)
+
+    existing_data = Map.get(remaining, :data, %{}) || %{}
+    Map.put(remaining, :data, Map.merge(existing_data, data_values))
+  end
+
+  # Dead parser compat fields — never read, safe to strip
+  @dead_fields ~w[srccallsign dstcallsign origpacket body header alive
+    posambiguity symboltable symbolcode messaging]a
+
+  defp strip_dead_fields(attrs) do
+    Map.drop(attrs, @dead_fields)
   end
 
   # Extract data from standard map-based data_extended
@@ -407,7 +377,6 @@ defmodule Aprsme.Packet do
     result =
       %{}
       |> put_symbol_fields(combined_data)
-      |> put_standard_parser_fields(combined_data)
       |> extract_weather_data(combined_data)
       |> put_weather_fields(combined_data)
       |> put_equipment_fields(combined_data)
@@ -861,21 +830,6 @@ defmodule Aprsme.Packet do
   defp normalize_format_field(format) when is_binary(format), do: format
   defp normalize_format_field(_), do: nil
 
-  # Extract standard parser compatibility fields
-  defp put_standard_parser_fields(map, data) do
-    map
-    |> maybe_put(:srccallsign, get_field_value(data, :srccallsign))
-    |> maybe_put(:dstcallsign, get_field_value(data, :dstcallsign))
-    |> maybe_put(:body, get_field_value(data, :body))
-    |> maybe_put(:origpacket, get_field_value(data, :origpacket))
-    |> maybe_put(:header, get_field_value(data, :header))
-    |> maybe_put(:alive, get_field_value(data, :alive))
-    |> maybe_put(:posambiguity, get_field_value(data, :posambiguity))
-    |> maybe_put(:symboltable, get_field_value(data, :symboltable))
-    |> maybe_put(:symbolcode, get_field_value(data, :symbolcode))
-    |> maybe_put(:messaging, get_field_value(data, :messaging))
-  end
-
   # Extract radio range field
   defp put_radio_range_field(map, data) do
     maybe_put(map, :radiorange, get_field_value(data, :radiorange))

lib/aprsme/packet_consumer.ex

@@ -486,9 +486,10 @@ defmodule Aprsme.PacketConsumer do
   end
 
   defp object_packet?(attrs) do
+    info_field = get_in(attrs, [:data, "information_field"])
+
     attrs[:data_type] == "object" or attrs["data_type"] == "object" or
-      (is_binary(attrs[:information_field]) and
+      (is_binary(info_field) and String.starts_with?(info_field, ";"))
-         String.starts_with?(attrs[:information_field], ";"))
   end
 
   defp item_packet?(attrs) do
@@ -496,9 +497,11 @@ defmodule Aprsme.PacketConsumer do
   end
 
   defp apply_object_fields(attrs) do
+    info_field = get_in(attrs, [:data, "information_field"])
+
     object_name =
       extract_object_name(attrs) ||
-        extract_object_name_from_info_field(attrs[:information_field])
+        extract_object_name_from_info_field(info_field)
 
     attrs
     |> Map.put(:object_name, object_name)
@@ -587,7 +590,6 @@ defmodule Aprsme.PacketConsumer do
     |> Map.put_new(:base_callsign, attrs[:sender])
     |> Map.put_new(:data_type, "unknown")
     |> Map.put_new(:destination, "")
-    |> Map.put_new(:information_field, "")
     |> Map.put_new(:path, "")
     |> Map.put_new(:ssid, "")
     |> Map.put_new(:data_extended, %{})
@@ -741,9 +743,7 @@ defmodule Aprsme.PacketConsumer do
       :rain_since_midnight,
       :snow,
       :speed,
-      :altitude,
+      :altitude
-      :posresolution,
-      :rain_midnight
     ]
 
     Enum.reduce(float_fields, attrs, fn field, acc ->

lib/aprsme/packet_field_whitelist.ex

@@ -9,51 +9,34 @@ defmodule Aprsme.PacketFieldWhitelist do
   # Using string list to handle both atom and string keys consistently
   @allowed_fields ~w[
     addressee
-    alive
     altitude
     aprs_messaging
     base_callsign
-    body
     comment
     course
     dao
+    data
     data_type
     destination
     device_identifier
-    dstcallsign
     equipment_type
-    format
     has_position
     has_weather
-    header
     humidity
-    information_field
     is_item
     is_object
     item_name
     lat
     location
     lon
-    luminosity
     manufacturer
     message_number
     message_text
-    messaging
     object_name
-    origpacket
     path
-    phg_directivity
-    phg_gain
-    phg_height
-    phg_power
-    posambiguity
-    position_ambiguity
-    posresolution
     pressure
-    radiorange
     rain_1h
     rain_24h
-    rain_midnight
     rain_since_midnight
     raw_packet
     received_at
@@ -61,15 +44,9 @@ defmodule Aprsme.PacketFieldWhitelist do
     sender
     snow
     speed
-    srccallsign
     ssid
     symbol_code
     symbol_table_id
-    symbolcode
-    symboltable
-    telemetry_bits
-    telemetry_seq
-    telemetry_vals
     temperature
     timestamp
     wind_direction

lib/aprsme/packet_sanitizer.ex

@@ -19,31 +19,27 @@ defmodule Aprsme.PacketSanitizer do
     timestamp: 50,
     message_number: 20,
     addressee: 50,
-    symboltable: 5,
-    symbolcode: 5,
-    srccallsign: 20,
-    dstcallsign: 20,
 
     # These fields can be longer but still have sanity limits
     path: 500,
     manufacturer: 100,
     equipment_type: 100,
-    format: 100,
     device_identifier: 255,
     item_name: 100,
     object_name: 100,
 
-    # Very long fields - these will be TEXT in DB but we still
+    # Very long fields
-    # want to prevent abuse with extremely long data
-    information_field: 5000,
     raw_packet: 5000,
     comment: 2000,
-    message_text: 2000,
+    message_text: 2000
-    body: 5000,
+  }
-    origpacket: 5000,
+
-    header: 1000,
+  # Max lengths for string values inside the JSONB `data` map
-    radiorange: 1000,
+  @data_string_max_lengths %{
-    telemetry_bits: 1000
+    "information_field" => 5000,
+    "radiorange" => 1000,
+    "telemetry_bits" => 1000,
+    "format" => 100
   }
 
   @doc """
@@ -57,6 +53,10 @@ defmodule Aprsme.PacketSanitizer do
     end)
   end
 
+  defp sanitize_field(:data, value) when is_map(value) do
+    sanitize_data_map(value)
+  end
+
   defp sanitize_field(key, value) when is_binary(value) do
     case Map.get(@max_lengths, key) do
       nil ->
@@ -70,6 +70,24 @@ defmodule Aprsme.PacketSanitizer do
 
   defp sanitize_field(_key, value), do: value
 
+  defp sanitize_data_map(data) when is_map(data) do
+    Enum.reduce(data, %{}, fn {key, value}, acc ->
+      sanitized =
+        case {is_binary(value), Map.get(@data_string_max_lengths, key)} do
+          {true, nil} -> strip_null_bytes(value)
+          {true, max_length} -> value |> strip_null_bytes() |> truncate_string(max_length)
+          _ -> value
+        end
+
+      Map.put(acc, key, sanitized)
+    end)
+  end
+
+  # PostgreSQL JSONB does not support \u0000 (null bytes)
+  defp strip_null_bytes(string) when is_binary(string) do
+    String.replace(string, <<0x00>>, "")
+  end
+
   defp truncate_string(string, max_length) when byte_size(string) <= max_length do
     string
   end

lib/aprsme/packets/query_builder.ex

@@ -125,7 +125,7 @@ defmodule Aprsme.Packets.QueryBuilder do
         rain_24h: p.rain_24h,
         rain_since_midnight: p.rain_since_midnight,
         snow: p.snow,
-        luminosity: p.luminosity
+        luminosity: fragment("(?->>'luminosity')::integer", p.data)
       }
   end
 

lib/aprsme_web/controllers/api/v1/json/callsign_json.ex

@@ -31,7 +31,7 @@ defmodule AprsmeWeb.Api.V1.CallsignJSON do
       destination: packet.destination,
       path: packet.path,
       data_type: packet.data_type,
-      information_field: packet.information_field,
+      information_field: get_in(packet.data || %{}, ["information_field"]),
       raw_packet: sanitize_raw_packet(packet.raw_packet),
       received_at: packet.received_at,
       region: packet.region,

lib/aprsme_web/live/api_docs_live.ex

@@ -105,7 +105,7 @@ defmodule AprsmeWeb.ApiDocsLive do
             "destination" => packet.destination,
             "path" => packet.path,
             "data_type" => packet.data_type,
-            "information_field" => packet.information_field,
+            "information_field" => get_in(packet.data || %{}, ["information_field"]),
             "raw_packet" => sanitize_raw_packet(packet.raw_packet),
             "received_at" => packet.received_at,
             "region" => packet.region,

lib/aprsme_web/live/info_live/show.html.heex

@@ -143,38 +143,38 @@
                   </dd>
                 </div>
               <% end %>
-              <%= if @packet.phg_power || @packet.phg_height || @packet.phg_gain do %>
+              <%= if (@packet.data || %{})["phg_power"] || (@packet.data || %{})["phg_height"] || (@packet.data || %{})["phg_gain"] do %>
                 <div class="col-span-2">
                   <dt class="text-xs font-medium text-gray-500 dark:text-gray-400 mb-2">
                     {gettext("PHG (Power-Height-Gain)")}
                   </dt>
                   <dd class="grid grid-cols-2 gap-2 text-sm">
-                    <%= if @packet.phg_power do %>
+                    <%= if (@packet.data || %{})["phg_power"] do %>
                       <div>
                         <span class="text-gray-500 dark:text-gray-400">{gettext("Power:")}</span>
-                        <span class="font-semibold">{@packet.phg_power} W</span>
+                        <span class="font-semibold">{(@packet.data || %{})["phg_power"]} W</span>
                       </div>
                     <% end %>
-                    <%= if @packet.phg_height do %>
+                    <%= if (@packet.data || %{})["phg_height"] do %>
                       <div>
                         <span class="text-gray-500 dark:text-gray-400">{gettext("HAAT:")}</span>
-                        <span class="font-semibold">{@packet.phg_height} ft</span>
+                        <span class="font-semibold">{(@packet.data || %{})["phg_height"]} ft</span>
                       </div>
                     <% end %>
-                    <%= if @packet.phg_gain do %>
+                    <%= if (@packet.data || %{})["phg_gain"] do %>
                       <div>
                         <span class="text-gray-500 dark:text-gray-400">{gettext("Gain:")}</span>
-                        <span class="font-semibold">{@packet.phg_gain} dBi</span>
+                        <span class="font-semibold">{(@packet.data || %{})["phg_gain"]} dBi</span>
                       </div>
                     <% end %>
-                    <%= if @packet.phg_directivity do %>
+                    <%= if (@packet.data || %{})["phg_directivity"] do %>
                       <div>
                         <span class="text-gray-500 dark:text-gray-400">{gettext("Dir:")}</span>
                         <span class="font-semibold">
-                          <%= if @packet.phg_directivity == 0 or @packet.phg_directivity == 360 do %>
+                          <%= if (@packet.data || %{})["phg_directivity"] == 0 or (@packet.data || %{})["phg_directivity"] == 360 do %>
                             {gettext("Omni")}
                           <% else %>
-                            {@packet.phg_directivity}°
+                            {(@packet.data || %{})["phg_directivity"]}°
                           <% end %>
                         </span>
                       </div>

lib/aprsme_web/live/map_live/packet_utils.ex

@@ -119,7 +119,7 @@ defmodule AprsmeWeb.MapLive.PacketUtils do
       where: p.sender == ^callsign,
       where:
         fragment(
-          "? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL",
+          "? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ?->>'luminosity' IS NOT NULL OR ?->>'rain_midnight' IS NOT NULL",
           p.temperature,
           p.humidity,
           p.pressure,
@@ -128,9 +128,9 @@ defmodule AprsmeWeb.MapLive.PacketUtils do
           p.wind_gust,
           p.rain_1h,
           p.rain_24h,
-          p.rain_midnight,
+          p.snow,
-          p.luminosity,
+          p.data,
-          p.snow_24h
+          p.data
         ),
       select: fragment("1"),
       limit: 1

lib/aprsme_web/live/packets_live/callsign_view.html.heex

@@ -60,12 +60,13 @@
                   </td>
                   <td>
                     <span class="text-sm font-mono">
-                      <%= if String.length(packet.information_field || "") > 50 do %>
+                      <% info_field = (packet.data || %{})["information_field"] || "" %>
-                        <span title={packet.information_field}>
+                      <%= if String.length(info_field) > 50 do %>
-                          {String.slice(packet.information_field, 0, 50)}...
+                        <span title={info_field}>
+                          {String.slice(info_field, 0, 50)}...
                         </span>
                       <% else %>
-                        {packet.information_field}
+                        {info_field}
                       <% end %>
                     </span>
                   </td>

lib/aprsme_web/live/shared/packet_utils.ex

@@ -103,8 +103,7 @@ defmodule AprsmeWeb.Live.Shared.PacketUtils do
       :rain_1h,
       :rain_24h,
       :rain_since_midnight,
-      :snow,
+      :snow
-      :luminosity
     ]
 
     Enum.any?(weather_fields, fn field ->

lib/aprsme_web/live/weather_live/callsign_view.ex

@@ -193,7 +193,7 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
         rain_1h: rain_1h_value,
         rain_24h: rain_24h_value,
         rain_since_midnight: rain_since_midnight_value,
-        luminosity: pkt.luminosity
+        luminosity: get_in(pkt.data || %{}, ["luminosity"])
       }
     end)
     |> Jason.encode!()

priv/repo/migrations/20260220230000_consolidate_packets_columns.exs

@@ -0,0 +1,468 @@
+defmodule Aprsme.Repo.Migrations.ConsolidatePacketsColumns do
+  use Ecto.Migration
+
+  @moduledoc """
+  Consolidates the packets table by:
+  - Dropping 10 dead parser-compat columns (never read)
+  - Moving 14 display-only columns into a single `data` JSONB column
+  - Fixing the notify_packets_insert() trigger to read from `data` JSONB
+
+  The packets table is ephemeral (refills from APRS-IS stream) so data loss
+  from DROP/recreate is acceptable.
+  """
+
+  def up do
+    execute("SET LOCAL statement_timeout = '0'")
+
+    # Drop existing partitioned table
+    execute("DROP TABLE IF EXISTS packets CASCADE")
+
+    # Recreate without the 24 dropped/moved columns, plus new `data` JSONB column
+    execute("""
+    CREATE TABLE packets (
+      id uuid NOT NULL,
+      base_callsign varchar,
+      data_type varchar,
+      destination varchar,
+      path text,
+      sender varchar,
+      ssid varchar,
+      data_extended jsonb,
+      inserted_at timestamp NOT NULL,
+      updated_at timestamp NOT NULL,
+      received_at timestamp NOT NULL,
+      region varchar,
+      lat numeric,
+      lon numeric,
+      has_position boolean DEFAULT false,
+      item_name text,
+      object_name text,
+      is_item boolean DEFAULT false,
+      is_object boolean DEFAULT false,
+      dao jsonb,
+      raw_packet text,
+      symbol_code varchar,
+      symbol_table_id varchar,
+      comment text,
+      "timestamp" varchar,
+      aprs_messaging boolean DEFAULT false,
+      temperature double precision,
+      humidity double precision,
+      wind_speed double precision,
+      wind_direction integer,
+      wind_gust double precision,
+      pressure double precision,
+      rain_1h double precision,
+      rain_24h double precision,
+      rain_since_midnight double precision,
+      manufacturer text,
+      equipment_type text,
+      course integer,
+      speed double precision,
+      altitude double precision,
+      addressee text,
+      message_text text,
+      message_number varchar,
+      location geometry(Point,4326),
+      device_identifier text,
+      snow double precision,
+      has_weather boolean DEFAULT false,
+      data jsonb DEFAULT '{}'::jsonb,
+      PRIMARY KEY (id, received_at)
+    ) PARTITION BY RANGE (received_at)
+    """)
+
+    # Default partition
+    execute("CREATE TABLE packets_default PARTITION OF packets DEFAULT")
+
+    # Daily partitions for recent past + near future
+    today = Date.utc_today()
+
+    for offset <- -8..2 do
+      date = Date.add(today, offset)
+      name = "packets_#{Calendar.strftime(date, "%Y%m%d")}"
+      next_date = Date.add(date, 1)
+
+      execute(
+        "CREATE TABLE IF NOT EXISTS #{name} PARTITION OF packets FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next_date} 00:00:00')"
+      )
+    end
+
+    create_indexes()
+    update_notify_trigger()
+    create_triggers("packets")
+  end
+
+  def down do
+    execute("SET LOCAL statement_timeout = '0'")
+    execute("DROP TABLE IF EXISTS packets CASCADE")
+
+    # Recreate original table with all columns (non-partitioned for simplicity)
+    execute("""
+    CREATE TABLE packets (
+      id uuid NOT NULL PRIMARY KEY,
+      base_callsign varchar,
+      data_type varchar,
+      destination varchar,
+      information_field text,
+      path text,
+      sender varchar,
+      ssid varchar,
+      data_extended jsonb,
+      inserted_at timestamp NOT NULL,
+      updated_at timestamp NOT NULL,
+      received_at timestamp NOT NULL,
+      region varchar,
+      lat numeric,
+      lon numeric,
+      has_position boolean DEFAULT false,
+      item_name text,
+      object_name text,
+      is_item boolean DEFAULT false,
+      is_object boolean DEFAULT false,
+      dao jsonb,
+      raw_packet text,
+      symbol_code varchar,
+      symbol_table_id varchar,
+      comment text,
+      "timestamp" varchar,
+      aprs_messaging boolean DEFAULT false,
+      temperature double precision,
+      humidity double precision,
+      wind_speed double precision,
+      wind_direction integer,
+      wind_gust double precision,
+      pressure double precision,
+      rain_1h double precision,
+      rain_24h double precision,
+      rain_since_midnight double precision,
+      manufacturer text,
+      equipment_type text,
+      course integer,
+      speed double precision,
+      altitude double precision,
+      addressee text,
+      message_text text,
+      message_number varchar,
+      location geometry(Point,4326),
+      device_identifier text,
+      luminosity integer,
+      snow double precision,
+      phg_power integer,
+      phg_height integer,
+      phg_gain integer,
+      phg_directivity integer,
+      has_weather boolean DEFAULT false,
+      rain_midnight double precision,
+      position_ambiguity integer,
+      posresolution double precision,
+      format text,
+      telemetry_seq integer,
+      telemetry_vals integer[],
+      telemetry_bits text,
+      radiorange text,
+      srccallsign varchar,
+      dstcallsign varchar,
+      body text,
+      origpacket text,
+      header text,
+      alive integer DEFAULT 1,
+      posambiguity integer,
+      symboltable varchar,
+      symbolcode varchar,
+      messaging integer
+    )
+    """)
+
+    restore_notify_trigger()
+    create_triggers("packets")
+  end
+
+  defp create_indexes do
+    # Time-series
+    execute("CREATE INDEX idx_packets_received_desc ON packets (received_at DESC)")
+    execute("CREATE INDEX idx_packets_received_brin ON packets USING brin (received_at)")
+
+    # Sender/Callsign
+    execute("CREATE INDEX idx_packets_sender_received ON packets (sender, received_at DESC)")
+
+    execute(
+      "CREATE INDEX idx_packets_upper_sender ON packets (upper(sender::text), received_at DESC)"
+    )
+
+    execute("CREATE INDEX idx_packets_sender_pattern ON packets (sender text_pattern_ops)")
+
+    execute(
+      "CREATE INDEX idx_packets_base_callsign_pattern ON packets (base_callsign text_pattern_ops)"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_base_callsign_time ON packets (base_callsign, received_at DESC)"
+    )
+
+    execute("CREATE INDEX idx_packets_sender_id_desc ON packets (sender, id DESC)")
+
+    execute(
+      "CREATE INDEX idx_packets_upper_base_callsign ON packets (upper(base_callsign::text))"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_sender_trgm ON packets USING gin (sender gin_trgm_ops) WHERE sender IS NOT NULL"
+    )
+
+    # Position/Geographic
+    execute(
+      "CREATE INDEX idx_packets_positioned ON packets (received_at DESC, lat, lon, sender) WHERE has_position = true"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_position_time ON packets (has_position, received_at DESC) WHERE has_position = true"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_location ON packets USING gist (location) WHERE has_position = true"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_location_geography ON packets USING gist ((location::geography))"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_spatial_temporal ON packets (received_at DESC, base_callsign) WHERE has_position = true"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_sender_position ON packets (sender, received_at DESC) WHERE lat IS NOT NULL AND lon IS NOT NULL"
+    )
+
+    execute("CREATE INDEX idx_packets_lat ON packets (lat)")
+    execute("CREATE INDEX idx_packets_lon ON packets (lon)")
+
+    # Region
+    execute("CREATE INDEX idx_packets_region ON packets (region)")
+    execute("CREATE INDEX idx_packets_region_received ON packets (region, received_at)")
+
+    execute(
+      "CREATE INDEX idx_packets_region_position_time ON packets (region, has_position, received_at DESC) WHERE has_position = true AND region IS NOT NULL"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_region_data_type ON packets (region, data_type, received_at)"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_region_position ON packets (region, has_position, received_at)"
+    )
+
+    # Weather
+    execute(
+      "CREATE INDEX idx_packets_weather ON packets (sender, received_at DESC) WHERE temperature IS NOT NULL OR humidity IS NOT NULL OR pressure IS NOT NULL OR wind_speed IS NOT NULL OR wind_direction IS NOT NULL OR rain_1h IS NOT NULL"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_has_weather ON packets (has_weather, sender, received_at DESC) WHERE has_weather = true"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_temperature ON packets (temperature) WHERE temperature IS NOT NULL"
+    )
+
+    execute("CREATE INDEX idx_packets_humidity ON packets (humidity) WHERE humidity IS NOT NULL")
+    execute("CREATE INDEX idx_packets_pressure ON packets (pressure) WHERE pressure IS NOT NULL")
+
+    execute(
+      "CREATE INDEX idx_packets_wind_speed ON packets (wind_speed) WHERE wind_speed IS NOT NULL"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_received_temp ON packets (received_at, temperature) WHERE temperature IS NOT NULL"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_sender_temp ON packets (sender, temperature) WHERE temperature IS NOT NULL"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_weather_history ON packets (sender, data_type, received_at) WHERE data_type = 'weather'"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_weather_selective ON packets (received_at DESC) WHERE data_type IN ('weather', 'Weather', 'WX', 'wx')"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_weather_symbol ON packets (received_at DESC, lat, lon) WHERE (symbol_table_id = '/' AND symbol_code = '_') OR (symbol_table_id = E'\\\\' AND symbol_code = '_')"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_sender_weather_lookup ON packets (upper(sender::text), received_at DESC) INCLUDE (data_type, symbol_table_id, symbol_code) WHERE data_type = 'weather' OR (symbol_table_id = '/' AND symbol_code = '_') OR (symbol_table_id = E'\\\\' AND symbol_code = '_')"
+    )
+
+    # Data type & Device
+    execute(
+      "CREATE INDEX idx_packets_datatype_sender_time ON packets (data_type, sender, received_at DESC)"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_device_identifier ON packets (device_identifier) WHERE device_identifier IS NOT NULL"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_device_received ON packets (device_identifier, received_at) WHERE device_identifier IS NOT NULL"
+    )
+
+    # Path
+    execute(
+      "CREATE INDEX idx_packets_path_pattern ON packets (path text_pattern_ops) WHERE path IS NOT NULL AND path <> ''"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_path_trgm ON packets USING gin (path gin_trgm_ops) WHERE path IS NOT NULL AND path <> '' AND path !~ '^TCPIP' AND path !~ ',TCPIP'"
+    )
+
+    # Digipeater/stations heard
+    execute(
+      "CREATE INDEX idx_packets_digipeater ON packets (sender, received_at DESC) WHERE location IS NOT NULL"
+    )
+
+    execute(
+      "CREATE INDEX idx_packets_stations_heard ON packets (received_at DESC, sender, lat, lon) WHERE path IS NOT NULL AND path <> '' AND path !~ '^TCPIP' AND path !~ ',TCPIP' AND lat IS NOT NULL AND lon IS NOT NULL"
+    )
+
+    # Symbol
+    execute(
+      "CREATE INDEX idx_packets_symbol_time ON packets (symbol_table_id, symbol_code, received_at)"
+    )
+
+    # JSONB data column — GIN index for key lookups
+    execute(
+      "CREATE INDEX idx_packets_data_gin ON packets USING gin (data) WHERE data IS NOT NULL AND data <> '{}'::jsonb"
+    )
+  end
+
+  # Updated trigger reads PHG/luminosity/rain_midnight from data JSONB
+  defp update_notify_trigger do
+    execute("DROP FUNCTION IF EXISTS notify_packets_insert() CASCADE")
+
+    execute("""
+    CREATE OR REPLACE FUNCTION notify_packets_insert() RETURNS trigger AS $$
+    DECLARE
+      payload TEXT;
+    BEGIN
+      payload := json_build_object(
+        'id', NEW.id,
+        'sender', NEW.sender,
+        'ssid', NEW.ssid,
+        'base_callsign', NEW.base_callsign,
+        'lat', NEW.lat,
+        'lon', NEW.lon,
+        'altitude', NEW.altitude,
+        'course', NEW.course,
+        'speed', NEW.speed,
+        'symbol_table_id', NEW.symbol_table_id,
+        'symbol_code', NEW.symbol_code,
+        'device_identifier', NEW.device_identifier,
+        'path', NEW.path,
+        'data_type', NEW.data_type,
+        'received_at', NEW.received_at,
+        'comment', NEW.comment,
+        'raw_packet', NEW.raw_packet,
+        'phg_power', NEW.data->>'phg_power',
+        'phg_height', NEW.data->>'phg_height',
+        'phg_gain', NEW.data->>'phg_gain',
+        'phg_directivity', NEW.data->>'phg_directivity',
+        'temperature', NEW.temperature,
+        'humidity', NEW.humidity,
+        'pressure', NEW.pressure,
+        'wind_speed', NEW.wind_speed,
+        'wind_direction', NEW.wind_direction,
+        'wind_gust', NEW.wind_gust,
+        'rain_1h', NEW.rain_1h,
+        'rain_24h', NEW.rain_24h,
+        'rain_midnight', NEW.data->>'rain_midnight',
+        'luminosity', NEW.data->>'luminosity',
+        'snow', NEW.snow,
+        'has_weather', NEW.has_weather,
+        'inserted_at', NEW.inserted_at
+      )::text;
+
+      PERFORM pg_notify('aprs_packets', payload);
+
+      RETURN NEW;
+    END;
+    $$ LANGUAGE plpgsql;
+    """)
+  end
+
+  # Restore original trigger for rollback
+  defp restore_notify_trigger do
+    execute("DROP FUNCTION IF EXISTS notify_packets_insert() CASCADE")
+
+    execute("""
+    CREATE OR REPLACE FUNCTION notify_packets_insert() RETURNS trigger AS $$
+    DECLARE
+      payload TEXT;
+    BEGIN
+      payload := json_build_object(
+        'id', NEW.id,
+        'sender', NEW.sender,
+        'ssid', NEW.ssid,
+        'base_callsign', NEW.base_callsign,
+        'lat', NEW.lat,
+        'lon', NEW.lon,
+        'altitude', NEW.altitude,
+        'course', NEW.course,
+        'speed', NEW.speed,
+        'symbol_table_id', NEW.symbol_table_id,
+        'symbol_code', NEW.symbol_code,
+        'device_identifier', NEW.device_identifier,
+        'path', NEW.path,
+        'data_type', NEW.data_type,
+        'received_at', NEW.received_at,
+        'comment', NEW.comment,
+        'raw_packet', NEW.raw_packet,
+        'phg_power', NEW.phg_power,
+        'phg_height', NEW.phg_height,
+        'phg_gain', NEW.phg_gain,
+        'phg_directivity', NEW.phg_directivity,
+        'temperature', NEW.temperature,
+        'humidity', NEW.humidity,
+        'pressure', NEW.pressure,
+        'wind_speed', NEW.wind_speed,
+        'wind_direction', NEW.wind_direction,
+        'wind_gust', NEW.wind_gust,
+        'rain_1h', NEW.rain_1h,
+        'rain_24h', NEW.rain_24h,
+        'rain_midnight', NEW.rain_midnight,
+        'luminosity', NEW.luminosity,
+        'snow', NEW.snow,
+        'has_weather', NEW.has_weather,
+        'inserted_at', NEW.inserted_at
+      )::text;
+
+      PERFORM pg_notify('aprs_packets', payload);
+
+      RETURN NEW;
+    END;
+    $$ LANGUAGE plpgsql;
+    """)
+  end
+
+  defp create_triggers(table) do
+    execute(
+      "CREATE TRIGGER update_has_weather_trigger BEFORE INSERT OR UPDATE ON #{table} FOR EACH ROW EXECUTE FUNCTION update_has_weather()"
+    )
+
+    execute(
+      "CREATE TRIGGER packets_notify_insert AFTER INSERT ON #{table} FOR EACH ROW EXECUTE FUNCTION notify_packets_insert()"
+    )
+
+    execute(
+      "CREATE TRIGGER packet_insert_sequence AFTER INSERT ON #{table} FOR EACH ROW EXECUTE FUNCTION increment_packet_sequence()"
+    )
+
+    execute(
+      "CREATE TRIGGER packet_delete_sequence AFTER DELETE ON #{table} FOR EACH ROW EXECUTE FUNCTION decrement_packet_sequence()"
+    )
+  end
+end

test/aprsme/encoding_utils_test.exs

@@ -41,7 +41,7 @@ defmodule Aprsme.EncodingUtilsTest do
   end
 
   describe "sanitize_packet/1" do
-    test "sanitizes information_field" do
+    test "sanitizes string values in data map" do
       invalid_info = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>
 
       packet = %Packet{
@@ -49,18 +49,20 @@ defmodule Aprsme.EncodingUtilsTest do
         sender: "TEST-1",
         path: "TCPIP*",
         destination: "APX100",
-        information_field: invalid_info,
         data_type: :position,
         base_callsign: "TEST",
         ssid: "1",
+        data: %{"information_field" => invalid_info, "radiorange" => "0050"},
         data_extended: %{comment: "Valid comment"}
       }
 
       sanitized = EncodingUtils.sanitize_packet(packet)
 
-      assert String.valid?(sanitized.information_field)
+      assert String.valid?(sanitized.data["information_field"])
-      assert String.contains?(sanitized.information_field, "Hello")
+      assert String.contains?(sanitized.data["information_field"], "Hello")
-      assert String.contains?(sanitized.information_field, "World")
+      assert String.contains?(sanitized.data["information_field"], "World")
+      # Non-string values in data map are preserved
+      assert sanitized.data["radiorange"] == "0050"
     end
 
     test "sanitizes comment in data_extended" do
@@ -71,7 +73,6 @@ defmodule Aprsme.EncodingUtilsTest do
         sender: "TEST-1",
         path: "TCPIP*",
         destination: "APX100",
-        information_field: "Valid info",
         data_type: :position,
         base_callsign: "TEST",
         ssid: "1",
@@ -90,7 +91,6 @@ defmodule Aprsme.EncodingUtilsTest do
         sender: "TEST-1",
         path: "TCPIP*",
         destination: "APX100",
-        information_field: "Valid info",
         data_type: :position,
         base_callsign: "TEST",
         ssid: "1",
@@ -100,7 +100,6 @@ defmodule Aprsme.EncodingUtilsTest do
       sanitized = EncodingUtils.sanitize_packet(packet)
 
       assert sanitized.data_extended == nil
-      assert sanitized.information_field == "Valid info"
     end
   end
 

test/aprsme/enhanced_parser_test.exs

@@ -4,7 +4,7 @@ defmodule Aprsme.EnhancedParserTest do
   alias Aprsme.Packet
 
   describe "enhanced parser field extraction" do
-    test "extracts standard parser compatibility fields" do
+    test "dead parser compat fields are dropped (not preserved)" do
       attrs = %{
         srccallsign: "W1AW",
         dstcallsign: "APRS",
@@ -26,19 +26,20 @@ defmodule Aprsme.EnhancedParserTest do
 
       result = Packet.extract_additional_data(attrs, "W1AW>APRS:!1234.56N/12345.67W-Test")
 
-      assert result[:srccallsign] == "W1AW"
+      # Dead fields should not be in result
-      assert result[:dstcallsign] == "APRS"
+      refute Map.has_key?(result, :srccallsign)
-      assert result[:body] == "!1234.56N/12345.67W-Test"
+      refute Map.has_key?(result, :dstcallsign)
-      assert result[:origpacket] == "W1AW>APRS:!1234.56N/12345.67W-Test"
+      refute Map.has_key?(result, :body)
-      assert result[:header] == "W1AW>APRS"
+      refute Map.has_key?(result, :origpacket)
-      assert result[:alive] == 1
+      refute Map.has_key?(result, :header)
-      assert result[:posambiguity] == 0
+      refute Map.has_key?(result, :alive)
-      assert result[:symboltable] == "/"
+      refute Map.has_key?(result, :posambiguity)
-      assert result[:symbolcode] == "-"
+      refute Map.has_key?(result, :symboltable)
-      assert result[:messaging] == 0
+      refute Map.has_key?(result, :symbolcode)
+      refute Map.has_key?(result, :messaging)
     end
 
-    test "extracts radio range field" do
+    test "extracts radio range into data map" do
       attrs = %{
         radiorange: "0050",
         sender: "W1AW",
@@ -51,7 +52,7 @@ defmodule Aprsme.EnhancedParserTest do
 
       result = Packet.extract_additional_data(attrs)
 
-      assert result[:radiorange] == "0050"
+      assert result[:data]["radiorange"] == "0050"
     end
 
     test "handles weather data from wx field" do
@@ -82,7 +83,7 @@ defmodule Aprsme.EnhancedParserTest do
       assert result[:wind_direction] == 180
     end
 
-    test "handles PHG data in string format" do
+    test "handles PHG data in string format and moves to data map" do
       attrs = %{
         sender: "W1AW",
         base_callsign: "W1AW",
@@ -98,13 +99,14 @@ defmodule Aprsme.EnhancedParserTest do
       result = Packet.extract_additional_data(attrs)
 
       # PHG 5430: power=5^2=25W, height=4->160ft, gain=3dB, dir=0->omni
-      assert result[:phg_power] == 25
+      # PHG fields should be in the data map
-      assert result[:phg_height] == 160
+      assert result[:data]["phg_power"] == 25
-      assert result[:phg_gain] == 3
+      assert result[:data]["phg_height"] == 160
-      assert result[:phg_directivity] == 0
+      assert result[:data]["phg_gain"] == 3
+      assert result[:data]["phg_directivity"] == 0
     end
 
-    test "handles PHG data in legacy map format" do
+    test "handles PHG data in legacy map format and moves to data map" do
       attrs = %{
         sender: "W1AW",
         base_callsign: "W1AW",
@@ -124,15 +126,103 @@ defmodule Aprsme.EnhancedParserTest do
 
       result = Packet.extract_additional_data(attrs)
 
-      assert result[:phg_power] == 25
+      assert result[:data]["phg_power"] == 25
-      assert result[:phg_height] == 160
+      assert result[:data]["phg_height"] == 160
-      assert result[:phg_gain] == 3
+      assert result[:data]["phg_gain"] == 3
-      assert result[:phg_directivity] == 0
+      assert result[:data]["phg_directivity"] == 0
+    end
+
+    test "collect_into_data_map sweeps all 14 fields into data" do
+      attrs = %{
+        sender: "W1AW",
+        base_callsign: "W1AW",
+        ssid: "0",
+        destination: "APRS",
+        data_type: "position",
+        received_at: DateTime.utc_now(),
+        data_extended: %{
+          phg: "5430",
+          format: :uncompressed,
+          posresolution: 18.52,
+          position_ambiguity: 0,
+          comment: "Test PHG5430"
+        }
+      }
+
+      result = Packet.extract_additional_data(attrs)
+
+      # These fields should be in the data map, not at top level
+      assert is_map(result[:data])
+      assert result[:data]["phg_power"] == 25
+      assert result[:data]["format"] == "uncompressed"
+      assert result[:data]["posresolution"] == 18.52
+      assert result[:data]["position_ambiguity"] == 0
+
+      # They should NOT be at top level
+      refute Map.has_key?(result, :phg_power)
+      refute Map.has_key?(result, :format)
+      refute Map.has_key?(result, :posresolution)
+      refute Map.has_key?(result, :position_ambiguity)
+    end
+
+    test "telemetry fields are moved into data map" do
+      attrs = %{
+        sender: "W1AW",
+        base_callsign: "W1AW",
+        ssid: "0",
+        destination: "APRS",
+        data_type: "telemetry",
+        received_at: DateTime.utc_now(),
+        data_extended: %{
+          telemetry: %{
+            seq: 123,
+            vals: [1, 2, 3, 4, 5],
+            bits: "11111111"
+          }
+        }
+      }
+
+      result = Packet.extract_additional_data(attrs)
+
+      assert result[:data]["telemetry_seq"] == 123
+      assert result[:data]["telemetry_vals"] == [1, 2, 3, 4, 5]
+      assert result[:data]["telemetry_bits"] == "11111111"
+
+      refute Map.has_key?(result, :telemetry_seq)
+      refute Map.has_key?(result, :telemetry_vals)
+      refute Map.has_key?(result, :telemetry_bits)
+    end
+
+    test "luminosity and rain_midnight are moved into data map" do
+      attrs = %{
+        sender: "WX1STN",
+        base_callsign: "WX1STN",
+        ssid: "0",
+        destination: "APRS",
+        data_type: "weather",
+        received_at: DateTime.utc_now(),
+        data_extended: %{
+          wx: %{
+            temperature: 72.5,
+            luminosity: 500,
+            rain_midnight: 0.5
+          }
+        }
+      }
+
+      result = Packet.extract_additional_data(attrs)
+
+      # luminosity and rain_midnight should be in data map
+      assert result[:data]["luminosity"] == 500
+      assert result[:data]["rain_midnight"] == 0.5
+
+      # temperature stays at top level
+      assert result[:temperature] == 72.5
     end
   end
 
   describe "enhanced parser packet storage" do
-    test "stores packet with enhanced parser fields" do
+    test "stores packet with data map fields" do
       attrs = %{
         sender: "W1AW-1",
         base_callsign: "W1AW",
@@ -146,36 +236,17 @@ defmodule Aprsme.EnhancedParserTest do
         symbol_code: "-",
         symbol_table_id: "/",
         comment: "Test station",
-        # Enhanced parser fields
+        radiorange: "0050",
-        srccallsign: "W1AW-1",
+        information_field: "!1234.56N/12345.67W-Test"
-        dstcallsign: "APRS",
-        body: "!1234.56N/12345.67W-Test RNG0050",
-        origpacket: "W1AW-1>APRS:!1234.56N/12345.67W-Test RNG0050",
-        header: "W1AW-1>APRS",
-        alive: 1,
-        posambiguity: 0,
-        symboltable: "/",
-        symbolcode: "-",
-        messaging: 0,
-        radiorange: "0050"
       }
 
       processed_attrs = Packet.extract_additional_data(attrs)
 
       assert {:ok, packet} = Aprsme.Packets.store_packet(processed_attrs)
 
-      # Verify enhanced fields are stored
+      # radiorange and information_field should be in data map
-      assert packet.srccallsign == "W1AW-1"
+      assert packet.data["radiorange"] == "0050"
-      assert packet.dstcallsign == "APRS"
+      assert packet.data["information_field"] == "!1234.56N/12345.67W-Test"
-      assert packet.body == "!1234.56N/12345.67W-Test RNG0050"
-      assert packet.origpacket == "W1AW-1>APRS:!1234.56N/12345.67W-Test RNG0050"
-      assert packet.header == "W1AW-1>APRS"
-      assert packet.alive == 1
-      assert packet.posambiguity == 0
-      assert packet.symboltable == "/"
-      assert packet.symbolcode == "-"
-      assert packet.messaging == 0
-      assert packet.radiorange == "0050"
     end
   end
 end

test/aprsme/packet_parsing_test.exs

@@ -22,15 +22,15 @@ defmodule Aprsme.PacketParsingTest do
       # Verify altitude was extracted
       assert extracted_attrs[:altitude] == 680.0
 
-      # Verify PHG data was extracted into individual fields
+      # Verify PHG data was extracted into the data map
       # PHG5 = 25W (5^2)
-      assert extracted_attrs[:phg_power] == 25
+      assert extracted_attrs.data["phg_power"] == 25
       # PHG x5x = 320 ft
-      assert extracted_attrs[:phg_height] == 320
+      assert extracted_attrs.data["phg_height"] == 320
       # PHG xx3 = 3 dBi
-      assert extracted_attrs[:phg_gain] == 3
+      assert extracted_attrs.data["phg_gain"] == 3
       # PHG xxx0 = 0 degrees (not 360 for omni in our implementation)
-      assert extracted_attrs[:phg_directivity] == 0
+      assert extracted_attrs.data["phg_directivity"] == 0
 
       # Verify comment has altitude and PHG removed
       assert extracted_attrs[:comment] == "Collin Cty Wide Digi"
@@ -73,10 +73,10 @@ defmodule Aprsme.PacketParsingTest do
       # Check that fields were properly set
       changes = changeset.changes
       assert changes.altitude == 680.0
-      assert changes.phg_power == 25
+      assert changes.data["phg_power"] == 25
-      assert changes.phg_height == 320
+      assert changes.data["phg_height"] == 320
-      assert changes.phg_gain == 3
+      assert changes.data["phg_gain"] == 3
-      assert changes.phg_directivity == 0
+      assert changes.data["phg_directivity"] == 0
       assert changes.comment == "Collin Cty Wide Digi"
     end
   end

test/aprsme/packet_test.exs

@@ -19,10 +19,10 @@ defmodule Aprsme.PacketTest do
 
       result = Packet.extract_additional_data(attrs, raw_packet)
 
-      assert result.telemetry_seq == 5
+      assert result.data["telemetry_seq"] == 5
       # 12.80 rounds to 13
-      assert result.telemetry_vals == [13, 0, 0, 0, 0]
+      assert result.data["telemetry_vals"] == [13, 0, 0, 0, 0]
-      assert result.telemetry_bits == "00000000"
+      assert result.data["telemetry_bits"] == "00000000"
     end
 
     test "handles mixed integer and string telemetry_vals" do
@@ -38,10 +38,10 @@ defmodule Aprsme.PacketTest do
 
       result = Packet.extract_additional_data(attrs, "test_packet")
 
-      assert result.telemetry_seq == 10
+      assert result.data["telemetry_seq"] == 10
       # Mixed types converted properly
-      assert result.telemetry_vals == [180, 38, 0, 88, 165]
+      assert result.data["telemetry_vals"] == [180, 38, 0, 88, 165]
-      assert result.telemetry_bits == "10101010"
+      assert result.data["telemetry_bits"] == "10101010"
     end
 
     test "handles float telemetry_vals" do
@@ -56,9 +56,9 @@ defmodule Aprsme.PacketTest do
 
       result = Packet.extract_additional_data(attrs, "test_packet")
 
-      assert result.telemetry_seq == 1
+      assert result.data["telemetry_seq"] == 1
       # Floats rounded to integers
-      assert result.telemetry_vals == [13, 37, 0, 89, 165]
+      assert result.data["telemetry_vals"] == [13, 37, 0, 89, 165]
     end
 
     test "handles invalid telemetry_vals gracefully" do
@@ -73,9 +73,9 @@ defmodule Aprsme.PacketTest do
 
       result = Packet.extract_additional_data(attrs, "test_packet")
 
-      assert result.telemetry_seq == 2
+      assert result.data["telemetry_seq"] == 2
       # Invalid values become 0
-      assert result.telemetry_vals == [0, 0, 0, 13, 0]
+      assert result.data["telemetry_vals"] == [0, 0, 0, 13, 0]
     end
 
     test "handles missing telemetry data" do
@@ -86,9 +86,9 @@ defmodule Aprsme.PacketTest do
 
       result = Packet.extract_additional_data(attrs, "test_packet")
 
-      assert Map.get(result, :telemetry_seq) == nil
+      assert get_in(result, [:data, "telemetry_seq"]) == nil
-      assert Map.get(result, :telemetry_vals) == nil
+      assert get_in(result, [:data, "telemetry_vals"]) == nil
-      assert Map.get(result, :telemetry_bits) == nil
+      assert get_in(result, [:data, "telemetry_bits"]) == nil
     end
   end
 end

test/aprsme/packets_encoding_test.exs

@@ -21,7 +21,7 @@ defmodule Aprsme.PacketsEncodingTest do
       }
 
       assert {:ok, stored_packet} = Packets.store_packet(packet_data)
-      assert stored_packet.information_field == "Valid UTF-8: ñ, é, 中文, 🚀"
+      assert stored_packet.data["information_field"] == "Valid UTF-8: ñ, é, 中文, 🚀"
       assert stored_packet.comment == "More UTF-8: αβγ, русский"
     end
 
@@ -68,8 +68,6 @@ defmodule Aprsme.PacketsEncodingTest do
       }
 
       assert {:ok, stored_packet} = Packets.store_packet(packet_data)
-      # Empty strings may be stored as nil or empty string depending on Ecto behavior
-      assert stored_packet.information_field in [nil, ""]
       assert stored_packet.path in [nil, ""]
       assert is_nil(stored_packet.comment)
     end
@@ -90,8 +88,8 @@ defmodule Aprsme.PacketsEncodingTest do
       }
 
       assert {:ok, stored_packet} = Packets.store_packet(packet_data)
-      # Control characters should be filtered out
+      # Control characters should be filtered out, field is now in data map
-      assert stored_packet.information_field == "Control chars:"
+      assert stored_packet.data["information_field"] == "Control chars:"
     end
   end