Consolidate 24 packet columns into JSONB data field
Drop 10 dead parser-compat columns (srccallsign, dstcallsign, origpacket, body, header, alive, posambiguity, symboltable, symbolcode, messaging) and move 14 display-only columns into a single `data` JSONB column (PHG, telemetry, radiorange, information_field, format, posresolution, position_ambiguity, luminosity, rain_midnight). Reduces row width from ~70 to ~48 columns. Table is ephemeral so migration uses DROP/recreate. Also fixes snow_24h -> snow bug in weather check query and adds null byte stripping for JSONB string sanitization.
This commit is contained in:
parent
0e53bc475d
commit
ca05b497b9
19 changed files with 778 additions and 282 deletions
|
|
@ -201,8 +201,7 @@ defmodule Aprsme.EncodingUtils do
|
|||
:rain_1h,
|
||||
:rain_24h,
|
||||
:rain_since_midnight,
|
||||
:snow,
|
||||
:luminosity
|
||||
:snow
|
||||
]
|
||||
end
|
||||
|
||||
|
|
@ -211,71 +210,83 @@ defmodule Aprsme.EncodingUtils do
|
|||
|
||||
## Examples
|
||||
|
||||
iex> result = Aprsme.EncodingUtils.sanitize_packet(%{"information_field" => <<0, 65, 66, 67>>, "data_extended" => %{"comment" => <<0, 68, 69, 70>>}})
|
||||
iex> result["information_field"] == "ABC"
|
||||
iex> result = Aprsme.EncodingUtils.sanitize_packet(%{"comment" => <<0, 65, 66, 67>>, "data_extended" => %{"comment" => <<0, 68, 69, 70>>}})
|
||||
iex> result["comment"] == "ABC"
|
||||
true
|
||||
iex> result["data_extended"]["comment"] == "DEF"
|
||||
true
|
||||
"""
|
||||
@spec sanitize_packet(struct() | map()) :: struct() | map()
|
||||
def sanitize_packet(%Aprsme.Packet{} = packet) do
|
||||
sanitized_data =
|
||||
case packet.data do
|
||||
nil -> nil
|
||||
data when is_map(data) -> sanitize_packet_strings(data)
|
||||
other -> other
|
||||
end
|
||||
|
||||
%{
|
||||
packet
|
||||
| information_field: sanitize_string(packet.information_field),
|
||||
| data: sanitized_data,
|
||||
data_extended: sanitize_data_extended(packet.data_extended)
|
||||
}
|
||||
end
|
||||
|
||||
def sanitize_packet(packet) when is_map(packet) do
|
||||
# Handle all known string fields, checking for both atom and string keys
|
||||
string_fields = [
|
||||
:information_field,
|
||||
:comment,
|
||||
:path,
|
||||
:raw_packet,
|
||||
:destination,
|
||||
:sender,
|
||||
:base_callsign,
|
||||
:ssid,
|
||||
:manufacturer,
|
||||
:equipment_type,
|
||||
:message_text,
|
||||
:addressee,
|
||||
:symbol_code,
|
||||
:symbol_table_id,
|
||||
:dao,
|
||||
:timestamp,
|
||||
:device_identifier
|
||||
]
|
||||
packet
|
||||
|> sanitize_string_fields()
|
||||
|> sanitize_nested_map(:data, &sanitize_packet_strings/1)
|
||||
|> sanitize_nested_map(:data_extended, &sanitize_data_extended/1)
|
||||
end
|
||||
|
||||
# Sanitize all string fields
|
||||
sanitized =
|
||||
Enum.reduce(string_fields, packet, fn field, acc ->
|
||||
atom_key = field
|
||||
string_key = to_string(field)
|
||||
@string_fields [
|
||||
:comment,
|
||||
:path,
|
||||
:raw_packet,
|
||||
:destination,
|
||||
:sender,
|
||||
:base_callsign,
|
||||
:ssid,
|
||||
:manufacturer,
|
||||
:equipment_type,
|
||||
:message_text,
|
||||
:addressee,
|
||||
:symbol_code,
|
||||
:symbol_table_id,
|
||||
:dao,
|
||||
:timestamp,
|
||||
:device_identifier
|
||||
]
|
||||
|
||||
cond do
|
||||
Map.has_key?(acc, atom_key) ->
|
||||
Map.update(acc, atom_key, nil, &sanitize_string/1)
|
||||
defp sanitize_string_fields(packet) do
|
||||
Enum.reduce(@string_fields, packet, fn field, acc ->
|
||||
string_key = to_string(field)
|
||||
|
||||
Map.has_key?(acc, string_key) ->
|
||||
Map.update(acc, string_key, nil, &sanitize_string/1)
|
||||
cond do
|
||||
Map.has_key?(acc, field) ->
|
||||
Map.update(acc, field, nil, &sanitize_string/1)
|
||||
|
||||
true ->
|
||||
acc
|
||||
end
|
||||
end)
|
||||
Map.has_key?(acc, string_key) ->
|
||||
Map.update(acc, string_key, nil, &sanitize_string/1)
|
||||
|
||||
true ->
|
||||
acc
|
||||
end
|
||||
end)
|
||||
end
|
||||
|
||||
defp sanitize_nested_map(packet, key, sanitizer) do
|
||||
string_key = to_string(key)
|
||||
|
||||
# Handle data_extended separately
|
||||
cond do
|
||||
Map.has_key?(sanitized, :data_extended) ->
|
||||
Map.update(sanitized, :data_extended, nil, &sanitize_data_extended/1)
|
||||
Map.has_key?(packet, key) ->
|
||||
Map.update(packet, key, nil, &if(is_nil(&1), do: nil, else: sanitizer.(&1)))
|
||||
|
||||
Map.has_key?(sanitized, "data_extended") ->
|
||||
Map.update(sanitized, "data_extended", nil, &sanitize_data_extended/1)
|
||||
Map.has_key?(packet, string_key) ->
|
||||
Map.update(packet, string_key, nil, &if(is_nil(&1), do: nil, else: sanitizer.(&1)))
|
||||
|
||||
true ->
|
||||
sanitized
|
||||
packet
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@ defmodule Aprsme.Packet do
|
|||
field(:base_callsign, :string)
|
||||
field(:data_type, :string)
|
||||
field(:destination, :string)
|
||||
field(:information_field, :string)
|
||||
field(:path, :string)
|
||||
field(:sender, :string)
|
||||
field(:ssid, :string)
|
||||
|
|
@ -45,55 +44,22 @@ defmodule Aprsme.Packet do
|
|||
field(:snow, :float)
|
||||
|
||||
# Equipment/status information
|
||||
field(:luminosity, :integer)
|
||||
field(:manufacturer, :string)
|
||||
field(:equipment_type, :string)
|
||||
field(:course, :integer)
|
||||
field(:speed, :float)
|
||||
field(:altitude, :float)
|
||||
|
||||
# Position ambiguity level (0-4)
|
||||
field(:position_ambiguity, :integer)
|
||||
|
||||
# Position resolution and format
|
||||
field(:posresolution, :float)
|
||||
field(:format, :string)
|
||||
|
||||
# PHG (Power-Height-Gain) fields
|
||||
field(:phg_power, :integer)
|
||||
field(:phg_height, :integer)
|
||||
field(:phg_gain, :integer)
|
||||
field(:phg_directivity, :integer)
|
||||
|
||||
# Message-specific fields
|
||||
field(:addressee, :string)
|
||||
field(:message_text, :string)
|
||||
field(:message_number, :string)
|
||||
|
||||
# Telemetry fields
|
||||
field(:telemetry_seq, :integer)
|
||||
field(:telemetry_vals, {:array, :integer})
|
||||
field(:telemetry_bits, :string)
|
||||
|
||||
# Radio range field
|
||||
field(:radiorange, :string)
|
||||
|
||||
# Standard parser compatibility fields
|
||||
field(:srccallsign, :string)
|
||||
field(:dstcallsign, :string)
|
||||
field(:body, :string)
|
||||
field(:origpacket, :string)
|
||||
field(:header, :string)
|
||||
field(:alive, :integer, default: 1)
|
||||
field(:posambiguity, :integer)
|
||||
field(:symboltable, :string)
|
||||
field(:symbolcode, :string)
|
||||
field(:messaging, :integer)
|
||||
|
||||
# Additional weather fields
|
||||
field(:rain_midnight, :float)
|
||||
field(:has_weather, :boolean, default: false)
|
||||
|
||||
# JSONB column for display-only fields (PHG, telemetry, radiorange, etc.)
|
||||
field(:data, :map, default: %{})
|
||||
|
||||
field(:device_identifier, :string)
|
||||
|
||||
# APRS Items/Objects support
|
||||
|
|
@ -123,7 +89,6 @@ defmodule Aprsme.Packet do
|
|||
:base_callsign,
|
||||
:data_type,
|
||||
:destination,
|
||||
:information_field,
|
||||
:path,
|
||||
:sender,
|
||||
:ssid,
|
||||
|
|
@ -149,38 +114,16 @@ defmodule Aprsme.Packet do
|
|||
:rain_24h,
|
||||
:rain_since_midnight,
|
||||
:snow,
|
||||
:luminosity,
|
||||
:manufacturer,
|
||||
:equipment_type,
|
||||
:course,
|
||||
:speed,
|
||||
:altitude,
|
||||
:position_ambiguity,
|
||||
:posresolution,
|
||||
:format,
|
||||
:phg_power,
|
||||
:phg_height,
|
||||
:phg_gain,
|
||||
:phg_directivity,
|
||||
:addressee,
|
||||
:message_text,
|
||||
:message_number,
|
||||
:telemetry_seq,
|
||||
:telemetry_vals,
|
||||
:telemetry_bits,
|
||||
:radiorange,
|
||||
:srccallsign,
|
||||
:dstcallsign,
|
||||
:body,
|
||||
:origpacket,
|
||||
:header,
|
||||
:alive,
|
||||
:posambiguity,
|
||||
:symboltable,
|
||||
:symbolcode,
|
||||
:messaging,
|
||||
:rain_midnight,
|
||||
:has_weather,
|
||||
:data,
|
||||
:device_identifier,
|
||||
:item_name,
|
||||
:object_name,
|
||||
|
|
@ -345,8 +288,12 @@ defmodule Aprsme.Packet do
|
|||
%{}
|
||||
end
|
||||
|
||||
# Merge all extracted data
|
||||
merge_extracted_data(base_attrs, additional_data, attrs)
|
||||
# Merge all extracted data, then sweep display-only fields into data map
|
||||
# and strip dead parser compat fields
|
||||
base_attrs
|
||||
|> merge_extracted_data(additional_data, attrs)
|
||||
|> collect_into_data_map()
|
||||
|> strip_dead_fields()
|
||||
end
|
||||
|
||||
defp add_symbol_data(base_attrs, attrs) do
|
||||
|
|
@ -366,16 +313,39 @@ defmodule Aprsme.Packet do
|
|||
put_telemetry_fields(%{}, attrs)
|
||||
end
|
||||
|
||||
# Process standard parser fields and radio range from top-level attrs
|
||||
parser_data =
|
||||
%{}
|
||||
|> put_standard_parser_fields(attrs)
|
||||
|> put_radio_range_field(attrs)
|
||||
# Process radio range from top-level attrs
|
||||
radio_range_data = put_radio_range_field(%{}, attrs)
|
||||
|
||||
base_attrs
|
||||
|> Map.merge(additional_data)
|
||||
|> Map.merge(telemetry_data)
|
||||
|> Map.merge(parser_data)
|
||||
|> Map.merge(radio_range_data)
|
||||
end
|
||||
|
||||
# Fields that get swept into the JSONB `data` column
|
||||
@data_fields ~w[telemetry_seq telemetry_vals telemetry_bits phg_power phg_height
|
||||
phg_gain phg_directivity radiorange information_field format posresolution
|
||||
position_ambiguity luminosity rain_midnight]a
|
||||
|
||||
defp collect_into_data_map(attrs) do
|
||||
{data_values, remaining} =
|
||||
Enum.reduce(@data_fields, {%{}, attrs}, fn field, {data, rest} ->
|
||||
case Map.pop(rest, field) do
|
||||
{nil, rest} -> {data, rest}
|
||||
{value, rest} -> {Map.put(data, to_string(field), value), rest}
|
||||
end
|
||||
end)
|
||||
|
||||
existing_data = Map.get(remaining, :data, %{}) || %{}
|
||||
Map.put(remaining, :data, Map.merge(existing_data, data_values))
|
||||
end
|
||||
|
||||
# Dead parser compat fields — never read, safe to strip
|
||||
@dead_fields ~w[srccallsign dstcallsign origpacket body header alive
|
||||
posambiguity symboltable symbolcode messaging]a
|
||||
|
||||
defp strip_dead_fields(attrs) do
|
||||
Map.drop(attrs, @dead_fields)
|
||||
end
|
||||
|
||||
# Extract data from standard map-based data_extended
|
||||
|
|
@ -407,7 +377,6 @@ defmodule Aprsme.Packet do
|
|||
result =
|
||||
%{}
|
||||
|> put_symbol_fields(combined_data)
|
||||
|> put_standard_parser_fields(combined_data)
|
||||
|> extract_weather_data(combined_data)
|
||||
|> put_weather_fields(combined_data)
|
||||
|> put_equipment_fields(combined_data)
|
||||
|
|
@ -861,21 +830,6 @@ defmodule Aprsme.Packet do
|
|||
defp normalize_format_field(format) when is_binary(format), do: format
|
||||
defp normalize_format_field(_), do: nil
|
||||
|
||||
# Extract standard parser compatibility fields
|
||||
defp put_standard_parser_fields(map, data) do
|
||||
map
|
||||
|> maybe_put(:srccallsign, get_field_value(data, :srccallsign))
|
||||
|> maybe_put(:dstcallsign, get_field_value(data, :dstcallsign))
|
||||
|> maybe_put(:body, get_field_value(data, :body))
|
||||
|> maybe_put(:origpacket, get_field_value(data, :origpacket))
|
||||
|> maybe_put(:header, get_field_value(data, :header))
|
||||
|> maybe_put(:alive, get_field_value(data, :alive))
|
||||
|> maybe_put(:posambiguity, get_field_value(data, :posambiguity))
|
||||
|> maybe_put(:symboltable, get_field_value(data, :symboltable))
|
||||
|> maybe_put(:symbolcode, get_field_value(data, :symbolcode))
|
||||
|> maybe_put(:messaging, get_field_value(data, :messaging))
|
||||
end
|
||||
|
||||
# Extract radio range field
|
||||
defp put_radio_range_field(map, data) do
|
||||
maybe_put(map, :radiorange, get_field_value(data, :radiorange))
|
||||
|
|
|
|||
|
|
@ -486,9 +486,10 @@ defmodule Aprsme.PacketConsumer do
|
|||
end
|
||||
|
||||
defp object_packet?(attrs) do
|
||||
info_field = get_in(attrs, [:data, "information_field"])
|
||||
|
||||
attrs[:data_type] == "object" or attrs["data_type"] == "object" or
|
||||
(is_binary(attrs[:information_field]) and
|
||||
String.starts_with?(attrs[:information_field], ";"))
|
||||
(is_binary(info_field) and String.starts_with?(info_field, ";"))
|
||||
end
|
||||
|
||||
defp item_packet?(attrs) do
|
||||
|
|
@ -496,9 +497,11 @@ defmodule Aprsme.PacketConsumer do
|
|||
end
|
||||
|
||||
defp apply_object_fields(attrs) do
|
||||
info_field = get_in(attrs, [:data, "information_field"])
|
||||
|
||||
object_name =
|
||||
extract_object_name(attrs) ||
|
||||
extract_object_name_from_info_field(attrs[:information_field])
|
||||
extract_object_name_from_info_field(info_field)
|
||||
|
||||
attrs
|
||||
|> Map.put(:object_name, object_name)
|
||||
|
|
@ -587,7 +590,6 @@ defmodule Aprsme.PacketConsumer do
|
|||
|> Map.put_new(:base_callsign, attrs[:sender])
|
||||
|> Map.put_new(:data_type, "unknown")
|
||||
|> Map.put_new(:destination, "")
|
||||
|> Map.put_new(:information_field, "")
|
||||
|> Map.put_new(:path, "")
|
||||
|> Map.put_new(:ssid, "")
|
||||
|> Map.put_new(:data_extended, %{})
|
||||
|
|
@ -741,9 +743,7 @@ defmodule Aprsme.PacketConsumer do
|
|||
:rain_since_midnight,
|
||||
:snow,
|
||||
:speed,
|
||||
:altitude,
|
||||
:posresolution,
|
||||
:rain_midnight
|
||||
:altitude
|
||||
]
|
||||
|
||||
Enum.reduce(float_fields, attrs, fn field, acc ->
|
||||
|
|
|
|||
|
|
@ -9,51 +9,34 @@ defmodule Aprsme.PacketFieldWhitelist do
|
|||
# Using string list to handle both atom and string keys consistently
|
||||
@allowed_fields ~w[
|
||||
addressee
|
||||
alive
|
||||
altitude
|
||||
aprs_messaging
|
||||
base_callsign
|
||||
body
|
||||
comment
|
||||
course
|
||||
dao
|
||||
data
|
||||
data_type
|
||||
destination
|
||||
device_identifier
|
||||
dstcallsign
|
||||
equipment_type
|
||||
format
|
||||
has_position
|
||||
has_weather
|
||||
header
|
||||
humidity
|
||||
information_field
|
||||
is_item
|
||||
is_object
|
||||
item_name
|
||||
lat
|
||||
location
|
||||
lon
|
||||
luminosity
|
||||
manufacturer
|
||||
message_number
|
||||
message_text
|
||||
messaging
|
||||
object_name
|
||||
origpacket
|
||||
path
|
||||
phg_directivity
|
||||
phg_gain
|
||||
phg_height
|
||||
phg_power
|
||||
posambiguity
|
||||
position_ambiguity
|
||||
posresolution
|
||||
pressure
|
||||
radiorange
|
||||
rain_1h
|
||||
rain_24h
|
||||
rain_midnight
|
||||
rain_since_midnight
|
||||
raw_packet
|
||||
received_at
|
||||
|
|
@ -61,15 +44,9 @@ defmodule Aprsme.PacketFieldWhitelist do
|
|||
sender
|
||||
snow
|
||||
speed
|
||||
srccallsign
|
||||
ssid
|
||||
symbol_code
|
||||
symbol_table_id
|
||||
symbolcode
|
||||
symboltable
|
||||
telemetry_bits
|
||||
telemetry_seq
|
||||
telemetry_vals
|
||||
temperature
|
||||
timestamp
|
||||
wind_direction
|
||||
|
|
|
|||
|
|
@ -19,31 +19,27 @@ defmodule Aprsme.PacketSanitizer do
|
|||
timestamp: 50,
|
||||
message_number: 20,
|
||||
addressee: 50,
|
||||
symboltable: 5,
|
||||
symbolcode: 5,
|
||||
srccallsign: 20,
|
||||
dstcallsign: 20,
|
||||
|
||||
# These fields can be longer but still have sanity limits
|
||||
path: 500,
|
||||
manufacturer: 100,
|
||||
equipment_type: 100,
|
||||
format: 100,
|
||||
device_identifier: 255,
|
||||
item_name: 100,
|
||||
object_name: 100,
|
||||
|
||||
# Very long fields - these will be TEXT in DB but we still
|
||||
# want to prevent abuse with extremely long data
|
||||
information_field: 5000,
|
||||
# Very long fields
|
||||
raw_packet: 5000,
|
||||
comment: 2000,
|
||||
message_text: 2000,
|
||||
body: 5000,
|
||||
origpacket: 5000,
|
||||
header: 1000,
|
||||
radiorange: 1000,
|
||||
telemetry_bits: 1000
|
||||
message_text: 2000
|
||||
}
|
||||
|
||||
# Max lengths for string values inside the JSONB `data` map
|
||||
@data_string_max_lengths %{
|
||||
"information_field" => 5000,
|
||||
"radiorange" => 1000,
|
||||
"telemetry_bits" => 1000,
|
||||
"format" => 100
|
||||
}
|
||||
|
||||
@doc """
|
||||
|
|
@ -57,6 +53,10 @@ defmodule Aprsme.PacketSanitizer do
|
|||
end)
|
||||
end
|
||||
|
||||
defp sanitize_field(:data, value) when is_map(value) do
|
||||
sanitize_data_map(value)
|
||||
end
|
||||
|
||||
defp sanitize_field(key, value) when is_binary(value) do
|
||||
case Map.get(@max_lengths, key) do
|
||||
nil ->
|
||||
|
|
@ -70,6 +70,24 @@ defmodule Aprsme.PacketSanitizer do
|
|||
|
||||
defp sanitize_field(_key, value), do: value
|
||||
|
||||
defp sanitize_data_map(data) when is_map(data) do
|
||||
Enum.reduce(data, %{}, fn {key, value}, acc ->
|
||||
sanitized =
|
||||
case {is_binary(value), Map.get(@data_string_max_lengths, key)} do
|
||||
{true, nil} -> strip_null_bytes(value)
|
||||
{true, max_length} -> value |> strip_null_bytes() |> truncate_string(max_length)
|
||||
_ -> value
|
||||
end
|
||||
|
||||
Map.put(acc, key, sanitized)
|
||||
end)
|
||||
end
|
||||
|
||||
# PostgreSQL JSONB does not support \u0000 (null bytes)
|
||||
defp strip_null_bytes(string) when is_binary(string) do
|
||||
String.replace(string, <<0x00>>, "")
|
||||
end
|
||||
|
||||
defp truncate_string(string, max_length) when byte_size(string) <= max_length do
|
||||
string
|
||||
end
|
||||
|
|
|
|||
|
|
@ -125,7 +125,7 @@ defmodule Aprsme.Packets.QueryBuilder do
|
|||
rain_24h: p.rain_24h,
|
||||
rain_since_midnight: p.rain_since_midnight,
|
||||
snow: p.snow,
|
||||
luminosity: p.luminosity
|
||||
luminosity: fragment("(?->>'luminosity')::integer", p.data)
|
||||
}
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ defmodule AprsmeWeb.Api.V1.CallsignJSON do
|
|||
destination: packet.destination,
|
||||
path: packet.path,
|
||||
data_type: packet.data_type,
|
||||
information_field: packet.information_field,
|
||||
information_field: get_in(packet.data || %{}, ["information_field"]),
|
||||
raw_packet: sanitize_raw_packet(packet.raw_packet),
|
||||
received_at: packet.received_at,
|
||||
region: packet.region,
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ defmodule AprsmeWeb.ApiDocsLive do
|
|||
"destination" => packet.destination,
|
||||
"path" => packet.path,
|
||||
"data_type" => packet.data_type,
|
||||
"information_field" => packet.information_field,
|
||||
"information_field" => get_in(packet.data || %{}, ["information_field"]),
|
||||
"raw_packet" => sanitize_raw_packet(packet.raw_packet),
|
||||
"received_at" => packet.received_at,
|
||||
"region" => packet.region,
|
||||
|
|
|
|||
|
|
@ -143,38 +143,38 @@
|
|||
</dd>
|
||||
</div>
|
||||
<% end %>
|
||||
<%= if @packet.phg_power || @packet.phg_height || @packet.phg_gain do %>
|
||||
<%= if (@packet.data || %{})["phg_power"] || (@packet.data || %{})["phg_height"] || (@packet.data || %{})["phg_gain"] do %>
|
||||
<div class="col-span-2">
|
||||
<dt class="text-xs font-medium text-gray-500 dark:text-gray-400 mb-2">
|
||||
{gettext("PHG (Power-Height-Gain)")}
|
||||
</dt>
|
||||
<dd class="grid grid-cols-2 gap-2 text-sm">
|
||||
<%= if @packet.phg_power do %>
|
||||
<%= if (@packet.data || %{})["phg_power"] do %>
|
||||
<div>
|
||||
<span class="text-gray-500 dark:text-gray-400">{gettext("Power:")}</span>
|
||||
<span class="font-semibold">{@packet.phg_power} W</span>
|
||||
<span class="font-semibold">{(@packet.data || %{})["phg_power"]} W</span>
|
||||
</div>
|
||||
<% end %>
|
||||
<%= if @packet.phg_height do %>
|
||||
<%= if (@packet.data || %{})["phg_height"] do %>
|
||||
<div>
|
||||
<span class="text-gray-500 dark:text-gray-400">{gettext("HAAT:")}</span>
|
||||
<span class="font-semibold">{@packet.phg_height} ft</span>
|
||||
<span class="font-semibold">{(@packet.data || %{})["phg_height"]} ft</span>
|
||||
</div>
|
||||
<% end %>
|
||||
<%= if @packet.phg_gain do %>
|
||||
<%= if (@packet.data || %{})["phg_gain"] do %>
|
||||
<div>
|
||||
<span class="text-gray-500 dark:text-gray-400">{gettext("Gain:")}</span>
|
||||
<span class="font-semibold">{@packet.phg_gain} dBi</span>
|
||||
<span class="font-semibold">{(@packet.data || %{})["phg_gain"]} dBi</span>
|
||||
</div>
|
||||
<% end %>
|
||||
<%= if @packet.phg_directivity do %>
|
||||
<%= if (@packet.data || %{})["phg_directivity"] do %>
|
||||
<div>
|
||||
<span class="text-gray-500 dark:text-gray-400">{gettext("Dir:")}</span>
|
||||
<span class="font-semibold">
|
||||
<%= if @packet.phg_directivity == 0 or @packet.phg_directivity == 360 do %>
|
||||
<%= if (@packet.data || %{})["phg_directivity"] == 0 or (@packet.data || %{})["phg_directivity"] == 360 do %>
|
||||
{gettext("Omni")}
|
||||
<% else %>
|
||||
{@packet.phg_directivity}°
|
||||
{(@packet.data || %{})["phg_directivity"]}°
|
||||
<% end %>
|
||||
</span>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ defmodule AprsmeWeb.MapLive.PacketUtils do
|
|||
where: p.sender == ^callsign,
|
||||
where:
|
||||
fragment(
|
||||
"? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL",
|
||||
"? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ? IS NOT NULL OR ?->>'luminosity' IS NOT NULL OR ?->>'rain_midnight' IS NOT NULL",
|
||||
p.temperature,
|
||||
p.humidity,
|
||||
p.pressure,
|
||||
|
|
@ -128,9 +128,9 @@ defmodule AprsmeWeb.MapLive.PacketUtils do
|
|||
p.wind_gust,
|
||||
p.rain_1h,
|
||||
p.rain_24h,
|
||||
p.rain_midnight,
|
||||
p.luminosity,
|
||||
p.snow_24h
|
||||
p.snow,
|
||||
p.data,
|
||||
p.data
|
||||
),
|
||||
select: fragment("1"),
|
||||
limit: 1
|
||||
|
|
|
|||
|
|
@ -60,12 +60,13 @@
|
|||
</td>
|
||||
<td>
|
||||
<span class="text-sm font-mono">
|
||||
<%= if String.length(packet.information_field || "") > 50 do %>
|
||||
<span title={packet.information_field}>
|
||||
{String.slice(packet.information_field, 0, 50)}...
|
||||
<% info_field = (packet.data || %{})["information_field"] || "" %>
|
||||
<%= if String.length(info_field) > 50 do %>
|
||||
<span title={info_field}>
|
||||
{String.slice(info_field, 0, 50)}...
|
||||
</span>
|
||||
<% else %>
|
||||
{packet.information_field}
|
||||
{info_field}
|
||||
<% end %>
|
||||
</span>
|
||||
</td>
|
||||
|
|
|
|||
|
|
@ -103,8 +103,7 @@ defmodule AprsmeWeb.Live.Shared.PacketUtils do
|
|||
:rain_1h,
|
||||
:rain_24h,
|
||||
:rain_since_midnight,
|
||||
:snow,
|
||||
:luminosity
|
||||
:snow
|
||||
]
|
||||
|
||||
Enum.any?(weather_fields, fn field ->
|
||||
|
|
|
|||
|
|
@ -193,7 +193,7 @@ defmodule AprsmeWeb.WeatherLive.CallsignView do
|
|||
rain_1h: rain_1h_value,
|
||||
rain_24h: rain_24h_value,
|
||||
rain_since_midnight: rain_since_midnight_value,
|
||||
luminosity: pkt.luminosity
|
||||
luminosity: get_in(pkt.data || %{}, ["luminosity"])
|
||||
}
|
||||
end)
|
||||
|> Jason.encode!()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,468 @@
|
|||
defmodule Aprsme.Repo.Migrations.ConsolidatePacketsColumns do
|
||||
use Ecto.Migration
|
||||
|
||||
@moduledoc """
|
||||
Consolidates the packets table by:
|
||||
- Dropping 10 dead parser-compat columns (never read)
|
||||
- Moving 14 display-only columns into a single `data` JSONB column
|
||||
- Fixing the notify_packets_insert() trigger to read from `data` JSONB
|
||||
|
||||
The packets table is ephemeral (refills from APRS-IS stream) so data loss
|
||||
from DROP/recreate is acceptable.
|
||||
"""
|
||||
|
||||
def up do
|
||||
execute("SET LOCAL statement_timeout = '0'")
|
||||
|
||||
# Drop existing partitioned table
|
||||
execute("DROP TABLE IF EXISTS packets CASCADE")
|
||||
|
||||
# Recreate without the 24 dropped/moved columns, plus new `data` JSONB column
|
||||
execute("""
|
||||
CREATE TABLE packets (
|
||||
id uuid NOT NULL,
|
||||
base_callsign varchar,
|
||||
data_type varchar,
|
||||
destination varchar,
|
||||
path text,
|
||||
sender varchar,
|
||||
ssid varchar,
|
||||
data_extended jsonb,
|
||||
inserted_at timestamp NOT NULL,
|
||||
updated_at timestamp NOT NULL,
|
||||
received_at timestamp NOT NULL,
|
||||
region varchar,
|
||||
lat numeric,
|
||||
lon numeric,
|
||||
has_position boolean DEFAULT false,
|
||||
item_name text,
|
||||
object_name text,
|
||||
is_item boolean DEFAULT false,
|
||||
is_object boolean DEFAULT false,
|
||||
dao jsonb,
|
||||
raw_packet text,
|
||||
symbol_code varchar,
|
||||
symbol_table_id varchar,
|
||||
comment text,
|
||||
"timestamp" varchar,
|
||||
aprs_messaging boolean DEFAULT false,
|
||||
temperature double precision,
|
||||
humidity double precision,
|
||||
wind_speed double precision,
|
||||
wind_direction integer,
|
||||
wind_gust double precision,
|
||||
pressure double precision,
|
||||
rain_1h double precision,
|
||||
rain_24h double precision,
|
||||
rain_since_midnight double precision,
|
||||
manufacturer text,
|
||||
equipment_type text,
|
||||
course integer,
|
||||
speed double precision,
|
||||
altitude double precision,
|
||||
addressee text,
|
||||
message_text text,
|
||||
message_number varchar,
|
||||
location geometry(Point,4326),
|
||||
device_identifier text,
|
||||
snow double precision,
|
||||
has_weather boolean DEFAULT false,
|
||||
data jsonb DEFAULT '{}'::jsonb,
|
||||
PRIMARY KEY (id, received_at)
|
||||
) PARTITION BY RANGE (received_at)
|
||||
""")
|
||||
|
||||
# Default partition
|
||||
execute("CREATE TABLE packets_default PARTITION OF packets DEFAULT")
|
||||
|
||||
# Daily partitions for recent past + near future
|
||||
today = Date.utc_today()
|
||||
|
||||
for offset <- -8..2 do
|
||||
date = Date.add(today, offset)
|
||||
name = "packets_#{Calendar.strftime(date, "%Y%m%d")}"
|
||||
next_date = Date.add(date, 1)
|
||||
|
||||
execute(
|
||||
"CREATE TABLE IF NOT EXISTS #{name} PARTITION OF packets FOR VALUES FROM ('#{date} 00:00:00') TO ('#{next_date} 00:00:00')"
|
||||
)
|
||||
end
|
||||
|
||||
create_indexes()
|
||||
update_notify_trigger()
|
||||
create_triggers("packets")
|
||||
end
|
||||
|
||||
def down do
|
||||
execute("SET LOCAL statement_timeout = '0'")
|
||||
execute("DROP TABLE IF EXISTS packets CASCADE")
|
||||
|
||||
# Recreate original table with all columns (non-partitioned for simplicity)
|
||||
execute("""
|
||||
CREATE TABLE packets (
|
||||
id uuid NOT NULL PRIMARY KEY,
|
||||
base_callsign varchar,
|
||||
data_type varchar,
|
||||
destination varchar,
|
||||
information_field text,
|
||||
path text,
|
||||
sender varchar,
|
||||
ssid varchar,
|
||||
data_extended jsonb,
|
||||
inserted_at timestamp NOT NULL,
|
||||
updated_at timestamp NOT NULL,
|
||||
received_at timestamp NOT NULL,
|
||||
region varchar,
|
||||
lat numeric,
|
||||
lon numeric,
|
||||
has_position boolean DEFAULT false,
|
||||
item_name text,
|
||||
object_name text,
|
||||
is_item boolean DEFAULT false,
|
||||
is_object boolean DEFAULT false,
|
||||
dao jsonb,
|
||||
raw_packet text,
|
||||
symbol_code varchar,
|
||||
symbol_table_id varchar,
|
||||
comment text,
|
||||
"timestamp" varchar,
|
||||
aprs_messaging boolean DEFAULT false,
|
||||
temperature double precision,
|
||||
humidity double precision,
|
||||
wind_speed double precision,
|
||||
wind_direction integer,
|
||||
wind_gust double precision,
|
||||
pressure double precision,
|
||||
rain_1h double precision,
|
||||
rain_24h double precision,
|
||||
rain_since_midnight double precision,
|
||||
manufacturer text,
|
||||
equipment_type text,
|
||||
course integer,
|
||||
speed double precision,
|
||||
altitude double precision,
|
||||
addressee text,
|
||||
message_text text,
|
||||
message_number varchar,
|
||||
location geometry(Point,4326),
|
||||
device_identifier text,
|
||||
luminosity integer,
|
||||
snow double precision,
|
||||
phg_power integer,
|
||||
phg_height integer,
|
||||
phg_gain integer,
|
||||
phg_directivity integer,
|
||||
has_weather boolean DEFAULT false,
|
||||
rain_midnight double precision,
|
||||
position_ambiguity integer,
|
||||
posresolution double precision,
|
||||
format text,
|
||||
telemetry_seq integer,
|
||||
telemetry_vals integer[],
|
||||
telemetry_bits text,
|
||||
radiorange text,
|
||||
srccallsign varchar,
|
||||
dstcallsign varchar,
|
||||
body text,
|
||||
origpacket text,
|
||||
header text,
|
||||
alive integer DEFAULT 1,
|
||||
posambiguity integer,
|
||||
symboltable varchar,
|
||||
symbolcode varchar,
|
||||
messaging integer
|
||||
)
|
||||
""")
|
||||
|
||||
restore_notify_trigger()
|
||||
create_triggers("packets")
|
||||
end
|
||||
|
||||
defp create_indexes do
|
||||
# Time-series
|
||||
execute("CREATE INDEX idx_packets_received_desc ON packets (received_at DESC)")
|
||||
execute("CREATE INDEX idx_packets_received_brin ON packets USING brin (received_at)")
|
||||
|
||||
# Sender/Callsign
|
||||
execute("CREATE INDEX idx_packets_sender_received ON packets (sender, received_at DESC)")
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_upper_sender ON packets (upper(sender::text), received_at DESC)"
|
||||
)
|
||||
|
||||
execute("CREATE INDEX idx_packets_sender_pattern ON packets (sender text_pattern_ops)")
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_base_callsign_pattern ON packets (base_callsign text_pattern_ops)"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_base_callsign_time ON packets (base_callsign, received_at DESC)"
|
||||
)
|
||||
|
||||
execute("CREATE INDEX idx_packets_sender_id_desc ON packets (sender, id DESC)")
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_upper_base_callsign ON packets (upper(base_callsign::text))"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_sender_trgm ON packets USING gin (sender gin_trgm_ops) WHERE sender IS NOT NULL"
|
||||
)
|
||||
|
||||
# Position/Geographic
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_positioned ON packets (received_at DESC, lat, lon, sender) WHERE has_position = true"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_position_time ON packets (has_position, received_at DESC) WHERE has_position = true"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_location ON packets USING gist (location) WHERE has_position = true"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_location_geography ON packets USING gist ((location::geography))"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_spatial_temporal ON packets (received_at DESC, base_callsign) WHERE has_position = true"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_sender_position ON packets (sender, received_at DESC) WHERE lat IS NOT NULL AND lon IS NOT NULL"
|
||||
)
|
||||
|
||||
execute("CREATE INDEX idx_packets_lat ON packets (lat)")
|
||||
execute("CREATE INDEX idx_packets_lon ON packets (lon)")
|
||||
|
||||
# Region
|
||||
execute("CREATE INDEX idx_packets_region ON packets (region)")
|
||||
execute("CREATE INDEX idx_packets_region_received ON packets (region, received_at)")
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_region_position_time ON packets (region, has_position, received_at DESC) WHERE has_position = true AND region IS NOT NULL"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_region_data_type ON packets (region, data_type, received_at)"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_region_position ON packets (region, has_position, received_at)"
|
||||
)
|
||||
|
||||
# Weather
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_weather ON packets (sender, received_at DESC) WHERE temperature IS NOT NULL OR humidity IS NOT NULL OR pressure IS NOT NULL OR wind_speed IS NOT NULL OR wind_direction IS NOT NULL OR rain_1h IS NOT NULL"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_has_weather ON packets (has_weather, sender, received_at DESC) WHERE has_weather = true"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_temperature ON packets (temperature) WHERE temperature IS NOT NULL"
|
||||
)
|
||||
|
||||
execute("CREATE INDEX idx_packets_humidity ON packets (humidity) WHERE humidity IS NOT NULL")
|
||||
execute("CREATE INDEX idx_packets_pressure ON packets (pressure) WHERE pressure IS NOT NULL")
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_wind_speed ON packets (wind_speed) WHERE wind_speed IS NOT NULL"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_received_temp ON packets (received_at, temperature) WHERE temperature IS NOT NULL"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_sender_temp ON packets (sender, temperature) WHERE temperature IS NOT NULL"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_weather_history ON packets (sender, data_type, received_at) WHERE data_type = 'weather'"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_weather_selective ON packets (received_at DESC) WHERE data_type IN ('weather', 'Weather', 'WX', 'wx')"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_weather_symbol ON packets (received_at DESC, lat, lon) WHERE (symbol_table_id = '/' AND symbol_code = '_') OR (symbol_table_id = E'\\\\' AND symbol_code = '_')"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_sender_weather_lookup ON packets (upper(sender::text), received_at DESC) INCLUDE (data_type, symbol_table_id, symbol_code) WHERE data_type = 'weather' OR (symbol_table_id = '/' AND symbol_code = '_') OR (symbol_table_id = E'\\\\' AND symbol_code = '_')"
|
||||
)
|
||||
|
||||
# Data type & Device
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_datatype_sender_time ON packets (data_type, sender, received_at DESC)"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_device_identifier ON packets (device_identifier) WHERE device_identifier IS NOT NULL"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_device_received ON packets (device_identifier, received_at) WHERE device_identifier IS NOT NULL"
|
||||
)
|
||||
|
||||
# Path
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_path_pattern ON packets (path text_pattern_ops) WHERE path IS NOT NULL AND path <> ''"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_path_trgm ON packets USING gin (path gin_trgm_ops) WHERE path IS NOT NULL AND path <> '' AND path !~ '^TCPIP' AND path !~ ',TCPIP'"
|
||||
)
|
||||
|
||||
# Digipeater/stations heard
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_digipeater ON packets (sender, received_at DESC) WHERE location IS NOT NULL"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_stations_heard ON packets (received_at DESC, sender, lat, lon) WHERE path IS NOT NULL AND path <> '' AND path !~ '^TCPIP' AND path !~ ',TCPIP' AND lat IS NOT NULL AND lon IS NOT NULL"
|
||||
)
|
||||
|
||||
# Symbol
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_symbol_time ON packets (symbol_table_id, symbol_code, received_at)"
|
||||
)
|
||||
|
||||
# JSONB data column — GIN index for key lookups
|
||||
execute(
|
||||
"CREATE INDEX idx_packets_data_gin ON packets USING gin (data) WHERE data IS NOT NULL AND data <> '{}'::jsonb"
|
||||
)
|
||||
end
|
||||
|
||||
# Updated trigger reads PHG/luminosity/rain_midnight from data JSONB
|
||||
defp update_notify_trigger do
|
||||
execute("DROP FUNCTION IF EXISTS notify_packets_insert() CASCADE")
|
||||
|
||||
execute("""
|
||||
CREATE OR REPLACE FUNCTION notify_packets_insert() RETURNS trigger AS $$
|
||||
DECLARE
|
||||
payload TEXT;
|
||||
BEGIN
|
||||
payload := json_build_object(
|
||||
'id', NEW.id,
|
||||
'sender', NEW.sender,
|
||||
'ssid', NEW.ssid,
|
||||
'base_callsign', NEW.base_callsign,
|
||||
'lat', NEW.lat,
|
||||
'lon', NEW.lon,
|
||||
'altitude', NEW.altitude,
|
||||
'course', NEW.course,
|
||||
'speed', NEW.speed,
|
||||
'symbol_table_id', NEW.symbol_table_id,
|
||||
'symbol_code', NEW.symbol_code,
|
||||
'device_identifier', NEW.device_identifier,
|
||||
'path', NEW.path,
|
||||
'data_type', NEW.data_type,
|
||||
'received_at', NEW.received_at,
|
||||
'comment', NEW.comment,
|
||||
'raw_packet', NEW.raw_packet,
|
||||
'phg_power', NEW.data->>'phg_power',
|
||||
'phg_height', NEW.data->>'phg_height',
|
||||
'phg_gain', NEW.data->>'phg_gain',
|
||||
'phg_directivity', NEW.data->>'phg_directivity',
|
||||
'temperature', NEW.temperature,
|
||||
'humidity', NEW.humidity,
|
||||
'pressure', NEW.pressure,
|
||||
'wind_speed', NEW.wind_speed,
|
||||
'wind_direction', NEW.wind_direction,
|
||||
'wind_gust', NEW.wind_gust,
|
||||
'rain_1h', NEW.rain_1h,
|
||||
'rain_24h', NEW.rain_24h,
|
||||
'rain_midnight', NEW.data->>'rain_midnight',
|
||||
'luminosity', NEW.data->>'luminosity',
|
||||
'snow', NEW.snow,
|
||||
'has_weather', NEW.has_weather,
|
||||
'inserted_at', NEW.inserted_at
|
||||
)::text;
|
||||
|
||||
PERFORM pg_notify('aprs_packets', payload);
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
end
|
||||
|
||||
# Restore original trigger for rollback
|
||||
defp restore_notify_trigger do
|
||||
execute("DROP FUNCTION IF EXISTS notify_packets_insert() CASCADE")
|
||||
|
||||
execute("""
|
||||
CREATE OR REPLACE FUNCTION notify_packets_insert() RETURNS trigger AS $$
|
||||
DECLARE
|
||||
payload TEXT;
|
||||
BEGIN
|
||||
payload := json_build_object(
|
||||
'id', NEW.id,
|
||||
'sender', NEW.sender,
|
||||
'ssid', NEW.ssid,
|
||||
'base_callsign', NEW.base_callsign,
|
||||
'lat', NEW.lat,
|
||||
'lon', NEW.lon,
|
||||
'altitude', NEW.altitude,
|
||||
'course', NEW.course,
|
||||
'speed', NEW.speed,
|
||||
'symbol_table_id', NEW.symbol_table_id,
|
||||
'symbol_code', NEW.symbol_code,
|
||||
'device_identifier', NEW.device_identifier,
|
||||
'path', NEW.path,
|
||||
'data_type', NEW.data_type,
|
||||
'received_at', NEW.received_at,
|
||||
'comment', NEW.comment,
|
||||
'raw_packet', NEW.raw_packet,
|
||||
'phg_power', NEW.phg_power,
|
||||
'phg_height', NEW.phg_height,
|
||||
'phg_gain', NEW.phg_gain,
|
||||
'phg_directivity', NEW.phg_directivity,
|
||||
'temperature', NEW.temperature,
|
||||
'humidity', NEW.humidity,
|
||||
'pressure', NEW.pressure,
|
||||
'wind_speed', NEW.wind_speed,
|
||||
'wind_direction', NEW.wind_direction,
|
||||
'wind_gust', NEW.wind_gust,
|
||||
'rain_1h', NEW.rain_1h,
|
||||
'rain_24h', NEW.rain_24h,
|
||||
'rain_midnight', NEW.rain_midnight,
|
||||
'luminosity', NEW.luminosity,
|
||||
'snow', NEW.snow,
|
||||
'has_weather', NEW.has_weather,
|
||||
'inserted_at', NEW.inserted_at
|
||||
)::text;
|
||||
|
||||
PERFORM pg_notify('aprs_packets', payload);
|
||||
|
||||
RETURN NEW;
|
||||
END;
|
||||
$$ LANGUAGE plpgsql;
|
||||
""")
|
||||
end
|
||||
|
||||
defp create_triggers(table) do
|
||||
execute(
|
||||
"CREATE TRIGGER update_has_weather_trigger BEFORE INSERT OR UPDATE ON #{table} FOR EACH ROW EXECUTE FUNCTION update_has_weather()"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE TRIGGER packets_notify_insert AFTER INSERT ON #{table} FOR EACH ROW EXECUTE FUNCTION notify_packets_insert()"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE TRIGGER packet_insert_sequence AFTER INSERT ON #{table} FOR EACH ROW EXECUTE FUNCTION increment_packet_sequence()"
|
||||
)
|
||||
|
||||
execute(
|
||||
"CREATE TRIGGER packet_delete_sequence AFTER DELETE ON #{table} FOR EACH ROW EXECUTE FUNCTION decrement_packet_sequence()"
|
||||
)
|
||||
end
|
||||
end
|
||||
|
|
@ -41,7 +41,7 @@ defmodule Aprsme.EncodingUtilsTest do
|
|||
end
|
||||
|
||||
describe "sanitize_packet/1" do
|
||||
test "sanitizes information_field" do
|
||||
test "sanitizes string values in data map" do
|
||||
invalid_info = <<72, 101, 108, 108, 111, 211, 87, 111, 114, 108, 100>>
|
||||
|
||||
packet = %Packet{
|
||||
|
|
@ -49,18 +49,20 @@ defmodule Aprsme.EncodingUtilsTest do
|
|||
sender: "TEST-1",
|
||||
path: "TCPIP*",
|
||||
destination: "APX100",
|
||||
information_field: invalid_info,
|
||||
data_type: :position,
|
||||
base_callsign: "TEST",
|
||||
ssid: "1",
|
||||
data: %{"information_field" => invalid_info, "radiorange" => "0050"},
|
||||
data_extended: %{comment: "Valid comment"}
|
||||
}
|
||||
|
||||
sanitized = EncodingUtils.sanitize_packet(packet)
|
||||
|
||||
assert String.valid?(sanitized.information_field)
|
||||
assert String.contains?(sanitized.information_field, "Hello")
|
||||
assert String.contains?(sanitized.information_field, "World")
|
||||
assert String.valid?(sanitized.data["information_field"])
|
||||
assert String.contains?(sanitized.data["information_field"], "Hello")
|
||||
assert String.contains?(sanitized.data["information_field"], "World")
|
||||
# Non-string values in data map are preserved
|
||||
assert sanitized.data["radiorange"] == "0050"
|
||||
end
|
||||
|
||||
test "sanitizes comment in data_extended" do
|
||||
|
|
@ -71,7 +73,6 @@ defmodule Aprsme.EncodingUtilsTest do
|
|||
sender: "TEST-1",
|
||||
path: "TCPIP*",
|
||||
destination: "APX100",
|
||||
information_field: "Valid info",
|
||||
data_type: :position,
|
||||
base_callsign: "TEST",
|
||||
ssid: "1",
|
||||
|
|
@ -90,7 +91,6 @@ defmodule Aprsme.EncodingUtilsTest do
|
|||
sender: "TEST-1",
|
||||
path: "TCPIP*",
|
||||
destination: "APX100",
|
||||
information_field: "Valid info",
|
||||
data_type: :position,
|
||||
base_callsign: "TEST",
|
||||
ssid: "1",
|
||||
|
|
@ -100,7 +100,6 @@ defmodule Aprsme.EncodingUtilsTest do
|
|||
sanitized = EncodingUtils.sanitize_packet(packet)
|
||||
|
||||
assert sanitized.data_extended == nil
|
||||
assert sanitized.information_field == "Valid info"
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ defmodule Aprsme.EnhancedParserTest do
|
|||
alias Aprsme.Packet
|
||||
|
||||
describe "enhanced parser field extraction" do
|
||||
test "extracts standard parser compatibility fields" do
|
||||
test "dead parser compat fields are dropped (not preserved)" do
|
||||
attrs = %{
|
||||
srccallsign: "W1AW",
|
||||
dstcallsign: "APRS",
|
||||
|
|
@ -26,19 +26,20 @@ defmodule Aprsme.EnhancedParserTest do
|
|||
|
||||
result = Packet.extract_additional_data(attrs, "W1AW>APRS:!1234.56N/12345.67W-Test")
|
||||
|
||||
assert result[:srccallsign] == "W1AW"
|
||||
assert result[:dstcallsign] == "APRS"
|
||||
assert result[:body] == "!1234.56N/12345.67W-Test"
|
||||
assert result[:origpacket] == "W1AW>APRS:!1234.56N/12345.67W-Test"
|
||||
assert result[:header] == "W1AW>APRS"
|
||||
assert result[:alive] == 1
|
||||
assert result[:posambiguity] == 0
|
||||
assert result[:symboltable] == "/"
|
||||
assert result[:symbolcode] == "-"
|
||||
assert result[:messaging] == 0
|
||||
# Dead fields should not be in result
|
||||
refute Map.has_key?(result, :srccallsign)
|
||||
refute Map.has_key?(result, :dstcallsign)
|
||||
refute Map.has_key?(result, :body)
|
||||
refute Map.has_key?(result, :origpacket)
|
||||
refute Map.has_key?(result, :header)
|
||||
refute Map.has_key?(result, :alive)
|
||||
refute Map.has_key?(result, :posambiguity)
|
||||
refute Map.has_key?(result, :symboltable)
|
||||
refute Map.has_key?(result, :symbolcode)
|
||||
refute Map.has_key?(result, :messaging)
|
||||
end
|
||||
|
||||
test "extracts radio range field" do
|
||||
test "extracts radio range into data map" do
|
||||
attrs = %{
|
||||
radiorange: "0050",
|
||||
sender: "W1AW",
|
||||
|
|
@ -51,7 +52,7 @@ defmodule Aprsme.EnhancedParserTest do
|
|||
|
||||
result = Packet.extract_additional_data(attrs)
|
||||
|
||||
assert result[:radiorange] == "0050"
|
||||
assert result[:data]["radiorange"] == "0050"
|
||||
end
|
||||
|
||||
test "handles weather data from wx field" do
|
||||
|
|
@ -82,7 +83,7 @@ defmodule Aprsme.EnhancedParserTest do
|
|||
assert result[:wind_direction] == 180
|
||||
end
|
||||
|
||||
test "handles PHG data in string format" do
|
||||
test "handles PHG data in string format and moves to data map" do
|
||||
attrs = %{
|
||||
sender: "W1AW",
|
||||
base_callsign: "W1AW",
|
||||
|
|
@ -98,13 +99,14 @@ defmodule Aprsme.EnhancedParserTest do
|
|||
result = Packet.extract_additional_data(attrs)
|
||||
|
||||
# PHG 5430: power=5^2=25W, height=4->160ft, gain=3dB, dir=0->omni
|
||||
assert result[:phg_power] == 25
|
||||
assert result[:phg_height] == 160
|
||||
assert result[:phg_gain] == 3
|
||||
assert result[:phg_directivity] == 0
|
||||
# PHG fields should be in the data map
|
||||
assert result[:data]["phg_power"] == 25
|
||||
assert result[:data]["phg_height"] == 160
|
||||
assert result[:data]["phg_gain"] == 3
|
||||
assert result[:data]["phg_directivity"] == 0
|
||||
end
|
||||
|
||||
test "handles PHG data in legacy map format" do
|
||||
test "handles PHG data in legacy map format and moves to data map" do
|
||||
attrs = %{
|
||||
sender: "W1AW",
|
||||
base_callsign: "W1AW",
|
||||
|
|
@ -124,15 +126,103 @@ defmodule Aprsme.EnhancedParserTest do
|
|||
|
||||
result = Packet.extract_additional_data(attrs)
|
||||
|
||||
assert result[:phg_power] == 25
|
||||
assert result[:phg_height] == 160
|
||||
assert result[:phg_gain] == 3
|
||||
assert result[:phg_directivity] == 0
|
||||
assert result[:data]["phg_power"] == 25
|
||||
assert result[:data]["phg_height"] == 160
|
||||
assert result[:data]["phg_gain"] == 3
|
||||
assert result[:data]["phg_directivity"] == 0
|
||||
end
|
||||
|
||||
test "collect_into_data_map sweeps all 14 fields into data" do
|
||||
attrs = %{
|
||||
sender: "W1AW",
|
||||
base_callsign: "W1AW",
|
||||
ssid: "0",
|
||||
destination: "APRS",
|
||||
data_type: "position",
|
||||
received_at: DateTime.utc_now(),
|
||||
data_extended: %{
|
||||
phg: "5430",
|
||||
format: :uncompressed,
|
||||
posresolution: 18.52,
|
||||
position_ambiguity: 0,
|
||||
comment: "Test PHG5430"
|
||||
}
|
||||
}
|
||||
|
||||
result = Packet.extract_additional_data(attrs)
|
||||
|
||||
# These fields should be in the data map, not at top level
|
||||
assert is_map(result[:data])
|
||||
assert result[:data]["phg_power"] == 25
|
||||
assert result[:data]["format"] == "uncompressed"
|
||||
assert result[:data]["posresolution"] == 18.52
|
||||
assert result[:data]["position_ambiguity"] == 0
|
||||
|
||||
# They should NOT be at top level
|
||||
refute Map.has_key?(result, :phg_power)
|
||||
refute Map.has_key?(result, :format)
|
||||
refute Map.has_key?(result, :posresolution)
|
||||
refute Map.has_key?(result, :position_ambiguity)
|
||||
end
|
||||
|
||||
test "telemetry fields are moved into data map" do
|
||||
attrs = %{
|
||||
sender: "W1AW",
|
||||
base_callsign: "W1AW",
|
||||
ssid: "0",
|
||||
destination: "APRS",
|
||||
data_type: "telemetry",
|
||||
received_at: DateTime.utc_now(),
|
||||
data_extended: %{
|
||||
telemetry: %{
|
||||
seq: 123,
|
||||
vals: [1, 2, 3, 4, 5],
|
||||
bits: "11111111"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = Packet.extract_additional_data(attrs)
|
||||
|
||||
assert result[:data]["telemetry_seq"] == 123
|
||||
assert result[:data]["telemetry_vals"] == [1, 2, 3, 4, 5]
|
||||
assert result[:data]["telemetry_bits"] == "11111111"
|
||||
|
||||
refute Map.has_key?(result, :telemetry_seq)
|
||||
refute Map.has_key?(result, :telemetry_vals)
|
||||
refute Map.has_key?(result, :telemetry_bits)
|
||||
end
|
||||
|
||||
test "luminosity and rain_midnight are moved into data map" do
|
||||
attrs = %{
|
||||
sender: "WX1STN",
|
||||
base_callsign: "WX1STN",
|
||||
ssid: "0",
|
||||
destination: "APRS",
|
||||
data_type: "weather",
|
||||
received_at: DateTime.utc_now(),
|
||||
data_extended: %{
|
||||
wx: %{
|
||||
temperature: 72.5,
|
||||
luminosity: 500,
|
||||
rain_midnight: 0.5
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result = Packet.extract_additional_data(attrs)
|
||||
|
||||
# luminosity and rain_midnight should be in data map
|
||||
assert result[:data]["luminosity"] == 500
|
||||
assert result[:data]["rain_midnight"] == 0.5
|
||||
|
||||
# temperature stays at top level
|
||||
assert result[:temperature] == 72.5
|
||||
end
|
||||
end
|
||||
|
||||
describe "enhanced parser packet storage" do
|
||||
test "stores packet with enhanced parser fields" do
|
||||
test "stores packet with data map fields" do
|
||||
attrs = %{
|
||||
sender: "W1AW-1",
|
||||
base_callsign: "W1AW",
|
||||
|
|
@ -146,36 +236,17 @@ defmodule Aprsme.EnhancedParserTest do
|
|||
symbol_code: "-",
|
||||
symbol_table_id: "/",
|
||||
comment: "Test station",
|
||||
# Enhanced parser fields
|
||||
srccallsign: "W1AW-1",
|
||||
dstcallsign: "APRS",
|
||||
body: "!1234.56N/12345.67W-Test RNG0050",
|
||||
origpacket: "W1AW-1>APRS:!1234.56N/12345.67W-Test RNG0050",
|
||||
header: "W1AW-1>APRS",
|
||||
alive: 1,
|
||||
posambiguity: 0,
|
||||
symboltable: "/",
|
||||
symbolcode: "-",
|
||||
messaging: 0,
|
||||
radiorange: "0050"
|
||||
radiorange: "0050",
|
||||
information_field: "!1234.56N/12345.67W-Test"
|
||||
}
|
||||
|
||||
processed_attrs = Packet.extract_additional_data(attrs)
|
||||
|
||||
assert {:ok, packet} = Aprsme.Packets.store_packet(processed_attrs)
|
||||
|
||||
# Verify enhanced fields are stored
|
||||
assert packet.srccallsign == "W1AW-1"
|
||||
assert packet.dstcallsign == "APRS"
|
||||
assert packet.body == "!1234.56N/12345.67W-Test RNG0050"
|
||||
assert packet.origpacket == "W1AW-1>APRS:!1234.56N/12345.67W-Test RNG0050"
|
||||
assert packet.header == "W1AW-1>APRS"
|
||||
assert packet.alive == 1
|
||||
assert packet.posambiguity == 0
|
||||
assert packet.symboltable == "/"
|
||||
assert packet.symbolcode == "-"
|
||||
assert packet.messaging == 0
|
||||
assert packet.radiorange == "0050"
|
||||
# radiorange and information_field should be in data map
|
||||
assert packet.data["radiorange"] == "0050"
|
||||
assert packet.data["information_field"] == "!1234.56N/12345.67W-Test"
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -22,15 +22,15 @@ defmodule Aprsme.PacketParsingTest do
|
|||
# Verify altitude was extracted
|
||||
assert extracted_attrs[:altitude] == 680.0
|
||||
|
||||
# Verify PHG data was extracted into individual fields
|
||||
# Verify PHG data was extracted into the data map
|
||||
# PHG5 = 25W (5^2)
|
||||
assert extracted_attrs[:phg_power] == 25
|
||||
assert extracted_attrs.data["phg_power"] == 25
|
||||
# PHG x5x = 320 ft
|
||||
assert extracted_attrs[:phg_height] == 320
|
||||
assert extracted_attrs.data["phg_height"] == 320
|
||||
# PHG xx3 = 3 dBi
|
||||
assert extracted_attrs[:phg_gain] == 3
|
||||
assert extracted_attrs.data["phg_gain"] == 3
|
||||
# PHG xxx0 = 0 degrees (not 360 for omni in our implementation)
|
||||
assert extracted_attrs[:phg_directivity] == 0
|
||||
assert extracted_attrs.data["phg_directivity"] == 0
|
||||
|
||||
# Verify comment has altitude and PHG removed
|
||||
assert extracted_attrs[:comment] == "Collin Cty Wide Digi"
|
||||
|
|
@ -73,10 +73,10 @@ defmodule Aprsme.PacketParsingTest do
|
|||
# Check that fields were properly set
|
||||
changes = changeset.changes
|
||||
assert changes.altitude == 680.0
|
||||
assert changes.phg_power == 25
|
||||
assert changes.phg_height == 320
|
||||
assert changes.phg_gain == 3
|
||||
assert changes.phg_directivity == 0
|
||||
assert changes.data["phg_power"] == 25
|
||||
assert changes.data["phg_height"] == 320
|
||||
assert changes.data["phg_gain"] == 3
|
||||
assert changes.data["phg_directivity"] == 0
|
||||
assert changes.comment == "Collin Cty Wide Digi"
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -19,10 +19,10 @@ defmodule Aprsme.PacketTest do
|
|||
|
||||
result = Packet.extract_additional_data(attrs, raw_packet)
|
||||
|
||||
assert result.telemetry_seq == 5
|
||||
assert result.data["telemetry_seq"] == 5
|
||||
# 12.80 rounds to 13
|
||||
assert result.telemetry_vals == [13, 0, 0, 0, 0]
|
||||
assert result.telemetry_bits == "00000000"
|
||||
assert result.data["telemetry_vals"] == [13, 0, 0, 0, 0]
|
||||
assert result.data["telemetry_bits"] == "00000000"
|
||||
end
|
||||
|
||||
test "handles mixed integer and string telemetry_vals" do
|
||||
|
|
@ -38,10 +38,10 @@ defmodule Aprsme.PacketTest do
|
|||
|
||||
result = Packet.extract_additional_data(attrs, "test_packet")
|
||||
|
||||
assert result.telemetry_seq == 10
|
||||
assert result.data["telemetry_seq"] == 10
|
||||
# Mixed types converted properly
|
||||
assert result.telemetry_vals == [180, 38, 0, 88, 165]
|
||||
assert result.telemetry_bits == "10101010"
|
||||
assert result.data["telemetry_vals"] == [180, 38, 0, 88, 165]
|
||||
assert result.data["telemetry_bits"] == "10101010"
|
||||
end
|
||||
|
||||
test "handles float telemetry_vals" do
|
||||
|
|
@ -56,9 +56,9 @@ defmodule Aprsme.PacketTest do
|
|||
|
||||
result = Packet.extract_additional_data(attrs, "test_packet")
|
||||
|
||||
assert result.telemetry_seq == 1
|
||||
assert result.data["telemetry_seq"] == 1
|
||||
# Floats rounded to integers
|
||||
assert result.telemetry_vals == [13, 37, 0, 89, 165]
|
||||
assert result.data["telemetry_vals"] == [13, 37, 0, 89, 165]
|
||||
end
|
||||
|
||||
test "handles invalid telemetry_vals gracefully" do
|
||||
|
|
@ -73,9 +73,9 @@ defmodule Aprsme.PacketTest do
|
|||
|
||||
result = Packet.extract_additional_data(attrs, "test_packet")
|
||||
|
||||
assert result.telemetry_seq == 2
|
||||
assert result.data["telemetry_seq"] == 2
|
||||
# Invalid values become 0
|
||||
assert result.telemetry_vals == [0, 0, 0, 13, 0]
|
||||
assert result.data["telemetry_vals"] == [0, 0, 0, 13, 0]
|
||||
end
|
||||
|
||||
test "handles missing telemetry data" do
|
||||
|
|
@ -86,9 +86,9 @@ defmodule Aprsme.PacketTest do
|
|||
|
||||
result = Packet.extract_additional_data(attrs, "test_packet")
|
||||
|
||||
assert Map.get(result, :telemetry_seq) == nil
|
||||
assert Map.get(result, :telemetry_vals) == nil
|
||||
assert Map.get(result, :telemetry_bits) == nil
|
||||
assert get_in(result, [:data, "telemetry_seq"]) == nil
|
||||
assert get_in(result, [:data, "telemetry_vals"]) == nil
|
||||
assert get_in(result, [:data, "telemetry_bits"]) == nil
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ defmodule Aprsme.PacketsEncodingTest do
|
|||
}
|
||||
|
||||
assert {:ok, stored_packet} = Packets.store_packet(packet_data)
|
||||
assert stored_packet.information_field == "Valid UTF-8: ñ, é, 中文, 🚀"
|
||||
assert stored_packet.data["information_field"] == "Valid UTF-8: ñ, é, 中文, 🚀"
|
||||
assert stored_packet.comment == "More UTF-8: αβγ, русский"
|
||||
end
|
||||
|
||||
|
|
@ -68,8 +68,6 @@ defmodule Aprsme.PacketsEncodingTest do
|
|||
}
|
||||
|
||||
assert {:ok, stored_packet} = Packets.store_packet(packet_data)
|
||||
# Empty strings may be stored as nil or empty string depending on Ecto behavior
|
||||
assert stored_packet.information_field in [nil, ""]
|
||||
assert stored_packet.path in [nil, ""]
|
||||
assert is_nil(stored_packet.comment)
|
||||
end
|
||||
|
|
@ -90,8 +88,8 @@ defmodule Aprsme.PacketsEncodingTest do
|
|||
}
|
||||
|
||||
assert {:ok, stored_packet} = Packets.store_packet(packet_data)
|
||||
# Control characters should be filtered out
|
||||
assert stored_packet.information_field == "Control chars:"
|
||||
# Control characters should be filtered out, field is now in data map
|
||||
assert stored_packet.data["information_field"] == "Control chars:"
|
||||
end
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue