Graham McIntire
31ea64aa6c
Drop the pre-insert aprs_messages broadcast in Is.dispatch (subscribers already get a richer payload via postgres:aprsme_packets after insert). Switch PacketsLive.CallsignView to the per-callsign packets:<CS> topic so it only receives relevant packets instead of filtering every one. In PacketConsumer: reuse the received_at stamped in Is.dispatch instead of calling DateTime.utc_now/0 per packet; drop the duplicate struct_to_map pass (Is.dispatch already handled it); fold coordinate validation and Geo.Point construction into set_lat_lon + create_location_geometry so coords are normalized once; extract device_identifier from the already- normalized attrs; pre-build the broadcast payload once (identifier pick, lat/lon aliases, routing callsign) so the async broadcast task no longer does Map.drop/Map.merge per packet. Rewrite PacketSanitizer.sanitize_packet / sanitize_data_map with Map.new/2 instead of Enum.reduce + Map.put — one allocation per packet instead of N. Also compute has_weather in Packet.changeset/2 so direct-changeset inserts (tests, backfills) populate it the same way the GenStage pipeline does.
121 lines
3.3 KiB
Elixir
121 lines
3.3 KiB
Elixir
defmodule Aprsme.PacketSanitizer do
|
|
@moduledoc """
|
|
Sanitizes packet data to ensure it fits within database constraints.
|
|
This module provides truncation for long strings to prevent database errors.
|
|
"""
|
|
|
|
# Define max lengths for fields that might still have constraints
|
|
# Even though we're migrating to text, this provides safety
|
|
@max_lengths %{
|
|
# Keep reasonable limits for key fields
|
|
base_callsign: 20,
|
|
sender: 20,
|
|
destination: 20,
|
|
ssid: 10,
|
|
data_type: 50,
|
|
symbol_code: 5,
|
|
symbol_table_id: 5,
|
|
region: 50,
|
|
timestamp: 50,
|
|
message_number: 20,
|
|
addressee: 50,
|
|
|
|
# These fields can be longer but still have sanity limits
|
|
path: 500,
|
|
manufacturer: 100,
|
|
equipment_type: 100,
|
|
device_identifier: 255,
|
|
item_name: 100,
|
|
object_name: 100,
|
|
|
|
# Very long fields
|
|
raw_packet: 5000,
|
|
comment: 2000,
|
|
message_text: 2000
|
|
}
|
|
|
|
# Max lengths for string values inside the JSONB `data` map
|
|
@data_string_max_lengths %{
|
|
"information_field" => 5000,
|
|
"radiorange" => 1000,
|
|
"telemetry_bits" => 1000,
|
|
"format" => 100
|
|
}
|
|
|
|
@doc """
|
|
Sanitizes a packet map by truncating string fields that exceed maximum lengths.
|
|
"""
|
|
@spec sanitize_packet(map()) :: map()
|
|
def sanitize_packet(packet) when is_map(packet) do
|
|
Map.new(packet, fn {key, value} -> {key, sanitize_field(key, value)} end)
|
|
end
|
|
|
|
defp sanitize_field(:data, value) when is_map(value) do
|
|
sanitize_data_map(value)
|
|
end
|
|
|
|
defp sanitize_field(key, value) when is_binary(value) do
|
|
case Map.get(@max_lengths, key) do
|
|
nil ->
|
|
# No limit defined, return as-is
|
|
value
|
|
|
|
max_length ->
|
|
truncate_string(value, max_length)
|
|
end
|
|
end
|
|
|
|
defp sanitize_field(_key, value), do: value
|
|
|
|
defp sanitize_data_map(data) when is_map(data) do
|
|
Map.new(data, fn {key, value} -> {key, sanitize_data_value(key, value)} end)
|
|
end
|
|
|
|
defp sanitize_data_value(key, value) when is_binary(value) do
|
|
case Map.get(@data_string_max_lengths, key) do
|
|
nil -> strip_null_bytes(value)
|
|
max_length -> value |> strip_null_bytes() |> truncate_string(max_length)
|
|
end
|
|
end
|
|
|
|
defp sanitize_data_value(_key, value), do: value
|
|
|
|
# PostgreSQL JSONB does not support \u0000 (null bytes)
|
|
defp strip_null_bytes(string) when is_binary(string) do
|
|
String.replace(string, <<0x00>>, "")
|
|
end
|
|
|
|
defp truncate_string(string, max_length) when byte_size(string) <= max_length do
|
|
string
|
|
end
|
|
|
|
defp truncate_string(string, max_length) do
|
|
# Use binary_part to safely truncate at byte boundaries
|
|
# This prevents splitting UTF-8 characters
|
|
truncated = binary_part(string, 0, max_length)
|
|
|
|
# Ensure we don't end in the middle of a UTF-8 character
|
|
if String.valid?(truncated) do
|
|
truncated
|
|
else
|
|
truncate_to_valid_utf8(string, max_length)
|
|
end
|
|
end
|
|
|
|
defp truncate_to_valid_utf8(string, max_length) do
|
|
# Work backwards from max_length to find a valid UTF-8 boundary
|
|
truncate_to_valid_utf8(string, max_length - 1, max_length)
|
|
end
|
|
|
|
defp truncate_to_valid_utf8(_string, 0, _original_max), do: ""
|
|
|
|
defp truncate_to_valid_utf8(string, current_length, original_max) do
|
|
truncated = binary_part(string, 0, current_length)
|
|
|
|
if String.valid?(truncated) do
|
|
truncated
|
|
else
|
|
truncate_to_valid_utf8(string, current_length - 1, original_max)
|
|
end
|
|
end
|
|
end
|