Graham McIntire
ca05b497b9
Drop 10 dead parser-compat columns (srccallsign, dstcallsign, origpacket, body, header, alive, posambiguity, symboltable, symbolcode, messaging) and move 14 display-only columns into a single `data` JSONB column (PHG, telemetry, radiorange, information_field, format, posresolution, position_ambiguity, luminosity, rain_midnight). Reduces row width from ~70 to ~48 columns. Table is ephemeral so migration uses DROP/recreate. Also fixes snow_24h -> snow bug in weather check query and adds null byte stripping for JSONB string sanitization.
124 lines
3.4 KiB
Elixir
124 lines
3.4 KiB
Elixir
defmodule Aprsme.PacketSanitizer do
|
|
@moduledoc """
|
|
Sanitizes packet data to ensure it fits within database constraints.
|
|
This module provides truncation for long strings to prevent database errors.
|
|
"""
|
|
|
|
# Define max lengths for fields that might still have constraints
|
|
# Even though we're migrating to text, this provides safety
|
|
@max_lengths %{
|
|
# Keep reasonable limits for key fields
|
|
base_callsign: 20,
|
|
sender: 20,
|
|
destination: 20,
|
|
ssid: 10,
|
|
data_type: 50,
|
|
symbol_code: 5,
|
|
symbol_table_id: 5,
|
|
region: 50,
|
|
timestamp: 50,
|
|
message_number: 20,
|
|
addressee: 50,
|
|
|
|
# These fields can be longer but still have sanity limits
|
|
path: 500,
|
|
manufacturer: 100,
|
|
equipment_type: 100,
|
|
device_identifier: 255,
|
|
item_name: 100,
|
|
object_name: 100,
|
|
|
|
# Very long fields
|
|
raw_packet: 5000,
|
|
comment: 2000,
|
|
message_text: 2000
|
|
}
|
|
|
|
# Max lengths for string values inside the JSONB `data` map
|
|
@data_string_max_lengths %{
|
|
"information_field" => 5000,
|
|
"radiorange" => 1000,
|
|
"telemetry_bits" => 1000,
|
|
"format" => 100
|
|
}
|
|
|
|
@doc """
|
|
Sanitizes a packet map by truncating string fields that exceed maximum lengths.
|
|
"""
|
|
@spec sanitize_packet(map()) :: map()
|
|
def sanitize_packet(packet) when is_map(packet) do
|
|
Enum.reduce(packet, %{}, fn {key, value}, acc ->
|
|
sanitized_value = sanitize_field(key, value)
|
|
Map.put(acc, key, sanitized_value)
|
|
end)
|
|
end
|
|
|
|
defp sanitize_field(:data, value) when is_map(value) do
|
|
sanitize_data_map(value)
|
|
end
|
|
|
|
defp sanitize_field(key, value) when is_binary(value) do
|
|
case Map.get(@max_lengths, key) do
|
|
nil ->
|
|
# No limit defined, return as-is
|
|
value
|
|
|
|
max_length ->
|
|
truncate_string(value, max_length)
|
|
end
|
|
end
|
|
|
|
defp sanitize_field(_key, value), do: value
|
|
|
|
defp sanitize_data_map(data) when is_map(data) do
|
|
Enum.reduce(data, %{}, fn {key, value}, acc ->
|
|
sanitized =
|
|
case {is_binary(value), Map.get(@data_string_max_lengths, key)} do
|
|
{true, nil} -> strip_null_bytes(value)
|
|
{true, max_length} -> value |> strip_null_bytes() |> truncate_string(max_length)
|
|
_ -> value
|
|
end
|
|
|
|
Map.put(acc, key, sanitized)
|
|
end)
|
|
end
|
|
|
|
# PostgreSQL JSONB does not support \u0000 (null bytes)
|
|
defp strip_null_bytes(string) when is_binary(string) do
|
|
String.replace(string, <<0x00>>, "")
|
|
end
|
|
|
|
defp truncate_string(string, max_length) when byte_size(string) <= max_length do
|
|
string
|
|
end
|
|
|
|
defp truncate_string(string, max_length) do
|
|
# Use binary_part to safely truncate at byte boundaries
|
|
# This prevents splitting UTF-8 characters
|
|
truncated = binary_part(string, 0, max_length)
|
|
|
|
# Ensure we don't end in the middle of a UTF-8 character
|
|
if String.valid?(truncated) do
|
|
truncated
|
|
else
|
|
truncate_to_valid_utf8(string, max_length)
|
|
end
|
|
end
|
|
|
|
defp truncate_to_valid_utf8(string, max_length) do
|
|
# Work backwards from max_length to find a valid UTF-8 boundary
|
|
truncate_to_valid_utf8(string, max_length - 1, max_length)
|
|
end
|
|
|
|
defp truncate_to_valid_utf8(_string, 0, _original_max), do: ""
|
|
|
|
defp truncate_to_valid_utf8(string, current_length, original_max) do
|
|
truncated = binary_part(string, 0, current_length)
|
|
|
|
if String.valid?(truncated) do
|
|
truncated
|
|
else
|
|
truncate_to_valid_utf8(string, current_length - 1, original_max)
|
|
end
|
|
end
|
|
end
|