From 706721b06f67ad1e2f55c50e8d8d7d2cc360c00d Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Sat, 9 Aug 2025 16:54:38 -0400
Subject: [PATCH] fix: resolve VARCHAR(255) database errors by migrating to
 TEXT fields

- Create migration to change string fields from VARCHAR(255) to TEXT type
- Add PacketSanitizer module for UTF-8 safe string truncation
- Update PacketConsumer to sanitize data before database insertion
- Prevents database errors from overly long packet data fields
---
 CHANGELOG.md                                  |   6 +
 lib/aprsme/packet_consumer.ex                 |   3 +
 lib/aprsme/packet_sanitizer.ex                | 106 ++++++++++++++++++
 ...000_increase_all_string_fields_to_text.exs |  31 +++++
 4 files changed, 146 insertions(+)
 create mode 100644 lib/aprsme/packet_sanitizer.ex
 create mode 100644 priv/repo/migrations/20250809000000_increase_all_string_fields_to_text.exs

diff --git a/CHANGELOG.md b/CHANGELOG.md
index cfedf01..6b26c88 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,12 +14,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   - `dao` field to store DAO (Datum As Offset) extension data for extra position precision
 - Automatic detection of items/objects during packet processing
 - Database indexes for efficient item/object queries
+- Packet data sanitization to prevent database field overflow errors
+  - Created `Aprsme.PacketSanitizer` module to truncate overly long strings
+  - Implemented UTF-8 safe string truncation
 
 ### Fixed
 - Fixed packet storage failure in production due to non-schema fields from updated APRS parser
   - Remove `type`, `digipeaters`, `daodatumbyte`, `mbits`, `message`, `phg`, `wx`, `resultcode`, `resultmsg` fields before database insertion
   - Enhanced error logging to capture sample packet data when batch inserts fail
   - Created helper function `remove_non_schema_fields/1` to centralize field removal logic
+- Fixed VARCHAR(255) database errors by migrating string fields to TEXT type
+  - Migration increases field sizes for: comment, message_text, raw_packet, body, origpacket, header, radiorange, item_name, object_name, telemetry_bits, device_identifier, format
+  - Added sanitization in PacketConsumer to truncate strings before insertion as safety measure
 
 ### Added
 - Database query optimization with 10 new performance indexes
diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex
index 98b9acf..e930ac4 100644
--- a/lib/aprsme/packet_consumer.ex
+++ b/lib/aprsme/packet_consumer.ex
@@ -306,6 +306,9 @@ defmodule Aprsme.PacketConsumer do
     # Detect and set item/object fields
     attrs = detect_item_or_object(attrs)
 
+    # Sanitize packet data to prevent database field overflow
+    attrs = Aprsme.PacketSanitizer.sanitize_packet(attrs)
+
     # Normalize data_type to string if it's an atom
     attrs = normalize_data_type(attrs)
 
diff --git a/lib/aprsme/packet_sanitizer.ex b/lib/aprsme/packet_sanitizer.ex
new file mode 100644
index 0000000..085fdfc
--- /dev/null
+++ b/lib/aprsme/packet_sanitizer.ex
@@ -0,0 +1,106 @@
+defmodule Aprsme.PacketSanitizer do
+  @moduledoc """
+  Sanitizes packet data to ensure it fits within database constraints.
+  This module provides truncation for long strings to prevent database errors.
+  """
+
+  # Define max lengths for fields that might still have constraints
+  # Even though we're migrating to text, this provides safety
+  @max_lengths %{
+    # Keep reasonable limits for key fields
+    base_callsign: 20,
+    sender: 20,
+    destination: 20,
+    ssid: 10,
+    data_type: 50,
+    symbol_code: 5,
+    symbol_table_id: 5,
+    region: 50,
+    timestamp: 50,
+    message_number: 20,
+    addressee: 50,
+    symboltable: 5,
+    symbolcode: 5,
+    srccallsign: 20,
+    dstcallsign: 20,
+
+    # These fields can be longer but still have sanity limits
+    path: 500,
+    manufacturer: 100,
+    equipment_type: 100,
+    format: 100,
+    device_identifier: 255,
+    item_name: 100,
+    object_name: 100,
+
+    # Very long fields - these will be TEXT in DB but we still
+    # want to prevent abuse with extremely long data
+    information_field: 5000,
+    raw_packet: 5000,
+    comment: 2000,
+    message_text: 2000,
+    body: 5000,
+    origpacket: 5000,
+    header: 1000,
+    radiorange: 1000,
+    telemetry_bits: 1000
+  }
+
+  @doc """
+  Sanitizes a packet map by truncating string fields that exceed maximum lengths.
+  """
+  @spec sanitize_packet(map()) :: map()
+  def sanitize_packet(packet) when is_map(packet) do
+    Enum.reduce(packet, %{}, fn {key, value}, acc ->
+      sanitized_value = sanitize_field(key, value)
+      Map.put(acc, key, sanitized_value)
+    end)
+  end
+
+  defp sanitize_field(key, value) when is_binary(value) do
+    case Map.get(@max_lengths, key) do
+      nil ->
+        # No limit defined, return as-is
+        value
+
+      max_length ->
+        truncate_string(value, max_length)
+    end
+  end
+
+  defp sanitize_field(_key, value), do: value
+
+  defp truncate_string(string, max_length) when byte_size(string) <= max_length do
+    string
+  end
+
+  defp truncate_string(string, max_length) do
+    # Use binary_part to safely truncate at byte boundaries
+    # This prevents splitting UTF-8 characters
+    truncated = binary_part(string, 0, max_length)
+
+    # Ensure we don't end in the middle of a UTF-8 character
+    if String.valid?(truncated) do
+      truncated
+    else
+      truncate_to_valid_utf8(string, max_length)
+    end
+  end
+
+  defp truncate_to_valid_utf8(string, max_length) do
+    # Work backwards from max_length to find a valid UTF-8 boundary
+    truncate_to_valid_utf8(string, max_length - 1, max_length)
+  end
+
+  defp truncate_to_valid_utf8(_string, 0, _original_max), do: ""
+
+  defp truncate_to_valid_utf8(string, current_length, original_max) do
+    truncated = binary_part(string, 0, current_length)
+
+    if String.valid?(truncated) do
+      truncated
+    else
+      truncate_to_valid_utf8(string, current_length - 1, original_max)
+    end
+  end
+end
diff --git a/priv/repo/migrations/20250809000000_increase_all_string_fields_to_text.exs b/priv/repo/migrations/20250809000000_increase_all_string_fields_to_text.exs
new file mode 100644
index 0000000..4a8f971
--- /dev/null
+++ b/priv/repo/migrations/20250809000000_increase_all_string_fields_to_text.exs
@@ -0,0 +1,31 @@
+defmodule Aprsme.Repo.Migrations.IncreaseAllStringFieldsToText do
+  use Ecto.Migration
+
+  def change do
+    alter table(:packets) do
+      # Fields that commonly have longer content
+      modify :comment, :text, from: :string
+      modify :message_text, :text, from: :string
+      modify :raw_packet, :text, from: :string
+      modify :body, :text, from: :string
+      modify :origpacket, :text, from: :string
+      modify :header, :text, from: :string
+
+      # Radio range can be long
+      modify :radiorange, :text, from: :string
+
+      # Object/item names can potentially be long
+      modify :item_name, :text, from: :string
+      modify :object_name, :text, from: :string
+
+      # Telemetry bits field
+      modify :telemetry_bits, :text, from: :string
+
+      # Device identifier could be long
+      modify :device_identifier, :text, from: :string
+
+      # Format field might have long descriptions
+      modify :format, :text, from: :string
+    end
+  end
+end