From 584402274c647da7dc72fca6876a691f9deaa1af Mon Sep 17 00:00:00 2001
From: Graham McIntire <graham@mcintire.me>
Date: Mon, 4 Aug 2025 09:33:37 -0500
Subject: [PATCH] fix: Remove all non-schema fields from parsed packets to fix
 database insertion
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The updated APRS parser (commit 69d0ab5) adds several fields that don't exist in our database schema:
- type, digipeaters, daodatumbyte, mbits, message (duplicate of message_text)
- phg (we store individual PHG fields), wx (weather object)
- resultcode, resultmsg, gpsfixstatus

Created a centralized helper function to remove all non-schema fields before database insertion.
Also enhanced error logging to capture sample packet data for easier debugging.

This fixes the production issue where packets were failing to save to the database.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 CHANGELOG.md                  |  6 +++++
 lib/aprsme/packet_consumer.ex | 44 ++++++++++++++++++++++++++++++-----
 2 files changed, 44 insertions(+), 6 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d0d2ca1..113d58e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+- Fixed packet storage failure in production due to non-schema fields from updated APRS parser
+  - Remove `type`, `digipeaters`, `daodatumbyte`, `mbits`, `message`, `phg`, `wx`, `resultcode`, `resultmsg` fields before database insertion
+  - Enhanced error logging to capture sample packet data when batch inserts fail
+  - Created helper function `remove_non_schema_fields/1` to centralize field removal logic
+
 ### Added
 - Database query optimization with 10 new performance indexes
 - Functional index on upper(sender) for case-insensitive searches
diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex
index a82dc84..eee9768 100644
--- a/lib/aprsme/packet_consumer.ex
+++ b/lib/aprsme/packet_consumer.ex
@@ -237,6 +237,14 @@ defmodule Aprsme.PacketConsumer do
     case Repo.insert_all(Aprsme.Packet, valid_packets, insert_opts) do
       {:error, error} ->
         Logger.error("Batch insert failed: #{inspect(error)}")
+
+        # Log sample packet to help debug field issues
+        if length(valid_packets) > 0 do
+          sample_packet = List.first(valid_packets)
+          Logger.error("Sample packet fields: #{inspect(Map.keys(sample_packet))}")
+          Logger.error("Sample packet data: #{inspect(sample_packet)}")
+        end
+
         {0, length(packets)}
 
       {inserted_count, _} ->
@@ -321,12 +329,8 @@ defmodule Aprsme.PacketConsumer do
     |> Map.delete(:data_extended)
     |> normalize_numeric_types()
     |> truncate_datetimes_to_second()
-    # Explicitly remove raw_weather_data to prevent insert_all errors
-    |> Map.delete(:raw_weather_data)
-    |> Map.delete("raw_weather_data")
-    # Remove gpsfixstatus field that was added by the parser but not in schema
-    |> Map.delete(:gpsfixstatus)
-    |> Map.delete("gpsfixstatus")
+    # Remove all non-schema fields
+    |> remove_non_schema_fields()
     # Create PostGIS geometry for location field
     |> create_location_geometry()
   rescue
@@ -353,6 +357,34 @@ defmodule Aprsme.PacketConsumer do
     end
   end
 
+  # Helper function to remove fields that exist in parser output but not in database schema
+  defp remove_non_schema_fields(attrs) do
+    attrs
+    # Remove fields that parser adds but aren't in our schema
+    |> Map.delete(:type)
+    |> Map.delete("type")
+    |> Map.delete(:digipeaters)
+    |> Map.delete("digipeaters")
+    |> Map.delete(:daodatumbyte)
+    |> Map.delete("daodatumbyte")
+    |> Map.delete(:mbits)
+    |> Map.delete("mbits")
+    |> Map.delete(:message)
+    |> Map.delete("message")
+    |> Map.delete(:phg)
+    |> Map.delete("phg")
+    |> Map.delete(:wx)
+    |> Map.delete("wx")
+    |> Map.delete(:resultcode)
+    |> Map.delete("resultcode")
+    |> Map.delete(:resultmsg)
+    |> Map.delete("resultmsg")
+    |> Map.delete(:gpsfixstatus)
+    |> Map.delete("gpsfixstatus")
+    |> Map.delete(:raw_weather_data)
+    |> Map.delete("raw_weather_data")
+  end
+
   # Helper functions for coordinate validation and point creation
   defp valid_coordinates?(lat, lon) do
     lat = normalize_coordinate(lat)