diff --git a/CHANGELOG.md b/CHANGELOG.md index d0d2ca1..113d58e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- Fixed packet storage failure in production due to non-schema fields from updated APRS parser + - Remove `type`, `digipeaters`, `daodatumbyte`, `mbits`, `message`, `phg`, `wx`, `resultcode`, `resultmsg` fields before database insertion + - Enhanced error logging to capture sample packet data when batch inserts fail + - Created helper function `remove_non_schema_fields/1` to centralize field removal logic + ### Added - Database query optimization with 10 new performance indexes - Functional index on upper(sender) for case-insensitive searches diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex index a82dc84..eee9768 100644 --- a/lib/aprsme/packet_consumer.ex +++ b/lib/aprsme/packet_consumer.ex @@ -237,6 +237,14 @@ defmodule Aprsme.PacketConsumer do case Repo.insert_all(Aprsme.Packet, valid_packets, insert_opts) do {:error, error} -> Logger.error("Batch insert failed: #{inspect(error)}") + + # Log sample packet to help debug field issues + if length(valid_packets) > 0 do + sample_packet = List.first(valid_packets) + Logger.error("Sample packet fields: #{inspect(Map.keys(sample_packet))}") + Logger.error("Sample packet data: #{inspect(sample_packet)}") + end + {0, length(packets)} {inserted_count, _} -> @@ -321,12 +329,8 @@ defmodule Aprsme.PacketConsumer do |> Map.delete(:data_extended) |> normalize_numeric_types() |> truncate_datetimes_to_second() - # Explicitly remove raw_weather_data to prevent insert_all errors - |> Map.delete(:raw_weather_data) - |> Map.delete("raw_weather_data") - # Remove gpsfixstatus field that was added by the parser but not in schema - |> Map.delete(:gpsfixstatus) - |> Map.delete("gpsfixstatus") + # Remove all non-schema fields + |> remove_non_schema_fields() # Create PostGIS geometry for location field |> create_location_geometry() rescue @@ -353,6 +357,34 @@ defmodule Aprsme.PacketConsumer do end end + # Helper function to remove fields that exist in parser output but not in database schema + defp remove_non_schema_fields(attrs) do + attrs + # Remove fields that parser adds but aren't in our schema + |> Map.delete(:type) + |> Map.delete("type") + |> Map.delete(:digipeaters) + |> Map.delete("digipeaters") + |> Map.delete(:daodatumbyte) + |> Map.delete("daodatumbyte") + |> Map.delete(:mbits) + |> Map.delete("mbits") + |> Map.delete(:message) + |> Map.delete("message") + |> Map.delete(:phg) + |> Map.delete("phg") + |> Map.delete(:wx) + |> Map.delete("wx") + |> Map.delete(:resultcode) + |> Map.delete("resultcode") + |> Map.delete(:resultmsg) + |> Map.delete("resultmsg") + |> Map.delete(:gpsfixstatus) + |> Map.delete("gpsfixstatus") + |> Map.delete(:raw_weather_data) + |> Map.delete("raw_weather_data") + end + # Helper functions for coordinate validation and point creation defp valid_coordinates?(lat, lon) do lat = normalize_coordinate(lat)