From 9b207e0d6735896003212e6aacfd6e7fd39a9023 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Sun, 3 Aug 2025 09:19:31 -0500 Subject: [PATCH] updates with parser updates --- .claude_session_summary.md | 84 ----- CHANGELOG.md | 9 + assets/js/app.js | 23 +- assets/js/map.ts | 9 + lib/aprsme/packets.ex | 8 +- .../components/layouts/root.html.heex | 2 +- .../live/map_live/historical_loader.ex | 12 + lib/aprsme_web/live/map_live/index.ex | 44 ++- .../live/map_live/packet_batcher.ex | 2 - ...003155_add_parser_compatibility_fields.exs | 21 ++ test-docker-build.sh | 16 - test/aprsme/cluster/leader_election_test.exs | 8 +- test/aprsme/packets_oldest_test.exs | 17 +- .../live/map_live/historical_loading_test.exs | 72 ++-- test/aprsme_web/time_helpers_test.exs | 2 +- .../historical_loading_integration_test.exs | 310 +++++++++--------- vendor/aprs | 2 +- 17 files changed, 317 insertions(+), 324 deletions(-) delete mode 100644 .claude_session_summary.md create mode 100644 priv/repo/migrations/20250803003155_add_parser_compatibility_fields.exs delete mode 100755 test-docker-build.sh diff --git a/.claude_session_summary.md b/.claude_session_summary.md deleted file mode 100644 index 7efa1e4..0000000 --- a/.claude_session_summary.md +++ /dev/null @@ -1,84 +0,0 @@ -# Claude Session Summary - August 2, 2025 - -## Session Overview -This session focused on updating the APRS.me codebase to handle improved APRS parser features based on updates from the vendor/aprs library. - -## Completed Tasks - -### 1. Enhanced APRS Parser Integration -- **Initial Request**: "the parser has been improved again with these updates, please update the code to handle them" -- **Parser Improvements Implemented**: - - Added 11 new standard parser fields for better compatibility - - Implemented comprehensive weather data extraction with dedicated wx field - - Fixed PHG parsing to return string format instead of map structure - - Added radiorange (RNG) field parsing from comments - - Enhanced comment processing with proper data extraction and cleaning - - Improved compressed position parsing with APRS messaging capability - -### 2. Database Schema Updates -- Created migration `20250801231447_add_enhanced_parser_fields.exs` -- Added 20 new fields to packets table: - - Standard parser fields: srccallsign, dstcallsign, body, origpacket, header, alive, posambiguity, symboltable, symbolcode, messaging - - Radio range field: radiorange - - Weather fields: rain_midnight, has_weather (some already existed) - -### 3. Code Updates in lib/aprsme/packet.ex -- Added `put_standard_parser_fields/2` function to extract new parser compatibility fields -- Added `put_radio_range_field/2` function for radiorange extraction -- Enhanced `extract_weather_data/2` to prioritize new wx field from improved parser -- Updated `put_phg_fields/2` to handle both string format ("1060") and legacy map structure -- Added `parse_phg_string/2` for parsing 4-character PHG strings - -### 4. Comprehensive Testing -- Created `test/aprsme/enhanced_parser_test.exs` with 6 tests -- Tests cover: - - Standard parser field extraction - - Radio range field extraction - - Weather data from wx field - - PHG data in both string and map formats - - Full packet storage with enhanced fields -- All 467 tests passing with 0 failures - -### 5. Previous Work in Session -Before the parser update task, the session included: -- Improving test coverage from 41.62% to meet 90% threshold -- Writing comprehensive test suites for Aprsme.Packets, Cluster.LeaderElection, and Cluster.ConnectionManager modules -- Fixing production issues including: - - Buffer full errors (actually PacketConsumer crashes) - - Telemetry_vals data type mismatches - - Missing position_ambiguity column -- Optimizing test suite performance (48% speed improvement) -- Multiple git commits and pushes throughout - -## Key Technical Details - -### Parser Field Mappings -The enhanced parser now provides these additional fields that are mapped to database columns: -- `posambiguity` → position_ambiguity (0-4 level indicator) -- `wx` → dedicated weather data field (prioritized over other weather fields) -- `radiorange` → RNG field from comments (e.g., "RNG0050") -- PHG data now comes as string "1060" instead of map structure - -### Backward Compatibility -All changes maintain backward compatibility: -- Weather extraction checks multiple field locations (wx, weather, weather_report, raw_weather_data) -- PHG parsing handles both string and map formats -- Standard fields are extracted from top-level attributes if present - -## Current State -- All code changes committed and pushed to main branch -- Database migration successfully applied -- Test suite fully passing -- Code formatted and no compilation warnings -- Ready for deployment to production - -## Remaining Tasks -From the todo list: -1. Write tests for Aprsme.Is module (APRS-IS connection) - pending -2. Run final coverage report to verify improvements - pending - -## Notes for Next Session -- The enhanced parser integration is complete and tested -- Consider deploying to production and monitoring for any issues with the new fields -- The Aprsme.Is module still needs test coverage -- May want to verify that the parser improvements are working correctly with live APRS data \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index d084d17..d0d2ca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Supports ambiguity levels 0-4 as defined in APRS specification - Enhanced APRS parser integration with latest improvements - Added `posresolution` field showing position accuracy in meters (18.52m for uncompressed, 0.291m for compressed) +- Upgraded vendor/aprs parser to version 0.1.5+ with comprehensive compatibility improvements + - Added standard APRS parser fields: posambiguity, format, alive, symboltable, symbolcode, messaging + - Enhanced weather data extraction with dedicated `wx` field + - Added radio range (RNG) parsing from comments + - Improved PHG data format handling (now returns string representation) + - Added telemetry fields (seq, vals, bits) extraction + - Better UTF-8 handling and error messages for invalid position data + - Fixed compressed latitude calculation and third-party traffic parsing + - Added support for alternate compressed position formats - Added 20 new database fields for enhanced parser compatibility - Implemented weather data extraction with dedicated `wx` field support - Enhanced PHG parsing to handle both string format and legacy map structure diff --git a/assets/js/app.js b/assets/js/app.js index 428bce0..a86f3c3 100644 --- a/assets/js/app.js +++ b/assets/js/app.js @@ -1,3 +1,5 @@ +console.log("app.js loading..."); + // If you want to use Phoenix channels, run `mix help phx.gen.channel` // to get started and then uncomment the line below. // import "./user_socket.js" @@ -124,23 +126,28 @@ let BodyClassHook = { }, }; -// APRS Map Hook +// APRS MapAPRSMap Hook let Hooks = {}; // Map hooks - load map bundle when needed +// Store original mounted function before creating wrapper +const originalMapMounted = MapAPRSMap.mounted; Hooks.APRSMap = { ...MapAPRSMap, mounted() { + console.log("APRSMap wrapper mounted() called"); const self = this; if (window.VendorLoader && !window.mapBundleLoaded) { + console.log("Loading map bundle..."); // Load map bundle and wait for it to complete const script = document.createElement('script'); script.src = window.VendorLoader.mapBundleUrl; script.onload = () => { + console.log("Map bundle loaded, calling original mounted"); window.mapBundleLoaded = true; // Now call the original mounted function - if (MapAPRSMap.mounted) { - MapAPRSMap.mounted.call(self); + if (originalMapMounted) { + originalMapMounted.call(self); } }; script.onerror = () => { @@ -148,9 +155,10 @@ Hooks.APRSMap = { }; document.head.appendChild(script); } else { + console.log("Map bundle already loaded, calling original mounted"); // Map bundle already loaded, proceed immediately - if (MapAPRSMap.mounted) { - MapAPRSMap.mounted.call(this); + if (originalMapMounted) { + originalMapMounted.call(this); } } } @@ -176,7 +184,7 @@ Hooks.InfoMap = { }; document.head.appendChild(script); } else { - // Map bundle already loaded, proceed immediately + // MapAPRSMap bundle already loaded, proceed immediately if (InfoMap.mounted) { InfoMap.mounted.call(this); } @@ -263,7 +271,7 @@ window.localStorage.setItem("theme", theme); window.reRenderAllCharts = () => { // Store all chart instances globally so we can access them if (!window.chartInstances) { - window.chartInstances = new Map(); + window.chartInstances = new MapAPRSMap(); } // Re-render all stored chart instances @@ -310,6 +318,7 @@ document.addEventListener("DOMContentLoaded", () => { }); }); +console.log("Creating LiveSocket with hooks:", Object.keys(Hooks)); let liveSocket = new LiveSocket("/live", Socket, { longPollFallbackMs: 2500, params: { _csrf_token: csrfToken }, diff --git a/assets/js/map.ts b/assets/js/map.ts index e4953ca..34e1065 100644 --- a/assets/js/map.ts +++ b/assets/js/map.ts @@ -64,6 +64,7 @@ import { let MapAPRSMap = { mounted() { const self = this as unknown as LiveViewHookContext; + console.log("APRSMap hook mounted() called on element:", this.el); // Initialize error tracking self.errors = []; self.initializationAttempts = 0; @@ -438,6 +439,7 @@ let MapAPRSMap = { if (self.map) { saveMapState(self.map, self.pushEvent.bind(self)); // Also send bounds to server to trigger historical loading + console.log("Calling sendBoundsToServer after map_ready (retry)"); self.sendBoundsToServer(); } // Also trigger map state update after a delay @@ -1059,6 +1061,11 @@ let MapAPRSMap = { self.handleEvent( "add_historical_packets_batch", (data: { packets: MarkerData[]; batch: number; is_final: boolean }) => { + console.log("Received add_historical_packets_batch event:", { + packetCount: data.packets?.length || 0, + batch: data.batch, + is_final: data.is_final + }); try { if (data.packets && Array.isArray(data.packets)) { // Process all packets immediately for maximum speed @@ -1325,6 +1332,7 @@ let MapAPRSMap = { sendBoundsToServer() { const self = this as unknown as LiveViewHookContext; + console.log("sendBoundsToServer called, map:", !!self.map, "isDestroyed:", self.isDestroyed); if (!self.map || self.isDestroyed) return; try { @@ -1351,6 +1359,7 @@ let MapAPRSMap = { }, zoom: zoom, }; + console.log("Sending bounds_changed event:", boundsData); self.pushEvent("bounds_changed", boundsData); } } catch (error) { diff --git a/lib/aprsme/packets.ex b/lib/aprsme/packets.ex index aa6adc2..0d516d0 100644 --- a/lib/aprsme/packets.ex +++ b/lib/aprsme/packets.ex @@ -435,6 +435,10 @@ defmodule Aprsme.Packets do @impl true @spec get_recent_packets(map()) :: [struct()] def get_recent_packets(opts \\ %{}) do + require Logger + + Logger.debug("Packets.get_recent_packets called with opts: #{inspect(opts)}") + # Use hours_back from opts if provided, otherwise default to 24 hours hours_back = Map.get(opts, :hours_back, 24) time_ago = DateTime.add(DateTime.utc_now(), -hours_back * 3600, :second) @@ -475,7 +479,9 @@ defmodule Aprsme.Packets do |> offset(^offset) |> QueryBuilder.with_coordinates() - Repo.all(query) + result = Repo.all(query) + Logger.debug("Packets.get_recent_packets returning #{length(result)} packets") + result end @doc """ diff --git a/lib/aprsme_web/components/layouts/root.html.heex b/lib/aprsme_web/components/layouts/root.html.heex index c576c66..c669f70 100644 --- a/lib/aprsme_web/components/layouts/root.html.heex +++ b/lib/aprsme_web/components/layouts/root.html.heex @@ -76,7 +76,7 @@ -