diff --git a/.formatter.exs b/.formatter.exs index b1c74c3..1d4dd1f 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -2,5 +2,6 @@ import_deps: [:ecto, :ecto_sql, :phoenix, :stream_data], subdirectories: ["priv/*/migrations"], plugins: [Phoenix.LiveView.HTMLFormatter, Styler], - inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"] + inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"], + line_length: 120 ] diff --git a/aprsme.sql b/aprsme.sql deleted file mode 100644 index e69de29..0000000 diff --git a/insert_performance_optimizations.md b/insert_performance_optimizations.md deleted file mode 100644 index eeb5d73..0000000 --- a/insert_performance_optimizations.md +++ /dev/null @@ -1,221 +0,0 @@ -# INSERT Performance Optimizations for Slow Queries - -## Problem Analysis - -The server was experiencing slow INSERT queries with durations of 6+ seconds: -``` -LOG: duration: 6069.632 ms execute ecto_insert_all_packets: INSERT INTO "packets" -``` - -## Root Causes Identified - -1. **Heavy Index Maintenance**: Multiple indexes were being updated on each INSERT -2. **Inefficient Batch Processing**: Excessive overhead in packet preparation -3. **Suboptimal Batch Sizes**: Fixed batch sizes not adapting to system load -4. **Redundant Field Processing**: Expensive operations during INSERT preparation - -## Optimizations Implemented - -### 1. Fast Packet Preparation ✅ - -**File**: `/Users/graham/dev/aprs.me/lib/aprsme/packet_consumer.ex` - -**Created optimized packet preparation pipeline**: -```elixir -# Old approach - expensive processing -defp prepare_packet_for_insert(packet_data) do - # ... extensive processing including data extraction, - # normalization, sanitization, etc. -end - -# New approach - minimal essential processing -defp prepare_packet_for_insert_fast(packet_data, current_time) do - attrs = if is_struct(packet_data), do: Map.from_struct(packet_data), else: packet_data - - attrs - |> Map.put(:received_at, current_time) - |> Map.put(:inserted_at, current_time) - |> Map.put(:updated_at, current_time) - |> extract_essential_fields() - |> create_location_geometry_fast() - |> validate_essential_fields() -end -``` - -**Benefits**: -- ~70% reduction in packet preparation time -- Single timestamp calculation per batch -- Essential fields only extraction -- Fast validation with pattern matching - -### 2. INSERT Performance Optimizer ✅ - -**File**: `/Users/graham/dev/aprs.me/lib/aprsme/performance/insert_optimizer.ex` (NEW) - -**Adaptive batch sizing based on INSERT performance**: -```elixir -# Dynamic batch size optimization -defp calculate_optimal_batch_size(avg_throughput, avg_duration, current_batch_size) do - cond do - # If throughput is low and duration is high, reduce batch size - avg_throughput < 50 and avg_duration > 5000 -> - max(@min_batch_size, round(current_batch_size * 0.8)) - - # If throughput is good and duration is acceptable, increase batch size - avg_throughput > 200 and avg_duration < 2000 -> - min(@max_batch_size, round(current_batch_size * 1.2)) - - true -> current_batch_size - end -end -``` - -**Features**: -- Monitors INSERT throughput (packets/second) -- Adjusts batch size based on performance -- Optimizes INSERT options (returning: false, on_conflict: :nothing) -- Telemetry integration for monitoring - -### 3. Optimized INSERT Options ✅ - -**Before**: -```elixir -Repo.insert_all(Aprsme.Packet, valid_packets, returning: [:id]) -``` - -**After**: -```elixir -insert_options = Aprsme.Performance.InsertOptimizer.get_insert_options() -Repo.insert_all(Aprsme.Packet, valid_packets, insert_options) - -# Options include: -%{ - returning: false, # Don't return IDs unless needed - on_conflict: :nothing, # Skip conflicts instead of raising - timeout: 15_000 # Appropriate timeout -} -``` - -**Benefits**: -- Eliminates unnecessary ID return processing -- Handles conflicts gracefully -- Prevents timeout issues - -### 4. Index Optimization for INSERT Performance ✅ - -**File**: `/Users/graham/dev/aprs.me/priv/repo/migrations/20250714210000_optimize_insert_performance.exs` - -**Removed heavy indexes that slow INSERTs**: -```sql --- Dropped expensive covering index -DROP INDEX packets_sender_received_covering_idx; - --- Replaced with lighter, more selective indexes -CREATE INDEX packets_weather_selective_idx ON packets(received_at DESC) -WHERE data_type IN ('weather', 'Weather', 'WX', 'wx'); - -CREATE INDEX packets_device_recent_idx ON packets(device_identifier, received_at DESC) -WHERE device_identifier IS NOT NULL; - -CREATE INDEX packets_location_selective_idx ON packets USING GIST (location) -WHERE has_position = true; -``` - -**Benefits**: -- Reduced index maintenance overhead during INSERTs -- Maintained query performance for common patterns -- Selective indexes only on relevant data - -### 5. Batch Processing Improvements ✅ - -**Enhanced batch processing**: -```elixir -# Old - fixed batch size -|> Enum.chunk_every(50) - -# New - adaptive batch sizing -batch_size = Aprsme.Performance.InsertOptimizer.get_optimal_batch_size() -|> Enum.chunk_every(batch_size) -``` - -**Optimized reduce operation**: -```elixir -# Single-pass validation and preparation -defp prepare_packets_batch(packets, current_time) do - packets - |> Enum.reduce({[], 0}, fn packet_data, {valid_acc, invalid_count} -> - case prepare_packet_for_insert_fast(packet_data, current_time) do - nil -> {valid_acc, invalid_count + 1} - attrs -> {[attrs | valid_acc], invalid_count} - end - end) -end -``` - -### 6. Performance Monitoring & Telemetry ✅ - -**Added comprehensive INSERT performance metrics**: -```elixir -# Telemetry events -:telemetry.execute([:aprsme, :insert_optimizer, :batch_size], %{value: new_batch_size}) -:telemetry.execute([:aprsme, :insert_optimizer, :throughput], %{value: avg_throughput}) -:telemetry.execute([:aprsme, :insert_optimizer, :duration], %{value: avg_duration}) -``` - -**LiveDashboard integration**: -- INSERT throughput monitoring (packets/second) -- Batch size optimization tracking -- Duration trend analysis -- Performance optimization events - -## Performance Impact - -### Before Optimization: -- INSERT duration: 6+ seconds -- Fixed batch size: 50 packets -- Heavy index maintenance -- Extensive packet processing overhead - -### After Optimization: -- **Expected INSERT duration**: 1-2 seconds (70% improvement) -- **Adaptive batch size**: 100-500 packets based on load -- **Reduced index overhead**: Selective indexes only -- **Minimal processing**: Essential fields only during INSERT - -## Implementation Status - -✅ **Fast packet preparation** - Implemented with pattern matching optimization -✅ **INSERT performance optimizer** - Adaptive batch sizing with telemetry -✅ **Optimized INSERT options** - Reduced returning overhead -✅ **Index optimization** - Selective indexes for better INSERT performance -✅ **Batch processing improvements** - Single-pass validation and preparation -✅ **Performance monitoring** - Comprehensive telemetry integration -✅ **All tests passing** - 351 tests verified - -## Monitoring - -Use the following telemetry metrics to monitor INSERT performance: - -1. **`aprsme.insert_optimizer.throughput`** - Packets per second throughput -2. **`aprsme.insert_optimizer.duration`** - INSERT duration per batch -3. **`aprsme.insert_optimizer.batch_size`** - Current optimized batch size -4. **`aprsme.insert_optimizer.optimizations`** - Number of adjustments made - -## Next Steps - -1. **Monitor production performance** - Watch INSERT durations and throughput -2. **Tune thresholds** - Adjust optimization thresholds based on real data -3. **Consider connection pooling** - If needed, optimize database connections -4. **Index maintenance** - Monitor if additional index optimizations are needed - -## Files Modified - -1. **`lib/aprsme/packet_consumer.ex`** - Fast packet preparation and batch processing -2. **`lib/aprsme/performance/insert_optimizer.ex`** - NEW: Adaptive INSERT optimization -3. **`priv/repo/migrations/20250714210000_optimize_insert_performance.exs`** - Index optimization -4. **`lib/aprsme/application.ex`** - Added InsertOptimizer to supervision tree -5. **`lib/aprsme_web/telemetry.ex`** - INSERT performance telemetry - -## Expected Results - -These optimizations should reduce INSERT query times from 6+ seconds to 1-2 seconds, representing a **70% performance improvement** in packet insertion workloads. \ No newline at end of file diff --git a/lib/aprsme/circuit_breaker.ex b/lib/aprsme/circuit_breaker.ex index f1d5417..c2bc19e 100644 --- a/lib/aprsme/circuit_breaker.ex +++ b/lib/aprsme/circuit_breaker.ex @@ -221,7 +221,11 @@ defmodule Aprsme.CircuitBreaker do } end - defp calculate_current_state(%{state: :open, last_failure_time: last_failure_time, recovery_timeout: recovery_timeout}) do + defp calculate_current_state(%{ + state: :open, + last_failure_time: last_failure_time, + recovery_timeout: recovery_timeout + }) do if last_failure_time && DateTime.diff(DateTime.utc_now(), last_failure_time, :millisecond) >= recovery_timeout do :half_open else diff --git a/lib/aprsme/encoding_utils.ex b/lib/aprsme/encoding_utils.ex index 6a5af5f..ff29805 100644 --- a/lib/aprsme/encoding_utils.ex +++ b/lib/aprsme/encoding_utils.ex @@ -56,7 +56,7 @@ defmodule Aprsme.EncodingUtils do [codepoint] -> # Allow printable characters and common whitespace # Remove C0 controls (0x00-0x1F except tab, newline, carriage return) - # Remove C1 controls (0x80-0x9F) + # Remove C1 controls (0x80-0x9F) # Remove DEL (0x7F) cond do # Tab @@ -138,8 +138,9 @@ defmodule Aprsme.EncodingUtils do # Helper to check if a float is finite (not infinity or NaN) defp finite_float?(float) when is_float(float) do - # NaN != NaN - float != :infinity and float != :neg_infinity and float == float + # In Elixir, we can't have infinity or NaN in regular floats + # This function is kept for defensive programming + true end defp finite_float?(_), do: false diff --git a/lib/aprsme/error_handler.ex b/lib/aprsme/error_handler.ex index 696b6ea..fa2e5d4 100644 --- a/lib/aprsme/error_handler.ex +++ b/lib/aprsme/error_handler.ex @@ -330,10 +330,10 @@ defmodule Aprsme.ErrorHandler do defp categorize_and_handle_error(error, context) do cond do - is_database_error?(error) -> + database_error?(error) -> handle_database_error(error, context) - is_network_error?(error) -> + network_error?(error) -> handle_network_error(error, :unknown_service, context) true -> @@ -349,19 +349,19 @@ defmodule Aprsme.ErrorHandler do end end - defp is_database_error?(%CastError{}), do: true - defp is_database_error?(%Ecto.ConstraintError{}), do: true - defp is_database_error?(%Postgrex.Error{}), do: true - defp is_database_error?(%DBConnection.ConnectionError{}), do: true - defp is_database_error?(%Ecto.QueryError{}), do: true - defp is_database_error?(_), do: false + defp database_error?(%CastError{}), do: true + defp database_error?(%Ecto.ConstraintError{}), do: true + defp database_error?(%Postgrex.Error{}), do: true + defp database_error?(%DBConnection.ConnectionError{}), do: true + defp database_error?(%Ecto.QueryError{}), do: true + defp database_error?(_), do: false - defp is_network_error?(%Req.TransportError{}), do: true - defp is_network_error?(%{reason: :circuit_open}), do: true - defp is_network_error?(%{reason: :timeout}), do: true - defp is_network_error?(%RuntimeError{message: "circuit_open"}), do: true - defp is_network_error?(%RuntimeError{message: "timeout"}), do: true - defp is_network_error?(_), do: false + defp network_error?(%Req.TransportError{}), do: true + defp network_error?(%{reason: :circuit_open}), do: true + defp network_error?(%{reason: :timeout}), do: true + defp network_error?(%RuntimeError{message: "circuit_open"}), do: true + defp network_error?(%RuntimeError{message: "timeout"}), do: true + defp network_error?(_), do: false defp store_bad_packet(packet_data, reason) do # This would store the bad packet for later analysis diff --git a/lib/aprsme/is/is.ex b/lib/aprsme/is/is.ex index 049a848..8785913 100644 --- a/lib/aprsme/is/is.ex +++ b/lib/aprsme/is/is.ex @@ -402,7 +402,12 @@ defmodule Aprsme.Is do case connect_to_aprs_is(state.server, state.port) do {:ok, socket} -> - case send_login_string(socket, state.login_params.user_id, state.login_params.passcode, state.login_params.filter) do + case send_login_string( + socket, + state.login_params.user_id, + state.login_params.passcode, + state.login_params.filter + ) do :ok -> Logger.info("Successfully reconnected to APRS-IS") timer = create_timer(@aprs_timeout) diff --git a/lib/aprsme/packet_replay.ex b/lib/aprsme/packet_replay.ex index 0d371c0..351912d 100644 --- a/lib/aprsme/packet_replay.ex +++ b/lib/aprsme/packet_replay.ex @@ -192,7 +192,9 @@ defmodule Aprsme.PacketReplay do ] # Log the start of replay with map bounds - Logger.info("Starting packet replay for user #{state.user_id} in map area #{inspect(state.bounds)} for the last hour") + Logger.info( + "Starting packet replay for user #{state.user_id} in map area #{inspect(state.bounds)} for the last hour" + ) # Send notification to client that replay is starting Endpoint.broadcast(state.replay_topic, "replay_started", %{ diff --git a/lib/aprsme/packets.ex b/lib/aprsme/packets.ex index a40f73f..d27f63a 100644 --- a/lib/aprsme/packets.ex +++ b/lib/aprsme/packets.ex @@ -30,7 +30,9 @@ defmodule Aprsme.Packets do packet_attrs = sanitized_packet_data - |> Packet.extract_additional_data(sanitized_packet_data[:raw_packet] || sanitized_packet_data["raw_packet"] || "") + |> Packet.extract_additional_data( + sanitized_packet_data[:raw_packet] || sanitized_packet_data["raw_packet"] || "" + ) |> normalize_packet_attrs() |> set_received_at() |> patch_lat_lon_from_data_extended() diff --git a/lib/aprsme_web/components/core_components.ex b/lib/aprsme_web/components/core_components.ex index d80179c..f4bb908 100644 --- a/lib/aprsme_web/components/core_components.ex +++ b/lib/aprsme_web/components/core_components.ex @@ -195,17 +195,8 @@ defmodule AprsmeWeb.CoreComponents do {@title}
{msg}
-