diff --git a/config/config.exs b/config/config.exs index 06c4c00..9830370 100644 --- a/config/config.exs +++ b/config/config.exs @@ -62,12 +62,12 @@ config :aprsme, packet_pipeline: [ # Larger buffer since inserts are async max_buffer_size: 5000, - # ~32KB per packet, stays within work_mem - batch_size: 500, - # 2 seconds - longer timeout for larger batches - batch_timeout: 2000, - # Higher demand to keep pipeline flowing - max_demand: 750, + # Smaller batch size to reduce memory pressure + batch_size: 100, + # 1 second timeout for smaller batches + batch_timeout: 1000, + # Adjust demand for smaller batches + max_demand: 300, # Number of parallel consumers for better throughput num_consumers: 3 ] diff --git a/lib/aprsme/packet_consumer.ex b/lib/aprsme/packet_consumer.ex index e2d0eff..56da883 100644 --- a/lib/aprsme/packet_consumer.ex +++ b/lib/aprsme/packet_consumer.ex @@ -122,8 +122,8 @@ defmodule Aprsme.PacketConsumer do start_time = System.monotonic_time(:millisecond) # Chunk size optimized for PostgreSQL work_mem=16MB - # With ~32KB per packet, we can fit ~500 packets in work_mem - chunk_size = Application.get_env(:aprsme, :packet_pipeline)[:batch_size] || 500 + # Using smaller batches to reduce memory pressure + chunk_size = Application.get_env(:aprsme, :packet_pipeline)[:batch_size] || 100 # Use Stream for memory-efficient processing results = @@ -167,7 +167,7 @@ defmodule Aprsme.PacketConsumer do end # Always do minor GC after large batches to prevent memory accumulation - if length(packets) > 500 do + if length(packets) > 100 do :erlang.garbage_collect(self(), type: :minor) end