graham/aprs.me
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix constant garbage collection warnings by monitoring per-process memory

Browse source 
The memory monitoring was incorrectly using total VM memory instead of
per-process memory, causing false positives whenever any process in the
system allocated memory.

Changes:
- Changed memory monitoring to per-process instead of total VM memory
- Reduced thresholds to reasonable per-process values (50MB diff, 100MB total)
- Increased minor GC trigger from 100 to 1000 packets to prevent GC after
  every normal batch

This prevents detecting memory changes from other processes and eliminates
the constant GC warnings while still maintaining proper memory management.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Graham McIntire 2025-07-20 07:55:28 -05:00
parent c8bd7980c2
commit d3a5ed33f2
No known key found for this signature in database
1 changed files with 10 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
lib/aprsme/packet_consumer.ex
View file

@ -117,8 +117,8 @@ defmodule Aprsme.PacketConsumer do
defp process_batch(packets) do defp process_batch(packets) do
require Logger require Logger
# Monitor memory usage before processing # Monitor memory usage before processing (only this process)
memory_before = :erlang.memory(:total) {:memory, memory_before} = Process.info(self(), :memory)
start_time = System.monotonic_time(:millisecond) start_time = System.monotonic_time(:millisecond)
# Chunk size optimized for PostgreSQL work_mem=16MB # Chunk size optimized for PostgreSQL work_mem=16MB
@ -140,17 +140,17 @@ defmodule Aprsme.PacketConsumer do
end_time = System.monotonic_time(:millisecond) end_time = System.monotonic_time(:millisecond)
duration = end_time - start_time duration = end_time - start_time
# Monitor memory usage after processing # Monitor memory usage after processing (only this process)
memory_after = :erlang.memory(:total) {:memory, memory_after} = Process.info(self(), :memory)
memory_diff = memory_after - memory_before memory_diff = memory_after - memory_before
# Get current process memory info # Get current process memory info
process_info = Process.info(self(), [:memory, :heap_size, :total_heap_size]) process_info = Process.info(self(), [:memory, :heap_size, :total_heap_size])
# Force garbage collection if memory usage is high # Force garbage collection if memory usage is high
# 500MB threshold for memory diff (we have 15GB available) # 50MB threshold for memory diff (per process)
# 1GB threshold for process memory # 100MB threshold for process memory
if memory_diff > 524_288_000 or process_info[:memory] > 1_073_741_824 do if memory_diff > 52_428_800 or process_info[:memory] > 104_857_600 do
:erlang.garbage_collect() :erlang.garbage_collect()
Logger.warning("High memory usage detected, forced garbage collection", Logger.warning("High memory usage detected, forced garbage collection",
@ -166,8 +166,9 @@ defmodule Aprsme.PacketConsumer do
) )
end end
# Always do minor GC after large batches to prevent memory accumulation # Only do minor GC after very large batches to prevent memory accumulation
if length(packets) > 100 do # This should rarely trigger with batch_size of 100
if length(packets) > 1000 do
:erlang.garbage_collect(self(), type: :minor) :erlang.garbage_collect(self(), type: :minor)
end end