From 400e444eaac8baab370d01c43c9c9a6ec8403ed7 Mon Sep 17 00:00:00 2001 From: Graham McIntire Date: Tue, 15 Jul 2025 13:34:15 -0500 Subject: [PATCH] Fix packet_counters table bloat with sequence-based counter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add aggressive autovacuum settings for packet_counters table - Replace table-based counter with PostgreSQL sequence to eliminate bloat - Sequences are designed for high-frequency counter operations - Maintains backward compatibility through get_packet_count() function The packet_counters table was experiencing severe bloat due to 350K+ updates on a single row. PostgreSQL sequences avoid this issue entirely. 🤖 Generated with [Claude Code](https://claude.ai/code) Co-Authored-By: Claude --- ...17_optimize_packet_counters_autovacuum.exs | 52 ++++++ ...9_replace_packet_counter_with_sequence.exs | 160 ++++++++++++++++++ 2 files changed, 212 insertions(+) create mode 100644 priv/repo/migrations/20250715183117_optimize_packet_counters_autovacuum.exs create mode 100644 priv/repo/migrations/20250715183139_replace_packet_counter_with_sequence.exs diff --git a/priv/repo/migrations/20250715183117_optimize_packet_counters_autovacuum.exs b/priv/repo/migrations/20250715183117_optimize_packet_counters_autovacuum.exs new file mode 100644 index 0000000..cdbc2fc --- /dev/null +++ b/priv/repo/migrations/20250715183117_optimize_packet_counters_autovacuum.exs @@ -0,0 +1,52 @@ +defmodule Aprsme.Repo.Migrations.OptimizePacketCountersAutovacuum do + use Ecto.Migration + + def up do + # Set aggressive autovacuum settings for the packet_counters table + # This table has only one row that gets updated frequently + execute """ + ALTER TABLE packet_counters SET ( + autovacuum_vacuum_scale_factor = 0.0, + autovacuum_vacuum_threshold = 100, + autovacuum_analyze_scale_factor = 0.0, + autovacuum_analyze_threshold = 100, + autovacuum_vacuum_cost_delay = 0, + autovacuum_vacuum_cost_limit = 10000 + ); + """ + + # Add a fillfactor to reduce page splits during updates + execute """ + ALTER TABLE packet_counters SET (fillfactor = 50); + """ + + # Note: VACUUM FULL cannot run in a transaction, so we'll skip it here + # You can run it manually after the migration: + # VACUUM FULL packet_counters; + + # Create a more efficient counter mechanism using unlogged table for high-frequency updates + # This is an alternative approach if the above doesn't solve the bloat issue + execute """ + COMMENT ON TABLE packet_counters IS 'Single-row table for packet counts. Configured with aggressive autovacuum due to high update frequency.'; + """ + end + + def down do + # Reset to default autovacuum settings + execute """ + ALTER TABLE packet_counters RESET ( + autovacuum_vacuum_scale_factor, + autovacuum_vacuum_threshold, + autovacuum_analyze_scale_factor, + autovacuum_analyze_threshold, + autovacuum_vacuum_cost_delay, + autovacuum_vacuum_cost_limit, + fillfactor + ); + """ + + execute """ + COMMENT ON TABLE packet_counters IS NULL; + """ + end +end diff --git a/priv/repo/migrations/20250715183139_replace_packet_counter_with_sequence.exs b/priv/repo/migrations/20250715183139_replace_packet_counter_with_sequence.exs new file mode 100644 index 0000000..05bd55e --- /dev/null +++ b/priv/repo/migrations/20250715183139_replace_packet_counter_with_sequence.exs @@ -0,0 +1,160 @@ +defmodule Aprsme.Repo.Migrations.ReplacePacketCounterWithSequence do + use Ecto.Migration + + def up do + # Get current count before making changes + execute """ + DO $$ + DECLARE + current_count BIGINT; + BEGIN + SELECT count INTO current_count FROM packet_counters WHERE counter_type = 'total_packets'; + + -- Ensure count is at least 1 (sequences can't start at 0) + IF current_count IS NULL OR current_count < 1 THEN + current_count := 1; + END IF; + + -- Create a sequence starting from current count + EXECUTE format('CREATE SEQUENCE packet_count_seq START WITH %s', current_count); + END $$; + """ + + # Drop the triggers that update the table + execute "DROP TRIGGER IF EXISTS packet_insert_counter ON packets;" + execute "DROP TRIGGER IF EXISTS packet_delete_counter ON packets;" + + # Create new trigger functions that use the sequence + execute """ + CREATE OR REPLACE FUNCTION increment_packet_sequence() + RETURNS TRIGGER AS $$ + BEGIN + PERFORM nextval('packet_count_seq'); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + + execute """ + CREATE OR REPLACE FUNCTION decrement_packet_sequence() + RETURNS TRIGGER AS $$ + BEGIN + PERFORM setval('packet_count_seq', GREATEST(0, currval('packet_count_seq') - 1), true); + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + """ + + # Create new triggers + execute """ + CREATE TRIGGER packet_insert_sequence + AFTER INSERT ON packets + FOR EACH ROW + EXECUTE FUNCTION increment_packet_sequence(); + """ + + execute """ + CREATE TRIGGER packet_delete_sequence + AFTER DELETE ON packets + FOR EACH ROW + EXECUTE FUNCTION decrement_packet_sequence(); + """ + + # Update the get_packet_count function to use the sequence + execute """ + CREATE OR REPLACE FUNCTION get_packet_count() + RETURNS BIGINT AS $$ + BEGIN + RETURN currval('packet_count_seq'); + EXCEPTION + WHEN object_not_in_prerequisite_state THEN + -- If currval hasn't been called yet in this session, get the last value + RETURN last_value FROM packet_count_seq; + END; + $$ LANGUAGE plpgsql STABLE; + """ + + # Keep the packet_counters table for now but stop updating it + execute """ + COMMENT ON TABLE packet_counters IS 'DEPRECATED: This table is no longer actively updated. Use get_packet_count() function which reads from packet_count_seq sequence.'; + """ + end + + def down do + # Restore the original triggers + execute "DROP TRIGGER IF EXISTS packet_insert_sequence ON packets;" + execute "DROP TRIGGER IF EXISTS packet_delete_sequence ON packets;" + + # Restore original trigger functions + execute """ + CREATE OR REPLACE FUNCTION increment_packet_counter() + RETURNS TRIGGER AS $$ + BEGIN + UPDATE packet_counters + SET count = count + 1, + updated_at = NOW() + WHERE counter_type = 'total_packets'; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + """ + + execute """ + CREATE OR REPLACE FUNCTION decrement_packet_counter() + RETURNS TRIGGER AS $$ + BEGIN + UPDATE packet_counters + SET count = GREATEST(0, count - 1), + updated_at = NOW() + WHERE counter_type = 'total_packets'; + RETURN OLD; + END; + $$ LANGUAGE plpgsql; + """ + + # Update the count in the table from the sequence + execute """ + UPDATE packet_counters + SET count = (SELECT last_value FROM packet_count_seq), + updated_at = NOW() + WHERE counter_type = 'total_packets'; + """ + + # Restore triggers + execute """ + CREATE TRIGGER packet_insert_counter + AFTER INSERT ON packets + FOR EACH ROW + EXECUTE FUNCTION increment_packet_counter(); + """ + + execute """ + CREATE TRIGGER packet_delete_counter + AFTER DELETE ON packets + FOR EACH ROW + EXECUTE FUNCTION decrement_packet_counter(); + """ + + # Restore original get_packet_count function + execute """ + CREATE OR REPLACE FUNCTION get_packet_count() + RETURNS BIGINT AS $$ + BEGIN + RETURN (SELECT count FROM packet_counters WHERE counter_type = 'total_packets'); + END; + $$ LANGUAGE plpgsql STABLE; + """ + + # Drop the sequence + execute "DROP SEQUENCE IF EXISTS packet_count_seq;" + + # Remove functions + execute "DROP FUNCTION IF EXISTS increment_packet_sequence();" + execute "DROP FUNCTION IF EXISTS decrement_packet_sequence();" + + # Remove comment + execute """ + COMMENT ON TABLE packet_counters IS NULL; + """ + end +end