graham/aprs.me
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix migration timeout issues for large tables

Browse source 
- Add statement_timeout and lock_timeout settings to migration
- Remove generated column addition that requires full table rewrite
- Replace with functional index to avoid table rewrite
- Create separate migration for generated column with 2-hour timeout
- Add MIGRATION_TIMEOUT environment variable support (default 2 hours)
- Configure extended timeouts in Release.migrate for large operations

The main migration now uses only CONCURRENTLY operations that don't lock
the table. The generated column addition is deferred to a separate migration
that can be run during a maintenance window.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Graham McIntire 2025-07-27 10:44:16 -05:00
parent 9189579a02
commit 4f7aabbce9
No known key found for this signature in database
3 changed files with 142 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
lib/aprsme/release.ex
View file

@ -22,17 +22,24 @@ defmodule Aprsme.Release do
# Run migrations with distributed lock
cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false)
# Configure longer timeouts for migration operations
migration_timeout = String.to_integer(System.get_env("MIGRATION_TIMEOUT", "7200000")) # 2 hours default
if cluster_enabled do
Logger.info("Running migrations with distributed lock...")
# Ensure repo is started for advisory lock
{:ok, _} = Aprsme.Repo.start_link()
# Ensure repo is started for advisory lock with extended timeout
repo_config = Aprsme.Repo.config()
|> Keyword.put(:timeout, migration_timeout)
|> Keyword.put(:pool_timeout, 60_000)
{:ok, _} = Aprsme.Repo.start_link(repo_config)
Aprsme.MigrationLock.with_lock(Aprsme.Repo, fn ->
{:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true))
run_migrations_with_timeout(migration_timeout)
end)
else
Logger.info("Running migrations without lock (single node)...")
{:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true))
run_migrations_with_timeout(migration_timeout)
end
end
@ -82,6 +89,23 @@ defmodule Aprsme.Release do
Application.get_env(:aprsme, :deployed_at) || DateTime.utc_now()
end
defp run_migrations_with_timeout(timeout) do
require Logger
Logger.info("Running migrations with timeout: #{timeout}ms")
# Run with extended timeout configuration
repo_config = Aprsme.Repo.config()
|> Keyword.put(:timeout, timeout)
|> Keyword.put(:pool_timeout, 60_000)
{:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, fn repo ->
# Set session-level timeout for this connection
Ecto.Adapters.SQL.query!(repo, "SET statement_timeout = '#{div(timeout, 1000)}s'")
Ecto.Migrator.run(repo, :up, all: true)
end, [timeout: timeout])
end
defp read_deployment_timestamp do
case File.read("/app/deployed_at.txt") do
{:ok, timestamp} ->

55
priv/repo/migrations/20250726215048_add_optimized_indexes.exs
View file

@ -3,7 +3,13 @@ defmodule Aprsme.Repo.Migrations.AddOptimizedIndexes do
@disable_ddl_transaction true
@disable_migration_lock true
# Increase timeout for large table operations
@migration_timeout :timer.minutes(30)
def up do
# Set statement timeout for this migration session
execute "SET statement_timeout = '30min';"
execute "SET lock_timeout = '10s';"
# 1. Functional index for case-insensitive callsign searches
# This replaces the need for upper() in queries
execute """
@ -54,25 +60,32 @@ defmodule Aprsme.Repo.Migrations.AddOptimizedIndexes do
WHERE lat IS NOT NULL AND lon IS NOT NULL
"""
# 7. Add has_weather computed column for better query performance
execute """
ALTER TABLE packets
ADD COLUMN IF NOT EXISTS has_weather boolean
GENERATED ALWAYS AS (
temperature IS NOT NULL
OR humidity IS NOT NULL
OR pressure IS NOT NULL
OR wind_speed IS NOT NULL
OR wind_direction IS NOT NULL
OR rain_1h IS NOT NULL
) STORED
"""
# 7. Skip the generated column for now - it requires full table rewrite
# Will be added in a separate migration with better handling
# execute """
# ALTER TABLE packets
# ADD COLUMN IF NOT EXISTS has_weather boolean
# GENERATED ALWAYS AS (
# temperature IS NOT NULL
# OR humidity IS NOT NULL
# OR pressure IS NOT NULL
# OR wind_speed IS NOT NULL
# OR wind_direction IS NOT NULL
# OR rain_1h IS NOT NULL
# ) STORED
# """
# 8. Index on the new has_weather column
# 8. Create functional index instead of relying on generated column
# This avoids the need for a full table rewrite
execute """
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_packets_has_weather
ON packets (has_weather, sender, received_at DESC)
WHERE has_weather = true
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_packets_has_weather_functional
ON packets (sender, received_at DESC)
WHERE (temperature IS NOT NULL
OR humidity IS NOT NULL
OR pressure IS NOT NULL
OR wind_speed IS NOT NULL
OR wind_direction IS NOT NULL
OR rain_1h IS NOT NULL)
"""
# 9. Composite index for region-based queries with position
@ -90,14 +103,18 @@ defmodule Aprsme.Repo.Migrations.AddOptimizedIndexes do
# Analyze the table to update statistics after adding indexes
execute "ANALYZE packets;"
# Reset statement timeout
execute "RESET statement_timeout;"
execute "RESET lock_timeout;"
end
def down do
# Drop indexes in reverse order
execute "DROP INDEX IF EXISTS idx_packets_datatype_sender_time;"
execute "DROP INDEX IF EXISTS idx_packets_region_position_time;"
execute "DROP INDEX IF EXISTS idx_packets_has_weather;"
execute "ALTER TABLE packets DROP COLUMN IF EXISTS has_weather;"
execute "DROP INDEX IF EXISTS idx_packets_has_weather_functional;"
# execute "ALTER TABLE packets DROP COLUMN IF EXISTS has_weather;"
execute "DROP INDEX IF EXISTS idx_packets_sender_position;"
execute "DROP INDEX IF EXISTS idx_packets_weather;"
execute "DROP INDEX IF EXISTS idx_packets_base_callsign_time;"

78
priv/repo/migrations/20250727000000_add_has_weather_column.exs Normal file
View file

@ -0,0 +1,78 @@
defmodule Aprsme.Repo.Migrations.AddHasWeatherColumn do
use Ecto.Migration
@disable_ddl_transaction true
@disable_migration_lock true
def up do
# This migration adds a generated column which requires a full table rewrite
# It should be run during a maintenance window or when load is low
# Set aggressive timeout for very large table
execute "SET statement_timeout = '2h';"
execute "SET lock_timeout = '30s';"
# Log progress
execute "SELECT NOW() AS start_time, 'Starting has_weather column addition' AS status;"
# Add the column - this will rewrite the entire table
# Consider running this during off-peak hours
execute """
ALTER TABLE packets
ADD COLUMN IF NOT EXISTS has_weather boolean
GENERATED ALWAYS AS (
temperature IS NOT NULL
OR humidity IS NOT NULL
OR pressure IS NOT NULL
OR wind_speed IS NOT NULL
OR wind_direction IS NOT NULL
OR rain_1h IS NOT NULL
) STORED
"""
# Create index on the new column
execute """
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_packets_has_weather
ON packets (has_weather, sender, received_at DESC)
WHERE has_weather = true
"""
# Drop the functional index if it exists (replaced by column-based index)
execute "DROP INDEX IF EXISTS idx_packets_has_weather_functional;"
# Update statistics
execute "ANALYZE packets (has_weather);"
# Reset timeouts
execute "RESET statement_timeout;"
execute "RESET lock_timeout;"
# Log completion
execute "SELECT NOW() AS end_time, 'Completed has_weather column addition' AS status;"
end
def down do
execute "SET statement_timeout = '2h';"
execute "SET lock_timeout = '30s';"
# Drop the index first
execute "DROP INDEX IF EXISTS idx_packets_has_weather;"
# Drop the column (this will also rewrite the table)
execute "ALTER TABLE packets DROP COLUMN IF EXISTS has_weather;"
# Recreate the functional index
execute """
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_packets_has_weather_functional
ON packets (sender, received_at DESC)
WHERE (temperature IS NOT NULL
OR humidity IS NOT NULL
OR pressure IS NOT NULL
OR wind_speed IS NOT NULL
OR wind_direction IS NOT NULL
OR rain_1h IS NOT NULL)
"""
execute "RESET statement_timeout;"
execute "RESET lock_timeout;"
end
end