diff --git a/lib/aprsme/release.ex b/lib/aprsme/release.ex index d575143..9d1ac7b 100644 --- a/lib/aprsme/release.ex +++ b/lib/aprsme/release.ex @@ -22,17 +22,24 @@ defmodule Aprsme.Release do # Run migrations with distributed lock cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false) + # Configure longer timeouts for migration operations + migration_timeout = String.to_integer(System.get_env("MIGRATION_TIMEOUT", "7200000")) # 2 hours default + if cluster_enabled do Logger.info("Running migrations with distributed lock...") - # Ensure repo is started for advisory lock - {:ok, _} = Aprsme.Repo.start_link() + # Ensure repo is started for advisory lock with extended timeout + repo_config = Aprsme.Repo.config() + |> Keyword.put(:timeout, migration_timeout) + |> Keyword.put(:pool_timeout, 60_000) + + {:ok, _} = Aprsme.Repo.start_link(repo_config) Aprsme.MigrationLock.with_lock(Aprsme.Repo, fn -> - {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true)) + run_migrations_with_timeout(migration_timeout) end) else Logger.info("Running migrations without lock (single node)...") - {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true)) + run_migrations_with_timeout(migration_timeout) end end @@ -82,6 +89,23 @@ defmodule Aprsme.Release do Application.get_env(:aprsme, :deployed_at) || DateTime.utc_now() end + defp run_migrations_with_timeout(timeout) do + require Logger + + Logger.info("Running migrations with timeout: #{timeout}ms") + + # Run with extended timeout configuration + repo_config = Aprsme.Repo.config() + |> Keyword.put(:timeout, timeout) + |> Keyword.put(:pool_timeout, 60_000) + + {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, fn repo -> + # Set session-level timeout for this connection + Ecto.Adapters.SQL.query!(repo, "SET statement_timeout = '#{div(timeout, 1000)}s'") + Ecto.Migrator.run(repo, :up, all: true) + end, [timeout: timeout]) + end + defp read_deployment_timestamp do case File.read("/app/deployed_at.txt") do {:ok, timestamp} -> diff --git a/priv/repo/migrations/20250726215048_add_optimized_indexes.exs b/priv/repo/migrations/20250726215048_add_optimized_indexes.exs index 27cee69..b1a41e0 100644 --- a/priv/repo/migrations/20250726215048_add_optimized_indexes.exs +++ b/priv/repo/migrations/20250726215048_add_optimized_indexes.exs @@ -3,7 +3,13 @@ defmodule Aprsme.Repo.Migrations.AddOptimizedIndexes do @disable_ddl_transaction true @disable_migration_lock true + # Increase timeout for large table operations + @migration_timeout :timer.minutes(30) + def up do + # Set statement timeout for this migration session + execute "SET statement_timeout = '30min';" + execute "SET lock_timeout = '10s';" # 1. Functional index for case-insensitive callsign searches # This replaces the need for upper() in queries execute """ @@ -54,25 +60,32 @@ defmodule Aprsme.Repo.Migrations.AddOptimizedIndexes do WHERE lat IS NOT NULL AND lon IS NOT NULL """ - # 7. Add has_weather computed column for better query performance - execute """ - ALTER TABLE packets - ADD COLUMN IF NOT EXISTS has_weather boolean - GENERATED ALWAYS AS ( - temperature IS NOT NULL - OR humidity IS NOT NULL - OR pressure IS NOT NULL - OR wind_speed IS NOT NULL - OR wind_direction IS NOT NULL - OR rain_1h IS NOT NULL - ) STORED - """ + # 7. Skip the generated column for now - it requires full table rewrite + # Will be added in a separate migration with better handling + # execute """ + # ALTER TABLE packets + # ADD COLUMN IF NOT EXISTS has_weather boolean + # GENERATED ALWAYS AS ( + # temperature IS NOT NULL + # OR humidity IS NOT NULL + # OR pressure IS NOT NULL + # OR wind_speed IS NOT NULL + # OR wind_direction IS NOT NULL + # OR rain_1h IS NOT NULL + # ) STORED + # """ - # 8. Index on the new has_weather column + # 8. Create functional index instead of relying on generated column + # This avoids the need for a full table rewrite execute """ - CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_packets_has_weather - ON packets (has_weather, sender, received_at DESC) - WHERE has_weather = true + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_packets_has_weather_functional + ON packets (sender, received_at DESC) + WHERE (temperature IS NOT NULL + OR humidity IS NOT NULL + OR pressure IS NOT NULL + OR wind_speed IS NOT NULL + OR wind_direction IS NOT NULL + OR rain_1h IS NOT NULL) """ # 9. Composite index for region-based queries with position @@ -90,14 +103,18 @@ defmodule Aprsme.Repo.Migrations.AddOptimizedIndexes do # Analyze the table to update statistics after adding indexes execute "ANALYZE packets;" + + # Reset statement timeout + execute "RESET statement_timeout;" + execute "RESET lock_timeout;" end def down do # Drop indexes in reverse order execute "DROP INDEX IF EXISTS idx_packets_datatype_sender_time;" execute "DROP INDEX IF EXISTS idx_packets_region_position_time;" - execute "DROP INDEX IF EXISTS idx_packets_has_weather;" - execute "ALTER TABLE packets DROP COLUMN IF EXISTS has_weather;" + execute "DROP INDEX IF EXISTS idx_packets_has_weather_functional;" + # execute "ALTER TABLE packets DROP COLUMN IF EXISTS has_weather;" execute "DROP INDEX IF EXISTS idx_packets_sender_position;" execute "DROP INDEX IF EXISTS idx_packets_weather;" execute "DROP INDEX IF EXISTS idx_packets_base_callsign_time;" diff --git a/priv/repo/migrations/20250727000000_add_has_weather_column.exs b/priv/repo/migrations/20250727000000_add_has_weather_column.exs new file mode 100644 index 0000000..fed7b08 --- /dev/null +++ b/priv/repo/migrations/20250727000000_add_has_weather_column.exs @@ -0,0 +1,78 @@ +defmodule Aprsme.Repo.Migrations.AddHasWeatherColumn do + use Ecto.Migration + @disable_ddl_transaction true + @disable_migration_lock true + + def up do + # This migration adds a generated column which requires a full table rewrite + # It should be run during a maintenance window or when load is low + + # Set aggressive timeout for very large table + execute "SET statement_timeout = '2h';" + execute "SET lock_timeout = '30s';" + + # Log progress + execute "SELECT NOW() AS start_time, 'Starting has_weather column addition' AS status;" + + # Add the column - this will rewrite the entire table + # Consider running this during off-peak hours + execute """ + ALTER TABLE packets + ADD COLUMN IF NOT EXISTS has_weather boolean + GENERATED ALWAYS AS ( + temperature IS NOT NULL + OR humidity IS NOT NULL + OR pressure IS NOT NULL + OR wind_speed IS NOT NULL + OR wind_direction IS NOT NULL + OR rain_1h IS NOT NULL + ) STORED + """ + + # Create index on the new column + execute """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_packets_has_weather + ON packets (has_weather, sender, received_at DESC) + WHERE has_weather = true + """ + + # Drop the functional index if it exists (replaced by column-based index) + execute "DROP INDEX IF EXISTS idx_packets_has_weather_functional;" + + # Update statistics + execute "ANALYZE packets (has_weather);" + + # Reset timeouts + execute "RESET statement_timeout;" + execute "RESET lock_timeout;" + + # Log completion + execute "SELECT NOW() AS end_time, 'Completed has_weather column addition' AS status;" + end + + def down do + execute "SET statement_timeout = '2h';" + execute "SET lock_timeout = '30s';" + + # Drop the index first + execute "DROP INDEX IF EXISTS idx_packets_has_weather;" + + # Drop the column (this will also rewrite the table) + execute "ALTER TABLE packets DROP COLUMN IF EXISTS has_weather;" + + # Recreate the functional index + execute """ + CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_packets_has_weather_functional + ON packets (sender, received_at DESC) + WHERE (temperature IS NOT NULL + OR humidity IS NOT NULL + OR pressure IS NOT NULL + OR wind_speed IS NOT NULL + OR wind_direction IS NOT NULL + OR rain_1h IS NOT NULL) + """ + + execute "RESET statement_timeout;" + execute "RESET lock_timeout;" + end +end \ No newline at end of file