diff --git a/k8s/README-migrations.md b/k8s/README-migrations.md new file mode 100644 index 0000000..5235943 --- /dev/null +++ b/k8s/README-migrations.md @@ -0,0 +1,69 @@ +# Database Migration Strategy for Kubernetes + +This document describes how database migrations are handled in the Kubernetes deployment to ensure only one node runs migrations at a time. + +## Overview + +In a distributed environment with multiple pods, we need to ensure that database migrations only run once, not multiple times concurrently. We use two strategies: + +1. **PostgreSQL Advisory Locks** - For runtime migration coordination +2. **Init Containers** - For deployment-time migrations (recommended) + +## Implementation + +### 1. PostgreSQL Advisory Locks + +The `Aprsme.MigrationLock` module uses PostgreSQL advisory locks to ensure only one node can run migrations at a time: + +- Uses `pg_try_advisory_lock()` to acquire a non-blocking lock +- Other nodes wait for the lock to be released +- Automatically releases the lock after migrations complete + +### 2. Init Container (Recommended) + +The StatefulSet can be configured with an init container that runs migrations before the main pods start: + +```bash +# Apply the init container patch +kubectl patch statefulset aprs -n aprs --patch-file k8s/statefulset-init-container-patch.yaml +``` + +This approach: +- Runs migrations sequentially before any pods start +- Prevents race conditions +- Makes migration failures visible in pod events + +### 3. Auto-Migration Disabled in Cluster Mode + +When `CLUSTER_ENABLED=true`, automatic migrations on startup are disabled to prevent race conditions. + +## Manual Migration + +To run migrations manually: + +```bash +# Run on the first pod +kubectl exec -it aprs-0 -n aprs -- /app/bin/migrate + +# Or create a one-off job +kubectl run migrate-job --rm -it --image=ghcr.io/aprsme/aprs.me:latest \ + --env="DATABASE_URL=$DATABASE_URL" \ + --env="SECRET_KEY_BASE=$SECRET_KEY_BASE" \ + --env="MIX_ENV=prod" \ + --env="SKIP_DB_CREATE=true" \ + --restart=Never \ + -- /app/bin/migrate +``` + +## Configuration + +Environment variables: +- `CLUSTER_ENABLED=true` - Enables distributed locking +- `SKIP_DB_CREATE=true` - Skips database creation (for PgBouncer) + +## Best Practices + +1. **Use Init Containers** for production deployments +2. **Test migrations** in a staging environment first +3. **Monitor migration logs** during deployments +4. **Have rollback plan** ready with `mix ecto.rollback` \ No newline at end of file diff --git a/k8s/statefulset-init-container-patch.yaml b/k8s/statefulset-init-container-patch.yaml new file mode 100644 index 0000000..e9e46a0 --- /dev/null +++ b/k8s/statefulset-init-container-patch.yaml @@ -0,0 +1,37 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: aprs + namespace: aprs +spec: + template: + spec: + initContainers: + - name: migrate + image: ghcr.io/aprsme/aprs.me:latest + imagePullPolicy: Always + command: ["/app/bin/migrate"] + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + key: database-url-pgbouncer + name: aprs-secret + - name: SECRET_KEY_BASE + valueFrom: + secretKeyRef: + key: secret-key-base + name: aprs-secret + - name: MIX_ENV + value: prod + - name: SKIP_DB_CREATE + value: "true" + - name: CLUSTER_ENABLED + value: "false" # Disable clustering for migration runner + resources: + limits: + cpu: 200m + memory: 256Mi + requests: + cpu: 100m + memory: 128Mi \ No newline at end of file diff --git a/lib/aprsme/application.ex b/lib/aprsme/application.ex index 1752dee..f37dbb7 100644 --- a/lib/aprsme/application.ex +++ b/lib/aprsme/application.ex @@ -91,7 +91,20 @@ defmodule Aprsme.Application do defp migrate do auto_migrate = Application.get_env(:aprsme, :auto_migrate, true) - do_migrate(auto_migrate) + cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false) + + # In cluster mode, prefer init containers or manual migration + # to avoid race conditions between nodes + if auto_migrate and not cluster_enabled do + do_migrate(true) + else + require Logger + if cluster_enabled do + Logger.info("Skipping auto-migration in cluster mode") + else + Logger.info("Auto-migration disabled") + end + end # Gettext translations are automatically compiled during Mix compilation rescue diff --git a/lib/aprsme/migration_lock.ex b/lib/aprsme/migration_lock.ex new file mode 100644 index 0000000..29ed076 --- /dev/null +++ b/lib/aprsme/migration_lock.ex @@ -0,0 +1,98 @@ +defmodule Aprsme.MigrationLock do + @moduledoc """ + Provides distributed locking for database migrations using PostgreSQL advisory locks. + This ensures only one node runs migrations at a time in a clustered environment. + """ + + require Logger + + # Use a consistent lock ID for migrations + # This is a 64-bit integer that's unlikely to conflict with other locks + @migration_lock_id 8_764_293_847_291 + + @doc """ + Attempts to acquire an exclusive advisory lock and run migrations. + Returns :ok if migrations were run successfully or :skipped if another node holds the lock. + """ + def with_lock(repo, fun) do + case acquire_lock(repo) do + :ok -> + try do + Logger.info("Acquired migration lock, running migrations...") + result = fun.() + Logger.info("Migrations completed successfully") + result + after + release_lock(repo) + Logger.info("Released migration lock") + end + + :locked -> + Logger.info("Another node is running migrations, waiting...") + wait_for_migrations(repo) + :skipped + end + end + + defp acquire_lock(repo) do + # Try to acquire an exclusive advisory lock (non-blocking) + query = "SELECT pg_try_advisory_lock($1)" + + case repo.query(query, [@migration_lock_id]) do + {:ok, %{rows: [[true]]}} -> + :ok + + {:ok, %{rows: [[false]]}} -> + :locked + + error -> + Logger.error("Failed to acquire migration lock: #{inspect(error)}") + :locked + end + end + + defp release_lock(repo) do + query = "SELECT pg_advisory_unlock($1)" + + case repo.query(query, [@migration_lock_id]) do + {:ok, %{rows: [[true]]}} -> + :ok + + error -> + Logger.error("Failed to release migration lock: #{inspect(error)}") + error + end + end + + defp wait_for_migrations(repo) do + # Wait for up to 60 seconds for migrations to complete + wait_for_migrations(repo, 60) + end + + defp wait_for_migrations(_repo, 0) do + Logger.warn("Timeout waiting for migrations to complete") + :timeout + end + + defp wait_for_migrations(repo, retries) do + # Check if lock is still held + query = "SELECT pg_try_advisory_lock($1)" + + case repo.query(query, [@migration_lock_id]) do + {:ok, %{rows: [[true]]}} -> + # We got the lock, which means migrations are done + release_lock(repo) + Logger.info("Migrations completed on another node") + :ok + + {:ok, %{rows: [[false]]}} -> + # Lock is still held, wait and retry + Process.sleep(1_000) + wait_for_migrations(repo, retries - 1) + + error -> + Logger.error("Error checking migration lock: #{inspect(error)}") + :error + end + end +end \ No newline at end of file diff --git a/lib/aprsme/release.ex b/lib/aprsme/release.ex index 110b118..d575143 100644 --- a/lib/aprsme/release.ex +++ b/lib/aprsme/release.ex @@ -19,8 +19,21 @@ defmodule Aprsme.Release do create_database() end - # Run migrations - {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true)) + # Run migrations with distributed lock + cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false) + + if cluster_enabled do + Logger.info("Running migrations with distributed lock...") + # Ensure repo is started for advisory lock + {:ok, _} = Aprsme.Repo.start_link() + + Aprsme.MigrationLock.with_lock(Aprsme.Repo, fn -> + {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true)) + end) + else + Logger.info("Running migrations without lock (single node)...") + {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true)) + end end defp create_database do