Implement distributed migration locking for cluster deployments

- Add MigrationLock module using PostgreSQL advisory locks
- Prevent concurrent migrations across multiple nodes
- Update Release.migrate to use distributed lock when cluster enabled
- Disable auto-migrations in cluster mode to avoid race conditions
- Add init container configuration for Kubernetes deployments
- Document migration strategies in k8s/README-migrations.md

This ensures only one node runs database migrations at a time, preventing
conflicts and race conditions in distributed deployments.

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <noreply@anthropic.com>

k8s/README-migrations.md

@@ -0,0 +1,69 @@
+# Database Migration Strategy for Kubernetes
+
+This document describes how database migrations are handled in the Kubernetes deployment to ensure only one node runs migrations at a time.
+
+## Overview
+
+In a distributed environment with multiple pods, we need to ensure that database migrations only run once, not multiple times concurrently. We use two strategies:
+
+1. **PostgreSQL Advisory Locks** - For runtime migration coordination
+2. **Init Containers** - For deployment-time migrations (recommended)
+
+## Implementation
+
+### 1. PostgreSQL Advisory Locks
+
+The `Aprsme.MigrationLock` module uses PostgreSQL advisory locks to ensure only one node can run migrations at a time:
+
+- Uses `pg_try_advisory_lock()` to acquire a non-blocking lock
+- Other nodes wait for the lock to be released
+- Automatically releases the lock after migrations complete
+
+### 2. Init Container (Recommended)
+
+The StatefulSet can be configured with an init container that runs migrations before the main pods start:
+
+```bash
+# Apply the init container patch
+kubectl patch statefulset aprs -n aprs --patch-file k8s/statefulset-init-container-patch.yaml
+```
+
+This approach:
+- Runs migrations sequentially before any pods start
+- Prevents race conditions
+- Makes migration failures visible in pod events
+
+### 3. Auto-Migration Disabled in Cluster Mode
+
+When `CLUSTER_ENABLED=true`, automatic migrations on startup are disabled to prevent race conditions.
+
+## Manual Migration
+
+To run migrations manually:
+
+```bash
+# Run on the first pod
+kubectl exec -it aprs-0 -n aprs -- /app/bin/migrate
+
+# Or create a one-off job
+kubectl run migrate-job --rm -it --image=ghcr.io/aprsme/aprs.me:latest \
+  --env="DATABASE_URL=$DATABASE_URL" \
+  --env="SECRET_KEY_BASE=$SECRET_KEY_BASE" \
+  --env="MIX_ENV=prod" \
+  --env="SKIP_DB_CREATE=true" \
+  --restart=Never \
+  -- /app/bin/migrate
+```
+
+## Configuration
+
+Environment variables:
+- `CLUSTER_ENABLED=true` - Enables distributed locking
+- `SKIP_DB_CREATE=true` - Skips database creation (for PgBouncer)
+
+## Best Practices
+
+1. **Use Init Containers** for production deployments
+2. **Test migrations** in a staging environment first
+3. **Monitor migration logs** during deployments
+4. **Have rollback plan** ready with `mix ecto.rollback`

k8s/statefulset-init-container-patch.yaml

@@ -0,0 +1,37 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: aprs
+  namespace: aprs
+spec:
+  template:
+    spec:
+      initContainers:
+      - name: migrate
+        image: ghcr.io/aprsme/aprs.me:latest
+        imagePullPolicy: Always
+        command: ["/app/bin/migrate"]
+        env:
+        - name: DATABASE_URL
+          valueFrom:
+            secretKeyRef:
+              key: database-url-pgbouncer
+              name: aprs-secret
+        - name: SECRET_KEY_BASE
+          valueFrom:
+            secretKeyRef:
+              key: secret-key-base
+              name: aprs-secret
+        - name: MIX_ENV
+          value: prod
+        - name: SKIP_DB_CREATE
+          value: "true"
+        - name: CLUSTER_ENABLED
+          value: "false"  # Disable clustering for migration runner
+        resources:
+          limits:
+            cpu: 200m
+            memory: 256Mi
+          requests:
+            cpu: 100m
+            memory: 128Mi

lib/aprsme/application.ex

@@ -91,7 +91,20 @@ defmodule Aprsme.Application do
 
   defp migrate do
     auto_migrate = Application.get_env(:aprsme, :auto_migrate, true)
-    do_migrate(auto_migrate)
+    cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false)
+    
+    # In cluster mode, prefer init containers or manual migration
+    # to avoid race conditions between nodes
+    if auto_migrate and not cluster_enabled do
+      do_migrate(true)
+    else
+      require Logger
+      if cluster_enabled do
+        Logger.info("Skipping auto-migration in cluster mode")
+      else
+        Logger.info("Auto-migration disabled")
+      end
+    end
 
     # Gettext translations are automatically compiled during Mix compilation
   rescue

lib/aprsme/migration_lock.ex

@@ -0,0 +1,98 @@
+defmodule Aprsme.MigrationLock do
+  @moduledoc """
+  Provides distributed locking for database migrations using PostgreSQL advisory locks.
+  This ensures only one node runs migrations at a time in a clustered environment.
+  """
+
+  require Logger
+
+  # Use a consistent lock ID for migrations
+  # This is a 64-bit integer that's unlikely to conflict with other locks
+  @migration_lock_id 8_764_293_847_291
+
+  @doc """
+  Attempts to acquire an exclusive advisory lock and run migrations.
+  Returns :ok if migrations were run successfully or :skipped if another node holds the lock.
+  """
+  def with_lock(repo, fun) do
+    case acquire_lock(repo) do
+      :ok ->
+        try do
+          Logger.info("Acquired migration lock, running migrations...")
+          result = fun.()
+          Logger.info("Migrations completed successfully")
+          result
+        after
+          release_lock(repo)
+          Logger.info("Released migration lock")
+        end
+
+      :locked ->
+        Logger.info("Another node is running migrations, waiting...")
+        wait_for_migrations(repo)
+        :skipped
+    end
+  end
+
+  defp acquire_lock(repo) do
+    # Try to acquire an exclusive advisory lock (non-blocking)
+    query = "SELECT pg_try_advisory_lock($1)"
+    
+    case repo.query(query, [@migration_lock_id]) do
+      {:ok, %{rows: [[true]]}} ->
+        :ok
+
+      {:ok, %{rows: [[false]]}} ->
+        :locked
+
+      error ->
+        Logger.error("Failed to acquire migration lock: #{inspect(error)}")
+        :locked
+    end
+  end
+
+  defp release_lock(repo) do
+    query = "SELECT pg_advisory_unlock($1)"
+    
+    case repo.query(query, [@migration_lock_id]) do
+      {:ok, %{rows: [[true]]}} ->
+        :ok
+
+      error ->
+        Logger.error("Failed to release migration lock: #{inspect(error)}")
+        error
+    end
+  end
+
+  defp wait_for_migrations(repo) do
+    # Wait for up to 60 seconds for migrations to complete
+    wait_for_migrations(repo, 60)
+  end
+
+  defp wait_for_migrations(_repo, 0) do
+    Logger.warn("Timeout waiting for migrations to complete")
+    :timeout
+  end
+
+  defp wait_for_migrations(repo, retries) do
+    # Check if lock is still held
+    query = "SELECT pg_try_advisory_lock($1)"
+    
+    case repo.query(query, [@migration_lock_id]) do
+      {:ok, %{rows: [[true]]}} ->
+        # We got the lock, which means migrations are done
+        release_lock(repo)
+        Logger.info("Migrations completed on another node")
+        :ok
+
+      {:ok, %{rows: [[false]]}} ->
+        # Lock is still held, wait and retry
+        Process.sleep(1_000)
+        wait_for_migrations(repo, retries - 1)
+
+      error ->
+        Logger.error("Error checking migration lock: #{inspect(error)}")
+        :error
+    end
+  end
+end

lib/aprsme/release.ex

@@ -19,8 +19,21 @@ defmodule Aprsme.Release do
       create_database()
     end
 
-    # Run migrations
-    {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true))
+    # Run migrations with distributed lock
+    cluster_enabled = Application.get_env(:aprsme, :cluster_enabled, false)
+    
+    if cluster_enabled do
+      Logger.info("Running migrations with distributed lock...")
+      # Ensure repo is started for advisory lock
+      {:ok, _} = Aprsme.Repo.start_link()
+      
+      Aprsme.MigrationLock.with_lock(Aprsme.Repo, fn ->
+        {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true))
+      end)
+    else
+      Logger.info("Running migrations without lock (single node)...")
+      {:ok, _, _} = Ecto.Migrator.with_repo(Aprsme.Repo, &Ecto.Migrator.run(&1, :up, all: true))
+    end
   end
 
   defp create_database do