diff --git a/scripts/restore_prod_to_local.sh b/scripts/restore_prod_to_local.sh index 36210f43..6acc95d0 100755 --- a/scripts/restore_prod_to_local.sh +++ b/scripts/restore_prod_to_local.sh @@ -1,31 +1,35 @@ #!/usr/bin/env bash # -# Pull the latest production Postgres dump off the DB server and restore -# it into the local `prop_dev` database, replacing whatever is there. -# -# The production server runs scripts/backup_prop_db.sh on a daily cron, -# which writes pg_dump custom-format dumps to /backups/prop/ as -# `prop_YYYYMMDD_HHMMSS.dump`. We SSH in, pick the newest one, scp it -# locally, then drop/recreate prop_dev and restore into it. +# Dump the production Postgres database and restore it into the local +# `prop_dev`, replacing whatever is there. Pulls prod credentials from +# `.envrc` (direnv handles the export) so there is no duplicated config. # # Usage: # # scripts/restore_prod_to_local.sh -# REMOTE_HOST=my-db-host REMOTE_USER=graham scripts/restore_prod_to_local.sh # # Before running: stop `mix phx.server` so Ecto doesn't hold connections. +# Requires direnv (or `source .envrc`) so PROP_PROD_DB_URL is set. set -euo pipefail -REMOTE_HOST="${REMOTE_HOST:-db-prop}" -REMOTE_USER="${REMOTE_USER:-$USER}" -REMOTE_BACKUP_DIR="${REMOTE_BACKUP_DIR:-/backups/prop}" +: "${PROP_PROD_DB_URL:?not set — run via direnv or 'source .envrc' first}" + LOCAL_DB="${LOCAL_DB:-prop_dev}" LOCAL_USER="${LOCAL_USER:-postgres}" LOCAL_HOST="${LOCAL_HOST:-localhost}" LOCAL_TMP="${LOCAL_TMP:-/tmp}" -ssh_target="${REMOTE_USER}@${REMOTE_HOST}" +# Prod runs Postgres 18; pg_dump refuses to talk to a newer server than +# its own version. Pin to the Homebrew @18 binaries so this works on a +# machine whose default PATH still points at an older series. +PG_BIN="${PG_BIN:-/opt/homebrew/opt/postgresql@18/bin}" +PG_DUMP="${PG_BIN}/pg_dump" +PG_RESTORE="${PG_BIN}/pg_restore" +PSQL="${PG_BIN}/psql" + +timestamp=$(date -u +'%Y%m%d_%H%M%S') +dump_file="${LOCAL_TMP}/prop_prod_${timestamp}.dump" log() { printf '%s restore_prod_to_local: %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*" @@ -36,55 +40,58 @@ die() { exit 1 } -command -v ssh >/dev/null || die "ssh not on PATH" -command -v scp >/dev/null || die "scp not on PATH" -command -v pg_restore >/dev/null || die "pg_restore not on PATH (install postgresql client tools)" -command -v psql >/dev/null || die "psql not on PATH" +[[ -x "$PG_DUMP" ]] || die "$PG_DUMP missing — brew install postgresql@18" +[[ -x "$PG_RESTORE" ]] || die "$PG_RESTORE missing — brew install postgresql@18" +[[ -x "$PSQL" ]] || die "$PSQL missing — brew install postgresql@18" -log "looking up latest dump on ${ssh_target}:${REMOTE_BACKUP_DIR}" -remote_file=$(ssh "$ssh_target" "ls -1t ${REMOTE_BACKUP_DIR}/prop_*.dump 2>/dev/null | head -n1") -[[ -n "$remote_file" ]] || die "no prop_*.dump files found in ${REMOTE_BACKUP_DIR} on ${REMOTE_HOST}" - -filename=$(basename "$remote_file") -local_file="${LOCAL_TMP}/${filename}" -log "latest dump: ${remote_file}" - -# Always remove the downloaded dump, even on mid-run failure. Saves disk -# on laptops where dumps are multi-GB. +# Always remove the downloaded dump, even on mid-run failure. Keeps +# multi-GB dumps from accumulating in /tmp. cleanup() { - [[ -n "${local_file:-}" && -f "$local_file" ]] && rm -f "$local_file" + [[ -n "${dump_file:-}" && -f "$dump_file" ]] && rm -f "$dump_file" } trap cleanup EXIT -log "downloading to ${local_file}" -scp "${ssh_target}:${remote_file}" "$local_file" +log "dumping prod -> ${dump_file}" +# --verbose streams per-object progress to stderr so the run isn't a +# black box against a multi-GB remote DB. +"$PG_DUMP" \ + --dbname="$PROP_PROD_DB_URL" \ + --format=custom \ + --compress=6 \ + --no-owner \ + --no-acl \ + --verbose \ + --file="$dump_file" + +size=$(stat -f '%z' "$dump_file" 2>/dev/null || stat -c '%s' "$dump_file") +log "dump complete ($(numfmt --to=iec "$size" 2>/dev/null || echo "$size bytes"))" # Kick any existing connections off the DB before dropping it — otherwise -# `DROP DATABASE` fails with "is being accessed by other users". This is -# aggressive, but it's a dev box and the user was told to stop the server. +# `DROP DATABASE` fails with "is being accessed by other users". log "terminating existing connections to ${LOCAL_DB}" -psql -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 </dev/null +"$PSQL" -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 </dev/null SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${LOCAL_DB}' AND pid <> pg_backend_pid(); SQL log "dropping and recreating ${LOCAL_DB}" -psql -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 </dev/null +"$PSQL" -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 </dev/null DROP DATABASE IF EXISTS ${LOCAL_DB}; CREATE DATABASE ${LOCAL_DB} OWNER ${LOCAL_USER}; SQL -# --no-owner / --no-acl so ownership in the prod dump doesn't cause role -# errors locally. -j 4 parallelises table data restore. -log "restoring dump into ${LOCAL_DB}" -pg_restore \ +# -j 4 parallelises table data restore; --verbose prints each object as +# workers pick it up so progress is visible. +log "restoring into ${LOCAL_DB}" +"$PG_RESTORE" \ -h "$LOCAL_HOST" \ -U "$LOCAL_USER" \ -d "$LOCAL_DB" \ --no-owner \ --no-acl \ + --verbose \ -j 4 \ - "$local_file" + "$dump_file" -log "done — ${LOCAL_DB} is a copy of prod as of $(basename "$filename" .dump)" +log "done — ${LOCAL_DB} now mirrors prod as of ${timestamp}"