chore(scripts): source restore script config from .envrc, pin pg tools

Swap the SSH+scp-from-backup approach for a direct pg_dump against the
prod Postgres using PROP_PROD_DB_URL already in .envrc. Pin the
pg_dump/pg_restore/psql binaries to the @18 series so they match the
prod server version, and stream per-object progress via --verbose on
both tools.
This commit is contained in:
Graham McIntire 2026-04-18 09:36:36 -05:00
parent fa0f2828a6
commit 154cb967a1
No known key found for this signature in database
GPG key ID: F4ABF488E6029E59

View file

@ -1,31 +1,35 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# #
# Pull the latest production Postgres dump off the DB server and restore # Dump the production Postgres database and restore it into the local
# it into the local `prop_dev` database, replacing whatever is there. # `prop_dev`, replacing whatever is there. Pulls prod credentials from
# # `.envrc` (direnv handles the export) so there is no duplicated config.
# The production server runs scripts/backup_prop_db.sh on a daily cron,
# which writes pg_dump custom-format dumps to /backups/prop/ as
# `prop_YYYYMMDD_HHMMSS.dump`. We SSH in, pick the newest one, scp it
# locally, then drop/recreate prop_dev and restore into it.
# #
# Usage: # Usage:
# #
# scripts/restore_prod_to_local.sh # scripts/restore_prod_to_local.sh
# REMOTE_HOST=my-db-host REMOTE_USER=graham scripts/restore_prod_to_local.sh
# #
# Before running: stop `mix phx.server` so Ecto doesn't hold connections. # Before running: stop `mix phx.server` so Ecto doesn't hold connections.
# Requires direnv (or `source .envrc`) so PROP_PROD_DB_URL is set.
set -euo pipefail set -euo pipefail
REMOTE_HOST="${REMOTE_HOST:-db-prop}" : "${PROP_PROD_DB_URL:?not set — run via direnv or 'source .envrc' first}"
REMOTE_USER="${REMOTE_USER:-$USER}"
REMOTE_BACKUP_DIR="${REMOTE_BACKUP_DIR:-/backups/prop}"
LOCAL_DB="${LOCAL_DB:-prop_dev}" LOCAL_DB="${LOCAL_DB:-prop_dev}"
LOCAL_USER="${LOCAL_USER:-postgres}" LOCAL_USER="${LOCAL_USER:-postgres}"
LOCAL_HOST="${LOCAL_HOST:-localhost}" LOCAL_HOST="${LOCAL_HOST:-localhost}"
LOCAL_TMP="${LOCAL_TMP:-/tmp}" LOCAL_TMP="${LOCAL_TMP:-/tmp}"
ssh_target="${REMOTE_USER}@${REMOTE_HOST}" # Prod runs Postgres 18; pg_dump refuses to talk to a newer server than
# its own version. Pin to the Homebrew @18 binaries so this works on a
# machine whose default PATH still points at an older series.
PG_BIN="${PG_BIN:-/opt/homebrew/opt/postgresql@18/bin}"
PG_DUMP="${PG_BIN}/pg_dump"
PG_RESTORE="${PG_BIN}/pg_restore"
PSQL="${PG_BIN}/psql"
timestamp=$(date -u +'%Y%m%d_%H%M%S')
dump_file="${LOCAL_TMP}/prop_prod_${timestamp}.dump"
log() { log() {
printf '%s restore_prod_to_local: %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*" printf '%s restore_prod_to_local: %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*"
@ -36,55 +40,58 @@ die() {
exit 1 exit 1
} }
command -v ssh >/dev/null || die "ssh not on PATH" [[ -x "$PG_DUMP" ]] || die "$PG_DUMP missing — brew install postgresql@18"
command -v scp >/dev/null || die "scp not on PATH" [[ -x "$PG_RESTORE" ]] || die "$PG_RESTORE missing — brew install postgresql@18"
command -v pg_restore >/dev/null || die "pg_restore not on PATH (install postgresql client tools)" [[ -x "$PSQL" ]] || die "$PSQL missing — brew install postgresql@18"
command -v psql >/dev/null || die "psql not on PATH"
log "looking up latest dump on ${ssh_target}:${REMOTE_BACKUP_DIR}" # Always remove the downloaded dump, even on mid-run failure. Keeps
remote_file=$(ssh "$ssh_target" "ls -1t ${REMOTE_BACKUP_DIR}/prop_*.dump 2>/dev/null | head -n1") # multi-GB dumps from accumulating in /tmp.
[[ -n "$remote_file" ]] || die "no prop_*.dump files found in ${REMOTE_BACKUP_DIR} on ${REMOTE_HOST}"
filename=$(basename "$remote_file")
local_file="${LOCAL_TMP}/${filename}"
log "latest dump: ${remote_file}"
# Always remove the downloaded dump, even on mid-run failure. Saves disk
# on laptops where dumps are multi-GB.
cleanup() { cleanup() {
[[ -n "${local_file:-}" && -f "$local_file" ]] && rm -f "$local_file" [[ -n "${dump_file:-}" && -f "$dump_file" ]] && rm -f "$dump_file"
} }
trap cleanup EXIT trap cleanup EXIT
log "downloading to ${local_file}" log "dumping prod -> ${dump_file}"
scp "${ssh_target}:${remote_file}" "$local_file" # --verbose streams per-object progress to stderr so the run isn't a
# black box against a multi-GB remote DB.
"$PG_DUMP" \
--dbname="$PROP_PROD_DB_URL" \
--format=custom \
--compress=6 \
--no-owner \
--no-acl \
--verbose \
--file="$dump_file"
size=$(stat -f '%z' "$dump_file" 2>/dev/null || stat -c '%s' "$dump_file")
log "dump complete ($(numfmt --to=iec "$size" 2>/dev/null || echo "$size bytes"))"
# Kick any existing connections off the DB before dropping it — otherwise # Kick any existing connections off the DB before dropping it — otherwise
# `DROP DATABASE` fails with "is being accessed by other users". This is # `DROP DATABASE` fails with "is being accessed by other users".
# aggressive, but it's a dev box and the user was told to stop the server.
log "terminating existing connections to ${LOCAL_DB}" log "terminating existing connections to ${LOCAL_DB}"
psql -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 <<SQL >/dev/null "$PSQL" -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 <<SQL >/dev/null
SELECT pg_terminate_backend(pid) SELECT pg_terminate_backend(pid)
FROM pg_stat_activity FROM pg_stat_activity
WHERE datname = '${LOCAL_DB}' AND pid <> pg_backend_pid(); WHERE datname = '${LOCAL_DB}' AND pid <> pg_backend_pid();
SQL SQL
log "dropping and recreating ${LOCAL_DB}" log "dropping and recreating ${LOCAL_DB}"
psql -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 <<SQL >/dev/null "$PSQL" -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 <<SQL >/dev/null
DROP DATABASE IF EXISTS ${LOCAL_DB}; DROP DATABASE IF EXISTS ${LOCAL_DB};
CREATE DATABASE ${LOCAL_DB} OWNER ${LOCAL_USER}; CREATE DATABASE ${LOCAL_DB} OWNER ${LOCAL_USER};
SQL SQL
# --no-owner / --no-acl so ownership in the prod dump doesn't cause role # -j 4 parallelises table data restore; --verbose prints each object as
# errors locally. -j 4 parallelises table data restore. # workers pick it up so progress is visible.
log "restoring dump into ${LOCAL_DB}" log "restoring into ${LOCAL_DB}"
pg_restore \ "$PG_RESTORE" \
-h "$LOCAL_HOST" \ -h "$LOCAL_HOST" \
-U "$LOCAL_USER" \ -U "$LOCAL_USER" \
-d "$LOCAL_DB" \ -d "$LOCAL_DB" \
--no-owner \ --no-owner \
--no-acl \ --no-acl \
--verbose \
-j 4 \ -j 4 \
"$local_file" "$dump_file"
log "done — ${LOCAL_DB} is a copy of prod as of $(basename "$filename" .dump)" log "done — ${LOCAL_DB} now mirrors prod as of ${timestamp}"