chore(scripts): source restore script config from .envrc, pin pg tools
Swap the SSH+scp-from-backup approach for a direct pg_dump against the prod Postgres using PROP_PROD_DB_URL already in .envrc. Pin the pg_dump/pg_restore/psql binaries to the @18 series so they match the prod server version, and stream per-object progress via --verbose on both tools.
This commit is contained in:
parent
fa0f2828a6
commit
154cb967a1
1 changed files with 46 additions and 39 deletions
|
|
@ -1,31 +1,35 @@
|
||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
#
|
#
|
||||||
# Pull the latest production Postgres dump off the DB server and restore
|
# Dump the production Postgres database and restore it into the local
|
||||||
# it into the local `prop_dev` database, replacing whatever is there.
|
# `prop_dev`, replacing whatever is there. Pulls prod credentials from
|
||||||
#
|
# `.envrc` (direnv handles the export) so there is no duplicated config.
|
||||||
# The production server runs scripts/backup_prop_db.sh on a daily cron,
|
|
||||||
# which writes pg_dump custom-format dumps to /backups/prop/ as
|
|
||||||
# `prop_YYYYMMDD_HHMMSS.dump`. We SSH in, pick the newest one, scp it
|
|
||||||
# locally, then drop/recreate prop_dev and restore into it.
|
|
||||||
#
|
#
|
||||||
# Usage:
|
# Usage:
|
||||||
#
|
#
|
||||||
# scripts/restore_prod_to_local.sh
|
# scripts/restore_prod_to_local.sh
|
||||||
# REMOTE_HOST=my-db-host REMOTE_USER=graham scripts/restore_prod_to_local.sh
|
|
||||||
#
|
#
|
||||||
# Before running: stop `mix phx.server` so Ecto doesn't hold connections.
|
# Before running: stop `mix phx.server` so Ecto doesn't hold connections.
|
||||||
|
# Requires direnv (or `source .envrc`) so PROP_PROD_DB_URL is set.
|
||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
REMOTE_HOST="${REMOTE_HOST:-db-prop}"
|
: "${PROP_PROD_DB_URL:?not set — run via direnv or 'source .envrc' first}"
|
||||||
REMOTE_USER="${REMOTE_USER:-$USER}"
|
|
||||||
REMOTE_BACKUP_DIR="${REMOTE_BACKUP_DIR:-/backups/prop}"
|
|
||||||
LOCAL_DB="${LOCAL_DB:-prop_dev}"
|
LOCAL_DB="${LOCAL_DB:-prop_dev}"
|
||||||
LOCAL_USER="${LOCAL_USER:-postgres}"
|
LOCAL_USER="${LOCAL_USER:-postgres}"
|
||||||
LOCAL_HOST="${LOCAL_HOST:-localhost}"
|
LOCAL_HOST="${LOCAL_HOST:-localhost}"
|
||||||
LOCAL_TMP="${LOCAL_TMP:-/tmp}"
|
LOCAL_TMP="${LOCAL_TMP:-/tmp}"
|
||||||
|
|
||||||
ssh_target="${REMOTE_USER}@${REMOTE_HOST}"
|
# Prod runs Postgres 18; pg_dump refuses to talk to a newer server than
|
||||||
|
# its own version. Pin to the Homebrew @18 binaries so this works on a
|
||||||
|
# machine whose default PATH still points at an older series.
|
||||||
|
PG_BIN="${PG_BIN:-/opt/homebrew/opt/postgresql@18/bin}"
|
||||||
|
PG_DUMP="${PG_BIN}/pg_dump"
|
||||||
|
PG_RESTORE="${PG_BIN}/pg_restore"
|
||||||
|
PSQL="${PG_BIN}/psql"
|
||||||
|
|
||||||
|
timestamp=$(date -u +'%Y%m%d_%H%M%S')
|
||||||
|
dump_file="${LOCAL_TMP}/prop_prod_${timestamp}.dump"
|
||||||
|
|
||||||
log() {
|
log() {
|
||||||
printf '%s restore_prod_to_local: %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*"
|
printf '%s restore_prod_to_local: %s\n' "$(date -u +'%Y-%m-%dT%H:%M:%SZ')" "$*"
|
||||||
|
|
@ -36,55 +40,58 @@ die() {
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
|
||||||
command -v ssh >/dev/null || die "ssh not on PATH"
|
[[ -x "$PG_DUMP" ]] || die "$PG_DUMP missing — brew install postgresql@18"
|
||||||
command -v scp >/dev/null || die "scp not on PATH"
|
[[ -x "$PG_RESTORE" ]] || die "$PG_RESTORE missing — brew install postgresql@18"
|
||||||
command -v pg_restore >/dev/null || die "pg_restore not on PATH (install postgresql client tools)"
|
[[ -x "$PSQL" ]] || die "$PSQL missing — brew install postgresql@18"
|
||||||
command -v psql >/dev/null || die "psql not on PATH"
|
|
||||||
|
|
||||||
log "looking up latest dump on ${ssh_target}:${REMOTE_BACKUP_DIR}"
|
# Always remove the downloaded dump, even on mid-run failure. Keeps
|
||||||
remote_file=$(ssh "$ssh_target" "ls -1t ${REMOTE_BACKUP_DIR}/prop_*.dump 2>/dev/null | head -n1")
|
# multi-GB dumps from accumulating in /tmp.
|
||||||
[[ -n "$remote_file" ]] || die "no prop_*.dump files found in ${REMOTE_BACKUP_DIR} on ${REMOTE_HOST}"
|
|
||||||
|
|
||||||
filename=$(basename "$remote_file")
|
|
||||||
local_file="${LOCAL_TMP}/${filename}"
|
|
||||||
log "latest dump: ${remote_file}"
|
|
||||||
|
|
||||||
# Always remove the downloaded dump, even on mid-run failure. Saves disk
|
|
||||||
# on laptops where dumps are multi-GB.
|
|
||||||
cleanup() {
|
cleanup() {
|
||||||
[[ -n "${local_file:-}" && -f "$local_file" ]] && rm -f "$local_file"
|
[[ -n "${dump_file:-}" && -f "$dump_file" ]] && rm -f "$dump_file"
|
||||||
}
|
}
|
||||||
trap cleanup EXIT
|
trap cleanup EXIT
|
||||||
|
|
||||||
log "downloading to ${local_file}"
|
log "dumping prod -> ${dump_file}"
|
||||||
scp "${ssh_target}:${remote_file}" "$local_file"
|
# --verbose streams per-object progress to stderr so the run isn't a
|
||||||
|
# black box against a multi-GB remote DB.
|
||||||
|
"$PG_DUMP" \
|
||||||
|
--dbname="$PROP_PROD_DB_URL" \
|
||||||
|
--format=custom \
|
||||||
|
--compress=6 \
|
||||||
|
--no-owner \
|
||||||
|
--no-acl \
|
||||||
|
--verbose \
|
||||||
|
--file="$dump_file"
|
||||||
|
|
||||||
|
size=$(stat -f '%z' "$dump_file" 2>/dev/null || stat -c '%s' "$dump_file")
|
||||||
|
log "dump complete ($(numfmt --to=iec "$size" 2>/dev/null || echo "$size bytes"))"
|
||||||
|
|
||||||
# Kick any existing connections off the DB before dropping it — otherwise
|
# Kick any existing connections off the DB before dropping it — otherwise
|
||||||
# `DROP DATABASE` fails with "is being accessed by other users". This is
|
# `DROP DATABASE` fails with "is being accessed by other users".
|
||||||
# aggressive, but it's a dev box and the user was told to stop the server.
|
|
||||||
log "terminating existing connections to ${LOCAL_DB}"
|
log "terminating existing connections to ${LOCAL_DB}"
|
||||||
psql -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 <<SQL >/dev/null
|
"$PSQL" -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 <<SQL >/dev/null
|
||||||
SELECT pg_terminate_backend(pid)
|
SELECT pg_terminate_backend(pid)
|
||||||
FROM pg_stat_activity
|
FROM pg_stat_activity
|
||||||
WHERE datname = '${LOCAL_DB}' AND pid <> pg_backend_pid();
|
WHERE datname = '${LOCAL_DB}' AND pid <> pg_backend_pid();
|
||||||
SQL
|
SQL
|
||||||
|
|
||||||
log "dropping and recreating ${LOCAL_DB}"
|
log "dropping and recreating ${LOCAL_DB}"
|
||||||
psql -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 <<SQL >/dev/null
|
"$PSQL" -h "$LOCAL_HOST" -U "$LOCAL_USER" -d postgres -v ON_ERROR_STOP=1 <<SQL >/dev/null
|
||||||
DROP DATABASE IF EXISTS ${LOCAL_DB};
|
DROP DATABASE IF EXISTS ${LOCAL_DB};
|
||||||
CREATE DATABASE ${LOCAL_DB} OWNER ${LOCAL_USER};
|
CREATE DATABASE ${LOCAL_DB} OWNER ${LOCAL_USER};
|
||||||
SQL
|
SQL
|
||||||
|
|
||||||
# --no-owner / --no-acl so ownership in the prod dump doesn't cause role
|
# -j 4 parallelises table data restore; --verbose prints each object as
|
||||||
# errors locally. -j 4 parallelises table data restore.
|
# workers pick it up so progress is visible.
|
||||||
log "restoring dump into ${LOCAL_DB}"
|
log "restoring into ${LOCAL_DB}"
|
||||||
pg_restore \
|
"$PG_RESTORE" \
|
||||||
-h "$LOCAL_HOST" \
|
-h "$LOCAL_HOST" \
|
||||||
-U "$LOCAL_USER" \
|
-U "$LOCAL_USER" \
|
||||||
-d "$LOCAL_DB" \
|
-d "$LOCAL_DB" \
|
||||||
--no-owner \
|
--no-owner \
|
||||||
--no-acl \
|
--no-acl \
|
||||||
|
--verbose \
|
||||||
-j 4 \
|
-j 4 \
|
||||||
"$local_file"
|
"$dump_file"
|
||||||
|
|
||||||
log "done — ${LOCAL_DB} is a copy of prod as of $(basename "$filename" .dump)"
|
log "done — ${LOCAL_DB} now mirrors prod as of ${timestamp}"
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue