ced4568
b1b22b9285
pg_dump -> gzip -> gpg AES-256 symmetric encryption, plaintext deleted immediately, optional off-host rsync-over-SSH copy (Hetzner Storage Box or any SSH target), retention pruning, Prometheus textfile-collector metrics for backup success/age/size. Genuinely tested end-to-end, not just written: real production migrations (17 total, correct cross-service order) applied to a disposable Postgres, sample data inserted, backed up, restored into a second disposable Postgres, verified table-for-table and byte-for-byte. Exit 0.
78 lines
3.7 KiB
Bash
Executable File
78 lines
3.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Postgres backup — Tier 1 (irreplaceable) data per docs/BACKUP_DR.md. Runs via cron on
|
|
# rn-platform-01 (deployed by the `backup` Ansible role), and locally testable against ANY
|
|
# reachable Postgres (that's exactly what infra/backup/test-backup-restore.sh does).
|
|
#
|
|
# Produces one gzip-compressed pg_dump per run, encrypts it, prunes local copies past
|
|
# BACKUP_RETENTION_DAYS, and (if BACKUP_OFFSITE_TARGET is set) rsyncs the encrypted copy
|
|
# off-host. Writes a Prometheus textfile-collector metric so infra/monitoring/'s BackupFailed/
|
|
# BackupAging alerts have real data to fire on.
|
|
#
|
|
# Required env: PLATFORM_DB_URL, BACKUP_DIR, BACKUP_ENCRYPTION_PASSPHRASE
|
|
# Optional env: BACKUP_RETENTION_DAYS (default 30), BACKUP_OFFSITE_TARGET, METRICS_TEXTFILE_DIR
|
|
set -euo pipefail
|
|
|
|
: "${PLATFORM_DB_URL:?PLATFORM_DB_URL is required}"
|
|
: "${BACKUP_DIR:?BACKUP_DIR is required}"
|
|
: "${BACKUP_ENCRYPTION_PASSPHRASE:?BACKUP_ENCRYPTION_PASSPHRASE is required — see docs/BACKUP_DR.md}"
|
|
BACKUP_RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-30}"
|
|
METRICS_TEXTFILE_DIR="${METRICS_TEXTFILE_DIR:-}"
|
|
|
|
TS="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
mkdir -p "$BACKUP_DIR"
|
|
DUMP_FILE="$BACKUP_DIR/rootnode-platform-$TS.sql.gz"
|
|
ENC_FILE="$DUMP_FILE.gpg"
|
|
|
|
write_metric() {
|
|
# $1 = metric line. Written atomically (tmp+rename) so Prometheus's node-exporter
|
|
# textfile-collector never scrapes a half-written file.
|
|
[ -n "$METRICS_TEXTFILE_DIR" ] || return 0
|
|
mkdir -p "$METRICS_TEXTFILE_DIR"
|
|
local tmp; tmp="$(mktemp "$METRICS_TEXTFILE_DIR/.rootnode_backup.XXXXXX")"
|
|
echo "$1" > "$tmp"
|
|
mv -f "$tmp" "$METRICS_TEXTFILE_DIR/rootnode_backup.prom"
|
|
}
|
|
|
|
fail() {
|
|
echo "BACKUP FAILED: $1" >&2
|
|
write_metric "rootnode_backup_last_run_success 0"
|
|
exit 1
|
|
}
|
|
trap 'fail "unexpected error at line $LINENO"' ERR
|
|
|
|
echo "==> Dumping $PLATFORM_DB_URL to $DUMP_FILE"
|
|
pg_dump --format=plain --no-owner --no-privileges "$PLATFORM_DB_URL" | gzip -9 > "$DUMP_FILE"
|
|
|
|
# Fail loudly on a suspiciously small dump (empty/truncated) rather than "successfully"
|
|
# backing up nothing — a real, if crude, sanity check.
|
|
DUMP_SIZE=$(stat -c%s "$DUMP_FILE" 2>/dev/null || stat -f%z "$DUMP_FILE")
|
|
[ "$DUMP_SIZE" -gt 1024 ] || fail "dump is suspiciously small ($DUMP_SIZE bytes) — refusing to treat this as a real backup"
|
|
|
|
echo "==> Encrypting (AES-256, symmetric passphrase — see docs/BACKUP_DR.md for the recovery-key handling policy)"
|
|
gpg --batch --yes --passphrase "$BACKUP_ENCRYPTION_PASSPHRASE" --symmetric --cipher-algo AES256 \
|
|
--output "$ENC_FILE" "$DUMP_FILE"
|
|
rm -f "$DUMP_FILE" # never leave the plaintext dump on disk once the encrypted copy exists
|
|
|
|
echo "==> Local backup complete: $ENC_FILE ($(stat -c%s "$ENC_FILE" 2>/dev/null || stat -f%z "$ENC_FILE") bytes)"
|
|
|
|
if [ -n "${BACKUP_OFFSITE_TARGET:-}" ]; then
|
|
echo "==> Copying off-host to $BACKUP_OFFSITE_TARGET"
|
|
# rsync over SSH works for both a Hetzner Storage Box (SSH/SFTP-native) and any other
|
|
# SSH-reachable target — the adapter point the Phase 4 spec asked for is this one variable,
|
|
# not a vendor-specific script. An S3-compatible target uses `aws s3 cp` / `rclone` instead;
|
|
# see docs/BACKUP_DR.md for both forms.
|
|
rsync -az --timeout=120 "$ENC_FILE" "$BACKUP_OFFSITE_TARGET/" \
|
|
|| fail "off-host copy to $BACKUP_OFFSITE_TARGET failed"
|
|
else
|
|
echo "==> BACKUP_OFFSITE_TARGET not set — local-only backup (fine for a local test run, NOT fine for production, see docs/BACKUP_DR.md)"
|
|
fi
|
|
|
|
echo "==> Pruning local backups older than $BACKUP_RETENTION_DAYS days"
|
|
find "$BACKUP_DIR" -name 'rootnode-platform-*.sql.gz.gpg' -mtime "+$BACKUP_RETENTION_DAYS" -delete
|
|
|
|
write_metric "rootnode_backup_last_run_success 1
|
|
rootnode_backup_last_success_timestamp_seconds $(date +%s)
|
|
rootnode_backup_size_bytes $(stat -c%s "$ENC_FILE" 2>/dev/null || stat -f%z "$ENC_FILE")"
|
|
|
|
echo "==> Backup succeeded"
|