#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" DRILL_RUN_ID="drill-$(date -u +%Y%m%dT%H%M%SZ)-$$" CONTAINER_NAME="hyb-oa-backup-drill-$$" DRILL_DIR="$(mktemp -d)" POSTGRES_USER="drill_user" POSTGRES_PASSWORD="drill_password" POSTGRES_DB="hyb_oa_restore_shadow" cleanup() { docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true if [[ -n "${DRILL_DIR:-}" && -d "$DRILL_DIR" && "$DRILL_DIR" == /tmp/* ]]; then rm -rf -- "$DRILL_DIR" fi } trap cleanup EXIT require_command() { if ! command -v "$1" >/dev/null 2>&1; then echo "[ERR] required command missing: $1" >&2 exit 1 fi } for command_name in docker pg_dump pg_restore psql tar sha256sum; do require_command "$command_name" done if ! command -v node >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then echo "[ERR] required command missing: node or python3" >&2 exit 1 fi START_EPOCH="$(date +%s)" SOURCE_STORAGE="$DRILL_DIR/source-storage" RESTORED_STORAGE="$DRILL_DIR/restored-storage" NONEMPTY_STORAGE="$DRILL_DIR/nonempty-storage" BACKUP_ROOT="$DRILL_DIR/backups" mkdir -p "$SOURCE_STORAGE/nested" "$NONEMPTY_STORAGE" "$BACKUP_ROOT" printf 'synthetic attachment for restore drill\n' >"$SOURCE_STORAGE/nested/marker.txt" printf 'must not be overwritten\n' >"$NONEMPTY_STORAGE/existing.txt" docker run -d --rm --name "$CONTAINER_NAME" \ -e "POSTGRES_USER=$POSTGRES_USER" \ -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \ -e "POSTGRES_DB=$POSTGRES_DB" \ -p 127.0.0.1::5432 postgres:16-alpine >/dev/null for _ in $(seq 1 60); do if docker exec "$CONTAINER_NAME" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then break fi sleep 1 done docker exec "$CONTAINER_NAME" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null HOST_PORT="$(docker port "$CONTAINER_NAME" 5432/tcp | awk -F: 'NR==1 {print $NF}')" DATABASE_URL="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@127.0.0.1:$HOST_PORT/$POSTGRES_DB" export DATABASE_URL STORAGE_DIR_HOST="$SOURCE_STORAGE" BACKUP_DIR="$BACKUP_ROOT" BACKUP_RUN_ID="$DRILL_RUN_ID" export DB_TOOL_MODE="docker" DB_CONTAINER="$CONTAINER_NAME" POSTGRES_USER POSTGRES_PASSWORD POSTGRES_DB psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -qc 'CREATE TABLE restore_drill_marker (value text NOT NULL)' psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -qc "INSERT INTO restore_drill_marker(value) VALUES ('synthetic-round-trip')" "$ROOT_DIR/scripts/backup-all.sh" BATCH_DIR="$BACKUP_ROOT/hyb-oa-backup-$DRILL_RUN_ID" DB_BACKUP="$BATCH_DIR/database.dump" STORAGE_BACKUP="$BATCH_DIR/storage.tar.gz" if command -v node >/dev/null 2>&1; then node - "$BATCH_DIR/manifest.json" "$DB_BACKUP" "$STORAGE_BACKUP" <<'NODE' const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); const [manifestPath, dbPath, storagePath] = process.argv.slice(2); const manifestText = fs.readFileSync(manifestPath, "utf8"); const manifest = JSON.parse(manifestText); if (manifest.status !== "complete" || manifest.schemaVersion !== 1) throw new Error("manifest contract mismatch"); if (path.isAbsolute(manifest.artifacts.database.file) || path.isAbsolute(manifest.artifacts.storage.file)) { throw new Error("manifest contains absolute artifact path"); } if (/postgresql:|DATABASE_URL|drill_password/.test(manifestText)) throw new Error("manifest contains sensitive connection data"); const sha256 = (file) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex"); if (sha256(dbPath) !== manifest.artifacts.database.sha256) throw new Error("database checksum mismatch"); if (sha256(storagePath) !== manifest.artifacts.storage.sha256) throw new Error("storage checksum mismatch"); NODE else python3 - "$BATCH_DIR/manifest.json" "$DB_BACKUP" "$STORAGE_BACKUP" <<'PYTHON' import hashlib import json import os import re import sys manifest_path, db_path, storage_path = sys.argv[1:] with open(manifest_path, encoding="utf-8") as manifest_file: manifest_text = manifest_file.read() manifest = json.loads(manifest_text) if manifest.get("status") != "complete" or manifest.get("schemaVersion") != 1: raise RuntimeError("manifest contract mismatch") for artifact in (manifest["artifacts"]["database"], manifest["artifacts"]["storage"]): if os.path.isabs(artifact["file"]): raise RuntimeError("manifest contains absolute artifact path") if re.search(r"postgresql:|DATABASE_URL|drill_password", manifest_text): raise RuntimeError("manifest contains sensitive connection data") def sha256(file_path): digest = hashlib.sha256() with open(file_path, "rb") as artifact_file: for block in iter(lambda: artifact_file.read(1024 * 1024), b""): digest.update(block) return digest.hexdigest() if sha256(db_path) != manifest["artifacts"]["database"]["sha256"]: raise RuntimeError("database checksum mismatch") if sha256(storage_path) != manifest["artifacts"]["storage"]["sha256"]: raise RuntimeError("storage checksum mismatch") PYTHON fi psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -qc 'DROP TABLE restore_drill_marker' docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$CONTAINER_NAME" \ createdb -U "$POSTGRES_USER" hyb_oa_prod if DATABASE_URL="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@127.0.0.1:$HOST_PORT/hyb_oa_prod" \ POSTGRES_DB=hyb_oa_prod RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE=hyb_oa_prod \ "$ROOT_DIR/scripts/restore-db.sh" "$DB_BACKUP" >/dev/null 2>&1; then echo "[ERR] production-like DB name was accepted as a shadow target" >&2 exit 1 fi CORRUPT_DB="$DRILL_DIR/corrupt.dump" head -c 32 "$DB_BACKUP" >"$CORRUPT_DB" if RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ "$ROOT_DIR/scripts/restore-db.sh" "$CORRUPT_DB" >/dev/null 2>&1; then echo "[ERR] corrupt DB dump was accepted" >&2 exit 1 fi if RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE=wrong_target \ "$ROOT_DIR/scripts/restore-db.sh" "$DB_BACKUP" >/dev/null 2>&1; then echo "[ERR] mismatched DB confirmation was accepted" >&2 exit 1 fi RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ "$ROOT_DIR/scripts/restore-db.sh" "$DB_BACKUP" RESTORED_VALUE="$(psql "$DATABASE_URL" -Atqc 'SELECT value FROM restore_drill_marker')" if [[ "$RESTORED_VALUE" != "synthetic-round-trip" ]]; then echo "[ERR] DB round-trip marker mismatch" >&2 exit 1 fi if RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" "$STORAGE_BACKUP" "$NONEMPTY_STORAGE" >/dev/null 2>&1; then echo "[ERR] non-empty storage target was accepted" >&2 exit 1 fi if RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" \ "$STORAGE_BACKUP" "$ROOT_DIR/deploy/trialrun/data/storage" >/dev/null 2>&1; then echo "[ERR] default live storage path was accepted as a shadow target" >&2 exit 1 fi RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" "$STORAGE_BACKUP" "$RESTORED_STORAGE" cmp "$SOURCE_STORAGE/nested/marker.txt" "$RESTORED_STORAGE/nested/marker.txt" CORRUPT_TAR="$DRILL_DIR/corrupt.tar.gz" head -c 32 "$STORAGE_BACKUP" >"$CORRUPT_TAR" if RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" "$CORRUPT_TAR" "$DRILL_DIR/corrupt-target" >/dev/null 2>&1; then echo "[ERR] corrupt storage archive was accepted" >&2 exit 1 fi DANGEROUS_TAR="$DRILL_DIR/dangerous.tar.gz" tar -czf "$DANGEROUS_TAR" --transform='s|nested/marker.txt|../escape.txt|' \ -C "$SOURCE_STORAGE" nested/marker.txt if RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" "$DANGEROUS_TAR" "$DRILL_DIR/dangerous-target" >/dev/null 2>&1; then echo "[ERR] dangerous storage archive path was accepted" >&2 exit 1 fi END_EPOCH="$(date +%s)" ELAPSED_SECONDS="$((END_EPOCH - START_EPOCH))" echo "[PASS] isolated backup/restore drill completed" echo "[INFO] local synthetic RTO seconds: $ELAPSED_SECONDS"