hyb-oa/scripts/qa/run-backup-restore-drill.sh

#!/usr/bin/env bash
set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
DRILL_RUN_ID="drill-$(date -u +%Y%m%dT%H%M%SZ)-$$"
CONTAINER_NAME="hyb-oa-backup-drill-$$"
DRILL_DIR="$(mktemp -d)"
POSTGRES_USER="drill_user"
POSTGRES_PASSWORD="drill_password"
POSTGRES_DB="hyb_oa_restore_shadow"

cleanup() {
  docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
  if [[ -n "${DRILL_DIR:-}" && -d "$DRILL_DIR" && "$DRILL_DIR" == /tmp/* ]]; then
    rm -rf -- "$DRILL_DIR"
  fi
}
trap cleanup EXIT

require_command() {
  if ! command -v "$1" >/dev/null 2>&1; then
    echo "[ERR] required command missing: $1" >&2
    exit 1
  fi
}

for command_name in docker pg_dump pg_restore psql tar sha256sum; do
  require_command "$command_name"
done
if ! command -v node >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then
  echo "[ERR] required command missing: node or python3" >&2
  exit 1
fi

START_EPOCH="$(date +%s)"
SOURCE_STORAGE="$DRILL_DIR/source-storage"
RESTORED_STORAGE="$DRILL_DIR/restored-storage"
NONEMPTY_STORAGE="$DRILL_DIR/nonempty-storage"
BACKUP_ROOT="$DRILL_DIR/backups"
mkdir -p "$SOURCE_STORAGE/nested" "$NONEMPTY_STORAGE" "$BACKUP_ROOT"
printf 'synthetic attachment for restore drill\n' >"$SOURCE_STORAGE/nested/marker.txt"
printf 'must not be overwritten\n' >"$NONEMPTY_STORAGE/existing.txt"

if BUILD_SHA=not-a-commit SOURCE_IMAGE_DIGEST= \
  BACKUP_DIR="$BACKUP_ROOT" BACKUP_RUN_ID=invalid-source \
  "$ROOT_DIR/scripts/backup-all.sh" >/dev/null 2>&1; then
  echo "[ERR] invalid backup source identity was accepted" >&2
  exit 1
fi

if env -u STORAGE_DIR_HOST -u STORAGE_DIR \
  OUT_FILE="$DRILL_DIR/implicit-storage.tar.gz" \
  "$ROOT_DIR/scripts/backup-storage.sh" >/dev/null 2>&1; then
  echo "[ERR] implicit storage backup source was accepted" >&2
  exit 1
fi

if env -u DB_CONTAINER DB_TOOL_MODE=docker DATABASE_URL=postgresql://invalid \
  POSTGRES_DB=invalid POSTGRES_USER=invalid POSTGRES_PASSWORD=invalid \
  OUT_FILE="$DRILL_DIR/implicit-db.dump" \
  "$ROOT_DIR/scripts/backup-db.sh" >/dev/null 2>&1; then
  echo "[ERR] implicit DB container was accepted" >&2
  exit 1
fi

if command -v node >/dev/null 2>&1; then
  if deploy_error="$(DEPLOY_SERVER=unused POSTGRES_DATA_DIR=./relative \
    STORAGE_HOST_DIR=/srv/hyb-oa/storage DEPLOY_DATA_MODE=existing \
    "$ROOT_DIR/deploy/trialrun/deploy.sh" 2>&1)"; then
    echo "[ERR] formal deploy accepted a relative PostgreSQL data path" >&2
    exit 1
  fi
  if [[ "$deploy_error" != *"POSTGRES_DATA_DIR"* ]]; then
    echo "[ERR] relative deploy path did not fail at the data-path guard" >&2
    exit 1
  fi

  if deploy_error="$(DEPLOY_SERVER=unused POSTGRES_DATA_DIR=/srv/hyb-oa/data \
    STORAGE_HOST_DIR=/srv/hyb-oa/data DEPLOY_DATA_MODE=existing \
    "$ROOT_DIR/deploy/trialrun/deploy.sh" 2>&1)"; then
    echo "[ERR] formal deploy accepted overlapping data paths" >&2
    exit 1
  fi
  if [[ "$deploy_error" != *"必须相互独立"* ]]; then
    echo "[ERR] overlapping deploy paths did not fail at the isolation guard" >&2
    exit 1
  fi

  if deploy_error="$(DEPLOY_SERVER=unused REMOTE_DIR=/srv/hyb-oa/app \
    POSTGRES_DATA_DIR=/srv/hyb-oa/app/data/postgres \
    STORAGE_HOST_DIR=/srv/hyb-oa/storage DEPLOY_DATA_MODE=existing \
    "$ROOT_DIR/deploy/trialrun/deploy.sh" 2>&1)"; then
    echo "[ERR] formal deploy accepted PostgreSQL data nested in the source sync root" >&2
    exit 1
  fi
  if [[ "$deploy_error" != *"POSTGRES_DATA_DIR 不得与 REMOTE_DIR"* ]]; then
    echo "[ERR] nested PostgreSQL path did not fail at the source-sync isolation guard" >&2
    exit 1
  fi

  if deploy_error="$(DEPLOY_SERVER=unused REMOTE_DIR=/srv/hyb-oa/app \
    POSTGRES_DATA_DIR=/srv/hyb-oa/postgres \
    STORAGE_HOST_DIR=/srv/hyb-oa/app/data/storage DEPLOY_DATA_MODE=existing \
    "$ROOT_DIR/deploy/trialrun/deploy.sh" 2>&1)"; then
    echo "[ERR] formal deploy accepted attachment storage nested in the source sync root" >&2
    exit 1
  fi
  if [[ "$deploy_error" != *"STORAGE_HOST_DIR 不得与 REMOTE_DIR"* ]]; then
    echo "[ERR] nested storage path did not fail at the source-sync isolation guard" >&2
    exit 1
  fi

  MUTATED_PORT_COMPOSE="$DRILL_DIR/docker-compose-db-port.yml"
  sed '/  db:/a\    ports:\n      - "5432:5432"' \
    "$ROOT_DIR/deploy/trialrun/docker-compose.yml" >"$MUTATED_PORT_COMPOSE"
  if ENV_CONTRACT_COMPOSE_PATH="$MUTATED_PORT_COMPOSE" \
    node "$ROOT_DIR/deploy/trialrun/validate-env-contract.js" >/dev/null 2>&1; then
    echo "[ERR] unsafe DB port publication passed the contract" >&2
    exit 1
  fi

  MUTATED_NAME_COMPOSE="$DRILL_DIR/docker-compose-fixed-name.yml"
  sed '/  db:/a\    container_name: forbidden-fixed-name' \
    "$ROOT_DIR/deploy/trialrun/docker-compose.yml" >"$MUTATED_NAME_COMPOSE"
  if ENV_CONTRACT_COMPOSE_PATH="$MUTATED_NAME_COMPOSE" \
    node "$ROOT_DIR/deploy/trialrun/validate-env-contract.js" >/dev/null 2>&1; then
    echo "[ERR] fixed Compose container name passed the contract" >&2
    exit 1
  fi

  MUTATED_DEPLOY="$DRILL_DIR/deploy-without-project.sh"
  sed 's/-p "$COMPOSE_PROJECT_NAME" //g' \
    "$ROOT_DIR/deploy/trialrun/deploy.sh" >"$MUTATED_DEPLOY"
  if ENV_CONTRACT_DEPLOY_SCRIPT_PATH="$MUTATED_DEPLOY" \
    node "$ROOT_DIR/deploy/trialrun/validate-env-contract.js" >/dev/null 2>&1; then
    echo "[ERR] deployment without an explicit Compose project passed the contract" >&2
    exit 1
  fi
fi

docker run -d --rm --name "$CONTAINER_NAME" \
  -e "POSTGRES_USER=$POSTGRES_USER" \
  -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \
  -e "POSTGRES_DB=$POSTGRES_DB" \
  -p 127.0.0.1::5432 postgres:16-alpine >/dev/null

for _ in $(seq 1 60); do
  if docker exec "$CONTAINER_NAME" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null 2>&1; then
    break
  fi
  sleep 1
done
docker exec "$CONTAINER_NAME" pg_isready -U "$POSTGRES_USER" -d "$POSTGRES_DB" >/dev/null

HOST_PORT="$(docker port "$CONTAINER_NAME" 5432/tcp | awk -F: 'NR==1 {print $NF}')"
DATABASE_URL="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@127.0.0.1:$HOST_PORT/$POSTGRES_DB"
export DATABASE_URL STORAGE_DIR_HOST="$SOURCE_STORAGE" BACKUP_DIR="$BACKUP_ROOT" BACKUP_RUN_ID="$DRILL_RUN_ID"
export DB_TOOL_MODE="docker" DB_CONTAINER="$CONTAINER_NAME" POSTGRES_USER POSTGRES_PASSWORD POSTGRES_DB
if [[ "${DRILL_SOURCE_ID_MODE:-both}" == "digest_only" ]]; then
  export BUILD_SHA=""
else
  export BUILD_SHA="abcdef1234567890abcdef1234567890abcdef12"
fi
export SOURCE_IMAGE_DIGEST="sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"

psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -qc 'CREATE TABLE restore_drill_marker (value text NOT NULL)'
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -qc "INSERT INTO restore_drill_marker(value) VALUES ('synthetic-round-trip')"

"$ROOT_DIR/scripts/backup-all.sh"
BATCH_DIR="$BACKUP_ROOT/hyb-oa-backup-$DRILL_RUN_ID"
DB_BACKUP="$BATCH_DIR/database.dump"
STORAGE_BACKUP="$BATCH_DIR/storage.tar.gz"

if command -v node >/dev/null 2>&1; then
  node - "$BATCH_DIR/manifest.json" "$DB_BACKUP" "$STORAGE_BACKUP" "$BUILD_SHA" "$SOURCE_IMAGE_DIGEST" <<'NODE'
const crypto = require("node:crypto");
const fs = require("node:fs");
const path = require("node:path");
const [manifestPath, dbPath, storagePath, expectedBuildSha, expectedImageDigest] = process.argv.slice(2);
const manifestText = fs.readFileSync(manifestPath, "utf8");
const manifest = JSON.parse(manifestText);
if (manifest.status !== "complete" || manifest.schemaVersion !== 1) throw new Error("manifest contract mismatch");
if (manifest.buildSha !== (expectedBuildSha || null) || manifest.sourceImageDigest !== expectedImageDigest) {
  throw new Error("manifest source identity mismatch");
}
if (path.isAbsolute(manifest.artifacts.database.file) || path.isAbsolute(manifest.artifacts.storage.file)) {
  throw new Error("manifest contains absolute artifact path");
}
if (/postgresql:|DATABASE_URL|drill_password/.test(manifestText)) throw new Error("manifest contains sensitive connection data");
const sha256 = (file) => crypto.createHash("sha256").update(fs.readFileSync(file)).digest("hex");
if (sha256(dbPath) !== manifest.artifacts.database.sha256) throw new Error("database checksum mismatch");
if (sha256(storagePath) !== manifest.artifacts.storage.sha256) throw new Error("storage checksum mismatch");
NODE
else
  python3 - "$BATCH_DIR/manifest.json" "$DB_BACKUP" "$STORAGE_BACKUP" "$BUILD_SHA" "$SOURCE_IMAGE_DIGEST" <<'PYTHON'
import hashlib
import json
import os
import re
import sys

manifest_path, db_path, storage_path, expected_build_sha, expected_image_digest = sys.argv[1:]
with open(manifest_path, encoding="utf-8") as manifest_file:
    manifest_text = manifest_file.read()
manifest = json.loads(manifest_text)
if manifest.get("status") != "complete" or manifest.get("schemaVersion") != 1:
    raise RuntimeError("manifest contract mismatch")
if manifest.get("buildSha") != (expected_build_sha or None) or manifest.get("sourceImageDigest") != expected_image_digest:
    raise RuntimeError("manifest source identity mismatch")
for artifact in (manifest["artifacts"]["database"], manifest["artifacts"]["storage"]):
    if os.path.isabs(artifact["file"]):
        raise RuntimeError("manifest contains absolute artifact path")
if re.search(r"postgresql:|DATABASE_URL|drill_password", manifest_text):
    raise RuntimeError("manifest contains sensitive connection data")

def sha256(file_path):
    digest = hashlib.sha256()
    with open(file_path, "rb") as artifact_file:
        for block in iter(lambda: artifact_file.read(1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()

if sha256(db_path) != manifest["artifacts"]["database"]["sha256"]:
    raise RuntimeError("database checksum mismatch")
if sha256(storage_path) != manifest["artifacts"]["storage"]["sha256"]:
    raise RuntimeError("storage checksum mismatch")
PYTHON
fi

psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -qc 'DROP TABLE restore_drill_marker'

docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$CONTAINER_NAME" \
  createdb -U "$POSTGRES_USER" hyb_oa_prod
if DATABASE_URL="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@127.0.0.1:$HOST_PORT/hyb_oa_prod" \
  POSTGRES_DB=hyb_oa_prod RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE=hyb_oa_prod \
  "$ROOT_DIR/scripts/restore-db.sh" "$DB_BACKUP" >/dev/null 2>&1; then
  echo "[ERR] production-like DB name was accepted as a shadow target" >&2
  exit 1
fi

CORRUPT_DB="$DRILL_DIR/corrupt.dump"
head -c 32 "$DB_BACKUP" >"$CORRUPT_DB"
if RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
  "$ROOT_DIR/scripts/restore-db.sh" "$CORRUPT_DB" >/dev/null 2>&1; then
  echo "[ERR] corrupt DB dump was accepted" >&2
  exit 1
fi

if RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE=wrong_target \
  "$ROOT_DIR/scripts/restore-db.sh" "$DB_BACKUP" >/dev/null 2>&1; then
  echo "[ERR] mismatched DB confirmation was accepted" >&2
  exit 1
fi

RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
  "$ROOT_DIR/scripts/restore-db.sh" "$DB_BACKUP"

RESTORED_VALUE="$(psql "$DATABASE_URL" -Atqc 'SELECT value FROM restore_drill_marker')"
if [[ "$RESTORED_VALUE" != "synthetic-round-trip" ]]; then
  echo "[ERR] DB round-trip marker mismatch" >&2
  exit 1
fi

if RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" "$STORAGE_BACKUP" "$NONEMPTY_STORAGE" >/dev/null 2>&1; then
  echo "[ERR] non-empty storage target was accepted" >&2
  exit 1
fi

if RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" \
  "$STORAGE_BACKUP" "$ROOT_DIR/deploy/trialrun/data/storage" >/dev/null 2>&1; then
  echo "[ERR] default live storage path was accepted as a shadow target" >&2
  exit 1
fi

RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" "$STORAGE_BACKUP" "$RESTORED_STORAGE"
cmp "$SOURCE_STORAGE/nested/marker.txt" "$RESTORED_STORAGE/nested/marker.txt"

CORRUPT_TAR="$DRILL_DIR/corrupt.tar.gz"
head -c 32 "$STORAGE_BACKUP" >"$CORRUPT_TAR"
if RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" "$CORRUPT_TAR" "$DRILL_DIR/corrupt-target" >/dev/null 2>&1; then
  echo "[ERR] corrupt storage archive was accepted" >&2
  exit 1
fi

DANGEROUS_TAR="$DRILL_DIR/dangerous.tar.gz"
tar -czf "$DANGEROUS_TAR" --transform='s|nested/marker.txt|../escape.txt|' \
  -C "$SOURCE_STORAGE" nested/marker.txt
if RESTORE_SCOPE=shadow "$ROOT_DIR/scripts/restore-storage.sh" "$DANGEROUS_TAR" "$DRILL_DIR/dangerous-target" >/dev/null 2>&1; then
  echo "[ERR] dangerous storage archive path was accepted" >&2
  exit 1
fi

END_EPOCH="$(date +%s)"
ELAPSED_SECONDS="$((END_EPOCH - START_EPOCH))"
echo "[PASS] isolated backup/restore drill completed"
echo "[INFO] local synthetic RTO seconds: $ELAPSED_SECONDS"