#!/usr/bin/env bash set -eEuo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" REMOTE_DIR="${REMOTE_DIR:-$ROOT_DIR}" REMOTE_ENV_FILE="${REMOTE_ENV_FILE:-${REMOTE_DIR}/deploy/trialrun/.env}" COMPOSE_FILE="${COMPOSE_FILE:-deploy/trialrun/docker-compose.yml}" COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-}" POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:-}" STORAGE_HOST_DIR="${STORAGE_HOST_DIR:-}" BUILD_SHA="${BUILD_SHA:-}" VITE_BUILD_SHA="${VITE_BUILD_SHA:-}" AUTOMATIC_BACKUP_DIR="${AUTOMATIC_BACKUP_DIR:-}" AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}" OFFHOST_TARGET="${OFFHOST_TARGET:-}" OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}" OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}" ALERT_WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-}" ALERT_HELPER="${ALERT_HELPER:-$ROOT_DIR/scripts/send-wecom-group-alert.py}" AUTOMATIC_BACKUP_KEEP="${AUTOMATIC_BACKUP_KEEP:-7}" AUTOMATIC_BACKUP_MAX_AGE_SECONDS="${AUTOMATIC_BACKUP_MAX_AGE_SECONDS:-86400}" MODE="${1:-run}" CURRENT_PHASE="preflight" SERVER_CONTAINER="" DB_CONTAINER="" SUMMARY_FILE="" COMPOSE_CMD=() fail() { echo "[automatic-backup][ERROR] $1" >&2 return 1 } validate_absolute_path() { local name="$1" local value="$2" if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then fail "$name must be an explicit safe absolute path" fi if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \ || "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then fail "$name contains an unsafe path segment" fi } validate_managed_directory_path() { local name="$1" local value="$2" validate_absolute_path "$name" "$value" [[ "$value" =~ ^/[^/]+/[^/]+ ]] \ || fail "$name must be a dedicated path below a top-level directory" case "$value" in /bin/*|/boot/*|/dev/*|/etc/*|/lib/*|/lib64/*|/proc/*|/run/*|/sbin/*|/sys/*|/usr/*) fail "$name must not be inside a system-managed directory" ;; esac } validate_common() { local name value for name in REMOTE_DIR REMOTE_ENV_FILE POSTGRES_DATA_DIR STORAGE_HOST_DIR \ AUTOMATIC_BACKUP_DIR AUTOMATIC_BACKUP_STATE_DIR; do value="${!name:-}" validate_absolute_path "$name" "$value" done validate_managed_directory_path AUTOMATIC_BACKUP_DIR "$AUTOMATIC_BACKUP_DIR" validate_managed_directory_path AUTOMATIC_BACKUP_STATE_DIR "$AUTOMATIC_BACKUP_STATE_DIR" [[ -d "$REMOTE_DIR" && -f "$REMOTE_ENV_FILE" ]] || fail "deployment root or env file is missing" [[ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]] || fail "active PGDATA marker is missing" [[ -d "$STORAGE_HOST_DIR" ]] || fail "active storage directory is missing" [[ "$COMPOSE_PROJECT_NAME" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "Compose project name is invalid" [[ "$COMPOSE_FILE" =~ ^[A-Za-z0-9._/-]+$ && "$COMPOSE_FILE" != /* \ && "/$COMPOSE_FILE/" != *"/../"* && "/$COMPOSE_FILE/" != *"/./"* ]] \ || fail "Compose file must be a safe relative path" [[ "$BUILD_SHA" =~ ^[0-9a-fA-F]{40,64}$ ]] || fail "full build SHA is invalid" BUILD_SHA="${BUILD_SHA,,}" VITE_BUILD_SHA="${VITE_BUILD_SHA,,}" [[ "$VITE_BUILD_SHA" =~ ^[0-9a-f]{12}$ && "${BUILD_SHA:0:12}" == "$VITE_BUILD_SHA" ]] \ || fail "frontend build SHA must match the full build prefix" [[ "$AUTOMATIC_BACKUP_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "backup retention must be positive" [[ "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "stale threshold must be positive" validate_absolute_path ALERT_WEBHOOK_FILE "$ALERT_WEBHOOK_FILE" validate_absolute_path ALERT_HELPER "$ALERT_HELPER" [[ -f "$ALERT_WEBHOOK_FILE" && ! -L "$ALERT_WEBHOOK_FILE" \ && "$(stat -c %a "$ALERT_WEBHOOK_FILE")" == "600" ]] \ || fail "alert webhook file must exist with mode 0600" [[ -f "$ALERT_HELPER" && ! -L "$ALERT_HELPER" ]] || fail "alert helper is missing" if [[ "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR" \ || "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR"/* \ || "$AUTOMATIC_BACKUP_STATE_DIR" == "$AUTOMATIC_BACKUP_DIR"/* ]]; then fail "backup and state directories must be independent" fi mkdir -p "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR" [[ -d "$AUTOMATIC_BACKUP_DIR" && ! -L "$AUTOMATIC_BACKUP_DIR" \ && -d "$AUTOMATIC_BACKUP_STATE_DIR" && ! -L "$AUTOMATIC_BACKUP_STATE_DIR" ]] \ || fail "backup and state directories must not be symbolic links" chmod 0700 "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR" command -v python3 >/dev/null 2>&1 || fail "python3 is required" } validate_transfer_config() { validate_absolute_path OFFHOST_SSH_KEY "$OFFHOST_SSH_KEY" validate_absolute_path OFFHOST_KNOWN_HOSTS "$OFFHOST_KNOWN_HOSTS" [[ -f "$OFFHOST_SSH_KEY" && ! -L "$OFFHOST_SSH_KEY" ]] || fail "off-host private key is missing" [[ "$(stat -c %a "$OFFHOST_SSH_KEY")" == "600" ]] || fail "off-host private key mode must be 0600" [[ -f "$OFFHOST_KNOWN_HOSTS" && -s "$OFFHOST_KNOWN_HOSTS" ]] || fail "pinned known-hosts file is missing" [[ "$OFFHOST_TARGET" =~ ^[a-z_][a-z0-9_-]*@[A-Za-z0-9._-]+$ ]] \ || fail "off-host target must use a safe user@alias value" command -v ssh >/dev/null 2>&1 || fail "ssh is required" command -v rsync >/dev/null 2>&1 || fail "rsync is required" } select_compose() { if docker compose version >/dev/null 2>&1; then COMPOSE_CMD=(docker compose) elif command -v docker-compose >/dev/null 2>&1; then COMPOSE_CMD=(docker-compose) else fail "Compose runtime is unavailable" fi } compose() { POSTGRES_DATA_DIR="$POSTGRES_DATA_DIR" \ STORAGE_HOST_DIR="$STORAGE_HOST_DIR" \ BUILD_SHA="$BUILD_SHA" \ VITE_BUILD_SHA="$VITE_BUILD_SHA" \ "${COMPOSE_CMD[@]}" -p "$COMPOSE_PROJECT_NAME" \ -f "$REMOTE_DIR/$COMPOSE_FILE" --env-file "$REMOTE_ENV_FILE" "$@" } container_env() { local container_id="$1" local key="$2" docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$container_id" \ | sed -n "s/^${key}=//p" | head -n 1 } mount_source() { local container_id="$1" local destination="$2" docker inspect -f '{{json .Mounts}}' "$container_id" \ | python3 -c ' import json import sys destination = sys.argv[1] mounts = json.load(sys.stdin) matches = [mount.get("Source", "") for mount in mounts if mount.get("Destination") == destination] if len(matches) != 1 or not matches[0]: raise SystemExit(1) print(matches[0]) ' "$destination" } resolve_active_runtime() { select_compose local -a servers=() dbs=() mapfile -t servers < <(compose ps -q server) mapfile -t dbs < <(compose ps -q db) [[ "${#servers[@]}" -eq 1 && -n "${servers[0]}" ]] || fail "expected exactly one active server container" [[ "${#dbs[@]}" -eq 1 && -n "${dbs[0]}" ]] || fail "expected exactly one active database container" SERVER_CONTAINER="${servers[0]}" DB_CONTAINER="${dbs[0]}" [[ "$(docker inspect -f '{{.State.Status}}' "$SERVER_CONTAINER")" == "running" ]] \ || fail "server container is not running" [[ "$(docker inspect -f '{{.State.Status}}' "$DB_CONTAINER")" == "running" ]] \ || fail "database container is not running" local actual_build storage_destination actual_storage actual_pgdata actual_build="$(container_env "$SERVER_CONTAINER" BUILD_SHA)" storage_destination="$(container_env "$SERVER_CONTAINER" STORAGE_DIR)" [[ -n "$storage_destination" && "$storage_destination" == /* ]] || fail "server storage destination is invalid" actual_storage="$(mount_source "$SERVER_CONTAINER" "$storage_destination")" actual_pgdata="$(mount_source "$DB_CONTAINER" /var/lib/postgresql/data)" [[ "$actual_build" == "$BUILD_SHA" ]] || fail "active server build does not match the installed backup contract" [[ "$actual_storage" == "$STORAGE_HOST_DIR" ]] || fail "active storage mount does not match the installed backup contract" [[ "$actual_pgdata" == "$POSTGRES_DATA_DIR" ]] || fail "active PGDATA mount does not match the installed backup contract" } write_summary() { local status="$1" local finding_count="$2" local detail="$3" local output_file="$AUTOMATIC_BACKUP_STATE_DIR/last-run.json" local temp_file="$AUTOMATIC_BACKUP_STATE_DIR/.last-run.json.tmp.$$" python3 - "$temp_file" "$status" "$finding_count" "$detail" "$VITE_BUILD_SHA" <<'PYTHON' import json import os import sys from datetime import datetime, timezone file_path, status, finding_count, detail, build_sha = sys.argv[1:] finding_count = int(finding_count) findings = [] if finding_count: findings.append({ "source": "automatic_backup", "status": status, "severity": "P0", "detail": detail, }) payload = { "issueId": "AUTO_BACKUP", "status": status, "conclusion": status, "findingCount": finding_count, "findings": findings, "generatedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "reportDir": "host-backup-state", "buildSha": build_sha, "privacy": "status/count/category only; no credentials, host addresses, absolute paths, file names, or payloads", } with open(file_path, "x", encoding="utf-8") as handle: json.dump(payload, handle, ensure_ascii=True, indent=2) handle.write("\n") os.chmod(file_path, 0o600) PYTHON mv -f -- "$temp_file" "$output_file" SUMMARY_FILE="$output_file" } send_summary_alert() { local state_name="$1" [[ -n "$SUMMARY_FILE" && -f "$SUMMARY_FILE" ]] || return 1 python3 "$ALERT_HELPER" \ --webhook-file "$ALERT_WEBHOOK_FILE" \ --summary-file "$SUMMARY_FILE" \ --state-file "$AUTOMATIC_BACKUP_STATE_DIR/$state_name" >/dev/null } capture_command() { local phase="$1" shift local temporary="$AUTOMATIC_BACKUP_STATE_DIR/.${phase}.diagnostic.$$" : >"$temporary" chmod 0600 "$temporary" if "$@" >"$temporary" 2>&1; then rm -f -- "$temporary" return 0 fi mv -f -- "$temporary" "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log" chmod 0600 "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log" return 1 } on_error() { local exit_code="$?" trap - ERR set +e write_summary "FAIL_AUTOMATIC_BACKUP" 1 "phase_${CURRENT_PHASE}_failed" send_summary_alert "automatic-backup-alert-state.json" || true echo "[automatic-backup][FAIL] phase=${CURRENT_PHASE}" >&2 exit "$exit_code" } prune_local_backups() { local current_name="$1" local -a entries=() mapfile -t entries < <(find "$AUTOMATIC_BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d \ -name 'hyb-oa-backup-auto-*' -printf '%f\n' | LC_ALL=C sort) local remove_count=$((${#entries[@]} - AUTOMATIC_BACKUP_KEEP)) if (( remove_count <= 0 )); then return 0 fi local index entry target for ((index = 0; index < remove_count; index += 1)); do entry="${entries[$index]}" [[ "$entry" != "$current_name" && "$entry" =~ ^hyb-oa-backup-auto-[0-9]{8}T[0-9]{6}Z$ ]] \ || fail "local retention target failed safety validation" target="$AUTOMATIC_BACKUP_DIR/$entry" [[ -d "$target" && ! -L "$target" ]] || fail "local retention target is invalid" rm -rf -- "$target" done } write_success_state() { local run_id="$1" local manifest_sha="$2" local temp_file="$AUTOMATIC_BACKUP_STATE_DIR/.last-success.json.tmp.$$" python3 - "$temp_file" "$run_id" "$manifest_sha" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON' import json import os import sys from datetime import datetime, timezone file_path, run_id, manifest_sha, project, build_sha = sys.argv[1:] payload = { "schemaVersion": 1, "status": "complete", "completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), "runId": run_id, "manifestSha256": manifest_sha, "composeProject": project, "buildSha": build_sha, } with open(file_path, "x", encoding="utf-8") as handle: json.dump(payload, handle, ensure_ascii=True, indent=2) handle.write("\n") os.chmod(file_path, 0o600) PYTHON mv -f -- "$temp_file" "$AUTOMATIC_BACKUP_STATE_DIR/last-success.json" } run_backup() { validate_common validate_transfer_config exec 9>"$AUTOMATIC_BACKUP_STATE_DIR/automatic-backup.lock" flock -n 9 || fail "another automatic backup is already running" trap on_error ERR CURRENT_PHASE="runtime_identity" resolve_active_runtime local database_url postgres_db postgres_user postgres_password parsed_identity url_user url_db database_url="$(container_env "$SERVER_CONTAINER" DATABASE_URL)" postgres_db="$(container_env "$DB_CONTAINER" POSTGRES_DB)" postgres_user="$(container_env "$DB_CONTAINER" POSTGRES_USER)" postgres_password="$(container_env "$DB_CONTAINER" POSTGRES_PASSWORD)" [[ -n "$database_url" && -n "$postgres_db" && -n "$postgres_user" && -n "$postgres_password" ]] \ || fail "active database identity is incomplete" parsed_identity="$(printf '%s' "$database_url" | python3 -c ' import sys from urllib.parse import unquote, urlparse value = urlparse(sys.stdin.read()) if value.scheme not in {"postgres", "postgresql"} or not value.username or not value.path.strip("/"): raise SystemExit(1) database_name = unquote(value.path.strip("/")) print(f"{unquote(value.username)}\t{database_name}") ')" IFS=$'\t' read -r url_user url_db <<<"$parsed_identity" [[ "$url_user" == "$postgres_user" && "$url_db" == "$postgres_db" ]] \ || fail "active server and database container identities do not match" local run_id batch_name batch_dir source_image_digest manifest_sha run_id="auto-$(date -u +%Y%m%dT%H%M%SZ)" batch_name="hyb-oa-backup-$run_id" batch_dir="$AUTOMATIC_BACKUP_DIR/$batch_name" source_image_digest="$(docker inspect -f '{{.Image}}' "$SERVER_CONTAINER")" [[ "$source_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "active server image digest is invalid" CURRENT_PHASE="paired_backup" if ! ( export DATABASE_URL="$database_url" export DB_TOOL_MODE=docker DB_CONTAINER export POSTGRES_DB="$postgres_db" POSTGRES_USER="$postgres_user" export POSTGRES_PASSWORD="$postgres_password" export STORAGE_DIR_HOST="$STORAGE_HOST_DIR" BACKUP_DIR="$AUTOMATIC_BACKUP_DIR" export BACKUP_RUN_ID="$run_id" BUILD_SHA SOURCE_IMAGE_DIGEST="$source_image_digest" capture_command paired_backup "$ROOT_DIR/scripts/backup-all.sh" ); then fail "paired backup command failed" fi [[ -d "$batch_dir" && -f "$batch_dir/manifest.json" ]] || fail "complete local batch was not published" manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')" local -a ssh_options=( -i "$OFFHOST_SSH_KEY" -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o "UserKnownHostsFile=$OFFHOST_KNOWN_HOSTS" -o ConnectTimeout=15 ) local rsync_shell printf -v rsync_shell 'ssh -i %q -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=%q -o ConnectTimeout=15' \ "$OFFHOST_SSH_KEY" "$OFFHOST_KNOWN_HOSTS" CURRENT_PHASE="offhost_transfer" if ! capture_command offhost_transfer rsync -a --partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= -e "$rsync_shell" \ "$batch_dir/" "$OFFHOST_TARGET:${batch_name}.partial/"; then fail "off-host transfer failed" fi CURRENT_PHASE="offhost_finalize" if ! capture_command offhost_finalize ssh "${ssh_options[@]}" "$OFFHOST_TARGET" \ "hyb-backup-finalize $run_id $manifest_sha"; then fail "off-host finalize failed" fi CURRENT_PHASE="success_state" write_success_state "$run_id" "$manifest_sha" prune_local_backups "$batch_name" write_summary "PASS_AUTOMATIC_BACKUP_COMPLETE" 0 "offhost_verified" send_summary_alert "automatic-backup-alert-state.json" || true rm -f -- "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log" trap - ERR echo "[automatic-backup][PASS] paired batch finalized off-host" } check_stale() { validate_common trap on_error ERR CURRENT_PHASE="stale_check" resolve_active_runtime local success_file="$AUTOMATIC_BACKUP_STATE_DIR/last-success.json" local stale_result if ! stale_result="$(python3 - "$success_file" "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON' import json import os import re import sys from datetime import datetime, timezone file_path, max_age, expected_project, expected_build = sys.argv[1:] if not os.path.isfile(file_path): raise SystemExit(1) with open(file_path, "r", encoding="utf-8") as handle: payload = json.load(handle) if payload.get("schemaVersion") != 1 or payload.get("status") != "complete": raise SystemExit(1) if payload.get("composeProject") != expected_project: raise SystemExit(1) if not re.fullmatch(r"auto-[0-9]{8}T[0-9]{6}Z", payload.get("runId", "")): raise SystemExit(1) if not re.fullmatch(r"[0-9a-f]{64}", payload.get("manifestSha256", "")): raise SystemExit(1) if not re.fullmatch(r"[0-9a-fA-F]{40,64}", payload.get("buildSha", "")): raise SystemExit(1) completed = datetime.fromisoformat(payload["completedAt"].replace("Z", "+00:00")) age = (datetime.now(timezone.utc) - completed).total_seconds() if age < 0 or age > int(max_age): raise SystemExit(1) print("BUILD_MISMATCH" if payload.get("buildSha") != expected_build else "FRESH") PYTHON )"; then fail "latest off-host success is missing, invalid, project-mismatched, or stale" fi if [[ "$stale_result" == "BUILD_MISMATCH" ]]; then write_summary "PASS_WITH_FINDINGS" 1 "recovery_point_build_differs" send_summary_alert "automatic-backup-alert-state.json" || true trap - ERR echo "[automatic-backup][PASS_WITH_FINDINGS] off-host recovery point is fresh but build differs" return 0 fi write_summary "PASS_AUTOMATIC_BACKUP_FRESH" 0 "fresh_offhost_recovery_point" send_summary_alert "automatic-backup-alert-state.json" || true trap - ERR echo "[automatic-backup][PASS] off-host recovery point is fresh" } remind_drill() { validate_common trap on_error ERR CURRENT_PHASE="quarterly_reminder" resolve_active_runtime local quarter quarter="$(python3 - <<'PYTHON' from datetime import datetime, timezone now = datetime.now(timezone.utc) print(f"{now.year}-Q{((now.month - 1) // 3) + 1}") PYTHON )" write_summary "PASS_WITH_FINDINGS" 1 "quarterly_shadow_restore_due_${quarter}" send_summary_alert "automatic-backup-drill-alert-state.json" trap - ERR echo "[automatic-backup][PASS] quarterly restore reminder evaluated" } case "$MODE" in run) run_backup ;; check-stale) check_stale ;; remind-drill) remind_drill ;; --help|-h) echo "Usage: run-automatic-backup.sh [run|check-stale|remind-drill]" ;; *) fail "unsupported mode" ;; esac