建立每日自动异机备份基线（07-27-automatic-offhost-backup）

2026-07-27 10:50:01 +08:00
@@ -47,6 +47,14 @@
  - [ ] 同一批次包含 `database.dump`、`storage.tar.gz`、`manifest.json`，两份 SHA-256 校验通过
  - [ ] 完整批次已复制到获批异机/异盘，并记录保留周期
  - [ ] 同批次已在隔离影子目标恢复：migration 数、关键业务表逐表计数、附件普通文件数均与冻结基线一致，兼容镜像 `/ready` 通过
+- [ ] 自动异机备份（未完成远端门禁前保持未勾选）：
+  - [ ] **EXTERNAL_GATED**：ARM 已获批安装 rsync、密码锁定专用账号、forced-command key 和独立 inbox/archive/weekly；普通 shell、rsync sender/read/delete 均被拒绝
+  - [ ] **EXTERNAL_GATED**：生产专用私钥为 `0600`，known-hosts 指纹已人工核对，安装器显式绑定活动 project、PGDATA、附件、完整/短构建 SHA、备份/状态目录和远端身份
+  - [ ] `install-automatic-backup-timer.sh install` 只安装 disabled units；首批真实批次及 ARM manifest/SHA-256 核对通过后才显式执行 `enable`
+  - [ ] daily/watchdog/drill-reminder 三组 timer enabled/active，daily service 当前 inactive；未通过手工启动真实告警来伪造验证
+  - [ ] 最近一次异机成功不超过 24 小时；生产自动日批次为 7 个，ARM 自动日批次为 7 个、周检查点为 4 个，历史手工批次未被轮转
+  - [ ] **EXTERNAL_GATED**：可控传输失败仅产生一条脱敏企微告警，恢复链路后新批次成功；同一失败重复运行已降噪
+  - [ ] 本季度已由第二名内部人员按 runbook 首部恢复卡在 ARM 隔离目标完成影子恢复走查；timer 未自动恢复生产
 - [ ] 停机复制方案已冻结：旧 PostgreSQL 完全停止后复制到空新目录并保留所有权/权限/链接/扩展属性；旧目录原位保留且观察期内不再写入

 ## B. 发布（Deploy）
@@ -0,0 +1,227 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+REMOTE_DIR="${REMOTE_DIR:-/opt/hyb-oa-trialrun}"
+REMOTE_ENV_FILE="${REMOTE_ENV_FILE:-${REMOTE_DIR}/deploy/trialrun/.env}"
+COMPOSE_FILE="${COMPOSE_FILE:-deploy/trialrun/docker-compose.yml}"
+COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-}"
+POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:-}"
+STORAGE_HOST_DIR="${STORAGE_HOST_DIR:-}"
+BUILD_SHA="${BUILD_SHA:-}"
+VITE_BUILD_SHA="${VITE_BUILD_SHA:-}"
+AUTOMATIC_BACKUP_DIR="${AUTOMATIC_BACKUP_DIR:-}"
+AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}"
+OFFHOST_TARGET="${OFFHOST_TARGET:-}"
+OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}"
+OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}"
+ALERT_RECIPIENT_EMPLOYEE_NO="${ALERT_RECIPIENT_EMPLOYEE_NO:-HYD000006}"
+BACKUP_RUN_AT="${BACKUP_RUN_AT:-02:30}"
+WATCHDOG_RUN_AT="${WATCHDOG_RUN_AT:-07:10}"
+SERVICE_PREFIX="${SERVICE_PREFIX:-hyb-oa-automatic-backup}"
+MODE="${1:-install}"
+
+RUNNER="${REMOTE_DIR}/scripts/run-automatic-backup.sh"
+SYSTEMD_DIR="/etc/systemd/system"
+
+fail() {
+  echo "[automatic-backup-installer][ERROR] $1" >&2
+  exit 1
+}
+
+need_root() {
+  [[ "$(id -u)" == "0" ]] || fail "root privileges are required"
+}
+
+validate_absolute_path() {
+  local name="$1"
+  local value="$2"
+  if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then
+    fail "$name must be an explicit safe absolute path"
+  fi
+  if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \
+    || "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then
+    fail "$name contains an unsafe path segment"
+  fi
+}
+
+validate_managed_directory_path() {
+  local name="$1"
+  local value="$2"
+  validate_absolute_path "$name" "$value"
+  [[ "$value" =~ ^/[^/]+/[^/]+ ]] \
+    || fail "$name must be a dedicated path below a top-level directory"
+  case "$value" in
+    /bin/*|/boot/*|/dev/*|/etc/*|/lib/*|/lib64/*|/proc/*|/run/*|/sbin/*|/sys/*|/usr/*)
+      fail "$name must not be inside a system-managed directory"
+      ;;
+  esac
+}
+
+paths_overlap() {
+  local first="$1"
+  local second="$2"
+  [[ "$first" == "$second" || "$first" == "$second"/* || "$second" == "$first"/* ]]
+}
+
+validate_inputs() {
+  local name value
+  for name in REMOTE_DIR REMOTE_ENV_FILE POSTGRES_DATA_DIR STORAGE_HOST_DIR \
+    AUTOMATIC_BACKUP_DIR AUTOMATIC_BACKUP_STATE_DIR OFFHOST_SSH_KEY OFFHOST_KNOWN_HOSTS; do
+    value="${!name:-}"
+    validate_absolute_path "$name" "$value"
+  done
+  validate_managed_directory_path AUTOMATIC_BACKUP_DIR "$AUTOMATIC_BACKUP_DIR"
+  validate_managed_directory_path AUTOMATIC_BACKUP_STATE_DIR "$AUTOMATIC_BACKUP_STATE_DIR"
+  [[ -d "$REMOTE_DIR" && -f "$REMOTE_ENV_FILE" && -x "$RUNNER" ]] \
+    || fail "deployment root, env file, or backup runner is missing"
+  [[ -f "$POSTGRES_DATA_DIR/PG_VERSION" && -d "$STORAGE_HOST_DIR" ]] \
+    || fail "active data directories are invalid"
+  [[ "$COMPOSE_PROJECT_NAME" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "Compose project name is invalid"
+  [[ "$COMPOSE_FILE" =~ ^[A-Za-z0-9._/-]+$ && "$COMPOSE_FILE" != /* \
+    && "/$COMPOSE_FILE/" != *"/../"* && "/$COMPOSE_FILE/" != *"/./"* ]] \
+    || fail "Compose file must be a safe relative path"
+  [[ "$BUILD_SHA" =~ ^[0-9a-fA-F]{40,64}$ ]] || fail "full build SHA is invalid"
+  BUILD_SHA="${BUILD_SHA,,}"
+  VITE_BUILD_SHA="${VITE_BUILD_SHA,,}"
+  [[ "$VITE_BUILD_SHA" =~ ^[0-9a-f]{12}$ && "${BUILD_SHA:0:12}" == "$VITE_BUILD_SHA" ]] \
+    || fail "frontend build SHA must match the full build prefix"
+  [[ "$OFFHOST_TARGET" =~ ^[a-z_][a-z0-9_-]*@[A-Za-z0-9._-]+$ ]] \
+    || fail "off-host target must use a safe user@alias value"
+  [[ -f "$OFFHOST_SSH_KEY" && ! -L "$OFFHOST_SSH_KEY" \
+    && "$(stat -c %a "$OFFHOST_SSH_KEY")" == "600" ]] \
+    || fail "off-host private key must exist with mode 0600"
+  [[ -s "$OFFHOST_KNOWN_HOSTS" ]] || fail "pinned known-hosts file is missing"
+  [[ "$BACKUP_RUN_AT" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || fail "backup time is invalid"
+  [[ "$WATCHDOG_RUN_AT" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || fail "watchdog time is invalid"
+  [[ "$SERVICE_PREFIX" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "service prefix is invalid"
+  [[ "$ALERT_RECIPIENT_EMPLOYEE_NO" =~ ^[A-Za-z0-9_-]{1,80}$ ]] || fail "alert recipient id is invalid"
+  local -a isolated_paths=(
+    "$POSTGRES_DATA_DIR" "$STORAGE_HOST_DIR" "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR"
+  )
+  local first_index second_index
+  for ((first_index = 0; first_index < ${#isolated_paths[@]}; first_index += 1)); do
+    for ((second_index = first_index + 1; second_index < ${#isolated_paths[@]}; second_index += 1)); do
+      if paths_overlap "${isolated_paths[$first_index]}" "${isolated_paths[$second_index]}"; then
+        fail "data, backup, and state directories must be independent"
+      fi
+    done
+  done
+}
+
+environment_lines() {
+  cat <<ENVIRONMENT
+Environment=REMOTE_DIR=${REMOTE_DIR}
+Environment=REMOTE_ENV_FILE=${REMOTE_ENV_FILE}
+Environment=COMPOSE_FILE=${COMPOSE_FILE}
+Environment=COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}
+Environment=POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR}
+Environment=STORAGE_HOST_DIR=${STORAGE_HOST_DIR}
+Environment=BUILD_SHA=${BUILD_SHA}
+Environment=VITE_BUILD_SHA=${VITE_BUILD_SHA}
+Environment=AUTOMATIC_BACKUP_DIR=${AUTOMATIC_BACKUP_DIR}
+Environment=AUTOMATIC_BACKUP_STATE_DIR=${AUTOMATIC_BACKUP_STATE_DIR}
+Environment=OFFHOST_TARGET=${OFFHOST_TARGET}
+Environment=OFFHOST_SSH_KEY=${OFFHOST_SSH_KEY}
+Environment=OFFHOST_KNOWN_HOSTS=${OFFHOST_KNOWN_HOSTS}
+Environment=ALERT_RECIPIENT_EMPLOYEE_NO=${ALERT_RECIPIENT_EMPLOYEE_NO}
+ENVIRONMENT
+}
+
+write_service() {
+  local suffix="$1"
+  local mode="$2"
+  local description="$3"
+  local service_file="$SYSTEMD_DIR/${SERVICE_PREFIX}-${suffix}.service"
+  cat >"$service_file" <<SERVICE
+[Unit]
+Description=${description}
+After=docker.service network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+WorkingDirectory=${REMOTE_DIR}
+UMask=0077
+$(environment_lines)
+ExecStart=/bin/bash ${RUNNER} ${mode}
+SERVICE
+}
+
+write_timer() {
+  local suffix="$1"
+  local calendar="$2"
+  local description="$3"
+  local timer_file="$SYSTEMD_DIR/${SERVICE_PREFIX}-${suffix}.timer"
+  cat >"$timer_file" <<TIMER
+[Unit]
+Description=${description}
+
+[Timer]
+OnCalendar=${calendar}
+Persistent=true
+Unit=${SERVICE_PREFIX}-${suffix}.service
+
+[Install]
+WantedBy=timers.target
+TIMER
+}
+
+install_units() {
+  need_root
+  validate_inputs
+  mkdir -p "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR"
+  [[ -d "$AUTOMATIC_BACKUP_DIR" && ! -L "$AUTOMATIC_BACKUP_DIR" \
+    && -d "$AUTOMATIC_BACKUP_STATE_DIR" && ! -L "$AUTOMATIC_BACKUP_STATE_DIR" ]] \
+    || fail "backup and state directories must not be symbolic links"
+  chmod 0700 "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR"
+
+  write_service daily run "hyb-oa automatic paired off-host backup"
+  write_service watchdog check-stale "hyb-oa automatic backup freshness watchdog"
+  write_service drill-reminder remind-drill "hyb-oa quarterly shadow restore reminder"
+  write_timer daily "*-*-* ${BACKUP_RUN_AT}:00" "Run hyb-oa automatic off-host backup daily"
+  write_timer watchdog "*-*-* ${WATCHDOG_RUN_AT}:00" "Check hyb-oa off-host backup freshness daily"
+  write_timer drill-reminder "*-01,04,07,10-01 09:00:00" "Remind quarterly hyb-oa shadow restore drill"
+  systemctl daemon-reload
+  echo "[automatic-backup-installer][PASS] units installed disabled"
+}
+
+enable_units() {
+  install_units
+  systemctl enable --now \
+    "${SERVICE_PREFIX}-daily.timer" \
+    "${SERVICE_PREFIX}-watchdog.timer" \
+    "${SERVICE_PREFIX}-drill-reminder.timer"
+  echo "[automatic-backup-installer][PASS] timers enabled"
+}
+
+uninstall_units() {
+  need_root
+  [[ "$SERVICE_PREFIX" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "service prefix is invalid"
+  local suffix
+  for suffix in daily watchdog drill-reminder; do
+    systemctl disable --now "${SERVICE_PREFIX}-${suffix}.timer" >/dev/null 2>&1 || true
+    systemctl stop "${SERVICE_PREFIX}-${suffix}.service" >/dev/null 2>&1 || true
+    rm -f -- "$SYSTEMD_DIR/${SERVICE_PREFIX}-${suffix}.timer" \
+      "$SYSTEMD_DIR/${SERVICE_PREFIX}-${suffix}.service"
+  done
+  systemctl daemon-reload
+  echo "[automatic-backup-installer][PASS] units removed; backups and credentials retained"
+}
+
+case "$MODE" in
+  install)
+    install_units
+    ;;
+  enable)
+    enable_units
+    ;;
+  uninstall|--uninstall)
+    uninstall_units
+    ;;
+  --help|-h)
+    echo "Usage: <explicit deployment/data/backup/SSH env> bash install-automatic-backup-timer.sh [install|enable|uninstall]"
+    ;;
+  *)
+    fail "unsupported mode"
+    ;;
+esac
@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BACKUP_USER="${BACKUP_USER:-hybbackup}"
+BACKUP_ROOT="${BACKUP_ROOT:-}"
+PUBLIC_KEY_FILE="${PUBLIC_KEY_FILE:-}"
+RECEIVER_SOURCE="${RECEIVER_SOURCE:-}"
+RECEIVER_TARGET="${RECEIVER_TARGET:-/usr/local/sbin/hyb-oa-offhost-receiver}"
+CONFIG_FILE="${CONFIG_FILE:-/etc/hyb-oa-offhost-backup.conf}"
+DAILY_KEEP="${DAILY_KEEP:-7}"
+WEEKLY_KEEP="${WEEKLY_KEEP:-4}"
+MODE="${1:-install}"
+
+fail() {
+  echo "[offhost-receiver-installer][ERROR] $1" >&2
+  exit 1
+}
+
+need_root() {
+  [[ "$(id -u)" == "0" ]] || fail "root privileges are required"
+}
+
+validate_absolute_path() {
+  local name="$1"
+  local value="$2"
+  if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then
+    fail "$name must be an explicit safe absolute path"
+  fi
+  if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \
+    || "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then
+    fail "$name contains an unsafe path segment"
+  fi
+}
+
+validate_backup_root() {
+  validate_absolute_path BACKUP_ROOT "$BACKUP_ROOT"
+  [[ "$BACKUP_ROOT" =~ ^/[^/]+/[^/]+ ]] \
+    || fail "backup root must be a dedicated path below a top-level directory"
+  case "$BACKUP_ROOT" in
+    /bin/*|/boot/*|/dev/*|/etc/*|/lib/*|/lib64/*|/proc/*|/run/*|/sbin/*|/sys/*|/usr/*)
+      fail "backup root must not be inside a system-managed directory"
+      ;;
+  esac
+  [[ ! -L "$BACKUP_ROOT" ]] || fail "backup root must not be a symbolic link"
+  if [[ -d "$BACKUP_ROOT" ]]; then
+    local unmanaged_entry
+    unmanaged_entry="$(find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 \
+      ! -name inbox ! -name archive ! -name weekly -print -quit)"
+    [[ -z "$unmanaged_entry" ]] || fail "backup root contains an unmanaged top-level entry"
+  fi
+}
+
+validate_inputs() {
+  [[ "$BACKUP_USER" =~ ^[a-z_][a-z0-9_-]{0,30}$ ]] || fail "backup user is invalid"
+  validate_backup_root
+  validate_absolute_path PUBLIC_KEY_FILE "$PUBLIC_KEY_FILE"
+  validate_absolute_path RECEIVER_SOURCE "$RECEIVER_SOURCE"
+  validate_absolute_path RECEIVER_TARGET "$RECEIVER_TARGET"
+  validate_absolute_path CONFIG_FILE "$CONFIG_FILE"
+  [[ -f "$PUBLIC_KEY_FILE" && ! -L "$PUBLIC_KEY_FILE" ]] || fail "public key file is missing"
+  [[ -f "$RECEIVER_SOURCE" && ! -L "$RECEIVER_SOURCE" ]] || fail "receiver source is missing"
+  [[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ && "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] \
+    || fail "retention values must be positive"
+  command -v rsync >/dev/null 2>&1 || fail "rsync must be installed before receiver setup"
+  command -v rrsync >/dev/null 2>&1 || fail "rrsync must be installed before receiver setup"
+  command -v python3 >/dev/null 2>&1 || fail "python3 is required"
+}
+
+read_public_key() {
+  local key_type key_body remainder extra_line_count
+  extra_line_count="$(awk 'NF { count += 1 } END { print count + 0 }' "$PUBLIC_KEY_FILE")"
+  [[ "$extra_line_count" == "1" ]] || fail "public key file must contain exactly one non-empty key"
+  read -r key_type key_body remainder <"$PUBLIC_KEY_FILE"
+  [[ "$key_type" == "ssh-ed25519" || "$key_type" == "[email protected]" ]] \
+    || fail "only Ed25519 public keys are accepted"
+  [[ "$key_body" =~ ^[A-Za-z0-9+/=]+$ ]] || fail "public key body is invalid"
+  printf '%s %s' "$key_type" "$key_body"
+}
+
+install_receiver() {
+  need_root
+  validate_inputs
+
+  if ! id "$BACKUP_USER" >/dev/null 2>&1; then
+    useradd --create-home --shell /bin/bash "$BACKUP_USER"
+  fi
+  passwd -l "$BACKUP_USER" >/dev/null 2>&1 || true
+
+  local home_dir inbox_dir archive_dir weekly_dir ssh_dir auth_file public_key rrsync_bin
+  home_dir="$(getent passwd "$BACKUP_USER" | awk -F: '{print $6}')"
+  validate_absolute_path BACKUP_USER_HOME "$home_dir"
+  inbox_dir="$BACKUP_ROOT/inbox"
+  archive_dir="$BACKUP_ROOT/archive"
+  weekly_dir="$BACKUP_ROOT/weekly"
+  mkdir -p "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh"
+  [[ -d "$BACKUP_ROOT" && ! -L "$BACKUP_ROOT" \
+    && -d "$inbox_dir" && ! -L "$inbox_dir" \
+    && -d "$archive_dir" && ! -L "$archive_dir" \
+    && -d "$weekly_dir" && ! -L "$weekly_dir" ]] \
+    || fail "backup receiver directories must not be symbolic links"
+  chown -R "$BACKUP_USER:$BACKUP_USER" "$BACKUP_ROOT" "$home_dir/.ssh"
+  chmod 0700 "$BACKUP_ROOT" "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh"
+
+  install -o root -g root -m 0755 "$RECEIVER_SOURCE" "$RECEIVER_TARGET"
+  rrsync_bin="$(command -v rrsync)"
+  cat >"$CONFIG_FILE" <<CONFIG
+BACKUP_INBOX_DIR=${inbox_dir}
+BACKUP_ARCHIVE_DIR=${archive_dir}
+BACKUP_WEEKLY_DIR=${weekly_dir}
+RRSYNC_BIN=${rrsync_bin}
+PYTHON_BIN=python3
+DAILY_KEEP=${DAILY_KEEP}
+WEEKLY_KEEP=${WEEKLY_KEEP}
+CONFIG
+  chown root:root "$CONFIG_FILE"
+  chmod 0644 "$CONFIG_FILE"
+
+  ssh_dir="$home_dir/.ssh"
+  auth_file="$ssh_dir/authorized_keys"
+  if [[ -s "$auth_file" ]] && ! grep -Fq 'hyb-oa-managed-offhost-backup' "$auth_file"; then
+    fail "dedicated backup account already has unmanaged authorized keys"
+  fi
+  public_key="$(read_public_key)"
+  cat >"$auth_file" <<AUTH
+# hyb-oa-managed-offhost-backup
+command="${RECEIVER_TARGET}",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty,no-user-rc ${public_key}
+AUTH
+  chown "$BACKUP_USER:$BACKUP_USER" "$auth_file"
+  chmod 0600 "$auth_file"
+
+  runuser -u "$BACKUP_USER" -- env RECEIVER_CONFIG_FILE="$CONFIG_FILE" \
+    "$RECEIVER_TARGET" validate-config >/dev/null
+  echo "[offhost-receiver-installer][PASS] restricted receiver installed"
+}
+
+uninstall_receiver() {
+  need_root
+  [[ "$BACKUP_USER" =~ ^[a-z_][a-z0-9_-]{0,30}$ ]] || fail "backup user is invalid"
+  validate_absolute_path RECEIVER_TARGET "$RECEIVER_TARGET"
+  validate_absolute_path CONFIG_FILE "$CONFIG_FILE"
+  if id "$BACKUP_USER" >/dev/null 2>&1; then
+    local home_dir auth_file
+    home_dir="$(getent passwd "$BACKUP_USER" | awk -F: '{print $6}')"
+    auth_file="$home_dir/.ssh/authorized_keys"
+    if [[ -f "$auth_file" ]] && grep -Fq 'hyb-oa-managed-offhost-backup' "$auth_file"; then
+      rm -f -- "$auth_file"
+    fi
+  fi
+  rm -f -- "$RECEIVER_TARGET" "$CONFIG_FILE"
+  echo "[offhost-receiver-installer][PASS] receiver authorization removed; archives retained"
+}
+
+case "$MODE" in
+  install)
+    install_receiver
+    ;;
+  uninstall|--uninstall)
+    uninstall_receiver
+    ;;
+  --help|-h)
+    echo "Usage: BACKUP_ROOT=<absolute> PUBLIC_KEY_FILE=<absolute> RECEIVER_SOURCE=<absolute> bash install-offhost-backup-receiver.sh [install|uninstall]"
+    ;;
+  *)
+    fail "unsupported mode"
+    ;;
+esac
@@ -2,6 +2,27 @@

 > 目标：按本文档在一台新机器完成 **启动 → 迁移 → 初始化管理员 → 验收 → 备份 → 恢复 → 回滚** 演练。

+## 紧急恢复卡（复制到公司知识库的一页版）
+
+适用情形：生产主机不可恢复、数据库或附件损坏，且已由负责人批准进入事故恢复。主操作人为系统管理员，复核人为另一名已完成季度演练的内部人员；两人均不得在聊天或工单中粘贴凭证、业务正文、真实目录或原始日志。
+
+1. 先暂停外部入口和新业务写入，不删除故障主机、旧数据目录或 ARM 归档；记录故障时间、最近成功批次编号类别和恢复负责人。
+2. 在 ARM 的只读归档中选择最近一个 `complete` 批次，复核三文件、manifest、SHA-256 与来源标识；先复制到隔离恢复工作区，不直接对归档原件操作。
+3. 在获批的新主机/隔离目标依次执行以下三个入口；尖括号值只从受控运维配置取得，不写回本文：
+
+```bash
+RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE=<shadow-db> <explicit-db-env> \
+  ./scripts/restore-db.sh <verified-batch>/database.dump
+RESTORE_SCOPE=shadow ./scripts/restore-storage.sh \
+  <verified-batch>/storage.tar.gz <empty-shadow-storage>
+curl -fsS <shadow-ready-url>
+```
+
+4. 复核 `/ready`、运行构建标识、成功 migration 数、组织/权限/报销/付款等关键表计数、附件普通文件数及 DB/附件关联；任何一项不一致都停止切换。
+5. 真实生产恢复仍是 **EXTERNAL_GATED**：另建或更新获批 Trellis task，冻结新数据目录、回滚点和停机窗口后才切换。恢复后保留 ARM 归档和故障主机原数据，记录 RTO/RPO 与结论。
+
+本卡必须由第二名内部人员每季度只读走查一次，并实际在 ARM 隔离目标完成人工影子恢复；自动 timer 只提醒，不自动恢复。
+
 ## 0A. 如果官网 `hyb-web` 已在同机运行（推荐方案）
 - 官网继续保持 `hyinda.com / www.hyinda.com` 静态托管，不要改官网目录和配置。
 - OA 新增独立二级域名，例如 `oa.hyinda.com`。
@@ -205,6 +226,31 @@ manifest 必须至少记录一个真实来源标识：活动 `/ready` 有构建

 整批复制到获批的异机/异盘位置并记录保留周期；不能只复制 DB 或附件中的一份。

+#### 每日自动异机备份（当前仅本地实现，远端启用为 EXTERNAL_GATED）
+
+- 每日 02:30 由生产主机解析唯一活动 Compose project 和真实 DB/附件挂载，生成同批三文件，ARM 校验并原子归档成功后才写成功状态和执行源端轮转。
+- 生产保留 7 个自动日批次；ARM 保留 7 个日批次和 4 个周检查点。07:10 watchdog 检查最近 24 小时异机成功记录；季度 timer 只发送人工影子恢复提醒。
+- ARM 使用专用密码锁定账号和 forced-command key：rsync 只能写 inbox，不能读取/删除 archive，也没有普通 shell。生产固定专用 key 和人工核对的 known-hosts，禁止 `accept-new`。
+- 失败告警只包含阶段、状态、finding 数和类别建议。同一失败降噪；首次失败、状态恶化、新 finding 和每季度首次恢复提醒才发送。
+
+安装顺序必须是：先获批 ARM 端安装 rsync、受限身份和 receiver，用合成批次验证；再获批生产端安装 units，手工完成首个真实批次；最后显式执行生产安装器的 `enable`。两个安装器默认 `install` 均不代表已启用，不得把本地合同通过写成远端完成。
+
+```bash
+# ARM：参数从受控配置取得；install 会写账号/authorized_keys/receiver，需单独批准。
+BACKUP_ROOT=<absolute-backup-root> PUBLIC_KEY_FILE=<dedicated-public-key> \
+RECEIVER_SOURCE=<repo-receiver-script> \
+  bash deploy/trialrun/install-offhost-backup-receiver.sh install
+
+# 生产：先只安装 disabled units；路径、项目、构建和 SSH 身份必须绑定活动实例。
+<explicit-project-data-build-and-ssh-env> \
+  bash deploy/trialrun/install-automatic-backup-timer.sh install
+
+# 首批真实备份、ARM 哈希和脱敏告警均经单独确认后，才允许启用三组 timer。
+<same-explicit-env> bash deploy/trialrun/install-automatic-backup-timer.sh enable
+```
+
+卸载只撤销 timer/receiver 授权，保留源端批次、状态、ARM archive 和凭证文件；专用账号/密钥的最终销毁另走获批清理。
+
 ### 1.5.2 本地隔离恢复演练

 开发机具备 Docker、PostgreSQL 客户端，以及 Node.js 或 Python 3 时，使用合成数据运行：
@@ -19,6 +19,16 @@ const timerScriptPath = path.resolve(
  process.env.ENV_CONTRACT_TIMER_SCRIPT_PATH
    || path.join(rootDir, 'deploy/trialrun/install-warch3-ops-timer.sh'),
 );
+const automaticBackupTimerScriptPath = path.join(
+  rootDir,
+  'deploy/trialrun/install-automatic-backup-timer.sh',
+);
+const offhostReceiverInstallerPath = path.join(
+  rootDir,
+  'deploy/trialrun/install-offhost-backup-receiver.sh',
+);
+const automaticBackupRunnerPath = path.join(rootDir, 'scripts/run-automatic-backup.sh');
+const offhostReceiverPath = path.join(rootDir, 'scripts/receive-offhost-backup.sh');
 const nginxPaths = [
  path.join(rootDir, 'deploy/trialrun/nginx-hyb-oa.conf'),
  path.join(rootDir, 'deploy/trialrun/nginx.conf.example'),
@@ -143,6 +153,10 @@ const runbookContent = readFile(runbookPath);
 const checklistContent = readFile(checklistPath);
 const deployScriptContent = readFile(deployScriptPath);
 const timerScriptContent = readFile(timerScriptPath);
+const automaticBackupTimerScriptContent = readFile(automaticBackupTimerScriptPath);
+const offhostReceiverInstallerContent = readFile(offhostReceiverInstallerPath);
+const automaticBackupRunnerContent = readFile(automaticBackupRunnerPath);
+const offhostReceiverContent = readFile(offhostReceiverPath);
 const nginxContents = nginxPaths.map((filePath) => ({
  filePath,
  content: readFile(filePath),
@@ -274,6 +288,82 @@ for (const requiredFragment of [
  }
 }

+for (const requiredFragment of [
+  'MODE="${1:-install}"',
+  'write_service daily run',
+  'write_service watchdog check-stale',
+  'write_service drill-reminder remind-drill',
+  'write_timer daily',
+  'write_timer watchdog',
+  'write_timer drill-reminder',
+  'Environment=COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}',
+  'Environment=POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR}',
+  'Environment=STORAGE_HOST_DIR=${STORAGE_HOST_DIR}',
+  'Environment=BUILD_SHA=${BUILD_SHA}',
+  'Environment=VITE_BUILD_SHA=${VITE_BUILD_SHA}',
+  'Environment=AUTOMATIC_BACKUP_DIR=${AUTOMATIC_BACKUP_DIR}',
+  'Environment=AUTOMATIC_BACKUP_STATE_DIR=${AUTOMATIC_BACKUP_STATE_DIR}',
+  'Environment=OFFHOST_TARGET=${OFFHOST_TARGET}',
+  'Environment=OFFHOST_SSH_KEY=${OFFHOST_SSH_KEY}',
+  'Environment=OFFHOST_KNOWN_HOSTS=${OFFHOST_KNOWN_HOSTS}',
+  'install)\n    install_units',
+  'enable)\n    enable_units',
+]) {
+  if (!automaticBackupTimerScriptContent.includes(requiredFragment)) {
+    errors.push(`install-automatic-backup-timer.sh 缺少自动备份安装合同: ${requiredFragment}`);
+  }
+}
+
+for (const requiredFragment of [
+  'COMPOSE_CMD=(docker compose)',
+  '-p "$COMPOSE_PROJECT_NAME"',
+  'flock -n 9',
+  '"$actual_storage" == "$STORAGE_HOST_DIR"',
+  '"$actual_pgdata" == "$POSTGRES_DATA_DIR"',
+  '--partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo=',
+  'StrictHostKeyChecking=yes',
+  'hyb-backup-finalize $run_id $manifest_sha',
+  'write_success_state "$run_id" "$manifest_sha"',
+  'prune_local_backups "$batch_name"',
+  '--summary-file=-',
+]) {
+  if (!automaticBackupRunnerContent.includes(requiredFragment)) {
+    errors.push(`run-automatic-backup.sh 缺少运行时安全合同: ${requiredFragment}`);
+  }
+}
+
+for (const requiredFragment of [
+  '"$original_command" == rsync\\ --server\\ *',
+  '--sender([[:space:]]|$)',
+  'exec "$RRSYNC_BIN" -wo "$BACKUP_INBOX_DIR"',
+  'hyb-backup-finalize',
+  'batch must contain exactly three regular files',
+  'manifest artifact checksum mismatch',
+  'mv -- "$partial_dir" "$archive_dir"',
+  'prune_named_directories "$BACKUP_ARCHIVE_DIR"',
+  'prune_named_directories "$BACKUP_WEEKLY_DIR"',
+]) {
+  if (!offhostReceiverContent.includes(requiredFragment)) {
+    errors.push(`receive-offhost-backup.sh 缺少受限接收合同: ${requiredFragment}`);
+  }
+}
+
+for (const requiredFragment of [
+  'MODE="${1:-install}"',
+  'passwd -l "$BACKUP_USER"',
+  'command="${RECEIVER_TARGET}",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty,no-user-rc',
+  'BACKUP_INBOX_DIR=${inbox_dir}',
+  'BACKUP_ARCHIVE_DIR=${archive_dir}',
+  'BACKUP_WEEKLY_DIR=${weekly_dir}',
+  'archives retained',
+  'backup root contains an unmanaged top-level entry',
+  'backup root must not be inside a system-managed directory',
+]) {
+  if (!offhostReceiverInstallerContent.includes(requiredFragment)) {
+    errors.push(`install-offhost-backup-receiver.sh 缺少受限身份安装合同: ${requiredFragment}`);
+  }
+}
+
 for (const requiredFragment of [
  'validate_data_path POSTGRES_DATA_DIR "$POSTGRES_DATA_DIR"',
  'validate_data_path STORAGE_HOST_DIR "$STORAGE_HOST_DIR"',