完善异机备份告警与同日恢复点取代（07-27-automatic-offhost-backup）
这个提交包含在：
@@ -31,7 +31,7 @@
|
||||
- [ ] 正式 deploy 使用绝对数据路径；既有实例使用 `DEPLOY_DATA_MODE=existing`，PGDATA 已存在且含 `PG_VERSION`，附件目录已存在
|
||||
- [ ] 已证明没有两个 PostgreSQL 实例同时挂载同一 PGDATA；任何不一致均停止发布
|
||||
- [ ] 已记录活动 server/db 容器、不可变镜像 ID（及可用 repo digest）、Caddy 目标、挂载源、环境摘要，不记录 env 正文
|
||||
- [ ] 已冻结成功 migration 数量（当前预期 58）和关键业务表逐表计数基线
|
||||
- [ ] 已现场只读冻结成功 migration 数量和关键业务表逐表计数基线；当前生产证据为 61 项、本地为 62 项，本次待上线只应新增 1 项，现场结果不符立即停止
|
||||
- [ ] 确认当前版本已打 tag：`git tag -a trialrun-YYYYMMDD-NN -m "trialrun release"`
|
||||
- [ ] 本地门禁：`./scripts/check-minimal.sh`
|
||||
- [ ] 部署契约校验：`node deploy/trialrun/validate-env-contract.js`
|
||||
@@ -48,12 +48,14 @@
|
||||
- [ ] 完整批次已复制到获批异机/异盘，并记录保留周期
|
||||
- [ ] 同批次已在隔离影子目标恢复：migration 数、关键业务表逐表计数、附件普通文件数均与冻结基线一致，兼容镜像 `/ready` 通过
|
||||
- [ ] 自动异机备份（未完成远端门禁前保持未勾选）：
|
||||
- [ ] **EXTERNAL_GATED**：ARM 已获批安装 rsync、密码锁定专用账号、forced-command key 和独立 inbox/archive/weekly；普通 shell、rsync sender/read/delete 均被拒绝
|
||||
- [ ] **EXTERNAL_GATED**：生产专用私钥为 `0600`，known-hosts 指纹已人工核对，安装器显式绑定活动 project、PGDATA、附件、完整/短构建 SHA、备份/状态目录和远端身份
|
||||
- [ ] **EXTERNAL_GATED**：用户已在独立企微运维群创建机器人，并分别把 Webhook 写入生产、ARM 的 root-only `0600` 文件；未向 Codex、仓库、普通 `.env`、日志或聊天提供正文
|
||||
- [ ] **EXTERNAL_GATED**：ARM 已获批安装 rsync、密码锁定专用账号、forced-command key、独立 inbox/archive/weekly 和 26 小时 dead-man；普通 shell、rsync sender/read/delete 均被拒绝
|
||||
- [ ] **EXTERNAL_GATED**：生产专用私钥为 `0600`，known-hosts 指纹已人工核对，安装器显式绑定活动 project、PGDATA、附件、完整/短构建 SHA、备份/状态目录、远端身份和 Webhook 文件路径
|
||||
- [ ] `install-automatic-backup-timer.sh install` 只安装 disabled units；首批真实批次及 ARM manifest/SHA-256 核对通过后才显式执行 `enable`
|
||||
- [ ] daily/watchdog/drill-reminder 三组 timer enabled/active，daily service 当前 inactive；未通过手工启动真实告警来伪造验证
|
||||
- [ ] 生产 daily/watchdog/drill-reminder 与 ARM dead-man 四组 timer enabled/active，各 oneshot service 当前 inactive；未通过只调用机器人发送入口来伪造验证
|
||||
- [ ] 最近一次异机成功不超过 24 小时；生产自动日批次为 7 个，ARM 自动日批次为 7 个、周检查点为 4 个，历史手工批次未被轮转
|
||||
- [ ] **EXTERNAL_GATED**：可控传输失败仅产生一条脱敏企微告警，恢复链路后新批次成功；同一失败重复运行已降噪
|
||||
- [ ] **EXTERNAL_GATED**：已真正停止一次源端 timer 或故意破坏一次传输，ARM dead-man 仅产生一条脱敏企微告警；恢复 finalize 后收到一次恢复通知，同一失败重复运行已降噪
|
||||
- [ ] 若调整过 `BACKUP_RUN_AT`，已人工确认调整后的次日 UTC 分桶和 26 小时窗口，未把一次预期空窗误判为长期故障
|
||||
- [ ] 本季度已由第二名内部人员按 runbook 首部恢复卡在 ARM 隔离目标完成影子恢复走查；timer 未自动恢复生产
|
||||
- [ ] 停机复制方案已冻结：旧 PostgreSQL 完全停止后复制到空新目录并保留所有权/权限/链接/扩展属性；旧目录原位保留且观察期内不再写入
|
||||
|
||||
|
||||
@@ -14,7 +14,8 @@ AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}"
|
||||
OFFHOST_TARGET="${OFFHOST_TARGET:-}"
|
||||
OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}"
|
||||
OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}"
|
||||
ALERT_RECIPIENT_EMPLOYEE_NO="${ALERT_RECIPIENT_EMPLOYEE_NO:-HYD000006}"
|
||||
ALERT_WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-}"
|
||||
ALERT_HELPER="${ALERT_HELPER:-${REMOTE_DIR}/scripts/send-wecom-group-alert.py}"
|
||||
BACKUP_RUN_AT="${BACKUP_RUN_AT:-02:30}"
|
||||
WATCHDOG_RUN_AT="${WATCHDOG_RUN_AT:-07:10}"
|
||||
SERVICE_PREFIX="${SERVICE_PREFIX:-hyb-oa-automatic-backup}"
|
||||
@@ -66,7 +67,8 @@ paths_overlap() {
|
||||
validate_inputs() {
|
||||
local name value
|
||||
for name in REMOTE_DIR REMOTE_ENV_FILE POSTGRES_DATA_DIR STORAGE_HOST_DIR \
|
||||
AUTOMATIC_BACKUP_DIR AUTOMATIC_BACKUP_STATE_DIR OFFHOST_SSH_KEY OFFHOST_KNOWN_HOSTS; do
|
||||
AUTOMATIC_BACKUP_DIR AUTOMATIC_BACKUP_STATE_DIR OFFHOST_SSH_KEY OFFHOST_KNOWN_HOSTS \
|
||||
ALERT_WEBHOOK_FILE ALERT_HELPER; do
|
||||
value="${!name:-}"
|
||||
validate_absolute_path "$name" "$value"
|
||||
done
|
||||
@@ -91,10 +93,14 @@ validate_inputs() {
|
||||
&& "$(stat -c %a "$OFFHOST_SSH_KEY")" == "600" ]] \
|
||||
|| fail "off-host private key must exist with mode 0600"
|
||||
[[ -s "$OFFHOST_KNOWN_HOSTS" ]] || fail "pinned known-hosts file is missing"
|
||||
[[ -f "$ALERT_WEBHOOK_FILE" && ! -L "$ALERT_WEBHOOK_FILE" \
|
||||
&& "$(stat -c %a "$ALERT_WEBHOOK_FILE")" == "600" \
|
||||
&& "$(stat -c %u "$ALERT_WEBHOOK_FILE")" == "0" ]] \
|
||||
|| fail "alert webhook file must be root-owned with mode 0600"
|
||||
[[ -f "$ALERT_HELPER" && ! -L "$ALERT_HELPER" ]] || fail "alert helper is missing"
|
||||
[[ "$BACKUP_RUN_AT" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || fail "backup time is invalid"
|
||||
[[ "$WATCHDOG_RUN_AT" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || fail "watchdog time is invalid"
|
||||
[[ "$SERVICE_PREFIX" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "service prefix is invalid"
|
||||
[[ "$ALERT_RECIPIENT_EMPLOYEE_NO" =~ ^[A-Za-z0-9_-]{1,80}$ ]] || fail "alert recipient id is invalid"
|
||||
local -a isolated_paths=(
|
||||
"$POSTGRES_DATA_DIR" "$STORAGE_HOST_DIR" "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR"
|
||||
)
|
||||
@@ -123,7 +129,8 @@ Environment=AUTOMATIC_BACKUP_STATE_DIR=${AUTOMATIC_BACKUP_STATE_DIR}
|
||||
Environment=OFFHOST_TARGET=${OFFHOST_TARGET}
|
||||
Environment=OFFHOST_SSH_KEY=${OFFHOST_SSH_KEY}
|
||||
Environment=OFFHOST_KNOWN_HOSTS=${OFFHOST_KNOWN_HOSTS}
|
||||
Environment=ALERT_RECIPIENT_EMPLOYEE_NO=${ALERT_RECIPIENT_EMPLOYEE_NO}
|
||||
Environment=ALERT_WEBHOOK_FILE=${ALERT_WEBHOOK_FILE}
|
||||
Environment=ALERT_HELPER=${ALERT_HELPER}
|
||||
ENVIRONMENT
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,15 @@ PUBLIC_KEY_FILE="${PUBLIC_KEY_FILE:-}"
|
||||
RECEIVER_SOURCE="${RECEIVER_SOURCE:-}"
|
||||
RECEIVER_TARGET="${RECEIVER_TARGET:-/usr/local/sbin/hyb-oa-offhost-receiver}"
|
||||
CONFIG_FILE="${CONFIG_FILE:-/etc/hyb-oa-offhost-backup.conf}"
|
||||
ALERT_WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-}"
|
||||
ALERT_HELPER_SOURCE="${ALERT_HELPER_SOURCE:-}"
|
||||
ALERT_HELPER_TARGET="${ALERT_HELPER_TARGET:-/usr/local/sbin/hyb-oa-wecom-group-alert}"
|
||||
DEADMAN_SOURCE="${DEADMAN_SOURCE:-}"
|
||||
DEADMAN_TARGET="${DEADMAN_TARGET:-/usr/local/sbin/hyb-oa-offhost-deadman}"
|
||||
DEADMAN_MAX_AGE_SECONDS="${DEADMAN_MAX_AGE_SECONDS:-93600}"
|
||||
DEADMAN_RUN_AT="${DEADMAN_RUN_AT:-06:30}"
|
||||
DEADMAN_SERVICE="${DEADMAN_SERVICE:-hyb-oa-offhost-deadman}"
|
||||
SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}"
|
||||
DAILY_KEEP="${DAILY_KEEP:-7}"
|
||||
WEEKLY_KEEP="${WEEKLY_KEEP:-4}"
|
||||
MODE="${1:-install}"
|
||||
@@ -45,7 +54,7 @@ validate_backup_root() {
|
||||
if [[ -d "$BACKUP_ROOT" ]]; then
|
||||
local unmanaged_entry
|
||||
unmanaged_entry="$(find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 \
|
||||
! -name inbox ! -name archive ! -name weekly -print -quit)"
|
||||
! -name inbox ! -name archive ! -name weekly ! -name state -print -quit)"
|
||||
[[ -z "$unmanaged_entry" ]] || fail "backup root contains an unmanaged top-level entry"
|
||||
fi
|
||||
}
|
||||
@@ -57,8 +66,22 @@ validate_inputs() {
|
||||
validate_absolute_path RECEIVER_SOURCE "$RECEIVER_SOURCE"
|
||||
validate_absolute_path RECEIVER_TARGET "$RECEIVER_TARGET"
|
||||
validate_absolute_path CONFIG_FILE "$CONFIG_FILE"
|
||||
validate_absolute_path ALERT_WEBHOOK_FILE "$ALERT_WEBHOOK_FILE"
|
||||
validate_absolute_path ALERT_HELPER_SOURCE "$ALERT_HELPER_SOURCE"
|
||||
validate_absolute_path ALERT_HELPER_TARGET "$ALERT_HELPER_TARGET"
|
||||
validate_absolute_path DEADMAN_SOURCE "$DEADMAN_SOURCE"
|
||||
validate_absolute_path DEADMAN_TARGET "$DEADMAN_TARGET"
|
||||
[[ -f "$PUBLIC_KEY_FILE" && ! -L "$PUBLIC_KEY_FILE" ]] || fail "public key file is missing"
|
||||
[[ -f "$RECEIVER_SOURCE" && ! -L "$RECEIVER_SOURCE" ]] || fail "receiver source is missing"
|
||||
[[ -f "$ALERT_HELPER_SOURCE" && ! -L "$ALERT_HELPER_SOURCE" ]] || fail "alert helper source is missing"
|
||||
[[ -f "$DEADMAN_SOURCE" && ! -L "$DEADMAN_SOURCE" ]] || fail "dead-man source is missing"
|
||||
[[ -f "$ALERT_WEBHOOK_FILE" && ! -L "$ALERT_WEBHOOK_FILE" \
|
||||
&& "$(stat -c %a "$ALERT_WEBHOOK_FILE")" == "600" \
|
||||
&& "$(stat -c %u "$ALERT_WEBHOOK_FILE")" == "0" ]] \
|
||||
|| fail "alert webhook file must be root-owned with mode 0600"
|
||||
[[ "$DEADMAN_MAX_AGE_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "dead-man threshold must be positive"
|
||||
[[ "$DEADMAN_RUN_AT" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || fail "dead-man time is invalid"
|
||||
[[ "$DEADMAN_SERVICE" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "dead-man service name is invalid"
|
||||
[[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ && "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] \
|
||||
|| fail "retention values must be positive"
|
||||
command -v rsync >/dev/null 2>&1 || fail "rsync must be installed before receiver setup"
|
||||
@@ -86,22 +109,28 @@ install_receiver() {
|
||||
fi
|
||||
passwd -l "$BACKUP_USER" >/dev/null 2>&1 || true
|
||||
|
||||
local home_dir inbox_dir archive_dir weekly_dir ssh_dir auth_file public_key rrsync_bin
|
||||
local home_dir inbox_dir archive_dir weekly_dir state_dir ssh_dir auth_file public_key rrsync_bin
|
||||
home_dir="$(getent passwd "$BACKUP_USER" | awk -F: '{print $6}')"
|
||||
validate_absolute_path BACKUP_USER_HOME "$home_dir"
|
||||
inbox_dir="$BACKUP_ROOT/inbox"
|
||||
archive_dir="$BACKUP_ROOT/archive"
|
||||
weekly_dir="$BACKUP_ROOT/weekly"
|
||||
mkdir -p "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh"
|
||||
state_dir="$BACKUP_ROOT/state"
|
||||
mkdir -p "$inbox_dir" "$archive_dir" "$weekly_dir" "$state_dir" "$home_dir/.ssh"
|
||||
[[ -d "$BACKUP_ROOT" && ! -L "$BACKUP_ROOT" \
|
||||
&& -d "$inbox_dir" && ! -L "$inbox_dir" \
|
||||
&& -d "$archive_dir" && ! -L "$archive_dir" \
|
||||
&& -d "$weekly_dir" && ! -L "$weekly_dir" ]] \
|
||||
|| fail "backup receiver directories must not be symbolic links"
|
||||
chown -R "$BACKUP_USER:$BACKUP_USER" "$BACKUP_ROOT" "$home_dir/.ssh"
|
||||
chown "$BACKUP_USER:$BACKUP_USER" "$BACKUP_ROOT"
|
||||
chown -R "$BACKUP_USER:$BACKUP_USER" "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh"
|
||||
chmod 0700 "$BACKUP_ROOT" "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh"
|
||||
chown -R root:root "$state_dir"
|
||||
chmod 0700 "$state_dir"
|
||||
|
||||
install -o root -g root -m 0755 "$RECEIVER_SOURCE" "$RECEIVER_TARGET"
|
||||
install -o root -g root -m 0755 "$ALERT_HELPER_SOURCE" "$ALERT_HELPER_TARGET"
|
||||
install -o root -g root -m 0755 "$DEADMAN_SOURCE" "$DEADMAN_TARGET"
|
||||
rrsync_bin="$(command -v rrsync)"
|
||||
cat >"$CONFIG_FILE" <<CONFIG
|
||||
BACKUP_INBOX_DIR=${inbox_dir}
|
||||
@@ -130,7 +159,37 @@ AUTH
|
||||
|
||||
runuser -u "$BACKUP_USER" -- env RECEIVER_CONFIG_FILE="$CONFIG_FILE" \
|
||||
"$RECEIVER_TARGET" validate-config >/dev/null
|
||||
echo "[offhost-receiver-installer][PASS] restricted receiver installed"
|
||||
cat >"$SYSTEMD_DIR/${DEADMAN_SERVICE}.service" <<SERVICE
|
||||
[Unit]
|
||||
Description=hyb-oa independent off-host backup dead-man
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
UMask=0077
|
||||
ExecStart=/usr/bin/python3 ${DEADMAN_TARGET} --archive-root ${archive_dir} --state-dir ${state_dir} --webhook-file ${ALERT_WEBHOOK_FILE} --alert-helper ${ALERT_HELPER_TARGET} --max-age-seconds ${DEADMAN_MAX_AGE_SECONDS}
|
||||
SERVICE
|
||||
cat >"$SYSTEMD_DIR/${DEADMAN_SERVICE}.timer" <<TIMER
|
||||
[Unit]
|
||||
Description=Check hyb-oa off-host backup freshness independently
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* ${DEADMAN_RUN_AT}:00
|
||||
Persistent=true
|
||||
Unit=${DEADMAN_SERVICE}.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
TIMER
|
||||
systemctl daemon-reload
|
||||
echo "[offhost-receiver-installer][PASS] restricted receiver and disabled dead-man installed"
|
||||
}
|
||||
|
||||
enable_receiver() {
|
||||
install_receiver
|
||||
systemctl enable --now "${DEADMAN_SERVICE}.timer"
|
||||
echo "[offhost-receiver-installer][PASS] dead-man timer enabled"
|
||||
}
|
||||
|
||||
uninstall_receiver() {
|
||||
@@ -138,6 +197,12 @@ uninstall_receiver() {
|
||||
[[ "$BACKUP_USER" =~ ^[a-z_][a-z0-9_-]{0,30}$ ]] || fail "backup user is invalid"
|
||||
validate_absolute_path RECEIVER_TARGET "$RECEIVER_TARGET"
|
||||
validate_absolute_path CONFIG_FILE "$CONFIG_FILE"
|
||||
validate_absolute_path ALERT_HELPER_TARGET "$ALERT_HELPER_TARGET"
|
||||
validate_absolute_path DEADMAN_TARGET "$DEADMAN_TARGET"
|
||||
[[ "$DEADMAN_SERVICE" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "dead-man service name is invalid"
|
||||
systemctl disable --now "${DEADMAN_SERVICE}.timer" >/dev/null 2>&1 || true
|
||||
systemctl stop "${DEADMAN_SERVICE}.service" >/dev/null 2>&1 || true
|
||||
rm -f -- "$SYSTEMD_DIR/${DEADMAN_SERVICE}.timer" "$SYSTEMD_DIR/${DEADMAN_SERVICE}.service"
|
||||
if id "$BACKUP_USER" >/dev/null 2>&1; then
|
||||
local home_dir auth_file
|
||||
home_dir="$(getent passwd "$BACKUP_USER" | awk -F: '{print $6}')"
|
||||
@@ -146,19 +211,23 @@ uninstall_receiver() {
|
||||
rm -f -- "$auth_file"
|
||||
fi
|
||||
fi
|
||||
rm -f -- "$RECEIVER_TARGET" "$CONFIG_FILE"
|
||||
echo "[offhost-receiver-installer][PASS] receiver authorization removed; archives retained"
|
||||
rm -f -- "$RECEIVER_TARGET" "$CONFIG_FILE" "$ALERT_HELPER_TARGET" "$DEADMAN_TARGET"
|
||||
systemctl daemon-reload
|
||||
echo "[offhost-receiver-installer][PASS] receiver authorization and dead-man removed; archives retained"
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
install)
|
||||
install_receiver
|
||||
;;
|
||||
enable)
|
||||
enable_receiver
|
||||
;;
|
||||
uninstall|--uninstall)
|
||||
uninstall_receiver
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: BACKUP_ROOT=<absolute> PUBLIC_KEY_FILE=<absolute> RECEIVER_SOURCE=<absolute> bash install-offhost-backup-receiver.sh [install|uninstall]"
|
||||
echo "Usage: <explicit backup/key/helper/dead-man/webhook paths> bash install-offhost-backup-receiver.sh [install|enable|uninstall]"
|
||||
;;
|
||||
*)
|
||||
fail "unsupported mode"
|
||||
|
||||
+23
-6
@@ -226,29 +226,46 @@ manifest 必须至少记录一个真实来源标识：活动 `/ready` 有构建
|
||||
|
||||
整批复制到获批的异机/异盘位置并记录保留周期；不能只复制 DB 或附件中的一份。
|
||||
|
||||
#### 每日自动异机备份（当前仅本地实现，远端启用为 EXTERNAL_GATED）
|
||||
#### 每日自动异机备份（可靠启用闭环仍为 EXTERNAL_GATED）
|
||||
|
||||
- 每日 02:30 由生产主机解析唯一活动 Compose project 和真实 DB/附件挂载，生成同批三文件，ARM 校验并原子归档成功后才写成功状态和执行源端轮转。
|
||||
- 生产保留 7 个自动日批次；ARM 保留 7 个日批次和 4 个周检查点。07:10 watchdog 检查最近 24 小时异机成功记录；季度 timer 只发送人工影子恢复提醒。
|
||||
- 生产保留 7 个自动日批次；ARM 保留 7 个日批次和 4 个周检查点。07:10 源端 watchdog 检查最近 24 小时异机成功记录；ARM 独立 dead-man 检查最近 26 小时归档，生产整机或 OA server 停止时仍能告警；季度 timer 只发送人工影子恢复提醒。
|
||||
- ARM 使用专用密码锁定账号和 forced-command key：rsync 只能写 inbox，不能读取/删除 archive，也没有普通 shell。生产固定专用 key 和人工核对的 known-hosts，禁止 `accept-new`。
|
||||
- 失败告警只包含阶段、状态、finding 数和类别建议。同一失败降噪；首次失败、状态恶化、新 finding 和每季度首次恢复提醒才发送。
|
||||
- 同一 UTC 日重试使用新 run id 时，ARM 必须先完整校验候选，再原子取代当日旧批次并重建该周硬链接检查点；坏候选或发布失败不得触碰原有效批次。
|
||||
- 生产与 ARM 都直接使用独立企微运维群机器人，不依赖 OA server、数据库或个人工号。Webhook 正文只由用户手工写入两台主机的 root-only `0600` 文件；仓库、普通 `.env`、systemd unit、日志和聊天只能出现受控文件路径，不能出现 URL 或 key。
|
||||
- 失败告警只包含阶段、状态、finding 数和类别建议；原始命令诊断只保留在源端状态目录的 `0600` 最近失败日志。同一失败降噪；首次失败、状态恶化、新 finding、恢复通知和每季度首次恢复提醒才发送。新部署构建与最近恢复点 build SHA 不同只记 finding，缺失、损坏、超龄、未来时间或 Compose project 不同仍硬失败。
|
||||
|
||||
创建机器人后，由用户分别登录两台主机，在 root shell 中用静默输入把完整 Webhook 写入受控路径；不要把值作为命令参数、环境文件内容或聊天消息交给 Codex：
|
||||
|
||||
```bash
|
||||
umask 077
|
||||
read -rsp 'WeCom operations robot Webhook: ' HYB_OA_ALERT_WEBHOOK
|
||||
printf '\n'
|
||||
install -o root -g root -m 0600 /dev/null <controlled-webhook-file>
|
||||
printf '%s\n' "$HYB_OA_ALERT_WEBHOOK" > <controlled-webhook-file>
|
||||
unset HYB_OA_ALERT_WEBHOOK
|
||||
```
|
||||
|
||||
安装顺序必须是：先获批 ARM 端安装 rsync、受限身份和 receiver，用合成批次验证；再获批生产端安装 units，手工完成首个真实批次；最后显式执行生产安装器的 `enable`。两个安装器默认 `install` 均不代表已启用，不得把本地合同通过写成远端完成。
|
||||
|
||||
```bash
|
||||
# ARM：参数从受控配置取得；install 会写账号/authorized_keys/receiver，需单独批准。
|
||||
BACKUP_ROOT=<absolute-backup-root> PUBLIC_KEY_FILE=<dedicated-public-key> \
|
||||
RECEIVER_SOURCE=<repo-receiver-script> \
|
||||
RECEIVER_SOURCE=<repo-receiver-script> ALERT_WEBHOOK_FILE=<controlled-webhook-file> \
|
||||
ALERT_HELPER_SOURCE=<repo-group-alert-helper> DEADMAN_SOURCE=<repo-deadman-script> \
|
||||
bash deploy/trialrun/install-offhost-backup-receiver.sh install
|
||||
|
||||
# 生产：先只安装 disabled units；路径、项目、构建和 SSH 身份必须绑定活动实例。
|
||||
<explicit-project-data-build-and-ssh-env> \
|
||||
<explicit-project-data-build-ssh-and-webhook-file-env> \
|
||||
bash deploy/trialrun/install-automatic-backup-timer.sh install
|
||||
|
||||
# 首批真实备份、ARM 哈希和脱敏告警均经单独确认后，才允许启用三组 timer。
|
||||
<same-arm-explicit-env> bash deploy/trialrun/install-offhost-backup-receiver.sh enable
|
||||
<same-explicit-env> bash deploy/trialrun/install-automatic-backup-timer.sh enable
|
||||
```
|
||||
|
||||
`BACKUP_RUN_AT` 使用生产主机本地时区，ARM 归档按 UTC 日分桶。调整执行时刻的当天可能出现预期空窗；改动后必须人工确认次日 UTC 分桶、下一次 daily 运行和 26 小时 dead-man 窗口。真实告警验收必须真正停止一次源端 timer 或故意让一次传输失败，再确认 ARM 首次告警、重复降噪和恢复通知；只测试机器人能发消息不算通过。
|
||||
|
||||
卸载只撤销 timer/receiver 授权，保留源端批次、状态、ARM archive 和凭证文件；专用账号/密钥的最终销毁另走获批清理。
|
||||
|
||||
### 1.5.2 本地隔离恢复演练
|
||||
@@ -284,7 +301,7 @@ RESTORE_SCOPE=shadow \
|
||||
|
||||
`restore-db.sh` 和 `restore-storage.sh` 默认只接受 `shadow`。影子库名必须包含独立的 `shadow`、`drill` 或 `restore_test` 标记；数据库确认不匹配、库名不安全、附件目录非空、目标为默认试运行附件目录或归档路径危险时均在写入前退出。
|
||||
|
||||
真实格式影子恢复的验收不仅是 `/ready`：恢复后的 `_prisma_migrations` 成功项必须等于冻结基线（当前预期 58），预先列出的关键业务表计数必须逐表一致，附件解包后的普通文件数必须与活动源目录一致。任一项不一致即判定批次不可作为切换回滚点。
|
||||
真实格式影子恢复的验收不仅是 `/ready`：恢复后的 `_prisma_migrations` 成功项必须等于执行前现场冻结基线（当前生产证据为 61 项；本地 62 项仅表示下一次部署预计新增 1 项），预先列出的关键业务表计数必须逐表一致，附件解包后的普通文件数必须与活动源目录一致。任一项不一致即判定批次不可作为切换回滚点。
|
||||
|
||||
### 1.5.3 生产切换前冻结记录
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#!/usr/bin/env node
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const childProcess = require('node:child_process');
|
||||
|
||||
const rootDir = path.resolve(__dirname, '..', '..');
|
||||
const configPath = path.join(rootDir, 'server/src/config.js');
|
||||
@@ -29,6 +30,8 @@ const offhostReceiverInstallerPath = path.join(
|
||||
);
|
||||
const automaticBackupRunnerPath = path.join(rootDir, 'scripts/run-automatic-backup.sh');
|
||||
const offhostReceiverPath = path.join(rootDir, 'scripts/receive-offhost-backup.sh');
|
||||
const groupAlertHelperPath = path.join(rootDir, 'scripts/send-wecom-group-alert.py');
|
||||
const offhostDeadmanPath = path.join(rootDir, 'scripts/check-offhost-backup-deadman.py');
|
||||
const nginxPaths = [
|
||||
path.join(rootDir, 'deploy/trialrun/nginx-hyb-oa.conf'),
|
||||
path.join(rootDir, 'deploy/trialrun/nginx.conf.example'),
|
||||
@@ -157,6 +160,8 @@ const automaticBackupTimerScriptContent = readFile(automaticBackupTimerScriptPat
|
||||
const offhostReceiverInstallerContent = readFile(offhostReceiverInstallerPath);
|
||||
const automaticBackupRunnerContent = readFile(automaticBackupRunnerPath);
|
||||
const offhostReceiverContent = readFile(offhostReceiverPath);
|
||||
const groupAlertHelperContent = readFile(groupAlertHelperPath);
|
||||
const offhostDeadmanContent = readFile(offhostDeadmanPath);
|
||||
const nginxContents = nginxPaths.map((filePath) => ({
|
||||
filePath,
|
||||
content: readFile(filePath),
|
||||
@@ -185,6 +190,25 @@ for (const key of ['APP_HOST_PORT', 'POSTGRES_DATA_DIR', 'STORAGE_HOST_DIR']) {
|
||||
if (envExampleKeys.has('COMPOSE_PROJECT_NAME')) {
|
||||
errors.push('.env.example 不应持久化 COMPOSE_PROJECT_NAME，调用方必须通过 -p 显式选择项目');
|
||||
}
|
||||
for (const key of ['ALERT_WEBHOOK_FILE', 'WECOM_WEBHOOK_URL', 'WECOM_WEBHOOK_KEY']) {
|
||||
if (envExampleKeys.has(key)) {
|
||||
errors.push(`.env.example 不应持久化企微群机器人配置 ${key}`);
|
||||
}
|
||||
}
|
||||
const trackedFiles = childProcess.execFileSync('git', ['ls-files', '--cached', '--others', '--exclude-standard', '-z'], {
|
||||
cwd: rootDir,
|
||||
encoding: 'utf8',
|
||||
}).split('\0').filter(Boolean);
|
||||
const literalWebhookPattern = /https:\/\/qyapi\.weixin\.qq\.com\/cgi-bin\/webhook\/send\?key=/;
|
||||
for (const relativePath of trackedFiles) {
|
||||
const filePath = path.join(rootDir, relativePath);
|
||||
if (!fs.statSync(filePath).isFile()) continue;
|
||||
const content = fs.readFileSync(filePath);
|
||||
if (content.includes(0)) continue;
|
||||
if (literalWebhookPattern.test(content.toString('utf8'))) {
|
||||
errors.push(`${relativePath} 不得持久化完整企微群机器人 Webhook URL`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const serviceName of requiredServices) {
|
||||
const block = extractServiceBlock(composeContent, serviceName);
|
||||
@@ -306,6 +330,8 @@ for (const requiredFragment of [
|
||||
'Environment=OFFHOST_TARGET=${OFFHOST_TARGET}',
|
||||
'Environment=OFFHOST_SSH_KEY=${OFFHOST_SSH_KEY}',
|
||||
'Environment=OFFHOST_KNOWN_HOSTS=${OFFHOST_KNOWN_HOSTS}',
|
||||
'Environment=ALERT_WEBHOOK_FILE=${ALERT_WEBHOOK_FILE}',
|
||||
'Environment=ALERT_HELPER=${ALERT_HELPER}',
|
||||
'install)\n install_units',
|
||||
'enable)\n enable_units',
|
||||
]) {
|
||||
@@ -325,7 +351,10 @@ for (const requiredFragment of [
|
||||
'hyb-backup-finalize $run_id $manifest_sha',
|
||||
'write_success_state "$run_id" "$manifest_sha"',
|
||||
'prune_local_backups "$batch_name"',
|
||||
'--summary-file=-',
|
||||
'python3 "$ALERT_HELPER"',
|
||||
'--webhook-file "$ALERT_WEBHOOK_FILE"',
|
||||
'capture_command offhost_transfer rsync',
|
||||
'last-error.log',
|
||||
"docker inspect -f '{{json .Mounts}}'",
|
||||
'if len(matches) != 1 or not matches[0]:',
|
||||
]) {
|
||||
@@ -345,6 +374,8 @@ for (const requiredFragment of [
|
||||
'batch must contain exactly three regular files',
|
||||
'manifest artifact checksum mismatch',
|
||||
'mv -- "$partial_dir" "$archive_dir"',
|
||||
'replacement_rollback=',
|
||||
'publish_weekly_checkpoint',
|
||||
'prune_named_directories "$BACKUP_ARCHIVE_DIR"',
|
||||
'prune_named_directories "$BACKUP_WEEKLY_DIR"',
|
||||
]) {
|
||||
@@ -360,6 +391,9 @@ for (const requiredFragment of [
|
||||
'BACKUP_INBOX_DIR=${inbox_dir}',
|
||||
'BACKUP_ARCHIVE_DIR=${archive_dir}',
|
||||
'BACKUP_WEEKLY_DIR=${weekly_dir}',
|
||||
'DEADMAN_MAX_AGE_SECONDS="${DEADMAN_MAX_AGE_SECONDS:-93600}"',
|
||||
'ExecStart=/usr/bin/python3 ${DEADMAN_TARGET}',
|
||||
'systemctl enable --now "${DEADMAN_SERVICE}.timer"',
|
||||
'archives retained',
|
||||
'backup root contains an unmanaged top-level entry',
|
||||
'backup root must not be inside a system-managed directory',
|
||||
@@ -369,6 +403,30 @@ for (const requiredFragment of [
|
||||
}
|
||||
}
|
||||
|
||||
for (const requiredFragment of [
|
||||
'stat.S_IMODE(info.st_mode) != 0o600',
|
||||
'parsed.hostname != "qyapi.weixin.qq.com"',
|
||||
'HYB_WECOM_ALERT_TEST_MODE',
|
||||
'should_send(previous, status, category)',
|
||||
'不要在群内粘贴原始日志',
|
||||
]) {
|
||||
if (!groupAlertHelperContent.includes(requiredFragment)) {
|
||||
errors.push(`send-wecom-group-alert.py 缺少独立脱敏告警合同: ${requiredFragment}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const requiredFragment of [
|
||||
'default=93600',
|
||||
'archive_manifest_invalid',
|
||||
'archive_time_in_future',
|
||||
'FAIL_OFFHOST_DEADMAN',
|
||||
'deadman-alert-state.json',
|
||||
]) {
|
||||
if (!offhostDeadmanContent.includes(requiredFragment)) {
|
||||
errors.push(`check-offhost-backup-deadman.py 缺少 ARM 独立看门狗合同: ${requiredFragment}`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const requiredFragment of [
|
||||
'validate_data_path POSTGRES_DATA_DIR "$POSTGRES_DATA_DIR"',
|
||||
'validate_data_path STORAGE_HOST_DIR "$STORAGE_HOST_DIR"',
|
||||
|
||||
在新工单中引用
屏蔽一个用户