From b78f2a11ce5ae5541891d5159cd49453c81d9550 Mon Sep 17 00:00:00 2001 From: shunli Date: Mon, 27 Jul 2026 10:50:01 +0800 Subject: [PATCH] =?UTF-8?q?=E5=BB=BA=E7=AB=8B=E6=AF=8F=E6=97=A5=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E5=BC=82=E6=9C=BA=E5=A4=87=E4=BB=BD=E5=9F=BA=E7=BA=BF?= =?UTF-8?q?=EF=BC=8807-27-automatic-offhost-backup=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .trellis/spec/hyb-oa/deployment-trialrun.md | 99 ++++ .../check.jsonl | 1 + .../07-27-automatic-offhost-backup/design.md | 76 ++++ .../implement.jsonl | 1 + .../implement.md | 57 +++ .../07-27-automatic-offhost-backup/prd.md | 63 +++ .../research/runtime-topology.md | 32 ++ .../07-27-automatic-offhost-backup/task.json | 26 ++ deploy/trialrun/checklist.md | 8 + .../install-automatic-backup-timer.sh | 227 ++++++++++ .../install-offhost-backup-receiver.sh | 166 +++++++ deploy/trialrun/runbook.md | 46 ++ deploy/trialrun/validate-env-contract.js | 90 ++++ docs/管理员手册.md | 10 + scripts/qa/run-automatic-backup-contract.sh | 375 ++++++++++++++++ scripts/receive-offhost-backup.sh | 242 ++++++++++ scripts/run-automatic-backup.sh | 425 ++++++++++++++++++ server/src/scripts/send-wecom-ops-alert.js | 69 ++- .../unit/scripts/send-wecom-ops-alert.test.js | 67 +++ 19 files changed, 2070 insertions(+), 10 deletions(-) create mode 100644 .trellis/tasks/07-27-automatic-offhost-backup/check.jsonl create mode 100644 .trellis/tasks/07-27-automatic-offhost-backup/design.md create mode 100644 .trellis/tasks/07-27-automatic-offhost-backup/implement.jsonl create mode 100644 .trellis/tasks/07-27-automatic-offhost-backup/implement.md create mode 100644 .trellis/tasks/07-27-automatic-offhost-backup/prd.md create mode 100644 .trellis/tasks/07-27-automatic-offhost-backup/research/runtime-topology.md create mode 100644 .trellis/tasks/07-27-automatic-offhost-backup/task.json create mode 100755 deploy/trialrun/install-automatic-backup-timer.sh create mode 100755 deploy/trialrun/install-offhost-backup-receiver.sh create mode 100755 scripts/qa/run-automatic-backup-contract.sh create mode 100755 scripts/receive-offhost-backup.sh create mode 100755 scripts/run-automatic-backup.sh diff --git a/.trellis/spec/hyb-oa/deployment-trialrun.md b/.trellis/spec/hyb-oa/deployment-trialrun.md index 9c3e0525..9db9d02c 100644 --- a/.trellis/spec/hyb-oa/deployment-trialrun.md +++ b/.trellis/spec/hyb-oa/deployment-trialrun.md @@ -912,3 +912,102 @@ SQL test "$(printf '%s\n' "$actual" | LC_ALL=C sort)" = \ "$(printf '%s\n' "$expected" | LC_ALL=C sort)" ``` + +## Scenario: Daily Restricted Off-Host Backup + +### 1. Scope / Trigger + +- Trigger: installing or changing automatic paired backups, off-host transfer, + retention, freshness alerts or quarterly restore reminders. +- Purpose: keep a 24-hour recovery point without giving the production host + ordinary shell or delete access to the ARM archive. + +### 2. Signatures + +- Production runner: `scripts/run-automatic-backup.sh` with `run`, + `check-stale` and `remind-drill` modes. +- ARM forced receiver: `scripts/receive-offhost-backup.sh`. +- Installers: `install-automatic-backup-timer.sh` and + `install-offhost-backup-receiver.sh`. +- Local contract: `bash scripts/qa/run-automatic-backup-contract.sh`. +- Production required environment: `REMOTE_DIR`, `REMOTE_ENV_FILE`, + `COMPOSE_PROJECT_NAME`, `POSTGRES_DATA_DIR`, `STORAGE_HOST_DIR`, `BUILD_SHA`, + `VITE_BUILD_SHA`, `AUTOMATIC_BACKUP_DIR`, `AUTOMATIC_BACKUP_STATE_DIR`, + `OFFHOST_TARGET`, `OFFHOST_SSH_KEY` and `OFFHOST_KNOWN_HOSTS`. +- Receiver config: independent absolute `BACKUP_INBOX_DIR`, + `BACKUP_ARCHIVE_DIR`, `BACKUP_WEEKLY_DIR`, `RRSYNC_BIN`, `PYTHON_BIN`, + `DAILY_KEEP` and `WEEKLY_KEEP`. + +### 3. Contracts + +- Resolve one active Compose server/database pair and verify its build, PGDATA, + storage mount and database identity before backup. Do not use fixed container + names or repository-relative runtime data. +- Transfer only a complete `auto-` three-file batch to ARM + inbox. The forced key allows write-only `rrsync` or an exact finalize command; + it rejects shell, sender/read and destructive rsync options. +- ARM revalidates manifest identity, exact regular-file set, sizes and SHA-256, + then atomically moves the batch to archive. Archive and weekly directories are + outside the rsync root; an archived name is never overwritten. +- Write production `last-success.json` and prune source copies only after ARM + finalize succeeds. Keep seven automatic source/ARM daily points and four ARM + weekly checkpoints; unrelated manual batches are outside automatic retention. +- Daily, freshness and quarterly reminder timers are separate. Alert summaries + contain categories only. AUTO_BACKUP sends the first failure, worsening/new + findings and the first quarterly reminder, while suppressing an identical + repeated failure or reminder. +- Installer `install` is disabled-by-default; `enable`, real transfer, real + WeCom alert and all server/account/key changes remain `EXTERNAL_GATED`. + +### 4. Validation & Error Matrix + +- Source mount/build/project drift, lock conflict or incomplete backup -> fail + before success state and source retention. +- Transfer/finalize failure, wrong/extra/link artifact, bad hash or second batch + for one UTC day -> leave no off-host success state and preserve source batch. +- Missing, stale or project/build-mismatched success record -> watchdog finding. +- Ordinary shell, rsync sender/read/delete or archive overwrite -> receiver + rejection without changing archive. +- Host/network/WeCom all unavailable at once -> self-hosted alert cannot prove + its own outage; this is a documented physical boundary, not a green state. + +### 5. Good/Base/Bad Cases + +- Good: one verified active source creates a complete batch, write-only rsync + resumes inside inbox, ARM finalize verifies and archives it, then production + writes success and prunes old automatic source points. +- Base: local implementation and synthetic contracts pass while both server + installers remain disabled and explicitly externally gated. +- Bad: give production a normal ARM login key, put archive below inbox, accept + a new host fingerprint automatically, or prune local recovery points after + rsync but before target-side finalize. + +### 6. Tests Required + +- Shell syntax plus `run-automatic-backup-contract.sh` using temporary files and + mocked Docker/SSH boundaries; it must never contact real hosts. +- Focused WeCom alert tests for stdin path redaction, preserved AUTO_BACKUP + issue identity, first/repeated failure and quarterly reminder behavior. +- Environment contract assertions for all four scripts, three timers, explicit + project/data/build/SSH binding and disabled-by-default installation. +- Real ARM setup, first paired batch, failure alert and quarterly shadow restore + are separately approved operational evidence. + +### 7. Wrong vs Correct + +#### Wrong + +```bash +rsync -a / backup-host:/ +rm -rf +``` + +#### Correct + +```bash +# Install only with explicit active identity; default remains disabled. + \ + bash deploy/trialrun/install-automatic-backup-timer.sh install +# The runner writes success and prunes only after restricted target finalize. +bash scripts/run-automatic-backup.sh run +``` diff --git a/.trellis/tasks/07-27-automatic-offhost-backup/check.jsonl b/.trellis/tasks/07-27-automatic-offhost-backup/check.jsonl new file mode 100644 index 00000000..9dd3234a --- /dev/null +++ b/.trellis/tasks/07-27-automatic-offhost-backup/check.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/07-27-automatic-offhost-backup/design.md b/.trellis/tasks/07-27-automatic-offhost-backup/design.md new file mode 100644 index 00000000..eb3539cf --- /dev/null +++ b/.trellis/tasks/07-27-automatic-offhost-backup/design.md @@ -0,0 +1,76 @@ +# 每日自动异机备份设计 + +## 总体边界 + +本任务不建设备份平台。生产主机负责生成和推送一个不可变成对批次，ARM 用受限 SSH 接收身份把传输区与归档区隔离；systemd 负责每日执行、缺失监测和季度提醒。 + +```text +production 02:30 + resolve active Compose identity + -> flock + -> backup-all.sh (DB + storage + manifest) + -> local verify + -> rsync to ARM inbox/.partial + -> restricted finalize + -> ARM verify exact files/manifest/hash + -> atomic move to archive/ + -> daily + weekly retention + -> write safe success state + -> prune local daily copies + +production 07:10 watchdog + -> last off-host success <= 24h ? pass : alert + +quarterly reminder + -> one redacted WeCom reminder per calendar quarter + -> human runs ARM shadow restore separately +``` + +## 生产备份编排 + +- 新增一个宿主 wrapper，所有正式输入由安装器写入 systemd：仓库根、Compose project/file/env、活动 PGDATA/附件目录、完整/短构建 SHA、备份根、状态根、SSH alias/key/known-hosts 和告警开关。 +- wrapper 通过 Compose label 解析唯一 server/db 容器，并把实际 DB/附件 mount、server build SHA 与安装时冻结值逐项比较。任一漂移在调用 `backup-all.sh` 前停止。 +- DB 名称/用户和密码只从活动容器环境在内存中取得，传给现有 Docker 备份路径；不得 source 漂移的旧部署 env，不得输出取值。 +- 每个自动批次使用 `auto-`，源端根和状态根均为显式安全绝对路径、权限 `0700`。`flock` 拒绝同一任务重入。 +- 只有 ARM finalize 成功后才写 `last-success.json` 并清理源端旧日批次。失败保留本次完整源批次供重试，不产生异机成功状态。 + +## ARM 受限接收边界 + +- ARM 一次性安装 rsync，创建密码锁定的专用账号、独立 inbox/archive/weekly 目录和一把专用公钥。该公钥的 `authorized_keys` 强制执行仓库提供的 receiver，并禁用 PTY、端口转发、agent 转发和 X11 转发。 +- receiver 只接受两类 `SSH_ORIGINAL_COMMAND`：rsync server 接收模式，或 `hyb-backup-finalize `。其他命令全部拒绝。 +- rsync 通过 `rrsync -wo` 只能进入 inbox；archive 和 weekly 位于 rrsync 根之外。即使生产端密钥泄露，也不能用 rsync 读取或删除已归档批次。 +- finalize 只接受 `auto-` 安全批次号和完整 manifest SHA。它要求 inbox 中恰有三个普通文件、无链接，manifest 的 runId/status/schema、相对文件名、大小和 SHA-256 全部匹配；已存在 archive 目标时拒绝覆盖。 +- 校验通过后在同一文件系统内原子移动到 archive。每个自然日最多接受一个正式自动批次进入保留计算，避免失陷源端通过短时间伪造大量批次提前挤掉历史。 +- archive 保留最近 7 个日批次；每个 ISO 周首次成功批次用只读硬链接快照形成周检查点，保留最近 4 周。清理不得删除最后一个 archive 或当前 finalize 批次。 +- 旧手工/审计批次不进入自动前缀和自动轮转范围。 + +## 传输与主机身份 + +- 生产 root 使用一把仅用于本接收器的私钥，权限 `0600`；SSH config 使用专用 alias，known_hosts 由人工核对的固定主机指纹生成，timer 固定 `StrictHostKeyChecking=yes`，不得使用 `accept-new`。 +- rsync 只发送完整源批次到同名 `.partial` inbox。网络中断可续传同一 partial；远端 archive 同名存在时 finalize 失败，不覆盖。 +- 仓库、任务和报告只记录 alias 类别、布尔值和批次数量，不记录地址、账号、密钥、公钥原文或完整运行目录。 + +## 告警与静默失败检测 + +- wrapper 无论成功或失败都生成字段白名单 summary；失败 trap 尝试通过活动 server 容器调用现有企微运维告警 CLI，并使用备份专属 state file，避免与 WARCH3 降噪状态互相覆盖。 +- 现有告警 CLI 小范围泛化为接受 stdin summary、保留 summary issueId，并为备份失败/过期提供正确建议；发送仍受既有 `WARCH3_OPS_ALERT_SEND=1` 双门控制。 +- 独立 watchdog timer 在每日巡检前读取宿主 `last-success.json`。文件缺失、格式错误、构建/项目不匹配或成功时间超过 24 小时均产生失败 summary 并尝试告警。 +- 若 Docker、整机网络或企微本身同时不可用，主机无法自证并外发，这是自托管告警的物理边界；watchdog 能覆盖 timer 未运行和短暂失败，但不替代外部主机存活监控。 +- 季度提醒以年份+季度作为 finding key，确保每季度一次且同季度不重复；只提醒人工影子恢复，不执行恢复。 + +## 安装与回滚 + +1. 本地先实现合成测试、受限 receiver、告警适配和两个安装器，不接触生产/ARM。 +2. 获批后先在 ARM 安装 rsync、受限账号/receiver/公钥和目标目录；使用无业务数据的合成批次验证拒绝 shell、拒绝 archive 覆盖和 finalize。 +3. 再在生产安装专用私钥/known_hosts、每日备份 service/timer、stale watchdog 和季度提醒；首次安装先保持 timer disabled。 +4. 手工启动一次获批自动备份，核对源/ARM 哈希、状态和无敏感输出；再 enable timers。 +5. 用不包含业务正文的可控失败验证企微告警后恢复链路，最终确认下次执行时间。 + +回滚只 disable/stop 新 timer、移除 unit 和专用密钥授权；保留已生成的完整备份、状态报告和 ARM archive，不删除历史数据。旧手工 `backup-all.sh` 入口保持兼容。 + +## 取舍 + +- 选择生产推送而非 ARM 拉取，是为了让一次任务同时掌握生成、传输、目标校验和即时告警结果。 +- 不给生产普通 ARM shell，而用 inbox + 强制 receiver + archive 隔离，抵消推送方案最主要的跨机删除风险。 +- 使用 rsync 而不是新备份服务；当前压缩批次规模较小，收益主要是断点续传和成熟 SSH 传输，不追求块级去重。 +- 不自动恢复。季度人工影子恢复才是可恢复性证据，日 timer 只负责生成恢复点。 diff --git a/.trellis/tasks/07-27-automatic-offhost-backup/implement.jsonl b/.trellis/tasks/07-27-automatic-offhost-backup/implement.jsonl new file mode 100644 index 00000000..9dd3234a --- /dev/null +++ b/.trellis/tasks/07-27-automatic-offhost-backup/implement.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/07-27-automatic-offhost-backup/implement.md b/.trellis/tasks/07-27-automatic-offhost-backup/implement.md new file mode 100644 index 00000000..b3e4c12a --- /dev/null +++ b/.trellis/tasks/07-27-automatic-offhost-backup/implement.md @@ -0,0 +1,57 @@ +# 每日自动异机备份实施计划 + +## 实施步骤 + +- [x] 实现宿主自动备份 wrapper：安全绝对路径、唯一 Compose 来源、活动 mount/build 校验、`flock`、成对备份、ARM 推送/finalize、成功状态和源端 7 日保留。 +- [x] 实现 ARM 受限 receiver：只允许 rrsync 写 inbox 和 finalize，严格校验三文件 manifest/大小/哈希、原子归档、7 日 + 4 周保留和最后成功保护。 +- [x] 实现生产与 ARM 安装器：专用账号/密钥/known_hosts 边界、systemd daily/watchdog/quarterly units、安装/卸载幂等和默认不触发真实任务。 +- [x] 泛化现有企微运维告警的 stdin summary、issueId 和备份建议，使用独立状态文件并补单元测试。 +- [x] 增加合成契约测试：来源漂移、锁冲突、不完整/损坏/重复批次、续传 partial、拒绝普通 shell、保留轮转、stale 和季度降噪。 +- [x] 更新部署环境契约、runbook、checklist、管理员手册的最小操作入口和 `.trellis/spec/hyb-oa/deployment-trialrun.md`；不新增平行长文档。 +- [x] 生成一页脱敏恢复卡内容，优先放在现有 runbook 的首部紧急段；由用户复制到公司知识库并安排第二人走查。 +- [ ] 完成本地全量门禁并提交；生产、ARM 安装另行请求批准。 +- [ ] 获批后先安装 ARM 接收端并用合成批次验证，再安装生产 units，手工运行首次真实自动批次，最后 enable timers。 +- [ ] 获批执行一次可控传输失败告警并恢复正常链路；不读取附件正文、不执行业务或企微其他写入。 + +## 预计文件边界 + +- 新增 `scripts/run-automatic-backup.sh`：生产 host 编排与 safe summary。 +- 新增 `scripts/receive-offhost-backup.sh`：ARM forced-command receiver、finalize 和 retention。 +- 新增 `deploy/trialrun/install-automatic-backup-timer.sh`：生产 units 安装/卸载。 +- 新增 `deploy/trialrun/install-offhost-backup-receiver.sh`：ARM rsync/账号/目录/公钥安装检查；软件安装和账号创建只在获批远端阶段执行。 +- 新增 `scripts/qa/run-automatic-backup-contract.sh`：纯合成正负向测试。 +- 修改 `server/src/scripts/send-wecom-ops-alert.js` 及其单测，仅做通用脱敏 summary 兼容。 +- 修改 `deploy/trialrun/validate-env-contract.js`、现有 runbook/checklist/管理员手册和部署规约。 + +## 验证命令 + +- `bash -n scripts/run-automatic-backup.sh scripts/receive-offhost-backup.sh deploy/trialrun/install-automatic-backup-timer.sh deploy/trialrun/install-offhost-backup-receiver.sh scripts/qa/run-automatic-backup-contract.sh` +- `bash scripts/qa/run-automatic-backup-contract.sh` +- `bash scripts/qa/run-backup-restore-drill.sh` +- `cd server && node --check src/scripts/send-wecom-ops-alert.js` +- `cd server && npm test -- --run tests/unit/scripts/send-wecom-ops-alert.test.js` +- `node deploy/trialrun/validate-env-contract.js` +- `python3 ./.trellis/scripts/task.py validate .trellis/tasks/07-27-automatic-offhost-backup` +- `node scripts/qa/check-trellis-governance.mjs` +- `node scripts/qa/check-sensitive-content.mjs` +- `node scripts/qa/audit-docs.mjs --check-registry` +- `node scripts/qa/check-process-contract.mjs --baseline --json` +- `git diff --check` + +## 外部门禁与回滚点 + +- 安装 ARM rsync/账号/authorized_keys/目录属于远端配置写入，须单独批准；不得复用普通登录账号或本地工作站私钥。 +- 安装生产 SSH key/config/systemd units、首次真实备份、真实异机传输和企微失败告警均须单独批准。 +- 首次真实执行前记录现有手工批次数量、磁盘余量、活动容器/mount/build 和现有 timer；只输出脱敏计数/布尔值。 +- 任何目标身份、主机指纹、活动 mount、manifest 或哈希不匹配立即停止，不猜路径、不接受新指纹、不清理源端批次。 +- 回滚新 timer/receiver 时只撤销调度和专用授权；ARM archive、源端完整批次和原 WARCH3 timer 保留。 + +## 本地实施与验证证据 + +- 已实现生产 wrapper、ARM forced receiver、两端安装器和 3 组 systemd timer；安装器默认只安装 disabled units。 +- `run-automatic-backup-contract.sh` 使用临时目录与 mock Docker/SSH/rsync 完成 28 项正负向合同，不连接真实主机。 +- 企微告警聚焦单测 15 项通过；stdin 路径被替换为安全类别，AUTO_BACKUP 首次失败/季度提醒发送且相同重复状态降噪。 +- `run-backup-restore-drill.sh` 本地合成 DB/附件往返通过，本次合成 RTO 为 8 秒；该结果不是生产 RTO 或 ARM 恢复证据。 +- `scripts/check-minimal.sh` 通过：server 162 个测试文件、1004 项测试全部通过，client typecheck/build 通过。 +- 环境合同、Trellis task/governance、敏感扫描、文档注册、流程合同和 `git diff --check` 均通过；文档注册保留既有 22 个未注册文件 warning，无本任务新增文档。 +- ARM 安装、生产 units、真实首批、timer enable、真实企微失败告警和第二人季度走查仍为外部门禁，未执行、不得写成已完成。 diff --git a/.trellis/tasks/07-27-automatic-offhost-backup/prd.md b/.trellis/tasks/07-27-automatic-offhost-backup/prd.md new file mode 100644 index 00000000..9bba28e5 --- /dev/null +++ b/.trellis/tasks/07-27-automatic-offhost-backup/prd.md @@ -0,0 +1,63 @@ +# 每日自动异机备份 + +## Goal + +为 5–10 人内部 OA 建立低维护的每日自动恢复点：生产每天生成同批数据库、附件和 manifest，完整校验后复制到 ARM 备份机，失败可被当天发现并告警；同时用简单保留策略和季度人工影子恢复降低主机故障与静默备份失效风险。 + +## Background + +- 现有 `scripts/backup-all.sh` 已能生成 DB、附件和脱敏 manifest 的完整批次，并在发布完整目录前验证 dump、tar 和 SHA-256。 +- 当前仓库没有任何生产调度器调用 `backup-all.sh`；唯一活动 timer 是 07:30 运维巡检，不执行备份。 +- 生产主机和 ARM 均有 systemd、SSH、`flock` 和 SHA-256 工具；生产有 rsync，ARM 当前没有 rsync。 +- 两台主机之间当前没有可直接复用的免密 SSH 别名或密钥链路，不能把本地工作站的 SSH 配置误当作服务器间自动化能力。 +- 当前手工正确批次和 ARM 影子恢复已证明格式可用，但不能满足 RPO 24 小时或自动失败发现。 +- 已确认采用“生产主动推送到 ARM 的专用受限接收身份”：生产保留即时传输结果，ARM 的已归档批次不暴露给 rsync 写入根。 + +## Scope + +- 生产每日成对备份、ARM 受限接收与目标端校验。 +- 源端 7 个日批次、ARM 7 个日批次加 4 个周检查点的安全轮转。 +- 每日 systemd 调度、独立 stale watchdog、失败企微告警和季度人工恢复提醒。 +- 两端安装器、合成契约测试、现有运维文档/规约更新和一页脱敏恢复卡。 + +## Requirements + +- 每日自动从唯一活动 Compose project 解析 DB 容器、活动附件挂载和实际构建标识，不读取旧项目、固定容器名或仓库默认附件目录。 +- 复用 `backup-all.sh` 生成完整批次；任何来源核验、dump、tar、manifest 或哈希检查失败时不得进入异机完成态。 +- 使用独立 systemd service/timer 和 `flock`，避免与部署、人工备份或自身上次运行重叠。 +- 完整批次通过固定主机指纹和专用受限密钥自动推送到 ARM；该身份无普通 shell，只能把文件写入 inbox 或请求校验/归档，不能通过 rsync 读取或删除 archive。 +- 传输端和目标端都要校验 manifest、文件大小和 SHA-256；同名已归档目标不得覆盖。中断传输可以在 inbox 内续传，但只有完整校验后才能原子进入 archive。 +- 生产保留最近 7 个成功日批次；ARM 至少保留最近 7 个日批次和 4 个周检查点。清理只能发生在本次异机成功确认之后，且不得删除最后一个已知成功批次。 +- 备份或异机校验失败必须生成脱敏状态报告，并复用现有企微运维告警能力；独立 stale watchdog 应在现有 07:30 巡检前检查最近一次异机成功时间，防止备份 timer 未运行或即时告警自身失败造成静默缺口。 +- 告警只包含阶段、状态、finding 数和脱敏建议，不包含数据库凭证、完整目录、SSH 信息、真实业务数据、附件名或原始命令输出。 +- 季度恢复保持人工触发，在 ARM 隔离目标执行；季度 timer 只发送一次脱敏提醒，不得自动运行恢复，更不得触碰生产库。 +- 形成一页脱敏恢复卡，说明主机故障后的负责人、备份位置类别、恢复顺序和验收项，供另一名人员存入公司知识库。 + +## Acceptance Criteria + +- [x] 本地合成测试证明每日 wrapper 只接受唯一活动来源、互斥锁有效，并只发布完整备份批次。 +- [x] 传输测试证明不完整批次、哈希不一致、重复批次和不可达 ARM 均失败，且不产生异机成功状态。 +- [x] 保留测试覆盖 7 个日批次、4 个周检查点、最后成功批次保护和仅在异机确认后清理。 +- [x] 告警测试证明首次失败和状态恶化会告警，重复同一失败降噪；07:30 stale 检查能发现超过 24 小时没有异机成功记录。 +- [ ] systemd 安装器显式绑定 Compose project、活动数据路径、构建标识、备份根和受限远端身份；安装后 timer enabled/active，但部署验证不手工执行真实告警。 +- [ ] 经单独生产授权后完成一次自动日备份：源端完整、ARM 目标端哈希一致、状态报告脱敏，且未读取附件正文或修改业务数据。 +- [ ] 模拟传输失败能在当天产生一条脱敏企微告警；测试后恢复正常链路并完成成功批次。 +- [ ] 恢复卡经第二名内部人员只读走查，能够指出恢复入口、影子目标和 `/ready`/关键计数验收项。 + +## Out of Scope + +- 不引入备份平台、对象存储、MinIO、云厂商 SDK、Kubernetes 或持续复制。 +- 不自动执行生产恢复、数据库 reset、迁移改写或附件回写。 +- 不把真实业务正文、附件清单、凭证、主机地址、完整目录或 SSH 私钥写入仓库、报告或聊天。 +- 不在本任务继续企微本人授权、员工角色、付款、报销或其他产品功能。 +- 不因自动备份完成而取消季度人工影子恢复。 + +## Test Plan + +- Shell 语法与合成文件系统测试，覆盖锁、来源解析、传输失败、目标重复、哈希和保留轮转。 +- 复用 `scripts/qa/run-backup-restore-drill.sh` 验证成对备份合同未回退。 +- 聚焦测试现有企微运维告警的备份状态与 stale finding 适配。 +- `node deploy/trialrun/validate-env-contract.js`。 +- `python3 ./.trellis/scripts/task.py validate .trellis/tasks/07-27-automatic-offhost-backup`。 +- Trellis governance、敏感扫描、文档注册/流程合同和 `git diff --check`。 +- 生产安装、首次自动批次、真实失败告警和 ARM 目标验证分别保持外部门禁。 diff --git a/.trellis/tasks/07-27-automatic-offhost-backup/research/runtime-topology.md b/.trellis/tasks/07-27-automatic-offhost-backup/research/runtime-topology.md new file mode 100644 index 00000000..80bbfda0 --- /dev/null +++ b/.trellis/tasks/07-27-automatic-offhost-backup/research/runtime-topology.md @@ -0,0 +1,32 @@ +# 自动异机备份运行拓扑调查 + +## 边界 + +- 调查日期：2026-07-27。 +- 仅执行只读命令；未创建账号、密钥、目录、timer 或备份，未安装软件、传输数据或发送告警。 +- 只记录工具可用性、批次数量和连接布尔值，不记录主机地址、账号、完整路径、凭证或业务数据。 + +## 仓库事实 + +- `scripts/backup-all.sh` 只被手工文档和合成恢复演练调用，没有生产调度入口。 +- 完整批次由 `database.dump`、`storage.tar.gz` 和 `manifest.json` 组成；最终目录只在 dump/tar 可读取且 SHA-256 生成后发布。 +- DB Docker 路径要求显式容器和数据库参数；附件路径要求显式 `STORAGE_DIR_HOST`，不存在历史默认回退。 +- 07:30 WARCH3 timer 只运行每日巡检。现有告警 CLI 可读取通用 summary 的 `issueId`、`status`、`findingCount` 和报告目录，但帮助、状态文件和部分建议文案仍绑定 WARCH3，备份复用前需要小范围泛化或独立状态文件。 + +## 主机能力 + +| 能力 | 生产主机 | ARM 备份机 | +| --- | --- | --- | +| systemd / SSH / flock / SHA-256 | 可用 | 可用 | +| rsync | 可用 | 当前不可用 | +| 现成跨主机免密别名/链路 | 无 | 无 | + +- 生产当前存在少量历史/手工批次，但没有定时策略。 +- 正确异机批次此前已复制并完成影子恢复；本轮默认目录探测没有把它当作新自动化根，说明自动化必须显式配置目标根，不能猜历史位置。 + +## 设计影响 + +- 无论推送或拉取，都需要一次性安装/配置 ARM rsync 并建立专用受限身份，不能复用本地工作站密钥作为生产事实。 +- 直接给生产保存 ARM 普通登录密钥虽然最省事，但会让生产失陷扩展为异机备份可删除，不应采用。 +- 生产推送更容易在同一任务内获得传输退出码并触发即时告警；要用无交互 shell、仅允许新批次写入的受限接收身份，并让已归档批次脱离该身份的删除权限。 +- ARM 拉取让生产主机没有 ARM 凭证，隔离更强；代价是调度和告警分散在两台机器，需要额外心跳或状态回传才能保证当天发现失败。 diff --git a/.trellis/tasks/07-27-automatic-offhost-backup/task.json b/.trellis/tasks/07-27-automatic-offhost-backup/task.json new file mode 100644 index 00000000..d7f80d25 --- /dev/null +++ b/.trellis/tasks/07-27-automatic-offhost-backup/task.json @@ -0,0 +1,26 @@ +{ + "id": "automatic-offhost-backup", + "name": "automatic-offhost-backup", + "title": "每日自动异机备份", + "description": "为内部 OA 建立每日成对备份、ARM 异机复制、保留轮转、失败告警与季度恢复提醒的低维护自动化基线。", + "status": "in_progress", + "dev_type": null, + "scope": "scripts/backup automation; deploy/trialrun backup receiver and systemd installers; server ops-alert adapter/tests; deployment runbook/checklist/spec evidence", + "package": null, + "priority": "P0", + "creator": "shunli", + "assignee": "shunli", + "createdAt": "2026-07-27", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/deploy/trialrun/checklist.md b/deploy/trialrun/checklist.md index c1354b65..dafb0354 100644 --- a/deploy/trialrun/checklist.md +++ b/deploy/trialrun/checklist.md @@ -47,6 +47,14 @@ - [ ] 同一批次包含 `database.dump`、`storage.tar.gz`、`manifest.json`，两份 SHA-256 校验通过 - [ ] 完整批次已复制到获批异机/异盘，并记录保留周期 - [ ] 同批次已在隔离影子目标恢复：migration 数、关键业务表逐表计数、附件普通文件数均与冻结基线一致，兼容镜像 `/ready` 通过 +- [ ] 自动异机备份（未完成远端门禁前保持未勾选）： + - [ ] **EXTERNAL_GATED**：ARM 已获批安装 rsync、密码锁定专用账号、forced-command key 和独立 inbox/archive/weekly；普通 shell、rsync sender/read/delete 均被拒绝 + - [ ] **EXTERNAL_GATED**：生产专用私钥为 `0600`，known-hosts 指纹已人工核对，安装器显式绑定活动 project、PGDATA、附件、完整/短构建 SHA、备份/状态目录和远端身份 + - [ ] `install-automatic-backup-timer.sh install` 只安装 disabled units；首批真实批次及 ARM manifest/SHA-256 核对通过后才显式执行 `enable` + - [ ] daily/watchdog/drill-reminder 三组 timer enabled/active，daily service 当前 inactive；未通过手工启动真实告警来伪造验证 + - [ ] 最近一次异机成功不超过 24 小时；生产自动日批次为 7 个，ARM 自动日批次为 7 个、周检查点为 4 个，历史手工批次未被轮转 + - [ ] **EXTERNAL_GATED**：可控传输失败仅产生一条脱敏企微告警，恢复链路后新批次成功；同一失败重复运行已降噪 + - [ ] 本季度已由第二名内部人员按 runbook 首部恢复卡在 ARM 隔离目标完成影子恢复走查；timer 未自动恢复生产 - [ ] 停机复制方案已冻结：旧 PostgreSQL 完全停止后复制到空新目录并保留所有权/权限/链接/扩展属性；旧目录原位保留且观察期内不再写入 ## B. 发布（Deploy） diff --git a/deploy/trialrun/install-automatic-backup-timer.sh b/deploy/trialrun/install-automatic-backup-timer.sh new file mode 100755 index 00000000..2a80326f --- /dev/null +++ b/deploy/trialrun/install-automatic-backup-timer.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash +set -euo pipefail + +REMOTE_DIR="${REMOTE_DIR:-/opt/hyb-oa-trialrun}" +REMOTE_ENV_FILE="${REMOTE_ENV_FILE:-${REMOTE_DIR}/deploy/trialrun/.env}" +COMPOSE_FILE="${COMPOSE_FILE:-deploy/trialrun/docker-compose.yml}" +COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-}" +POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:-}" +STORAGE_HOST_DIR="${STORAGE_HOST_DIR:-}" +BUILD_SHA="${BUILD_SHA:-}" +VITE_BUILD_SHA="${VITE_BUILD_SHA:-}" +AUTOMATIC_BACKUP_DIR="${AUTOMATIC_BACKUP_DIR:-}" +AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}" +OFFHOST_TARGET="${OFFHOST_TARGET:-}" +OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}" +OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}" +ALERT_RECIPIENT_EMPLOYEE_NO="${ALERT_RECIPIENT_EMPLOYEE_NO:-HYD000006}" +BACKUP_RUN_AT="${BACKUP_RUN_AT:-02:30}" +WATCHDOG_RUN_AT="${WATCHDOG_RUN_AT:-07:10}" +SERVICE_PREFIX="${SERVICE_PREFIX:-hyb-oa-automatic-backup}" +MODE="${1:-install}" + +RUNNER="${REMOTE_DIR}/scripts/run-automatic-backup.sh" +SYSTEMD_DIR="/etc/systemd/system" + +fail() { + echo "[automatic-backup-installer][ERROR] $1" >&2 + exit 1 +} + +need_root() { + [[ "$(id -u)" == "0" ]] || fail "root privileges are required" +} + +validate_absolute_path() { + local name="$1" + local value="$2" + if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then + fail "$name must be an explicit safe absolute path" + fi + if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \ + || "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then + fail "$name contains an unsafe path segment" + fi +} + +validate_managed_directory_path() { + local name="$1" + local value="$2" + validate_absolute_path "$name" "$value" + [[ "$value" =~ ^/[^/]+/[^/]+ ]] \ + || fail "$name must be a dedicated path below a top-level directory" + case "$value" in + /bin/*|/boot/*|/dev/*|/etc/*|/lib/*|/lib64/*|/proc/*|/run/*|/sbin/*|/sys/*|/usr/*) + fail "$name must not be inside a system-managed directory" + ;; + esac +} + +paths_overlap() { + local first="$1" + local second="$2" + [[ "$first" == "$second" || "$first" == "$second"/* || "$second" == "$first"/* ]] +} + +validate_inputs() { + local name value + for name in REMOTE_DIR REMOTE_ENV_FILE POSTGRES_DATA_DIR STORAGE_HOST_DIR \ + AUTOMATIC_BACKUP_DIR AUTOMATIC_BACKUP_STATE_DIR OFFHOST_SSH_KEY OFFHOST_KNOWN_HOSTS; do + value="${!name:-}" + validate_absolute_path "$name" "$value" + done + validate_managed_directory_path AUTOMATIC_BACKUP_DIR "$AUTOMATIC_BACKUP_DIR" + validate_managed_directory_path AUTOMATIC_BACKUP_STATE_DIR "$AUTOMATIC_BACKUP_STATE_DIR" + [[ -d "$REMOTE_DIR" && -f "$REMOTE_ENV_FILE" && -x "$RUNNER" ]] \ + || fail "deployment root, env file, or backup runner is missing" + [[ -f "$POSTGRES_DATA_DIR/PG_VERSION" && -d "$STORAGE_HOST_DIR" ]] \ + || fail "active data directories are invalid" + [[ "$COMPOSE_PROJECT_NAME" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "Compose project name is invalid" + [[ "$COMPOSE_FILE" =~ ^[A-Za-z0-9._/-]+$ && "$COMPOSE_FILE" != /* \ + && "/$COMPOSE_FILE/" != *"/../"* && "/$COMPOSE_FILE/" != *"/./"* ]] \ + || fail "Compose file must be a safe relative path" + [[ "$BUILD_SHA" =~ ^[0-9a-fA-F]{40,64}$ ]] || fail "full build SHA is invalid" + BUILD_SHA="${BUILD_SHA,,}" + VITE_BUILD_SHA="${VITE_BUILD_SHA,,}" + [[ "$VITE_BUILD_SHA" =~ ^[0-9a-f]{12}$ && "${BUILD_SHA:0:12}" == "$VITE_BUILD_SHA" ]] \ + || fail "frontend build SHA must match the full build prefix" + [[ "$OFFHOST_TARGET" =~ ^[a-z_][a-z0-9_-]*@[A-Za-z0-9._-]+$ ]] \ + || fail "off-host target must use a safe user@alias value" + [[ -f "$OFFHOST_SSH_KEY" && ! -L "$OFFHOST_SSH_KEY" \ + && "$(stat -c %a "$OFFHOST_SSH_KEY")" == "600" ]] \ + || fail "off-host private key must exist with mode 0600" + [[ -s "$OFFHOST_KNOWN_HOSTS" ]] || fail "pinned known-hosts file is missing" + [[ "$BACKUP_RUN_AT" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || fail "backup time is invalid" + [[ "$WATCHDOG_RUN_AT" =~ ^([01][0-9]|2[0-3]):[0-5][0-9]$ ]] || fail "watchdog time is invalid" + [[ "$SERVICE_PREFIX" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "service prefix is invalid" + [[ "$ALERT_RECIPIENT_EMPLOYEE_NO" =~ ^[A-Za-z0-9_-]{1,80}$ ]] || fail "alert recipient id is invalid" + local -a isolated_paths=( + "$POSTGRES_DATA_DIR" "$STORAGE_HOST_DIR" "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR" + ) + local first_index second_index + for ((first_index = 0; first_index < ${#isolated_paths[@]}; first_index += 1)); do + for ((second_index = first_index + 1; second_index < ${#isolated_paths[@]}; second_index += 1)); do + if paths_overlap "${isolated_paths[$first_index]}" "${isolated_paths[$second_index]}"; then + fail "data, backup, and state directories must be independent" + fi + done + done +} + +environment_lines() { + cat <"$service_file" <"$timer_file" </dev/null 2>&1 || true + systemctl stop "${SERVICE_PREFIX}-${suffix}.service" >/dev/null 2>&1 || true + rm -f -- "$SYSTEMD_DIR/${SERVICE_PREFIX}-${suffix}.timer" \ + "$SYSTEMD_DIR/${SERVICE_PREFIX}-${suffix}.service" + done + systemctl daemon-reload + echo "[automatic-backup-installer][PASS] units removed; backups and credentials retained" +} + +case "$MODE" in + install) + install_units + ;; + enable) + enable_units + ;; + uninstall|--uninstall) + uninstall_units + ;; + --help|-h) + echo "Usage: bash install-automatic-backup-timer.sh [install|enable|uninstall]" + ;; + *) + fail "unsupported mode" + ;; +esac diff --git a/deploy/trialrun/install-offhost-backup-receiver.sh b/deploy/trialrun/install-offhost-backup-receiver.sh new file mode 100755 index 00000000..b4fe4bab --- /dev/null +++ b/deploy/trialrun/install-offhost-backup-receiver.sh @@ -0,0 +1,166 @@ +#!/usr/bin/env bash +set -euo pipefail + +BACKUP_USER="${BACKUP_USER:-hybbackup}" +BACKUP_ROOT="${BACKUP_ROOT:-}" +PUBLIC_KEY_FILE="${PUBLIC_KEY_FILE:-}" +RECEIVER_SOURCE="${RECEIVER_SOURCE:-}" +RECEIVER_TARGET="${RECEIVER_TARGET:-/usr/local/sbin/hyb-oa-offhost-receiver}" +CONFIG_FILE="${CONFIG_FILE:-/etc/hyb-oa-offhost-backup.conf}" +DAILY_KEEP="${DAILY_KEEP:-7}" +WEEKLY_KEEP="${WEEKLY_KEEP:-4}" +MODE="${1:-install}" + +fail() { + echo "[offhost-receiver-installer][ERROR] $1" >&2 + exit 1 +} + +need_root() { + [[ "$(id -u)" == "0" ]] || fail "root privileges are required" +} + +validate_absolute_path() { + local name="$1" + local value="$2" + if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then + fail "$name must be an explicit safe absolute path" + fi + if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \ + || "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then + fail "$name contains an unsafe path segment" + fi +} + +validate_backup_root() { + validate_absolute_path BACKUP_ROOT "$BACKUP_ROOT" + [[ "$BACKUP_ROOT" =~ ^/[^/]+/[^/]+ ]] \ + || fail "backup root must be a dedicated path below a top-level directory" + case "$BACKUP_ROOT" in + /bin/*|/boot/*|/dev/*|/etc/*|/lib/*|/lib64/*|/proc/*|/run/*|/sbin/*|/sys/*|/usr/*) + fail "backup root must not be inside a system-managed directory" + ;; + esac + [[ ! -L "$BACKUP_ROOT" ]] || fail "backup root must not be a symbolic link" + if [[ -d "$BACKUP_ROOT" ]]; then + local unmanaged_entry + unmanaged_entry="$(find "$BACKUP_ROOT" -mindepth 1 -maxdepth 1 \ + ! -name inbox ! -name archive ! -name weekly -print -quit)" + [[ -z "$unmanaged_entry" ]] || fail "backup root contains an unmanaged top-level entry" + fi +} + +validate_inputs() { + [[ "$BACKUP_USER" =~ ^[a-z_][a-z0-9_-]{0,30}$ ]] || fail "backup user is invalid" + validate_backup_root + validate_absolute_path PUBLIC_KEY_FILE "$PUBLIC_KEY_FILE" + validate_absolute_path RECEIVER_SOURCE "$RECEIVER_SOURCE" + validate_absolute_path RECEIVER_TARGET "$RECEIVER_TARGET" + validate_absolute_path CONFIG_FILE "$CONFIG_FILE" + [[ -f "$PUBLIC_KEY_FILE" && ! -L "$PUBLIC_KEY_FILE" ]] || fail "public key file is missing" + [[ -f "$RECEIVER_SOURCE" && ! -L "$RECEIVER_SOURCE" ]] || fail "receiver source is missing" + [[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ && "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] \ + || fail "retention values must be positive" + command -v rsync >/dev/null 2>&1 || fail "rsync must be installed before receiver setup" + command -v rrsync >/dev/null 2>&1 || fail "rrsync must be installed before receiver setup" + command -v python3 >/dev/null 2>&1 || fail "python3 is required" +} + +read_public_key() { + local key_type key_body remainder extra_line_count + extra_line_count="$(awk 'NF { count += 1 } END { print count + 0 }' "$PUBLIC_KEY_FILE")" + [[ "$extra_line_count" == "1" ]] || fail "public key file must contain exactly one non-empty key" + read -r key_type key_body remainder <"$PUBLIC_KEY_FILE" + [[ "$key_type" == "ssh-ed25519" || "$key_type" == "sk-ssh-ed25519@openssh.com" ]] \ + || fail "only Ed25519 public keys are accepted" + [[ "$key_body" =~ ^[A-Za-z0-9+/=]+$ ]] || fail "public key body is invalid" + printf '%s %s' "$key_type" "$key_body" +} + +install_receiver() { + need_root + validate_inputs + + if ! id "$BACKUP_USER" >/dev/null 2>&1; then + useradd --create-home --shell /bin/bash "$BACKUP_USER" + fi + passwd -l "$BACKUP_USER" >/dev/null 2>&1 || true + + local home_dir inbox_dir archive_dir weekly_dir ssh_dir auth_file public_key rrsync_bin + home_dir="$(getent passwd "$BACKUP_USER" | awk -F: '{print $6}')" + validate_absolute_path BACKUP_USER_HOME "$home_dir" + inbox_dir="$BACKUP_ROOT/inbox" + archive_dir="$BACKUP_ROOT/archive" + weekly_dir="$BACKUP_ROOT/weekly" + mkdir -p "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh" + [[ -d "$BACKUP_ROOT" && ! -L "$BACKUP_ROOT" \ + && -d "$inbox_dir" && ! -L "$inbox_dir" \ + && -d "$archive_dir" && ! -L "$archive_dir" \ + && -d "$weekly_dir" && ! -L "$weekly_dir" ]] \ + || fail "backup receiver directories must not be symbolic links" + chown -R "$BACKUP_USER:$BACKUP_USER" "$BACKUP_ROOT" "$home_dir/.ssh" + chmod 0700 "$BACKUP_ROOT" "$inbox_dir" "$archive_dir" "$weekly_dir" "$home_dir/.ssh" + + install -o root -g root -m 0755 "$RECEIVER_SOURCE" "$RECEIVER_TARGET" + rrsync_bin="$(command -v rrsync)" + cat >"$CONFIG_FILE" <"$auth_file" </dev/null + echo "[offhost-receiver-installer][PASS] restricted receiver installed" +} + +uninstall_receiver() { + need_root + [[ "$BACKUP_USER" =~ ^[a-z_][a-z0-9_-]{0,30}$ ]] || fail "backup user is invalid" + validate_absolute_path RECEIVER_TARGET "$RECEIVER_TARGET" + validate_absolute_path CONFIG_FILE "$CONFIG_FILE" + if id "$BACKUP_USER" >/dev/null 2>&1; then + local home_dir auth_file + home_dir="$(getent passwd "$BACKUP_USER" | awk -F: '{print $6}')" + auth_file="$home_dir/.ssh/authorized_keys" + if [[ -f "$auth_file" ]] && grep -Fq 'hyb-oa-managed-offhost-backup' "$auth_file"; then + rm -f -- "$auth_file" + fi + fi + rm -f -- "$RECEIVER_TARGET" "$CONFIG_FILE" + echo "[offhost-receiver-installer][PASS] receiver authorization removed; archives retained" +} + +case "$MODE" in + install) + install_receiver + ;; + uninstall|--uninstall) + uninstall_receiver + ;; + --help|-h) + echo "Usage: BACKUP_ROOT= PUBLIC_KEY_FILE= RECEIVER_SOURCE= bash install-offhost-backup-receiver.sh [install|uninstall]" + ;; + *) + fail "unsupported mode" + ;; +esac diff --git a/deploy/trialrun/runbook.md b/deploy/trialrun/runbook.md index 847fc815..2f045046 100644 --- a/deploy/trialrun/runbook.md +++ b/deploy/trialrun/runbook.md @@ -2,6 +2,27 @@ > 目标：按本文档在一台新机器完成 **启动 → 迁移 → 初始化管理员 → 验收 → 备份 → 恢复 → 回滚** 演练。 +## 紧急恢复卡（复制到公司知识库的一页版） + +适用情形：生产主机不可恢复、数据库或附件损坏，且已由负责人批准进入事故恢复。主操作人为系统管理员，复核人为另一名已完成季度演练的内部人员；两人均不得在聊天或工单中粘贴凭证、业务正文、真实目录或原始日志。 + +1. 先暂停外部入口和新业务写入，不删除故障主机、旧数据目录或 ARM 归档；记录故障时间、最近成功批次编号类别和恢复负责人。 +2. 在 ARM 的只读归档中选择最近一个 `complete` 批次，复核三文件、manifest、SHA-256 与来源标识；先复制到隔离恢复工作区，不直接对归档原件操作。 +3. 在获批的新主机/隔离目标依次执行以下三个入口；尖括号值只从受控运维配置取得，不写回本文： + +```bash +RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE= \ + ./scripts/restore-db.sh /database.dump +RESTORE_SCOPE=shadow ./scripts/restore-storage.sh \ + /storage.tar.gz +curl -fsS +``` + +4. 复核 `/ready`、运行构建标识、成功 migration 数、组织/权限/报销/付款等关键表计数、附件普通文件数及 DB/附件关联；任何一项不一致都停止切换。 +5. 真实生产恢复仍是 **EXTERNAL_GATED**：另建或更新获批 Trellis task，冻结新数据目录、回滚点和停机窗口后才切换。恢复后保留 ARM 归档和故障主机原数据，记录 RTO/RPO 与结论。 + +本卡必须由第二名内部人员每季度只读走查一次，并实际在 ARM 隔离目标完成人工影子恢复；自动 timer 只提醒，不自动恢复。 + ## 0A. 如果官网 `hyb-web` 已在同机运行（推荐方案） - 官网继续保持 `hyinda.com / www.hyinda.com` 静态托管，不要改官网目录和配置。 - OA 新增独立二级域名，例如 `oa.hyinda.com`。 @@ -205,6 +226,31 @@ manifest 必须至少记录一个真实来源标识：活动 `/ready` 有构建 整批复制到获批的异机/异盘位置并记录保留周期；不能只复制 DB 或附件中的一份。 +#### 每日自动异机备份（当前仅本地实现，远端启用为 EXTERNAL_GATED） + +- 每日 02:30 由生产主机解析唯一活动 Compose project 和真实 DB/附件挂载，生成同批三文件，ARM 校验并原子归档成功后才写成功状态和执行源端轮转。 +- 生产保留 7 个自动日批次；ARM 保留 7 个日批次和 4 个周检查点。07:10 watchdog 检查最近 24 小时异机成功记录；季度 timer 只发送人工影子恢复提醒。 +- ARM 使用专用密码锁定账号和 forced-command key：rsync 只能写 inbox，不能读取/删除 archive，也没有普通 shell。生产固定专用 key 和人工核对的 known-hosts，禁止 `accept-new`。 +- 失败告警只包含阶段、状态、finding 数和类别建议。同一失败降噪；首次失败、状态恶化、新 finding 和每季度首次恢复提醒才发送。 + +安装顺序必须是：先获批 ARM 端安装 rsync、受限身份和 receiver，用合成批次验证；再获批生产端安装 units，手工完成首个真实批次；最后显式执行生产安装器的 `enable`。两个安装器默认 `install` 均不代表已启用，不得把本地合同通过写成远端完成。 + +```bash +# ARM：参数从受控配置取得；install 会写账号/authorized_keys/receiver，需单独批准。 +BACKUP_ROOT= PUBLIC_KEY_FILE= \ +RECEIVER_SOURCE= \ + bash deploy/trialrun/install-offhost-backup-receiver.sh install + +# 生产：先只安装 disabled units；路径、项目、构建和 SSH 身份必须绑定活动实例。 + \ + bash deploy/trialrun/install-automatic-backup-timer.sh install + +# 首批真实备份、ARM 哈希和脱敏告警均经单独确认后，才允许启用三组 timer。 + bash deploy/trialrun/install-automatic-backup-timer.sh enable +``` + +卸载只撤销 timer/receiver 授权，保留源端批次、状态、ARM archive 和凭证文件；专用账号/密钥的最终销毁另走获批清理。 + ### 1.5.2 本地隔离恢复演练 开发机具备 Docker、PostgreSQL 客户端，以及 Node.js 或 Python 3 时，使用合成数据运行： diff --git a/deploy/trialrun/validate-env-contract.js b/deploy/trialrun/validate-env-contract.js index bc78ae7e..d12b56b3 100755 --- a/deploy/trialrun/validate-env-contract.js +++ b/deploy/trialrun/validate-env-contract.js @@ -19,6 +19,16 @@ const timerScriptPath = path.resolve( process.env.ENV_CONTRACT_TIMER_SCRIPT_PATH || path.join(rootDir, 'deploy/trialrun/install-warch3-ops-timer.sh'), ); +const automaticBackupTimerScriptPath = path.join( + rootDir, + 'deploy/trialrun/install-automatic-backup-timer.sh', +); +const offhostReceiverInstallerPath = path.join( + rootDir, + 'deploy/trialrun/install-offhost-backup-receiver.sh', +); +const automaticBackupRunnerPath = path.join(rootDir, 'scripts/run-automatic-backup.sh'); +const offhostReceiverPath = path.join(rootDir, 'scripts/receive-offhost-backup.sh'); const nginxPaths = [ path.join(rootDir, 'deploy/trialrun/nginx-hyb-oa.conf'), path.join(rootDir, 'deploy/trialrun/nginx.conf.example'), @@ -143,6 +153,10 @@ const runbookContent = readFile(runbookPath); const checklistContent = readFile(checklistPath); const deployScriptContent = readFile(deployScriptPath); const timerScriptContent = readFile(timerScriptPath); +const automaticBackupTimerScriptContent = readFile(automaticBackupTimerScriptPath); +const offhostReceiverInstallerContent = readFile(offhostReceiverInstallerPath); +const automaticBackupRunnerContent = readFile(automaticBackupRunnerPath); +const offhostReceiverContent = readFile(offhostReceiverPath); const nginxContents = nginxPaths.map((filePath) => ({ filePath, content: readFile(filePath), @@ -274,6 +288,82 @@ for (const requiredFragment of [ } } +for (const requiredFragment of [ + 'MODE="${1:-install}"', + 'write_service daily run', + 'write_service watchdog check-stale', + 'write_service drill-reminder remind-drill', + 'write_timer daily', + 'write_timer watchdog', + 'write_timer drill-reminder', + 'Environment=COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME}', + 'Environment=POSTGRES_DATA_DIR=${POSTGRES_DATA_DIR}', + 'Environment=STORAGE_HOST_DIR=${STORAGE_HOST_DIR}', + 'Environment=BUILD_SHA=${BUILD_SHA}', + 'Environment=VITE_BUILD_SHA=${VITE_BUILD_SHA}', + 'Environment=AUTOMATIC_BACKUP_DIR=${AUTOMATIC_BACKUP_DIR}', + 'Environment=AUTOMATIC_BACKUP_STATE_DIR=${AUTOMATIC_BACKUP_STATE_DIR}', + 'Environment=OFFHOST_TARGET=${OFFHOST_TARGET}', + 'Environment=OFFHOST_SSH_KEY=${OFFHOST_SSH_KEY}', + 'Environment=OFFHOST_KNOWN_HOSTS=${OFFHOST_KNOWN_HOSTS}', + 'install)\n install_units', + 'enable)\n enable_units', +]) { + if (!automaticBackupTimerScriptContent.includes(requiredFragment)) { + errors.push(`install-automatic-backup-timer.sh 缺少自动备份安装合同: ${requiredFragment}`); + } +} + +for (const requiredFragment of [ + 'COMPOSE_CMD=(docker compose)', + '-p "$COMPOSE_PROJECT_NAME"', + 'flock -n 9', + '"$actual_storage" == "$STORAGE_HOST_DIR"', + '"$actual_pgdata" == "$POSTGRES_DATA_DIR"', + '--partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo=', + 'StrictHostKeyChecking=yes', + 'hyb-backup-finalize $run_id $manifest_sha', + 'write_success_state "$run_id" "$manifest_sha"', + 'prune_local_backups "$batch_name"', + '--summary-file=-', +]) { + if (!automaticBackupRunnerContent.includes(requiredFragment)) { + errors.push(`run-automatic-backup.sh 缺少运行时安全合同: ${requiredFragment}`); + } +} + +for (const requiredFragment of [ + '"$original_command" == rsync\\ --server\\ *', + '--sender([[:space:]]|$)', + 'exec "$RRSYNC_BIN" -wo "$BACKUP_INBOX_DIR"', + 'hyb-backup-finalize', + 'batch must contain exactly three regular files', + 'manifest artifact checksum mismatch', + 'mv -- "$partial_dir" "$archive_dir"', + 'prune_named_directories "$BACKUP_ARCHIVE_DIR"', + 'prune_named_directories "$BACKUP_WEEKLY_DIR"', +]) { + if (!offhostReceiverContent.includes(requiredFragment)) { + errors.push(`receive-offhost-backup.sh 缺少受限接收合同: ${requiredFragment}`); + } +} + +for (const requiredFragment of [ + 'MODE="${1:-install}"', + 'passwd -l "$BACKUP_USER"', + 'command="${RECEIVER_TARGET}",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty,no-user-rc', + 'BACKUP_INBOX_DIR=${inbox_dir}', + 'BACKUP_ARCHIVE_DIR=${archive_dir}', + 'BACKUP_WEEKLY_DIR=${weekly_dir}', + 'archives retained', + 'backup root contains an unmanaged top-level entry', + 'backup root must not be inside a system-managed directory', +]) { + if (!offhostReceiverInstallerContent.includes(requiredFragment)) { + errors.push(`install-offhost-backup-receiver.sh 缺少受限身份安装合同: ${requiredFragment}`); + } +} + for (const requiredFragment of [ 'validate_data_path POSTGRES_DATA_DIR "$POSTGRES_DATA_DIR"', 'validate_data_path STORAGE_HOST_DIR "$STORAGE_HOST_DIR"', diff --git a/docs/管理员手册.md b/docs/管理员手册.md index d6dd5b37..36868239 100644 --- a/docs/管理员手册.md +++ b/docs/管理员手册.md @@ -356,6 +356,16 @@ WAPC2 请假/补卡只读 POC 操作口径： 6. 管理员查看审计日志中是否有异常导出、权限拒绝或付款动作。 7. 记录当天新增问题、影响等级、处理人、结论和是否需要暂停。 +自动异机备份运维口径： + +1. 自动备份以 OA 仓库 runbook 首部“紧急恢复卡”为唯一短入口；公司知识库保存该卡副本，并指定系统管理员和一名替补操作人。 +2. 每日 02:30 生成数据库、附件、manifest 同批恢复点并推送 ARM；07:10 检查最近 24 小时异机成功状态。生产保留 7 个自动日批次，ARM 保留 7 个日批次和 4 个周检查点。 +3. ARM 专用身份只能写 inbox 和请求 finalize，不能登录 shell、读取或删除 archive。任何人不得为了省事改用普通 ARM 登录账号或关闭 known-hosts 固定校验。 +4. 企微只收脱敏异常：首次失败、恶化或新 finding 发送；同一失败降噪。季度提醒只要求人工在 ARM 隔离目标恢复，不会自动恢复生产。 +5. 当前仓库实现和本地合成合同不等于服务器已安装。ARM receiver、生产 units、首次真实批次、timer enable 和真实失败告警均为 **EXTERNAL_GATED**，必须逐阶段批准并回填证据。 +6. 每季度由替补操作人按恢复卡走查，至少能指出归档类别、三条恢复入口、`/ready` 和关键计数验收项。不能独立说明时，本季度恢复门禁不通过。 +7. 卸载 timer 或 receiver 时保留历史 archive 和源端完整批次；账号、密钥、备份数据的删除必须另行批准。 + WROAD2/WARCH3 企微受控上线每日巡检： 1. 试用范围固定为测试账号 `SMOKE002 -> SMOKE101`，管理员观察和告警收件人为 `HYD000006`；`HYD000006` 是 `wangshunli` 的员工号，`wangshunli` 只作为姓名/账号别名记录。试用周期 3 天；WARCH3 可通过 `hyb-oa-warch3-ops.timer` 每天 07:30 自动触发。 diff --git a/scripts/qa/run-automatic-backup-contract.sh b/scripts/qa/run-automatic-backup-contract.sh new file mode 100755 index 00000000..7da8c7ad --- /dev/null +++ b/scripts/qa/run-automatic-backup-contract.sh @@ -0,0 +1,375 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +TEST_DIR="$(mktemp -d)" +PASS_COUNT=0 + +cleanup() { + if [[ -n "${TEST_DIR:-}" && -d "$TEST_DIR" && "$TEST_DIR" == /tmp/* ]]; then + rm -rf -- "$TEST_DIR" + fi +} +trap cleanup EXIT + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + printf '[PASS] %s\n' "$1" +} + +expect_failure() { + local label="$1" + shift + if "$@" >"$TEST_DIR/last-command.out" 2>&1; then + printf '[FAIL] expected rejection: %s\n' "$label" >&2 + exit 1 + fi + pass "$label" +} + +make_layout() { + local root="$1" + mkdir -p "$root/inbox" "$root/archive" "$root/weekly" +} + +make_batch() { + local target="$1" + local run_id="$2" + mkdir -p "$target" + printf 'synthetic database payload\n' >"$target/database.dump" + printf 'synthetic storage payload\n' >"$target/storage.tar.gz" + python3 - "$target" "$run_id" <<'PYTHON' +import hashlib +import json +import os +import sys + +target, run_id = sys.argv[1:] +def artifact(file_name): + file_path = os.path.join(target, file_name) + with open(file_path, "rb") as handle: + content = handle.read() + return {"file": file_name, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()} + +manifest = { + "schemaVersion": 1, + "runId": run_id, + "createdAt": "2026-01-01T00:00:00Z", + "buildSha": "a" * 40, + "sourceImageDigest": None, + "status": "complete", + "artifacts": { + "database": artifact("database.dump"), + "storage": artifact("storage.tar.gz"), + }, +} +with open(os.path.join(target, "manifest.json"), "x", encoding="utf-8") as handle: + json.dump(manifest, handle, indent=2) + handle.write("\n") +PYTHON +} + +receiver() { + local layout="$1" + shift + env RECEIVER_TEST_MODE=1 RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \ + BACKUP_INBOX_DIR="$layout/inbox" BACKUP_ARCHIVE_DIR="$layout/archive" \ + BACKUP_WEEKLY_DIR="$layout/weekly" DAILY_KEEP=7 WEEKLY_KEEP=4 \ + "$ROOT_DIR/scripts/receive-offhost-backup.sh" "$@" +} + +CURRENT_RUN_ID="auto-$(date -u +%Y%m%dT%H%M%SZ)" +CURRENT_BATCH="hyb-oa-backup-$CURRENT_RUN_ID" + +VALID_LAYOUT="$TEST_DIR/receiver-valid" +make_layout "$VALID_LAYOUT" +make_batch "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID" +VALID_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')" +receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null +[[ -d "$VALID_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" ]] +pass "receiver finalizes one complete synthetic batch" + +receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null +pass "receiver treats an identical repeated finalize as idempotent" + +SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z" +if [[ "$SECOND_RUN_ID" == "$CURRENT_RUN_ID" ]]; then + SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z" +fi +SECOND_BATCH="hyb-oa-backup-$SECOND_RUN_ID" +make_batch "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID" +SECOND_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')" +expect_failure "receiver rejects a different automatic batch for the same UTC day" \ + receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA" + +negative_receiver_case() { + local label="$1" + local mutation="$2" + local layout="$TEST_DIR/receiver-negative-${PASS_COUNT}" + make_layout "$layout" + make_batch "$layout/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID" + case "$mutation" in + corrupt-artifact) printf 'changed\n' >>"$layout/inbox/${CURRENT_BATCH}.partial/database.dump" ;; + corrupt-manifest) printf ' ' >>"$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" ;; + extra-file) printf 'extra\n' >"$layout/inbox/${CURRENT_BATCH}.partial/extra.txt" ;; + symlink) rm "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"; ln -s database.dump "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" ;; + esac + local sha + sha="$(sha256sum "$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')" + if [[ "$mutation" == "corrupt-manifest" ]]; then + sha="$VALID_MANIFEST_SHA" + fi + expect_failure "$label" receiver "$layout" finalize-test "$CURRENT_RUN_ID" "$sha" + [[ ! -e "$layout/archive/$CURRENT_BATCH" ]] +} + +negative_receiver_case "receiver rejects an artifact checksum mismatch" corrupt-artifact +negative_receiver_case "receiver rejects a manifest checksum mismatch" corrupt-manifest +negative_receiver_case "receiver rejects an extra file" extra-file +negative_receiver_case "receiver rejects links in a batch" symlink + +RESUME_LAYOUT="$TEST_DIR/receiver-resume" +make_layout "$RESUME_LAYOUT" +make_batch "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID" +RESUME_MANIFEST_SHA="$(sha256sum "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')" +mv "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" "$TEST_DIR/resume-storage.tar.gz" +expect_failure "receiver keeps an incomplete partial batch out of archive" \ + receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA" +mv "$TEST_DIR/resume-storage.tar.gz" "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" +receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA" >/dev/null +[[ -d "$RESUME_LAYOUT/archive/$CURRENT_BATCH" ]] +pass "receiver finalizes the same partial directory after transfer resumes" + +FORCED_LAYOUT="$TEST_DIR/receiver-forced" +make_layout "$FORCED_LAYOUT" +expect_failure "receiver rejects rsync sender/read mode" env \ + RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \ + BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \ + BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \ + SSH_ORIGINAL_COMMAND="rsync --server --sender -logDtpre.iLsfxCIvu . ." \ + "$ROOT_DIR/scripts/receive-offhost-backup.sh" forced +expect_failure "receiver rejects a sender flag in a later rsync position" env \ + RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \ + BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \ + BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \ + SSH_ORIGINAL_COMMAND="rsync --server -logDtpre.iLsfxCIvu --sender . ." \ + "$ROOT_DIR/scripts/receive-offhost-backup.sh" forced +expect_failure "receiver rejects destructive rsync flags" env \ + RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \ + BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \ + BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \ + SSH_ORIGINAL_COMMAND="rsync --server --delete -logDtpre.iLsfxCIvu . ." \ + "$ROOT_DIR/scripts/receive-offhost-backup.sh" forced +expect_failure "receiver rejects an ordinary shell command" env \ + RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \ + BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \ + BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" SSH_ORIGINAL_COMMAND="sh -c id" \ + "$ROOT_DIR/scripts/receive-offhost-backup.sh" forced + +NESTED_LAYOUT="$TEST_DIR/receiver-nested" +mkdir -p "$NESTED_LAYOUT/inbox/archive" "$NESTED_LAYOUT/weekly" +expect_failure "receiver rejects archive nested inside the rsync inbox" env \ + RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \ + BACKUP_INBOX_DIR="$NESTED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$NESTED_LAYOUT/inbox/archive" \ + BACKUP_WEEKLY_DIR="$NESTED_LAYOUT/weekly" \ + "$ROOT_DIR/scripts/receive-offhost-backup.sh" validate-config + +RETENTION_LAYOUT="$TEST_DIR/receiver-retention" +make_layout "$RETENTION_LAYOUT" +for day in 01 02 03 04 05 06 07; do + mkdir "$RETENTION_LAYOUT/archive/hyb-oa-backup-auto-202601${day}T023000Z" +done +for week in 01 02 03 04; do + mkdir "$RETENTION_LAYOUT/weekly/week-2026-W${week}" +done +make_batch "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID" +RETENTION_SHA="$(sha256sum "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')" +receiver "$RETENTION_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RETENTION_SHA" >/dev/null +[[ "$(find "$RETENTION_LAYOUT/archive" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "7" ]] +[[ "$(find "$RETENTION_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "4" ]] +[[ -d "$RETENTION_LAYOUT/archive/$CURRENT_BATCH" ]] +pass "receiver retains seven daily and four weekly recovery points while protecting current success" + +EMPTY_SOURCE="$TEST_DIR/empty-rsync-source" +EMPTY_TARGET="$TEST_DIR/empty-rsync-target" +mkdir "$EMPTY_SOURCE" "$EMPTY_TARGET" +rsync -a --dry-run --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= "$EMPTY_SOURCE/" "$EMPTY_TARGET/" +pass "installed rsync accepts the production chmod contract" + +WRAPPER_ROOT="$TEST_DIR/wrapper-root" +WRAPPER_PG="$TEST_DIR/wrapper-pg" +WRAPPER_STORAGE="$TEST_DIR/wrapper-storage" +WRAPPER_BACKUPS="$TEST_DIR/wrapper-backups" +WRAPPER_STATE="$TEST_DIR/wrapper-state" +MOCK_BIN="$TEST_DIR/mock-bin" +mkdir -p "$WRAPPER_ROOT/deploy/trialrun" "$WRAPPER_PG" "$WRAPPER_STORAGE" \ + "$WRAPPER_BACKUPS" "$WRAPPER_STATE" "$MOCK_BIN" +printf '16\n' >"$WRAPPER_PG/PG_VERSION" +printf 'synthetic attachment\n' >"$WRAPPER_STORAGE/marker.txt" +printf 'synthetic env\n' >"$WRAPPER_ROOT/deploy/trialrun/.env" +ln -s "$ROOT_DIR/scripts" "$WRAPPER_ROOT/scripts" +printf 'private-key-fixture\n' >"$TEST_DIR/offhost-key" +chmod 0600 "$TEST_DIR/offhost-key" +printf 'backup-alias ssh-ed25519 fixture\n' >"$TEST_DIR/known-hosts" + +BUILD_SHA="abcdef1234567890abcdef1234567890abcdef12" +VITE_BUILD_SHA="${BUILD_SHA:0:12}" +MOCK_ALERT_LOG="$TEST_DIR/mock-alert.log" +export MOCK_PG="$WRAPPER_PG" MOCK_STORAGE="$WRAPPER_STORAGE" MOCK_BUILD_SHA="$BUILD_SHA" MOCK_ALERT_LOG + +cat >"$MOCK_BIN/docker" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then exit 0; fi +if [[ "${1:-}" == "compose" ]]; then + args=" $* " + if [[ "$args" == *" ps -q server "* ]]; then echo server-id; exit 0; fi + if [[ "$args" == *" ps -q db "* ]]; then echo db-id; exit 0; fi + if [[ "$args" == *" exec -T server "* ]]; then cat >>"$MOCK_ALERT_LOG"; exit 0; fi +fi +if [[ "${1:-}" == "inspect" ]]; then + template="${3:-}" + container="${4:-}" + case "$template" in + *State.Status*) echo running ;; + *Config.Env*) + if [[ "$container" == "server-id" ]]; then + printf 'BUILD_SHA=%s\nSTORAGE_DIR=/app/storage\nDATABASE_URL=postgresql://backup_user:fixture@db:5432/hyb_oa\n' "$MOCK_BUILD_SHA" + else + printf 'POSTGRES_DB=hyb_oa\nPOSTGRES_USER=backup_user\nPOSTGRES_PASSWORD=fixture\n' + fi + ;; + *Mounts*) + if [[ "$container" == "server-id" ]]; then + printf '/app/storage\t%s\n' "$MOCK_STORAGE" + else + printf '/var/lib/postgresql/data\t%s\n' "$MOCK_PG" + fi + ;; + *Image*) printf 'sha256:%064d\n' 0 ;; + *) exit 1 ;; + esac + exit 0 +fi +if [[ "${1:-}" == "exec" ]]; then + if [[ " $* " == *" pg_dump "* ]]; then printf 'synthetic-dump\n'; exit 0; fi + if [[ " $* " == *" pg_restore --list "* ]]; then cat >/dev/null; exit 0; fi +fi +exit 1 +MOCK + +cat >"$MOCK_BIN/rsync" <<'MOCK' +#!/usr/bin/env bash +exit "${MOCK_RSYNC_EXIT:-1}" +MOCK +cat >"$MOCK_BIN/ssh" <<'MOCK' +#!/usr/bin/env bash +exit "${MOCK_SSH_EXIT:-0}" +MOCK +chmod +x "$MOCK_BIN/docker" "$MOCK_BIN/rsync" "$MOCK_BIN/ssh" + +wrapper_env=( + env PATH="$MOCK_BIN:$PATH" REMOTE_DIR="$WRAPPER_ROOT" + REMOTE_ENV_FILE="$WRAPPER_ROOT/deploy/trialrun/.env" + COMPOSE_PROJECT_NAME="hyb-oa-current" POSTGRES_DATA_DIR="$WRAPPER_PG" + STORAGE_HOST_DIR="$WRAPPER_STORAGE" BUILD_SHA="$BUILD_SHA" VITE_BUILD_SHA="$VITE_BUILD_SHA" + AUTOMATIC_BACKUP_DIR="$WRAPPER_BACKUPS" AUTOMATIC_BACKUP_STATE_DIR="$WRAPPER_STATE" + OFFHOST_TARGET="hybbackup@backup-alias" OFFHOST_SSH_KEY="$TEST_DIR/offhost-key" + OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_RECIPIENT_EMPLOYEE_NO="OPS001" +) + +exec 8>"$WRAPPER_STATE/automatic-backup.lock" +flock -n 8 +expect_failure "wrapper rejects a concurrent automatic-backup run" \ + "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" run +flock -u 8 + +rm -f "$WRAPPER_STATE/last-success.json" +for day in 01 02 03 04 05 06 07; do + mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z" +done +expect_failure "unreachable rsync fails without writing off-host success" \ + "${wrapper_env[@]}" MOCK_RSYNC_EXIT=1 "$ROOT_DIR/scripts/run-automatic-backup.sh" run +[[ ! -e "$WRAPPER_STATE/last-success.json" ]] +[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "8" ]] +pass "source retention does not run before off-host finalize succeeds" +python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON' +import json +import os +import sys +with open(sys.argv[1], encoding="utf-8") as handle: + payload = json.load(handle) +serialized = json.dumps(payload) +assert payload["issueId"] == "AUTO_BACKUP" +assert payload["status"] == "FAIL_AUTOMATIC_BACKUP" +assert not any(value in serialized for value in ("/tmp/", "postgresql://", "backup-alias", "fixture")) +PYTHON +pass "wrapper failure summary is category-only and redacted" + +find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} + +for day in 01 02 03 04 05 06 07; do + mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z" +done +"${wrapper_env[@]}" MOCK_RSYNC_EXIT=0 MOCK_SSH_EXIT=0 \ + "$ROOT_DIR/scripts/run-automatic-backup.sh" run >/dev/null +[[ -f "$WRAPPER_STATE/last-success.json" ]] +[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "7" ]] +pass "off-host finalize success writes state and then prunes source retention to seven" + +rm -f "$WRAPPER_STATE/last-success.json" +expect_failure "stale watchdog rejects a missing success record" \ + "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale + +python3 - "$WRAPPER_STATE/last-success.json" "$BUILD_SHA" <<'PYTHON' +import json +import sys +from datetime import datetime, timezone +with open(sys.argv[1], "w", encoding="utf-8") as handle: + json.dump({ + "schemaVersion": 1, + "status": "complete", + "completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "runId": "auto-synthetic", + "manifestSha256": "0" * 64, + "composeProject": "hyb-oa-current", + "buildSha": sys.argv[2], + }, handle) +PYTHON +cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json" +python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON' +import json +import sys +with open(sys.argv[1], encoding="utf-8") as handle: + payload = json.load(handle) +payload["buildSha"] = "0" * 40 +with open(sys.argv[1], "w", encoding="utf-8") as handle: + json.dump(payload, handle) +PYTHON +expect_failure "stale watchdog rejects a success record from another build" \ + "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale +mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json" +"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null +pass "stale watchdog accepts a fresh project/build-bound success record" + +DRIFT_BUILD_SHA="0123456789abcdef0123456789abcdef01234567" +expect_failure "wrapper rejects an active runtime build that drifted from its installed contract" \ + "${wrapper_env[@]}" BUILD_SHA="$DRIFT_BUILD_SHA" VITE_BUILD_SHA="${DRIFT_BUILD_SHA:0:12}" \ + "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale + +"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null +FIRST_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")" +"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null +SECOND_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")" +[[ "$FIRST_REMINDER" == "$SECOND_REMINDER" && "$FIRST_REMINDER" == quarterly_shadow_restore_due_* ]] +pass "quarterly reminder finding key is stable within the same quarter" + +expect_failure "receiver installer uninstall rejects an unsafe target path" env \ + BACKUP_USER=hybbackup RECEIVER_TARGET=/ CONFIG_FILE=/etc/hyb-oa-test.conf \ + "$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" uninstall + +expect_failure "receiver installer rejects a system-managed backup root before mutation" env \ + BACKUP_ROOT=/etc/hyb-oa-backups PUBLIC_KEY_FILE="$TEST_DIR/known-hosts" \ + RECEIVER_SOURCE="$ROOT_DIR/scripts/receive-offhost-backup.sh" \ + "$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" install + +printf '[PASS] automatic off-host backup contract completed (%d checks)\n' "$PASS_COUNT" diff --git a/scripts/receive-offhost-backup.sh b/scripts/receive-offhost-backup.sh new file mode 100755 index 00000000..6d4163bb --- /dev/null +++ b/scripts/receive-offhost-backup.sh @@ -0,0 +1,242 @@ +#!/usr/bin/env bash +set -euo pipefail + +CONFIG_FILE="${RECEIVER_CONFIG_FILE:-/etc/hyb-oa-offhost-backup.conf}" +if [[ -f "$CONFIG_FILE" ]]; then + # Root-owned installer output with path-only assignments. + # shellcheck disable=SC1090 + source "$CONFIG_FILE" +fi + +BACKUP_INBOX_DIR="${BACKUP_INBOX_DIR:-}" +BACKUP_ARCHIVE_DIR="${BACKUP_ARCHIVE_DIR:-}" +BACKUP_WEEKLY_DIR="${BACKUP_WEEKLY_DIR:-}" +RRSYNC_BIN="${RRSYNC_BIN:-/usr/bin/rrsync}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +DAILY_KEEP="${DAILY_KEEP:-7}" +WEEKLY_KEEP="${WEEKLY_KEEP:-4}" + +fail() { + echo "[offhost-receiver][ERROR] $1" >&2 + exit 1 +} + +validate_absolute_path() { + local name="$1" + local value="$2" + if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then + fail "$name must be an explicit safe absolute path" + fi + if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \ + || "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then + fail "$name contains an unsafe path segment" + fi +} + +validate_layout() { + local name value + for name in BACKUP_INBOX_DIR BACKUP_ARCHIVE_DIR BACKUP_WEEKLY_DIR; do + value="${!name:-}" + validate_absolute_path "$name" "$value" + [[ -d "$value" && ! -L "$value" ]] || fail "$name is not an existing regular directory" + done + if [[ "$BACKUP_INBOX_DIR" == "$BACKUP_ARCHIVE_DIR" \ + || "$BACKUP_INBOX_DIR" == "$BACKUP_WEEKLY_DIR" \ + || "$BACKUP_ARCHIVE_DIR" == "$BACKUP_WEEKLY_DIR" ]]; then + fail "backup receiver directories must be distinct" + fi + local first second + for first in "$BACKUP_INBOX_DIR" "$BACKUP_ARCHIVE_DIR" "$BACKUP_WEEKLY_DIR"; do + for second in "$BACKUP_INBOX_DIR" "$BACKUP_ARCHIVE_DIR" "$BACKUP_WEEKLY_DIR"; do + if [[ "$first" != "$second" && ( "$first" == "$second"/* || "$second" == "$first"/* ) ]]; then + fail "backup receiver directories must not be nested" + fi + done + done + if [[ "$(stat -c %d "$BACKUP_INBOX_DIR")" != "$(stat -c %d "$BACKUP_ARCHIVE_DIR")" \ + || "$(stat -c %d "$BACKUP_ARCHIVE_DIR")" != "$(stat -c %d "$BACKUP_WEEKLY_DIR")" ]]; then + fail "receiver directories must share a filesystem for atomic finalize and weekly hardlinks" + fi + [[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "DAILY_KEEP must be positive" + [[ "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "WEEKLY_KEEP must be positive" + command -v "$PYTHON_BIN" >/dev/null 2>&1 || fail "python3 is required" +} + +validate_run_id() { + local run_id="$1" + [[ "$run_id" =~ ^auto-[0-9]{8}T[0-9]{6}Z$ ]] \ + || fail "run id must use auto-YYYYMMDDTHHMMSSZ" + local run_day="${run_id#auto-}" + run_day="${run_day%%T*}" + [[ "$run_day" == "$(date -u +%Y%m%d)" || "$run_day" == "$(date -u -d yesterday +%Y%m%d)" ]] \ + || fail "run id UTC day is outside the accepted receiver window" +} + +validate_batch() { + local batch_dir="$1" + local run_id="$2" + local expected_manifest_sha="$3" + + [[ -d "$batch_dir" && ! -L "$batch_dir" ]] || fail "partial batch is missing" + [[ "$expected_manifest_sha" =~ ^[0-9a-f]{64}$ ]] || fail "manifest SHA-256 is invalid" + + local entry_count regular_count + entry_count="$(find "$batch_dir" -mindepth 1 -maxdepth 1 -printf '.' | wc -c | tr -d ' ')" + regular_count="$(find "$batch_dir" -mindepth 1 -maxdepth 1 -type f -printf '.' | wc -c | tr -d ' ')" + [[ "$entry_count" == "3" && "$regular_count" == "3" ]] \ + || fail "batch must contain exactly three regular files" + for file_name in database.dump storage.tar.gz manifest.json; do + [[ -f "$batch_dir/$file_name" && ! -L "$batch_dir/$file_name" ]] \ + || fail "batch required file is missing" + done + + local actual_manifest_sha + actual_manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')" + [[ "$actual_manifest_sha" == "$expected_manifest_sha" ]] \ + || fail "manifest checksum mismatch" + + "$PYTHON_BIN" - "$batch_dir" "$run_id" <<'PYTHON' +import hashlib +import json +import os +import sys + +batch_dir, expected_run_id = sys.argv[1:] +manifest_path = os.path.join(batch_dir, "manifest.json") +with open(manifest_path, "r", encoding="utf-8") as handle: + manifest = json.load(handle) + +if manifest.get("schemaVersion") != 1: + raise SystemExit("unsupported manifest schema") +if manifest.get("status") != "complete" or manifest.get("runId") != expected_run_id: + raise SystemExit("manifest identity mismatch") +if not manifest.get("buildSha") and not manifest.get("sourceImageDigest"): + raise SystemExit("manifest source identity missing") + +artifacts = manifest.get("artifacts") or {} +expected = { + "database": "database.dump", + "storage": "storage.tar.gz", +} +for key, expected_name in expected.items(): + artifact = artifacts.get(key) or {} + if artifact.get("file") != expected_name: + raise SystemExit("manifest artifact name mismatch") + file_path = os.path.join(batch_dir, expected_name) + if os.path.getsize(file_path) != artifact.get("bytes"): + raise SystemExit("manifest artifact size mismatch") + digest = hashlib.sha256() + with open(file_path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() != artifact.get("sha256"): + raise SystemExit("manifest artifact checksum mismatch") +PYTHON +} + +prune_named_directories() { + local root="$1" + local prefix="$2" + local keep="$3" + local -a entries=() + mapfile -t entries < <(find "$root" -mindepth 1 -maxdepth 1 -type d \ + -name "${prefix}*" -printf '%f\n' | LC_ALL=C sort) + local remove_count=$((${#entries[@]} - keep)) + if (( remove_count <= 0 )); then + return 0 + fi + local index entry target + for ((index = 0; index < remove_count; index += 1)); do + entry="${entries[$index]}" + target="$root/$entry" + [[ "$entry" == "$prefix"* && -d "$target" && ! -L "$target" ]] \ + || fail "retention target failed safety validation" + rm -rf -- "$target" + done +} + +finalize_batch() { + local run_id="$1" + local expected_manifest_sha="$2" + validate_run_id "$run_id" + validate_layout + + local batch_name="hyb-oa-backup-$run_id" + local partial_dir="$BACKUP_INBOX_DIR/${batch_name}.partial" + local archive_dir="$BACKUP_ARCHIVE_DIR/$batch_name" + local already_archived=0 + if [[ -e "$archive_dir" ]]; then + [[ -d "$archive_dir" && ! -L "$archive_dir" ]] || fail "archive target is invalid" + validate_batch "$archive_dir" "$run_id" "$expected_manifest_sha" + already_archived=1 + fi + + local run_day="${run_id#auto-}" + run_day="${run_day%%T*}" + local existing_for_day + existing_for_day="$(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 -type d \ + -name "hyb-oa-backup-auto-${run_day}T*Z" ! -name "$batch_name" -print -quit)" + [[ -z "$existing_for_day" ]] || fail "another automatic archive already exists for this UTC day" + + if [[ "$already_archived" == "0" ]]; then + validate_batch "$partial_dir" "$run_id" "$expected_manifest_sha" + chmod 0400 "$partial_dir/database.dump" "$partial_dir/storage.tar.gz" "$partial_dir/manifest.json" + mv -- "$partial_dir" "$archive_dir" + fi + + local iso_week + iso_week="$($PYTHON_BIN - "$run_day" <<'PYTHON' +from datetime import datetime +import sys +day = datetime.strptime(sys.argv[1], "%Y%m%d").date() +iso = day.isocalendar() +print(f"week-{iso.year:04d}-W{iso.week:02d}") +PYTHON +)" + local weekly_dir="$BACKUP_WEEKLY_DIR/$iso_week" + if [[ ! -e "$weekly_dir" ]]; then + cp -al -- "$archive_dir" "$weekly_dir" + fi + + prune_named_directories "$BACKUP_ARCHIVE_DIR" "hyb-oa-backup-auto-" "$DAILY_KEEP" + prune_named_directories "$BACKUP_WEEKLY_DIR" "week-" "$WEEKLY_KEEP" + echo "[offhost-receiver][PASS] finalized batch with verified manifest" +} + +run_forced_command() { + validate_layout + local original_command="${SSH_ORIGINAL_COMMAND:-}" + if [[ "$original_command" == rsync\ --server\ * ]]; then + if [[ "$original_command" =~ (^|[[:space:]])--sender([[:space:]]|$) ]]; then + fail "read access is forbidden" + fi + if [[ "$original_command" =~ (^|[[:space:]])--(delete|delete-before|delete-after|delete-during|delete-delay|delete-excluded|remove-source-files|remove-sent-files|inplace)(=|[[:space:]]|$) ]]; then + fail "destructive rsync option is forbidden" + fi + [[ -x "$RRSYNC_BIN" ]] || fail "rrsync is required" + exec "$RRSYNC_BIN" -wo "$BACKUP_INBOX_DIR" + fi + + if [[ "$original_command" =~ ^hyb-backup-finalize[[:space:]](auto-[0-9]{8}T[0-9]{6}Z)[[:space:]]([0-9a-f]{64})$ ]]; then + finalize_batch "${BASH_REMATCH[1]}" "${BASH_REMATCH[2]}" + return 0 + fi + fail "unsupported forced command" +} + +case "${1:-forced}" in + forced) + run_forced_command + ;; + finalize-test) + [[ "${RECEIVER_TEST_MODE:-0}" == "1" ]] || fail "test mode is disabled" + finalize_batch "${2:-}" "${3:-}" + ;; + validate-config) + validate_layout + echo "[offhost-receiver][PASS] configuration valid" + ;; + *) + fail "unsupported mode" + ;; +esac diff --git a/scripts/run-automatic-backup.sh b/scripts/run-automatic-backup.sh new file mode 100755 index 00000000..ffb7d326 --- /dev/null +++ b/scripts/run-automatic-backup.sh @@ -0,0 +1,425 @@ +#!/usr/bin/env bash +set -eEuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +REMOTE_DIR="${REMOTE_DIR:-$ROOT_DIR}" +REMOTE_ENV_FILE="${REMOTE_ENV_FILE:-${REMOTE_DIR}/deploy/trialrun/.env}" +COMPOSE_FILE="${COMPOSE_FILE:-deploy/trialrun/docker-compose.yml}" +COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-}" +POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:-}" +STORAGE_HOST_DIR="${STORAGE_HOST_DIR:-}" +BUILD_SHA="${BUILD_SHA:-}" +VITE_BUILD_SHA="${VITE_BUILD_SHA:-}" +AUTOMATIC_BACKUP_DIR="${AUTOMATIC_BACKUP_DIR:-}" +AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}" +OFFHOST_TARGET="${OFFHOST_TARGET:-}" +OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}" +OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}" +AUTOMATIC_BACKUP_KEEP="${AUTOMATIC_BACKUP_KEEP:-7}" +AUTOMATIC_BACKUP_MAX_AGE_SECONDS="${AUTOMATIC_BACKUP_MAX_AGE_SECONDS:-86400}" +ALERT_RECIPIENT_EMPLOYEE_NO="${ALERT_RECIPIENT_EMPLOYEE_NO:-HYD000006}" +MODE="${1:-run}" + +CURRENT_PHASE="preflight" +SERVER_CONTAINER="" +DB_CONTAINER="" +SUMMARY_FILE="" +COMPOSE_CMD=() + +fail() { + echo "[automatic-backup][ERROR] $1" >&2 + return 1 +} + +validate_absolute_path() { + local name="$1" + local value="$2" + if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then + fail "$name must be an explicit safe absolute path" + fi + if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \ + || "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then + fail "$name contains an unsafe path segment" + fi +} + +validate_managed_directory_path() { + local name="$1" + local value="$2" + validate_absolute_path "$name" "$value" + [[ "$value" =~ ^/[^/]+/[^/]+ ]] \ + || fail "$name must be a dedicated path below a top-level directory" + case "$value" in + /bin/*|/boot/*|/dev/*|/etc/*|/lib/*|/lib64/*|/proc/*|/run/*|/sbin/*|/sys/*|/usr/*) + fail "$name must not be inside a system-managed directory" + ;; + esac +} + +validate_common() { + local name value + for name in REMOTE_DIR REMOTE_ENV_FILE POSTGRES_DATA_DIR STORAGE_HOST_DIR \ + AUTOMATIC_BACKUP_DIR AUTOMATIC_BACKUP_STATE_DIR; do + value="${!name:-}" + validate_absolute_path "$name" "$value" + done + validate_managed_directory_path AUTOMATIC_BACKUP_DIR "$AUTOMATIC_BACKUP_DIR" + validate_managed_directory_path AUTOMATIC_BACKUP_STATE_DIR "$AUTOMATIC_BACKUP_STATE_DIR" + [[ -d "$REMOTE_DIR" && -f "$REMOTE_ENV_FILE" ]] || fail "deployment root or env file is missing" + [[ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]] || fail "active PGDATA marker is missing" + [[ -d "$STORAGE_HOST_DIR" ]] || fail "active storage directory is missing" + [[ "$COMPOSE_PROJECT_NAME" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "Compose project name is invalid" + [[ "$COMPOSE_FILE" =~ ^[A-Za-z0-9._/-]+$ && "$COMPOSE_FILE" != /* \ + && "/$COMPOSE_FILE/" != *"/../"* && "/$COMPOSE_FILE/" != *"/./"* ]] \ + || fail "Compose file must be a safe relative path" + [[ "$BUILD_SHA" =~ ^[0-9a-fA-F]{40,64}$ ]] || fail "full build SHA is invalid" + BUILD_SHA="${BUILD_SHA,,}" + VITE_BUILD_SHA="${VITE_BUILD_SHA,,}" + [[ "$VITE_BUILD_SHA" =~ ^[0-9a-f]{12}$ && "${BUILD_SHA:0:12}" == "$VITE_BUILD_SHA" ]] \ + || fail "frontend build SHA must match the full build prefix" + [[ "$AUTOMATIC_BACKUP_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "backup retention must be positive" + [[ "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "stale threshold must be positive" + [[ "$ALERT_RECIPIENT_EMPLOYEE_NO" =~ ^[A-Za-z0-9_-]{1,80}$ ]] || fail "alert recipient id is invalid" + if [[ "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR" \ + || "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR"/* \ + || "$AUTOMATIC_BACKUP_STATE_DIR" == "$AUTOMATIC_BACKUP_DIR"/* ]]; then + fail "backup and state directories must be independent" + fi + mkdir -p "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR" + [[ -d "$AUTOMATIC_BACKUP_DIR" && ! -L "$AUTOMATIC_BACKUP_DIR" \ + && -d "$AUTOMATIC_BACKUP_STATE_DIR" && ! -L "$AUTOMATIC_BACKUP_STATE_DIR" ]] \ + || fail "backup and state directories must not be symbolic links" + chmod 0700 "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR" + command -v python3 >/dev/null 2>&1 || fail "python3 is required" +} + +validate_transfer_config() { + validate_absolute_path OFFHOST_SSH_KEY "$OFFHOST_SSH_KEY" + validate_absolute_path OFFHOST_KNOWN_HOSTS "$OFFHOST_KNOWN_HOSTS" + [[ -f "$OFFHOST_SSH_KEY" && ! -L "$OFFHOST_SSH_KEY" ]] || fail "off-host private key is missing" + [[ "$(stat -c %a "$OFFHOST_SSH_KEY")" == "600" ]] || fail "off-host private key mode must be 0600" + [[ -f "$OFFHOST_KNOWN_HOSTS" && -s "$OFFHOST_KNOWN_HOSTS" ]] || fail "pinned known-hosts file is missing" + [[ "$OFFHOST_TARGET" =~ ^[a-z_][a-z0-9_-]*@[A-Za-z0-9._-]+$ ]] \ + || fail "off-host target must use a safe user@alias value" + command -v ssh >/dev/null 2>&1 || fail "ssh is required" + command -v rsync >/dev/null 2>&1 || fail "rsync is required" +} + +select_compose() { + if docker compose version >/dev/null 2>&1; then + COMPOSE_CMD=(docker compose) + elif command -v docker-compose >/dev/null 2>&1; then + COMPOSE_CMD=(docker-compose) + else + fail "Compose runtime is unavailable" + fi +} + +compose() { + POSTGRES_DATA_DIR="$POSTGRES_DATA_DIR" \ + STORAGE_HOST_DIR="$STORAGE_HOST_DIR" \ + BUILD_SHA="$BUILD_SHA" \ + VITE_BUILD_SHA="$VITE_BUILD_SHA" \ + "${COMPOSE_CMD[@]}" -p "$COMPOSE_PROJECT_NAME" \ + -f "$REMOTE_DIR/$COMPOSE_FILE" --env-file "$REMOTE_ENV_FILE" "$@" +} + +container_env() { + local container_id="$1" + local key="$2" + docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$container_id" \ + | sed -n "s/^${key}=//p" | head -n 1 +} + +mount_source() { + local container_id="$1" + local destination="$2" + docker inspect -f '{{range .Mounts}}{{println .Destination "\t" .Source}}{{end}}' "$container_id" \ + | awk -F '\t' -v expected="$destination" '$1 == expected { print $2 }' +} + +resolve_active_runtime() { + select_compose + local -a servers=() dbs=() + mapfile -t servers < <(compose ps -q server) + mapfile -t dbs < <(compose ps -q db) + [[ "${#servers[@]}" -eq 1 && -n "${servers[0]}" ]] || fail "expected exactly one active server container" + [[ "${#dbs[@]}" -eq 1 && -n "${dbs[0]}" ]] || fail "expected exactly one active database container" + SERVER_CONTAINER="${servers[0]}" + DB_CONTAINER="${dbs[0]}" + [[ "$(docker inspect -f '{{.State.Status}}' "$SERVER_CONTAINER")" == "running" ]] \ + || fail "server container is not running" + [[ "$(docker inspect -f '{{.State.Status}}' "$DB_CONTAINER")" == "running" ]] \ + || fail "database container is not running" + + local actual_build storage_destination actual_storage actual_pgdata + actual_build="$(container_env "$SERVER_CONTAINER" BUILD_SHA)" + storage_destination="$(container_env "$SERVER_CONTAINER" STORAGE_DIR)" + [[ -n "$storage_destination" && "$storage_destination" == /* ]] || fail "server storage destination is invalid" + actual_storage="$(mount_source "$SERVER_CONTAINER" "$storage_destination")" + actual_pgdata="$(mount_source "$DB_CONTAINER" /var/lib/postgresql/data)" + [[ "$actual_build" == "$BUILD_SHA" ]] || fail "active server build does not match the installed backup contract" + [[ "$actual_storage" == "$STORAGE_HOST_DIR" ]] || fail "active storage mount does not match the installed backup contract" + [[ "$actual_pgdata" == "$POSTGRES_DATA_DIR" ]] || fail "active PGDATA mount does not match the installed backup contract" +} + +write_summary() { + local status="$1" + local finding_count="$2" + local detail="$3" + local output_file="$AUTOMATIC_BACKUP_STATE_DIR/last-run.json" + local temp_file="$AUTOMATIC_BACKUP_STATE_DIR/.last-run.json.tmp.$$" + python3 - "$temp_file" "$status" "$finding_count" "$detail" "$VITE_BUILD_SHA" <<'PYTHON' +import json +import os +import sys +from datetime import datetime, timezone + +file_path, status, finding_count, detail, build_sha = sys.argv[1:] +finding_count = int(finding_count) +findings = [] +if finding_count: + findings.append({ + "source": "automatic_backup", + "status": status, + "severity": "P0", + "detail": detail, + }) +payload = { + "issueId": "AUTO_BACKUP", + "status": status, + "conclusion": status, + "findingCount": finding_count, + "findings": findings, + "generatedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "reportDir": "host-backup-state", + "buildSha": build_sha, + "privacy": "status/count/category only; no credentials, host addresses, absolute paths, file names, or payloads", +} +with open(file_path, "x", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=True, indent=2) + handle.write("\n") +os.chmod(file_path, 0o600) +PYTHON + mv -f -- "$temp_file" "$output_file" + SUMMARY_FILE="$output_file" +} + +send_summary_alert() { + local state_name="$1" + [[ -n "$SUMMARY_FILE" && -f "$SUMMARY_FILE" ]] || return 1 + [[ -n "$SERVER_CONTAINER" ]] || resolve_active_runtime + compose exec -T server sh -lc \ + "cd /app/server && WARCH3_OPS_ALERT_SEND=1 npm run send:wecom-ops-alert -- --summary-file=- --recipient-employee-no=${ALERT_RECIPIENT_EMPLOYEE_NO} --send --state-file=/app/artifacts/qa-flow/${state_name} --format=json" \ + <"$SUMMARY_FILE" >/dev/null +} + +on_error() { + local exit_code="$?" + trap - ERR + set +e + write_summary "FAIL_AUTOMATIC_BACKUP" 1 "phase_${CURRENT_PHASE}_failed" + send_summary_alert "automatic-backup-alert-state.json" || true + echo "[automatic-backup][FAIL] phase=${CURRENT_PHASE}" >&2 + exit "$exit_code" +} + +prune_local_backups() { + local current_name="$1" + local -a entries=() + mapfile -t entries < <(find "$AUTOMATIC_BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d \ + -name 'hyb-oa-backup-auto-*' -printf '%f\n' | LC_ALL=C sort) + local remove_count=$((${#entries[@]} - AUTOMATIC_BACKUP_KEEP)) + if (( remove_count <= 0 )); then + return 0 + fi + local index entry target + for ((index = 0; index < remove_count; index += 1)); do + entry="${entries[$index]}" + [[ "$entry" != "$current_name" && "$entry" =~ ^hyb-oa-backup-auto-[0-9]{8}T[0-9]{6}Z$ ]] \ + || fail "local retention target failed safety validation" + target="$AUTOMATIC_BACKUP_DIR/$entry" + [[ -d "$target" && ! -L "$target" ]] || fail "local retention target is invalid" + rm -rf -- "$target" + done +} + +write_success_state() { + local run_id="$1" + local manifest_sha="$2" + local temp_file="$AUTOMATIC_BACKUP_STATE_DIR/.last-success.json.tmp.$$" + python3 - "$temp_file" "$run_id" "$manifest_sha" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON' +import json +import os +import sys +from datetime import datetime, timezone + +file_path, run_id, manifest_sha, project, build_sha = sys.argv[1:] +payload = { + "schemaVersion": 1, + "status": "complete", + "completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "runId": run_id, + "manifestSha256": manifest_sha, + "composeProject": project, + "buildSha": build_sha, +} +with open(file_path, "x", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=True, indent=2) + handle.write("\n") +os.chmod(file_path, 0o600) +PYTHON + mv -f -- "$temp_file" "$AUTOMATIC_BACKUP_STATE_DIR/last-success.json" +} + +run_backup() { + validate_common + validate_transfer_config + exec 9>"$AUTOMATIC_BACKUP_STATE_DIR/automatic-backup.lock" + flock -n 9 || fail "another automatic backup is already running" + trap on_error ERR + + CURRENT_PHASE="runtime_identity" + resolve_active_runtime + + local database_url postgres_db postgres_user postgres_password parsed_identity url_user url_db + database_url="$(container_env "$SERVER_CONTAINER" DATABASE_URL)" + postgres_db="$(container_env "$DB_CONTAINER" POSTGRES_DB)" + postgres_user="$(container_env "$DB_CONTAINER" POSTGRES_USER)" + postgres_password="$(container_env "$DB_CONTAINER" POSTGRES_PASSWORD)" + [[ -n "$database_url" && -n "$postgres_db" && -n "$postgres_user" && -n "$postgres_password" ]] \ + || fail "active database identity is incomplete" + parsed_identity="$(printf '%s' "$database_url" | python3 -c ' +import sys +from urllib.parse import unquote, urlparse +value = urlparse(sys.stdin.read()) +if value.scheme not in {"postgres", "postgresql"} or not value.username or not value.path.strip("/"): + raise SystemExit(1) +database_name = unquote(value.path.strip("/")) +print(f"{unquote(value.username)}\t{database_name}") +')" + IFS=$'\t' read -r url_user url_db <<<"$parsed_identity" + [[ "$url_user" == "$postgres_user" && "$url_db" == "$postgres_db" ]] \ + || fail "active server and database container identities do not match" + + local run_id batch_name batch_dir source_image_digest manifest_sha + run_id="auto-$(date -u +%Y%m%dT%H%M%SZ)" + batch_name="hyb-oa-backup-$run_id" + batch_dir="$AUTOMATIC_BACKUP_DIR/$batch_name" + source_image_digest="$(docker inspect -f '{{.Image}}' "$SERVER_CONTAINER")" + [[ "$source_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "active server image digest is invalid" + + CURRENT_PHASE="paired_backup" + if ! DATABASE_URL="$database_url" DB_TOOL_MODE=docker DB_CONTAINER="$DB_CONTAINER" \ + POSTGRES_DB="$postgres_db" POSTGRES_USER="$postgres_user" POSTGRES_PASSWORD="$postgres_password" \ + STORAGE_DIR_HOST="$STORAGE_HOST_DIR" BACKUP_DIR="$AUTOMATIC_BACKUP_DIR" \ + BACKUP_RUN_ID="$run_id" BUILD_SHA="$BUILD_SHA" SOURCE_IMAGE_DIGEST="$source_image_digest" \ + "$ROOT_DIR/scripts/backup-all.sh" >/dev/null 2>&1; then + fail "paired backup command failed" + fi + [[ -d "$batch_dir" && -f "$batch_dir/manifest.json" ]] || fail "complete local batch was not published" + manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')" + + local -a ssh_options=( + -i "$OFFHOST_SSH_KEY" + -o BatchMode=yes + -o IdentitiesOnly=yes + -o StrictHostKeyChecking=yes + -o "UserKnownHostsFile=$OFFHOST_KNOWN_HOSTS" + -o ConnectTimeout=15 + ) + local rsync_shell + printf -v rsync_shell 'ssh -i %q -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=%q -o ConnectTimeout=15' \ + "$OFFHOST_SSH_KEY" "$OFFHOST_KNOWN_HOSTS" + + CURRENT_PHASE="offhost_transfer" + if ! rsync -a --partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= -e "$rsync_shell" \ + "$batch_dir/" "$OFFHOST_TARGET:${batch_name}.partial/" >/dev/null 2>&1; then + fail "off-host transfer failed" + fi + + CURRENT_PHASE="offhost_finalize" + if ! ssh "${ssh_options[@]}" "$OFFHOST_TARGET" \ + "hyb-backup-finalize $run_id $manifest_sha" >/dev/null 2>&1; then + fail "off-host finalize failed" + fi + + CURRENT_PHASE="success_state" + write_success_state "$run_id" "$manifest_sha" + prune_local_backups "$batch_name" + write_summary "PASS_AUTOMATIC_BACKUP_COMPLETE" 0 "offhost_verified" + send_summary_alert "automatic-backup-alert-state.json" || true + trap - ERR + echo "[automatic-backup][PASS] paired batch finalized off-host" +} + +check_stale() { + validate_common + trap on_error ERR + CURRENT_PHASE="stale_check" + resolve_active_runtime + local success_file="$AUTOMATIC_BACKUP_STATE_DIR/last-success.json" + if ! python3 - "$success_file" "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON' +import json +import os +import sys +from datetime import datetime, timezone + +file_path, max_age, expected_project, expected_build = sys.argv[1:] +if not os.path.isfile(file_path): + raise SystemExit(1) +with open(file_path, "r", encoding="utf-8") as handle: + payload = json.load(handle) +if payload.get("schemaVersion") != 1 or payload.get("status") != "complete": + raise SystemExit(1) +if payload.get("composeProject") != expected_project or payload.get("buildSha") != expected_build: + raise SystemExit(1) +completed = datetime.fromisoformat(payload["completedAt"].replace("Z", "+00:00")) +age = (datetime.now(timezone.utc) - completed).total_seconds() +if age < 0 or age > int(max_age): + raise SystemExit(1) +PYTHON + then + fail "latest off-host success is missing, invalid, mismatched, or stale" + fi + write_summary "PASS_AUTOMATIC_BACKUP_FRESH" 0 "fresh_offhost_recovery_point" + send_summary_alert "automatic-backup-alert-state.json" || true + trap - ERR + echo "[automatic-backup][PASS] off-host recovery point is fresh" +} + +remind_drill() { + validate_common + trap on_error ERR + CURRENT_PHASE="quarterly_reminder" + resolve_active_runtime + local quarter + quarter="$(python3 - <<'PYTHON' +from datetime import datetime, timezone +now = datetime.now(timezone.utc) +print(f"{now.year}-Q{((now.month - 1) // 3) + 1}") +PYTHON +)" + write_summary "PASS_WITH_FINDINGS" 1 "quarterly_shadow_restore_due_${quarter}" + send_summary_alert "automatic-backup-drill-alert-state.json" + trap - ERR + echo "[automatic-backup][PASS] quarterly restore reminder evaluated" +} + +case "$MODE" in + run) + run_backup + ;; + check-stale) + check_stale + ;; + remind-drill) + remind_drill + ;; + --help|-h) + echo "Usage: run-automatic-backup.sh [run|check-stale|remind-drill]" + ;; + *) + fail "unsupported mode" + ;; +esac diff --git a/server/src/scripts/send-wecom-ops-alert.js b/server/src/scripts/send-wecom-ops-alert.js index 89d594ed..6a17b6e2 100644 --- a/server/src/scripts/send-wecom-ops-alert.js +++ b/server/src/scripts/send-wecom-ops-alert.js @@ -16,11 +16,11 @@ const SEND_CONFIRM_ENV = "WARCH3_OPS_ALERT_SEND"; function printHelp() { process.stdout.write(`Usage: node src/scripts/send-wecom-ops-alert.js [options] -QA-only WARCH3 企微巡检脱敏告警。 +QA/运维脱敏告警，支持 WARCH3 巡检和 AUTO_BACKUP 自动备份状态。 默认 dry-run：只生成预览和报告，不发送企微消息。 Options: - --summary-file= WARCH3/WROAD2 summary.json。 + --summary-file= WARCH3/WROAD2 summary.json；- 表示从 stdin 读取。 --recipient-employee-no= 收件员工号，可重复；默认 HYD000006。 --send 发送企微应用消息。 --force PASS 且无 finding 时也发送测试消息。 @@ -131,7 +131,12 @@ function assertAlertSendAllowed(options, env = process.env) { } } -function readSummary(summaryFile) { +function safeIssueId(value) { + const issueId = toSafeString(value, 40); + return /^[A-Z0-9_-]{1,40}$/.test(issueId) ? issueId : "WARCH3"; +} + +function readSummary(summaryFile, stdinText) { if (!summaryFile) { return { status: "PARTIAL_ALERT_NEEDS_CONFIG", @@ -141,12 +146,17 @@ function readSummary(summaryFile) { notes: "summary_file_missing", }; } - const resolved = path.resolve(summaryFile); - const raw = fs.readFileSync(resolved, "utf8"); + const fromStdin = summaryFile === "-"; + const resolved = fromStdin ? null : path.resolve(summaryFile); + const raw = fromStdin + ? (stdinText ?? fs.readFileSync(0, "utf8")) + : fs.readFileSync(resolved, "utf8"); const parsed = JSON.parse(raw); return { ...parsed, - reportDir: parsed.reportDir || path.dirname(resolved), + issueId: safeIssueId(parsed.issueId), + // stdin is an untrusted cross-host boundary. Never echo a supplied path. + reportDir: fromStdin ? "stdin-summary" : (parsed.reportDir || path.dirname(resolved)), }; } @@ -156,7 +166,20 @@ function shouldSendForSummary(summary, options) { } function nextStepForStatus(summary) { + const issueId = safeIssueId(summary?.issueId); const status = toSafeString(summary?.status || summary?.conclusion, 80); + const findingText = Array.isArray(summary?.findings) + ? summary.findings.map((finding) => toSafeString(finding?.detail, 160)).join(" ") + : ""; + if (issueId === "AUTO_BACKUP" && /quarterly_shadow_restore_due/i.test(findingText)) { + return "按恢复卡在 ARM 隔离目标执行人工影子恢复，并核对就绪状态与关键计数。"; + } + if (issueId === "AUTO_BACKUP" && ( + /FAIL|STALE|MISSING|INVALID|MISMATCH|BLOCKED|ERROR/i.test(status) + || Number(summary?.findingCount || 0) > 0 + )) { + return "检查自动备份任务、异机传输链路和最近一个已校验成功的恢复点。"; + } if (status === "SKIP/BLOCKED_BY_EXTERNAL_CONFIG") { return "检查 Secret、权限、可信 IP、可见范围或外部配置后复跑。"; } @@ -167,13 +190,13 @@ function nextStepForStatus(summary) { } function buildAlertMessage(summary) { - const issueId = toSafeString(summary?.issueId || "WARCH3", 40) || "WARCH3"; + const issueId = safeIssueId(summary?.issueId); const status = toSafeString(summary?.status || summary?.conclusion || "UNKNOWN", 80) || "UNKNOWN"; const findingCount = Number(summary?.findingCount || 0); const reportPath = toSafeString(summary?.reportDir || summary?.sourceReports?.wroad2 || "N/A", 500); const nextStep = nextStepForStatus(summary); return { - title: `OA 巡检 ${status}`, + title: issueId === "AUTO_BACKUP" ? `OA 自动备份 ${status}` : `OA 巡检 ${status}`, body: [ `阶段：${issueId}`, `状态：${status}`, @@ -252,6 +275,7 @@ function writeAlertState(stateFile, state) { function evaluateAlertDecision(summary, options = {}, priorState = null) { const currentState = buildAlertState(summary); + const issueId = safeIssueId(summary?.issueId); if (options.force) { return { shouldSend: true, @@ -267,8 +291,31 @@ function evaluateAlertDecision(summary, options = {}, priorState = null) { const previousRank = Number(priorState?.statusRank || statusRank(priorState?.status)); const previousKeys = new Set(Array.isArray(priorState?.findingKeys) ? priorState.findingKeys : []); const newFindingKeys = currentState.findingKeys.filter((key) => !previousKeys.has(key)); + const isAutomaticBackup = issueId === "AUTO_BACKUP"; + const isQuarterlyReminder = currentState.findingKeys.some((key) => /quarterly_shadow_restore_due/i.test(key)); + + if (isAutomaticBackup && isQuarterlyReminder && (!priorState || newFindingKeys.length > 0)) { + return { + shouldSend: true, + reason: "quarterly_restore_reminder", + newFindingKeys, + previousStatus: priorState?.status || null, + currentStatus: currentState.status, + currentState, + }; + } if (currentRank >= 4) { + if (isAutomaticBackup && priorState && currentRank <= previousRank && newFindingKeys.length === 0) { + return { + shouldSend: false, + reason: "repeated_automatic_backup_failure", + newFindingKeys, + previousStatus: priorState.status || null, + currentStatus: currentState.status, + currentState, + }; + } return { shouldSend: true, reason: "blocked_or_failed", @@ -412,6 +459,7 @@ async function executeOpsAlert(options, deps = {}) { const generatedAt = deps.generatedAt || new Date().toISOString(); const config = deps.config || getConfig(); const summary = deps.summary || readSummary(options.summaryFile); + const issueId = safeIssueId(summary?.issueId); const message = buildAlertMessage(summary); const priorState = Object.prototype.hasOwnProperty.call(deps, "priorState") ? deps.priorState @@ -497,7 +545,7 @@ async function executeOpsAlert(options, deps = {}) { } return { - issueId: "WARCH3", + issueId, mode: "wecom-ops-alert", generatedAt, status, @@ -513,7 +561,7 @@ async function executeOpsAlert(options, deps = {}) { }, stateWrite, summary: { - issueId: summary?.issueId || null, + issueId, status: summary?.status || summary?.conclusion || "UNKNOWN", findingCount: Number(summary?.findingCount || 0), reportDir: summary?.reportDir || null, @@ -572,6 +620,7 @@ module.exports = { evaluateAlertDecision, executeOpsAlert, parseArgs, + readSummary, readAlertState, sanitizeDeliveryResponse, shouldSendForSummary, diff --git a/server/tests/unit/scripts/send-wecom-ops-alert.test.js b/server/tests/unit/scripts/send-wecom-ops-alert.test.js index be1193ac..d58896c3 100644 --- a/server/tests/unit/scripts/send-wecom-ops-alert.test.js +++ b/server/tests/unit/scripts/send-wecom-ops-alert.test.js @@ -5,6 +5,7 @@ const { evaluateAlertDecision, executeOpsAlert, parseArgs, + readSummary, shouldSendForSummary, } = require("../../../src/scripts/send-wecom-ops-alert"); @@ -75,6 +76,72 @@ describe("send-wecom-ops-alert", () => { expect(message.body).not.toContain("手机号"); }); + it("reads stdin summary without preserving an injected absolute report path", () => { + const summary = readSummary("-", JSON.stringify({ + issueId: "AUTO_BACKUP", + status: "FAIL_AUTOMATIC_BACKUP", + findingCount: 1, + reportDir: `/private/${SENSITIVE_TOKEN}/state`, + })); + + expect(summary.issueId).toBe("AUTO_BACKUP"); + expect(summary.reportDir).toBe("stdin-summary"); + expect(JSON.stringify(summary)).not.toContain(SENSITIVE_TOKEN); + expect(JSON.stringify(summary)).not.toContain("/private/"); + }); + + it("keeps AUTO_BACKUP issue id and uses backup-specific failure guidance", async () => { + const summary = baseSummary({ + issueId: "AUTO_BACKUP", + status: "FAIL_AUTOMATIC_BACKUP", + reportDir: "host-backup-state", + }); + const message = buildAlertMessage(summary); + const report = await executeOpsAlert(baseOptions(), { + summary, + recipientRows: recipientRows(), + sendMessageImpl: vi.fn(), + env: {}, + config: {}, + }); + + expect(message.title).toContain("自动备份"); + expect(message.body).toContain("异机传输链路"); + expect(report.issueId).toBe("AUTO_BACKUP"); + expect(report.summary.issueId).toBe("AUTO_BACKUP"); + }); + + it("uses manual ARM shadow-restore guidance for quarterly reminder", () => { + const summary = baseSummary({ + issueId: "AUTO_BACKUP", + findings: [{ detail: "quarterly_shadow_restore_due_2026-Q3" }], + reportDir: "host-backup-state", + }); + const message = buildAlertMessage(summary); + const firstDecision = evaluateAlertDecision(summary, baseOptions(), null); + const repeatedDecision = evaluateAlertDecision(summary, baseOptions(), buildAlertState(summary)); + + expect(message.body).toContain("ARM 隔离目标执行人工影子恢复"); + expect(firstDecision.shouldSend).toBe(true); + expect(firstDecision.reason).toBe("quarterly_restore_reminder"); + expect(repeatedDecision.shouldSend).toBe(false); + }); + + it("sends the first automatic-backup failure and suppresses an identical repeat", () => { + const summary = baseSummary({ + issueId: "AUTO_BACKUP", + status: "FAIL_AUTOMATIC_BACKUP", + findings: [{ source: "automatic_backup", status: "FAIL_AUTOMATIC_BACKUP", detail: "phase_offhost_transfer_failed" }], + }); + const firstDecision = evaluateAlertDecision(summary, baseOptions(), null); + const repeatedDecision = evaluateAlertDecision(summary, baseOptions(), buildAlertState(summary)); + + expect(firstDecision.shouldSend).toBe(true); + expect(firstDecision.reason).toBe("blocked_or_failed"); + expect(repeatedDecision.shouldSend).toBe(false); + expect(repeatedDecision.reason).toBe("repeated_automatic_backup_failure"); + }); + it("keeps PASS without findings as no-send unless forced", () => { const summary = baseSummary({ status: "PASS_ARCHIVE_OPS_READY", findingCount: 0 }); expect(shouldSendForSummary(summary, baseOptions())).toBe(false);