From 9a624700d05c4d7832655b78cbc22eae6aafe74d Mon Sep 17 00:00:00 2001 From: shunli Date: Sun, 26 Jul 2026 21:47:44 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=8C=E6=88=90=E7=9C=9F=E5=AE=9E=E6=88=90?= =?UTF-8?q?=E5=AF=B9=E5=BC=82=E6=9C=BA=E5=A4=87=E4=BB=BD=EF=BC=8807-26-int?= =?UTF-8?q?ernal-ops-recovery-baseline=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .trellis/spec/hyb-oa/deployment-trialrun.md | 24 ++++++++++ .../implement.md | 3 +- .../prd.md | 4 +- .../research/live-read-only-baseline.md | 20 ++++++-- .../task.json | 2 +- deploy/trialrun/runbook.md | 4 +- scripts/backup-all.sh | 47 ++++++++++++++++++- scripts/qa/run-backup-restore-drill.sh | 42 ++++++++++++++++- 8 files changed, 133 insertions(+), 13 deletions(-) diff --git a/.trellis/spec/hyb-oa/deployment-trialrun.md b/.trellis/spec/hyb-oa/deployment-trialrun.md index ab07cc3a..22719281 100644 --- a/.trellis/spec/hyb-oa/deployment-trialrun.md +++ b/.trellis/spec/hyb-oa/deployment-trialrun.md @@ -530,6 +530,8 @@ bash scripts/qa/run-local-candidate.sh full - Paired backup: `DB_TOOL_MODE=docker DB_CONTAINER= ./scripts/backup-all.sh`. - Optional safe batch identifier: `BACKUP_RUN_ID=`. +- Manifest writer: host `node` is preferred; host `python3` is an equivalent + fallback for Docker-only runtime machines. - DB shadow restore requires `RESTORE_SCOPE=shadow`, `RESTORE_CONFIRM_DATABASE=` and `DATABASE_URL`. - Storage shadow restore: @@ -543,6 +545,13 @@ bash scripts/qa/run-local-candidate.sh full - A complete recovery point is one batch directory containing `database.dump`, `storage.tar.gz` and a final `manifest.json`. The manifest contains only safe relative names, byte sizes, SHA-256 hashes, batch time and optional build SHA. +- Manifest generation uses exclusive creation and preserves the same schema + under Node and Python 3; if neither runtime exists, fail before publishing the + final batch directory. +- For an approved live backup, do not assume the deployment `.env` still matches + the running database. Confirm that the selected PostgreSQL container's database + and user match at least one active application `DATABASE_URL`, then pass the + verified runtime values without logging their contents. - The DB dump must pass `pg_restore --list` and the storage archive must pass `tar -tzf` before the manifest may say `complete`. - DB restore queries `current_database()` and compares it exactly with @@ -571,13 +580,22 @@ bash scripts/qa/run-local-candidate.sh full - Corrupt dump/tar or dangerous/link archive member -> fail before restore. - Host PostgreSQL tools mismatch in `auto` mode -> docker fallback is allowed only when the explicit target container and required DB variables exist. +- Neither Node nor Python 3 is available -> fail manifest generation and remove + the temporary batch; do not publish a partial recovery point. +- Deployment `.env`, PostgreSQL container and active application connection do + not identify the same database/user -> stop the live backup; never guess + credentials or expose values in diagnostics. ### 5. Good/Base/Bad Cases - Good: one command creates a validated pair, the complete batch is copied off host, and a disposable shadow drill restores the DB marker and attachment hash. +- Good: a Docker-only host uses Python 3 to write the same manifest contract + after the active application connection is matched to the database container. - Base: backup succeeds locally but off-host copy and deployed `/ready` evidence remain explicitly pending; do not claim formal recovery readiness. +- Bad: source a stale deployment `.env` and treat a failed dump as a prompt to + guess or print credentials. - Bad: source the active trial `.env` and run a cleaning restore directly, or restore attachments into a non-empty live directory. @@ -589,6 +607,8 @@ bash scripts/qa/run-local-candidate.sh full target and dangerous tar path. - Validate manifest relative paths, absence of connection secrets and matching SHA-256 values. +- Run the synthetic drill with Node available and with a `PATH` that excludes + Node but includes Python 3; both paths must complete the same round trip. - Real-instance restore and `/ready` remain separately approved evidence and must never be fabricated from the local drill. @@ -598,6 +618,7 @@ bash scripts/qa/run-local-candidate.sh full ```bash source deploy/trialrun/.env +./scripts/backup-all.sh ./scripts/restore-db.sh some.dump ./scripts/restore-storage.sh some.tar.gz deploy/trialrun/data/storage ``` @@ -606,5 +627,8 @@ source deploy/trialrun/.env ```bash ./scripts/qa/run-backup-restore-drill.sh +# Before an approved live backup, compare the selected DB container identity +# with an active app connection using redacted boolean diagnostics. +DB_TOOL_MODE=docker DB_CONTAINER= ./scripts/backup-all.sh # Real restore remains an approved incident operation after shadow evidence. ``` diff --git a/.trellis/tasks/07-26-internal-ops-recovery-baseline/implement.md b/.trellis/tasks/07-26-internal-ops-recovery-baseline/implement.md index 527b5bbb..a6433418 100644 --- a/.trellis/tasks/07-26-internal-ops-recovery-baseline/implement.md +++ b/.trellis/tasks/07-26-internal-ops-recovery-baseline/implement.md @@ -10,7 +10,8 @@ - [x] 更新 runbook/checklist 和部署规约，区分普通备份、影子演练和真实事故恢复。 - [x] 运行任务测试计划和 Trellis 治理门禁；按实际结果记录本地 RTO，不伪造 `/ready` 或异机证据。 - [x] 在单独授权后完成真实运行来源、职责分离、异机副本和 `/ready` 的只读核查，并如实记录 `NO_GO` 缺口。 -- [ ] 经后续明确授权完成提交发布、成对异机备份、真实职责分配、部署/迁移和真实格式影子恢复后复验。 +- [x] 经明确授权完成提交发布和真实 DB＋附件＋manifest 成对异机备份，两端完整性复验通过。 +- [ ] 经后续明确授权完成真实职责分配、部署/迁移和真实格式影子恢复后复验。 ## 推荐文件边界 diff --git a/.trellis/tasks/07-26-internal-ops-recovery-baseline/prd.md b/.trellis/tasks/07-26-internal-ops-recovery-baseline/prd.md index 95691519..d2069078 100644 --- a/.trellis/tasks/07-26-internal-ops-recovery-baseline/prd.md +++ b/.trellis/tasks/07-26-internal-ops-recovery-baseline/prd.md @@ -75,8 +75,8 @@ - 获批只读核查结果见 `research/live-read-only-baseline.md`，结论为 `NO_GO_FOR_FORMAL_INTERNAL_USE`。 - 公开 HTTPS、TLS、HSTS 和 `/ready` 当前可用，但运行环境没有后端/前端构建标识，且远端迁移少于本地当前基线。 - 真实角色只有一个活跃账号，财务和付款复核候选均为 0，不满足 maker-checker。 -- 历史 DB/附件备份约 55 天未更新、无 manifest，且与运行数据位于同一设备；不能证明 RPO、异机副本或可恢复性。 -- 当前本地提交尚未进入 upstream；未执行生产备份、角色修改、部署、迁移或恢复，因此最后一项验收继续保持未勾选。 +- 只读核查时的历史 DB/附件备份约 55 天未更新、无 manifest，且与运行数据位于同一设备；获批后已生成新的 DB＋附件＋manifest 完整批次并复制到独立主机，两端校验一致。 +- 当前代码已在获批后以普通快进方式发布到 upstream，来源门禁为 `PASS`；尚未执行角色修改、部署、迁移或真实格式影子恢复，因此最后一项验收继续保持未勾选。 ## Notes diff --git a/.trellis/tasks/07-26-internal-ops-recovery-baseline/research/live-read-only-baseline.md b/.trellis/tasks/07-26-internal-ops-recovery-baseline/research/live-read-only-baseline.md index d5b93de0..08b5e373 100644 --- a/.trellis/tasks/07-26-internal-ops-recovery-baseline/research/live-read-only-baseline.md +++ b/.trellis/tasks/07-26-internal-ops-recovery-baseline/research/live-read-only-baseline.md @@ -27,7 +27,7 @@ | `BUILD_SHA` | 容器未配置 | 无法证明运行来源 | | 前端 build marker | 缺失 | 无法从静态资源证明运行来源 | | Prisma migration | 远端已应用 58，本地当前 61；最新项不匹配 | 远端落后，禁止直接宣称当前版本已部署 | -| 本地来源门禁 | `commit_not_in_upstream` | 当前本地提交尚未进入 upstream，标准部署前必须先发布提交 | +| 来源门禁复验 | `PASS`，当前提交已发布 | 已完成普通快进发布，范围 `325a795..7658051`，未使用覆盖参数 | ## 角色和付款职责 @@ -62,10 +62,22 @@ 服务当前可访问不等于运行与恢复底线通过。进入下一步前必须按顺序完成： -1. 将当前获批代码提交进入 upstream，确保部署来源门禁可通过。 -2. 在任何部署前，用新成对备份合同生成 DB + 附件 + manifest，并复制到异机/异盘。 +1. 已将当前获批代码以普通快进方式发布到 upstream，来源门禁复验为 `PASS`。 +2. 已在获批后用新成对备份合同生成 DB + 附件 + manifest，并复制到独立主机；两端 manifest 和产物校验通过。 3. 明确至少一名财务处理人和另一名 `manager`/`ceo` 付款复核人；系统管理职责单独分配，应急 `admin` 独立托管。 4. 通过获批部署任务升级代码和 3 项迁移，验证 `BUILD_SHA`、前端 marker、`/ready`、权限矩阵和回滚点。 5. 在独立影子目标恢复同一批次 DB/附件，验证应用 `/ready` 和关键附件引用，记录真实 RPO/RTO。 -以上动作分别涉及发布、远程备份、真实角色写入、部署和影子恢复，必须继续取得明确授权，不能由本次只读批准推导。 +发布和远程备份已分别获得授权并完成；真实角色写入、部署和影子恢复仍须继续取得明确授权，不能由既有批准推导。 + +## 后续获批写入：成对异机备份 + +- 执行日期：2026-07-26。 +- 授权范围：生成真实 DB、附件和 manifest 成对备份，并复制到用户指定的另一台主机。 +- 源机与目标机设备指纹不同；未记录主机名、地址、SSH 名称、账号或完整目录。 +- 完整批次：`hyb-oa-backup-20260726T133923Z-live`，包含 `database.dump`、`storage.tar.gz`、`manifest.json` 三个预期文件，总大小约 9.6 MiB。 +- 源端完整性检查通过；目标端重新计算 DB/附件 SHA-256 并与 manifest 核对通过；两端 manifest 哈希一致。 +- 目标批次目录权限为 `700`；传输失败产生的单文件残缺副本已精确删除，本机临时暂存数据已安全清理。 +- 运行主机没有宿主 Node.js，因此为成对备份入口增加 Python 3 manifest 等价回退；无 Node 路径已通过完整合成恢复演练，本地参考 RTO 仍为 6 秒。 +- 运行中的 PostgreSQL 容器参数与部署目录环境文件存在漂移；本次只在运行容器参数与活动服务数据库连接相互匹配后执行备份，未修改环境文件。 +- 本次未执行恢复、部署、迁移、角色修改、企微写入或业务写入；真实格式影子恢复和恢复后 `/ready` 仍无证据。 diff --git a/.trellis/tasks/07-26-internal-ops-recovery-baseline/task.json b/.trellis/tasks/07-26-internal-ops-recovery-baseline/task.json index 0ba4f383..47afddbb 100644 --- a/.trellis/tasks/07-26-internal-ops-recovery-baseline/task.json +++ b/.trellis/tasks/07-26-internal-ops-recovery-baseline/task.json @@ -32,6 +32,6 @@ "server/README.md", ".trellis/spec/hyb-oa/deployment-trialrun.md" ], - "notes": "本地安全基线已提交；获批真实环境只读核查结论 NO_GO：运行 commit 不可追溯、迁移落后、无财务/复核职责分离、备份约55天且同设备无manifest。提交发布、成对异机备份、角色写入、部署/迁移和影子恢复仍需另行授权。", + "notes": "本地安全基线已普通快进发布，来源门禁复验 PASS；获批真实成对异机备份已完成且两端校验一致。整体仍为 NO_GO：运行实例缺少构建标识、迁移落后、无财务/复核职责分离，角色写入、部署/迁移和真实格式影子恢复仍需另行授权。", "meta": {} } diff --git a/deploy/trialrun/runbook.md b/deploy/trialrun/runbook.md index fc2b5bcc..582fe96e 100644 --- a/deploy/trialrun/runbook.md +++ b/deploy/trialrun/runbook.md @@ -179,6 +179,8 @@ DB_CONTAINER=hyb-oa-trialrun-db \ ./scripts/backup-all.sh ``` +`backup-all.sh` 可使用宿主 Node.js 或 Python 3 生成同结构 manifest；Docker-only 主机没有 Node.js 时会自动使用 Python 3，两者都没有则失败且不发布完整批次。真实运行环境执行前，不得只相信可能漂移的 `.env`：先脱敏确认所选 PostgreSQL 容器的数据库/用户与至少一个活动应用连接一致，再把已验证的运行参数传给脚本；不在日志中输出参数值。 + 输出默认在 `deploy/trialrun/backups/hyb-oa-backup-/`，其中包含： - `database.dump` @@ -189,7 +191,7 @@ DB_CONTAINER=hyb-oa-trialrun-db \ ### 1.5.2 本地隔离恢复演练 -开发机具备 Docker、PostgreSQL 客户端和 Node.js 时，使用合成数据运行： +开发机具备 Docker、PostgreSQL 客户端，以及 Node.js 或 Python 3 时，使用合成数据运行： ```bash ./scripts/qa/run-backup-restore-drill.sh diff --git a/scripts/backup-all.sh b/scripts/backup-all.sh index d0eca120..b3f45c66 100755 --- a/scripts/backup-all.sh +++ b/scripts/backup-all.sh @@ -67,8 +67,13 @@ STORAGE_BYTES="$(wc -c <"$STORAGE_FILE" | tr -d ' ')" BUILD_SHA="$(git -C "$ROOT_DIR" rev-parse --short=12 HEAD 2>/dev/null || true)" CREATED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -node - "$TEMP_DIR/$MANIFEST_NAME" "$BACKUP_RUN_ID" "$CREATED_AT" "$BUILD_SHA" \ - "$DB_NAME" "$DB_BYTES" "$DB_SHA256" "$STORAGE_NAME" "$STORAGE_BYTES" "$STORAGE_SHA256" <<'NODE' +MANIFEST_ARGS=( + "$TEMP_DIR/$MANIFEST_NAME" "$BACKUP_RUN_ID" "$CREATED_AT" "$BUILD_SHA" + "$DB_NAME" "$DB_BYTES" "$DB_SHA256" "$STORAGE_NAME" "$STORAGE_BYTES" "$STORAGE_SHA256" +) + +if command -v node >/dev/null 2>&1; then + node - "${MANIFEST_ARGS[@]}" <<'NODE' const fs = require("node:fs"); const [manifestPath, runId, createdAt, buildSha, dbFile, dbBytes, dbSha256, storageFile, storageBytes, storageSha256] = process.argv.slice(2); const manifest = { @@ -84,6 +89,44 @@ const manifest = { }; fs.writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { encoding: "utf8", flag: "wx" }); NODE +elif command -v python3 >/dev/null 2>&1; then + python3 - "${MANIFEST_ARGS[@]}" <<'PYTHON' +import json +import sys + +( + manifest_path, + run_id, + created_at, + build_sha, + db_file, + db_bytes, + db_sha256, + storage_file, + storage_bytes, + storage_sha256, +) = sys.argv[1:] + +manifest = { + "schemaVersion": 1, + "runId": run_id, + "createdAt": created_at, + "buildSha": build_sha or None, + "status": "complete", + "artifacts": { + "database": {"file": db_file, "bytes": int(db_bytes), "sha256": db_sha256}, + "storage": {"file": storage_file, "bytes": int(storage_bytes), "sha256": storage_sha256}, + }, +} + +with open(manifest_path, "x", encoding="utf-8") as manifest_file: + json.dump(manifest, manifest_file, ensure_ascii=True, indent=2) + manifest_file.write("\n") +PYTHON +else + echo "[ERR] node or python3 is required to write the backup manifest" >&2 + exit 1 +fi mv "$TEMP_DIR" "$FINAL_DIR" TEMP_DIR="" diff --git a/scripts/qa/run-backup-restore-drill.sh b/scripts/qa/run-backup-restore-drill.sh index c5e6c437..73ef758c 100755 --- a/scripts/qa/run-backup-restore-drill.sh +++ b/scripts/qa/run-backup-restore-drill.sh @@ -24,9 +24,13 @@ require_command() { fi } -for command_name in docker pg_dump pg_restore psql tar sha256sum node; do +for command_name in docker pg_dump pg_restore psql tar sha256sum; do require_command "$command_name" done +if ! command -v node >/dev/null 2>&1 && ! command -v python3 >/dev/null 2>&1; then + echo "[ERR] required command missing: node or python3" >&2 + exit 1 +fi START_EPOCH="$(date +%s)" SOURCE_STORAGE="$DRILL_DIR/source-storage" @@ -64,7 +68,8 @@ BATCH_DIR="$BACKUP_ROOT/hyb-oa-backup-$DRILL_RUN_ID" DB_BACKUP="$BATCH_DIR/database.dump" STORAGE_BACKUP="$BATCH_DIR/storage.tar.gz" -node - "$BATCH_DIR/manifest.json" "$DB_BACKUP" "$STORAGE_BACKUP" <<'NODE' +if command -v node >/dev/null 2>&1; then + node - "$BATCH_DIR/manifest.json" "$DB_BACKUP" "$STORAGE_BACKUP" <<'NODE' const crypto = require("node:crypto"); const fs = require("node:fs"); const path = require("node:path"); @@ -80,6 +85,39 @@ const sha256 = (file) => crypto.createHash("sha256").update(fs.readFileSync(file if (sha256(dbPath) !== manifest.artifacts.database.sha256) throw new Error("database checksum mismatch"); if (sha256(storagePath) !== manifest.artifacts.storage.sha256) throw new Error("storage checksum mismatch"); NODE +else + python3 - "$BATCH_DIR/manifest.json" "$DB_BACKUP" "$STORAGE_BACKUP" <<'PYTHON' +import hashlib +import json +import os +import re +import sys + +manifest_path, db_path, storage_path = sys.argv[1:] +with open(manifest_path, encoding="utf-8") as manifest_file: + manifest_text = manifest_file.read() +manifest = json.loads(manifest_text) +if manifest.get("status") != "complete" or manifest.get("schemaVersion") != 1: + raise RuntimeError("manifest contract mismatch") +for artifact in (manifest["artifacts"]["database"], manifest["artifacts"]["storage"]): + if os.path.isabs(artifact["file"]): + raise RuntimeError("manifest contains absolute artifact path") +if re.search(r"postgresql:|DATABASE_URL|drill_password", manifest_text): + raise RuntimeError("manifest contains sensitive connection data") + +def sha256(file_path): + digest = hashlib.sha256() + with open(file_path, "rb") as artifact_file: + for block in iter(lambda: artifact_file.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + +if sha256(db_path) != manifest["artifacts"]["database"]["sha256"]: + raise RuntimeError("database checksum mismatch") +if sha256(storage_path) != manifest["artifacts"]["storage"]["sha256"]: + raise RuntimeError("storage checksum mismatch") +PYTHON +fi psql "$DATABASE_URL" -v ON_ERROR_STOP=1 -qc 'DROP TABLE restore_drill_marker'