From 5ec7db13b23614f21216dce82e86e7fc71248454 Mon Sep 17 00:00:00 2001 From: shunli Date: Tue, 25 Aug 2026 12:00:33 +0800 Subject: [PATCH] =?UTF-8?q?=E9=AA=8C=E8=AF=81(08-23-isolated-backup-restor?= =?UTF-8?q?e-drill):=20=E5=9B=BA=E5=8C=96=E9=9A=94=E7=A6=BB=E6=81=A2?= =?UTF-8?q?=E5=A4=8Drunner=E5=90=88=E5=90=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + .trellis/spec/hyb-oa/deployment-trialrun.md | 250 ++++++ .../check.jsonl | 1 + .../design.md | 124 +++ .../implement.jsonl | 1 + .../implement.md | 147 ++++ .../prd.md | 89 +++ .../research/restore-drill-evidence.md | 293 +++++++ .../task.json | 26 + .../offhost-shadow-restore-runner.template.sh | 608 +++++++++++++++ ...-offhost-shadow-restore-runner-contract.sh | 725 ++++++++++++++++++ ...date-offhost-shadow-restore-attachments.py | 67 ++ 12 files changed, 2334 insertions(+) create mode 100644 .trellis/tasks/08-23-isolated-backup-restore-drill/check.jsonl create mode 100644 .trellis/tasks/08-23-isolated-backup-restore-drill/design.md create mode 100644 .trellis/tasks/08-23-isolated-backup-restore-drill/implement.jsonl create mode 100644 .trellis/tasks/08-23-isolated-backup-restore-drill/implement.md create mode 100644 .trellis/tasks/08-23-isolated-backup-restore-drill/prd.md create mode 100644 .trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md create mode 100644 .trellis/tasks/08-23-isolated-backup-restore-drill/task.json create mode 100644 scripts/qa/offhost-shadow-restore-runner.template.sh create mode 100755 scripts/qa/run-offhost-shadow-restore-runner-contract.sh create mode 100755 scripts/qa/validate-offhost-shadow-restore-attachments.py diff --git a/.gitignore b/.gitignore index ad9ad76b..c7297c8e 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,9 @@ artifacts/ # Local sensitive business samples; never commit originals /input/ +# Private off-host restore render inputs and site-specific candidates +/.restore-drill-private/ + # Vitest coverage artifacts server/coverage server/.vitest diff --git a/.trellis/spec/hyb-oa/deployment-trialrun.md b/.trellis/spec/hyb-oa/deployment-trialrun.md index 09354be6..519897db 100644 --- a/.trellis/spec/hyb-oa/deployment-trialrun.md +++ b/.trellis/spec/hyb-oa/deployment-trialrun.md @@ -1210,6 +1210,256 @@ test "$(printf '%s\n' "$actual" | LC_ALL=C sort)" = \ "$(printf '%s\n' "$expected" | LC_ALL=C sort)" ``` +## Scenario: Off-Host Archive And Private-Copy Identity Validation + +### 1. Scope / Trigger + +- Trigger: an off-host recovery drill copies a formally archived backup batch + into a private temporary directory before database or attachment restore. +- Purpose: preserve the archive placement evidence at the source boundary while + validating a renamed private copy by its source-bound content identity. + +### 2. Signatures + +- Source validation: + `validate_source_batch() -> source_metadata`. +- Private-copy validation: + `validate_copied_batch(, )`. +- `source_metadata` contains only redacted comparison values: verified run id, + manifest/build/time identity, exact three-file set, byte sizes and SHA-256 + values. It must not contain credentials, business payloads or full paths. + +### 3. Contracts + +- A formal archive run directory must have a basename that exactly equals the + validated manifest run id. This detects a valid manifest placed under the + wrong archive run and is mandatory before any copy or restore write. +- A private copy may use an operator-owned neutral basename such as `batch`; + that basename is not archive provenance and must not be compared with the + manifest run id. +- The copied manifest run id must exactly equal the run id already verified at + the source. Internal self-consistency inside the copy is insufficient. +- The copy must repeat manifest schema/status, run-id format, exact + `manifest.json`/`database.dump`/`storage.tar.gz` regular-file set, build and + age gates, byte-size checks and SHA-256 checks. +- After repeated validation, every redacted copied metadata field must equal + `source_metadata`. Copy validation must never disable a source-side gate or + accept a newly invented identity. +- Formal archives remain read-only. A validation failure stops before creating + PostgreSQL, extracting attachments or modifying the archive, and cleanup must + prove zero private runtime and privilege-entry residue. + +### 4. Validation & Error Matrix + +- Source basename differs from manifest run id -> `batch_identity`; `NO_GO` + before copy. +- Copied manifest run id differs from the verified source run id, even when the + copy is internally self-consistent -> `copied_batch_invalid/batch_identity`; + `NO_GO` before restore runtime creation. +- Copied file set, bytes, SHA-256, build, time or other redacted metadata differs + from the source -> `copied_batch_invalid`; remove only the owned private + runtime and retain the archive unchanged. +- Private basename differs from the run id but all source-bound content checks + pass -> valid; continue to the separately approved restore step. +- Any cleanup residue remains -> attempt stays non-clean and cannot be reported + as recovery `GO`. + +### 5. Good/Base/Bad Cases + +- Good: source directory basename matches its manifest run id; the same batch + is copied to a private directory named `batch`, revalidated, and its redacted + metadata exactly equals the source record. +- Base: source and copy basenames happen to match; both still use their proper + validation modes and the copy remains bound to `source_metadata`. +- Bad: reuse source validation on the private directory and reject it only + because the operator chose a neutral basename. +- Bad: skip the copy basename check and then trust any internally consistent + replacement manifest without comparing it with the verified source run id. + +### 6. Tests Required + +- Positive: a formal source whose basename matches its manifest run id passes. +- Positive: an unchanged copy under a different private basename passes and + produces metadata equal to the source record. +- Negative: source basename/run-id mismatch fails before copy. +- Negative: copied content or checksum mutation fails before restore. +- Negative: a copied manifest with an internally consistent but source-different + run id fails, proving the source binding cannot be bypassed. +- Regression: run the existing synthetic backup/restore contract and assert no + PostgreSQL object, extracted attachment, private runtime or privilege entry is + left after every pre-runtime failure. + +### 7. Wrong vs Correct + +#### Wrong + +```text +validate_batch(source, require_basename_match=true) +copy source -> private/batch +validate_batch(private/batch, require_basename_match=true) +``` + +#### Correct + +```text +source_metadata = validate_source_batch(source_run_directory) +copy source_run_directory -> private/batch +validate_copied_batch(private/batch, source_metadata) +``` + +## Scenario: Shadow-Restore Failure Diagnostics Before Teardown + +### 1. Scope / Trigger + +- Trigger: a one-shot off-host shadow restore keeps raw tool output only in a + private runtime that must be deleted on every exit path. +- Purpose: preserve enough allowlisted stage evidence to locate a restore + failure without retaining or publishing raw stderr, paths, credentials, SQL + or restored object names. + +### 2. Signatures + +- Required stages: `dump_list`, `target_confirmation`, `restore_apply`, each + with the enum `PASS|FAIL|NOT_RUN`. +- Required execution fields: `restore_execution_path=DOCKER_STDIN|HOST_FILE|NOT_RUN`, + `dump_list_exit_code`, `restore_script_exit_code`, + `pg_restore_exit_code`, `pg_restore_error_count` and + `restore_error_category`. +- Exit-code fields are non-negative integers or `NOT_RUN`; + `pg_restore_error_count` additionally permits `UNKNOWN` when no safe numeric + summary is available. +- Tracked supply-chain entries: + `scripts/qa/offhost-shadow-restore-runner.template.sh`, + `scripts/qa/validate-offhost-shadow-restore-attachments.py` and + `bash scripts/qa/run-offhost-shadow-restore-runner-contract.sh`. +- Candidate render: + `bash scripts/qa/run-offhost-shadow-restore-runner-contract.sh render `; + it prints only PASS and the candidate SHA-256, never config values or paths. + +### 3. Contracts + +- Emit the fixed diagnostic fields on the failure path before runtime teardown. + Teardown remains in a trap/finally-equivalent path and must still run after + diagnostic emission; callers retain stdout, not the private raw log. +- An ARM restore runner fixes `restore_execution_path=DOCKER_STDIN`. A private + Docker proxy may observe the real `pg_restore --list` and apply calls and + persist only their stage name and numeric exit code; it must not record argv, + environment values or stderr. +- Run `dump_list` and target confirmation independently before the existing + restore script. A failed stage leaves every later stage `NOT_RUN`. +- Raw restore output remains mode `0600`, never leaves the off-host node and is + deleted with the owned runtime on success and failure. +- Any non-zero pg_restore apply exit is `FAIL/NO_GO`, even when a numeric + "errors ignored" count can be extracted. There is no partial-success + allowlist. Relaxing that rule requires a separately approved design proving + the exact restored object set. +- A diagnostic failure is an exercise-runner finding until evidence identifies + a dump or restore-content defect. It must not be summarized as proof that the + latest backup is unrecoverable. +- The tracked runner is a template, not a site-specific script. Remote address, + account, paths, container/database names, timer units, build identity and + migration fingerprint arrive only through a private JSON with an exact key + set. Rendering validates every value's type, charset and safety shape before + exclusively creating a candidate; runtime environment variables cannot + override those rendered constants. +- The exact rendered candidate must pass the tracked contract and preserve a + deterministic SHA-256 before upload. A workspace-only runner, hand-edited + candidate or contract failure is a hard upload blocker. +- Private render JSON and site-specific candidates must live outside the + repository or under the dedicated ignored `/.restore-drill-private/` + directory. The renderer checks both input and output with Git ignore rules + and fails before candidate creation when an in-repository path is not + ignored; operator memory is not a control. +- The mode-`0600` raw log is truncated exactly once at initialization. Container + start, dump list, target confirmation, restore apply and storage restore each + append a fixed stage marker and their private output, so a later failure does + not erase earlier-stage diagnostics before final teardown. + +### 4. Validation & Error Matrix + +- Dump list non-zero -> `dump_list=FAIL`, later stages `NOT_RUN`, category + `DUMP_LIST_NONZERO` / `dump_list_failed`. +- Target query non-zero or exact shadow name mismatch -> + `target_confirmation=FAIL`, apply `NOT_RUN`, category + `TARGET_CONFIRMATION_FAILED`. +- Independent list and target pass, but the restore script never invokes apply + -> category `APPLY_EXIT_CODE_MISSING`; fail closed. +- Apply exit non-zero -> `restore_apply=FAIL`, preserve the exact numeric exit + and safe numeric/`UNKNOWN` error count, category `RESTORE_APPLY_NONZERO`. +- Apply and script exit zero -> `restore_apply=PASS`; later migration, table and + attachment checks still decide the final recovery result. +- Diagnostics emitted only after cleanup, or cleanup residue non-zero -> the + attempt is non-clean and cannot be reported as `GO`. +- Missing/extra render key, newline/shell syntax, unsafe path/name, malformed + digest or pre-existing output -> render fails without publishing a partial + candidate. +- In-repository private config or candidate path not covered by Git ignore -> + render fails before the candidate exists. The dedicated ignored directory + remains valid for a locally protected render. +- Any raw-log truncating redirect after initialization, or a missing fixed stage + marker -> tracked contract failure; do not publish a candidate. +- Candidate differs from the deterministic render or has not passed the tracked + contract -> do not upload or install it. + +### 5. Good/Base/Bad Cases + +- Good: dump list and target pass, apply exits `23`; fixed fields show + `DOCKER_STDIN`, `23` and `RESTORE_APPLY_NONZERO` before teardown, then cleanup + reports zero residue without exposing the raw error. +- Base: all three stages pass with exit code `0`; the runner continues to + migration and attachment verification. +- Bad: emit only `database_restore_failed` after deleting the sole private log. +- Bad: parse and publish arbitrary pg_restore lines, or treat a non-zero exit as + success because the restored database appears partly populated. +- Bad: commit one host's concrete runner or reconstruct a supposedly equivalent + runner from chat/workspace immediately before an exercise. + +### 6. Tests Required + +- Directly drive the repository `scripts/restore-db.sh` with a real custom dump + through Docker stdin and host-file execution paths; do not replace the script + with an equivalent test implementation. +- Inject list failure, target mismatch and list-pass/apply-failure. Assert exact + stage enums, later-stage `NOT_RUN`, execution path and numeric exit code. +- Inject a non-zero apply exit and assert the diagnostic is emitted before + teardown, teardown passes, and the calling test can still read the fixed + fields afterwards. +- Scan captured diagnostics for fixture credentials, connection strings, SQL, + temporary paths and free-form tool output. +- Run the existing synthetic paired backup/restore and automatic off-host + contract suites after changing the diagnostic runner. +- Render the same private fixture twice and require byte equality; inject shell + syntax and require failure before the output file exists. +- Assert an unignored in-repository private config and an unignored candidate + path both fail before output creation; assert the dedicated ignored directory + accepts synthetic input and output without becoming trackable. +- Statically require exactly one raw-log truncation at initialization and fixed + append markers for container start, dump list, target confirmation, restore + apply and storage restore. +- Directly test source/copy identity, dynamic migration fingerprint and the + tracked attachment validator. Run `check-sensitive-content.mjs` explicitly on + all three tracked runner assets and require zero findings. + +### 7. Wrong vs Correct + +#### Wrong + +```text +restore-db > private.log || fail database_restore_failed +finally: delete private runtime +print result=NO_GO +``` + +#### Correct + +```text +tracked template + exact private config -> deterministic candidate + SHA-256 +contract PASS -> list -> target confirmation -> real restore path +capture only stage/path/exit/category enums +emit fixed diagnostics -> finally teardown raw private runtime +non-zero apply remains FAIL +``` + ## Scenario: Daily Restricted Off-Host Backup ### 1. Scope / Trigger diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/check.jsonl b/.trellis/tasks/08-23-isolated-backup-restore-drill/check.jsonl new file mode 100644 index 00000000..9dd3234a --- /dev/null +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/check.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/design.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/design.md new file mode 100644 index 00000000..72f04a23 --- /dev/null +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/design.md @@ -0,0 +1,124 @@ +# 隔离备份恢复演练设计 + +## 设计结论 + +采用“ARM 最新正式日归档 + 本轮私有临时副本 + 无宿主端口的一次性 PostgreSQL 16 + 空附件目录”的数据恢复方案。腾讯云生产只做演练前后公网 `/ready` 读取，不执行任何远端生产写操作；开发机不接收真实备份。 + +## 环境选择 + +| 方案 | 数据移动与风险 | 对生产影响 | 结论 | +| --- | --- | --- | --- | +| ARM 一次性影子恢复 | 备份已在 ARM，只在同机复制到私有临时区；明文恢复范围最小 | 无生产容器操作 | 采用 | +| 开发机本地 Docker | 需要把真实 DB/附件下载到个人开发机，扩大敏感数据副本 | 无运行影响，但数据治理风险高 | 不采用 | +| 腾讯云同机影子 Compose | 不需跨机下载，但会在真实生产宿主创建容器、目录和资源负载 | 生产宿主存在误操作和资源竞争风险 | 不采用 | + +ARM 在本设计中仍是备份与恢复演练节点，不是生产部署目标。一次性 PostgreSQL 不发布端口；本轮不启动 OA server。 + +## 数据流与信任边界 + +```text +ARM 正式日归档（只读） + │ 选择最新 complete 批次并校验 + ▼ +本轮 0700 临时副本 + ├── database.dump ──> 一次性 PostgreSQL 16 影子库 ──> migration/表计数检查 + └── storage.tar.gz ─> 本轮空附件目录 ───────────────> 文件数/引用/哈希检查 + │ + ▼ + 仅数量与状态的脱敏证据 +``` + +正式归档不被解包、改名、删除或直接用作写入目标。临时副本、影子数据库和恢复附件目录都由本轮唯一 run id 标识，清理只允许命中该标识。 + +## 恢复点合同 + +1. 从正式日归档中按 manifest `createdAt` 选择最新批次，而不是按目录 mtime 猜测。 +2. 候选目录只能包含三个普通文件；manifest 必须为 schema v1、`complete`，并通过 run id、文件名、字节数和 SHA-256 自校验。 +3. `buildSha` 必须唯一解析为本仓库已知提交，并且是演练开始时公网 `/ready` 实测当前构建的祖先或相同；同时保留 source image digest 是否存在的布尔证据，不输出完整 digest。相等状态仅用于度量发布后何时形成同源恢复点，不再决定原始备份是否可恢复。 +4. 演练开始时批次年龄不得超过 24 小时。候选不合格直接 `HOLD`，不自动降级到旧批次。 +5. 临时副本复制完成后再次验证三文件和 manifest，证明恢复输入与正式归档一致。 + +目录身份校验只适用于 ARM 正式归档源目录：其 basename 必须与 manifest `runId` 配对。临时副本允许使用本轮固定私有名称；对临时副本必须继续验证 manifest 合同、run id 格式、三文件集合、字节数、SHA-256、构建和时间，并与源目录首次校验得到的脱敏元数据完全一致，但不得要求临时目录 basename 冒充正式归档名称。 + +## 隔离恢复合同 + +### 数据库 + +- 使用随机一次性容器名和含 `drill` 的数据库名；容器无 `ports`，不加入现有业务网络，不挂载生产或 ARM 归档目录。 +- 使用 PostgreSQL 16，与现有恢复基线和脚本一致。 +- 调用 `scripts/restore-db.sh`，同时满足 `RESTORE_SCOPE=shadow`、精确数据库确认和显式 Docker 容器身份。 +- 恢复后只运行 `SELECT`：migration 成功/失败状态、固定关键表行数和附件引用聚合。不执行 Prisma migrate、seed、DDL 或修复语句。 +- 从 manifest 来源提交动态取得 migration 目录名称集合，与恢复库成功且未回滚的 `_prisma_migrations.migration_name` 做精确集合核对；不硬编码数量。再静态比较来源提交与当前构建的 migration 集合，记录“相同”或“当前有待升级 migration”。 + +固定 12 类表为：部门、员工、员工角色、采购申请、报销单、发票池、付款批次、合同、项目、采购附件、报销附件、付款回单附件。本次已知来源和当前构建现场均为 65 个 migration，但执行器必须使用动态名称集合，不能把 65 写成长期断言。 + +### 附件 + +- 调用 `scripts/restore-storage.sh` 恢复到不存在或为空的本轮目录。 +- 恢复前验证 tar 可读、无绝对路径、无 `..`、无软/硬链接；恢复后比较普通文件数。 +- 对使用本地存储的发票、采购、报销、付款回单和请假附件记录，只在影子库内读取 `storage_path`、大小和哈希用于机器校验，输出仅保留总数和不一致数。 +- `s3:` 引用不属于当前 tar 恢复范围；若最新备份中出现则 `FAIL` 并登记类别，不尝试访问外部对象存储。 + +## RTO/RPO 口径 + +- `RPO = drill_started_at - manifest.createdAt`。它表示使用该最新批次恢复时最多回退的数据时间窗口。 +- `data_restore_RTO = all_data_checks_passed_at - restore_copy_started_at`。起点位于全部只读前置门禁通过后、复制正式批次到临时工作区之前；终点为 DB、附件及关联检查全部通过。 +- `same_build_recovery_point_lag` 的正式口径需要可审计的发布完成时刻与 ARM 原子归档完成时刻；当前证据不具备后者，因此不得输出精确正式值。内部诊断可记录 `manifest_creation_after_contract_dropin = manifest.createdAt - automatic-backup service 构建 drop-in 最大 mtime`，但必须明确它既不是应用发布时刻，也不是 ARM 到达时刻。首次严格构建门禁失败仍证明发布后存在一段没有当前构建同源恢复点的窗口；该画像独立于数据 RPO 和数据恢复 RTO，即使恢复没有开始也保留。 +- 前置访问准备、等待人工输入和任务文档整理不计入 RTO；镜像拉取、批次复制、实际恢复和验证命令计入。 +- 本轮不启动应用，因此不得称为“完整 OA 恢复 RTO”；生产 `/ready` 只证明演练未影响在线服务。 + +## 状态判定 + +- `GO`：AC1–AC7 全部满足，可证明最新成对数据备份按其来源 schema 可恢复；若来源与当前 migration 集合不同，必须同时标注当前 schema 升级未验证，不能宣称完整灾难恢复 GO。 +- `PAUSE`：访问权限、磁盘空间、Docker/PostgreSQL 16 运行条件或清理验证不足，但未发现备份内容损坏。 +- `NO_GO`：最新批次超龄/损坏、来源提交未知或不是当前构建祖先/相同、DB restore 失败、来源 migration 集合或关键表异常、附件缺失/哈希不一致、DB 与附件不配对，或生产受到影响。 + +重试不得覆盖首次失败证据。若修正环境性问题后重试，必须使用新 run id 并保留首次脱敏 finding。 + +## Attempt 1 后的只读身份探针 + +首次恢复入口因最新 manifest `build_mismatch` 失败关闭后，不直接重装恢复入口。该失败证据保留，但严格相等门禁已被证明会在发布后自然备份到达前产生必然假阻断。后续先安装一个固定无参数、运行后自删除的 ARM root-owned 只读探针，权限仅用于读取 receiver 的路径配置和最新正式日归档： + +- 按 manifest `createdAt` 选择最新正式日批次，不自动改选旧批次。 +- 只输出 manifest 构建短标识、与当前生产短标识的相等状态、批次年龄、run id/目录配对状态、source image digest 是否存在，以及两份产物的文件类型、大小和 SHA-256 聚合校验状态；祖先关系在本地仓库只读验证，不要求 ARM 持有 Git 历史。 +- 不输出完整 run id、完整 digest、完整路径、字节数、文件名之外的业务信息或原始 manifest。 +- 不复制、不解包、不启动 Docker、不读取数据库内容或附件内容，不修改正式归档。 +- 成功或失败均删除探针自身和对应 sudoers；普通用户暂存文件在 SHA-256 精确匹配后另行清理。 + +探针结果不能替代恢复验收。最新 manifest 与当前生产不相等时，先判断其是否为当前构建祖先：是祖先且批次完整、新鲜时可进入原始恢复重试审批，同时记录同源恢复点滞后；未知或非祖先才维持 `NO_GO` 并转入备份链调查。手工触发生产备份始终是可撤回备选项，不是完成项。 + +## Attempt 3 后的数据库恢复诊断合同 + +Attempt 3 只留下 `database_restore_failed` 聚合类别，失败清理同时删除了权限 `0600` 的私有原始日志。删除潜在敏感日志是正确安全边界，但下一轮 runner 必须在清理前生成只含 allowlist 状态的阶段诊断，不能再次盲重试。 + +- 数据库恢复拆成 `dump_list`、`target_confirmation`、`restore_apply` 三个独立阶段；每个阶段只输出 `PASS`、`FAIL` 或 `NOT_RUN`，并保留稳定错误类别，不输出 pg_restore 原始 stderr、对象名、路径、凭据或 SQL。诊断必须在失败路径进入清理前写到标准输出，清理仍由 trap/finally 等价路径保证执行；输出先于清理不表示保留私有原始日志。 +- `dump_list` 使用与实际恢复相同的 PostgreSQL 16 容器和临时副本，只解析 custom dump 目录，不执行数据库写入。失败时分类为 `dump_list_failed` 并停止。 +- `target_confirmation` 只确认当前数据库名等于含 `drill` 标记的本轮目标，并确认容器连接可用；失败时分类为 `restore_target_confirmation_failed`。 +- 只有前两项通过才允许调用现有 `restore-db.sh`。调用必须记录执行路径枚举 `DOCKER_STDIN` 或 `HOST_FILE`、脚本退出码和真实 pg_restore apply 退出码；ARM runner 固定使用 `DOCKER_STDIN`，两条路径都由本地集成回归直接驱动仓库中的 `scripts/restore-db.sh`。此后非零分类为 `restore_apply_failed_after_list_pass`，证明 dump 可解析但实际应用失败；它仍不能自动等同于备份内容损坏。 +- 对 pg_restore 非零不设置“可忽略对象错误”白名单：即使私有日志末尾可脱敏提取到对象错误计数，任何非零仍为 `FAIL/NO_GO`。计数只输出非负整数或 `UNKNOWN`，不得据此把部分恢复升级为 `PASS`；未来若要允许部分成功，必须另行列出错误类别、证明对象集合完整并重新批准设计。 +- 私有原始日志继续保持 `0600` 且不得复制出 ARM；成功或失败都随本轮 runtime 删除。清理前将三项阶段状态写入最终标准输出，保证失败原因至少可定位到合同边界。 +- 合成回归必须分别制造 dump 列表失败、目标确认失败和列表通过但 apply 失败，并断言后续阶段保持 `NOT_RUN`、输出不含 fixture Secret/路径/SQL、容器与 runtime 清理为零；至少一条注入失败用例必须同时证明诊断在清理前已输出、清理完成后该诊断仍可从调用方捕获。成功回归直接以 PostgreSQL 16 custom dump 驱动真实 `scripts/restore-db.sh` 的 Docker stdin 与宿主文件两条路径，不重写等价恢复逻辑。 +- 该修复只改变演练器诊断能力，不放宽正式归档、源/副本身份、24 小时新鲜度、shadow 数据库、无端口、附件或清理门禁。任何 Attempt 4 实际恢复仍须使用新 run id、重新探测最新批次并另行批准。 + +## Attempt 4 前的 runner 持久化门禁提案 + +个人 `.trellis/workspace/` 中的 runner 和回归不会提交，不能作为 Attempt 4 的可重复证据。该限制不接受；实际恢复前应在另行批准的本地代码阶段完成以下持久化，不与 ARM 探针或恢复操作混在一起： + +- 将当前一次性恢复 runner 收敛为受版本控制、只含占位符和校验逻辑的 `scripts/qa/offhost-shadow-restore-runner.template.sh`，将附件关联校验器作为 `scripts/qa/validate-offhost-shadow-restore-attachments.py`。Attempt 4 候选必须由合同脚本使用私有 JSON 参数机械渲染，并与现有 `scripts/restore-db.sh`、`scripts/restore-storage.sh` 一起打包；不得从聊天或个人 workspace 重新生成等价实现。 +- 新增 `scripts/qa/run-offhost-shadow-restore-runner-contract.sh`，直接校验候选 runner 的 Bash 语法、依赖 SHA-256、固定无参数入口、源/副本身份重绑、动态 migration 集合、三阶段脱敏诊断、退出码、失败先诊断后清理及敏感输出边界。 +- 合同脚本直接以 PostgreSQL 16 custom dump 驱动真实 `scripts/restore-db.sh` 的 Docker stdin 和 host-file 两条路径，并覆盖 dump-list 失败、目标确认失败、apply 非零、后续阶段 `NOT_RUN` 和零残留。测试失败不得上传候选。 +- 远端地址、现场路径、远端账号、一次性容器/数据库命名、timer、构建前缀和 migration 集合指纹均不进入模板；由私有 JSON 提供并按字段类型、字符集和安全命名规则验证后替换。渲染器拒绝额外/缺失字段、换行、shell 语法、非绝对或逃逸路径和错误摘要格式，不从不受信环境变量覆盖。 +- 私有 JSON 和渲染后的现场候选只能位于仓库外，或位于仓库内专用的 `/.restore-drill-private/` ignored 目录。渲染器必须对输入和输出分别执行 Git ignore 判定；任一路径在仓库内但未被忽略时，在创建候选前失败关闭，不能依赖人工避免 `git add -A`。 +- 权限 `0600` 的私有原始日志只允许初始化时截断一次；容器启动、dump list、目标确认、restore apply 和附件恢复各写固定阶段标记并使用追加模式。这样后续阶段失败时仍能在本轮 runtime 内交叉核对之前阶段，最终清理和脱敏输出边界不变。 +- 上传前由合同脚本对实际候选文件生成/校验 SHA-256；ARM 安装前再次校验同一摘要。证据记录合同脚本提交、候选摘要和 PASS，不记录地址、路径或业务数据。 + +## 清理与回滚 + +- 生产无变更，无生产回滚步骤。 +- 成功或失败都只删除本轮 run id 对应的一次性容器、临时附件目录和临时批次副本；归档目录、receiver 管理目录和现有监控容器不在清理匹配范围。 +- 清理前再次解析容器名和临时根目录；任何对象不满足本轮标识、类型或父目录约束时停止自动清理。 +- 清理后确认无本轮容器、宿主端口、临时网络和临时目录残留；失败则结论至少为 `PAUSE`，交由管理员精确处置。 + +## 证据边界 + +执行证据写入当前 task 的 `research/restore-drill-evidence.md`，随后只在现有 OPS7 Runbook 或人工试运行报告中补一条脱敏状态摘要；不新建 `docs/` 平行文档。不得保存主机地址、SSH 别名、完整远端路径、Secret、连接串、真实账号、表内容、附件名或原始日志。 diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.jsonl b/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.jsonl new file mode 100644 index 00000000..9dd3234a --- /dev/null +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md new file mode 100644 index 00000000..410fcd00 --- /dev/null +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md @@ -0,0 +1,147 @@ +# 隔离备份恢复演练实施计划 + +## 执行前门禁 + +- [x] 确认任务已在用户审阅最终规划摘要后由 `task.py start` 进入 `in_progress`。 +- [x] 只读确认本地 `main`、当前生产构建事实、来源/当前 migration 集合和现有恢复脚本；现场两构建均为 65 且目录无变化，但后续执行不硬编码数量。 +- [x] 只读确认生产公网 `/ready` 正常；不读取生产数据库或执行生产写入。 +- [ ] 连接 ARM 后只读核对 receiver 配置有效、最新正式日批次身份、dead-man 状态、Docker 可用性和磁盘余量；不列出完整目录或原始 manifest。 +- [ ] 磁盘安全预算至少满足“批次总字节数 5 倍 + 2 GiB”；不足则 `PAUSE`。 +- [ ] 若当前账号没有对正式归档只读和本轮临时区/Docker 的最小权限，停止并给用户一条窄权限操作步骤；不得申请通用 root shell 或扩大既有健康监控 sudo 入口。 + +## 有序执行清单 + +1. 生成不含主机信息的本轮 run id，记录 UTC 开始时间和生产前置 `/ready` 状态。 +2. 在 ARM 正式日归档中选择 manifest 时间最新的 complete 批次，验证三文件、字节数、SHA-256 与 24 小时 RPO 门禁；在本地仓库确认来源提交是当前构建祖先或相同，并单独记录是否严格相等。 +3. 创建权限 `0700` 的本轮临时工作区；启动 RTO 计时，复制整个批次并对临时副本重新执行同一校验。 +4. 创建无宿主端口、无业务网络、无持久 volume 的一次性 PostgreSQL 16 容器；数据库名含 `drill`，凭据随机且仅存在本轮进程环境。 +5. 使用 `scripts/restore-db.sh` 的 shadow 合同恢复数据库；任何警告、确认不匹配或 restore 非零立即停止后续验证并进入清理。 +6. 从来源提交动态冻结 migration 名称集合；通过错误即停的只读 SQL 验证恢复库成功且未回滚的 migration 集合精确匹配来源、无失败/未知项，并静态记录来源与当前构建的 migration 差异；随后验证固定 12 类关键表并记录行数。 +7. 使用 `scripts/restore-storage.sh` 恢复附件到本轮空目录；比较归档与解包普通文件数。 +8. 在影子库中聚合本地附件引用，以不输出路径的校验程序验证路径安全、文件存在及适用的大小/SHA-256；记录引用总数和不一致数。 +9. 所有数据检查通过时停止 RTO 计时；按 manifest 时间计算 RPO，记录当前发布到首个同源恢复点的可用滞后，形成区分“原始数据可恢复性”和“当前 schema 升级状态”的 `GO/PAUSE/NO_GO` 脱敏结论。 +10. 清理本轮容器、临时网络（若有）、临时附件目录、临时批次副本和临时凭据；确认无本轮残留。 +11. 再次只读检查生产公网 `/ready` 和 ARM dead-man/健康 timer 状态，确认演练没有影响在线生产与现有监控。 +12. 将证据写入当前 task，并在既有事实源补充当前状态；不提交任何真实备份、日志或业务内容。 + +## Attempt 1 执行状态 + +- 最新 ARM 候选在 `select_latest_batch` 阶段因 manifest `build_mismatch` 失败关闭；复制、容器创建、数据库恢复和附件恢复均未开始。 +- 失败后的只读复核确认：公网 `/ready` 与生产自动备份 unit 的构建短标识一致，两个生产侧事实均为 `7b86ffb3`；因此当前不是计划事实源过期或备份 unit 冻结构建漂移。 +- ARM 一次性 root 入口和本轮 runtime 均为零残留，两个监控 timer 仍为 active；本轮 5 个普通用户暂存文件已按 SHA-256 精确匹配并删除，未宽泛删除目录。 +- Attempt 1 保持 `NO_GO`。在当时取得最新 ARM manifest 的脱敏构建短标识、run id 配对关系和 manifest 摘要一致性前，不进入第 3 步及之后的恢复流程。 + +## Attempt 1 后只读调查清单 + +1. 生成固定无参数的 root-owned ARM 只读身份探针和最小 sudoers，只允许读取 receiver 路径配置及最新正式日归档。 +2. 本地执行 `bash -n`、候选 SHA-256、sudoers candidate `visudo`（环境可用时）和禁止写入/恢复关键字人工复核。 +3. 上传到 ARM 普通用户私有暂存目录；由用户输入 sudo 密码安装固定入口，安装前再次核对全部候选 SHA-256 和系统 sudoers 语法。 +4. 运行一次固定入口，记录构建短标识、批次年龄、manifest 配对、产物聚合完整性和入口自清理状态。 +5. 对照公网 `/ready` 与生产自动备份 unit 的当前短标识：相等时记录同源恢复点已形成；不相等时在本地仓库核对是否为当前构建祖先，祖先且完整/新鲜仍可形成原始恢复重试方案，未知或非祖先才 `NO_GO`。 +6. 按 SHA-256 精确删除普通用户暂存文件；不得删除 ARM 正式归档或宽泛清理目录。 + +### 只读调查执行结果 + +- [x] 探针候选通过本地/ARM SHA-256、Bash 语法和 ARM sudoers candidate 校验。 +- [x] 固定入口只读读取最新正式批次并自删除；未复制、解包、启动容器或恢复数据。 +- [x] 最新批次 `ec682565` 在 24 小时内，run id/目录配对及 DB/附件大小、SHA-256 均通过，但与当前 `7b86ffb3` 不匹配。 +- [x] 只读对时确认自动备份合同刷新晚于最新批次生成；finding 收敛为“发布后尚未产生新构建恢复点”。 +- [x] root 入口残留为 0，3 个普通用户候选及暂存目录已按精确身份清理，两个 ARM 监控 timer 仍 active。 +- [x] 08-24 最新自然批次只读探针通过：来源与当前构建相同、年龄不超过 24 小时、run id 配对及 DB/附件 SHA-256 均通过；临时入口和暂存零残留。当前可提交恢复重试审批，尚未获得恢复授权。 + +### Attempt 2 恢复重试结果 + +- [x] 用户批准后安装并执行动态 migration 集合版固定恢复入口。 +- [x] ARM 正式归档最新批次首次门禁通过，并开始复制到本轮私有临时区。 +- [x] 临时副本复检因 runner 错把正式源目录 basename 身份规则应用到固定名 `batch` 而触发 `batch_identity`；失败阶段为 `create_isolated_runtime`，数据库容器和恢复均未开始。 +- [x] runtime、root 入口、6 个普通用户候选和暂存目录均精确清理为零残留，两个 ARM 监控 timer 保持 active。 +- [x] 修订 runner：正式源目录继续强制 basename/run id 配对；临时副本 manifest run id 重绑到源侧已验证 run id，并继续校验完整内容合同及源/副本元数据精确一致。补齐 2 正例 + 3 负例后重新执行只读身份探针，实际恢复另行批准。 + +### Attempt 3 准备批准边界 + +- [x] 用户已批准修复 runner，并重新执行 08-25 最新正式批次只读身份探针。 +- [x] 回归已覆盖：正式源通过、同内容改名副本通过、内容篡改副本失败、源 basename/run id 错配失败、副本 run id 与源侧已验证 run id 不同失败。 +- [x] Attempt 3 未复用 Attempt 2 新鲜度：08-25 最新批次年龄约 6 小时 11 分，来源与当前构建相同，run id 配对、大小与 SHA-256 均通过。 +- [x] 用户复核探针后明确批准 Attempt 3 实际恢复，范围仅限 ARM 隔离影子恢复。 + +### Attempt 3 恢复结果与停止点 + +- [x] 修复后的候选在本地和 ARM 通过 SHA-256、Bash 与 sudoers 语法校验后安装；正式归档门禁和临时副本源侧身份重绑通过。 +- [x] runner 在 `restore_database` 阶段因 `restore-db.sh` 返回非零而以 `database_restore_failed` 失败关闭；migration、12 类关键表、附件恢复与关联验证均未执行。 +- [x] 一次性容器、本轮 runtime、临时 root 入口和普通用户候选均精确清理为零残留；ARM 两个监控 timer 保持 active，生产演练后 `/ready` 仍为相同构建的 `PASS`。 +- [x] 在任何 Attempt 4 前修复失败诊断合同：分别记录 dump 列表校验与实际 restore 的脱敏阶段码，并确保失败清理前只保留安全聚合诊断；本地回归已通过，远端探针和实际恢复仍需重新批准。 + +### Attempt 4 前的修复清单 + +1. runner 在调用 `restore-db.sh` 前，以同一 PostgreSQL 16 容器对临时副本执行独立 `pg_restore --list`，记录 `dump_list=PASS|FAIL|NOT_RUN`。 +2. 独立确认影子目标数据库名与连接状态，记录 `target_confirmation=PASS|FAIL|NOT_RUN`；不输出数据库名或连接信息。 +3. 仅在前两项通过后执行实际 restore，记录 `restore_apply=PASS|FAIL|NOT_RUN`；失败类别区分为 `dump_list_failed`、`restore_target_confirmation_failed` 或 `restore_apply_failed_after_list_pass`。 +4. 原始 stderr 只进入本轮 `0600` 私有日志并随 runtime 删除；最终输出只含 allowlist 阶段状态和既有脱敏聚合。 +5. 补三条阶段负例、敏感输出负例和每条失败路径零残留回归；通过后重新执行最新批次身份探针，再请求 Attempt 4 实际恢复批准。 + +### Attempt 3 后本地修复结果 + +- [x] `dump_list`、`target_confirmation`、`restore_apply` 固定输出 `PASS/FAIL/NOT_RUN`，并增加执行路径、脚本/pg_restore 退出码、对象错误计数与稳定类别。 +- [x] 诊断在清理前输出；注入 apply 退出码 `23` 后，调用方在 runtime/access 清理完成后仍可读取阶段状态和退出码。 +- [x] partial-success 规则保持失败关闭：任何 pg_restore 非零均为 `FAIL/NO_GO`，不设置可忽略对象错误白名单。 +- [x] PostgreSQL 16 custom dump 直接驱动真实 `scripts/restore-db.sh` 的 Docker stdin 与 host-file 两条路径，均恢复成功。 +- [x] dump-list 失败、目标确认失败、apply 失败及敏感输出负例通过；既有成对备份/恢复回归和自动异机备份合同通过。 +- [x] 用户另行批准并完成 Attempt 4 前最新批次只读探针：构建 `SAME`、年龄约 7 小时 24 分、run id/产物完整性与 migration 集合门禁通过，入口及暂存零残留；尚未执行实际恢复。 + +### Attempt 4 前的持久化门禁 + +1. 将已验证的 runner 和附件校验器从个人 workspace 收敛为 `scripts/qa/` 下的跟踪文件，不改变恢复边界或生产脚本。 +2. 将八类 runner 回归收敛为可独立执行的 `run-offhost-shadow-restore-runner-contract.sh`，直接驱动真实 `restore-db.sh` 双路径。 +3. 合同脚本必须机械验证三阶段诊断、执行路径、两个退出码、非零失败关闭、源/副本身份、dynamic migration、敏感输出和清理顺序；对候选生成稳定 SHA-256。 +4. 运行新合同、既有成对备份/恢复、自动异机备份 74 项、Trellis 治理与敏感扫描；全部通过后才允许把跟踪文件打包为 Attempt 4 候选。 +5. 本阶段仅本地修改与测试；新批次探针结果和持久化门禁均通过后，再单独申请 Attempt 4 实际恢复。 + +### Attempt 4 前持久化门禁结果 + +- [x] runner 已收敛为 `scripts/qa/offhost-shadow-restore-runner.template.sh`；现场路径、账号、容器/数据库命名、timer、构建和 migration 指纹均为经严格校验的私有渲染字段。 +- [x] 附件关联校验器已收敛为 `scripts/qa/validate-offhost-shadow-restore-attachments.py`。 +- [x] `scripts/qa/run-offhost-shadow-restore-runner-contract.sh` 可独立复跑并提供 `render` 模式；相同私有配置生成确定性候选 SHA-256，shell 注入在输出候选前失败关闭。 +- [x] 源/副本身份、内容篡改、dynamic migration、附件安全、三阶段诊断、真实 restore 双路径、NOT_RUN、退出码、清理顺序和敏感输出合同全部通过。 +- [x] 新增三个文件由 `check-sensitive-content.mjs` 定向扫描，`3/3` 文件、finding `0`；本地测试容器残留 `0`。 +- [x] `.gitignore` 增加 `/.restore-drill-private/` 专用目录；合同分别证明仓库内未忽略的私有 JSON、未忽略的现场候选均在创建输出前失败，ignored 输入/输出正例通过。 +- [x] 私有原始日志保留唯一初始化截断，五个固定阶段均以标记分隔并追加写；静态合同拒绝后续阶段再次截断。 +- [x] 完整复跑 runner 合同、既有隔离备份/恢复演练和自动异机备份 74 项；task、Trellis、process、默认/定向敏感扫描及空白门禁通过，测试容器和本地私有路径残留均为 `0`。 +- [ ] Attempt 4 候选尚未渲染或上传；实际恢复前须再次选择最新批次并取得明确批准。 + +## 验证命令类别 + +- 本地/仓库：`bash -n` 覆盖备份、DB 恢复、附件恢复和 receiver 脚本；只读统计 migration 目录。 +- ARM 前置：receiver `validate-config`、systemd 公共状态、Docker 版本/容量、候选 manifest 脱敏校验。 +- 恢复输入：`sha256sum`、manifest schema/文件集/字节数校验、`pg_restore --list`、`tar -tzf`/`tar -tvzf`。 +- DB 验证：`psql -v ON_ERROR_STOP=1` 的 migration 名称/状态集合和固定表计数查询；本地用 Git 比较来源/当前 migration 目录，不执行 migrate。 +- 附件验证：`find` 普通文件计数及不输出路径的 Python 只读校验；只输出引用数和 mismatch 数。 +- 任务门禁： + - `python3 ./.trellis/scripts/task.py validate .trellis/tasks/08-23-isolated-backup-restore-drill` + - `node scripts/qa/check-trellis-governance.mjs` + - `node scripts/qa/check-process-contract.mjs --baseline --json` + - `node scripts/qa/audit-docs.mjs --check-registry` + - `node scripts/qa/check-sensitive-content.mjs` + - `git diff --check` + +具体远端命令在执行阶段依据 ARM 已安装的 root-owned receiver 配置生成，不把真实地址、账号、目录或凭据写入本计划。 + +## 停止条件 + +- 最新批次不存在、超过 24 小时、manifest 或 SHA-256 不一致，或来源提交未知/不是当前构建祖先或相同。仅“不严格相等”不再单独停止。 +- 权限、Docker、PostgreSQL 16 或磁盘预算不满足。 +- 命令目标可能指向腾讯云生产、ARM 正式归档、receiver 管理目录或既有容器/网络。 +- DB restore、migration、固定表、附件安全、文件引用、哈希/大小任一检查失败。 +- 生产 `/ready` 在演练期间异常，或现有 ARM dead-man/健康监控状态改变。 +- 清理对象身份不明确。此时保留现场并报告，不使用宽泛删除命令。 + +## 变更与证据文件边界 + +- 允许写入：当前 Trellis task、既有 OPS7 Runbook/人工试运行报告中的脱敏状态摘要、本轮 ARM 私有临时区和一次性 Docker 对象。 +- 禁止写入：产品代码、migration、生产/ARM 正式配置、生产 DB/附件、ARM 正式归档、企业微信。 +- 计划新增的 task 内证据：`research/restore-drill-evidence.md`；只在实施实际发生后创建。 + +## 审查与收口 + +- 本任务涉及真实备份数据和远端运维边界。执行完成后做一次只读复核，重点确认：生产零变更、归档未修改、RTO/RPO 口径正确、附件验证没有漏项、敏感内容未进入证据、临时对象已清理。 +- 若现有脚本需要修改，回到规划阶段补充设计和新的批准；不得在远端临时放宽 shadow/路径/确认门禁。 +- 门禁通过后用中文 commit 引用 `08-23-isolated-backup-restore-drill`，是否推送仍按用户当时授权执行。 diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/prd.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/prd.md new file mode 100644 index 00000000..5decbed0 --- /dev/null +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/prd.md @@ -0,0 +1,89 @@ +# 隔离备份恢复演练 + +## Goal + +在不覆盖、不暂停腾讯云生产 OA 的前提下，使用 ARM 备份机上最新的完整异机批次完成一次数据库与附件的隔离影子恢复，证明“备份存在”进一步达到“最新成对数据备份可恢复”，并记录实测数据恢复 RTO、RPO 和脱敏证据。 + +## Scope + +- 在 ARM 上选择并校验最新正式日归档，使用本轮临时副本恢复数据库与附件。 +- 验证 migration、固定关键表、附件数量与 DB/附件引用一致性，记录数据恢复 RTO/RPO。 +- 复核生产前后 `/ready`、ARM 监控状态和临时对象清理结果，形成当前 task 内脱敏证据并更新既有事实源摘要。 +- 不修改产品运行代码；允许把演练 runner 模板、附件校验器和可复跑合同收敛到 `scripts/qa/`。生产、正式归档、企业微信和现有监控均保持只读或零变更。 + +## Background + +- 腾讯云服务器是真实 OA 生产；ARM 只承担异机备份、外部健康监控和受控恢复演练，不是 OA 生产服务器。 +- 2026-08-23 只读复核确认：生产最近 14 天每日异机备份和新鲜度 watchdog 均成功，ARM dead-man 最近成功；企业微信中的“恢复通知”来自公网健康状态恢复，不代表执行过数据恢复。 +- 2026-07-27 曾在 ARM 使用当时的正确异机批次完成真实格式影子恢复，但不能替代对当前最新自动备份的可恢复性验证。 +- 首次演练时生产产品构建为 `7b86ffb`，已通过公网 `/ready` 和 migration pending=none；当时仓库与来源构建 `ec682565` 均有 65 个 Prisma migration 目录，二者之间 migration 目录无变化。该数字是本次现场事实，不再作为跨构建硬编码合同。 +- 现有恢复脚本默认只接受带 `shadow`、`drill` 或 `restore_test` 标记的数据库名，并拒绝生产式目标、确认值不一致、非空附件目标和危险归档路径。 + +## Requirements + +### R1. 恢复点选择与前置门禁 + +- 只选择 ARM 正式日归档中按 UTC 时间最新的一个完整批次，不触碰 inbox、临时事务目录、旧错误批次或周检查点。 +- 批次必须恰含 `database.dump`、`storage.tar.gz`、`manifest.json` 三个普通文件；manifest 必须为 schema v1、`complete`，相对文件名、字节数、SHA-256 和来源构建身份全部一致。 +- 批次年龄必须不超过 24 小时。manifest `buildSha` 必须唯一解析为本仓库已知提交，且该提交是演练开始时公网 `/ready` 实测当前构建的祖先或与其相同；未知提交、旁支/后继提交、超龄、损坏或文件集歧义判定 `NO_GO`，磁盘空间不足判定 `PAUSE`。两者都不得自动改选更旧批次掩盖问题。 +- 构建相等只表示“当前发布后已经形成同源恢复点”，不再作为数据备份可恢复性的必要条件。若来源是当前构建的祖先，必须单独记录发布后同源恢复点滞后 finding，不能把它误报为归档损坏。 +- 执行前后各做一次公网 `/ready` 只读检查，仅用于证明生产未受影响；该结果不得充当影子恢复后的应用启动证据。 + +### R2. ARM 隔离和数据保护 + +- 演练仅在 ARM 上创建本轮专属、权限 `0700` 的临时工作区、一次性 PostgreSQL 16 容器和空附件目录；不得在开发机下载真实备份。 +- PostgreSQL 不发布宿主端口，不加入现有 OA/监控网络，不复用任何现有容器名、数据库名、volume 或目录；影子数据库名必须含 `drill` 标记。 +- 先把所选批次复制到临时工作区并重新校验 manifest 与三文件 SHA-256，再对临时副本执行恢复；正式 ARM 归档始终只读，不直接解包或改名。 +- 不读取或复用生产 `.env`、JWT、Webhook、企业微信 Secret、真实数据库口令或业务账号；影子数据库只使用本轮随机临时凭据，凭据不进入聊天、仓库、报告或命令参数证据。 +- 含现场路径、命名或构建值的私有渲染 JSON 和渲染后候选不得进入 Git 历史；若位于仓库内，必须置于专用 ignored 目录，合同脚本须在创建候选前机械验证该 ignore 状态，未忽略即失败关闭。 + +### R3. 数据库恢复验证 + +- 使用现有 `scripts/restore-db.sh` 的 `RESTORE_SCOPE=shadow` 和精确数据库名确认门禁，将最新 `database.dump` 恢复到一次性 PostgreSQL 16 容器。 +- `pg_restore --list` 和实际 restore 必须成功。从 manifest 来源提交动态冻结其 Prisma migration 目录集合；恢复后 `_prisma_migrations` 中成功且未回滚的 migration 名称必须与该来源集合一致，并且不存在失败、回滚中或来源未知的 migration。不得用硬编码数量替代集合核对。 +- 同时比较来源提交与当前构建的 migration 目录集合：集合相同则记录“当前 schema 无额外升级步骤”；当前构建新增 migration 时，原始备份恢复仍可继续验证，但“升级到当前 schema”标为 `PAUSE/未验证`，另行设计仅针对影子库的 migration 演练，不在本轮临时执行。 +- 固定核对组织、人员、角色、采购、报销、发票池、付款批次、合同、项目及三类核心附件表的存在性和行数；只记录表名、数量和 PASS/FAIL，不记录业务字段或正文。 +- 所有验证 SQL 必须只读并开启错误即停；不执行 migration、seed、DDL、数据修补或业务写入。 + +### R4. 附件恢复与关联验证 + +- 使用现有 `scripts/restore-storage.sh` 的 shadow 门禁恢复 `storage.tar.gz` 到本轮空目录；归档路径、链接类型和目标空目录检查必须通过。 +- 归档普通文件数必须与解包后普通文件数一致；对数据库中本地存储型的发票、采购附件、报销附件、付款回单和请假附件引用，验证相对路径安全、目标文件存在，并在已有 SHA-256/大小字段时核对哈希或大小。 +- 发现 `s3:` 引用、缺失文件、路径逃逸、哈希/大小不一致或 DB 与附件批次不配对时判定 `NO_GO`；不输出文件名、原始路径、票据内容或附件内容。 + +### R5. 计时、证据和清理 + +- `RPO` 定义为演练开始 UTC 时间减 manifest `createdAt`；必须小于等于 24 小时才通过。本次实测值只用于当前恢复演练，不形成 SLA 承诺。 +- “数据恢复 RTO”从通过前置门禁后、开始复制批次到临时工作区之前计时，到 DB/附件恢复和全部数据一致性检查通过时结束；不得把历史合成演练耗时或生产 `/ready` 计入本轮 RTO。 +- 另行记录“同源恢复点可用滞后”：当前发布合同刷新时间到首个相同构建的 ARM 完整批次到达时间。该指标与数据 RPO、数据恢复 RTO 分开，不形成 SLA；即使恢复未执行，观测到的发布后空窗仍是有效演练发现。 +- 证据写入当前 Trellis task，只记录日期、角色、批次脱敏标识、构建短标识、年龄、耗时、数量、状态、finding 类别和清理结果；不保存远端地址、完整目录、凭据、真实账号、业务正文或原始 payload。 +- 成功或失败后只清理本轮明确创建的一次性容器、临时工作区和临时凭据；清理后确认无容器、端口、临时目录残留。任何目标身份不明确时停止清理并人工复核，绝不删除正式归档。 + +## Acceptance Criteria + +- [ ] AC1：ARM 最新正式日批次通过文件集、manifest、字节数和 SHA-256 校验，来源构建是当前构建的祖先或相同，且实测 RPO 不超过 24 小时；是否已形成当前构建同源恢复点单独记录。 +- [ ] AC2：腾讯云生产在演练前后 `/ready` 均正常，生产容器、数据库、附件、配置、timer 和业务数据零写入、零重启、零切换。 +- [ ] AC3：一次性 PostgreSQL 16 影子库无宿主端口，数据库恢复成功；恢复 migration 集合与 manifest 来源提交一致且无失败/未知项，来源与当前 migration 差异已明确；固定 12 类关键表均可读并形成仅含数量的脱敏摘要。 +- [ ] AC4：附件安全恢复成功，归档与解包普通文件数一致；所有纳入范围的本地附件引用均安全且存在，适用的哈希/大小核对无不一致。 +- [ ] AC5：记录本轮数据恢复 RTO、RPO、同源恢复点可用滞后、开始/结束时间、失败点或 finding、Go/Pause/No-Go 结论；即使恢复未开始也保留已观测空窗，不把任何指标表述为完整应用灾备 RTO 或 SLA。 +- [ ] AC6：演练结束后本轮容器、端口、网络和临时数据全部清理，ARM 正式归档与现有 dead-man/健康监控不受影响。 +- [ ] AC7：任务校验、Trellis 治理、流程合同、文档审计、敏感扫描和 `git diff --check` 通过；证据不含敏感信息。 + +## Out of Scope + +- 不在腾讯云生产执行 restore、停机、重启、数据库 reset、migration、DDL、seed、配置修改、timer 修改或业务写入。 +- 不触发新的生产备份，不删除、替换或轮转 ARM 正式归档，不发送企业微信测试消息。 +- 不把真实备份下载到开发机，不把 ARM 变成 OA 部署目标或长期运行 OA server。 +- 本轮不启动隔离 OA 应用容器、不做登录、报销详情或附件下载页面验证，也不对恢复库执行 Prisma migration；因此只形成“原始数据恢复 RTO”和静态 migration 兼容判断，完整应用恢复、schema 升级和切换 RTO 仍需未来灾难恢复专项验证。 +- 不修改备份/恢复产品代码；若现有脚本在真实最新批次上暴露缺陷，先停止并另行回到规划，不现场放宽安全门禁。 + +## Blocking Open Questions + +- 无。构建门禁已裁决为“来源提交是当前构建祖先或相同 + 批次年龄不超过 24 小时”；实施仍需用户审阅本次修订后的最终规划摘要并再次明确批准。 + +## Test Plan + +- 校验最新批次 manifest、三文件集合、字节数、SHA-256、构建身份和 24 小时新鲜度。 +- 通过现有 shadow 恢复脚本完成 PostgreSQL 16 和附件恢复，验证来源提交的 migration 集合、与当前构建的 migration 差异、固定 12 类表及附件引用聚合。 +- 复核无宿主端口、生产前后 `/ready`、ARM 监控未受影响和本轮临时对象零残留。 +- 运行 task validate、Trellis governance、process contract、docs registry、敏感扫描和 `git diff --check`。 diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md new file mode 100644 index 00000000..c51580fc --- /dev/null +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md @@ -0,0 +1,293 @@ +# 隔离备份恢复演练证据 + +## 证据边界 + +- 仅记录日期、角色、短构建标识、状态、finding 类别和清理结果。 +- 不记录远端地址、SSH 身份、完整目录、凭据、账号、原始 manifest、业务字段或附件内容。 +- 腾讯云是 OA 生产；ARM 仅是异机备份、外部健康探测和隔离恢复演练节点。 + +## Attempt 1：前置门禁失败关闭 + +日期：2026-08-23 + +| 检查项 | 结果 | +| --- | --- | +| 生产公网 `/ready`（演练前） | `PASS` | +| 计划基线构建短标识 | `7b86ffb` | +| 最新 ARM 正式日批次选择 | 已选择最新候选，未降级到旧批次 | +| manifest 来源构建门禁 | `FAIL`：`build_mismatch` | +| 失败阶段 | `select_latest_batch` | +| finding 类别 | `latest_batch_invalid` | +| 数据库/附件复制 | 未开始 | +| 一次性 PostgreSQL / 恢复写入 | 未创建、未执行 | +| runtime 清理 | `PASS` | +| 临时 root 入口清理 | `PASS` | +| 结论 | `NO_GO` | + +该结果只证明最新候选未通过来源构建门禁，不能单独证明备份内容损坏。首次失败证据必须保留，禁止自动改选更旧批次或放宽门禁。 + +## 失败后只读复核 + +日期：2026-08-23 + +| 检查项 | 结果 | +| --- | --- | +| 生产公网 `/ready` | `PASS`，构建短标识 `7b86ffb3` | +| 生产自动备份 timer | `active` | +| 生产自动备份 unit 冻结构建 | `7b86ffb3` | +| ARM dead-man timer | `active` | +| ARM 公网健康 timer | `active` | +| ARM 一次性 root 入口残留 | `0` | +| ARM 本轮 runtime 残留 | `0` | +| ARM 普通用户暂存候选 | 窄名称初扫发现 `1`；随后按本轮 5 个文件的 SHA-256 精确匹配并全部删除 | + +只读复核排除了“当前生产构建事实源过期”和“生产自动备份 unit 仍冻结旧构建”两种解释。ARM 上本轮 5 个普通用户暂存文件均经 SHA-256 与本地候选逐一匹配后精确删除，未执行宽泛目录删除。剩余调查重点是：最新 ARM 正式日批次是否来自当前生产自动备份链，以及 manifest 构建身份在生产生成、异机传输或 ARM 日归档选择中的哪一环发生漂移。 + +## Attempt 1 当时判定与重试门禁 + +- Attempt 1 仍为 `NO_GO`，当时 AC1、AC3、AC4、AC5 未满足；该次没有形成数据恢复 RTO/RPO 实测值。 +- 不重试恢复，不触碰 ARM 正式归档，不修改腾讯云生产数据或配置。 +- 下一步只允许以窄权限读取以下脱敏事实：最新 ARM manifest 的构建短标识、批次时间与最新生产成功状态是否指向同一 run id、manifest 摘要是否一致。 +- 若确认只是 ARM 最新批次来源/归档选择异常，另行规划修复备份链；若确认归档身份正确且仅演练器口径有误，先更新任务设计并重新批准。 +- 任何第二次恢复尝试前，必须更新规划、重新生成一次性入口并再次取得用户明确批准。 + +## 只读身份探针结果 + +日期：2026-08-23 + +| 检查项 | 结果 | +| --- | --- | +| 最新 ARM manifest 构建短标识 | `ec682565` | +| 当前生产/自动备份 unit 短标识 | `7b86ffb3` | +| 构建匹配 | `NO` | +| 最新批次年龄 | 约 18 小时，`within_24h=yes` | +| run id 与目录配对 | `PASS` | +| DB/附件 manifest 大小与 SHA-256 | `PASS` | +| source image digest | 存在 | +| manifest 摘要 | 已取得 12 位脱敏前缀，不保存完整摘要 | +| 探针临时 root 入口清理 | `PASS`，残留 `0` | +| 普通用户暂存清理 | 3 个候选均按 SHA-256 精确删除，目录残留 `0` | +| ARM 两个监控 timer | `active` | + +`ec682565` 是仓库可识别的历史提交，也是当前生产提交的祖先。随后只读对时确认：当前生产自动备份合同约 1.3 小时前才刷新，而最新 ARM 批次约 18 小时前生成；下一次自然调度约 14 小时后执行。因此最新 ARM 批次是在当前发布之前由当时活动构建生成，不支持“错误来源批次”或“传输损坏”的判断。 + +当前 finding 应归类为“发布后尚未产生新构建恢复点”。首次恢复入口按当时已批准的精确构建门禁正确失败关闭；该 `NO_GO` 是 Attempt 1 的真实历史结论，不覆盖也不回写。后续设计复盘确认，严格相等不是原始备份可恢复性的必要条件，新的准入合同见 2026-08-24 规划修订。 + +当时用户决策：不手工触发生产自动备份，等待既有 timer 自然生成并异机归档下一批恢复点。该决策不是完成项，也不永久排除手工备份；若后续自然批次异常，可重新作为需明确批准的备选项。 + +## 2026-08-24 设计复盘与有效产出 + +- 自 2026-08-23 对时到当前日期，下一次自然调度窗口已经过去；当前唯一可立即执行的动作是按新设计重新只读探测 ARM 最新批次，而不是继续等待。 +- Attempt 1 已经观测到：生产合同刷新后，到下一个自然备份抵达前，没有与当前构建严格相等的异机恢复点。该“同源恢复点可用滞后”是独立于数据 RPO/RTO 的有效恢复画像，即使数据库恢复未开始也必须保留。 +- `ec682565` 与 `7b86ffb3` 的 migration 目录均为 65 个，目录差异为 0；这证明本次历史批次与当前 schema 没有 migration 增量，但不能证明“65”适合作为长期硬编码断言。 +- 修订后的恢复点门禁为：manifest 来源提交是当前生产构建的已知祖先或相同，批次年龄不超过 24 小时，且 manifest/两份产物完整。严格相等只用于度量同源恢复点何时形成。 +- migration 验证改为来源提交目录名称集合与恢复库成功集合精确一致；来源/当前集合不同则把“升级到当前 schema”单独标为未验证，不在本轮临时执行 migration。 + +## 2026-08-24 最新自然批次只读探针 + +本节仅追加，不改写 Attempt 1 及 2026-08-23 既有证据。 + +| 检查项 | 结果 | +| --- | --- | +| 演练开始时公网 `/ready` | `PASS` | +| 公网当前构建唯一解析 | `PASS`，短标识 `7b86ffb3` | +| 生产自动备份 unit 与公网构建 | `MATCH`，timer `active` | +| ARM 最新批次构建短标识 | `7b86ffb3` | +| 来源与当前构建关系 | `SAME`；因此同时满足“祖先或相同” | +| 批次年龄 | `68491` 秒，约 19 小时 2 分，`within_24h=yes` | +| run id 与目录配对 | `PASS` | +| DB/附件 manifest 大小与 SHA-256 | `PASS` | +| source image digest | 存在 | +| manifest 摘要 | 已取得 12 位脱敏前缀，不保存完整摘要 | +| 来源/当前 migration 目录 | 均为 65 个，差异文件 `0` | +| 探针临时 root 入口 | 自清理 `PASS`，复核残留 `0` | +| 普通用户暂存 | 3 个候选按 SHA-256 精确删除，目录残留 `0` | +| ARM 两个监控 timer | `active` | + +同源恢复点画像：生产备份合同刷新距探针约 `94465` 秒；最新同源 manifest 距探针 `68491` 秒，因此该同源批次的 manifest 创建时间约在合同刷新后 `25974` 秒（约 7 小时 13 分）。当前证据没有直接采集 ARM 原子归档完成时刻，不能把该差值表述为精确“到达 ARM”耗时；只能证明同源批次创建滞后约 7 小时 13 分，并且在本次探针时已经可用。 + +执行方结论：最新自然批次满足修订后的来源关系、新鲜度、run id 配对和产物完整性门禁，可进入恢复重试审批。该结论不是恢复授权；本次只读探针没有复制或解包备份、创建 Docker 对象、恢复数据库/附件或修改生产与 ARM 正式归档。 + +## 2026-08-24 同源滞后锚点更正 + +本节追加说明前节 `25974` 秒的锚点，不改写既有记录。 + +- 起点实际是生产 `automatic-backup` service 构建 drop-in 的最大文件 mtime；终点是最新 manifest `createdAt`。 +- 该差值可称为内部诊断指标“合同 drop-in 文件刷新到同源 manifest 创建”，不能称为应用发布完成到恢复点到达 ARM 的精确耗时。 +- 当前证据没有采集 ARM 原子归档完成时刻，因此正式 `same_build_recovery_point_lag` 只能保留为“存在发布后空窗”的定性 finding，`7h13m` 不对外引用为精确恢复画像。 +- 2026-08-23 “下一次自然调度约 14 小时后”的估算存在时区处理错误：探针把 systemd 输出的本地时区时间直接附为 UTC，导致约多算 8 小时。修正后自然调度约在当时探针后 6 小时，与 2026-08-24 最新 manifest 创建时间一致；没有证据表明中间发生非计划手工备份。 + +## Attempt 2：临时副本目录身份误用 + +日期：2026-08-24 + +本节仅追加，不改写 Attempt 1、只读探针或锚点更正记录。 + +| 检查项 | 结果 | +| --- | --- | +| 最新正式批次前置门禁 | `PASS` | +| 正式归档读取/修改 | 只读使用、零修改 | +| 临时副本创建 | 已复制到本轮私有临时区 | +| 失败阶段 | `create_isolated_runtime` | +| finding 类别 | `copied_batch_invalid` / `batch_identity` | +| PostgreSQL 容器 | 未创建 | +| DB/附件恢复 | 未开始 | +| migration/关键表/附件关联验证 | 未执行 | +| runtime 清理 | `PASS`，复核残留 `0` | +| 临时 root 入口清理 | `PASS`，复核残留 `0` | +| 普通用户暂存清理 | 6 个候选按 SHA-256 精确删除，目录残留 `0` | +| ARM 两个监控 timer | `active` | +| 结论 | `NO_GO` | + +根因：runner 的 `validate_batch` 同时承担正式源目录身份和批次内容校验。正式源目录 basename 必须与 manifest `runId` 配对是正确安全门禁；复制后的私有临时目录固定名为 `batch`，内容仍保持同一 manifest/run id，却被同一个 basename 断言错误拒绝。该 finding 不表示备份损坏，也没有进入数据库或附件恢复。 + +修订边界：正式源目录继续执行 basename/run id 强校验；临时副本只跳过目录 basename 断言，manifest、run id 格式、文件集、构建、时间、大小、SHA-256 和与源元数据相等比较全部保留。任何重试前必须补“正式源通过、同内容改名副本通过、任一内容篡改失败”的合成回归并再次取得明确批准。 + +## 2026-08-25 Attempt 2 finding 分类补充 + +本节只追加分类，不改写 Attempt 2 原始结果。 + +- Attempt 2 finding 类别明确为 `runner_defect`，具体子类为“源目录身份断言误用于临时副本”。 +- Attempt 1 finding 是“发布后尚未形成同源恢复点”；Attempt 2 finding 是演练器缺陷。两者均未证明备份不可恢复，不能合并表述为“备份连续失败两次”。 +- Attempt 2 的正式批次前置门禁 `PASS`、临时复制完成、恢复前失败关闭、正式归档零修改和所有临时对象零残留，证明安全门禁在错误发生后按设计收口。 +- 临时副本修订不是简单放宽：副本 manifest `runId` 必须重绑到源侧已验证的 run id，并继续通过 manifest、文件集、构建、时间、字节数、SHA-256 及源/副本元数据相等校验。 +- 新增强制负例：源目录 basename 与 manifest run id 不配对必须 `NO_GO`；副本 manifest run id 即使内部自洽，但与源侧已验证 run id 不同也必须 `NO_GO`。 + +## 2026-08-25 runner 修复验证与最新批次探针 + +本节仅追加，不改写 Attempt 1、Attempt 2 或既有探针记录。 + +### Runner 修复验证 + +| 回归 | 结果 | +| --- | --- | +| 正式源 basename 与 manifest run id 配对 | `PASS` | +| 同内容改名副本重绑源侧 run id | `PASS` | +| 副本内容篡改 | 预期失败，`PASS` | +| 源 basename 与 manifest run id 错配 | 预期失败，`PASS` | +| 副本 manifest run id 内部自洽但不同于源 run id | 预期失败，`PASS` | +| 动态 migration 集合正/反例 | `PASS` | +| 既有隔离备份/恢复合同回归 | `PASS` | + +实现结果：正式源目录继续校验 basename/run id；临时副本 manifest run id 必须等于源侧已验证 run id，并继续验证 manifest、文件集、构建、时间、大小、SHA-256 及源/副本脱敏元数据完全一致。 + +### 08-25 最新自然批次 + +| 检查项 | 结果 | +| --- | --- | +| 演练开始时公网 `/ready` | `PASS` | +| 公网构建与自动备份 unit | `MATCH`，短标识 `7b86ffb3`，timer `active` | +| ARM 最新批次构建短标识 | `7b86ffb3` | +| 来源与当前构建关系 | `SAME` | +| 批次年龄 | `22249` 秒，约 6 小时 11 分，`within_24h=yes` | +| run id 与目录配对 | `PASS` | +| DB/附件 manifest 大小与 SHA-256 | `PASS` | +| source image digest | 存在 | +| manifest 摘要 | 已取得 12 位脱敏前缀，不保存完整摘要 | +| 来源/当前 migration 目录 | 均为 65 个，差异文件 `0` | +| 探针 root 入口及普通用户暂存 | 残留均为 `0` | +| ARM 两个监控 timer | `active` | + +该批次不是 Attempt 2 的旧候选，新鲜度和全部身份/完整性门禁已重新计算。当前满足 Attempt 3 恢复审批前提，但本次批准不包含实际恢复，未复制、解包或恢复该批次。 + +## Attempt 3：数据库恢复阶段失败关闭 + +日期：2026-08-25 + +本节仅追加，不改写 Attempt 1、Attempt 2 或既有探针记录。 + +| 检查项 | 结果 | +| --- | --- | +| 候选文件 SHA-256、Bash 与 sudoers 语法 | `PASS` | +| 最新正式批次前置门禁 | `PASS`；使用 08-25 新候选 | +| 正式归档读取/修改 | 只读取用；runner 没有归档写入路径 | +| 临时副本创建及源侧身份重绑 | `PASS`；否则不会进入数据库恢复阶段 | +| 失败阶段 | `restore_database` | +| runner 聚合类别 | `database_restore_failed` | +| migration/12 类表/附件恢复与关联验证 | 未执行 | +| runtime 清理 | `PASS`，复核目录残留 `0` | +| 一次性 PostgreSQL 容器清理 | `PASS`，复核残留 `0` | +| 临时 root 入口与库清理 | `PASS`，复核残留 `0` | +| ARM 两个监控 timer | `active` | +| 腾讯云生产公网 `/ready`（演练后） | `PASS`，构建短标识仍为 `7b86ffb3` | +| 结论 | `NO_GO` | + +Attempt 3 在数据库恢复命令返回非零后正确停止，未进入 migration、关键表、附件恢复或附件关联检查。当前输出不能区分 dump 列表校验失败、恢复命令失败、工具/参数合同不匹配或其他 runner 调用问题，因此该次 `NO_GO` 不能被改写为“备份不可恢复”，也不能据此给出数据恢复 RTO 或完整恢复结论。 + +本轮还暴露了独立的演练器可观测性缺陷：`restore-db.sh` 的详细输出仅写入权限 `0600` 的本轮私有日志，而失败清理随后删除该日志，最终只保留 `database_restore_failed` 聚合类别。安全清理是正确的，但失败关闭前必须先生成不含路径、凭据、业务内容或原始 SQL 的阶段化脱敏诊断；在该合同补齐并通过回归、重新审批前，不执行 Attempt 4，也不改选旧批次或放宽门禁。 + +措辞对齐说明：既有 Attempt 2 段落中的“临时副本只跳过目录 basename 断言”仅描述不把自建目录名当归档证据，不构成放宽。生效合同始终是副本 manifest run id 必须等于源侧已验证 run id，并且副本全部脱敏元数据与源侧首次校验结果完全一致。 + +## 2026-08-25 Attempt 3 后本地诊断修复 + +本节只追加本地 runner 与合成回归结果；未连接 ARM、未探测新批次、未执行 Attempt 4，也未修改腾讯云生产或 ARM 正式归档。 + +- runner 将数据库恢复拆为 `dump_list`、`target_confirmation`、`restore_apply` 三个阶段，只输出 `PASS/FAIL/NOT_RUN`、执行路径枚举、数字退出码、数字或 `UNKNOWN` 错误计数和稳定错误类别。 +- allowlist 诊断在 trap/finally 等价清理之前输出；原始 stderr 仍只存在于权限 `0600` 的本轮私有日志并随 runtime 删除。注入退出码 `23` 的回归证明：阶段诊断先输出，runtime/access 清理随后均为 `PASS`，调用方在清理后仍能读取数字退出码。 +- PostgreSQL 16 custom dump 直接驱动仓库 `scripts/restore-db.sh` 的 Docker stdin 路径成功；host-file 路径通过 PostgreSQL 16 工具执行同一真实脚本成功。没有在测试中重写等价恢复逻辑。 +- dump-list 失败、目标确认失败和 list/target 通过后 apply 失败三条负例均通过；后续阶段保持 `NOT_RUN`，apply 非零不会因对象错误计数较小而降级为成功。 +- 捕获诊断不含 fixture 凭据、连接串、SQL、临时路径或自由文本工具错误；既有合成成对备份/恢复回归通过，自动异机备份合同 `74` 项通过。 + +截至 Attempt 3，最新备份是否真的可恢复这个核心问题仍然没有答案。目前验证到的是演练器、恢复点身份门禁和每次失败后的安全收口，不是最新备份的数据可恢复性。Attempt 1 是发布后恢复点缺口；Attempt 2 是临时副本身份规则误用；Attempt 3 目前只能确定演练器可观测性不足，底层数据库恢复失败原因仍未知。三次均为 `NO_GO`，但不得把次数解释为“最新备份连续三次不可恢复”。 + +## 2026-08-25 Attempt 4 前最新批次只读探针 + +本节仅追加探针与本地 Git 静态核对结果；未复制、解包或恢复批次，未安装恢复 runner，未执行 Attempt 4。 + +| 检查项 | 结果 | +| --- | --- | +| 演练前公网 `/ready` | `PASS`，构建短标识 `7b86ffb3c9ef` | +| ARM 最新批次探针 | `PASS` | +| 最新批次构建短标识 | `7b86ffb3` | +| 来源提交唯一解析 | `PASS` | +| 来源与当前生产构建关系 | `SAME` | +| 批次年龄 | `26652` 秒，约 7 小时 24 分，`within_24h=yes` | +| run id 与目录配对 | `PASS` | +| DB/附件大小与 SHA-256 | `PASS` | +| source image digest | 存在 | +| manifest 脱敏摘要前缀 | `ee1f97a5b42d` | +| 来源/当前 migration 集合 | `65/65`，差异条目 `0` | +| 仓库 HEAD 相对生产构建的应用代码漂移 | 文件数 `0` | +| 探针 root 入口清理 | `PASS`，残留 `0` | +| 普通用户探针暂存 | 残留 `0` | +| ARM 两个监控 timer | `active` | + +该批次满足 Attempt 4 的构建关系、新鲜度、run id 配对、产物完整性和 schema 集合前置门禁；该结论仍不是恢复授权，也不回答最新备份是否可恢复。 + +恢复 runner 的持久化限制不予接受：截至本节时，修复后的 runner 与八类本地回归仍位于不提交的个人 workspace，仓库尚无可由其他执行者独立复跑的 runner 合同门禁。Attempt 4 实际恢复前必须先把 runner 的稳定来源和对应合同回归纳入 `scripts/qa/`，由合同脚本机械验证候选 SHA-256、三阶段诊断、源/副本身份、真实 `restore-db.sh` 双路径和失败清理；未完成前不安装 Attempt 4 恢复入口。 + +## 2026-08-25 runner 持久化合同结果 + +本节仅追加本地跟踪文件与合同回归结果；未连接 ARM、未渲染现场候选、未执行 Attempt 4。 + +新增跟踪文件： + +- `scripts/qa/offhost-shadow-restore-runner.template.sh` +- `scripts/qa/validate-offhost-shadow-restore-attachments.py` +- `scripts/qa/run-offhost-shadow-restore-runner-contract.sh` + +| 合同类别 | 结果 | +| --- | --- | +| 私有参数渲染、确定性候选 SHA-256、shell 注入拒绝 | `PASS` | +| 正式源身份与改名私有副本源侧重绑正例 | `PASS` | +| 源 basename/run id 错配、副本篡改、副本 run id 与源不同负例 | `PASS` | +| dynamic migration 集合指纹正/反例，无硬编码数量 | `PASS` | +| 附件本地路径/大小/SHA-256 正例，路径逃逸与 S3 负例 | `PASS` | +| 三阶段诊断、执行路径、两个退出码、后续 `NOT_RUN` | `PASS` | +| 真实 `scripts/restore-db.sh` Docker stdin 与 PostgreSQL 16 host-file 路径 | `PASS/PASS` | +| apply 非零失败关闭、诊断先于清理、清理后可读、无敏感自由文本 | `PASS` | + +持久化模板不含远端地址、个人用户目录、远端账号、现场路径、实际容器/数据库名、生产构建值或 migration 指纹；这些值只能来自私有 JSON，并在候选写出前通过精确字段集、类型、字符集、绝对路径、shadow 命名和摘要格式校验。`check-sensitive-content.mjs` 对三个新增文件定向扫描结果为 `PASS`，扫描文件 `3`、finding `0`；本地测试容器残留 `0`。 + +Attempt 4 的 runner 产生过程已从个人 workspace 人工产物改为“跟踪模板 + 跟踪合同 + 私有参数确定性渲染 + 候选 SHA-256”。合同未通过不得生成或上传候选。该结果只补齐演练器可重复性，仍不构成最新备份可恢复性结论或 Attempt 4 恢复授权。 + +## 2026-08-25 私有渲染与阶段日志门禁补强 + +本节仅追加本地门禁修订与合成回归结果；未连接 ARM、未创建现场私有配置、未渲染或上传 Attempt 4 候选，也未执行实际恢复。 + +- `.gitignore` 已增加专用 `/.restore-drill-private/` 目录；合同分别注入仓库内未忽略的私有 JSON 和未忽略的候选输出路径，两者均在候选创建前按预期失败。ignored 目录内的合成输入/输出正例通过，测试文件在退出时清理。 +- 私有原始日志仍为 `0600` 且只在初始化时截断一次；PostgreSQL 容器启动、dump list、目标确认、restore apply 和附件恢复使用固定阶段标记追加。合同静态拒绝后续截断，最终 runtime 清理与只输出 allowlist 诊断的边界不变。 +- PostgreSQL 16 fixture 增加连续成功查询的稳定就绪门禁，避免镜像首次初始化临时服务关闭窗口造成合成回归假失败；真实 `scripts/restore-db.sh` 字节未修改，Docker stdin 和 host-file 两条路径仍直接执行仓库脚本并通过。 +- 本轮新增门禁回归全部 `PASS`；完成完整质量门禁并形成可追溯 Git 提交前，仍不得准备或上传 Attempt 4 候选。 + +完整本地门禁随后通过：runner 合同全部 `PASS`，既有隔离备份/恢复合成演练 `PASS`，自动异机备份合同 `74/74 PASS`；task validate、Trellis governance、process baseline、默认敏感扫描和三个新增文件定向敏感扫描均 `PASS`。文档审计错误 `0`，仅保留既有未登记文件 warning；`shellcheck` 在本机不可用，已以 Bash 语法、真实双路径集成回归和仓库合同覆盖。测试容器、本地私有渲染目录及候选残留均为 `0`。 diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/task.json b/.trellis/tasks/08-23-isolated-backup-restore-drill/task.json new file mode 100644 index 00000000..a3dfae25 --- /dev/null +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/task.json @@ -0,0 +1,26 @@ +{ + "id": "isolated-backup-restore-drill", + "name": "isolated-backup-restore-drill", + "title": "隔离备份恢复演练", + "description": "在不覆盖生产的隔离环境验证最新数据库与附件备份可恢复性，并记录RTO/RPO和脱敏证据。", + "status": "in_progress", + "dev_type": null, + "scope": "operations", + "package": null, + "priority": "P2", + "creator": "shunli", + "assignee": "shunli", + "createdAt": "2026-08-23", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/scripts/qa/offhost-shadow-restore-runner.template.sh b/scripts/qa/offhost-shadow-restore-runner.template.sh new file mode 100644 index 00000000..648127d0 --- /dev/null +++ b/scripts/qa/offhost-shadow-restore-runner.template.sh @@ -0,0 +1,608 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +umask 077 + +CONFIG_FILE="@@RECEIVER_CONFIG_FILE@@" +LIB_DIR="@@RESTORE_LIB_DIR@@" +SELF_PATH="@@SELF_PATH@@" +SUDOERS_PATH="@@SUDOERS_PATH@@" +SUDOERS_MAIN_FILE="@@SUDOERS_MAIN_FILE@@" +WORK_ROOT="@@WORK_ROOT@@" +WORK_PREFIX="@@WORK_PREFIX@@" +CONTAINER_PREFIX="@@CONTAINER_PREFIX@@" +SHADOW_DB_USER="@@SHADOW_DB_USER@@" +SHADOW_DB_NAME="@@SHADOW_DB_NAME@@" +POSTGRES_IMAGE="@@POSTGRES_IMAGE@@" +DEADMAN_TIMER_UNIT="@@DEADMAN_TIMER_UNIT@@" +HEALTH_TIMER_UNIT="@@HEALTH_TIMER_UNIT@@" +ARCHIVE_BATCH_PREFIX="@@ARCHIVE_BATCH_PREFIX@@" +RUN_ID_PREFIX="@@RUN_ID_PREFIX@@" +EXPECTED_DB_RESTORE_SHA="@@EXPECTED_DB_RESTORE_SHA@@" +EXPECTED_STORAGE_RESTORE_SHA="@@EXPECTED_STORAGE_RESTORE_SHA@@" +EXPECTED_ATTACHMENT_VALIDATOR_SHA="@@EXPECTED_ATTACHMENT_VALIDATOR_SHA@@" +EXPECTED_BUILD_PREFIX="@@EXPECTED_BUILD_PREFIX@@" +EXPECTED_MIGRATION_SET_SHA="@@EXPECTED_MIGRATION_SET_SHA@@" +PHASE="bootstrap" +WORK_DIR="" +CONTAINER_NAME="" +RUNTIME_CLEANED=0 +RESULT_STATUS="NO_GO" +RPO_SECONDS="" +RTO_SECONDS="" +MIGRATION_SUCCESS="" +MIGRATION_OTHER="" +MIGRATION_SET_MATCH="" +ARCHIVE_FILE_COUNT="" +RESTORED_FILE_COUNT="" +ATTACHMENT_TOTAL="" +ATTACHMENT_MISMATCH="" +TABLE_COUNTS="" +SOURCE_RUN_ID="" +DUMP_LIST_STATUS="NOT_RUN" +TARGET_CONFIRMATION_STATUS="NOT_RUN" +RESTORE_APPLY_STATUS="NOT_RUN" +RESTORE_EXECUTION_PATH="NOT_RUN" +DUMP_LIST_EXIT_CODE="NOT_RUN" +RESTORE_SCRIPT_EXIT_CODE="NOT_RUN" +PG_RESTORE_EXIT_CODE="NOT_RUN" +PG_RESTORE_ERROR_COUNT="NOT_RUN" +RESTORE_ERROR_CATEGORY="NOT_APPLICABLE" +FAILURE_CATEGORY="none" +DIAGNOSTICS_EMITTED=0 + +fail() { + FAILURE_CATEGORY="$1" + echo "[restore-drill][FAIL] phase=$PHASE category=$1" >&2 + exit 1 +} + +require_command() { + command -v "$1" >/dev/null 2>&1 || fail "required_command_missing" +} + +cleanup_runtime() { + local cleanup_ok=1 + local container_suffix="" + local work_suffix="" + set +e + if [[ -n "$CONTAINER_NAME" ]]; then + container_suffix="${CONTAINER_NAME#"$CONTAINER_PREFIX"}" + if [[ "$CONTAINER_NAME" == "$CONTAINER_PREFIX"* \ + && "$container_suffix" =~ ^[0-9TZ-]+$ ]]; then + docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + docker inspect "$CONTAINER_NAME" >/dev/null 2>&1 && cleanup_ok=0 + else + cleanup_ok=0 + fi + fi + if [[ -n "$WORK_DIR" ]]; then + work_suffix="${WORK_DIR#"$WORK_ROOT/$WORK_PREFIX"}" + if [[ "$WORK_DIR" == "$WORK_ROOT/$WORK_PREFIX"* \ + && "$work_suffix" =~ ^[0-9TZ-]+$ \ + && -d "$WORK_DIR" && ! -L "$WORK_DIR" ]]; then + rm -rf -- "$WORK_DIR" + [[ ! -e "$WORK_DIR" ]] || cleanup_ok=0 + elif [[ -e "$WORK_DIR" ]]; then + cleanup_ok=0 + fi + fi + set -e + [[ "$cleanup_ok" -eq 1 ]] +} + +cleanup_access() { + local cleanup_ok=1 + set +e + if [[ -f "$SUDOERS_PATH" && ! -L "$SUDOERS_PATH" \ + && "$(stat -c %u "$SUDOERS_PATH" 2>/dev/null)" == "0" ]]; then + rm -f -- "$SUDOERS_PATH" || cleanup_ok=0 + elif [[ -e "$SUDOERS_PATH" ]]; then + cleanup_ok=0 + fi + visudo -cf "$SUDOERS_MAIN_FILE" >/dev/null 2>&1 || cleanup_ok=0 + if [[ -d "$LIB_DIR" && ! -L "$LIB_DIR" \ + && "$(stat -c %u "$LIB_DIR" 2>/dev/null)" == "0" ]]; then + rm -rf -- "$LIB_DIR" || cleanup_ok=0 + elif [[ -e "$LIB_DIR" ]]; then + cleanup_ok=0 + fi + if [[ -f "$SELF_PATH" && ! -L "$SELF_PATH" \ + && "$(stat -c %u "$SELF_PATH" 2>/dev/null)" == "0" ]]; then + rm -f -- "$SELF_PATH" || cleanup_ok=0 + elif [[ -e "$SELF_PATH" ]]; then + cleanup_ok=0 + fi + set -e + [[ "$cleanup_ok" -eq 1 ]] +} + +emit_restore_diagnostics() { + if [[ "$DIAGNOSTICS_EMITTED" -eq 1 ]]; then + return 0 + fi + DIAGNOSTICS_EMITTED=1 + echo "failure_category=$FAILURE_CATEGORY" + echo "dump_list=$DUMP_LIST_STATUS" + echo "dump_list_exit_code=$DUMP_LIST_EXIT_CODE" + echo "target_confirmation=$TARGET_CONFIRMATION_STATUS" + echo "restore_execution_path=$RESTORE_EXECUTION_PATH" + echo "restore_apply=$RESTORE_APPLY_STATUS" + echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE" + echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE" + echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT" + echo "restore_error_category=$RESTORE_ERROR_CATEGORY" +} + +finish() { + local exit_code="$?" + local runtime_cleanup="PASS" + local access_cleanup="PASS" + trap - EXIT HUP INT TERM + # Emit only allowlisted stage diagnostics while the private runtime still + # exists. Raw stderr remains mode 0600 and is removed by cleanup_runtime. + emit_restore_diagnostics + if [[ "$RUNTIME_CLEANED" -ne 1 ]]; then + cleanup_runtime || runtime_cleanup="FAIL" + fi + cleanup_access || access_cleanup="FAIL" + if [[ "$runtime_cleanup" != "PASS" || "$access_cleanup" != "PASS" ]]; then + exit_code=1 + fi + echo "runtime_cleanup=$runtime_cleanup" + echo "temporary_access_cleanup=$access_cleanup" + if [[ "$exit_code" -eq 0 ]]; then + echo "result=$RESULT_STATUS" + echo "rpo_seconds=$RPO_SECONDS" + echo "data_restore_rto_seconds=$RTO_SECONDS" + echo "migration_success=$MIGRATION_SUCCESS" + echo "migration_other=$MIGRATION_OTHER" + echo "migration_set_match=$MIGRATION_SET_MATCH" + echo "archive_regular_files=$ARCHIVE_FILE_COUNT" + echo "restored_regular_files=$RESTORED_FILE_COUNT" + echo "attachment_reference_count=$ATTACHMENT_TOTAL" + echo "attachment_mismatch_count=$ATTACHMENT_MISMATCH" + printf '%s\n' "$TABLE_COUNTS" + else + echo "result=NO_GO" + fi + exit "$exit_code" +} +trap finish EXIT HUP INT TERM + +for command_name in docker python3 sha256sum tar find stat df cp openssl visudo systemctl; do + require_command "$command_name" +done + +PHASE="entry_contract" +[[ "$(id -u)" -eq 0 ]] || fail "root_required" +for absolute_path in "$CONFIG_FILE" "$LIB_DIR" "$SELF_PATH" "$SUDOERS_PATH" \ + "$SUDOERS_MAIN_FILE" "$WORK_ROOT"; do + [[ "$absolute_path" == /* && "$absolute_path" != "/" ]] \ + || fail "rendered_path_invalid" +done +[[ "$WORK_PREFIX" =~ ^[a-z0-9][a-z0-9-]*-$ \ + && "$CONTAINER_PREFIX" =~ ^[a-z0-9][a-z0-9-]*-$ \ + && "$SHADOW_DB_USER" =~ ^[a-z_][a-z0-9_]*$ \ + && "$SHADOW_DB_NAME" =~ (^|_)(shadow|drill|restore_test)(_|$) \ + && "$POSTGRES_IMAGE" =~ ^[a-z0-9._/-]+:[a-zA-Z0-9._-]+$ \ + && "$DEADMAN_TIMER_UNIT" =~ ^[a-zA-Z0-9_.@-]+[.]timer$ \ + && "$HEALTH_TIMER_UNIT" =~ ^[a-zA-Z0-9_.@-]+[.]timer$ \ + && "$ARCHIVE_BATCH_PREFIX" =~ ^[a-z0-9][a-z0-9-]*-$ \ + && "$RUN_ID_PREFIX" =~ ^[a-z0-9][a-z0-9-]*-$ ]] \ + || fail "rendered_identity_invalid" +[[ "$EXPECTED_DB_RESTORE_SHA" =~ ^[0-9a-f]{64}$ \ + && "$EXPECTED_STORAGE_RESTORE_SHA" =~ ^[0-9a-f]{64}$ \ + && "$EXPECTED_ATTACHMENT_VALIDATOR_SHA" =~ ^[0-9a-f]{64}$ \ + && "$EXPECTED_BUILD_PREFIX" =~ ^[0-9a-f]{12}$ \ + && "$EXPECTED_MIGRATION_SET_SHA" =~ ^[0-9a-f]{64}$ ]] \ + || fail "rendered_digest_invalid" +[[ -f "$CONFIG_FILE" && ! -L "$CONFIG_FILE" \ + && "$(stat -c %u "$CONFIG_FILE")" == "0" ]] || fail "receiver_config_invalid" +[[ -f "$LIB_DIR/restore-db.sh" && ! -L "$LIB_DIR/restore-db.sh" \ + && -f "$LIB_DIR/restore-storage.sh" && ! -L "$LIB_DIR/restore-storage.sh" \ + && -f "$LIB_DIR/validate-attachment-restore.py" \ + && ! -L "$LIB_DIR/validate-attachment-restore.py" ]] \ + || fail "restore_library_missing" +[[ "$(sha256sum "$LIB_DIR/restore-db.sh" | awk '{print $1}')" == "$EXPECTED_DB_RESTORE_SHA" ]] \ + || fail "db_restore_script_hash_mismatch" +[[ "$(sha256sum "$LIB_DIR/restore-storage.sh" | awk '{print $1}')" == "$EXPECTED_STORAGE_RESTORE_SHA" ]] \ + || fail "storage_restore_script_hash_mismatch" +[[ "$(sha256sum "$LIB_DIR/validate-attachment-restore.py" | awk '{print $1}')" == "$EXPECTED_ATTACHMENT_VALIDATOR_SHA" ]] \ + || fail "attachment_validator_hash_mismatch" + +# Root-owned receiver configuration contains path-only assignments. +# shellcheck disable=SC1090 +source "$CONFIG_FILE" +BACKUP_ARCHIVE_DIR="${BACKUP_ARCHIVE_DIR:-}" +[[ "$BACKUP_ARCHIVE_DIR" == /* && "$BACKUP_ARCHIVE_DIR" != "/" \ + && -d "$BACKUP_ARCHIVE_DIR" && ! -L "$BACKUP_ARCHIVE_DIR" ]] \ + || fail "archive_root_invalid" + +PHASE="monitor_preflight" +deadman_enabled_before="$(systemctl is-enabled "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)" +deadman_active_before="$(systemctl is-active "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)" +health_enabled_before="$(systemctl is-enabled "$HEALTH_TIMER_UNIT" 2>/dev/null || true)" +health_active_before="$(systemctl is-active "$HEALTH_TIMER_UNIT" 2>/dev/null || true)" +[[ "$deadman_enabled_before" == "enabled" && "$deadman_active_before" == "active" \ + && "$health_enabled_before" == "enabled" && "$health_active_before" == "active" ]] \ + || fail "monitor_preflight_not_ready" + +validate_batch() { + local batch_dir="$1" + local identity_mode="$2" + local expected_run_id="${3:-}" + python3 - "$batch_dir" "$EXPECTED_BUILD_PREFIX" "$identity_mode" \ + "$expected_run_id" "$ARCHIVE_BATCH_PREFIX" "$RUN_ID_PREFIX" <<'PY' +import datetime as dt +import hashlib +import json +import os +import re +import stat +import sys + +( + batch_dir, + build_prefix, + identity_mode, + expected_run_id, + archive_batch_prefix, + run_id_prefix, +) = sys.argv[1:] +if identity_mode not in {"source", "copy"}: + raise SystemExit("identity_mode") +if not os.path.isdir(batch_dir) or os.path.islink(batch_dir): + raise SystemExit("batch_type") +entries = set(os.listdir(batch_dir)) +expected = {"database.dump", "storage.tar.gz", "manifest.json"} +if entries != expected: + raise SystemExit("batch_files") +for name in expected: + path = os.path.join(batch_dir, name) + mode = os.lstat(path).st_mode + if not stat.S_ISREG(mode) or stat.S_ISLNK(mode): + raise SystemExit("artifact_type") +with open(os.path.join(batch_dir, "manifest.json"), encoding="utf-8") as handle: + manifest = json.load(handle) +run_id = manifest.get("runId") +if manifest.get("schemaVersion") != 1 or manifest.get("status") != "complete": + raise SystemExit("manifest_contract") +run_id_pattern = re.escape(run_id_prefix) + r"[0-9]{8}T[0-9]{6}Z" +if not isinstance(run_id, str) or not re.fullmatch(run_id_pattern, run_id): + raise SystemExit("run_id") +if identity_mode == "source": + if expected_run_id: + raise SystemExit("unexpected_expected_run_id") + if os.path.basename(batch_dir) != archive_batch_prefix + run_id: + raise SystemExit("source_batch_identity") +else: + if not re.fullmatch(run_id_pattern, expected_run_id): + raise SystemExit("expected_run_id") + if run_id != expected_run_id: + raise SystemExit("copied_batch_source_identity") +build = manifest.get("buildSha") +if not isinstance(build, str) or not re.fullmatch(r"[0-9a-f]{40,64}", build): + raise SystemExit("build_identity") +if not build.startswith(build_prefix): + raise SystemExit("build_mismatch") +created = manifest.get("createdAt") +try: + created_at = dt.datetime.strptime(created, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=dt.timezone.utc) +except Exception as exc: + raise SystemExit("created_at") from exc +artifacts = manifest.get("artifacts") or {} +total_bytes = 0 +for key, name in (("database", "database.dump"), ("storage", "storage.tar.gz")): + meta = artifacts.get(key) or {} + if meta.get("file") != name or os.path.isabs(str(meta.get("file") or "")): + raise SystemExit("artifact_name") + path = os.path.join(batch_dir, name) + size = os.path.getsize(path) + if size != meta.get("bytes"): + raise SystemExit("artifact_size") + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() != meta.get("sha256"): + raise SystemExit("artifact_hash") + total_bytes += size +print( + int(created_at.timestamp()), + total_bytes, + build[:8], + int(bool(manifest.get("sourceImageDigest"))), + run_id, +) +PY +} + +PHASE="select_latest_batch" +selected_batch="$(python3 - "$BACKUP_ARCHIVE_DIR" \ + "$ARCHIVE_BATCH_PREFIX" "$RUN_ID_PREFIX" <<'PY' +import os +import re +import sys + +root, archive_batch_prefix, run_id_prefix = sys.argv[1:] +batch_pattern = re.escape(archive_batch_prefix + run_id_prefix) + r"[0-9]{8}T[0-9]{6}Z" +candidates = [] +for entry in os.scandir(root): + if entry.is_symlink() or not entry.is_dir(follow_symlinks=False): + continue + if not re.fullmatch(batch_pattern, entry.name): + continue + candidates.append((entry.name, entry.path)) +if not candidates: + raise SystemExit(1) +candidates.sort(key=lambda item: item[0]) +print(candidates[-1][1]) +PY +)" || fail "latest_batch_not_found" + +batch_metadata="$(validate_batch "$selected_batch" source "")" || fail "latest_batch_invalid" +read -r created_epoch batch_total_bytes build_short source_digest_present SOURCE_RUN_ID <<<"$batch_metadata" +drill_started_epoch="$(date -u +%s)" +RPO_SECONDS="$((drill_started_epoch - created_epoch))" +[[ "$RPO_SECONDS" -ge 0 && "$RPO_SECONDS" -le 86400 ]] || fail "latest_batch_outside_rpo" +required_bytes="$((batch_total_bytes * 5 + 2147483648))" +available_kib="$(df -Pk "$WORK_ROOT" | awk 'NR==2 {print $4}')" +[[ "$available_kib" =~ ^[0-9]+$ ]] || fail "disk_capacity_unavailable" +available_bytes="$((available_kib * 1024))" +[[ "$available_bytes" -ge "$required_bytes" ]] || fail "disk_capacity_insufficient" + +PHASE="create_isolated_runtime" +run_id="$(date -u +%Y%m%dT%H%M%SZ)-$$-$RANDOM" +[[ "$run_id" =~ ^[0-9TZ-]+$ ]] || fail "run_id_invalid" +WORK_DIR="$WORK_ROOT/$WORK_PREFIX$run_id" +CONTAINER_NAME="$CONTAINER_PREFIX$run_id" +[[ ! -e "$WORK_DIR" ]] || fail "work_dir_exists" +mkdir -m 0700 "$WORK_DIR" +private_log="$WORK_DIR/private.log" +: >"$private_log" +chmod 0600 "$private_log" +rto_started_epoch="$(date -u +%s)" +cp -a -- "$selected_batch" "$WORK_DIR/batch" +copied_metadata="$(validate_batch "$WORK_DIR/batch" copy "$SOURCE_RUN_ID")" \ + || fail "copied_batch_invalid" +[[ "$copied_metadata" == "$batch_metadata" ]] || fail "copied_batch_identity_mismatch" + +env_file="$WORK_DIR/postgres.env" +postgres_password="$(openssl rand -hex 24)" +{ + echo "POSTGRES_USER=$SHADOW_DB_USER" + echo "POSTGRES_PASSWORD=$postgres_password" + echo "POSTGRES_DB=$SHADOW_DB_NAME" +} >"$env_file" +chmod 0600 "$env_file" +printf '%s\n' '[stage=postgres_container_start]' >>"$private_log" +docker run -d --rm --name "$CONTAINER_NAME" --network none --env-file "$env_file" \ + "$POSTGRES_IMAGE" >>"$private_log" 2>&1 || fail "postgres_container_start_failed" +for _ in $(seq 1 60); do + if docker exec "$CONTAINER_NAME" pg_isready -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ + >/dev/null 2>&1; then + break + fi + sleep 1 +done +docker exec "$CONTAINER_NAME" pg_isready -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ + >/dev/null 2>&1 || fail "postgres_not_ready" + +PHASE="restore_database" +DUMP_LIST_STATUS="FAIL" +RESTORE_EXECUTION_PATH="DOCKER_STDIN" +printf '%s\n' '[stage=dump_list]' >>"$private_log" +set +e +docker exec -i "$CONTAINER_NAME" pg_restore --list \ + <"$WORK_DIR/batch/database.dump" >>"$private_log" 2>&1 +DUMP_LIST_EXIT_CODE="$?" +set -e +if [[ "$DUMP_LIST_EXIT_CODE" -ne 0 ]]; then + RESTORE_ERROR_CATEGORY="DUMP_LIST_NONZERO" + fail "dump_list_failed" +fi +DUMP_LIST_STATUS="PASS" + +TARGET_CONFIRMATION_STATUS="FAIL" +printf '%s\n' '[stage=target_confirmation]' >>"$private_log" +set +e +actual_database="$(PGPASSWORD="$postgres_password" docker exec -e PGPASSWORD \ + "$CONTAINER_NAME" psql -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ + -Atqc 'SELECT current_database()' 2>>"$private_log")" +target_confirmation_exit_code="$?" +set -e +if [[ "$target_confirmation_exit_code" -ne 0 \ + || "$actual_database" != "$SHADOW_DB_NAME" ]]; then + RESTORE_ERROR_CATEGORY="TARGET_CONFIRMATION_FAILED" + fail "restore_target_confirmation_failed" +fi +TARGET_CONFIRMATION_STATUS="PASS" + +real_docker_path="$(command -v docker)" +docker_proxy_dir="$WORK_DIR/docker-proxy" +docker_proxy="$docker_proxy_dir/docker" +restore_diag_file="$WORK_DIR/restore-diag" +mkdir -m 0700 "$docker_proxy_dir" +cat >"$docker_proxy" <<'PROXY' +#!/usr/bin/env bash +set +e +"$RESTORE_REAL_DOCKER" "$@" +command_exit="$?" +is_pg_restore=0 +is_list=0 +for argument in "$@"; do + [[ "$argument" == "pg_restore" ]] && is_pg_restore=1 + [[ "$argument" == "--list" ]] && is_list=1 +done +if [[ "$is_pg_restore" -eq 1 ]]; then + if [[ "$is_list" -eq 1 ]]; then + printf 'script_dump_list_exit_code=%s\n' "$command_exit" >>"$RESTORE_DIAG_FILE" + else + printf 'restore_apply_exit_code=%s\n' "$command_exit" >>"$RESTORE_DIAG_FILE" + fi +fi +exit "$command_exit" +PROXY +chmod 0700 "$docker_proxy" +: >"$restore_diag_file" +chmod 0600 "$restore_diag_file" + +RESTORE_APPLY_STATUS="FAIL" +printf '%s\n' '[stage=restore_apply]' >>"$private_log" +set +e +RESTORE_REAL_DOCKER="$real_docker_path" \ +RESTORE_DIAG_FILE="$restore_diag_file" \ +PATH="$docker_proxy_dir:$PATH" \ +RESTORE_SCOPE=shadow \ + RESTORE_CONFIRM_DATABASE="$SHADOW_DB_NAME" \ + DB_TOOL_MODE=docker \ + DB_CONTAINER="$CONTAINER_NAME" \ + DATABASE_URL="postgresql://shadow.invalid/$SHADOW_DB_NAME" \ + POSTGRES_DB="$SHADOW_DB_NAME" \ + POSTGRES_USER="$SHADOW_DB_USER" \ + POSTGRES_PASSWORD="$postgres_password" \ + "$LIB_DIR/restore-db.sh" "$WORK_DIR/batch/database.dump" \ + >>"$private_log" 2>&1 +RESTORE_SCRIPT_EXIT_CODE="$?" +set -e + +script_dump_list_exit_code="$(awk -F= \ + '$1 == "script_dump_list_exit_code" {print $2}' "$restore_diag_file")" +apply_exit_code="$(awk -F= \ + '$1 == "restore_apply_exit_code" {print $2}' "$restore_diag_file")" +if [[ ! "$script_dump_list_exit_code" =~ ^[0-9]+$ \ + || "$script_dump_list_exit_code" -ne 0 ]]; then + RESTORE_ERROR_CATEGORY="SCRIPT_DUMP_LIST_NONZERO" + fail "restore_script_list_failed" +fi +if [[ ! "$apply_exit_code" =~ ^[0-9]+$ ]]; then + RESTORE_ERROR_CATEGORY="APPLY_EXIT_CODE_MISSING" + fail "restore_diagnostic_missing" +fi +PG_RESTORE_EXIT_CODE="$apply_exit_code" +if [[ "$PG_RESTORE_EXIT_CODE" -eq 0 ]]; then + PG_RESTORE_ERROR_COUNT="0" +else + PG_RESTORE_ERROR_COUNT="$(sed -nE \ + 's/^pg_restore: (warning: )?errors ignored on restore: ([0-9]+)$/\2/p' \ + "$private_log" | tail -n 1)" + [[ "$PG_RESTORE_ERROR_COUNT" =~ ^[0-9]+$ ]] \ + || PG_RESTORE_ERROR_COUNT="UNKNOWN" +fi +if [[ "$RESTORE_SCRIPT_EXIT_CODE" -ne 0 || "$PG_RESTORE_EXIT_CODE" -ne 0 ]]; then + RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO" + fail "restore_apply_failed_after_list_pass" +fi +RESTORE_APPLY_STATUS="PASS" +RESTORE_ERROR_CATEGORY="NONE" + +PHASE="verify_database" +migration_names="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \ + "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ + -At <<'SQL' | LC_ALL=C sort +SELECT migration_name +FROM "_prisma_migrations" +WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL +ORDER BY migration_name; +SQL +)" || fail "migration_name_query_failed" +[[ -n "$migration_names" ]] || fail "migration_set_empty" +if printf '%s\n' "$migration_names" | grep -Ev '^[0-9]{14}_[a-z0-9_]+$' >/dev/null; then + fail "migration_name_invalid" +fi +MIGRATION_SUCCESS="$(printf '%s\n' "$migration_names" | wc -l | tr -d ' ')" +MIGRATION_OTHER="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \ + "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ + -At <<'SQL' +SELECT count(*) +FROM "_prisma_migrations" +WHERE finished_at IS NULL OR rolled_back_at IS NOT NULL; +SQL +)" || fail "migration_state_query_failed" +[[ "$MIGRATION_OTHER" == "0" ]] || fail "migration_state_mismatch" +migration_set_sha="$(printf '%s\n' "$migration_names" | sha256sum | awk '{print $1}')" +[[ "$migration_set_sha" == "$EXPECTED_MIGRATION_SET_SHA" ]] \ + || fail "migration_set_mismatch" +MIGRATION_SET_MATCH="PASS" + +TABLE_COUNTS="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \ + "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ + -At -F = <<'SQL' | LC_ALL=C sort +SELECT key, value FROM (VALUES + ('departments', (SELECT count(*)::bigint FROM departments)), + ('employees', (SELECT count(*)::bigint FROM employees)), + ('employee_roles', (SELECT count(*)::bigint FROM employee_roles)), + ('purchase_requests', (SELECT count(*)::bigint FROM purchase_requests)), + ('reimbursement_forms', (SELECT count(*)::bigint FROM reimbursement_forms)), + ('invoice_pool', (SELECT count(*)::bigint FROM invoice_pool)), + ('payment_batches', (SELECT count(*)::bigint FROM payment_batches)), + ('contract_registry', (SELECT count(*)::bigint FROM contract_registry)), + ('projects', (SELECT count(*)::bigint FROM projects)), + ('purchase_attachments', (SELECT count(*)::bigint FROM purchase_attachments)), + ('reimbursement_attachments', (SELECT count(*)::bigint FROM reimbursement_attachments)), + ('payment_batch_receipt_attachments', (SELECT count(*)::bigint FROM payment_batch_receipt_attachments)) +) AS counts(key, value); +SQL +)" || fail "table_count_query_failed" +[[ "$(printf '%s\n' "$TABLE_COUNTS" | sed '/^$/d' | wc -l | tr -d ' ')" == "12" ]] \ + || fail "table_count_key_mismatch" + +PHASE="restore_storage" +restored_storage="$WORK_DIR/restored-storage" +printf '%s\n' '[stage=restore_storage]' >>"$private_log" +if ! RESTORE_SCOPE=shadow "$LIB_DIR/restore-storage.sh" \ + "$WORK_DIR/batch/storage.tar.gz" "$restored_storage" \ + >>"$private_log" 2>&1; then + fail "storage_restore_failed" +fi +ARCHIVE_FILE_COUNT="$(tar -tvzf "$WORK_DIR/batch/storage.tar.gz" \ + | awk 'substr($0,1,1)=="-" {count++} END {print count+0}')" +RESTORED_FILE_COUNT="$(find "$restored_storage" -type f -printf '.' | wc -c | tr -d ' ')" +[[ "$ARCHIVE_FILE_COUNT" == "$RESTORED_FILE_COUNT" ]] || fail "storage_file_count_mismatch" + +PHASE="verify_attachment_references" +attachment_result="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \ + "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ + -At -F $'\t' <<'SQL' | python3 "$LIB_DIR/validate-attachment-restore.py" "$restored_storage" +SELECT encode(convert_to(storage_path, 'UTF8'), 'base64'), file_size::text, coalesce(file_sha256, '') FROM invoice_pool +UNION ALL +SELECT encode(convert_to(storage_path, 'UTF8'), 'base64'), file_size::text, coalesce(file_sha256, '') FROM purchase_attachments +UNION ALL +SELECT encode(convert_to(storage_path, 'UTF8'), 'base64'), file_size::text, coalesce(file_sha256, '') FROM reimbursement_attachments +UNION ALL +SELECT encode(convert_to(storage_path, 'UTF8'), 'base64'), file_size::text, coalesce(file_sha256, '') FROM payment_batch_receipt_attachments +UNION ALL +SELECT encode(convert_to(attachment_storage_path, 'UTF8'), 'base64'), '', '' FROM leave_requests WHERE attachment_storage_path IS NOT NULL; +SQL +)" || fail "attachment_reference_query_failed" +read -r ATTACHMENT_TOTAL ATTACHMENT_MISMATCH attachment_s3_count <<<"$attachment_result" +[[ "$ATTACHMENT_TOTAL" =~ ^[0-9]+$ && "$ATTACHMENT_MISMATCH" == "0" \ + && "$attachment_s3_count" == "0" ]] || fail "attachment_reference_mismatch" + +PHASE="post_validation" +copied_metadata_after="$(validate_batch "$WORK_DIR/batch" copy "$SOURCE_RUN_ID")" \ + || fail "copied_batch_postcheck_failed" +source_metadata_after="$(validate_batch "$selected_batch" source "")" \ + || fail "source_archive_postcheck_failed" +[[ "$copied_metadata_after" == "$batch_metadata" \ + && "$source_metadata_after" == "$batch_metadata" ]] || fail "archive_identity_changed" +rto_finished_epoch="$(date -u +%s)" +RTO_SECONDS="$((rto_finished_epoch - rto_started_epoch))" + +deadman_enabled_after="$(systemctl is-enabled "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)" +deadman_active_after="$(systemctl is-active "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)" +health_enabled_after="$(systemctl is-enabled "$HEALTH_TIMER_UNIT" 2>/dev/null || true)" +health_active_after="$(systemctl is-active "$HEALTH_TIMER_UNIT" 2>/dev/null || true)" +[[ "$deadman_enabled_after" == "$deadman_enabled_before" \ + && "$deadman_active_after" == "$deadman_active_before" \ + && "$health_enabled_after" == "$health_enabled_before" \ + && "$health_active_after" == "$health_active_before" ]] \ + || fail "monitor_state_changed" + +PHASE="runtime_cleanup" +cleanup_runtime || fail "runtime_cleanup_failed" +RUNTIME_CLEANED=1 +RESULT_STATUS="GO" +PHASE="complete" +exit 0 diff --git a/scripts/qa/run-offhost-shadow-restore-runner-contract.sh b/scripts/qa/run-offhost-shadow-restore-runner-contract.sh new file mode 100755 index 00000000..fc8e6caf --- /dev/null +++ b/scripts/qa/run-offhost-shadow-restore-runner-contract.sh @@ -0,0 +1,725 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUNNER_TEMPLATE="$ROOT_DIR/scripts/qa/offhost-shadow-restore-runner.template.sh" +ATTACHMENT_VALIDATOR="$ROOT_DIR/scripts/qa/validate-offhost-shadow-restore-attachments.py" +RESTORE_DB="$ROOT_DIR/scripts/restore-db.sh" +RESTORE_STORAGE="$ROOT_DIR/scripts/restore-storage.sh" +TEST_DIR="$(mktemp -d)" +RUNNER="$TEST_DIR/rendered-runner.sh" +PRIVATE_REPO_TEST_DIR="$ROOT_DIR/.restore-drill-private/contract-$$-$RANDOM" +UNIGNORED_CONFIG="$ROOT_DIR/.restore-drill-private-contract-$$-$RANDOM.json" +UNIGNORED_OUTPUT="$ROOT_DIR/.restore-drill-private-contract-$$-$RANDOM.sh" +CONTAINER_NAME="$(basename "$0" .sh)-$RANDOM-$$" +POSTGRES_USER="fixture_user_$RANDOM" +POSTGRES_PASSWORD="$(openssl rand -hex 16)" +POSTGRES_DB="fixture_restore_drill_$RANDOM" + +cleanup() { + [[ -z "$CONTAINER_NAME" ]] \ + || docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true + rm -rf -- "$TEST_DIR" + rm -rf -- "$PRIVATE_REPO_TEST_DIR" + rm -f -- "$UNIGNORED_CONFIG" "$UNIGNORED_OUTPUT" +} +trap cleanup EXIT HUP INT TERM + +fail() { + echo "[runner-contract][FAIL] $1" >&2 + exit 1 +} + +pass() { + echo "[runner-contract][PASS] $1" +} + +require_ignored_if_inside_repo() { + local candidate_path="$1" + local resolved_parent resolved_path relative_path + resolved_parent="$(realpath -e "$(dirname "$candidate_path")")" \ + || return 1 + resolved_path="$resolved_parent/$(basename "$candidate_path")" + case "$resolved_path" in + "$ROOT_DIR"/*) + relative_path="${resolved_path#"$ROOT_DIR"/}" + git -C "$ROOT_DIR" check-ignore -q --no-index -- "$relative_path" \ + || return 1 + ;; + esac +} + +render_runner() { + local config_file="$1" + local output_file="$2" + require_ignored_if_inside_repo "$config_file" || return 1 + require_ignored_if_inside_repo "$output_file" || return 1 + python3 - "$RUNNER_TEMPLATE" "$config_file" "$output_file" <<'PY' +import json +import os +import pathlib +import re +import sys + +template_path, config_path, output_path = map(pathlib.Path, sys.argv[1:]) +template = template_path.read_text(encoding="utf-8") +config = json.loads(config_path.read_text(encoding="utf-8")) +tokens = set(re.findall(r"@@([A-Z0-9_]+)@@", template)) +if set(config) != tokens: + raise SystemExit("render_config_keys") +path_keys = { + "RECEIVER_CONFIG_FILE", + "RESTORE_LIB_DIR", + "SELF_PATH", + "SUDOERS_PATH", + "SUDOERS_MAIN_FILE", + "WORK_ROOT", +} +prefix_keys = {"WORK_PREFIX", "CONTAINER_PREFIX", "ARCHIVE_BATCH_PREFIX", "RUN_ID_PREFIX"} +sha_keys = { + "EXPECTED_DB_RESTORE_SHA", + "EXPECTED_STORAGE_RESTORE_SHA", + "EXPECTED_ATTACHMENT_VALIDATOR_SHA", + "EXPECTED_MIGRATION_SET_SHA", +} +for key, value in config.items(): + if not isinstance(value, str) or not value or "\n" in value or "\r" in value: + raise SystemExit("render_config_value") + if key in path_keys: + parsed = pathlib.PurePosixPath(value) + if not parsed.is_absolute() or any(part in {"", ".", ".."} for part in parsed.parts[1:]): + raise SystemExit("render_path_value") + if not re.fullmatch(r"/[A-Za-z0-9_./-]+", value): + raise SystemExit("render_path_charset") + elif key in prefix_keys and not re.fullmatch(r"[a-z0-9][a-z0-9-]*-", value): + raise SystemExit("render_prefix_value") + elif key == "SHADOW_DB_USER" and not re.fullmatch(r"[a-z_][a-z0-9_]*", value): + raise SystemExit("render_db_user") + elif key == "SHADOW_DB_NAME" and ( + not re.fullmatch(r"[a-z_][a-z0-9_]*", value) + or not re.search(r"(^|_)(shadow|drill|restore_test)(_|$)", value) + ): + raise SystemExit("render_db_name") + elif key == "POSTGRES_IMAGE" and not re.fullmatch(r"[a-z0-9._/-]+:[A-Za-z0-9._-]+", value): + raise SystemExit("render_postgres_image") + elif key in {"DEADMAN_TIMER_UNIT", "HEALTH_TIMER_UNIT"} and not re.fullmatch( + r"[A-Za-z0-9_.@-]+\.timer", value + ): + raise SystemExit("render_timer_unit") + elif key in sha_keys and not re.fullmatch(r"[0-9a-f]{64}", value): + raise SystemExit("render_sha") + elif key == "EXPECTED_BUILD_PREFIX" and not re.fullmatch(r"[0-9a-f]{12}", value): + raise SystemExit("render_build_prefix") + template = template.replace(f"@@{key}@@", value) +if re.search(r"@@[A-Z0-9_]+@@", template): + raise SystemExit("render_placeholder_remaining") +descriptor = os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o700) +with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(template) +PY +} + +if [[ "${1:-}" == "render" ]]; then + [[ "$#" -eq 3 ]] || fail "usage: $0 render " + render_runner "$2" "$3" || fail "private render paths must be outside the repository or ignored" + bash -n "$3" + candidate_sha="$(sha256sum "$3" | awk '{print $1}')" + echo "[runner-contract][PASS] rendered candidate syntax" + echo "candidate_sha256=$candidate_sha" + exit 0 +fi +[[ "$#" -eq 0 ]] || fail "usage: $0 [render ]" + +db_restore_sha="$(sha256sum "$RESTORE_DB" | awk '{print $1}')" +storage_restore_sha="$(sha256sum "$RESTORE_STORAGE" | awk '{print $1}')" +attachment_validator_sha="$(sha256sum "$ATTACHMENT_VALIDATOR" | awk '{print $1}')" +migration_fixture_names=$'20260101000000_fixture_one\n20260102000000_fixture_two' +migration_fixture_sha="$(printf '%s\n' "$migration_fixture_names" | sha256sum | awk '{print $1}')" +python3 - "$TEST_DIR/render-config.json" "$db_restore_sha" \ + "$storage_restore_sha" "$attachment_validator_sha" "$migration_fixture_sha" <<'PY' +import json +import pathlib +import sys + +output, db_sha, storage_sha, validator_sha, migration_sha = sys.argv[1:] +payload = { + "RECEIVER_CONFIG_FILE": "/synthetic/receiver.conf", + "RESTORE_LIB_DIR": "/synthetic/restore-lib", + "SELF_PATH": "/synthetic/restore-entry", + "SUDOERS_PATH": "/synthetic/restore-rule", + "SUDOERS_MAIN_FILE": "/synthetic/sudoers", + "WORK_ROOT": "/tmp", + "WORK_PREFIX": "restore-contract-work-", + "CONTAINER_PREFIX": "restore-contract-container-", + "SHADOW_DB_USER": "fixture_user", + "SHADOW_DB_NAME": "fixture_restore_drill", + "POSTGRES_IMAGE": "postgres:16-alpine", + "DEADMAN_TIMER_UNIT": "fixture-deadman.timer", + "HEALTH_TIMER_UNIT": "fixture-health.timer", + "ARCHIVE_BATCH_PREFIX": "fixture-backup-", + "RUN_ID_PREFIX": "auto-", + "EXPECTED_DB_RESTORE_SHA": db_sha, + "EXPECTED_STORAGE_RESTORE_SHA": storage_sha, + "EXPECTED_ATTACHMENT_VALIDATOR_SHA": validator_sha, + "EXPECTED_BUILD_PREFIX": "a" * 12, + "EXPECTED_MIGRATION_SET_SHA": migration_sha, +} +pathlib.Path(output).write_text(json.dumps(payload), encoding="utf-8") +PY + +cp "$TEST_DIR/render-config.json" "$UNIGNORED_CONFIG" +if render_runner "$UNIGNORED_CONFIG" "$TEST_DIR/unignored-config-output.sh" \ + >/dev/null 2>&1; then + fail "renderer accepted an unignored in-repository private config" +fi +[[ ! -e "$TEST_DIR/unignored-config-output.sh" ]] \ + || fail "renderer published output for an unignored private config" +pass "unignored in-repository private config fails before candidate creation" + +if render_runner "$TEST_DIR/render-config.json" "$UNIGNORED_OUTPUT" \ + >/dev/null 2>&1; then + fail "renderer accepted an unignored in-repository candidate path" +fi +[[ ! -e "$UNIGNORED_OUTPUT" ]] \ + || fail "renderer published an unignored in-repository candidate" +pass "unignored in-repository candidate path fails before creation" + +mkdir -m 0700 -p "$PRIVATE_REPO_TEST_DIR" +cp "$TEST_DIR/render-config.json" "$PRIVATE_REPO_TEST_DIR/render-config.json" +render_runner "$PRIVATE_REPO_TEST_DIR/render-config.json" \ + "$PRIVATE_REPO_TEST_DIR/rendered-runner.sh" +[[ -s "$PRIVATE_REPO_TEST_DIR/rendered-runner.sh" ]] \ + || fail "ignored private render path did not produce a candidate" +pass "ignored in-repository private config and candidate paths are accepted" + +render_runner "$TEST_DIR/render-config.json" "$RUNNER" +render_runner "$TEST_DIR/render-config.json" "$TEST_DIR/rendered-runner-repeat.sh" +cmp -s "$RUNNER" "$TEST_DIR/rendered-runner-repeat.sh" \ + || fail "identical private config did not render a deterministic candidate" +pass "identical private config renders a deterministic candidate SHA-256" +python3 - "$TEST_DIR/render-config.json" "$TEST_DIR/unsafe-render-config.json" <<'PY' +import json +import pathlib +import sys + +source, target = map(pathlib.Path, sys.argv[1:]) +payload = json.loads(source.read_text(encoding="utf-8")) +payload["SELF_PATH"] = '/synthetic/entry";unsafe-command' +target.write_text(json.dumps(payload), encoding="utf-8") +PY +if render_runner "$TEST_DIR/unsafe-render-config.json" \ + "$TEST_DIR/unsafe-rendered-runner.sh" >/dev/null 2>&1; then + fail "renderer accepted shell syntax in a runtime path" +fi +[[ ! -e "$TEST_DIR/unsafe-rendered-runner.sh" ]] \ + || fail "renderer published an unsafe partial candidate" +pass "renderer rejects shell syntax before publishing a candidate" + +bash -n "$RUNNER" "$RUNNER_TEMPLATE" "$RESTORE_DB" "$RESTORE_STORAGE" +python3 - "$ATTACHMENT_VALIDATOR" <<'PY' +import pathlib +import sys + +compile(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"), sys.argv[1], "exec") +PY +if rg -n '([0-9]{1,3}[.]){3}[0-9]{1,3}' \ + "$RUNNER_TEMPLATE" "$ATTACHMENT_VALIDATOR" >/dev/null; then + fail "tracked runner assets contain a site-specific value" +fi +python3 - "$0" <<'PY' +import ipaddress +import pathlib +import re +import sys + +text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +for raw in re.findall(r"(?/dev/null; then + fail "tracked runner assets contain the local account name" +fi +pass "tracked template renders one syntax-valid candidate without site-specific values" + +python3 - "$RUNNER_TEMPLATE" <<'PY' +import pathlib +import re +import sys + +text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +truncating = [ + line.strip() + for line in text.splitlines() + if re.search(r"(?)>\"\$private_log\"", line) +] +if truncating != [': >"$private_log"']: + raise SystemExit("private_log_truncation_contract") +required_markers = { + "[stage=postgres_container_start]", + "[stage=dump_list]", + "[stage=target_confirmation]", + "[stage=restore_apply]", + "[stage=restore_storage]", +} +missing = {marker for marker in required_markers if marker not in text} +if missing: + raise SystemExit("private_log_stage_markers_missing") +PY +pass "private raw log is initialized once and appends fixed stage markers" + +python3 - "$RUNNER" "$TEST_DIR/validate-batch.py" <<'PY' +import pathlib +import re +import sys + +runner_path, output_path = map(pathlib.Path, sys.argv[1:]) +text = runner_path.read_text(encoding="utf-8") +function_text = text.split("validate_batch() {", 1)[1] +match = re.search(r"<<'PY'\n(?P.*?)\nPY\n\}", function_text, re.S) +if not match: + raise SystemExit("validate_batch_body_missing") +output_path.write_text(match.group("body") + "\n", encoding="utf-8") +PY + +python3 - "$TEST_DIR/batches" <<'PY' +import hashlib +import json +import pathlib +import shutil +import sys + +root = pathlib.Path(sys.argv[1]) +run_id = "auto-20260825T000000Z" +source = root / f"fixture-backup-{run_id}" +source.mkdir(parents=True) +(source / "database.dump").write_bytes(b"database-fixture") +(source / "storage.tar.gz").write_bytes(b"storage-fixture") + +def artifact(name): + data = (source / name).read_bytes() + return {"file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()} + +manifest = { + "schemaVersion": 1, + "status": "complete", + "runId": run_id, + "createdAt": "2026-08-25T00:00:00Z", + "buildSha": "a" * 40, + "sourceImageDigest": "sha256:" + "c" * 64, + "artifacts": { + "database": artifact("database.dump"), + "storage": artifact("storage.tar.gz"), + }, +} +(source / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8") +shutil.copytree(source, root / "private-copy") +shutil.copytree(source, root / "wrong-source-name") +PY + +source_batch="$TEST_DIR/batches/fixture-backup-auto-20260825T000000Z" +copy_batch="$TEST_DIR/batches/private-copy" +validator_args=("$(printf 'a%.0s' {1..12})" "fixture-backup-" "auto-") +source_metadata="$(python3 "$TEST_DIR/validate-batch.py" "$source_batch" \ + "${validator_args[0]}" source '' "${validator_args[1]}" "${validator_args[2]}")" +copy_metadata="$(python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \ + "${validator_args[0]}" copy auto-20260825T000000Z \ + "${validator_args[1]}" "${validator_args[2]}")" +[[ "$source_metadata" == "$copy_metadata" ]] \ + || fail "source and renamed copy metadata differ" +pass "source identity and renamed private copy both pass with source binding" + +if python3 "$TEST_DIR/validate-batch.py" \ + "$TEST_DIR/batches/wrong-source-name" "${validator_args[0]}" source '' \ + "${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then + fail "source basename/run-id mismatch passed" +fi +pass "source basename/run-id mismatch fails closed" + +printf 'tamper' >>"$copy_batch/storage.tar.gz" +if python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \ + "${validator_args[0]}" copy auto-20260825T000000Z \ + "${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then + fail "copied content tamper passed" +fi +pass "copied content tamper fails closed" + +python3 - "$source_batch/storage.tar.gz" "$copy_batch/storage.tar.gz" \ + "$copy_batch/manifest.json" <<'PY' +import json +import pathlib +import sys + +source_storage, copied_storage, path = map(pathlib.Path, sys.argv[1:]) +copied_storage.write_bytes(source_storage.read_bytes()) +payload = json.loads(path.read_text(encoding="utf-8")) +payload["runId"] = "auto-20260825T010000Z" +path.write_text(json.dumps(payload), encoding="utf-8") +PY +if python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \ + "${validator_args[0]}" copy auto-20260825T000000Z \ + "${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then + fail "copy run-id different from verified source passed" +fi +pass "copy run-id different from verified source fails closed" + +rendered_migration_sha="$(sed -nE \ + 's/^EXPECTED_MIGRATION_SET_SHA="([0-9a-f]{64})"$/\1/p' "$RUNNER")" +[[ "$rendered_migration_sha" == "$migration_fixture_sha" ]] \ + || fail "rendered dynamic migration fingerprint mismatch" +mutated_migration_sha="$(printf '%s\n%s\n' "$migration_fixture_names" \ + '20260103000000_fixture_three' | sha256sum | awk '{print $1}')" +[[ "$mutated_migration_sha" != "$rendered_migration_sha" ]] \ + || fail "migration set mutation was not detected" +if rg -n 'EXPECTED_MIGRATIONS|migration[^\n]*==[[:space:]]*65' "$RUNNER" >/dev/null; then + fail "runner contains a hard-coded migration count" +fi +pass "dynamic migration fingerprint accepts exact set and rejects mutation" + +mkdir -p "$TEST_DIR/attachment-root/nested" +printf 'attachment-fixture' >"$TEST_DIR/attachment-root/nested/file.bin" +attachment_path_b64="$(printf 'nested/file.bin' | base64 -w0)" +attachment_size="$(stat -c %s "$TEST_DIR/attachment-root/nested/file.bin")" +attachment_sha="$(sha256sum "$TEST_DIR/attachment-root/nested/file.bin" | awk '{print $1}')" +attachment_ok="$(printf '%s\t%s\t%s\n' "$attachment_path_b64" \ + "$attachment_size" "$attachment_sha" \ + | python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")" +[[ "$attachment_ok" == "1 0 0" ]] \ + || fail "attachment validator rejected a valid local reference" +unsafe_path_b64="$(printf '../escape.bin' | base64 -w0)" +attachment_unsafe="$(printf '%s\t\t\n' "$unsafe_path_b64" \ + | python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")" +[[ "$attachment_unsafe" == "1 1 0" ]] \ + || fail "attachment validator accepted path traversal" +s3_path_b64="$(printf 's3:external-object' | base64 -w0)" +attachment_s3="$(printf '%s\t\t\n' "$s3_path_b64" \ + | python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")" +[[ "$attachment_s3" == "1 1 1" ]] \ + || fail "attachment validator accepted an external object reference" +pass "attachment validator accepts exact local hash and rejects traversal/S3" + +python3 - "$RUNNER" <<'PY' +import pathlib +import re +import sys + +text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") +finish = re.search(r"\nfinish\(\) \{(?P.*?)\n\}\ntrap finish", text, re.S) +if not finish: + raise SystemExit("finish function not found") +body = finish.group("body") +emit_position = body.find("\n emit_restore_diagnostics") +cleanup_position = body.find("\n cleanup_runtime ||") +if emit_position == -1: + raise SystemExit("diagnostic emission missing") +if cleanup_position == -1: + raise SystemExit("runtime cleanup call missing") +if emit_position > cleanup_position: + raise SystemExit("diagnostics are emitted after cleanup") +required = { + 'echo "dump_list=$DUMP_LIST_STATUS"', + 'echo "target_confirmation=$TARGET_CONFIRMATION_STATUS"', + 'echo "restore_execution_path=$RESTORE_EXECUTION_PATH"', + 'echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE"', + 'echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE"', + 'echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT"', + 'echo "restore_error_category=$RESTORE_ERROR_CATEGORY"', +} +missing = sorted(value for value in required if value not in text) +if missing: + raise SystemExit(f"diagnostic fields missing: {missing}") +if 'RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"' not in text: + raise SystemExit("non-zero apply classification missing") +PY +pass "diagnostics precede finally-equivalent cleanup and use allowlisted fields" + +docker run -d --rm --name "$CONTAINER_NAME" -p 127.0.0.1::5432 \ + -e "POSTGRES_USER=$POSTGRES_USER" \ + -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \ + -e "POSTGRES_DB=$POSTGRES_DB" \ + postgres:16-alpine >/dev/null + +ready_streak=0 +for _ in $(seq 1 60); do + if docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ + -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atqc 'SELECT 1' \ + >/dev/null 2>&1; then + ready_streak="$((ready_streak + 1))" + [[ "$ready_streak" -ge 2 ]] && break + else + ready_streak=0 + fi + sleep 1 +done +[[ "$ready_streak" -ge 2 ]] \ + || fail "PostgreSQL 16 fixture did not become stably ready" + +HOST_PORT="$(docker port "$CONTAINER_NAME" 5432/tcp \ + | awk -F: 'NR == 1 {print $NF}')" +[[ "$HOST_PORT" =~ ^[0-9]+$ ]] \ + || fail "loopback PostgreSQL 16 port unavailable" +HOST_DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${HOST_PORT}/${POSTGRES_DB}" + +docker exec -i "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ + -U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' >/dev/null +CREATE TABLE restore_contract_marker (value text NOT NULL); +INSERT INTO restore_contract_marker(value) VALUES ('direct-real-script'); +SQL +docker exec "$CONTAINER_NAME" pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + --format=c --no-owner --no-acl >"$TEST_DIR/fixture.dump" +docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ + -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -c 'DROP TABLE restore_contract_marker' >/dev/null + +DOCKER_DATABASE_URL="postgresql://shadow.invalid/$POSTGRES_DB" + +DB_TOOL_MODE=docker \ +DB_CONTAINER="$CONTAINER_NAME" \ +POSTGRES_DB="$POSTGRES_DB" \ +POSTGRES_USER="$POSTGRES_USER" \ +POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ +DATABASE_URL="$DOCKER_DATABASE_URL" \ +RESTORE_SCOPE=shadow \ +RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ + "$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null +[[ "$(docker exec "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -Atqc 'SELECT value FROM restore_contract_marker')" \ + == "direct-real-script" ]] || fail "Docker stdin restore result mismatch" +pass "real restore-db.sh Docker stdin path restores a PostgreSQL 16 custom dump" + +head -c 32 "$TEST_DIR/fixture.dump" >"$TEST_DIR/corrupt.dump" +if DB_TOOL_MODE=docker \ + DB_CONTAINER="$CONTAINER_NAME" \ + POSTGRES_DB="$POSTGRES_DB" \ + POSTGRES_USER="$POSTGRES_USER" \ + POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + DATABASE_URL="$DOCKER_DATABASE_URL" \ + RESTORE_SCOPE=shadow \ + RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ + "$RESTORE_DB" "$TEST_DIR/corrupt.dump" >/dev/null 2>&1; then + fail "corrupt custom dump passed list validation" +fi +pass "real restore-db.sh rejects a dump-list failure" + +if DB_TOOL_MODE=docker \ + DB_CONTAINER="$CONTAINER_NAME" \ + POSTGRES_DB="$POSTGRES_DB" \ + POSTGRES_USER="$POSTGRES_USER" \ + POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + DATABASE_URL="$DOCKER_DATABASE_URL" \ + RESTORE_SCOPE=shadow \ + RESTORE_CONFIRM_DATABASE=wrong_restore_drill \ + "$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null 2>&1; then + fail "wrong target confirmation was accepted" +fi +pass "real restore-db.sh rejects target confirmation mismatch" + +REAL_DOCKER="$(command -v docker)" +mkdir -m 0700 "$TEST_DIR/proxy" +cat >"$TEST_DIR/proxy/docker" <<'PROXY' +#!/usr/bin/env bash +set +e +is_pg_restore=0 +is_list=0 +for argument in "$@"; do + [[ "$argument" == "pg_restore" ]] && is_pg_restore=1 + [[ "$argument" == "--list" ]] && is_list=1 +done +if [[ "$is_pg_restore" -eq 1 && "$is_list" -eq 0 ]]; then + cat >/dev/null + exit 23 +fi +"$REAL_DOCKER" "$@" +exit "$?" +PROXY +chmod 0700 "$TEST_DIR/proxy/docker" + +set +e +proxy_output="$(REAL_DOCKER="$REAL_DOCKER" \ + PATH="$TEST_DIR/proxy:$PATH" \ + DB_TOOL_MODE=docker \ + DB_CONTAINER="$CONTAINER_NAME" \ + POSTGRES_DB="$POSTGRES_DB" \ + POSTGRES_USER="$POSTGRES_USER" \ + POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + DATABASE_URL="$DOCKER_DATABASE_URL" \ + RESTORE_SCOPE=shadow \ + RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ + "$RESTORE_DB" "$TEST_DIR/fixture.dump" 2>&1)" +proxy_exit="$?" +set -e +[[ "$proxy_exit" -eq 23 ]] || fail "apply failure exit code was not preserved" +[[ "$proxy_output" != *"$POSTGRES_PASSWORD"* \ + && "$proxy_output" != *"$DOCKER_DATABASE_URL"* ]] \ + || fail "apply failure output exposed fixture credentials" +pass "real restore-db.sh preserves an injected apply exit and exposes no fixture secret" + +mkdir -m 0700 "$TEST_DIR/host-tools" +cat >"$TEST_DIR/host-tools/psql" <<'HOST_PSQL' +#!/usr/bin/env bash +exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \ + postgres:16-alpine psql "$@" +HOST_PSQL +cat >"$TEST_DIR/host-tools/pg_restore" <<'HOST_RESTORE' +#!/usr/bin/env bash +exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \ + -v "$HOST_TOOL_MOUNT:$HOST_TOOL_MOUNT:ro" \ + postgres:16-alpine pg_restore "$@" +HOST_RESTORE +chmod 0700 "$TEST_DIR/host-tools/psql" "$TEST_DIR/host-tools/pg_restore" +export PGPASSWORD="$POSTGRES_PASSWORD" REAL_DOCKER HOST_TOOL_MOUNT="$TEST_DIR" +for _ in $(seq 1 60); do + if "$TEST_DIR/host-tools/psql" "$HOST_DATABASE_URL" \ + -Atqc 'SELECT 1' >/dev/null 2>&1; then + break + fi + sleep 1 +done +"$TEST_DIR/host-tools/psql" "$HOST_DATABASE_URL" \ + -Atqc 'SELECT 1' >/dev/null 2>&1 \ + || fail "host-path TCP fixture did not become ready" +docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ + -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -c 'DROP TABLE restore_contract_marker' >/dev/null +PATH="$TEST_DIR/host-tools:$PATH" \ +DB_TOOL_MODE=host \ +DATABASE_URL="$HOST_DATABASE_URL" \ +RESTORE_SCOPE=shadow \ +RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \ + "$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null +[[ "$(docker exec "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -Atqc 'SELECT value FROM restore_contract_marker')" == "direct-real-script" ]] \ + || fail "host-file restore result mismatch" +pass "real restore-db.sh host-file path restores through PostgreSQL 16 tools" + +docker rm -f "$CONTAINER_NAME" >/dev/null +if docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then + fail "fixture container cleanup failed" +fi +CONTAINER_NAME="" + +sed -n '1,/^trap finish EXIT HUP INT TERM$/p' "$RUNNER" \ + | sed '$d' >"$TEST_DIR/runner-finalization-functions.sh" + +capture_finalization_case() { + local output_file="$1" + local failure_category="$2" + local dump_status="$3" + local dump_exit="$4" + local target_status="$5" + local apply_status="$6" + local script_exit="$7" + local pg_restore_exit="$8" + local error_count="$9" + local error_category="${10}" + set +e + ( + # shellcheck disable=SC1090 + source "$TEST_DIR/runner-finalization-functions.sh" + cleanup_runtime() { + echo "cleanup_marker=runtime" + return 0 + } + cleanup_access() { + echo "cleanup_marker=access" + return 0 + } + FAILURE_CATEGORY="$failure_category" + DUMP_LIST_STATUS="$dump_status" + DUMP_LIST_EXIT_CODE="$dump_exit" + TARGET_CONFIRMATION_STATUS="$target_status" + RESTORE_EXECUTION_PATH="DOCKER_STDIN" + RESTORE_APPLY_STATUS="$apply_status" + RESTORE_SCRIPT_EXIT_CODE="$script_exit" + PG_RESTORE_EXIT_CODE="$pg_restore_exit" + PG_RESTORE_ERROR_COUNT="$error_count" + RESTORE_ERROR_CATEGORY="$error_category" + trap finish EXIT + exit 23 + ) >"$output_file" 2>&1 + local case_exit="$?" + set -e + [[ "$case_exit" -eq 23 ]] \ + || fail "finalization did not preserve the injected exit code" +} + +capture_finalization_case \ + "$TEST_DIR/captured-dump-list-failure" \ + dump_list_failed FAIL 1 NOT_RUN NOT_RUN NOT_RUN NOT_RUN NOT_RUN \ + DUMP_LIST_NONZERO +grep -Fxq 'dump_list=FAIL' "$TEST_DIR/captured-dump-list-failure" \ + || fail "dump-list failure stage missing" +grep -Fxq 'target_confirmation=NOT_RUN' \ + "$TEST_DIR/captured-dump-list-failure" \ + || fail "target stage ran after dump-list failure" +grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-dump-list-failure" \ + || fail "apply stage ran after dump-list failure" +pass "dump-list failure keeps target and apply stages NOT_RUN" + +capture_finalization_case \ + "$TEST_DIR/captured-target-failure" \ + restore_target_confirmation_failed PASS 0 FAIL NOT_RUN NOT_RUN NOT_RUN \ + NOT_RUN TARGET_CONFIRMATION_FAILED +grep -Fxq 'target_confirmation=FAIL' "$TEST_DIR/captured-target-failure" \ + || fail "target-confirmation failure stage missing" +grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-target-failure" \ + || fail "apply stage ran after target-confirmation failure" +pass "target-confirmation failure keeps apply stage NOT_RUN" + +set +e +( + # shellcheck disable=SC1090 + source "$TEST_DIR/runner-finalization-functions.sh" + cleanup_runtime() { + echo "cleanup_marker=runtime" + return 0 + } + cleanup_access() { + echo "cleanup_marker=access" + return 0 + } + FAILURE_CATEGORY="restore_apply_failed_after_list_pass" + DUMP_LIST_STATUS="PASS" + DUMP_LIST_EXIT_CODE="0" + TARGET_CONFIRMATION_STATUS="PASS" + RESTORE_EXECUTION_PATH="DOCKER_STDIN" + RESTORE_APPLY_STATUS="FAIL" + RESTORE_SCRIPT_EXIT_CODE="23" + PG_RESTORE_EXIT_CODE="23" + PG_RESTORE_ERROR_COUNT="UNKNOWN" + RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO" + trap finish EXIT + exit 23 +) >"$TEST_DIR/captured-finalization" 2>&1 +finalization_exit="$?" +set -e +[[ "$finalization_exit" -eq 23 ]] \ + || fail "finalization did not preserve the injected exit code" +diagnostic_line="$(grep -n '^restore_apply=FAIL$' \ + "$TEST_DIR/captured-finalization" | cut -d: -f1)" +cleanup_line="$(grep -n '^cleanup_marker=runtime$' \ + "$TEST_DIR/captured-finalization" | cut -d: -f1)" +[[ "$diagnostic_line" =~ ^[0-9]+$ && "$cleanup_line" =~ ^[0-9]+$ \ + && "$diagnostic_line" -lt "$cleanup_line" ]] \ + || fail "stage diagnostics were not emitted before teardown" +grep -Fxq 'runtime_cleanup=PASS' "$TEST_DIR/captured-finalization" \ + || fail "runtime teardown result was not retained" +grep -Fxq 'temporary_access_cleanup=PASS' "$TEST_DIR/captured-finalization" \ + || fail "access teardown result was not retained" +grep -Fxq 'pg_restore_exit_code=23' "$TEST_DIR/captured-finalization" \ + || fail "pg_restore exit code disappeared after teardown" +pass "injected failure emits readable stages before finally-equivalent teardown" + +if grep -Eiq 'postgresql://|SELECT[[:space:]]|/tmp/|local_fixture_password' \ + "$TEST_DIR/captured-dump-list-failure" \ + "$TEST_DIR/captured-target-failure" \ + "$TEST_DIR/captured-finalization"; then + fail "allowlisted diagnostics exposed a path, credential or SQL" +fi +pass "captured stage diagnostics contain no path, credential or SQL" + +echo "[runner-contract][PASS] all off-host shadow-restore runner contracts completed" diff --git a/scripts/qa/validate-offhost-shadow-restore-attachments.py b/scripts/qa/validate-offhost-shadow-restore-attachments.py new file mode 100755 index 00000000..abc38e8a --- /dev/null +++ b/scripts/qa/validate-offhost-shadow-restore-attachments.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import base64 +import hashlib +import os +import pathlib +import sys + + +def main() -> int: + if len(sys.argv) != 2: + return 2 + + root = os.path.realpath(sys.argv[1]) + total = 0 + mismatch = 0 + s3_count = 0 + + for raw in sys.stdin: + raw = raw.rstrip("\n") + if not raw: + continue + parts = raw.split("\t") + if len(parts) != 3: + mismatch += 1 + continue + + total += 1 + try: + storage_path = base64.b64decode(parts[0], validate=True).decode("utf-8") + except (ValueError, UnicodeDecodeError): + mismatch += 1 + continue + + if storage_path.startswith("s3:"): + s3_count += 1 + mismatch += 1 + continue + + pure_path = pathlib.PurePosixPath(storage_path) + if not storage_path or pure_path.is_absolute() or ".." in pure_path.parts: + mismatch += 1 + continue + + target = os.path.realpath(os.path.join(root, *pure_path.parts)) + if not target.startswith(root + os.sep) or not os.path.isfile(target): + mismatch += 1 + continue + + expected_size, expected_sha = parts[1], parts[2].lower() + if expected_size and str(os.path.getsize(target)) != expected_size: + mismatch += 1 + continue + + if expected_sha: + digest = hashlib.sha256() + with open(target, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() != expected_sha: + mismatch += 1 + + print(total, mismatch, s3_count) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())