diff --git a/.trellis/spec/hyb-oa/deployment-trialrun.md b/.trellis/spec/hyb-oa/deployment-trialrun.md index 030f80d0..85bbd9b1 100644 --- a/.trellis/spec/hyb-oa/deployment-trialrun.md +++ b/.trellis/spec/hyb-oa/deployment-trialrun.md @@ -1322,7 +1322,9 @@ validate_copied_batch(private/batch, source_metadata) - Required stages: `dump_list`, `target_confirmation`, `restore_apply`, each with the enum `PASS|FAIL|NOT_RUN`. - Required execution fields: `restore_execution_path=DOCKER_STDIN|HOST_FILE|NOT_RUN`, - `dump_list_exit_code`, `restore_script_exit_code`, + `dump_list_exit_code`, `target_confirmation_exit_code`, + `target_name_match=yes|no|NOT_RUN`, `target_confirmation_attempt_count`, + `target_confirmation_success_streak`, `restore_script_exit_code`, `pg_restore_exit_code`, `pg_restore_error_count` and `restore_error_category`. - Exit-code fields are non-negative integers or `NOT_RUN`; @@ -1347,6 +1349,18 @@ validate_copied_batch(private/batch, source_metadata) environment values or stderr. - Run `dump_list` and target confirmation independently before the existing restore script. A failed stage leaves every later stage `NOT_RUN`. +- For the official PostgreSQL image, target readiness must use a real query over + container-local TCP loopback (`psql -h 127.0.0.1`), not default Unix-socket + `pg_isready`. The initdb temporary server listens only on its socket, while + the final server accepts loopback TCP even under `--network none`. Require + two consecutive exact `current_database()` matches inside a bounded retry + window as additional stability evidence; a successful wrong name fails + immediately and a non-zero query resets the streak. +- Define target confirmation in the rendered runner and make the contract + extract or directly drive that same implementation. A fixture-only wait loop + that stabilizes PostgreSQL before testing a different runner path is + forbidden: it can make the contract pass while the deployed candidate still + contains the startup race. - Raw restore output remains mode `0600`, never leaves the off-host node and is deleted with the owned runtime on success and failure. - Any non-zero pg_restore apply exit is `FAIL/NO_GO`, even when a numeric @@ -1383,9 +1397,12 @@ validate_copied_batch(private/batch, source_metadata) - Dump list non-zero -> `dump_list=FAIL`, later stages `NOT_RUN`, category `DUMP_LIST_NONZERO` / `dump_list_failed`. -- Target query non-zero or exact shadow name mismatch -> - `target_confirmation=FAIL`, apply `NOT_RUN`, category - `TARGET_CONFIRMATION_FAILED`. +- Bounded TCP target queries remain non-zero -> `target_confirmation=FAIL`, + apply `NOT_RUN`, category `TARGET_QUERY_NONZERO`; preserve the last numeric + exit, attempt count and zero success streak. +- Target query succeeds but the exact shadow name differs -> fail immediately + with `target_name_match=no`, category `TARGET_NAME_MISMATCH`; do not retry a + confirmed wrong target. - Independent list and target pass, but the restore script never invokes apply -> category `APPLY_EXIT_CODE_MISSING`; fail closed. - Apply exit non-zero -> `restore_apply=FAIL`, preserve the exact numeric exit @@ -1415,11 +1432,16 @@ validate_copied_batch(private/batch, source_metadata) reports zero residue without exposing the raw error. - Base: all three stages pass with exit code `0`; the runner continues to migration and attachment verification. +- Good: Unix-socket availability alone does not pass; loopback TCP becomes + available, two exact target queries succeed consecutively, then apply runs. - Bad: emit only `database_restore_failed` after deleting the sole private log. - Bad: parse and publish arbitrary pg_restore lines, or treat a non-zero exit as success because the restored database appears partly populated. - Bad: commit one host's concrete runner or reconstruct a supposedly equivalent runner from chat/workspace immediately before an exercise. +- Bad: add two successful queries only to the fixture while the rendered runner + still accepts the first socket `pg_isready`, or use a fixed sleep as an ARM + timing workaround. ### 6. Tests Required @@ -1428,6 +1450,11 @@ validate_copied_batch(private/batch, source_metadata) with an equivalent test implementation. - Inject list failure, target mismatch and list-pass/apply-failure. Assert exact stage enums, later-stage `NOT_RUN`, execution path and numeric exit code. +- Extract the target-confirmation function from the rendered runner and drive + it with: socket-only availability/TCP failure, first-query recovery, streak + reset, exact-name mismatch and permanent bounded failure. Assert every probe + contains `-h 127.0.0.1`, name mismatch is not retried, and the real + PostgreSQL 16 fixture uses this same function before both restore paths. - Inject a non-zero apply exit and assert the diagnostic is emitted before teardown, teardown passes, and the calling test can still read the fixed fields afterwards. @@ -1455,6 +1482,8 @@ validate_copied_batch(private/batch, source_metadata) #### Wrong ```text +fixture: wait for two queries +runner: first socket pg_isready -> continue restore-db > private.log || fail database_restore_failed finally: delete private runtime print result=NO_GO @@ -1464,7 +1493,7 @@ print result=NO_GO ```text tracked template + exact private config -> deterministic candidate + SHA-256 -contract PASS -> list -> target confirmation -> real restore path +contract drives rendered TCP target function -> list -> target -> real restore path capture only stage/path/exit/category enums emit fixed diagnostics -> finally teardown raw private runtime non-zero apply remains FAIL diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/design.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/design.md index e5d528c1..639ba7a6 100644 --- a/.trellis/tasks/08-23-isolated-backup-restore-drill/design.md +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/design.md @@ -120,13 +120,15 @@ Attempt 4 在 `dump_list=PASS/0` 后、实际 restore apply 前以 `restore_targ 1. 删除 runner 的“首次 `pg_isready` 成功即通过”判定。`pg_isready` 最多可作为诊断,不能成为允许 restore 的充分条件。 2. `dump_list` 保持不连接目标数据库的独立阶段;其成功只证明 custom dump 目录可解析,不计入数据库稳定就绪次数。 -3. `target_confirmation` 在最多 60 次、每次间隔 1 秒的有界窗口内执行真实 `SELECT current_database()`。每次查询退出码非零时把连续成功计数重置为 `0`;查询成功且数据库名精确等于本轮 shadow 目标时累计,只有连续两次成功才 `PASS`;查询成功但名称不匹配立即失败,不继续重试。 -4. 脱敏输出增加 `target_confirmation_exit_code`、`target_name_match=yes|no|NOT_RUN`、`target_confirmation_attempt_count` 和 `target_confirmation_success_streak`。退出码取最后一次实际查询的非负整数;未执行为 `NOT_RUN`。错误类别区分 `TARGET_QUERY_NONZERO` 与 `TARGET_NAME_MISMATCH`,不得输出查询文本、数据库名、路径、凭据或原始 stderr。 -5. 稳定就绪函数必须定义在被渲染 runner 中;合同从 rendered runner 提取或直接驱动该同一实现,不得在 fixture 外围另写一套更宽松或更稳定的等待逻辑。fixture 自身如需等待,也必须通过同一实现进入后续真实脚本双路径回归。 -6. 回归至少覆盖:首次查询失败后连续两次成功可继续;一次成功后失败会重置 streak;数据库名错配立即失败;查询持续非零在第 60 次后失败;每条失败路径的 apply 保持 `NOT_RUN`、诊断先于清理且残留为 `0`;原有 dump、restore 双路径、身份、migration、附件和敏感输出合同继续全绿。 -7. 禁止用固定长 sleep、增加 ARM 特例、降低连续成功次数、跳过名称精确比较或只修改合同夹具来修复。若相同实现的本地合同通过而 ARM 仍在该阶段失败,保留环境差异 finding,依据新退出码和尝试计数继续调查,不现场放宽。 +3. `target_confirmation` 在容器内显式通过 TCP 回环 `127.0.0.1` 执行真实 `SELECT current_database()`,不用默认 Unix socket。官方 PostgreSQL 镜像的 initdb 临时服务器不监听 TCP,而正式服务器监听容器内回环;因此 TCP 成功是排除临时初始化服务器的结构性判据。`--network none` 不影响容器自己的 loopback,实际 restore 仍可继续沿现有 socket 路径,不修改 `scripts/restore-db.sh`。 +4. TCP 目标确认使用最多 60 次、每次间隔 1 秒的有界窗口作为额外保险。每次查询退出码非零时把连续成功计数重置为 `0`;查询成功且数据库名精确等于本轮 shadow 目标时累计,只有连续两次成功才 `PASS`;查询成功但名称不匹配立即失败,不继续重试。连续成功不是排除临时服务器的唯一依据,不能移除 TCP 回环判据后单独保留时序启发式。 +5. 脱敏输出增加 `target_confirmation_exit_code`、`target_name_match=yes|no|NOT_RUN`、`target_confirmation_attempt_count` 和 `target_confirmation_success_streak`。退出码取最后一次实际查询的非负整数;未执行为 `NOT_RUN`。错误类别区分 `TARGET_QUERY_NONZERO` 与 `TARGET_NAME_MISMATCH`,不得输出查询文本、数据库名、路径、凭据或原始 stderr。 +6. 稳定就绪函数必须定义在被渲染 runner 中;合同从 rendered runner 提取或直接驱动该同一实现,不得在 fixture 外围另写一套更宽松或更稳定的等待逻辑。fixture 自身如需等待,也必须通过同一实现进入后续真实脚本双路径回归。 +7. 实施前审计 runner 合同的每个覆盖机制,区分“直接执行跟踪脚本/校验器”“从 rendered runner 提取同一实现”“只做静态结构断言”和“合同自行重写等价逻辑”。最后一类必须消除或登记为明确限制,不能以组件测试通过推断未执行的 runner 编排行为。 +8. 回归至少覆盖:Unix socket 可用但 TCP 仍失败时不得通过;首次 TCP 查询失败后连续两次成功可继续;一次成功后失败会重置 streak;数据库名错配立即失败;查询持续非零在第 60 次后失败;每条失败路径的 apply 保持 `NOT_RUN`、诊断先于清理且残留为 `0`;原有 dump、restore 双路径、身份、migration、附件和敏感输出合同继续全绿。 +9. 禁止用固定长 sleep、增加 ARM 特例、降低连续成功次数、跳过 TCP 回环或名称精确比较、只修改合同夹具来修复。若相同实现的本地合同通过而 ARM 仍在该阶段失败,保留环境差异 finding,依据新退出码和尝试计数继续调查,不现场放宽。 -本节只是待审设计,不构成修改 runner、连接 ARM 或执行 Attempt 5 的授权。修复通过后仍须提交可追溯变更、重新选择当时最新批次、重跑只读身份探针、重新渲染并核对候选 SHA-256,再单独申请 Attempt 5 实际恢复。 +本节已于 2026-08-25 获准仅在本地实施和回归;授权不包含连接 ARM 或执行 Attempt 5。修复通过后仍须提交可追溯变更、重新选择当时最新批次、重跑只读身份探针、重新渲染并核对候选 SHA-256,再单独申请 Attempt 5 实际恢复。 ## 清理与回滚 diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md index 9ab553e1..6a8518bf 100644 --- a/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md @@ -111,17 +111,18 @@ - [x] Attempt 4 runtime、临时 root 入口和普通用户上传暂存均清理为 `PASS/0`;按预先裁决规则转入本地合同路径与 ARM 实际环境的只读差异调查,不直接修补 runner 或创建 Attempt 5。 - [x] 只读差异矩阵与本地精确复现收敛到 PostgreSQL 初始化就绪竞态:合同夹具等待连续两次真实查询,runner 仅等待首次 `pg_isready`;本地稳定数据库上的同一 target query 为 `PASS/0`,ARM 所有临时对象复核残留 `0`。 - [x] Attempt 4 后正式归档只读探针与执行前为同一构建和同一 manifest 脱敏摘要,run id 配对、产物 SHA-256、24 小时新鲜度均通过;探针入口与暂存残留 `0`,正式归档内容级零修改已补证。 -- [ ] 演练后腾讯云生产 `/ready` 尚未独立补证;须记录正常状态和 12 位构建前缀,不能由 ARM timer、归档探针或清理 PASS 代替。 -- [ ] 修复与任何 Attempt 5 尚未批准;先审阅 `design.md` 的“PostgreSQL 稳定就绪修订提案”,仅批准后才开始以下本地工作。 +- [x] 演练后腾讯云生产 `/ready` 已由执行方沿既有只读路径补证:状态 `PASS`、12 位构建前缀 `7b86ffb3c9ef`,与演练前一致;未记录地址或原始响应。 +- [x] 用户已批准仅在本地实施和回归 PostgreSQL 稳定就绪修复;该批准不包含连接 ARM 或执行 Attempt 5。 -### Attempt 5 前仅本地修复计划(待批准) +### Attempt 5 前仅本地修复结果 -1. 在跟踪 runner 中以最多 60 次、间隔 1 秒、连续两次真实 `SELECT current_database()` 成功且名称精确匹配,替换首次 `pg_isready` 即通过逻辑;名称错配立即失败,查询非零重置连续成功计数。 -2. 增加 target query 退出码、名称匹配、尝试次数和连续成功次数的 allowlist 诊断,区分查询非零与名称错配;私有原始日志、失败先诊断后清理和零残留边界不变。 -3. 让可复跑合同直接驱动 rendered runner 中同一稳定就绪实现;删除 fixture 外围与 runner 不同源的稳定化旁路。 -4. 补齐首次失败后恢复、streak 重置、名称错配立即失败、永久查询失败有界退出四类回归,并复跑真实 `restore-db.sh` Docker stdin/host-file、源/副本身份、dynamic migration、附件安全、敏感输出和清理合同。 -5. 确认 `scripts/restore-db.sh` 字节未改,运行 task、Trellis、process、docs、敏感扫描和空白门禁;通过后以中文提交引用本 task。 -6. 本地修复阶段不连接 ARM、不生成或安装 Attempt 5 现场入口。提交后重新选择最新批次并执行只读探针、重新渲染候选和核对 SHA-256;实际恢复须再次单独批准。 +- [x] 在跟踪 runner 中以容器内 TCP 回环 `127.0.0.1` 的真实 `SELECT current_database()` 作为结构性就绪判据,并以最多 60 次、间隔 1 秒、连续两次成功且名称精确匹配作为额外保险,替换首次 Unix socket `pg_isready` 即通过逻辑;名称错配立即失败,查询非零重置连续成功计数。 +- [x] 增加 target query 退出码、名称匹配、尝试次数和连续成功次数的 allowlist 诊断,区分查询非零与名称错配;私有原始日志、失败先诊断后清理和零残留边界不变。 +- [x] 可复跑合同直接提取并驱动 rendered runner 中同一稳定就绪函数;fixture 外围与 runner 不同源的等待循环已删除。 +- [x] “socket 可用但 TCP 不可用仍失败”、首次失败后恢复、streak 重置、名称错配立即失败、永久查询失败有界退出五类回归通过;真实 `restore-db.sh` Docker stdin/host-file、源/副本身份、dynamic migration、附件安全、敏感输出和清理合同继续通过。 +- [x] 已审计合同全部覆盖机制,未发现第二套自写等价恢复逻辑;完整 runner 的 migration/12 类表/附件编排仍只能由真实隔离恢复验证,已作为明确限制保留,不能由本地组件合同替代。 +- [x] `scripts/restore-db.sh` 相对生产构建字节未改;runner、成对备份/恢复、自动异机备份 74 项、task、Trellis、process、docs、敏感扫描和空白门禁均通过。 +- [ ] 本地修复阶段不连接 ARM、不生成或安装 Attempt 5 现场入口。提交后重新选择最新批次并执行只读探针、重新渲染候选和核对 SHA-256;实际恢复须再次单独批准。 ## 验证命令类别 diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md index b2386372..33676bd8 100644 --- a/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md @@ -400,3 +400,30 @@ Attempt 4 的 runner 产生过程已从个人 workspace 人工产物改为“跟 Attempt 4 前后探针的最新构建短标识、run id 配对、产物完整性和 manifest 脱敏摘要前缀一致,后置探针仍在 24 小时内通过。由此正式归档的内容级零修改已取得现场只读补证;该结论仅覆盖正式归档,不替代恢复成功证明,也不回答最新备份是否可恢复。 腾讯云生产演练后 `/ready` 仍未独立补证。ARM 的 timer active、清理 PASS 和归档探针均不能推导生产在线状态;在取得 `/ready` 正常状态及 12 位构建前缀前,AC2 保持未满足,证据不得表述为生产前后均正常。 + +## Attempt 4 后生产只读补证与本地稳定就绪修复 + +日期:2026-08-25 + +执行方沿既有审计窄路径完成腾讯云生产演练后 `/ready` 只读核验:状态 `PASS`,12 位构建前缀 `7b86ffb3c9ef`,与演练前记录一致。地址和原始响应未写入本证据;该读取没有登录、写库、重启、切换或配置修改。至此 Attempt 4 的生产前后 `/ready` 均正常已补证,但不改变本轮 restore apply 未执行和最新备份可恢复性仍未知的结论。 + +用户随后批准的范围仅为本地 runner 修复与回归,不连接 ARM、不执行 Attempt 5。修复把 PostgreSQL 就绪从默认 Unix socket 的首次 `pg_isready` 改为容器内 TCP 回环 `127.0.0.1` 的真实 `SELECT current_database()`:initdb 临时服务器不监听 TCP,因此该判据结构性排除临时服务器;连续两次成功和最多 60 次有界重试只作为额外稳定保险。目标名称成功但不匹配立即失败,查询非零重置 streak;实际 `restore-db.sh` 仍走原路径且字节不改。 + +合同夹具与 runner 分叉审计结果: + +| 覆盖对象 | 合同机制 | 是否存在等价逻辑分叉 | +| --- | --- | --- | +| 私有配置渲染与候选 SHA | 直接使用跟踪模板和合同内唯一渲染器 | 否 | +| 正式源/临时副本身份 | 从 rendered runner 提取 `validate_batch` 同一实现 | 否 | +| PostgreSQL 目标就绪 | 从 rendered runner 提取同一 TCP 目标确认函数;真实 PostgreSQL 16 fixture 也调用该函数 | 原分叉已消除 | +| DB restore Docker stdin/host-file | 直接执行仓库 `scripts/restore-db.sh` | 否 | +| 附件引用安全 | 直接执行跟踪附件校验器 | 否 | +| 失败诊断与 finally 清理 | 从 rendered runner 提取初始化和 finalization 同一实现 | 否 | +| migration 指纹 | 对 rendered 值、目录集合变化和禁止硬编码做静态/合成断言 | 无等价重写;真实恢复集合仍待远端演练 | +| 12 类表、storage 与完整编排 | 本地只做组件合同,不执行含真实备份的完整 runner | 明确覆盖限制,不能推断远端成功 | + +就绪回归全部 `PASS`:socket 路径假定可用但 TCP 永久失败仍在第 60 次失败;首次查询失败后连续两次成功在第 3 次通过;一次成功后失败会清零 streak 并在第 4 次通过;目标名不匹配第 1 次立即失败;永久查询非零以 `TARGET_QUERY_NONZERO` 失败。真实 PostgreSQL 16 fixture 使用 extracted rendered-runner 函数就绪后,`restore-db.sh` Docker stdin 与 host-file 两条路径均成功;dump-list、目标错配、apply 非零、诊断先于清理、敏感输出负例继续通过。本地测试容器残留 `0`。 + +完整复跑结果:runner 合同全部 `PASS`,既有成对备份/恢复演练 `PASS`,自动异机备份合同 `74/74 PASS`;task validate、Trellis governance、process baseline、默认/定向敏感扫描和 `git diff --check` 均通过。文档审计错误 `0`,仅保留既有未登记文件 warning;`scripts/restore-db.sh` 相对生产构建无字节漂移,测试容器和生成缓存残留均为 `0`。 + +本地合同通过只证明演练器修复与组件路径,不回答最新正式备份是否可恢复。下一步仍须先提交可追溯变更,再动态选择最新正式批次、执行只读身份探针并重新渲染候选;Attempt 5 实际恢复必须另行批准。 diff --git a/scripts/qa/offhost-shadow-restore-runner.template.sh b/scripts/qa/offhost-shadow-restore-runner.template.sh index 648127d0..0a3be16d 100644 --- a/scripts/qa/offhost-shadow-restore-runner.template.sh +++ b/scripts/qa/offhost-shadow-restore-runner.template.sh @@ -44,6 +44,11 @@ TARGET_CONFIRMATION_STATUS="NOT_RUN" RESTORE_APPLY_STATUS="NOT_RUN" RESTORE_EXECUTION_PATH="NOT_RUN" DUMP_LIST_EXIT_CODE="NOT_RUN" +TARGET_CONFIRMATION_EXIT_CODE="NOT_RUN" +TARGET_NAME_MATCH="NOT_RUN" +TARGET_CONFIRMATION_ATTEMPT_COUNT="0" +TARGET_CONFIRMATION_SUCCESS_STREAK="0" +TARGET_CONFIRMATION_FAILURE_REASON="NOT_APPLICABLE" RESTORE_SCRIPT_EXIT_CODE="NOT_RUN" PG_RESTORE_EXIT_CODE="NOT_RUN" PG_RESTORE_ERROR_COUNT="NOT_RUN" @@ -61,6 +66,47 @@ require_command() { command -v "$1" >/dev/null 2>&1 || fail "required_command_missing" } +confirm_postgres_target() { + local actual_database="" + local query_exit_code=0 + local attempt=0 + TARGET_CONFIRMATION_EXIT_CODE="NOT_RUN" + TARGET_NAME_MATCH="NOT_RUN" + TARGET_CONFIRMATION_ATTEMPT_COUNT="0" + TARGET_CONFIRMATION_SUCCESS_STREAK="0" + TARGET_CONFIRMATION_FAILURE_REASON="TARGET_QUERY_NONZERO" + for attempt in $(seq 1 60); do + TARGET_CONFIRMATION_ATTEMPT_COUNT="$attempt" + set +e + actual_database="$(PGPASSWORD="$postgres_password" docker exec -e PGPASSWORD \ + "$CONTAINER_NAME" psql -h 127.0.0.1 -v ON_ERROR_STOP=1 \ + -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ + -Atqc 'SELECT current_database()' 2>>"$private_log")" + query_exit_code="$?" + set -e + TARGET_CONFIRMATION_EXIT_CODE="$query_exit_code" + if [[ "$query_exit_code" -ne 0 ]]; then + TARGET_NAME_MATCH="NOT_RUN" + TARGET_CONFIRMATION_SUCCESS_STREAK="0" + [[ "$attempt" -eq 60 ]] || sleep 1 + continue + fi + if [[ "$actual_database" != "$SHADOW_DB_NAME" ]]; then + TARGET_NAME_MATCH="no" + TARGET_CONFIRMATION_FAILURE_REASON="TARGET_NAME_MISMATCH" + return 1 + fi + TARGET_NAME_MATCH="yes" + TARGET_CONFIRMATION_SUCCESS_STREAK="$((TARGET_CONFIRMATION_SUCCESS_STREAK + 1))" + if [[ "$TARGET_CONFIRMATION_SUCCESS_STREAK" -ge 2 ]]; then + TARGET_CONFIRMATION_FAILURE_REASON="NONE" + return 0 + fi + sleep 1 + done + return 1 +} + cleanup_runtime() { local cleanup_ok=1 local container_suffix="" @@ -126,6 +172,10 @@ emit_restore_diagnostics() { echo "dump_list=$DUMP_LIST_STATUS" echo "dump_list_exit_code=$DUMP_LIST_EXIT_CODE" echo "target_confirmation=$TARGET_CONFIRMATION_STATUS" + echo "target_confirmation_exit_code=$TARGET_CONFIRMATION_EXIT_CODE" + echo "target_name_match=$TARGET_NAME_MATCH" + echo "target_confirmation_attempt_count=$TARGET_CONFIRMATION_ATTEMPT_COUNT" + echo "target_confirmation_success_streak=$TARGET_CONFIRMATION_SUCCESS_STREAK" echo "restore_execution_path=$RESTORE_EXECUTION_PATH" echo "restore_apply=$RESTORE_APPLY_STATUS" echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE" @@ -170,7 +220,7 @@ finish() { } trap finish EXIT HUP INT TERM -for command_name in docker python3 sha256sum tar find stat df cp openssl visudo systemctl; do +for command_name in docker python3 sha256sum tar find stat df cp openssl visudo systemctl seq; do require_command "$command_name" done @@ -379,15 +429,6 @@ chmod 0600 "$env_file" printf '%s\n' '[stage=postgres_container_start]' >>"$private_log" docker run -d --rm --name "$CONTAINER_NAME" --network none --env-file "$env_file" \ "$POSTGRES_IMAGE" >>"$private_log" 2>&1 || fail "postgres_container_start_failed" -for _ in $(seq 1 60); do - if docker exec "$CONTAINER_NAME" pg_isready -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ - >/dev/null 2>&1; then - break - fi - sleep 1 -done -docker exec "$CONTAINER_NAME" pg_isready -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ - >/dev/null 2>&1 || fail "postgres_not_ready" PHASE="restore_database" DUMP_LIST_STATUS="FAIL" @@ -406,15 +447,8 @@ DUMP_LIST_STATUS="PASS" TARGET_CONFIRMATION_STATUS="FAIL" printf '%s\n' '[stage=target_confirmation]' >>"$private_log" -set +e -actual_database="$(PGPASSWORD="$postgres_password" docker exec -e PGPASSWORD \ - "$CONTAINER_NAME" psql -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \ - -Atqc 'SELECT current_database()' 2>>"$private_log")" -target_confirmation_exit_code="$?" -set -e -if [[ "$target_confirmation_exit_code" -ne 0 \ - || "$actual_database" != "$SHADOW_DB_NAME" ]]; then - RESTORE_ERROR_CATEGORY="TARGET_CONFIRMATION_FAILED" +if ! confirm_postgres_target; then + RESTORE_ERROR_CATEGORY="$TARGET_CONFIRMATION_FAILURE_REASON" fail "restore_target_confirmation_failed" fi TARGET_CONFIRMATION_STATUS="PASS" diff --git a/scripts/qa/run-offhost-shadow-restore-runner-contract.sh b/scripts/qa/run-offhost-shadow-restore-runner-contract.sh index fc8e6caf..618bb81e 100755 --- a/scripts/qa/run-offhost-shadow-restore-runner-contract.sh +++ b/scripts/qa/run-offhost-shadow-restore-runner-contract.sh @@ -136,12 +136,13 @@ attachment_validator_sha="$(sha256sum "$ATTACHMENT_VALIDATOR" | awk '{print $1}' migration_fixture_names=$'20260101000000_fixture_one\n20260102000000_fixture_two' migration_fixture_sha="$(printf '%s\n' "$migration_fixture_names" | sha256sum | awk '{print $1}')" python3 - "$TEST_DIR/render-config.json" "$db_restore_sha" \ - "$storage_restore_sha" "$attachment_validator_sha" "$migration_fixture_sha" <<'PY' + "$storage_restore_sha" "$attachment_validator_sha" "$migration_fixture_sha" \ + "$POSTGRES_USER" "$POSTGRES_DB" <<'PY' import json import pathlib import sys -output, db_sha, storage_sha, validator_sha, migration_sha = sys.argv[1:] +output, db_sha, storage_sha, validator_sha, migration_sha, db_user, db_name = sys.argv[1:] payload = { "RECEIVER_CONFIG_FILE": "/synthetic/receiver.conf", "RESTORE_LIB_DIR": "/synthetic/restore-lib", @@ -151,8 +152,8 @@ payload = { "WORK_ROOT": "/tmp", "WORK_PREFIX": "restore-contract-work-", "CONTAINER_PREFIX": "restore-contract-container-", - "SHADOW_DB_USER": "fixture_user", - "SHADOW_DB_NAME": "fixture_restore_drill", + "SHADOW_DB_USER": db_user, + "SHADOW_DB_NAME": db_name, "POSTGRES_IMAGE": "postgres:16-alpine", "DEADMAN_TIMER_UNIT": "fixture-deadman.timer", "HEALTH_TIMER_UNIT": "fixture-health.timer", @@ -222,21 +223,18 @@ import sys compile(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"), sys.argv[1], "exec") PY -if rg -n '([0-9]{1,3}[.]){3}[0-9]{1,3}' \ - "$RUNNER_TEMPLATE" "$ATTACHMENT_VALIDATOR" >/dev/null; then - fail "tracked runner assets contain a site-specific value" -fi -python3 - "$0" <<'PY' +python3 - "$RUNNER_TEMPLATE" "$ATTACHMENT_VALIDATOR" "$0" <<'PY' import ipaddress import pathlib import re import sys -text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8") -for raw in re.findall(r"(?/dev/null; then + fail "socket-only pg_isready remains in rendered runner" +fi +pass "rendered runner exposes the single TCP target-confirmation implementation" + python3 - "$RUNNER" "$TEST_DIR/validate-batch.py" <<'PY' import pathlib import re @@ -423,6 +443,10 @@ if emit_position > cleanup_position: required = { 'echo "dump_list=$DUMP_LIST_STATUS"', 'echo "target_confirmation=$TARGET_CONFIRMATION_STATUS"', + 'echo "target_confirmation_exit_code=$TARGET_CONFIRMATION_EXIT_CODE"', + 'echo "target_name_match=$TARGET_NAME_MATCH"', + 'echo "target_confirmation_attempt_count=$TARGET_CONFIRMATION_ATTEMPT_COUNT"', + 'echo "target_confirmation_success_streak=$TARGET_CONFIRMATION_SUCCESS_STREAK"', 'echo "restore_execution_path=$RESTORE_EXECUTION_PATH"', 'echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE"', 'echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE"', @@ -437,26 +461,145 @@ if 'RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"' not in text: PY pass "diagnostics precede finally-equivalent cleanup and use allowlisted fields" +cat >"$TEST_DIR/mock-target-docker" <<'MOCK_DOCKER' +#!/usr/bin/env bash +set -euo pipefail +has_tcp_loopback=0 +previous="" +for argument in "$@"; do + if [[ "$previous" == "-h" && "$argument" == "127.0.0.1" ]]; then + has_tcp_loopback=1 + fi + previous="$argument" +done +if [[ "$has_tcp_loopback" -ne 1 ]]; then + printf '%s\n' "$MOCK_EXPECTED_DATABASE" + exit 0 +fi +index="$(cat "$MOCK_QUERY_STATE")" +index="$((index + 1))" +printf '%s\n' "$index" >"$MOCK_QUERY_STATE" +line="$(sed -n "${index}p" "$MOCK_QUERY_SEQUENCE")" +[[ -n "$line" ]] || exit 99 +exit_code="${line%%|*}" +result="${line#*|}" +[[ "$exit_code" =~ ^[0-9]+$ ]] || exit 98 +if [[ "$exit_code" -eq 0 ]]; then + printf '%s\n' "$result" +fi +exit "$exit_code" +MOCK_DOCKER +chmod 0700 "$TEST_DIR/mock-target-docker" + +capture_target_confirmation_case() { + local label="$1" + local sequence="$2" + local output_file="$3" + local sequence_file="$TEST_DIR/$label.sequence" + local state_file="$TEST_DIR/$label.state" + printf '%s' "$sequence" >"$sequence_file" + printf '0\n' >"$state_file" + ( + # shellcheck disable=SC1090 + source "$TEST_DIR/confirm-postgres-target.sh" + docker() { + "$TEST_DIR/mock-target-docker" "$@" + } + sleep() { + : + } + export MOCK_EXPECTED_DATABASE="$POSTGRES_DB" + export MOCK_QUERY_SEQUENCE="$sequence_file" + export MOCK_QUERY_STATE="$state_file" + private_log="$TEST_DIR/$label.private.log" + : >"$private_log" + postgres_password="$POSTGRES_PASSWORD" + CONTAINER_NAME="fixture-container" + SHADOW_DB_USER="$POSTGRES_USER" + SHADOW_DB_NAME="$POSTGRES_DB" + if confirm_postgres_target; then + function_exit=0 + else + function_exit="$?" + fi + echo "function_exit=$function_exit" + echo "target_confirmation_exit_code=$TARGET_CONFIRMATION_EXIT_CODE" + echo "target_name_match=$TARGET_NAME_MATCH" + echo "target_confirmation_attempt_count=$TARGET_CONFIRMATION_ATTEMPT_COUNT" + echo "target_confirmation_success_streak=$TARGET_CONFIRMATION_SUCCESS_STREAK" + echo "target_confirmation_failure_reason=$TARGET_CONFIRMATION_FAILURE_REASON" + ) >"$output_file" 2>&1 +} + +capture_target_confirmation_case recovery \ + "$(printf '1|\n0|%s\n0|%s\n' "$POSTGRES_DB" "$POSTGRES_DB")" \ + "$TEST_DIR/readiness-recovery.out" +grep -Fxq 'function_exit=0' "$TEST_DIR/readiness-recovery.out" \ + || fail "TCP target confirmation did not recover after an initial query failure" +grep -Fxq 'target_confirmation_attempt_count=3' "$TEST_DIR/readiness-recovery.out" \ + || fail "TCP target confirmation recovery attempt count mismatch" +grep -Fxq 'target_confirmation_success_streak=2' "$TEST_DIR/readiness-recovery.out" \ + || fail "TCP target confirmation recovery streak mismatch" +pass "rendered TCP target confirmation recovers only after two consecutive successes" + +capture_target_confirmation_case streak-reset \ + "$(printf '0|%s\n1|\n0|%s\n0|%s\n' \ + "$POSTGRES_DB" "$POSTGRES_DB" "$POSTGRES_DB")" \ + "$TEST_DIR/readiness-streak-reset.out" +grep -Fxq 'function_exit=0' "$TEST_DIR/readiness-streak-reset.out" \ + || fail "TCP target confirmation did not recover after streak reset" +grep -Fxq 'target_confirmation_attempt_count=4' \ + "$TEST_DIR/readiness-streak-reset.out" \ + || fail "query failure did not reset the consecutive-success streak" +pass "rendered TCP target confirmation resets its streak after a query failure" + +capture_target_confirmation_case name-mismatch \ + '0|wrong_restore_drill +' "$TEST_DIR/readiness-name-mismatch.out" +grep -Fxq 'function_exit=1' "$TEST_DIR/readiness-name-mismatch.out" \ + || fail "target-name mismatch did not fail" +grep -Fxq 'target_name_match=no' "$TEST_DIR/readiness-name-mismatch.out" \ + || fail "target-name mismatch diagnostic missing" +grep -Fxq 'target_confirmation_attempt_count=1' \ + "$TEST_DIR/readiness-name-mismatch.out" \ + || fail "target-name mismatch was retried" +grep -Fxq 'target_confirmation_failure_reason=TARGET_NAME_MISMATCH' \ + "$TEST_DIR/readiness-name-mismatch.out" \ + || fail "target-name mismatch category missing" +pass "rendered TCP target confirmation fails immediately on an exact-name mismatch" + +query_failures="" +for _ in $(seq 1 60); do + query_failures+=$'1|\n' +done +capture_target_confirmation_case tcp-unavailable "$query_failures" \ + "$TEST_DIR/readiness-tcp-unavailable.out" +grep -Fxq 'function_exit=1' "$TEST_DIR/readiness-tcp-unavailable.out" \ + || fail "socket-only mock passed without TCP loopback readiness" +grep -Fxq 'target_confirmation_attempt_count=60' \ + "$TEST_DIR/readiness-tcp-unavailable.out" \ + || fail "permanent TCP failure did not stop at the bounded attempt count" +grep -Fxq 'target_confirmation_failure_reason=TARGET_QUERY_NONZERO' \ + "$TEST_DIR/readiness-tcp-unavailable.out" \ + || fail "permanent TCP failure category missing" +pass "socket-only availability cannot bypass the rendered TCP readiness gate" + docker run -d --rm --name "$CONTAINER_NAME" -p 127.0.0.1::5432 \ -e "POSTGRES_USER=$POSTGRES_USER" \ -e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \ -e "POSTGRES_DB=$POSTGRES_DB" \ postgres:16-alpine >/dev/null -ready_streak=0 -for _ in $(seq 1 60); do - if docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ - -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atqc 'SELECT 1' \ - >/dev/null 2>&1; then - ready_streak="$((ready_streak + 1))" - [[ "$ready_streak" -ge 2 ]] && break - else - ready_streak=0 - fi - sleep 1 -done -[[ "$ready_streak" -ge 2 ]] \ - || fail "PostgreSQL 16 fixture did not become stably ready" +private_log="$TEST_DIR/fixture-private.log" +: >"$private_log" +postgres_password="$POSTGRES_PASSWORD" +SHADOW_DB_USER="$POSTGRES_USER" +SHADOW_DB_NAME="$POSTGRES_DB" +# shellcheck disable=SC1090 +source "$TEST_DIR/confirm-postgres-target.sh" +confirm_postgres_target \ + || fail "PostgreSQL 16 fixture did not pass rendered TCP target confirmation" +pass "real PostgreSQL 16 fixture readiness uses the rendered runner implementation" HOST_PORT="$(docker port "$CONTAINER_NAME" 5432/tcp \ | awk -F: 'NR == 1 {print $NF}')" @@ -572,16 +715,6 @@ exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \ HOST_RESTORE chmod 0700 "$TEST_DIR/host-tools/psql" "$TEST_DIR/host-tools/pg_restore" export PGPASSWORD="$POSTGRES_PASSWORD" REAL_DOCKER HOST_TOOL_MOUNT="$TEST_DIR" -for _ in $(seq 1 60); do - if "$TEST_DIR/host-tools/psql" "$HOST_DATABASE_URL" \ - -Atqc 'SELECT 1' >/dev/null 2>&1; then - break - fi - sleep 1 -done -"$TEST_DIR/host-tools/psql" "$HOST_DATABASE_URL" \ - -Atqc 'SELECT 1' >/dev/null 2>&1 \ - || fail "host-path TCP fixture did not become ready" docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \ -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ -c 'DROP TABLE restore_contract_marker' >/dev/null @@ -632,6 +765,20 @@ capture_finalization_case() { DUMP_LIST_STATUS="$dump_status" DUMP_LIST_EXIT_CODE="$dump_exit" TARGET_CONFIRMATION_STATUS="$target_status" + case "$target_status" in + PASS) + TARGET_CONFIRMATION_EXIT_CODE="0" + TARGET_NAME_MATCH="yes" + TARGET_CONFIRMATION_ATTEMPT_COUNT="2" + TARGET_CONFIRMATION_SUCCESS_STREAK="2" + ;; + FAIL) + TARGET_CONFIRMATION_EXIT_CODE="1" + TARGET_NAME_MATCH="NOT_RUN" + TARGET_CONFIRMATION_ATTEMPT_COUNT="60" + TARGET_CONFIRMATION_SUCCESS_STREAK="0" + ;; + esac RESTORE_EXECUTION_PATH="DOCKER_STDIN" RESTORE_APPLY_STATUS="$apply_status" RESTORE_SCRIPT_EXIT_CODE="$script_exit" @@ -663,9 +810,15 @@ pass "dump-list failure keeps target and apply stages NOT_RUN" capture_finalization_case \ "$TEST_DIR/captured-target-failure" \ restore_target_confirmation_failed PASS 0 FAIL NOT_RUN NOT_RUN NOT_RUN \ - NOT_RUN TARGET_CONFIRMATION_FAILED + NOT_RUN TARGET_QUERY_NONZERO grep -Fxq 'target_confirmation=FAIL' "$TEST_DIR/captured-target-failure" \ || fail "target-confirmation failure stage missing" +grep -Fxq 'target_confirmation_exit_code=1' \ + "$TEST_DIR/captured-target-failure" \ + || fail "target-confirmation failure exit code missing" +grep -Fxq 'target_confirmation_attempt_count=60' \ + "$TEST_DIR/captured-target-failure" \ + || fail "target-confirmation attempt count missing" grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-target-failure" \ || fail "apply stage ran after target-confirmation failure" pass "target-confirmation failure keeps apply stage NOT_RUN" @@ -686,6 +839,10 @@ set +e DUMP_LIST_STATUS="PASS" DUMP_LIST_EXIT_CODE="0" TARGET_CONFIRMATION_STATUS="PASS" + TARGET_CONFIRMATION_EXIT_CODE="0" + TARGET_NAME_MATCH="yes" + TARGET_CONFIRMATION_ATTEMPT_COUNT="2" + TARGET_CONFIRMATION_SUCCESS_STREAK="2" RESTORE_EXECUTION_PATH="DOCKER_STDIN" RESTORE_APPLY_STATUS="FAIL" RESTORE_SCRIPT_EXIT_CODE="23"