From 0b94b97a3a99915c69d0b9add51a446e4ac76d90 Mon Sep 17 00:00:00 2001 From: shunli Date: Tue, 25 Aug 2026 20:57:15 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=BC=BA(08-23-isolated-backup-restor?= =?UTF-8?q?e-drill):=20=E5=9B=BA=E5=8C=96=E5=85=AB=E9=98=B6=E6=AE=B5RTO?= =?UTF-8?q?=E5=90=88=E5=90=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .trellis/spec/hyb-oa/deployment-trialrun.md | 46 ++++- .../implement.md | 12 +- .../research/restore-drill-evidence.md | 14 ++ .../offhost-shadow-restore-runner.template.sh | 134 +++++++++++++- ...-offhost-shadow-restore-runner-contract.sh | 163 +++++++++++++++++- 5 files changed, 351 insertions(+), 18 deletions(-) diff --git a/.trellis/spec/hyb-oa/deployment-trialrun.md b/.trellis/spec/hyb-oa/deployment-trialrun.md index 85bbd9b1..523de906 100644 --- a/.trellis/spec/hyb-oa/deployment-trialrun.md +++ b/.trellis/spec/hyb-oa/deployment-trialrun.md @@ -1330,6 +1330,13 @@ validate_copied_batch(private/batch, source_metadata) - Exit-code fields are non-negative integers or `NOT_RUN`; `pg_restore_error_count` additionally permits `UNKNOWN` when no safe numeric summary is available. +- Required data-restore timing fields: + `rto_batch_copy_seconds`, `rto_postgres_container_start_seconds`, + `rto_dump_list_seconds`, `rto_target_confirmation_seconds`, + `rto_restore_apply_seconds`, `rto_database_verify_seconds`, + `rto_storage_attachment_verify_seconds`, `rto_final_integrity_seconds` and + `data_restore_rto_seconds`. A measured value is a non-negative integer; + an unstarted stage is `NOT_RUN`. - Tracked supply-chain entries: `scripts/qa/offhost-shadow-restore-runner.template.sh`, `scripts/qa/validate-offhost-shadow-restore-attachments.py` and @@ -1349,6 +1356,17 @@ validate_copied_batch(private/batch, source_metadata) environment values or stderr. - Run `dump_list` and target confirmation independently before the existing restore script. A failed stage leaves every later stage `NOT_RUN`. +- Measure the eight data-restore stages in this exact continuous order: + `batch_copy` -> `postgres_container_start` -> `dump_list` -> + `target_confirmation` -> `restore_apply` -> `database_verify` -> + `storage_attachment_verify` -> `final_integrity`. One timestamp closes the + current stage and starts the next, so no elapsed time is hidden between + fields. The total is computed from the measured stage fields and must equal + their exact sum. +- On failure, finalize the active stage and emit the completed/current numeric + durations plus later `NOT_RUN` fields before runtime teardown. A pre-copy + failure emits `NOT_RUN` for the total and every stage. Do not emit the total + a second time on the success-only summary path. - For the official PostgreSQL image, target readiness must use a real query over container-local TCP loopback (`psql -h 127.0.0.1`), not default Unix-socket `pg_isready`. The initdb temporary server listens only on its socket, while @@ -1399,7 +1417,8 @@ validate_copied_batch(private/batch, source_metadata) `DUMP_LIST_NONZERO` / `dump_list_failed`. - Bounded TCP target queries remain non-zero -> `target_confirmation=FAIL`, apply `NOT_RUN`, category `TARGET_QUERY_NONZERO`; preserve the last numeric - exit, attempt count and zero success streak. + exit, attempt count and zero success streak. All query retries and their + one-second intervals accrue only to `rto_target_confirmation_seconds`. - Target query succeeds but the exact shadow name differs -> fail immediately with `target_name_match=no`, category `TARGET_NAME_MISMATCH`; do not retry a confirmed wrong target. @@ -1411,6 +1430,10 @@ validate_copied_batch(private/batch, source_metadata) attachment checks still decide the final recovery result. - Diagnostics emitted only after cleanup, or cleanup residue non-zero -> the attempt is non-clean and cannot be reported as `GO`. +- A measured stage is negative/non-numeric, a completed stage is `NOT_RUN`, a + later stage is numeric after an earlier failure, or the stage sum differs + from `data_restore_rto_seconds` -> runner contract failure; do not render or + upload the candidate. - Missing/extra render key, newline/shell syntax, unsafe path/name, malformed digest or pre-existing output -> render fails without publishing a partial candidate. @@ -1434,6 +1457,12 @@ validate_copied_batch(private/batch, source_metadata) migration and attachment verification. - Good: Unix-socket availability alone does not pass; loopback TCP becomes available, two exact target queries succeed consecutively, then apply runs. +- Good: an apply failure records numeric copy/container/list/target/apply + durations, leaves database/storage/final verification `NOT_RUN`, emits their + exact partial sum, and only then tears down the private runtime. +- Base: a successful run emits all eight numeric durations once and their sum + exactly equals `data_restore_rto_seconds`; the value remains a data-only + exercise RTO, not a complete OA recovery RTO. - Bad: emit only `database_restore_failed` after deleting the sole private log. - Bad: parse and publish arbitrary pg_restore lines, or treat a non-zero exit as success because the restored database appears partly populated. @@ -1450,6 +1479,15 @@ validate_copied_batch(private/batch, source_metadata) with an equivalent test implementation. - Inject list failure, target mismatch and list-pass/apply-failure. Assert exact stage enums, later-stage `NOT_RUN`, execution path and numeric exit code. +- Drive the rendered runner's timing functions with deterministic timestamps. + Assert all eight successful stage durations and their exact total; inject + target-confirmation and restore-apply failures and assert completed/current + numeric values, later `NOT_RUN`, one total field and timing output before the + runtime cleanup marker. +- Statically assert that the bounded TCP confirmation call is enclosed by the + `dump_list -> target_confirmation` and + `target_confirmation -> restore_apply` timing transitions; a retry sleep + must not be charged to container start or restore apply. - Extract the target-confirmation function from the rendered runner and drive it with: socket-only availability/TCP failure, first-query recovery, streak reset, exact-name mismatch and permanent bounded failure. Assert every probe @@ -1485,6 +1523,7 @@ validate_copied_batch(private/batch, source_metadata) fixture: wait for two queries runner: first socket pg_isready -> continue restore-db > private.log || fail database_restore_failed +RTO = finish - start; stage gaps are inferred later finally: delete private runtime print result=NO_GO ``` @@ -1494,8 +1533,9 @@ print result=NO_GO ```text tracked template + exact private config -> deterministic candidate + SHA-256 contract drives rendered TCP target function -> list -> target -> real restore path -capture only stage/path/exit/category enums -emit fixed diagnostics -> finally teardown raw private runtime +continuous stage transitions -> exact stage sum + one data-only RTO +capture only stage/path/exit/category/duration allowlists +finalize active stage -> emit fixed diagnostics -> finally teardown raw private runtime non-zero apply remains FAIL ``` diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md index d9e1aeab..143749a1 100644 --- a/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/implement.md @@ -130,14 +130,16 @@ - [x] 最新批次来源与当前生产构建相同，探针时年龄 `64625` 秒且不超过 24 小时；run id 配对、DB/附件大小与 SHA-256、source digest 均通过。 - [x] 来源/当前 migration 目录为 `65/65`、差异 `0`；仓库 HEAD 相对生产构建的应用代码漂移文件数 `0`。 - [x] 探针 root 入口与普通用户暂存残留均为 `0`；按已验证私有配置取得的 dead-man/健康 timer 实际 unit 状态为 `active/active`。 -- [ ] RTO 八阶段实现尚未批准，因此未渲染 Attempt 5 最终候选、未生成最终候选 SHA-256，也未执行实际恢复。 +- [x] 本次探针完成时 RTO 八阶段实现尚未批准，因此当时未渲染 Attempt 5 最终候选、未生成最终候选 SHA-256，也未执行实际恢复；后续本地实现授权单独追加，不改写该时点事实。 -### Attempt 5 前 RTO 分阶段提案（待单独批准实现） +### Attempt 5 前 RTO 分阶段实施（已批准仅本地实现） - [x] 设计口径已固定为总 RTO 加八段串行秒数：批次复制、容器启动、dump list、目标确认、restore apply、migration/12 类表、附件恢复与关联校验、最终源/副本复核。 -- [ ] runner 增加固定 allowlist 阶段字段，成功时八段均为非负整数且精确加总为 `data_restore_rto_seconds`；失败时已完成段和当前失败段保留数字，后续段为 `NOT_RUN`。 -- [ ] 合同注入各阶段成功/失败，验证最长 TCP 轮询只进入目标确认阶段、后续 `NOT_RUN`、总数精确加总、输出不含路径/SQL/凭据；原有清理与恢复合同继续通过。 -- [ ] 未经单独批准，不修改 runner、重新渲染 Attempt 5 候选或执行实际恢复。 +- [x] runner 增加固定 allowlist 阶段字段，成功时八段均为非负整数且精确加总为 `data_restore_rto_seconds`；失败时已完成段和当前失败段保留数字，后续段为 `NOT_RUN`。 +- [x] 合同注入各阶段成功/失败，验证最长 TCP 轮询只进入目标确认阶段、后续 `NOT_RUN`、总数精确加总、输出不含路径/SQL/凭据；原有清理与恢复合同继续通过。 +- [x] 用户批准的当前边界仅为本地修改 runner、更新可复跑合同并执行回归；不连接 ARM、不重新渲染 Attempt 5 现场候选、不执行实际恢复。 +- [ ] 本轮八阶段计时与完整合同通过后，下一份满足 24 小时门禁的新鲜批次直接进入 Attempt 5 的候选渲染、SHA-256 核对、只读探针和实际恢复审批流程。此后发现的改进项，除非会导致 Attempt 5 失败或产生错误结论，否则只登记为后续任务，不再作为 Attempt 5 前置增强插队。 +- [x] 八阶段状态必须在失败路径清理前输出；阶段边界沿用 `dump_list`、`target_confirmation`、`restore_apply` 等既有诊断词汇，不建立第二套同义命名。 ## 验证命令类别 diff --git a/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md b/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md index 0b86dac1..765e9a28 100644 --- a/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md +++ b/.trellis/tasks/08-23-isolated-backup-restore-drill/research/restore-drill-evidence.md @@ -452,3 +452,17 @@ Attempt 4 前后探针的最新构建短标识、run id 配对、产物完整性 timer 状态只采用已验证私有配置中的实际 unit 名只读取得；一次使用猜测名称得到的无效读数已当场撤销，未进入证据结论。探针结果表明该最新批次继续满足构建、新鲜度、身份和完整性前置门禁，但仍不是恢复成功证明。 Attempt 5 最终候选尚未渲染。原因是用户提出的 RTO 分阶段记录已进入设计、但 runner 实现尚未另行批准；若先渲染，后续计时代码会改变候选字节和 SHA-256。必须先裁决并完成 RTO 八阶段本地合同，再从跟踪模板重新确定性渲染候选并核对 SHA-256，实际恢复仍须单独批准。 + +## Attempt 5 前八阶段 RTO 本地合同 + +日期：2026-08-25 + +用户随后批准的范围仅为本地实现八阶段 RTO 计时并执行合同回归；本阶段未连接 ARM、未渲染现场候选、未生成 Attempt 5 候选 SHA-256，也未执行实际恢复。此前探针批次的 `64625` 秒新鲜度只属于探针时点，后续不得复用，实际恢复前必须重新选择最新正式批次并重跑只读身份门禁。 + +runner 现在以连续边界依次记录 `batch_copy`、`postgres_container_start`、`dump_list`、`target_confirmation`、`restore_apply`、`database_verify`、`storage_attachment_verify` 和 `final_integrity`。同一个时间点关闭前一段并开启后一段；成功路径八段均为非负整数且精确加总为唯一的 `data_restore_rto_seconds`。失败路径先结算当前阶段，保留已完成和当前阶段的数字，后续阶段为 `NOT_RUN`，然后才清理私有 runtime。最长 60 次 TCP 目标确认及其 1 秒间隔只位于 `target_confirmation` 边界内。 + +可复跑合同直接驱动 rendered runner 的同一计时函数。确定性成功夹具得到八段 `3/5/2/11/7/13/17/3` 秒，总计 `61` 秒且总字段只输出一次；目标确认失败夹具保留 `2/4/1/9` 秒、后四段 `NOT_RUN`、部分总计 `16` 秒，并证明计时输出位于 runtime cleanup 之前；restore apply 失败夹具记录当前段 `12` 秒、后续三段 `NOT_RUN`、部分总计 `28` 秒。捕获输出不含路径、连接串、凭据或 SQL。 + +完整本地门禁结果：runner 合同全部 `PASS`；真实 PostgreSQL 16 fixture 和未修改的 `scripts/restore-db.sh` Docker stdin/host-file 双路径继续通过；既有成对备份/恢复演练 `PASS`；自动异机备份合同 `74/74 PASS`；task validate、Trellis governance、process baseline、docs registry、敏感扫描和 `git diff --check` 通过。文档审计错误 `0`，保留既有未登记文件 warning；本地测试容器残留 `0`，`scripts/restore-db.sh` 相对生产构建仍无字节漂移。 + +Attempt 5 前增强停止线从本节起生效：八阶段计时完成并通过合同后，下一份满足 24 小时门禁的新鲜批次直接进入最终候选渲染、SHA-256 核对、只读探针和实际恢复审批。此后发现的改进项，除非会导致 Attempt 5 失败或产生错误结论，否则只登记为后续任务，不再作为 Attempt 5 前置增强插队。本地合同通过仍只证明演练器合同，不回答最新正式备份是否可恢复。 diff --git a/scripts/qa/offhost-shadow-restore-runner.template.sh b/scripts/qa/offhost-shadow-restore-runner.template.sh index 0a3be16d..742419b0 100644 --- a/scripts/qa/offhost-shadow-restore-runner.template.sh +++ b/scripts/qa/offhost-shadow-restore-runner.template.sh @@ -29,7 +29,18 @@ CONTAINER_NAME="" RUNTIME_CLEANED=0 RESULT_STATUS="NO_GO" RPO_SECONDS="" -RTO_SECONDS="" +RTO_SECONDS="NOT_RUN" +RTO_BATCH_COPY_SECONDS="NOT_RUN" +RTO_POSTGRES_CONTAINER_START_SECONDS="NOT_RUN" +RTO_DUMP_LIST_SECONDS="NOT_RUN" +RTO_TARGET_CONFIRMATION_SECONDS="NOT_RUN" +RTO_RESTORE_APPLY_SECONDS="NOT_RUN" +RTO_DATABASE_VERIFY_SECONDS="NOT_RUN" +RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS="NOT_RUN" +RTO_FINAL_INTEGRITY_SECONDS="NOT_RUN" +RTO_ACTIVE_STAGE="" +RTO_STAGE_STARTED_EPOCH="" +RTO_FINALIZED=0 MIGRATION_SUCCESS="" MIGRATION_OTHER="" MIGRATION_SET_MATCH="" @@ -66,6 +77,94 @@ require_command() { command -v "$1" >/dev/null 2>&1 || fail "required_command_missing" } +rto_now() { + date -u +%s +} + +rto_set_stage_seconds() { + local stage="$1" + local seconds="$2" + case "$stage" in + batch_copy) RTO_BATCH_COPY_SECONDS="$seconds" ;; + postgres_container_start) RTO_POSTGRES_CONTAINER_START_SECONDS="$seconds" ;; + dump_list) RTO_DUMP_LIST_SECONDS="$seconds" ;; + target_confirmation) RTO_TARGET_CONFIRMATION_SECONDS="$seconds" ;; + restore_apply) RTO_RESTORE_APPLY_SECONDS="$seconds" ;; + database_verify) RTO_DATABASE_VERIFY_SECONDS="$seconds" ;; + storage_attachment_verify) RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS="$seconds" ;; + final_integrity) RTO_FINAL_INTEGRITY_SECONDS="$seconds" ;; + *) return 1 ;; + esac +} + +rto_begin() { + local stage="$1" + local now + [[ -z "$RTO_ACTIVE_STAGE" && "$RTO_FINALIZED" -eq 0 ]] || return 1 + now="$(rto_now)" + [[ "$now" =~ ^[0-9]+$ ]] || return 1 + rto_set_stage_seconds "$stage" "NOT_RUN" || return 1 + RTO_ACTIVE_STAGE="$stage" + RTO_STAGE_STARTED_EPOCH="$now" +} + +rto_close_active_at() { + local now="$1" + local elapsed + [[ -n "$RTO_ACTIVE_STAGE" && "$RTO_STAGE_STARTED_EPOCH" =~ ^[0-9]+$ \ + && "$now" =~ ^[0-9]+$ && "$now" -ge "$RTO_STAGE_STARTED_EPOCH" ]] \ + || return 1 + elapsed="$((now - RTO_STAGE_STARTED_EPOCH))" + rto_set_stage_seconds "$RTO_ACTIVE_STAGE" "$elapsed" || return 1 +} + +rto_transition() { + local expected_stage="$1" + local next_stage="$2" + local now + [[ "$RTO_ACTIVE_STAGE" == "$expected_stage" && "$RTO_FINALIZED" -eq 0 ]] \ + || return 1 + now="$(rto_now)" + rto_close_active_at "$now" || return 1 + rto_set_stage_seconds "$next_stage" "NOT_RUN" || return 1 + RTO_ACTIVE_STAGE="$next_stage" + RTO_STAGE_STARTED_EPOCH="$now" +} + +rto_finalize() { + local now + local value + local total=0 + local measured=0 + if [[ "$RTO_FINALIZED" -eq 1 ]]; then + return 0 + fi + if [[ -n "$RTO_ACTIVE_STAGE" ]]; then + now="$(rto_now)" + rto_close_active_at "$now" || return 1 + RTO_ACTIVE_STAGE="" + RTO_STAGE_STARTED_EPOCH="" + fi + for value in \ + "$RTO_BATCH_COPY_SECONDS" \ + "$RTO_POSTGRES_CONTAINER_START_SECONDS" \ + "$RTO_DUMP_LIST_SECONDS" \ + "$RTO_TARGET_CONFIRMATION_SECONDS" \ + "$RTO_RESTORE_APPLY_SECONDS" \ + "$RTO_DATABASE_VERIFY_SECONDS" \ + "$RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS" \ + "$RTO_FINAL_INTEGRITY_SECONDS"; do + if [[ "$value" =~ ^[0-9]+$ ]]; then + total="$((total + value))" + measured=1 + fi + done + if [[ "$measured" -eq 1 ]]; then + RTO_SECONDS="$total" + fi + RTO_FINALIZED=1 +} + confirm_postgres_target() { local actual_database="" local query_exit_code=0 @@ -182,6 +281,15 @@ emit_restore_diagnostics() { echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE" echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT" echo "restore_error_category=$RESTORE_ERROR_CATEGORY" + echo "rto_batch_copy_seconds=$RTO_BATCH_COPY_SECONDS" + echo "rto_postgres_container_start_seconds=$RTO_POSTGRES_CONTAINER_START_SECONDS" + echo "rto_dump_list_seconds=$RTO_DUMP_LIST_SECONDS" + echo "rto_target_confirmation_seconds=$RTO_TARGET_CONFIRMATION_SECONDS" + echo "rto_restore_apply_seconds=$RTO_RESTORE_APPLY_SECONDS" + echo "rto_database_verify_seconds=$RTO_DATABASE_VERIFY_SECONDS" + echo "rto_storage_attachment_verify_seconds=$RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS" + echo "rto_final_integrity_seconds=$RTO_FINAL_INTEGRITY_SECONDS" + echo "data_restore_rto_seconds=$RTO_SECONDS" } finish() { @@ -191,6 +299,10 @@ finish() { trap - EXIT HUP INT TERM # Emit only allowlisted stage diagnostics while the private runtime still # exists. Raw stderr remains mode 0600 and is removed by cleanup_runtime. + if ! rto_finalize; then + FAILURE_CATEGORY="rto_stage_finalize_failed" + exit_code=1 + fi emit_restore_diagnostics if [[ "$RUNTIME_CLEANED" -ne 1 ]]; then cleanup_runtime || runtime_cleanup="FAIL" @@ -204,7 +316,6 @@ finish() { if [[ "$exit_code" -eq 0 ]]; then echo "result=$RESULT_STATUS" echo "rpo_seconds=$RPO_SECONDS" - echo "data_restore_rto_seconds=$RTO_SECONDS" echo "migration_success=$MIGRATION_SUCCESS" echo "migration_other=$MIGRATION_OTHER" echo "migration_set_match=$MIGRATION_SET_MATCH" @@ -412,11 +523,13 @@ mkdir -m 0700 "$WORK_DIR" private_log="$WORK_DIR/private.log" : >"$private_log" chmod 0600 "$private_log" -rto_started_epoch="$(date -u +%s)" +rto_begin batch_copy || fail "rto_stage_begin_failed" cp -a -- "$selected_batch" "$WORK_DIR/batch" copied_metadata="$(validate_batch "$WORK_DIR/batch" copy "$SOURCE_RUN_ID")" \ || fail "copied_batch_invalid" [[ "$copied_metadata" == "$batch_metadata" ]] || fail "copied_batch_identity_mismatch" +rto_transition batch_copy postgres_container_start \ + || fail "rto_stage_transition_failed" env_file="$WORK_DIR/postgres.env" postgres_password="$(openssl rand -hex 24)" @@ -429,6 +542,8 @@ chmod 0600 "$env_file" printf '%s\n' '[stage=postgres_container_start]' >>"$private_log" docker run -d --rm --name "$CONTAINER_NAME" --network none --env-file "$env_file" \ "$POSTGRES_IMAGE" >>"$private_log" 2>&1 || fail "postgres_container_start_failed" +rto_transition postgres_container_start dump_list \ + || fail "rto_stage_transition_failed" PHASE="restore_database" DUMP_LIST_STATUS="FAIL" @@ -444,6 +559,8 @@ if [[ "$DUMP_LIST_EXIT_CODE" -ne 0 ]]; then fail "dump_list_failed" fi DUMP_LIST_STATUS="PASS" +rto_transition dump_list target_confirmation \ + || fail "rto_stage_transition_failed" TARGET_CONFIRMATION_STATUS="FAIL" printf '%s\n' '[stage=target_confirmation]' >>"$private_log" @@ -452,6 +569,8 @@ if ! confirm_postgres_target; then fail "restore_target_confirmation_failed" fi TARGET_CONFIRMATION_STATUS="PASS" +rto_transition target_confirmation restore_apply \ + || fail "rto_stage_transition_failed" real_docker_path="$(command -v docker)" docker_proxy_dir="$WORK_DIR/docker-proxy" @@ -530,6 +649,8 @@ if [[ "$RESTORE_SCRIPT_EXIT_CODE" -ne 0 || "$PG_RESTORE_EXIT_CODE" -ne 0 ]]; the fi RESTORE_APPLY_STATUS="PASS" RESTORE_ERROR_CATEGORY="NONE" +rto_transition restore_apply database_verify \ + || fail "rto_stage_transition_failed" PHASE="verify_database" migration_names="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \ @@ -581,6 +702,8 @@ SQL )" || fail "table_count_query_failed" [[ "$(printf '%s\n' "$TABLE_COUNTS" | sed '/^$/d' | wc -l | tr -d ' ')" == "12" ]] \ || fail "table_count_key_mismatch" +rto_transition database_verify storage_attachment_verify \ + || fail "rto_stage_transition_failed" PHASE="restore_storage" restored_storage="$WORK_DIR/restored-storage" @@ -613,6 +736,8 @@ SQL read -r ATTACHMENT_TOTAL ATTACHMENT_MISMATCH attachment_s3_count <<<"$attachment_result" [[ "$ATTACHMENT_TOTAL" =~ ^[0-9]+$ && "$ATTACHMENT_MISMATCH" == "0" \ && "$attachment_s3_count" == "0" ]] || fail "attachment_reference_mismatch" +rto_transition storage_attachment_verify final_integrity \ + || fail "rto_stage_transition_failed" PHASE="post_validation" copied_metadata_after="$(validate_batch "$WORK_DIR/batch" copy "$SOURCE_RUN_ID")" \ @@ -621,8 +746,7 @@ source_metadata_after="$(validate_batch "$selected_batch" source "")" \ || fail "source_archive_postcheck_failed" [[ "$copied_metadata_after" == "$batch_metadata" \ && "$source_metadata_after" == "$batch_metadata" ]] || fail "archive_identity_changed" -rto_finished_epoch="$(date -u +%s)" -RTO_SECONDS="$((rto_finished_epoch - rto_started_epoch))" +rto_finalize || fail "rto_stage_finalize_failed" deadman_enabled_after="$(systemctl is-enabled "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)" deadman_active_after="$(systemctl is-active "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)" diff --git a/scripts/qa/run-offhost-shadow-restore-runner-contract.sh b/scripts/qa/run-offhost-shadow-restore-runner-contract.sh index 618bb81e..59e1cdae 100755 --- a/scripts/qa/run-offhost-shadow-restore-runner-contract.sh +++ b/scripts/qa/run-offhost-shadow-restore-runner-contract.sh @@ -432,14 +432,17 @@ finish = re.search(r"\nfinish\(\) \{(?P.*?)\n\}\ntrap finish", text, re.S) if not finish: raise SystemExit("finish function not found") body = finish.group("body") +finalize_position = body.find("\n if ! rto_finalize; then") emit_position = body.find("\n emit_restore_diagnostics") cleanup_position = body.find("\n cleanup_runtime ||") +if finalize_position == -1: + raise SystemExit("RTO finalization missing") if emit_position == -1: raise SystemExit("diagnostic emission missing") if cleanup_position == -1: raise SystemExit("runtime cleanup call missing") -if emit_position > cleanup_position: - raise SystemExit("diagnostics are emitted after cleanup") +if not (finalize_position < emit_position < cleanup_position): + raise SystemExit("RTO/stage diagnostics are not finalized before cleanup") required = { 'echo "dump_list=$DUMP_LIST_STATUS"', 'echo "target_confirmation=$TARGET_CONFIRMATION_STATUS"', @@ -452,14 +455,41 @@ required = { 'echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE"', 'echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT"', 'echo "restore_error_category=$RESTORE_ERROR_CATEGORY"', + 'echo "rto_batch_copy_seconds=$RTO_BATCH_COPY_SECONDS"', + 'echo "rto_postgres_container_start_seconds=$RTO_POSTGRES_CONTAINER_START_SECONDS"', + 'echo "rto_dump_list_seconds=$RTO_DUMP_LIST_SECONDS"', + 'echo "rto_target_confirmation_seconds=$RTO_TARGET_CONFIRMATION_SECONDS"', + 'echo "rto_restore_apply_seconds=$RTO_RESTORE_APPLY_SECONDS"', + 'echo "rto_database_verify_seconds=$RTO_DATABASE_VERIFY_SECONDS"', + 'echo "rto_storage_attachment_verify_seconds=$RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS"', + 'echo "rto_final_integrity_seconds=$RTO_FINAL_INTEGRITY_SECONDS"', + 'echo "data_restore_rto_seconds=$RTO_SECONDS"', } missing = sorted(value for value in required if value not in text) if missing: raise SystemExit(f"diagnostic fields missing: {missing}") if 'RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"' not in text: raise SystemExit("non-zero apply classification missing") +transitions = [ + "rto_begin batch_copy", + "rto_transition batch_copy postgres_container_start", + "rto_transition postgres_container_start dump_list", + "rto_transition dump_list target_confirmation", + "rto_transition target_confirmation restore_apply", + "rto_transition restore_apply database_verify", + "rto_transition database_verify storage_attachment_verify", + "rto_transition storage_attachment_verify final_integrity", +] +positions = [text.find(value) for value in transitions] +if any(position == -1 for position in positions) or positions != sorted(positions): + raise SystemExit("RTO stage transitions missing or out of order") +target_call_position = text.find("if ! confirm_postgres_target; then") +if not positions[3] < target_call_position < positions[4]: + raise SystemExit("TCP target confirmation is outside its RTO stage boundary") +if text.count('echo "data_restore_rto_seconds=$RTO_SECONDS"') != 1: + raise SystemExit("data restore RTO must be emitted exactly once") PY -pass "diagnostics precede finally-equivalent cleanup and use allowlisted fields" +pass "eight-stage RTO and restore diagnostics precede cleanup with one vocabulary" cat >"$TEST_DIR/mock-target-docker" <<'MOCK_DOCKER' #!/usr/bin/env bash @@ -738,6 +768,107 @@ CONTAINER_NAME="" sed -n '1,/^trap finish EXIT HUP INT TERM$/p' "$RUNNER" \ | sed '$d' >"$TEST_DIR/runner-finalization-functions.sh" +( + # shellcheck disable=SC1090 + source "$TEST_DIR/runner-finalization-functions.sh" + RTO_TEST_NOW=100 + rto_now() { + printf '%s\n' "$RTO_TEST_NOW" + } + rto_begin batch_copy + RTO_TEST_NOW=103 + rto_transition batch_copy postgres_container_start + RTO_TEST_NOW=108 + rto_transition postgres_container_start dump_list + RTO_TEST_NOW=110 + rto_transition dump_list target_confirmation + RTO_TEST_NOW=121 + rto_transition target_confirmation restore_apply + RTO_TEST_NOW=128 + rto_transition restore_apply database_verify + RTO_TEST_NOW=141 + rto_transition database_verify storage_attachment_verify + RTO_TEST_NOW=158 + rto_transition storage_attachment_verify final_integrity + RTO_TEST_NOW=161 + rto_finalize + emit_restore_diagnostics +) >"$TEST_DIR/captured-rto-success" 2>&1 +for expected in \ + 'rto_batch_copy_seconds=3' \ + 'rto_postgres_container_start_seconds=5' \ + 'rto_dump_list_seconds=2' \ + 'rto_target_confirmation_seconds=11' \ + 'rto_restore_apply_seconds=7' \ + 'rto_database_verify_seconds=13' \ + 'rto_storage_attachment_verify_seconds=17' \ + 'rto_final_integrity_seconds=3' \ + 'data_restore_rto_seconds=61'; do + grep -Fxq "$expected" "$TEST_DIR/captured-rto-success" \ + || fail "successful eight-stage RTO total or boundary mismatch" +done +[[ "$(grep -c '^data_restore_rto_seconds=' "$TEST_DIR/captured-rto-success")" -eq 1 ]] \ + || fail "successful RTO total was not emitted exactly once" +pass "rendered runner measures eight contiguous stages with an exact total" + +set +e +( + # shellcheck disable=SC1090 + source "$TEST_DIR/runner-finalization-functions.sh" + cleanup_runtime() { + echo "cleanup_marker=runtime" + return 0 + } + cleanup_access() { + echo "cleanup_marker=access" + return 0 + } + RTO_BATCH_COPY_SECONDS="2" + RTO_POSTGRES_CONTAINER_START_SECONDS="4" + RTO_DUMP_LIST_SECONDS="1" + RTO_ACTIVE_STAGE="target_confirmation" + RTO_STAGE_STARTED_EPOCH="200" + RTO_TEST_NOW=209 + rto_now() { + printf '%s\n' "$RTO_TEST_NOW" + } + FAILURE_CATEGORY="restore_target_confirmation_failed" + DUMP_LIST_STATUS="PASS" + DUMP_LIST_EXIT_CODE="0" + TARGET_CONFIRMATION_STATUS="FAIL" + TARGET_CONFIRMATION_EXIT_CODE="1" + TARGET_CONFIRMATION_ATTEMPT_COUNT="60" + RESTORE_EXECUTION_PATH="DOCKER_STDIN" + RESTORE_ERROR_CATEGORY="TARGET_QUERY_NONZERO" + trap finish EXIT + exit 23 +) >"$TEST_DIR/captured-rto-target-failure" 2>&1 +rto_target_failure_exit="$?" +set -e +[[ "$rto_target_failure_exit" -eq 23 ]] \ + || fail "target-confirmation timing failure did not preserve exit code" +for expected in \ + 'rto_batch_copy_seconds=2' \ + 'rto_postgres_container_start_seconds=4' \ + 'rto_dump_list_seconds=1' \ + 'rto_target_confirmation_seconds=9' \ + 'rto_restore_apply_seconds=NOT_RUN' \ + 'rto_database_verify_seconds=NOT_RUN' \ + 'rto_storage_attachment_verify_seconds=NOT_RUN' \ + 'rto_final_integrity_seconds=NOT_RUN' \ + 'data_restore_rto_seconds=16'; do + grep -Fxq "$expected" "$TEST_DIR/captured-rto-target-failure" \ + || fail "target-confirmation failure RTO boundary mismatch" +done +rto_line="$(grep -n '^rto_target_confirmation_seconds=9$' \ + "$TEST_DIR/captured-rto-target-failure" | cut -d: -f1)" +rto_cleanup_line="$(grep -n '^cleanup_marker=runtime$' \ + "$TEST_DIR/captured-rto-target-failure" | cut -d: -f1)" +[[ "$rto_line" =~ ^[0-9]+$ && "$rto_cleanup_line" =~ ^[0-9]+$ \ + && "$rto_line" -lt "$rto_cleanup_line" ]] \ + || fail "failed-stage RTO was not emitted before teardown" +pass "target-confirmation failure preserves elapsed stages and later NOT_RUN before cleanup" + capture_finalization_case() { local output_file="$1" local failure_category="$2" @@ -805,7 +936,13 @@ grep -Fxq 'target_confirmation=NOT_RUN' \ || fail "target stage ran after dump-list failure" grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-dump-list-failure" \ || fail "apply stage ran after dump-list failure" -pass "dump-list failure keeps target and apply stages NOT_RUN" +grep -Fxq 'rto_batch_copy_seconds=NOT_RUN' \ + "$TEST_DIR/captured-dump-list-failure" \ + || fail "pre-RTO failure unexpectedly reported a measured stage" +grep -Fxq 'data_restore_rto_seconds=NOT_RUN' \ + "$TEST_DIR/captured-dump-list-failure" \ + || fail "pre-RTO failure unexpectedly reported a measured total" +pass "pre-RTO failure keeps timing, target and apply fields NOT_RUN" capture_finalization_case \ "$TEST_DIR/captured-target-failure" \ @@ -849,6 +986,16 @@ set +e PG_RESTORE_EXIT_CODE="23" PG_RESTORE_ERROR_COUNT="UNKNOWN" RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO" + RTO_BATCH_COPY_SECONDS="2" + RTO_POSTGRES_CONTAINER_START_SECONDS="4" + RTO_DUMP_LIST_SECONDS="1" + RTO_TARGET_CONFIRMATION_SECONDS="9" + RTO_ACTIVE_STAGE="restore_apply" + RTO_STAGE_STARTED_EPOCH="300" + RTO_TEST_NOW=312 + rto_now() { + printf '%s\n' "$RTO_TEST_NOW" + } trap finish EXIT exit 23 ) >"$TEST_DIR/captured-finalization" 2>&1 @@ -869,7 +1016,13 @@ grep -Fxq 'temporary_access_cleanup=PASS' "$TEST_DIR/captured-finalization" \ || fail "access teardown result was not retained" grep -Fxq 'pg_restore_exit_code=23' "$TEST_DIR/captured-finalization" \ || fail "pg_restore exit code disappeared after teardown" -pass "injected failure emits readable stages before finally-equivalent teardown" +grep -Fxq 'rto_restore_apply_seconds=12' "$TEST_DIR/captured-finalization" \ + || fail "restore-apply failure duration missing" +grep -Fxq 'rto_database_verify_seconds=NOT_RUN' "$TEST_DIR/captured-finalization" \ + || fail "database verification ran after restore-apply failure" +grep -Fxq 'data_restore_rto_seconds=28' "$TEST_DIR/captured-finalization" \ + || fail "restore-apply failure RTO total mismatch" +pass "injected restore-apply failure preserves diagnostics and RTO before teardown" if grep -Eiq 'postgresql://|SELECT[[:space:]]|/tmp/|local_fixture_password' \ "$TEST_DIR/captured-dump-list-failure" \