修复异机备份凭证与中断恢复(07-27-offhost-backup-final-fixes)
ci-e2e / e2e (push) Waiting to run
ci-minimal / check-minimal (push) Waiting to run

这个提交包含在:
2026-07-27 17:58:11 +08:00
父节点 f1e8cf138a
当前提交 01f1259557
共修改 13 个文件,包含 748 行新增和 16 行删除
+212
查看文件
@@ -128,6 +128,162 @@ expect_failure "receiver rolls back the daily archive when weekly checkpoint pub
[[ -d "$ROLLBACK_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$ROLLBACK_LAYOUT/archive/$SECOND_BATCH" ]]
pass "receiver restores the prior daily point after a later replacement step fails"
CURRENT_WEEK="$(python3 - "$CURRENT_RUN_ID" <<'PYTHON'
from datetime import datetime
import sys
day = datetime.strptime(sys.argv[1][5:13], "%Y%m%d").date().isocalendar()
print(f"week-{day.year:04d}-W{day.week:02d}")
PYTHON
)"
make_corrupt_partial() {
local layout="$1"
local run_id="$2"
local batch_name="hyb-oa-backup-$run_id"
make_batch "$layout/inbox/${batch_name}.partial" "$run_id"
printf 'corrupt candidate\n' >>"$layout/inbox/${batch_name}.partial/database.dump"
sha256sum "$layout/inbox/${batch_name}.partial/manifest.json" | awk '{print $1}'
}
DAILY_COMPLETE_LAYOUT="$TEST_DIR/receiver-daily-orphan-complete"
make_layout "$DAILY_COMPLETE_LAYOUT"
make_batch "$DAILY_COMPLETE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.101" "$CURRENT_RUN_ID"
make_batch "$DAILY_COMPLETE_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
DAILY_COMPLETE_SHA="$(make_corrupt_partial "$DAILY_COMPLETE_LAYOUT" "$THIRD_RUN_ID")"
expect_failure "receiver cleans a valid daily rollback after the replacement was published" \
receiver "$DAILY_COMPLETE_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_COMPLETE_SHA"
[[ ! -e "$DAILY_COMPLETE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.101" \
&& -d "$DAILY_COMPLETE_LAYOUT/archive/$SECOND_BATCH" ]]
[[ -d "$DAILY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" ]]
pass "receiver keeps the valid published daily point and rebuilds its weekly checkpoint"
DAILY_WEEKLY_FAIL_LAYOUT="$TEST_DIR/receiver-daily-orphan-weekly-failure"
make_layout "$DAILY_WEEKLY_FAIL_LAYOUT"
DAILY_WEEKLY_FAIL_ORPHAN="$DAILY_WEEKLY_FAIL_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.107"
make_batch "$DAILY_WEEKLY_FAIL_ORPHAN" "$CURRENT_RUN_ID"
make_batch "$DAILY_WEEKLY_FAIL_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
printf 'invalid weekly target\n' >"$DAILY_WEEKLY_FAIL_LAYOUT/weekly/$CURRENT_WEEK"
DAILY_WEEKLY_FAIL_SHA="$(make_corrupt_partial "$DAILY_WEEKLY_FAIL_LAYOUT" "$THIRD_RUN_ID")"
expect_failure "receiver keeps a daily rollback until weekly reconciliation succeeds" \
receiver "$DAILY_WEEKLY_FAIL_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_WEEKLY_FAIL_SHA"
[[ -d "$DAILY_WEEKLY_FAIL_ORPHAN" \
&& -d "$DAILY_WEEKLY_FAIL_LAYOUT/archive/$SECOND_BATCH" \
&& -f "$DAILY_WEEKLY_FAIL_LAYOUT/weekly/$CURRENT_WEEK" ]]
pass "receiver preserves the daily rollback when weekly reconciliation fails"
DAILY_RESTORE_LAYOUT="$TEST_DIR/receiver-daily-orphan-restore"
make_layout "$DAILY_RESTORE_LAYOUT"
make_batch "$DAILY_RESTORE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.102" "$CURRENT_RUN_ID"
DAILY_RESTORE_SHA="$(make_corrupt_partial "$DAILY_RESTORE_LAYOUT" "$SECOND_RUN_ID")"
expect_failure "receiver restores a valid daily rollback when the published archive is absent" \
receiver "$DAILY_RESTORE_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_RESTORE_SHA"
[[ -d "$DAILY_RESTORE_LAYOUT/archive/$CURRENT_BATCH" \
&& ! -e "$DAILY_RESTORE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.102" ]]
[[ -d "$DAILY_RESTORE_LAYOUT/weekly/$CURRENT_WEEK" ]]
pass "receiver restores the last daily point before rejecting the next corrupt candidate"
DAILY_AMBIGUOUS_LAYOUT="$TEST_DIR/receiver-daily-orphan-ambiguous"
make_layout "$DAILY_AMBIGUOUS_LAYOUT"
make_batch "$DAILY_AMBIGUOUS_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.103" "$CURRENT_RUN_ID"
make_batch "$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
printf 'corruption\n' >>"$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH/database.dump"
make_batch "$DAILY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
DAILY_AMBIGUOUS_SHA="$(sha256sum "$DAILY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
expect_failure "receiver fails closed when a daily rollback and invalid published archive coexist" \
receiver "$DAILY_AMBIGUOUS_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_AMBIGUOUS_SHA"
[[ -d "$DAILY_AMBIGUOUS_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.103" \
&& -d "$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" ]]
pass "receiver preserves both daily objects when replacement state is ambiguous"
DAILY_DUPLICATE_LAYOUT="$TEST_DIR/receiver-daily-orphan-duplicate"
make_layout "$DAILY_DUPLICATE_LAYOUT"
make_batch "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.104" "$CURRENT_RUN_ID"
make_batch "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.105" "$CURRENT_RUN_ID"
make_batch "$DAILY_DUPLICATE_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
DAILY_DUPLICATE_SHA="$(sha256sum "$DAILY_DUPLICATE_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
expect_failure "receiver fails closed on duplicate daily rollback orphans" \
receiver "$DAILY_DUPLICATE_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_DUPLICATE_SHA"
[[ -d "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.104" \
&& -d "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.105" ]]
pass "receiver preserves duplicate daily rollback evidence without choosing one"
DAILY_LINK_LAYOUT="$TEST_DIR/receiver-daily-orphan-link"
make_layout "$DAILY_LINK_LAYOUT"
make_batch "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
ln -s "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" \
"$DAILY_LINK_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.106"
make_batch "$DAILY_LINK_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
DAILY_LINK_SHA="$(sha256sum "$DAILY_LINK_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
expect_failure "receiver rejects a symbolic-link daily rollback orphan" \
receiver "$DAILY_LINK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_LINK_SHA"
[[ -L "$DAILY_LINK_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.106" \
&& -d "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" ]]
pass "receiver leaves linked daily rollback evidence untouched"
WEEKLY_ROLLBACK_LAYOUT="$TEST_DIR/receiver-weekly-orphan-rollback"
make_layout "$WEEKLY_ROLLBACK_LAYOUT"
make_batch "$WEEKLY_ROLLBACK_LAYOUT/weekly/${CURRENT_WEEK}.rollback.201" "$CURRENT_RUN_ID"
WEEKLY_ROLLBACK_SHA="$(make_corrupt_partial "$WEEKLY_ROLLBACK_LAYOUT" "$SECOND_RUN_ID")"
expect_failure "receiver restores a weekly rollback when the canonical checkpoint is absent" \
receiver "$WEEKLY_ROLLBACK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$WEEKLY_ROLLBACK_SHA"
[[ -d "$WEEKLY_ROLLBACK_LAYOUT/weekly/$CURRENT_WEEK" \
&& ! -e "$WEEKLY_ROLLBACK_LAYOUT/weekly/${CURRENT_WEEK}.rollback.201" ]]
pass "receiver restores the authoritative weekly rollback before the next finalize"
WEEKLY_COMPLETE_LAYOUT="$TEST_DIR/receiver-weekly-orphan-complete"
make_layout "$WEEKLY_COMPLETE_LAYOUT"
make_batch "$WEEKLY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" "$SECOND_RUN_ID"
make_batch "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.rollback.202" "$CURRENT_RUN_ID"
mkdir "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202"
printf 'interrupted staging\n' >"$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202/partial"
WEEKLY_COMPLETE_SHA="$(make_corrupt_partial "$WEEKLY_COMPLETE_LAYOUT" "$THIRD_RUN_ID")"
expect_failure "receiver cleans weekly rollback and staging after a valid checkpoint was published" \
receiver "$WEEKLY_COMPLETE_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_COMPLETE_SHA"
[[ -d "$WEEKLY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" \
&& ! -e "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.rollback.202" \
&& ! -e "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202" ]]
pass "receiver treats weekly staging as non-authoritative after a valid publish"
WEEKLY_REBUILD_LAYOUT="$TEST_DIR/receiver-weekly-orphan-rebuild"
make_layout "$WEEKLY_REBUILD_LAYOUT"
make_batch "$WEEKLY_REBUILD_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
mkdir "$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203"
printf 'interrupted staging\n' >"$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203/partial"
WEEKLY_REBUILD_SHA="$(make_corrupt_partial "$WEEKLY_REBUILD_LAYOUT" "$SECOND_RUN_ID")"
expect_failure "receiver rebuilds a missing weekly point from one valid daily archive" \
receiver "$WEEKLY_REBUILD_LAYOUT" finalize-test "$SECOND_RUN_ID" "$WEEKLY_REBUILD_SHA"
[[ -d "$WEEKLY_REBUILD_LAYOUT/weekly/$CURRENT_WEEK" \
&& ! -e "$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203" ]]
[[ "$(stat -c %i "$WEEKLY_REBUILD_LAYOUT/weekly/$CURRENT_WEEK/database.dump")" \
== "$(stat -c %i "$WEEKLY_REBUILD_LAYOUT/archive/$CURRENT_BATCH/database.dump")" ]]
pass "receiver rebuilds the weekly checkpoint from the unique valid daily source"
WEEKLY_AMBIGUOUS_LAYOUT="$TEST_DIR/receiver-weekly-orphan-ambiguous"
make_layout "$WEEKLY_AMBIGUOUS_LAYOUT"
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
mkdir "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/${CURRENT_WEEK}.new.204"
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
WEEKLY_AMBIGUOUS_SHA="$(sha256sum "$WEEKLY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
expect_failure "receiver fails closed when weekly staging has multiple daily sources" \
receiver "$WEEKLY_AMBIGUOUS_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_AMBIGUOUS_SHA"
[[ -d "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/${CURRENT_WEEK}.new.204" \
&& ! -e "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/$CURRENT_WEEK" ]]
pass "receiver preserves ambiguous weekly staging without choosing a daily source"
WEEKLY_INVALID_LAYOUT="$TEST_DIR/receiver-weekly-orphan-invalid"
make_layout "$WEEKLY_INVALID_LAYOUT"
make_batch "$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK" "$SECOND_RUN_ID"
printf 'corruption\n' >>"$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK/database.dump"
make_batch "$WEEKLY_INVALID_LAYOUT/weekly/${CURRENT_WEEK}.rollback.205" "$CURRENT_RUN_ID"
make_batch "$WEEKLY_INVALID_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
WEEKLY_INVALID_SHA="$(sha256sum "$WEEKLY_INVALID_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
expect_failure "receiver fails closed when a weekly rollback and invalid canonical coexist" \
receiver "$WEEKLY_INVALID_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_INVALID_SHA"
[[ -d "$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK" \
&& -d "$WEEKLY_INVALID_LAYOUT/weekly/${CURRENT_WEEK}.rollback.205" ]]
pass "receiver preserves both weekly objects when checkpoint state is ambiguous"
negative_receiver_case() {
local label="$1"
local mutation="$2"
@@ -241,7 +397,10 @@ printf 'backup-alias ssh-ed25519 fixture\n' >"$TEST_DIR/known-hosts"
BUILD_SHA="abcdef1234567890abcdef1234567890abcdef12"
VITE_BUILD_SHA="${BUILD_SHA:0:12}"
MOCK_ALERT_LOG="$TEST_DIR/mock-alert.log"
MOCK_DOCKER_ARGV_LOG="$TEST_DIR/mock-docker-argv.log"
MOCK_DOCKER_ENV_LOG="$TEST_DIR/mock-docker-env.log"
export MOCK_PG="$WRAPPER_PG" MOCK_STORAGE="$WRAPPER_STORAGE" MOCK_BUILD_SHA="$BUILD_SHA" MOCK_ALERT_LOG
export MOCK_DOCKER_ARGV_LOG MOCK_DOCKER_ENV_LOG
cat >"$MOCK_BIN/docker" <<'MOCK'
#!/usr/bin/env bash
@@ -278,8 +437,13 @@ if [[ "${1:-}" == "inspect" ]]; then
exit 0
fi
if [[ "${1:-}" == "exec" ]]; then
printf '%q ' "$@" >>"$MOCK_DOCKER_ARGV_LOG"
printf '\n' >>"$MOCK_DOCKER_ARGV_LOG"
printf 'PGPASSWORD=%s\n' "${PGPASSWORD:-}" >>"$MOCK_DOCKER_ENV_LOG"
if [[ " $* " == *" pg_dump "* ]]; then printf 'synthetic-dump\n'; exit 0; fi
if [[ " $* " == *" psql "* ]]; then printf '%s\n' "${POSTGRES_DB:-hyb_oa_restore_shadow}"; exit 0; fi
if [[ " $* " == *" pg_restore --list "* ]]; then cat >/dev/null; exit 0; fi
if [[ " $* " == *" pg_restore "* ]]; then cat >/dev/null; exit 0; fi
fi
exit 1
MOCK
@@ -365,6 +529,54 @@ pass "wrapper failure summary is category-only and redacted"
[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
grep -Fq 'synthetic transfer diagnostic' "$WRAPPER_STATE/last-error.log"
pass "wrapper retains the latest command diagnostic locally with mode 0600"
[[ -s "$MOCK_DOCKER_ARGV_LOG" && -s "$MOCK_DOCKER_ENV_LOG" ]]
! grep -Fq 'fixture' "$MOCK_DOCKER_ARGV_LOG"
! grep -Fq 'postgresql://' "$MOCK_DOCKER_ARGV_LOG"
grep -Fq 'PGPASSWORD=fixture' "$MOCK_DOCKER_ENV_LOG"
pass "wrapper and nightly Docker backup keep connection secrets out of captured argv"
: >"$MOCK_DOCKER_ARGV_LOG"
: >"$MOCK_DOCKER_ENV_LOG"
DB_TOOL_MODE=docker DATABASE_URL='postgresql://synthetic:argv-secret@db:5432/hyb_oa_restore_shadow' \
DB_CONTAINER=db-id POSTGRES_DB=hyb_oa_restore_shadow POSTGRES_USER=synthetic \
POSTGRES_PASSWORD=argv-secret OUT_FILE="$TEST_DIR/argv-backup.dump" PATH="$MOCK_BIN:$PATH" \
"$ROOT_DIR/scripts/backup-db.sh" >/dev/null
printf 'synthetic restore dump\n' >"$TEST_DIR/argv-restore.dump"
DB_TOOL_MODE=docker DATABASE_URL='postgresql://synthetic:argv-secret@db:5432/hyb_oa_restore_shadow' \
DB_CONTAINER=db-id POSTGRES_DB=hyb_oa_restore_shadow POSTGRES_USER=synthetic \
POSTGRES_PASSWORD=argv-secret RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE=hyb_oa_restore_shadow \
PATH="$MOCK_BIN:$PATH" "$ROOT_DIR/scripts/restore-db.sh" "$TEST_DIR/argv-restore.dump" >/dev/null
! grep -Fq 'argv-secret' "$MOCK_DOCKER_ARGV_LOG"
grep -Fq 'exec -e PGPASSWORD db-id pg_dump' "$MOCK_DOCKER_ARGV_LOG"
grep -Fq 'exec -e PGPASSWORD db-id psql' "$MOCK_DOCKER_ARGV_LOG"
grep -Fq 'exec -i -e PGPASSWORD db-id pg_restore' "$MOCK_DOCKER_ARGV_LOG"
[[ "$(grep -Fc 'PGPASSWORD=argv-secret' "$MOCK_DOCKER_ENV_LOG")" == "3" ]]
pass "Docker backup and both restore credential paths inherit PGPASSWORD without argv values"
! grep -Eq 'capture_command[[:space:]]+paired_backup[[:space:]]+env' "$ROOT_DIR/scripts/run-automatic-backup.sh"
! grep -Eq 'docker exec([^\n]|\\\n)*-e[[:space:]]+PGPASSWORD=' \
"$ROOT_DIR/scripts/backup-db.sh" "$ROOT_DIR/scripts/restore-db.sh"
pass "production backup scripts contain no value-bearing credential argv patterns"
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
for day in 01 02 03 04 05 06 07; do
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202602${day}T023000Z"
done
rm -f "$WRAPPER_STATE/last-success.json"
expect_failure "off-host finalize rejection fails the source run without publishing success" \
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=0 MOCK_SSH_EXIT=1 "$ROOT_DIR/scripts/run-automatic-backup.sh" run
[[ ! -e "$WRAPPER_STATE/last-success.json" ]]
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "8" ]]
[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
payload = json.load(handle)
assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
assert payload["findings"][0]["detail"] == "phase_offhost_finalize_failed"
PYTHON
pass "off-host finalize fail-closed preserves the source batch and emits a safe failure category"
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
for day in 01 02 03 04 05 06 07; do