修复异机备份凭证与中断恢复（07-27-offhost-backup-final-fixes）
这个提交包含在：
+1
-1
@@ -71,7 +71,7 @@ if [[ -z "$POSTGRES_DB" || -z "$POSTGRES_USER" || -z "$POSTGRES_PASSWORD" ]]; th
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$DB_CONTAINER" \
|
||||
if ! PGPASSWORD="$POSTGRES_PASSWORD" docker exec -e PGPASSWORD "$DB_CONTAINER" \
|
||||
pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" --format=c --no-owner --no-acl \
|
||||
>"$OUT_FILE"; then
|
||||
rm -f -- "$OUT_FILE"
|
||||
|
||||
@@ -128,6 +128,162 @@ expect_failure "receiver rolls back the daily archive when weekly checkpoint pub
|
||||
[[ -d "$ROLLBACK_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$ROLLBACK_LAYOUT/archive/$SECOND_BATCH" ]]
|
||||
pass "receiver restores the prior daily point after a later replacement step fails"
|
||||
|
||||
CURRENT_WEEK="$(python3 - "$CURRENT_RUN_ID" <<'PYTHON'
|
||||
from datetime import datetime
|
||||
import sys
|
||||
day = datetime.strptime(sys.argv[1][5:13], "%Y%m%d").date().isocalendar()
|
||||
print(f"week-{day.year:04d}-W{day.week:02d}")
|
||||
PYTHON
|
||||
)"
|
||||
|
||||
make_corrupt_partial() {
|
||||
local layout="$1"
|
||||
local run_id="$2"
|
||||
local batch_name="hyb-oa-backup-$run_id"
|
||||
make_batch "$layout/inbox/${batch_name}.partial" "$run_id"
|
||||
printf 'corrupt candidate\n' >>"$layout/inbox/${batch_name}.partial/database.dump"
|
||||
sha256sum "$layout/inbox/${batch_name}.partial/manifest.json" | awk '{print $1}'
|
||||
}
|
||||
|
||||
DAILY_COMPLETE_LAYOUT="$TEST_DIR/receiver-daily-orphan-complete"
|
||||
make_layout "$DAILY_COMPLETE_LAYOUT"
|
||||
make_batch "$DAILY_COMPLETE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.101" "$CURRENT_RUN_ID"
|
||||
make_batch "$DAILY_COMPLETE_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
|
||||
DAILY_COMPLETE_SHA="$(make_corrupt_partial "$DAILY_COMPLETE_LAYOUT" "$THIRD_RUN_ID")"
|
||||
expect_failure "receiver cleans a valid daily rollback after the replacement was published" \
|
||||
receiver "$DAILY_COMPLETE_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_COMPLETE_SHA"
|
||||
[[ ! -e "$DAILY_COMPLETE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.101" \
|
||||
&& -d "$DAILY_COMPLETE_LAYOUT/archive/$SECOND_BATCH" ]]
|
||||
[[ -d "$DAILY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" ]]
|
||||
pass "receiver keeps the valid published daily point and rebuilds its weekly checkpoint"
|
||||
|
||||
DAILY_WEEKLY_FAIL_LAYOUT="$TEST_DIR/receiver-daily-orphan-weekly-failure"
|
||||
make_layout "$DAILY_WEEKLY_FAIL_LAYOUT"
|
||||
DAILY_WEEKLY_FAIL_ORPHAN="$DAILY_WEEKLY_FAIL_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.107"
|
||||
make_batch "$DAILY_WEEKLY_FAIL_ORPHAN" "$CURRENT_RUN_ID"
|
||||
make_batch "$DAILY_WEEKLY_FAIL_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
|
||||
printf 'invalid weekly target\n' >"$DAILY_WEEKLY_FAIL_LAYOUT/weekly/$CURRENT_WEEK"
|
||||
DAILY_WEEKLY_FAIL_SHA="$(make_corrupt_partial "$DAILY_WEEKLY_FAIL_LAYOUT" "$THIRD_RUN_ID")"
|
||||
expect_failure "receiver keeps a daily rollback until weekly reconciliation succeeds" \
|
||||
receiver "$DAILY_WEEKLY_FAIL_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_WEEKLY_FAIL_SHA"
|
||||
[[ -d "$DAILY_WEEKLY_FAIL_ORPHAN" \
|
||||
&& -d "$DAILY_WEEKLY_FAIL_LAYOUT/archive/$SECOND_BATCH" \
|
||||
&& -f "$DAILY_WEEKLY_FAIL_LAYOUT/weekly/$CURRENT_WEEK" ]]
|
||||
pass "receiver preserves the daily rollback when weekly reconciliation fails"
|
||||
|
||||
DAILY_RESTORE_LAYOUT="$TEST_DIR/receiver-daily-orphan-restore"
|
||||
make_layout "$DAILY_RESTORE_LAYOUT"
|
||||
make_batch "$DAILY_RESTORE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.102" "$CURRENT_RUN_ID"
|
||||
DAILY_RESTORE_SHA="$(make_corrupt_partial "$DAILY_RESTORE_LAYOUT" "$SECOND_RUN_ID")"
|
||||
expect_failure "receiver restores a valid daily rollback when the published archive is absent" \
|
||||
receiver "$DAILY_RESTORE_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_RESTORE_SHA"
|
||||
[[ -d "$DAILY_RESTORE_LAYOUT/archive/$CURRENT_BATCH" \
|
||||
&& ! -e "$DAILY_RESTORE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.102" ]]
|
||||
[[ -d "$DAILY_RESTORE_LAYOUT/weekly/$CURRENT_WEEK" ]]
|
||||
pass "receiver restores the last daily point before rejecting the next corrupt candidate"
|
||||
|
||||
DAILY_AMBIGUOUS_LAYOUT="$TEST_DIR/receiver-daily-orphan-ambiguous"
|
||||
make_layout "$DAILY_AMBIGUOUS_LAYOUT"
|
||||
make_batch "$DAILY_AMBIGUOUS_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.103" "$CURRENT_RUN_ID"
|
||||
make_batch "$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
|
||||
printf 'corruption\n' >>"$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
||||
make_batch "$DAILY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
|
||||
DAILY_AMBIGUOUS_SHA="$(sha256sum "$DAILY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver fails closed when a daily rollback and invalid published archive coexist" \
|
||||
receiver "$DAILY_AMBIGUOUS_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_AMBIGUOUS_SHA"
|
||||
[[ -d "$DAILY_AMBIGUOUS_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.103" \
|
||||
&& -d "$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" ]]
|
||||
pass "receiver preserves both daily objects when replacement state is ambiguous"
|
||||
|
||||
DAILY_DUPLICATE_LAYOUT="$TEST_DIR/receiver-daily-orphan-duplicate"
|
||||
make_layout "$DAILY_DUPLICATE_LAYOUT"
|
||||
make_batch "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.104" "$CURRENT_RUN_ID"
|
||||
make_batch "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.105" "$CURRENT_RUN_ID"
|
||||
make_batch "$DAILY_DUPLICATE_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
||||
DAILY_DUPLICATE_SHA="$(sha256sum "$DAILY_DUPLICATE_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver fails closed on duplicate daily rollback orphans" \
|
||||
receiver "$DAILY_DUPLICATE_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_DUPLICATE_SHA"
|
||||
[[ -d "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.104" \
|
||||
&& -d "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.105" ]]
|
||||
pass "receiver preserves duplicate daily rollback evidence without choosing one"
|
||||
|
||||
DAILY_LINK_LAYOUT="$TEST_DIR/receiver-daily-orphan-link"
|
||||
make_layout "$DAILY_LINK_LAYOUT"
|
||||
make_batch "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
|
||||
ln -s "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" \
|
||||
"$DAILY_LINK_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.106"
|
||||
make_batch "$DAILY_LINK_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
||||
DAILY_LINK_SHA="$(sha256sum "$DAILY_LINK_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver rejects a symbolic-link daily rollback orphan" \
|
||||
receiver "$DAILY_LINK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_LINK_SHA"
|
||||
[[ -L "$DAILY_LINK_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.106" \
|
||||
&& -d "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" ]]
|
||||
pass "receiver leaves linked daily rollback evidence untouched"
|
||||
|
||||
WEEKLY_ROLLBACK_LAYOUT="$TEST_DIR/receiver-weekly-orphan-rollback"
|
||||
make_layout "$WEEKLY_ROLLBACK_LAYOUT"
|
||||
make_batch "$WEEKLY_ROLLBACK_LAYOUT/weekly/${CURRENT_WEEK}.rollback.201" "$CURRENT_RUN_ID"
|
||||
WEEKLY_ROLLBACK_SHA="$(make_corrupt_partial "$WEEKLY_ROLLBACK_LAYOUT" "$SECOND_RUN_ID")"
|
||||
expect_failure "receiver restores a weekly rollback when the canonical checkpoint is absent" \
|
||||
receiver "$WEEKLY_ROLLBACK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$WEEKLY_ROLLBACK_SHA"
|
||||
[[ -d "$WEEKLY_ROLLBACK_LAYOUT/weekly/$CURRENT_WEEK" \
|
||||
&& ! -e "$WEEKLY_ROLLBACK_LAYOUT/weekly/${CURRENT_WEEK}.rollback.201" ]]
|
||||
pass "receiver restores the authoritative weekly rollback before the next finalize"
|
||||
|
||||
WEEKLY_COMPLETE_LAYOUT="$TEST_DIR/receiver-weekly-orphan-complete"
|
||||
make_layout "$WEEKLY_COMPLETE_LAYOUT"
|
||||
make_batch "$WEEKLY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" "$SECOND_RUN_ID"
|
||||
make_batch "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.rollback.202" "$CURRENT_RUN_ID"
|
||||
mkdir "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202"
|
||||
printf 'interrupted staging\n' >"$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202/partial"
|
||||
WEEKLY_COMPLETE_SHA="$(make_corrupt_partial "$WEEKLY_COMPLETE_LAYOUT" "$THIRD_RUN_ID")"
|
||||
expect_failure "receiver cleans weekly rollback and staging after a valid checkpoint was published" \
|
||||
receiver "$WEEKLY_COMPLETE_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_COMPLETE_SHA"
|
||||
[[ -d "$WEEKLY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" \
|
||||
&& ! -e "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.rollback.202" \
|
||||
&& ! -e "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202" ]]
|
||||
pass "receiver treats weekly staging as non-authoritative after a valid publish"
|
||||
|
||||
WEEKLY_REBUILD_LAYOUT="$TEST_DIR/receiver-weekly-orphan-rebuild"
|
||||
make_layout "$WEEKLY_REBUILD_LAYOUT"
|
||||
make_batch "$WEEKLY_REBUILD_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
|
||||
mkdir "$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203"
|
||||
printf 'interrupted staging\n' >"$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203/partial"
|
||||
WEEKLY_REBUILD_SHA="$(make_corrupt_partial "$WEEKLY_REBUILD_LAYOUT" "$SECOND_RUN_ID")"
|
||||
expect_failure "receiver rebuilds a missing weekly point from one valid daily archive" \
|
||||
receiver "$WEEKLY_REBUILD_LAYOUT" finalize-test "$SECOND_RUN_ID" "$WEEKLY_REBUILD_SHA"
|
||||
[[ -d "$WEEKLY_REBUILD_LAYOUT/weekly/$CURRENT_WEEK" \
|
||||
&& ! -e "$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203" ]]
|
||||
[[ "$(stat -c %i "$WEEKLY_REBUILD_LAYOUT/weekly/$CURRENT_WEEK/database.dump")" \
|
||||
== "$(stat -c %i "$WEEKLY_REBUILD_LAYOUT/archive/$CURRENT_BATCH/database.dump")" ]]
|
||||
pass "receiver rebuilds the weekly checkpoint from the unique valid daily source"
|
||||
|
||||
WEEKLY_AMBIGUOUS_LAYOUT="$TEST_DIR/receiver-weekly-orphan-ambiguous"
|
||||
make_layout "$WEEKLY_AMBIGUOUS_LAYOUT"
|
||||
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
|
||||
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
|
||||
mkdir "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/${CURRENT_WEEK}.new.204"
|
||||
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
|
||||
WEEKLY_AMBIGUOUS_SHA="$(sha256sum "$WEEKLY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver fails closed when weekly staging has multiple daily sources" \
|
||||
receiver "$WEEKLY_AMBIGUOUS_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_AMBIGUOUS_SHA"
|
||||
[[ -d "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/${CURRENT_WEEK}.new.204" \
|
||||
&& ! -e "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/$CURRENT_WEEK" ]]
|
||||
pass "receiver preserves ambiguous weekly staging without choosing a daily source"
|
||||
|
||||
WEEKLY_INVALID_LAYOUT="$TEST_DIR/receiver-weekly-orphan-invalid"
|
||||
make_layout "$WEEKLY_INVALID_LAYOUT"
|
||||
make_batch "$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK" "$SECOND_RUN_ID"
|
||||
printf 'corruption\n' >>"$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK/database.dump"
|
||||
make_batch "$WEEKLY_INVALID_LAYOUT/weekly/${CURRENT_WEEK}.rollback.205" "$CURRENT_RUN_ID"
|
||||
make_batch "$WEEKLY_INVALID_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
|
||||
WEEKLY_INVALID_SHA="$(sha256sum "$WEEKLY_INVALID_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
||||
expect_failure "receiver fails closed when a weekly rollback and invalid canonical coexist" \
|
||||
receiver "$WEEKLY_INVALID_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_INVALID_SHA"
|
||||
[[ -d "$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK" \
|
||||
&& -d "$WEEKLY_INVALID_LAYOUT/weekly/${CURRENT_WEEK}.rollback.205" ]]
|
||||
pass "receiver preserves both weekly objects when checkpoint state is ambiguous"
|
||||
|
||||
negative_receiver_case() {
|
||||
local label="$1"
|
||||
local mutation="$2"
|
||||
@@ -241,7 +397,10 @@ printf 'backup-alias ssh-ed25519 fixture\n' >"$TEST_DIR/known-hosts"
|
||||
BUILD_SHA="abcdef1234567890abcdef1234567890abcdef12"
|
||||
VITE_BUILD_SHA="${BUILD_SHA:0:12}"
|
||||
MOCK_ALERT_LOG="$TEST_DIR/mock-alert.log"
|
||||
MOCK_DOCKER_ARGV_LOG="$TEST_DIR/mock-docker-argv.log"
|
||||
MOCK_DOCKER_ENV_LOG="$TEST_DIR/mock-docker-env.log"
|
||||
export MOCK_PG="$WRAPPER_PG" MOCK_STORAGE="$WRAPPER_STORAGE" MOCK_BUILD_SHA="$BUILD_SHA" MOCK_ALERT_LOG
|
||||
export MOCK_DOCKER_ARGV_LOG MOCK_DOCKER_ENV_LOG
|
||||
|
||||
cat >"$MOCK_BIN/docker" <<'MOCK'
|
||||
#!/usr/bin/env bash
|
||||
@@ -278,8 +437,13 @@ if [[ "${1:-}" == "inspect" ]]; then
|
||||
exit 0
|
||||
fi
|
||||
if [[ "${1:-}" == "exec" ]]; then
|
||||
printf '%q ' "$@" >>"$MOCK_DOCKER_ARGV_LOG"
|
||||
printf '\n' >>"$MOCK_DOCKER_ARGV_LOG"
|
||||
printf 'PGPASSWORD=%s\n' "${PGPASSWORD:-}" >>"$MOCK_DOCKER_ENV_LOG"
|
||||
if [[ " $* " == *" pg_dump "* ]]; then printf 'synthetic-dump\n'; exit 0; fi
|
||||
if [[ " $* " == *" psql "* ]]; then printf '%s\n' "${POSTGRES_DB:-hyb_oa_restore_shadow}"; exit 0; fi
|
||||
if [[ " $* " == *" pg_restore --list "* ]]; then cat >/dev/null; exit 0; fi
|
||||
if [[ " $* " == *" pg_restore "* ]]; then cat >/dev/null; exit 0; fi
|
||||
fi
|
||||
exit 1
|
||||
MOCK
|
||||
@@ -365,6 +529,54 @@ pass "wrapper failure summary is category-only and redacted"
|
||||
[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
|
||||
grep -Fq 'synthetic transfer diagnostic' "$WRAPPER_STATE/last-error.log"
|
||||
pass "wrapper retains the latest command diagnostic locally with mode 0600"
|
||||
[[ -s "$MOCK_DOCKER_ARGV_LOG" && -s "$MOCK_DOCKER_ENV_LOG" ]]
|
||||
! grep -Fq 'fixture' "$MOCK_DOCKER_ARGV_LOG"
|
||||
! grep -Fq 'postgresql://' "$MOCK_DOCKER_ARGV_LOG"
|
||||
grep -Fq 'PGPASSWORD=fixture' "$MOCK_DOCKER_ENV_LOG"
|
||||
pass "wrapper and nightly Docker backup keep connection secrets out of captured argv"
|
||||
|
||||
: >"$MOCK_DOCKER_ARGV_LOG"
|
||||
: >"$MOCK_DOCKER_ENV_LOG"
|
||||
DB_TOOL_MODE=docker DATABASE_URL='postgresql://synthetic:argv-secret@db:5432/hyb_oa_restore_shadow' \
|
||||
DB_CONTAINER=db-id POSTGRES_DB=hyb_oa_restore_shadow POSTGRES_USER=synthetic \
|
||||
POSTGRES_PASSWORD=argv-secret OUT_FILE="$TEST_DIR/argv-backup.dump" PATH="$MOCK_BIN:$PATH" \
|
||||
"$ROOT_DIR/scripts/backup-db.sh" >/dev/null
|
||||
printf 'synthetic restore dump\n' >"$TEST_DIR/argv-restore.dump"
|
||||
DB_TOOL_MODE=docker DATABASE_URL='postgresql://synthetic:argv-secret@db:5432/hyb_oa_restore_shadow' \
|
||||
DB_CONTAINER=db-id POSTGRES_DB=hyb_oa_restore_shadow POSTGRES_USER=synthetic \
|
||||
POSTGRES_PASSWORD=argv-secret RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE=hyb_oa_restore_shadow \
|
||||
PATH="$MOCK_BIN:$PATH" "$ROOT_DIR/scripts/restore-db.sh" "$TEST_DIR/argv-restore.dump" >/dev/null
|
||||
! grep -Fq 'argv-secret' "$MOCK_DOCKER_ARGV_LOG"
|
||||
grep -Fq 'exec -e PGPASSWORD db-id pg_dump' "$MOCK_DOCKER_ARGV_LOG"
|
||||
grep -Fq 'exec -e PGPASSWORD db-id psql' "$MOCK_DOCKER_ARGV_LOG"
|
||||
grep -Fq 'exec -i -e PGPASSWORD db-id pg_restore' "$MOCK_DOCKER_ARGV_LOG"
|
||||
[[ "$(grep -Fc 'PGPASSWORD=argv-secret' "$MOCK_DOCKER_ENV_LOG")" == "3" ]]
|
||||
pass "Docker backup and both restore credential paths inherit PGPASSWORD without argv values"
|
||||
|
||||
! grep -Eq 'capture_command[[:space:]]+paired_backup[[:space:]]+env' "$ROOT_DIR/scripts/run-automatic-backup.sh"
|
||||
! grep -Eq 'docker exec([^\n]|\\\n)*-e[[:space:]]+PGPASSWORD=' \
|
||||
"$ROOT_DIR/scripts/backup-db.sh" "$ROOT_DIR/scripts/restore-db.sh"
|
||||
pass "production backup scripts contain no value-bearing credential argv patterns"
|
||||
|
||||
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
|
||||
for day in 01 02 03 04 05 06 07; do
|
||||
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202602${day}T023000Z"
|
||||
done
|
||||
rm -f "$WRAPPER_STATE/last-success.json"
|
||||
expect_failure "off-host finalize rejection fails the source run without publishing success" \
|
||||
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=0 MOCK_SSH_EXIT=1 "$ROOT_DIR/scripts/run-automatic-backup.sh" run
|
||||
[[ ! -e "$WRAPPER_STATE/last-success.json" ]]
|
||||
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "8" ]]
|
||||
[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
|
||||
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
|
||||
import json
|
||||
import sys
|
||||
with open(sys.argv[1], encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
|
||||
assert payload["findings"][0]["detail"] == "phase_offhost_finalize_failed"
|
||||
PYTHON
|
||||
pass "off-host finalize fail-closed preserves the source batch and emits a safe failure category"
|
||||
|
||||
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
|
||||
for day in 01 02 03 04 05 06 07; do
|
||||
|
||||
@@ -15,6 +15,8 @@ RRSYNC_BIN="${RRSYNC_BIN:-/usr/bin/rrsync}"
|
||||
PYTHON_BIN="${PYTHON_BIN:-python3}"
|
||||
DAILY_KEEP="${DAILY_KEEP:-7}"
|
||||
WEEKLY_KEEP="${WEEKLY_KEEP:-4}"
|
||||
declare -A RECONCILED_WEEK_SOURCE=()
|
||||
declare -a DAILY_ORPHANS_PENDING_DELETE=()
|
||||
|
||||
fail() {
|
||||
echo "[offhost-receiver][ERROR] $1" >&2
|
||||
@@ -96,7 +98,8 @@ validate_batch() {
|
||||
[[ "$actual_manifest_sha" == "$expected_manifest_sha" ]] \
|
||||
|| fail "manifest checksum mismatch"
|
||||
|
||||
"$PYTHON_BIN" - "$batch_dir" "$run_id" <<'PYTHON'
|
||||
"$PYTHON_BIN" - "$batch_dir" "$run_id" <<'PYTHON' \
|
||||
|| fail "batch manifest or artifact validation failed"
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
@@ -135,6 +138,128 @@ for key, expected_name in expected.items():
|
||||
PYTHON
|
||||
}
|
||||
|
||||
validated_batch_identity() {
|
||||
local batch_dir="$1"
|
||||
[[ -d "$batch_dir" && ! -L "$batch_dir" ]] || fail "recovery candidate is not a regular directory"
|
||||
|
||||
local run_id manifest_sha identity
|
||||
run_id="$($PYTHON_BIN - "$batch_dir/manifest.json" <<'PYTHON'
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
with open(sys.argv[1], "r", encoding="utf-8") as handle:
|
||||
manifest = json.load(handle)
|
||||
run_id = manifest.get("runId")
|
||||
if not isinstance(run_id, str) or not re.fullmatch(r"auto-[0-9]{8}T[0-9]{6}Z", run_id):
|
||||
raise SystemExit("manifest run id is invalid")
|
||||
print(run_id)
|
||||
PYTHON
|
||||
)" || fail "recovery candidate manifest identity is invalid"
|
||||
manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')"
|
||||
validate_batch "$batch_dir" "$run_id" "$manifest_sha"
|
||||
identity="$($PYTHON_BIN - "$run_id" <<'PYTHON'
|
||||
from datetime import datetime
|
||||
import sys
|
||||
|
||||
run_id = sys.argv[1]
|
||||
day = run_id.removeprefix("auto-").split("T", 1)[0]
|
||||
iso = datetime.strptime(day, "%Y%m%d").date().isocalendar()
|
||||
print(f"{run_id}\t{day}\tweek-{iso.year:04d}-W{iso.week:02d}")
|
||||
PYTHON
|
||||
)" || fail "recovery candidate date identity is invalid"
|
||||
printf '%s\t%s\n' "$identity" "$manifest_sha"
|
||||
}
|
||||
|
||||
validate_temp_directory() {
|
||||
local target="$1"
|
||||
local root="$2"
|
||||
local pattern="$3"
|
||||
[[ "$(dirname -- "$target")" == "$root" ]] || fail "orphan target escaped its managed root"
|
||||
[[ "$(basename -- "$target")" =~ $pattern ]] || fail "orphan target name is invalid"
|
||||
[[ -d "$target" && ! -L "$target" ]] || fail "orphan target is not a regular directory"
|
||||
}
|
||||
|
||||
remove_temp_directory() {
|
||||
local target="$1"
|
||||
local root="$2"
|
||||
local pattern="$3"
|
||||
validate_temp_directory "$target" "$root" "$pattern"
|
||||
rm -rf -- "$target"
|
||||
}
|
||||
|
||||
canonical_daily_for_day() {
|
||||
local run_day="$1"
|
||||
local -a entries=()
|
||||
mapfile -t entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
|
||||
-name "hyb-oa-backup-auto-${run_day}T*Z" -print | LC_ALL=C sort)
|
||||
[[ "${#entries[@]}" -le 1 ]] || fail "daily archive invariant is ambiguous during recovery"
|
||||
if [[ "${#entries[@]}" -eq 1 ]]; then
|
||||
printf '%s\n' "${entries[0]}"
|
||||
fi
|
||||
}
|
||||
|
||||
reconcile_daily_orphans() {
|
||||
local -a orphans=()
|
||||
mapfile -t orphans < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
|
||||
-name '.daily-replacement-*' -print | LC_ALL=C sort)
|
||||
local orphan base run_day identity orphan_run_id orphan_day orphan_week orphan_sha
|
||||
local canonical canonical_identity canonical_run_id canonical_day canonical_week canonical_sha
|
||||
local -A seen_days=() identities=() canonicals=()
|
||||
for orphan in "${orphans[@]}"; do
|
||||
base="$(basename -- "$orphan")"
|
||||
[[ "$base" =~ ^\.daily-replacement-([0-9]{8})\.([0-9]+)$ ]] \
|
||||
|| fail "daily replacement orphan name is invalid"
|
||||
run_day="${BASH_REMATCH[1]}"
|
||||
[[ -z "${seen_days[$run_day]:-}" ]] || fail "multiple daily replacement orphans exist for one day"
|
||||
seen_days[$run_day]=1
|
||||
validate_temp_directory "$orphan" "$BACKUP_ARCHIVE_DIR" \
|
||||
'^\.daily-replacement-[0-9]{8}\.[0-9]+$'
|
||||
identity="$(validated_batch_identity "$orphan")" \
|
||||
|| fail "daily replacement orphan validation failed"
|
||||
IFS=$'\t' read -r orphan_run_id orphan_day orphan_week orphan_sha <<<"$identity"
|
||||
[[ "$orphan_day" == "$run_day" ]] || fail "daily replacement orphan day does not match its manifest"
|
||||
identities[$orphan]="$identity"
|
||||
|
||||
canonical="$(canonical_daily_for_day "$run_day")" \
|
||||
|| fail "daily archive lookup failed during recovery"
|
||||
canonicals[$run_day]="$canonical"
|
||||
if [[ -n "$canonical" ]]; then
|
||||
[[ -d "$canonical" && ! -L "$canonical" ]] || fail "daily archive recovery target is invalid"
|
||||
canonical_identity="$(validated_batch_identity "$canonical")" \
|
||||
|| fail "daily archive validation failed during recovery"
|
||||
IFS=$'\t' read -r canonical_run_id canonical_day canonical_week canonical_sha <<<"$canonical_identity"
|
||||
[[ "$canonical_day" == "$run_day" \
|
||||
&& "$(basename -- "$canonical")" == "hyb-oa-backup-$canonical_run_id" ]] \
|
||||
|| fail "daily archive identity is inconsistent during recovery"
|
||||
[[ "$canonical_week" == "$orphan_week" ]] || fail "daily replacement week identity is inconsistent"
|
||||
else
|
||||
canonical="$BACKUP_ARCHIVE_DIR/hyb-oa-backup-$orphan_run_id"
|
||||
[[ ! -e "$canonical" ]] || fail "daily recovery target already exists"
|
||||
fi
|
||||
done
|
||||
|
||||
for orphan in "${orphans[@]}"; do
|
||||
base="$(basename -- "$orphan")"
|
||||
[[ "$base" =~ ^\.daily-replacement-([0-9]{8})\.([0-9]+)$ ]] \
|
||||
|| fail "daily replacement orphan name changed during recovery"
|
||||
run_day="${BASH_REMATCH[1]}"
|
||||
identity="${identities[$orphan]}"
|
||||
IFS=$'\t' read -r orphan_run_id orphan_day orphan_week orphan_sha <<<"$identity"
|
||||
canonical="${canonicals[$run_day]}"
|
||||
if [[ -z "$canonical" ]]; then
|
||||
canonical="$BACKUP_ARCHIVE_DIR/hyb-oa-backup-$orphan_run_id"
|
||||
validate_temp_directory "$orphan" "$BACKUP_ARCHIVE_DIR" \
|
||||
'^\.daily-replacement-[0-9]{8}\.[0-9]+$'
|
||||
[[ ! -e "$canonical" ]] || fail "daily recovery target appeared during recovery"
|
||||
mv -- "$orphan" "$canonical"
|
||||
else
|
||||
DAILY_ORPHANS_PENDING_DELETE+=("$orphan")
|
||||
fi
|
||||
RECONCILED_WEEK_SOURCE[$orphan_week]="$canonical"
|
||||
done
|
||||
}
|
||||
|
||||
prune_named_directories() {
|
||||
local root="$1"
|
||||
local prefix="$2"
|
||||
@@ -186,6 +311,138 @@ publish_weekly_checkpoint() {
|
||||
fi
|
||||
}
|
||||
|
||||
unique_daily_source_for_week() {
|
||||
local expected_week="$1"
|
||||
local -a candidates=() entries=()
|
||||
mapfile -t entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 \
|
||||
-name 'hyb-oa-backup-auto-*' -print | LC_ALL=C sort)
|
||||
local entry identity run_id run_day iso_week manifest_sha
|
||||
for entry in "${entries[@]}"; do
|
||||
[[ -d "$entry" && ! -L "$entry" ]] || fail "daily archive candidate is invalid during weekly recovery"
|
||||
identity="$(validated_batch_identity "$entry")" \
|
||||
|| fail "daily archive validation failed during weekly recovery"
|
||||
IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
|
||||
[[ "$(basename -- "$entry")" == "hyb-oa-backup-$run_id" ]] \
|
||||
|| fail "daily archive name does not match its manifest"
|
||||
if [[ "$iso_week" == "$expected_week" ]]; then
|
||||
candidates+=("$entry")
|
||||
fi
|
||||
done
|
||||
[[ "${#candidates[@]}" -eq 1 ]] || fail "weekly recovery source is not unique"
|
||||
printf '%s\n' "${candidates[0]}"
|
||||
}
|
||||
|
||||
reconcile_weekly_orphans() {
|
||||
local -a entries=()
|
||||
mapfile -t entries < <(find "$BACKUP_WEEKLY_DIR" -mindepth 1 -maxdepth 1 \
|
||||
\( -name 'week-*-W*.new.*' -o -name 'week-*-W*.rollback.*' \) -print | LC_ALL=C sort)
|
||||
local -A new_by_week=() rollback_by_week=() weeks=()
|
||||
local entry base week kind
|
||||
for entry in "${entries[@]}"; do
|
||||
base="$(basename -- "$entry")"
|
||||
if [[ "$base" =~ ^(week-[0-9]{4}-W[0-9]{2})\.(new|rollback)\.([0-9]+)$ ]]; then
|
||||
week="${BASH_REMATCH[1]}"
|
||||
kind="${BASH_REMATCH[2]}"
|
||||
else
|
||||
fail "weekly orphan name is invalid"
|
||||
fi
|
||||
validate_temp_directory "$entry" "$BACKUP_WEEKLY_DIR" \
|
||||
'^week-[0-9]{4}-W[0-9]{2}\.(new|rollback)\.[0-9]+$'
|
||||
weeks[$week]=1
|
||||
if [[ "$kind" == "new" ]]; then
|
||||
[[ -z "${new_by_week[$week]:-}" ]] || fail "multiple weekly staging orphans exist"
|
||||
new_by_week[$week]="$entry"
|
||||
else
|
||||
[[ -z "${rollback_by_week[$week]:-}" ]] || fail "multiple weekly rollback orphans exist"
|
||||
rollback_by_week[$week]="$entry"
|
||||
fi
|
||||
done
|
||||
for week in "${!RECONCILED_WEEK_SOURCE[@]}"; do
|
||||
weeks[$week]=1
|
||||
done
|
||||
|
||||
local -A sources=()
|
||||
local canonical new_dir rollback_dir identity run_id run_day iso_week manifest_sha source
|
||||
for week in "${!weeks[@]}"; do
|
||||
canonical="$BACKUP_WEEKLY_DIR/$week"
|
||||
new_dir="${new_by_week[$week]:-}"
|
||||
rollback_dir="${rollback_by_week[$week]:-}"
|
||||
|
||||
if [[ -e "$canonical" ]]; then
|
||||
[[ -d "$canonical" && ! -L "$canonical" ]] || fail "weekly checkpoint is not a regular directory"
|
||||
identity="$(validated_batch_identity "$canonical")" \
|
||||
|| fail "weekly checkpoint validation failed during recovery"
|
||||
IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
|
||||
[[ "$iso_week" == "$week" ]] || fail "weekly checkpoint manifest belongs to another week"
|
||||
fi
|
||||
if [[ -n "$rollback_dir" ]]; then
|
||||
identity="$(validated_batch_identity "$rollback_dir")" \
|
||||
|| fail "weekly rollback validation failed"
|
||||
IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
|
||||
[[ "$iso_week" == "$week" ]] || fail "weekly rollback manifest belongs to another week"
|
||||
fi
|
||||
|
||||
source="${RECONCILED_WEEK_SOURCE[$week]:-}"
|
||||
if [[ -z "$source" && ! -e "$canonical" && -z "$rollback_dir" && -n "$new_dir" ]]; then
|
||||
source="$(unique_daily_source_for_week "$week")" \
|
||||
|| fail "weekly staging recovery source could not be resolved"
|
||||
fi
|
||||
if [[ -n "$source" ]]; then
|
||||
[[ -d "$source" && ! -L "$source" ]] || fail "weekly rebuild source is invalid"
|
||||
identity="$(validated_batch_identity "$source")" \
|
||||
|| fail "weekly rebuild source validation failed"
|
||||
IFS=$'\t' read -r run_id run_day iso_week manifest_sha <<<"$identity"
|
||||
[[ "$iso_week" == "$week" ]] || fail "weekly rebuild source belongs to another week"
|
||||
sources[$week]="$source"
|
||||
fi
|
||||
[[ -e "$canonical" || -n "$rollback_dir" || -n "$source" ]] \
|
||||
|| fail "weekly checkpoint has no authoritative recovery source"
|
||||
done
|
||||
|
||||
for week in "${!weeks[@]}"; do
|
||||
canonical="$BACKUP_WEEKLY_DIR/$week"
|
||||
new_dir="${new_by_week[$week]:-}"
|
||||
rollback_dir="${rollback_by_week[$week]:-}"
|
||||
|
||||
if [[ ! -e "$canonical" && -n "$rollback_dir" ]]; then
|
||||
validate_temp_directory "$rollback_dir" "$BACKUP_WEEKLY_DIR" \
|
||||
'^week-[0-9]{4}-W[0-9]{2}\.rollback\.[0-9]+$'
|
||||
mv -- "$rollback_dir" "$canonical"
|
||||
rollback_dir=""
|
||||
fi
|
||||
|
||||
source="${sources[$week]:-}"
|
||||
if [[ -n "$source" ]]; then
|
||||
[[ -d "$source" && ! -L "$source" ]] || fail "weekly rebuild source is invalid"
|
||||
if ! publish_weekly_checkpoint "$source" "$canonical" "$([[ -e "$canonical" ]] && echo 1 || echo 0)"; then
|
||||
fail "weekly checkpoint reconciliation failed"
|
||||
fi
|
||||
fi
|
||||
|
||||
[[ -e "$canonical" ]] || fail "weekly checkpoint is missing after reconciliation"
|
||||
if [[ -n "$rollback_dir" ]]; then
|
||||
remove_temp_directory "$rollback_dir" "$BACKUP_WEEKLY_DIR" \
|
||||
'^week-[0-9]{4}-W[0-9]{2}\.rollback\.[0-9]+$'
|
||||
fi
|
||||
if [[ -n "$new_dir" ]]; then
|
||||
remove_temp_directory "$new_dir" "$BACKUP_WEEKLY_DIR" \
|
||||
'^week-[0-9]{4}-W[0-9]{2}\.new\.[0-9]+$'
|
||||
fi
|
||||
done
|
||||
local pending_daily_orphan
|
||||
for pending_daily_orphan in "${DAILY_ORPHANS_PENDING_DELETE[@]}"; do
|
||||
remove_temp_directory "$pending_daily_orphan" "$BACKUP_ARCHIVE_DIR" \
|
||||
'^\.daily-replacement-[0-9]{8}\.[0-9]+$'
|
||||
done
|
||||
DAILY_ORPHANS_PENDING_DELETE=()
|
||||
RECONCILED_WEEK_SOURCE=()
|
||||
}
|
||||
|
||||
reconcile_interrupted_finalize() {
|
||||
reconcile_daily_orphans
|
||||
reconcile_weekly_orphans
|
||||
}
|
||||
|
||||
finalize_batch() {
|
||||
local run_id="$1"
|
||||
local expected_manifest_sha="$2"
|
||||
@@ -194,6 +451,7 @@ finalize_batch() {
|
||||
exec 9>"$BACKUP_ARCHIVE_DIR/.finalize.lock"
|
||||
chmod 0600 "$BACKUP_ARCHIVE_DIR/.finalize.lock"
|
||||
flock -n 9 || fail "another finalize operation is already running"
|
||||
reconcile_interrupted_finalize
|
||||
|
||||
local batch_name="hyb-oa-backup-$run_id"
|
||||
local partial_dir="$BACKUP_INBOX_DIR/${batch_name}.partial"
|
||||
|
||||
+2
-2
@@ -98,7 +98,7 @@ if [[ -z "$POSTGRES_DB" || -z "$POSTGRES_USER" || -z "$POSTGRES_PASSWORD" ]]; th
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ACTUAL_DATABASE="$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$DB_CONTAINER" \
|
||||
ACTUAL_DATABASE="$(PGPASSWORD="$POSTGRES_PASSWORD" docker exec -e PGPASSWORD "$DB_CONTAINER" \
|
||||
psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atqc 'SELECT current_database()')"
|
||||
|
||||
if [[ -z "$ACTUAL_DATABASE" || "$ACTUAL_DATABASE" != "$RESTORE_CONFIRM_DATABASE" ]]; then
|
||||
@@ -110,7 +110,7 @@ validate_shadow_database_name "$ACTUAL_DATABASE"
|
||||
docker exec -i "$DB_CONTAINER" pg_restore --list <"$BACKUP_FILE" >/dev/null
|
||||
|
||||
echo "[INFO] DB restore target confirmed: $ACTUAL_DATABASE (shadow)"
|
||||
docker exec -i -e PGPASSWORD="$POSTGRES_PASSWORD" "$DB_CONTAINER" \
|
||||
PGPASSWORD="$POSTGRES_PASSWORD" docker exec -i -e PGPASSWORD "$DB_CONTAINER" \
|
||||
pg_restore --clean --if-exists --no-owner --no-acl -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
||||
<"$BACKUP_FILE"
|
||||
|
||||
|
||||
@@ -341,12 +341,15 @@ print(f"{unquote(value.username)}\t{database_name}")
|
||||
[[ "$source_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "active server image digest is invalid"
|
||||
|
||||
CURRENT_PHASE="paired_backup"
|
||||
if ! capture_command paired_backup env \
|
||||
DATABASE_URL="$database_url" DB_TOOL_MODE=docker DB_CONTAINER="$DB_CONTAINER" \
|
||||
POSTGRES_DB="$postgres_db" POSTGRES_USER="$postgres_user" POSTGRES_PASSWORD="$postgres_password" \
|
||||
STORAGE_DIR_HOST="$STORAGE_HOST_DIR" BACKUP_DIR="$AUTOMATIC_BACKUP_DIR" \
|
||||
BACKUP_RUN_ID="$run_id" BUILD_SHA="$BUILD_SHA" SOURCE_IMAGE_DIGEST="$source_image_digest" \
|
||||
"$ROOT_DIR/scripts/backup-all.sh"; then
|
||||
if ! (
|
||||
export DATABASE_URL="$database_url"
|
||||
export DB_TOOL_MODE=docker DB_CONTAINER
|
||||
export POSTGRES_DB="$postgres_db" POSTGRES_USER="$postgres_user"
|
||||
export POSTGRES_PASSWORD="$postgres_password"
|
||||
export STORAGE_DIR_HOST="$STORAGE_HOST_DIR" BACKUP_DIR="$AUTOMATIC_BACKUP_DIR"
|
||||
export BACKUP_RUN_ID="$run_id" BUILD_SHA SOURCE_IMAGE_DIGEST="$source_image_digest"
|
||||
capture_command paired_backup "$ROOT_DIR/scripts/backup-all.sh"
|
||||
); then
|
||||
fail "paired backup command failed"
|
||||
fi
|
||||
[[ -d "$batch_dir" && -f "$batch_dir/manifest.json" ]] || fail "complete local batch was not published"
|
||||
|
||||
在新工单中引用
屏蔽一个用户