737 行
40 KiB
Bash
可执行文件
737 行
40 KiB
Bash
可执行文件
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
TEST_DIR="$(mktemp -d)"
|
|
PASS_COUNT=0
|
|
|
|
cleanup() {
|
|
if [[ -n "${TEST_DIR:-}" && -d "$TEST_DIR" && "$TEST_DIR" == /tmp/* ]]; then
|
|
rm -rf -- "$TEST_DIR"
|
|
fi
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
pass() {
|
|
PASS_COUNT=$((PASS_COUNT + 1))
|
|
printf '[PASS] %s\n' "$1"
|
|
}
|
|
|
|
expect_failure() {
|
|
local label="$1"
|
|
shift
|
|
if "$@" >"$TEST_DIR/last-command.out" 2>&1; then
|
|
printf '[FAIL] expected rejection: %s\n' "$label" >&2
|
|
exit 1
|
|
fi
|
|
pass "$label"
|
|
}
|
|
|
|
make_layout() {
|
|
local root="$1"
|
|
mkdir -p "$root/inbox" "$root/archive" "$root/weekly"
|
|
}
|
|
|
|
make_batch() {
|
|
local target="$1"
|
|
local run_id="$2"
|
|
mkdir -p "$target"
|
|
printf 'synthetic database payload\n' >"$target/database.dump"
|
|
printf 'synthetic storage payload\n' >"$target/storage.tar.gz"
|
|
python3 - "$target" "$run_id" <<'PYTHON'
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
target, run_id = sys.argv[1:]
|
|
def artifact(file_name):
|
|
file_path = os.path.join(target, file_name)
|
|
with open(file_path, "rb") as handle:
|
|
content = handle.read()
|
|
return {"file": file_name, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()}
|
|
|
|
manifest = {
|
|
"schemaVersion": 1,
|
|
"runId": run_id,
|
|
"createdAt": "2026-01-01T00:00:00Z",
|
|
"buildSha": "a" * 40,
|
|
"sourceImageDigest": None,
|
|
"status": "complete",
|
|
"artifacts": {
|
|
"database": artifact("database.dump"),
|
|
"storage": artifact("storage.tar.gz"),
|
|
},
|
|
}
|
|
with open(os.path.join(target, "manifest.json"), "x", encoding="utf-8") as handle:
|
|
json.dump(manifest, handle, indent=2)
|
|
handle.write("\n")
|
|
PYTHON
|
|
}
|
|
|
|
receiver() {
|
|
local layout="$1"
|
|
shift
|
|
env RECEIVER_TEST_MODE=1 RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
|
BACKUP_INBOX_DIR="$layout/inbox" BACKUP_ARCHIVE_DIR="$layout/archive" \
|
|
BACKUP_WEEKLY_DIR="$layout/weekly" DAILY_KEEP=7 WEEKLY_KEEP=4 \
|
|
"$ROOT_DIR/scripts/receive-offhost-backup.sh" "$@"
|
|
}
|
|
|
|
CURRENT_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z"
|
|
CURRENT_BATCH="hyb-oa-backup-$CURRENT_RUN_ID"
|
|
|
|
VALID_LAYOUT="$TEST_DIR/receiver-valid"
|
|
make_layout "$VALID_LAYOUT"
|
|
make_batch "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
|
VALID_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
|
|
[[ -d "$VALID_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" ]]
|
|
pass "receiver finalizes one complete synthetic batch"
|
|
|
|
receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
|
|
pass "receiver treats an identical repeated finalize as idempotent"
|
|
|
|
SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z"
|
|
SECOND_BATCH="hyb-oa-backup-$SECOND_RUN_ID"
|
|
make_batch "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
|
SECOND_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA" >/dev/null
|
|
[[ ! -e "$VALID_LAYOUT/archive/$CURRENT_BATCH" && -d "$VALID_LAYOUT/archive/$SECOND_BATCH" ]]
|
|
WEEKLY_BATCH="$(find "$VALID_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -print -quit)"
|
|
[[ -n "$WEEKLY_BATCH" ]]
|
|
[[ "$(stat -c %i "$WEEKLY_BATCH/database.dump")" == "$(stat -c %i "$VALID_LAYOUT/archive/$SECOND_BATCH/database.dump")" ]]
|
|
pass "receiver atomically replaces a different batch for the same UTC day and rebuilds its weekly checkpoint"
|
|
|
|
THIRD_RUN_ID="auto-$(date -u +%Y%m%d)T000002Z"
|
|
THIRD_BATCH="hyb-oa-backup-$THIRD_RUN_ID"
|
|
make_batch "$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
|
|
printf 'corrupt candidate\n' >>"$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial/database.dump"
|
|
THIRD_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
expect_failure "receiver validates a same-day replacement before touching the current archive" \
|
|
receiver "$VALID_LAYOUT" finalize-test "$THIRD_RUN_ID" "$THIRD_MANIFEST_SHA"
|
|
[[ -d "$VALID_LAYOUT/archive/$SECOND_BATCH" && ! -e "$VALID_LAYOUT/archive/$THIRD_BATCH" ]]
|
|
pass "receiver preserves the current daily archive when a replacement candidate is corrupt"
|
|
|
|
ROLLBACK_LAYOUT="$TEST_DIR/receiver-replacement-rollback"
|
|
make_layout "$ROLLBACK_LAYOUT"
|
|
make_batch "$ROLLBACK_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
|
ROLLBACK_FIRST_SHA="$(sha256sum "$ROLLBACK_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
receiver "$ROLLBACK_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$ROLLBACK_FIRST_SHA" >/dev/null
|
|
ROLLBACK_WEEKLY="$(find "$ROLLBACK_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -print -quit)"
|
|
find "$ROLLBACK_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -exec rm -rf -- {} +
|
|
printf 'invalid weekly target\n' >"$ROLLBACK_WEEKLY"
|
|
make_batch "$ROLLBACK_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
|
ROLLBACK_SECOND_SHA="$(sha256sum "$ROLLBACK_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
expect_failure "receiver rolls back the daily archive when weekly checkpoint publication fails" \
|
|
receiver "$ROLLBACK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$ROLLBACK_SECOND_SHA"
|
|
[[ -d "$ROLLBACK_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$ROLLBACK_LAYOUT/archive/$SECOND_BATCH" ]]
|
|
pass "receiver restores the prior daily point after a later replacement step fails"
|
|
|
|
CURRENT_WEEK="$(python3 - "$CURRENT_RUN_ID" <<'PYTHON'
|
|
from datetime import datetime
|
|
import sys
|
|
day = datetime.strptime(sys.argv[1][5:13], "%Y%m%d").date().isocalendar()
|
|
print(f"week-{day.year:04d}-W{day.week:02d}")
|
|
PYTHON
|
|
)"
|
|
|
|
make_corrupt_partial() {
|
|
local layout="$1"
|
|
local run_id="$2"
|
|
local batch_name="hyb-oa-backup-$run_id"
|
|
make_batch "$layout/inbox/${batch_name}.partial" "$run_id"
|
|
printf 'corrupt candidate\n' >>"$layout/inbox/${batch_name}.partial/database.dump"
|
|
sha256sum "$layout/inbox/${batch_name}.partial/manifest.json" | awk '{print $1}'
|
|
}
|
|
|
|
DAILY_COMPLETE_LAYOUT="$TEST_DIR/receiver-daily-orphan-complete"
|
|
make_layout "$DAILY_COMPLETE_LAYOUT"
|
|
make_batch "$DAILY_COMPLETE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.101" "$CURRENT_RUN_ID"
|
|
make_batch "$DAILY_COMPLETE_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
|
|
DAILY_COMPLETE_SHA="$(make_corrupt_partial "$DAILY_COMPLETE_LAYOUT" "$THIRD_RUN_ID")"
|
|
expect_failure "receiver cleans a valid daily rollback after the replacement was published" \
|
|
receiver "$DAILY_COMPLETE_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_COMPLETE_SHA"
|
|
[[ ! -e "$DAILY_COMPLETE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.101" \
|
|
&& -d "$DAILY_COMPLETE_LAYOUT/archive/$SECOND_BATCH" ]]
|
|
[[ -d "$DAILY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" ]]
|
|
pass "receiver keeps the valid published daily point and rebuilds its weekly checkpoint"
|
|
|
|
DAILY_WEEKLY_FAIL_LAYOUT="$TEST_DIR/receiver-daily-orphan-weekly-failure"
|
|
make_layout "$DAILY_WEEKLY_FAIL_LAYOUT"
|
|
DAILY_WEEKLY_FAIL_ORPHAN="$DAILY_WEEKLY_FAIL_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.107"
|
|
make_batch "$DAILY_WEEKLY_FAIL_ORPHAN" "$CURRENT_RUN_ID"
|
|
make_batch "$DAILY_WEEKLY_FAIL_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
|
|
printf 'invalid weekly target\n' >"$DAILY_WEEKLY_FAIL_LAYOUT/weekly/$CURRENT_WEEK"
|
|
DAILY_WEEKLY_FAIL_SHA="$(make_corrupt_partial "$DAILY_WEEKLY_FAIL_LAYOUT" "$THIRD_RUN_ID")"
|
|
expect_failure "receiver keeps a daily rollback until weekly reconciliation succeeds" \
|
|
receiver "$DAILY_WEEKLY_FAIL_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_WEEKLY_FAIL_SHA"
|
|
[[ -d "$DAILY_WEEKLY_FAIL_ORPHAN" \
|
|
&& -d "$DAILY_WEEKLY_FAIL_LAYOUT/archive/$SECOND_BATCH" \
|
|
&& -f "$DAILY_WEEKLY_FAIL_LAYOUT/weekly/$CURRENT_WEEK" ]]
|
|
pass "receiver preserves the daily rollback when weekly reconciliation fails"
|
|
|
|
DAILY_RESTORE_LAYOUT="$TEST_DIR/receiver-daily-orphan-restore"
|
|
make_layout "$DAILY_RESTORE_LAYOUT"
|
|
make_batch "$DAILY_RESTORE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.102" "$CURRENT_RUN_ID"
|
|
DAILY_RESTORE_SHA="$(make_corrupt_partial "$DAILY_RESTORE_LAYOUT" "$SECOND_RUN_ID")"
|
|
expect_failure "receiver restores a valid daily rollback when the published archive is absent" \
|
|
receiver "$DAILY_RESTORE_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_RESTORE_SHA"
|
|
[[ -d "$DAILY_RESTORE_LAYOUT/archive/$CURRENT_BATCH" \
|
|
&& ! -e "$DAILY_RESTORE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.102" ]]
|
|
[[ -d "$DAILY_RESTORE_LAYOUT/weekly/$CURRENT_WEEK" ]]
|
|
pass "receiver restores the last daily point before rejecting the next corrupt candidate"
|
|
|
|
DAILY_AMBIGUOUS_LAYOUT="$TEST_DIR/receiver-daily-orphan-ambiguous"
|
|
make_layout "$DAILY_AMBIGUOUS_LAYOUT"
|
|
make_batch "$DAILY_AMBIGUOUS_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.103" "$CURRENT_RUN_ID"
|
|
make_batch "$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
|
|
printf 'corruption\n' >>"$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
|
make_batch "$DAILY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
|
|
DAILY_AMBIGUOUS_SHA="$(sha256sum "$DAILY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
expect_failure "receiver fails closed when a daily rollback and invalid published archive coexist" \
|
|
receiver "$DAILY_AMBIGUOUS_LAYOUT" finalize-test "$THIRD_RUN_ID" "$DAILY_AMBIGUOUS_SHA"
|
|
[[ -d "$DAILY_AMBIGUOUS_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.103" \
|
|
&& -d "$DAILY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" ]]
|
|
pass "receiver preserves both daily objects when replacement state is ambiguous"
|
|
|
|
DAILY_DUPLICATE_LAYOUT="$TEST_DIR/receiver-daily-orphan-duplicate"
|
|
make_layout "$DAILY_DUPLICATE_LAYOUT"
|
|
make_batch "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.104" "$CURRENT_RUN_ID"
|
|
make_batch "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.105" "$CURRENT_RUN_ID"
|
|
make_batch "$DAILY_DUPLICATE_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
|
DAILY_DUPLICATE_SHA="$(sha256sum "$DAILY_DUPLICATE_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
expect_failure "receiver fails closed on duplicate daily rollback orphans" \
|
|
receiver "$DAILY_DUPLICATE_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_DUPLICATE_SHA"
|
|
[[ -d "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.104" \
|
|
&& -d "$DAILY_DUPLICATE_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.105" ]]
|
|
pass "receiver preserves duplicate daily rollback evidence without choosing one"
|
|
|
|
DAILY_LINK_LAYOUT="$TEST_DIR/receiver-daily-orphan-link"
|
|
make_layout "$DAILY_LINK_LAYOUT"
|
|
make_batch "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
|
|
ln -s "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" \
|
|
"$DAILY_LINK_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.106"
|
|
make_batch "$DAILY_LINK_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
|
|
DAILY_LINK_SHA="$(sha256sum "$DAILY_LINK_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
expect_failure "receiver rejects a symbolic-link daily rollback orphan" \
|
|
receiver "$DAILY_LINK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$DAILY_LINK_SHA"
|
|
[[ -L "$DAILY_LINK_LAYOUT/archive/.daily-replacement-${CURRENT_RUN_ID:5:8}.106" \
|
|
&& -d "$DAILY_LINK_LAYOUT/archive/$CURRENT_BATCH" ]]
|
|
pass "receiver leaves linked daily rollback evidence untouched"
|
|
|
|
WEEKLY_ROLLBACK_LAYOUT="$TEST_DIR/receiver-weekly-orphan-rollback"
|
|
make_layout "$WEEKLY_ROLLBACK_LAYOUT"
|
|
make_batch "$WEEKLY_ROLLBACK_LAYOUT/weekly/${CURRENT_WEEK}.rollback.201" "$CURRENT_RUN_ID"
|
|
WEEKLY_ROLLBACK_SHA="$(make_corrupt_partial "$WEEKLY_ROLLBACK_LAYOUT" "$SECOND_RUN_ID")"
|
|
expect_failure "receiver restores a weekly rollback when the canonical checkpoint is absent" \
|
|
receiver "$WEEKLY_ROLLBACK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$WEEKLY_ROLLBACK_SHA"
|
|
[[ -d "$WEEKLY_ROLLBACK_LAYOUT/weekly/$CURRENT_WEEK" \
|
|
&& ! -e "$WEEKLY_ROLLBACK_LAYOUT/weekly/${CURRENT_WEEK}.rollback.201" ]]
|
|
pass "receiver restores the authoritative weekly rollback before the next finalize"
|
|
|
|
WEEKLY_COMPLETE_LAYOUT="$TEST_DIR/receiver-weekly-orphan-complete"
|
|
make_layout "$WEEKLY_COMPLETE_LAYOUT"
|
|
make_batch "$WEEKLY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" "$SECOND_RUN_ID"
|
|
make_batch "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.rollback.202" "$CURRENT_RUN_ID"
|
|
mkdir "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202"
|
|
printf 'interrupted staging\n' >"$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202/partial"
|
|
WEEKLY_COMPLETE_SHA="$(make_corrupt_partial "$WEEKLY_COMPLETE_LAYOUT" "$THIRD_RUN_ID")"
|
|
expect_failure "receiver cleans weekly rollback and staging after a valid checkpoint was published" \
|
|
receiver "$WEEKLY_COMPLETE_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_COMPLETE_SHA"
|
|
[[ -d "$WEEKLY_COMPLETE_LAYOUT/weekly/$CURRENT_WEEK" \
|
|
&& ! -e "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.rollback.202" \
|
|
&& ! -e "$WEEKLY_COMPLETE_LAYOUT/weekly/${CURRENT_WEEK}.new.202" ]]
|
|
pass "receiver treats weekly staging as non-authoritative after a valid publish"
|
|
|
|
WEEKLY_REBUILD_LAYOUT="$TEST_DIR/receiver-weekly-orphan-rebuild"
|
|
make_layout "$WEEKLY_REBUILD_LAYOUT"
|
|
make_batch "$WEEKLY_REBUILD_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
|
|
mkdir "$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203"
|
|
printf 'interrupted staging\n' >"$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203/partial"
|
|
WEEKLY_REBUILD_SHA="$(make_corrupt_partial "$WEEKLY_REBUILD_LAYOUT" "$SECOND_RUN_ID")"
|
|
expect_failure "receiver rebuilds a missing weekly point from one valid daily archive" \
|
|
receiver "$WEEKLY_REBUILD_LAYOUT" finalize-test "$SECOND_RUN_ID" "$WEEKLY_REBUILD_SHA"
|
|
[[ -d "$WEEKLY_REBUILD_LAYOUT/weekly/$CURRENT_WEEK" \
|
|
&& ! -e "$WEEKLY_REBUILD_LAYOUT/weekly/${CURRENT_WEEK}.new.203" ]]
|
|
[[ "$(stat -c %i "$WEEKLY_REBUILD_LAYOUT/weekly/$CURRENT_WEEK/database.dump")" \
|
|
== "$(stat -c %i "$WEEKLY_REBUILD_LAYOUT/archive/$CURRENT_BATCH/database.dump")" ]]
|
|
pass "receiver rebuilds the weekly checkpoint from the unique valid daily source"
|
|
|
|
WEEKLY_AMBIGUOUS_LAYOUT="$TEST_DIR/receiver-weekly-orphan-ambiguous"
|
|
make_layout "$WEEKLY_AMBIGUOUS_LAYOUT"
|
|
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/archive/$CURRENT_BATCH" "$CURRENT_RUN_ID"
|
|
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/archive/$SECOND_BATCH" "$SECOND_RUN_ID"
|
|
mkdir "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/${CURRENT_WEEK}.new.204"
|
|
make_batch "$WEEKLY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
|
|
WEEKLY_AMBIGUOUS_SHA="$(sha256sum "$WEEKLY_AMBIGUOUS_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
expect_failure "receiver fails closed when weekly staging has multiple daily sources" \
|
|
receiver "$WEEKLY_AMBIGUOUS_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_AMBIGUOUS_SHA"
|
|
[[ -d "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/${CURRENT_WEEK}.new.204" \
|
|
&& ! -e "$WEEKLY_AMBIGUOUS_LAYOUT/weekly/$CURRENT_WEEK" ]]
|
|
pass "receiver preserves ambiguous weekly staging without choosing a daily source"
|
|
|
|
WEEKLY_INVALID_LAYOUT="$TEST_DIR/receiver-weekly-orphan-invalid"
|
|
make_layout "$WEEKLY_INVALID_LAYOUT"
|
|
make_batch "$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK" "$SECOND_RUN_ID"
|
|
printf 'corruption\n' >>"$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK/database.dump"
|
|
make_batch "$WEEKLY_INVALID_LAYOUT/weekly/${CURRENT_WEEK}.rollback.205" "$CURRENT_RUN_ID"
|
|
make_batch "$WEEKLY_INVALID_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
|
|
WEEKLY_INVALID_SHA="$(sha256sum "$WEEKLY_INVALID_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
expect_failure "receiver fails closed when a weekly rollback and invalid canonical coexist" \
|
|
receiver "$WEEKLY_INVALID_LAYOUT" finalize-test "$THIRD_RUN_ID" "$WEEKLY_INVALID_SHA"
|
|
[[ -d "$WEEKLY_INVALID_LAYOUT/weekly/$CURRENT_WEEK" \
|
|
&& -d "$WEEKLY_INVALID_LAYOUT/weekly/${CURRENT_WEEK}.rollback.205" ]]
|
|
pass "receiver preserves both weekly objects when checkpoint state is ambiguous"
|
|
|
|
negative_receiver_case() {
|
|
local label="$1"
|
|
local mutation="$2"
|
|
local layout="$TEST_DIR/receiver-negative-${PASS_COUNT}"
|
|
make_layout "$layout"
|
|
make_batch "$layout/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
|
case "$mutation" in
|
|
corrupt-artifact) printf 'changed\n' >>"$layout/inbox/${CURRENT_BATCH}.partial/database.dump" ;;
|
|
corrupt-manifest) printf ' ' >>"$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" ;;
|
|
extra-file) printf 'extra\n' >"$layout/inbox/${CURRENT_BATCH}.partial/extra.txt" ;;
|
|
symlink) rm "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"; ln -s database.dump "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" ;;
|
|
esac
|
|
local sha
|
|
sha="$(sha256sum "$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
if [[ "$mutation" == "corrupt-manifest" ]]; then
|
|
sha="$VALID_MANIFEST_SHA"
|
|
fi
|
|
expect_failure "$label" receiver "$layout" finalize-test "$CURRENT_RUN_ID" "$sha"
|
|
[[ ! -e "$layout/archive/$CURRENT_BATCH" ]]
|
|
}
|
|
|
|
negative_receiver_case "receiver rejects an artifact checksum mismatch" corrupt-artifact
|
|
negative_receiver_case "receiver rejects a manifest checksum mismatch" corrupt-manifest
|
|
negative_receiver_case "receiver rejects an extra file" extra-file
|
|
negative_receiver_case "receiver rejects links in a batch" symlink
|
|
|
|
RESUME_LAYOUT="$TEST_DIR/receiver-resume"
|
|
make_layout "$RESUME_LAYOUT"
|
|
make_batch "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
|
RESUME_MANIFEST_SHA="$(sha256sum "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
mv "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" "$TEST_DIR/resume-storage.tar.gz"
|
|
expect_failure "receiver keeps an incomplete partial batch out of archive" \
|
|
receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA"
|
|
mv "$TEST_DIR/resume-storage.tar.gz" "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"
|
|
receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA" >/dev/null
|
|
[[ -d "$RESUME_LAYOUT/archive/$CURRENT_BATCH" ]]
|
|
pass "receiver finalizes the same partial directory after transfer resumes"
|
|
|
|
FORCED_LAYOUT="$TEST_DIR/receiver-forced"
|
|
make_layout "$FORCED_LAYOUT"
|
|
expect_failure "receiver rejects rsync sender/read mode" env \
|
|
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
|
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
|
|
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
|
|
SSH_ORIGINAL_COMMAND="rsync --server --sender -logDtpre.iLsfxCIvu . ." \
|
|
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
|
|
expect_failure "receiver rejects a sender flag in a later rsync position" env \
|
|
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
|
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
|
|
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
|
|
SSH_ORIGINAL_COMMAND="rsync --server -logDtpre.iLsfxCIvu --sender . ." \
|
|
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
|
|
expect_failure "receiver rejects destructive rsync flags" env \
|
|
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
|
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
|
|
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
|
|
SSH_ORIGINAL_COMMAND="rsync --server --delete -logDtpre.iLsfxCIvu . ." \
|
|
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
|
|
expect_failure "receiver rejects an ordinary shell command" env \
|
|
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
|
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
|
|
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" SSH_ORIGINAL_COMMAND="sh -c id" \
|
|
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
|
|
|
|
NESTED_LAYOUT="$TEST_DIR/receiver-nested"
|
|
mkdir -p "$NESTED_LAYOUT/inbox/archive" "$NESTED_LAYOUT/weekly"
|
|
expect_failure "receiver rejects archive nested inside the rsync inbox" env \
|
|
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
|
|
BACKUP_INBOX_DIR="$NESTED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$NESTED_LAYOUT/inbox/archive" \
|
|
BACKUP_WEEKLY_DIR="$NESTED_LAYOUT/weekly" \
|
|
"$ROOT_DIR/scripts/receive-offhost-backup.sh" validate-config
|
|
|
|
RETENTION_LAYOUT="$TEST_DIR/receiver-retention"
|
|
make_layout "$RETENTION_LAYOUT"
|
|
for day in 01 02 03 04 05 06 07; do
|
|
mkdir "$RETENTION_LAYOUT/archive/hyb-oa-backup-auto-202601${day}T023000Z"
|
|
done
|
|
for week in 01 02 03 04; do
|
|
mkdir "$RETENTION_LAYOUT/weekly/week-2026-W${week}"
|
|
done
|
|
make_batch "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
|
|
RETENTION_SHA="$(sha256sum "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
|
|
receiver "$RETENTION_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RETENTION_SHA" >/dev/null
|
|
[[ "$(find "$RETENTION_LAYOUT/archive" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "7" ]]
|
|
[[ "$(find "$RETENTION_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "4" ]]
|
|
[[ -d "$RETENTION_LAYOUT/archive/$CURRENT_BATCH" ]]
|
|
pass "receiver retains seven daily and four weekly recovery points while protecting current success"
|
|
|
|
EMPTY_SOURCE="$TEST_DIR/empty-rsync-source"
|
|
EMPTY_TARGET="$TEST_DIR/empty-rsync-target"
|
|
mkdir "$EMPTY_SOURCE" "$EMPTY_TARGET"
|
|
rsync -a --dry-run --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= "$EMPTY_SOURCE/" "$EMPTY_TARGET/"
|
|
pass "installed rsync accepts the production chmod contract"
|
|
|
|
WRAPPER_ROOT="$TEST_DIR/wrapper-root"
|
|
WRAPPER_PG="$TEST_DIR/wrapper-pg"
|
|
WRAPPER_STORAGE="$TEST_DIR/wrapper-storage"
|
|
WRAPPER_BACKUPS="$TEST_DIR/wrapper-backups"
|
|
WRAPPER_STATE="$TEST_DIR/wrapper-state"
|
|
MOCK_BIN="$TEST_DIR/mock-bin"
|
|
mkdir -p "$WRAPPER_ROOT/deploy/trialrun" "$WRAPPER_PG" "$WRAPPER_STORAGE" \
|
|
"$WRAPPER_BACKUPS" "$WRAPPER_STATE" "$MOCK_BIN"
|
|
printf '16\n' >"$WRAPPER_PG/PG_VERSION"
|
|
printf 'synthetic attachment\n' >"$WRAPPER_STORAGE/marker.txt"
|
|
printf 'synthetic env\n' >"$WRAPPER_ROOT/deploy/trialrun/.env"
|
|
ln -s "$ROOT_DIR/scripts" "$WRAPPER_ROOT/scripts"
|
|
printf 'private-key-fixture\n' >"$TEST_DIR/offhost-key"
|
|
chmod 0600 "$TEST_DIR/offhost-key"
|
|
printf 'backup-alias ssh-ed25519 fixture\n' >"$TEST_DIR/known-hosts"
|
|
|
|
BUILD_SHA="abcdef1234567890abcdef1234567890abcdef12"
|
|
VITE_BUILD_SHA="${BUILD_SHA:0:12}"
|
|
MOCK_ALERT_LOG="$TEST_DIR/mock-alert.log"
|
|
MOCK_DOCKER_ARGV_LOG="$TEST_DIR/mock-docker-argv.log"
|
|
MOCK_DOCKER_ENV_LOG="$TEST_DIR/mock-docker-env.log"
|
|
export MOCK_PG="$WRAPPER_PG" MOCK_STORAGE="$WRAPPER_STORAGE" MOCK_BUILD_SHA="$BUILD_SHA" MOCK_ALERT_LOG
|
|
export MOCK_DOCKER_ARGV_LOG MOCK_DOCKER_ENV_LOG
|
|
|
|
cat >"$MOCK_BIN/docker" <<'MOCK'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then exit 0; fi
|
|
if [[ "${1:-}" == "compose" ]]; then
|
|
args=" $* "
|
|
if [[ "$args" == *" ps -q server "* ]]; then echo server-id; exit 0; fi
|
|
if [[ "$args" == *" ps -q db "* ]]; then echo db-id; exit 0; fi
|
|
if [[ "$args" == *" exec -T server "* ]]; then exit 99; fi
|
|
fi
|
|
if [[ "${1:-}" == "inspect" ]]; then
|
|
template="${3:-}"
|
|
container="${4:-}"
|
|
case "$template" in
|
|
*State.Status*) echo running ;;
|
|
*Config.Env*)
|
|
if [[ "$container" == "server-id" ]]; then
|
|
printf 'BUILD_SHA=%s\nSTORAGE_DIR=/app/storage\nDATABASE_URL=postgresql://backup_user:fixture@db:5432/hyb_oa\n' "$MOCK_BUILD_SHA"
|
|
else
|
|
printf 'POSTGRES_DB=hyb_oa\nPOSTGRES_USER=backup_user\nPOSTGRES_PASSWORD=fixture\n'
|
|
fi
|
|
;;
|
|
*json*.Mounts*)
|
|
if [[ "$container" == "server-id" ]]; then
|
|
printf '[{"Destination":"/app/storage","Source":"%s"}]\n' "$MOCK_STORAGE"
|
|
else
|
|
printf '[{"Destination":"/var/lib/postgresql/data","Source":"%s"}]\n' "$MOCK_PG"
|
|
fi
|
|
;;
|
|
*Image*) printf 'sha256:%064d\n' 0 ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
exit 0
|
|
fi
|
|
if [[ "${1:-}" == "exec" ]]; then
|
|
printf '%q ' "$@" >>"$MOCK_DOCKER_ARGV_LOG"
|
|
printf '\n' >>"$MOCK_DOCKER_ARGV_LOG"
|
|
printf 'PGPASSWORD=%s\n' "${PGPASSWORD:-}" >>"$MOCK_DOCKER_ENV_LOG"
|
|
if [[ " $* " == *" pg_dump "* ]]; then printf 'synthetic-dump\n'; exit 0; fi
|
|
if [[ " $* " == *" psql "* ]]; then printf '%s\n' "${POSTGRES_DB:-hyb_oa_restore_shadow}"; exit 0; fi
|
|
if [[ " $* " == *" pg_restore --list "* ]]; then cat >/dev/null; exit 0; fi
|
|
if [[ " $* " == *" pg_restore "* ]]; then cat >/dev/null; exit 0; fi
|
|
fi
|
|
exit 1
|
|
MOCK
|
|
|
|
cat >"$MOCK_BIN/rsync" <<'MOCK'
|
|
#!/usr/bin/env bash
|
|
echo 'synthetic transfer diagnostic' >&2
|
|
exit "${MOCK_RSYNC_EXIT:-1}"
|
|
MOCK
|
|
cat >"$MOCK_BIN/ssh" <<'MOCK'
|
|
#!/usr/bin/env bash
|
|
exit "${MOCK_SSH_EXIT:-0}"
|
|
MOCK
|
|
chmod +x "$MOCK_BIN/docker" "$MOCK_BIN/rsync" "$MOCK_BIN/ssh"
|
|
WECOM_TEST_HOST='qyapi.weixin.qq.com'
|
|
WECOM_TEST_PATH='/cgi-bin/webhook/send'
|
|
printf 'https://%s%s?key=%s\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
|
|
'00000000-0000-0000-0000-000000000000' >"$TEST_DIR/wecom-webhook"
|
|
chmod 0600 "$TEST_DIR/wecom-webhook"
|
|
ALERT_CAPTURE="$TEST_DIR/alert-capture.jsonl"
|
|
cat >"$TEST_DIR/failure-summary.json" <<'JSON'
|
|
{"issueId":"AUTO_BACKUP","status":"FAIL_AUTOMATIC_BACKUP","conclusion":"FAIL_AUTOMATIC_BACKUP","findingCount":1,"findings":[{"source":"automatic_backup","status":"FAIL_AUTOMATIC_BACKUP","severity":"P0","detail":"synthetic_failure"}]}
|
|
JSON
|
|
chmod 0600 "$TEST_DIR/failure-summary.json"
|
|
chmod 0644 "$TEST_DIR/wecom-webhook"
|
|
expect_failure "group alert helper rejects a webhook file that is not mode 0600" env \
|
|
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
|
|
python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook" \
|
|
--summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
|
|
chmod 0600 "$TEST_DIR/wecom-webhook"
|
|
ln -s "$TEST_DIR/wecom-webhook" "$TEST_DIR/wecom-webhook-link"
|
|
expect_failure "group alert helper rejects a symbolic-link webhook file" env \
|
|
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
|
|
python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook-link" \
|
|
--summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
|
|
printf 'https://%s%s?key=%s&extra=forbidden\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
|
|
'00000000-0000-0000-0000-000000000000' >"$TEST_DIR/wecom-webhook-invalid"
|
|
chmod 0600 "$TEST_DIR/wecom-webhook-invalid"
|
|
expect_failure "group alert helper rejects a webhook URL with an unexpected query field" env \
|
|
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
|
|
python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook-invalid" \
|
|
--summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
|
|
|
|
wrapper_env=(
|
|
env PATH="$MOCK_BIN:$PATH" REMOTE_DIR="$WRAPPER_ROOT"
|
|
REMOTE_ENV_FILE="$WRAPPER_ROOT/deploy/trialrun/.env"
|
|
COMPOSE_PROJECT_NAME="hyb-oa-current" POSTGRES_DATA_DIR="$WRAPPER_PG"
|
|
STORAGE_HOST_DIR="$WRAPPER_STORAGE" BUILD_SHA="$BUILD_SHA" VITE_BUILD_SHA="$VITE_BUILD_SHA"
|
|
AUTOMATIC_BACKUP_DIR="$WRAPPER_BACKUPS" AUTOMATIC_BACKUP_STATE_DIR="$WRAPPER_STATE"
|
|
OFFHOST_TARGET="hybbackup@backup-alias" OFFHOST_SSH_KEY="$TEST_DIR/offhost-key"
|
|
OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_WEBHOOK_FILE="$TEST_DIR/wecom-webhook"
|
|
ALERT_HELPER="$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
|
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE"
|
|
)
|
|
|
|
exec 8>"$WRAPPER_STATE/automatic-backup.lock"
|
|
flock -n 8
|
|
expect_failure "wrapper rejects a concurrent automatic-backup run" \
|
|
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" run
|
|
flock -u 8
|
|
|
|
rm -f "$WRAPPER_STATE/last-success.json"
|
|
for day in 01 02 03 04 05 06 07; do
|
|
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z"
|
|
done
|
|
expect_failure "unreachable rsync fails without writing off-host success" \
|
|
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=1 "$ROOT_DIR/scripts/run-automatic-backup.sh" run
|
|
[[ ! -e "$WRAPPER_STATE/last-success.json" ]]
|
|
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "8" ]]
|
|
pass "source retention does not run before off-host finalize succeeds"
|
|
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
|
|
import json
|
|
import os
|
|
import sys
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
serialized = json.dumps(payload)
|
|
assert payload["issueId"] == "AUTO_BACKUP"
|
|
assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
|
|
assert not any(value in serialized for value in ("/tmp/", "postgresql://", "backup-alias", "fixture"))
|
|
PYTHON
|
|
pass "wrapper failure summary is category-only and redacted"
|
|
[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
|
|
grep -Fq 'synthetic transfer diagnostic' "$WRAPPER_STATE/last-error.log"
|
|
pass "wrapper retains the latest command diagnostic locally with mode 0600"
|
|
[[ -s "$MOCK_DOCKER_ARGV_LOG" && -s "$MOCK_DOCKER_ENV_LOG" ]]
|
|
! grep -Fq 'fixture' "$MOCK_DOCKER_ARGV_LOG"
|
|
! grep -Fq 'postgresql://' "$MOCK_DOCKER_ARGV_LOG"
|
|
grep -Fq 'PGPASSWORD=fixture' "$MOCK_DOCKER_ENV_LOG"
|
|
pass "wrapper and nightly Docker backup keep connection secrets out of captured argv"
|
|
|
|
: >"$MOCK_DOCKER_ARGV_LOG"
|
|
: >"$MOCK_DOCKER_ENV_LOG"
|
|
DB_TOOL_MODE=docker DATABASE_URL='postgresql://synthetic:argv-secret@db:5432/hyb_oa_restore_shadow' \
|
|
DB_CONTAINER=db-id POSTGRES_DB=hyb_oa_restore_shadow POSTGRES_USER=synthetic \
|
|
POSTGRES_PASSWORD=argv-secret OUT_FILE="$TEST_DIR/argv-backup.dump" PATH="$MOCK_BIN:$PATH" \
|
|
"$ROOT_DIR/scripts/backup-db.sh" >/dev/null
|
|
printf 'synthetic restore dump\n' >"$TEST_DIR/argv-restore.dump"
|
|
DB_TOOL_MODE=docker DATABASE_URL='postgresql://synthetic:argv-secret@db:5432/hyb_oa_restore_shadow' \
|
|
DB_CONTAINER=db-id POSTGRES_DB=hyb_oa_restore_shadow POSTGRES_USER=synthetic \
|
|
POSTGRES_PASSWORD=argv-secret RESTORE_SCOPE=shadow RESTORE_CONFIRM_DATABASE=hyb_oa_restore_shadow \
|
|
PATH="$MOCK_BIN:$PATH" "$ROOT_DIR/scripts/restore-db.sh" "$TEST_DIR/argv-restore.dump" >/dev/null
|
|
! grep -Fq 'argv-secret' "$MOCK_DOCKER_ARGV_LOG"
|
|
grep -Fq 'exec -e PGPASSWORD db-id pg_dump' "$MOCK_DOCKER_ARGV_LOG"
|
|
grep -Fq 'exec -e PGPASSWORD db-id psql' "$MOCK_DOCKER_ARGV_LOG"
|
|
grep -Fq 'exec -i -e PGPASSWORD db-id pg_restore' "$MOCK_DOCKER_ARGV_LOG"
|
|
[[ "$(grep -Fc 'PGPASSWORD=argv-secret' "$MOCK_DOCKER_ENV_LOG")" == "3" ]]
|
|
pass "Docker backup and both restore credential paths inherit PGPASSWORD without argv values"
|
|
|
|
! grep -Eq 'capture_command[[:space:]]+paired_backup[[:space:]]+env' "$ROOT_DIR/scripts/run-automatic-backup.sh"
|
|
! grep -Eq 'docker exec([^\n]|\\\n)*-e[[:space:]]+PGPASSWORD=' \
|
|
"$ROOT_DIR/scripts/backup-db.sh" "$ROOT_DIR/scripts/restore-db.sh"
|
|
pass "production backup scripts contain no value-bearing credential argv patterns"
|
|
|
|
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
|
|
for day in 01 02 03 04 05 06 07; do
|
|
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202602${day}T023000Z"
|
|
done
|
|
rm -f "$WRAPPER_STATE/last-success.json"
|
|
expect_failure "off-host finalize rejection fails the source run without publishing success" \
|
|
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=0 MOCK_SSH_EXIT=1 "$ROOT_DIR/scripts/run-automatic-backup.sh" run
|
|
[[ ! -e "$WRAPPER_STATE/last-success.json" ]]
|
|
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "8" ]]
|
|
[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
|
|
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
|
|
import json
|
|
import sys
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
|
|
assert payload["findings"][0]["detail"] == "phase_offhost_finalize_failed"
|
|
PYTHON
|
|
pass "off-host finalize fail-closed preserves the source batch and emits a safe failure category"
|
|
|
|
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
|
|
for day in 01 02 03 04 05 06 07; do
|
|
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z"
|
|
done
|
|
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=0 MOCK_SSH_EXIT=0 \
|
|
"$ROOT_DIR/scripts/run-automatic-backup.sh" run >/dev/null
|
|
[[ -f "$WRAPPER_STATE/last-success.json" ]]
|
|
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "7" ]]
|
|
pass "off-host finalize success writes state and then prunes source retention to seven"
|
|
[[ ! -e "$WRAPPER_STATE/last-error.log" ]]
|
|
pass "wrapper clears the retained diagnostic after a successful off-host run"
|
|
|
|
rm -f "$WRAPPER_STATE/last-success.json"
|
|
expect_failure "stale watchdog rejects a missing success record" \
|
|
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
|
|
|
python3 - "$WRAPPER_STATE/last-success.json" "$BUILD_SHA" <<'PYTHON'
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
|
json.dump({
|
|
"schemaVersion": 1,
|
|
"status": "complete",
|
|
"completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
"runId": "auto-20260727T000000Z",
|
|
"manifestSha256": "0" * 64,
|
|
"composeProject": "hyb-oa-current",
|
|
"buildSha": sys.argv[2],
|
|
}, handle)
|
|
PYTHON
|
|
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
|
|
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
|
|
import json
|
|
import sys
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
payload["buildSha"] = "0" * 40
|
|
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
|
json.dump(payload, handle)
|
|
PYTHON
|
|
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
|
|
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
|
|
import json
|
|
import sys
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
assert payload["status"] == "PASS_WITH_FINDINGS"
|
|
assert payload["findings"][0]["detail"] == "recovery_point_build_differs"
|
|
PYTHON
|
|
pass "stale watchdog downgrades a fresh recovery point from another build to one finding"
|
|
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
|
|
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
|
|
pass "stale watchdog accepts a fresh project/build-bound success record"
|
|
|
|
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
|
|
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
|
|
import json
|
|
import sys
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
payload["composeProject"] = "different-project"
|
|
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
|
json.dump(payload, handle)
|
|
PYTHON
|
|
expect_failure "stale watchdog hard-fails a fresh record from another Compose project" \
|
|
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
|
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
|
|
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
|
|
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
|
|
import json
|
|
import sys
|
|
with open(sys.argv[1], encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
payload["manifestSha256"] = "invalid"
|
|
with open(sys.argv[1], "w", encoding="utf-8") as handle:
|
|
json.dump(payload, handle)
|
|
PYTHON
|
|
expect_failure "stale watchdog hard-fails a malformed success record" \
|
|
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
|
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
|
|
|
|
DEADMAN_LAYOUT="$TEST_DIR/deadman"
|
|
mkdir -p "$DEADMAN_LAYOUT/archive" "$DEADMAN_LAYOUT/state"
|
|
cp -a "$VALID_LAYOUT/archive/$SECOND_BATCH" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
|
|
DEADMAN_CAPTURE="$TEST_DIR/deadman-alerts.jsonl"
|
|
deadman_env=(env HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$DEADMAN_CAPTURE")
|
|
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
|
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
|
|
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
|
|
[[ ! -e "$DEADMAN_CAPTURE" ]]
|
|
pass "ARM dead-man accepts a fresh verified archive without sending a healthy baseline"
|
|
DEADMAN_CORRUPT_STATE="$TEST_DIR/deadman-corrupt-state"
|
|
DEADMAN_CORRUPT_CAPTURE="$TEST_DIR/deadman-corrupt-alerts.jsonl"
|
|
mkdir "$DEADMAN_CORRUPT_STATE"
|
|
chmod 0600 "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
|
printf 'corruption\n' >>"$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
|
expect_failure "ARM dead-man rejects a corrupt newest finalized archive" env \
|
|
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$DEADMAN_CORRUPT_CAPTURE" \
|
|
python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
|
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_CORRUPT_STATE" \
|
|
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
|
cp "$VALID_LAYOUT/archive/$SECOND_BATCH/database.dump" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
|
chmod 0400 "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
|
|
pass "ARM dead-man detects archive corruption independently of source state"
|
|
mv "$DEADMAN_LAYOUT/archive/$SECOND_BATCH" "$TEST_DIR/deadman-withheld-batch"
|
|
expect_failure "ARM dead-man detects that the source no longer produces a finalized archive" \
|
|
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
|
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
|
|
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
|
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "1" ]]
|
|
expect_failure "ARM dead-man suppresses an identical repeated failure alert" \
|
|
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
|
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
|
|
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
|
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "1" ]]
|
|
pass "ARM dead-man keeps repeated identical failure alerts deduplicated"
|
|
mv "$TEST_DIR/deadman-withheld-batch" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
|
|
"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
|
|
--archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
|
|
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
|
|
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "2" ]]
|
|
pass "ARM dead-man sends one recovery notification after finalized archives resume"
|
|
python3 - "$DEADMAN_CAPTURE" <<'PYTHON'
|
|
import json
|
|
import sys
|
|
|
|
messages = [json.loads(line)["text"]["content"] for line in open(sys.argv[1], encoding="utf-8")]
|
|
assert messages[0].startswith("内部 OA 运维告警\n"), messages
|
|
assert messages[1].startswith("内部 OA 运维恢复通知\n"), messages
|
|
assert all("OA 公网健康" not in message for message in messages), messages
|
|
PYTHON
|
|
pass "shared helper keeps dead-man alert and recovery titles unchanged"
|
|
|
|
DRIFT_BUILD_SHA="0123456789abcdef0123456789abcdef01234567"
|
|
expect_failure "wrapper rejects an active runtime build that drifted from its installed contract" \
|
|
"${wrapper_env[@]}" BUILD_SHA="$DRIFT_BUILD_SHA" VITE_BUILD_SHA="${DRIFT_BUILD_SHA:0:12}" \
|
|
"$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
|
|
|
|
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null
|
|
FIRST_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")"
|
|
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null
|
|
SECOND_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")"
|
|
[[ "$FIRST_REMINDER" == "$SECOND_REMINDER" && "$FIRST_REMINDER" == quarterly_shadow_restore_due_* ]]
|
|
pass "quarterly reminder finding key is stable within the same quarter"
|
|
|
|
expect_failure "receiver installer uninstall rejects an unsafe target path" env \
|
|
BACKUP_USER=hybbackup RECEIVER_TARGET=/ CONFIG_FILE=/etc/hyb-oa-test.conf \
|
|
"$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" uninstall
|
|
|
|
expect_failure "receiver installer rejects a system-managed backup root before mutation" env \
|
|
BACKUP_ROOT=/etc/hyb-oa-backups PUBLIC_KEY_FILE="$TEST_DIR/known-hosts" \
|
|
RECEIVER_SOURCE="$ROOT_DIR/scripts/receive-offhost-backup.sh" \
|
|
"$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" install
|
|
|
|
printf '[PASS] automatic off-host backup contract completed (%d checks)\n' "$PASS_COUNT"
|