文件
hyb-oa/scripts/qa/run-automatic-backup-contract.sh
T
wangsli 9875bc37a4
ci-e2e / e2e (push) Waiting to run
ci-minimal / check-minimal (push) Waiting to run
修复自动备份真实挂载解析(07-27-automatic-offhost-backup)
2026-07-27 11:19:13 +08:00

376 行
16 KiB
Bash
可执行文件

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
TEST_DIR="$(mktemp -d)"
PASS_COUNT=0
cleanup() {
if [[ -n "${TEST_DIR:-}" && -d "$TEST_DIR" && "$TEST_DIR" == /tmp/* ]]; then
rm -rf -- "$TEST_DIR"
fi
}
trap cleanup EXIT
pass() {
PASS_COUNT=$((PASS_COUNT + 1))
printf '[PASS] %s\n' "$1"
}
expect_failure() {
local label="$1"
shift
if "$@" >"$TEST_DIR/last-command.out" 2>&1; then
printf '[FAIL] expected rejection: %s\n' "$label" >&2
exit 1
fi
pass "$label"
}
make_layout() {
local root="$1"
mkdir -p "$root/inbox" "$root/archive" "$root/weekly"
}
make_batch() {
local target="$1"
local run_id="$2"
mkdir -p "$target"
printf 'synthetic database payload\n' >"$target/database.dump"
printf 'synthetic storage payload\n' >"$target/storage.tar.gz"
python3 - "$target" "$run_id" <<'PYTHON'
import hashlib
import json
import os
import sys
target, run_id = sys.argv[1:]
def artifact(file_name):
file_path = os.path.join(target, file_name)
with open(file_path, "rb") as handle:
content = handle.read()
return {"file": file_name, "bytes": len(content), "sha256": hashlib.sha256(content).hexdigest()}
manifest = {
"schemaVersion": 1,
"runId": run_id,
"createdAt": "2026-01-01T00:00:00Z",
"buildSha": "a" * 40,
"sourceImageDigest": None,
"status": "complete",
"artifacts": {
"database": artifact("database.dump"),
"storage": artifact("storage.tar.gz"),
},
}
with open(os.path.join(target, "manifest.json"), "x", encoding="utf-8") as handle:
json.dump(manifest, handle, indent=2)
handle.write("\n")
PYTHON
}
receiver() {
local layout="$1"
shift
env RECEIVER_TEST_MODE=1 RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
BACKUP_INBOX_DIR="$layout/inbox" BACKUP_ARCHIVE_DIR="$layout/archive" \
BACKUP_WEEKLY_DIR="$layout/weekly" DAILY_KEEP=7 WEEKLY_KEEP=4 \
"$ROOT_DIR/scripts/receive-offhost-backup.sh" "$@"
}
CURRENT_RUN_ID="auto-$(date -u +%Y%m%dT%H%M%SZ)"
CURRENT_BATCH="hyb-oa-backup-$CURRENT_RUN_ID"
VALID_LAYOUT="$TEST_DIR/receiver-valid"
make_layout "$VALID_LAYOUT"
make_batch "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
VALID_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
[[ -d "$VALID_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$VALID_LAYOUT/inbox/${CURRENT_BATCH}.partial" ]]
pass "receiver finalizes one complete synthetic batch"
receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
pass "receiver treats an identical repeated finalize as idempotent"
SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z"
if [[ "$SECOND_RUN_ID" == "$CURRENT_RUN_ID" ]]; then
SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z"
fi
SECOND_BATCH="hyb-oa-backup-$SECOND_RUN_ID"
make_batch "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
SECOND_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
expect_failure "receiver rejects a different automatic batch for the same UTC day" \
receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA"
negative_receiver_case() {
local label="$1"
local mutation="$2"
local layout="$TEST_DIR/receiver-negative-${PASS_COUNT}"
make_layout "$layout"
make_batch "$layout/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
case "$mutation" in
corrupt-artifact) printf 'changed\n' >>"$layout/inbox/${CURRENT_BATCH}.partial/database.dump" ;;
corrupt-manifest) printf ' ' >>"$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" ;;
extra-file) printf 'extra\n' >"$layout/inbox/${CURRENT_BATCH}.partial/extra.txt" ;;
symlink) rm "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"; ln -s database.dump "$layout/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" ;;
esac
local sha
sha="$(sha256sum "$layout/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
if [[ "$mutation" == "corrupt-manifest" ]]; then
sha="$VALID_MANIFEST_SHA"
fi
expect_failure "$label" receiver "$layout" finalize-test "$CURRENT_RUN_ID" "$sha"
[[ ! -e "$layout/archive/$CURRENT_BATCH" ]]
}
negative_receiver_case "receiver rejects an artifact checksum mismatch" corrupt-artifact
negative_receiver_case "receiver rejects a manifest checksum mismatch" corrupt-manifest
negative_receiver_case "receiver rejects an extra file" extra-file
negative_receiver_case "receiver rejects links in a batch" symlink
RESUME_LAYOUT="$TEST_DIR/receiver-resume"
make_layout "$RESUME_LAYOUT"
make_batch "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
RESUME_MANIFEST_SHA="$(sha256sum "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
mv "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz" "$TEST_DIR/resume-storage.tar.gz"
expect_failure "receiver keeps an incomplete partial batch out of archive" \
receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA"
mv "$TEST_DIR/resume-storage.tar.gz" "$RESUME_LAYOUT/inbox/${CURRENT_BATCH}.partial/storage.tar.gz"
receiver "$RESUME_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RESUME_MANIFEST_SHA" >/dev/null
[[ -d "$RESUME_LAYOUT/archive/$CURRENT_BATCH" ]]
pass "receiver finalizes the same partial directory after transfer resumes"
FORCED_LAYOUT="$TEST_DIR/receiver-forced"
make_layout "$FORCED_LAYOUT"
expect_failure "receiver rejects rsync sender/read mode" env \
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
SSH_ORIGINAL_COMMAND="rsync --server --sender -logDtpre.iLsfxCIvu . ." \
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
expect_failure "receiver rejects a sender flag in a later rsync position" env \
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
SSH_ORIGINAL_COMMAND="rsync --server -logDtpre.iLsfxCIvu --sender . ." \
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
expect_failure "receiver rejects destructive rsync flags" env \
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" \
SSH_ORIGINAL_COMMAND="rsync --server --delete -logDtpre.iLsfxCIvu . ." \
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
expect_failure "receiver rejects an ordinary shell command" env \
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
BACKUP_INBOX_DIR="$FORCED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$FORCED_LAYOUT/archive" \
BACKUP_WEEKLY_DIR="$FORCED_LAYOUT/weekly" SSH_ORIGINAL_COMMAND="sh -c id" \
"$ROOT_DIR/scripts/receive-offhost-backup.sh" forced
NESTED_LAYOUT="$TEST_DIR/receiver-nested"
mkdir -p "$NESTED_LAYOUT/inbox/archive" "$NESTED_LAYOUT/weekly"
expect_failure "receiver rejects archive nested inside the rsync inbox" env \
RECEIVER_CONFIG_FILE="$TEST_DIR/no-receiver-config" \
BACKUP_INBOX_DIR="$NESTED_LAYOUT/inbox" BACKUP_ARCHIVE_DIR="$NESTED_LAYOUT/inbox/archive" \
BACKUP_WEEKLY_DIR="$NESTED_LAYOUT/weekly" \
"$ROOT_DIR/scripts/receive-offhost-backup.sh" validate-config
RETENTION_LAYOUT="$TEST_DIR/receiver-retention"
make_layout "$RETENTION_LAYOUT"
for day in 01 02 03 04 05 06 07; do
mkdir "$RETENTION_LAYOUT/archive/hyb-oa-backup-auto-202601${day}T023000Z"
done
for week in 01 02 03 04; do
mkdir "$RETENTION_LAYOUT/weekly/week-2026-W${week}"
done
make_batch "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
RETENTION_SHA="$(sha256sum "$RETENTION_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
receiver "$RETENTION_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$RETENTION_SHA" >/dev/null
[[ "$(find "$RETENTION_LAYOUT/archive" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "7" ]]
[[ "$(find "$RETENTION_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d | wc -l)" == "4" ]]
[[ -d "$RETENTION_LAYOUT/archive/$CURRENT_BATCH" ]]
pass "receiver retains seven daily and four weekly recovery points while protecting current success"
EMPTY_SOURCE="$TEST_DIR/empty-rsync-source"
EMPTY_TARGET="$TEST_DIR/empty-rsync-target"
mkdir "$EMPTY_SOURCE" "$EMPTY_TARGET"
rsync -a --dry-run --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= "$EMPTY_SOURCE/" "$EMPTY_TARGET/"
pass "installed rsync accepts the production chmod contract"
WRAPPER_ROOT="$TEST_DIR/wrapper-root"
WRAPPER_PG="$TEST_DIR/wrapper-pg"
WRAPPER_STORAGE="$TEST_DIR/wrapper-storage"
WRAPPER_BACKUPS="$TEST_DIR/wrapper-backups"
WRAPPER_STATE="$TEST_DIR/wrapper-state"
MOCK_BIN="$TEST_DIR/mock-bin"
mkdir -p "$WRAPPER_ROOT/deploy/trialrun" "$WRAPPER_PG" "$WRAPPER_STORAGE" \
"$WRAPPER_BACKUPS" "$WRAPPER_STATE" "$MOCK_BIN"
printf '16\n' >"$WRAPPER_PG/PG_VERSION"
printf 'synthetic attachment\n' >"$WRAPPER_STORAGE/marker.txt"
printf 'synthetic env\n' >"$WRAPPER_ROOT/deploy/trialrun/.env"
ln -s "$ROOT_DIR/scripts" "$WRAPPER_ROOT/scripts"
printf 'private-key-fixture\n' >"$TEST_DIR/offhost-key"
chmod 0600 "$TEST_DIR/offhost-key"
printf 'backup-alias ssh-ed25519 fixture\n' >"$TEST_DIR/known-hosts"
BUILD_SHA="abcdef1234567890abcdef1234567890abcdef12"
VITE_BUILD_SHA="${BUILD_SHA:0:12}"
MOCK_ALERT_LOG="$TEST_DIR/mock-alert.log"
export MOCK_PG="$WRAPPER_PG" MOCK_STORAGE="$WRAPPER_STORAGE" MOCK_BUILD_SHA="$BUILD_SHA" MOCK_ALERT_LOG
cat >"$MOCK_BIN/docker" <<'MOCK'
#!/usr/bin/env bash
set -euo pipefail
if [[ "${1:-}" == "compose" && "${2:-}" == "version" ]]; then exit 0; fi
if [[ "${1:-}" == "compose" ]]; then
args=" $* "
if [[ "$args" == *" ps -q server "* ]]; then echo server-id; exit 0; fi
if [[ "$args" == *" ps -q db "* ]]; then echo db-id; exit 0; fi
if [[ "$args" == *" exec -T server "* ]]; then cat >>"$MOCK_ALERT_LOG"; exit 0; fi
fi
if [[ "${1:-}" == "inspect" ]]; then
template="${3:-}"
container="${4:-}"
case "$template" in
*State.Status*) echo running ;;
*Config.Env*)
if [[ "$container" == "server-id" ]]; then
printf 'BUILD_SHA=%s\nSTORAGE_DIR=/app/storage\nDATABASE_URL=postgresql://backup_user:fixture@db:5432/hyb_oa\n' "$MOCK_BUILD_SHA"
else
printf 'POSTGRES_DB=hyb_oa\nPOSTGRES_USER=backup_user\nPOSTGRES_PASSWORD=fixture\n'
fi
;;
*json*.Mounts*)
if [[ "$container" == "server-id" ]]; then
printf '[{"Destination":"/app/storage","Source":"%s"}]\n' "$MOCK_STORAGE"
else
printf '[{"Destination":"/var/lib/postgresql/data","Source":"%s"}]\n' "$MOCK_PG"
fi
;;
*Image*) printf 'sha256:%064d\n' 0 ;;
*) exit 1 ;;
esac
exit 0
fi
if [[ "${1:-}" == "exec" ]]; then
if [[ " $* " == *" pg_dump "* ]]; then printf 'synthetic-dump\n'; exit 0; fi
if [[ " $* " == *" pg_restore --list "* ]]; then cat >/dev/null; exit 0; fi
fi
exit 1
MOCK
cat >"$MOCK_BIN/rsync" <<'MOCK'
#!/usr/bin/env bash
exit "${MOCK_RSYNC_EXIT:-1}"
MOCK
cat >"$MOCK_BIN/ssh" <<'MOCK'
#!/usr/bin/env bash
exit "${MOCK_SSH_EXIT:-0}"
MOCK
chmod +x "$MOCK_BIN/docker" "$MOCK_BIN/rsync" "$MOCK_BIN/ssh"
wrapper_env=(
env PATH="$MOCK_BIN:$PATH" REMOTE_DIR="$WRAPPER_ROOT"
REMOTE_ENV_FILE="$WRAPPER_ROOT/deploy/trialrun/.env"
COMPOSE_PROJECT_NAME="hyb-oa-current" POSTGRES_DATA_DIR="$WRAPPER_PG"
STORAGE_HOST_DIR="$WRAPPER_STORAGE" BUILD_SHA="$BUILD_SHA" VITE_BUILD_SHA="$VITE_BUILD_SHA"
AUTOMATIC_BACKUP_DIR="$WRAPPER_BACKUPS" AUTOMATIC_BACKUP_STATE_DIR="$WRAPPER_STATE"
OFFHOST_TARGET="hybbackup@backup-alias" OFFHOST_SSH_KEY="$TEST_DIR/offhost-key"
OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_RECIPIENT_EMPLOYEE_NO="OPS001"
)
exec 8>"$WRAPPER_STATE/automatic-backup.lock"
flock -n 8
expect_failure "wrapper rejects a concurrent automatic-backup run" \
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" run
flock -u 8
rm -f "$WRAPPER_STATE/last-success.json"
for day in 01 02 03 04 05 06 07; do
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z"
done
expect_failure "unreachable rsync fails without writing off-host success" \
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=1 "$ROOT_DIR/scripts/run-automatic-backup.sh" run
[[ ! -e "$WRAPPER_STATE/last-success.json" ]]
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "8" ]]
pass "source retention does not run before off-host finalize succeeds"
python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
import json
import os
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
payload = json.load(handle)
serialized = json.dumps(payload)
assert payload["issueId"] == "AUTO_BACKUP"
assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
assert not any(value in serialized for value in ("/tmp/", "postgresql://", "backup-alias", "fixture"))
PYTHON
pass "wrapper failure summary is category-only and redacted"
find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
for day in 01 02 03 04 05 06 07; do
mkdir "$WRAPPER_BACKUPS/hyb-oa-backup-auto-202601${day}T023000Z"
done
"${wrapper_env[@]}" MOCK_RSYNC_EXIT=0 MOCK_SSH_EXIT=0 \
"$ROOT_DIR/scripts/run-automatic-backup.sh" run >/dev/null
[[ -f "$WRAPPER_STATE/last-success.json" ]]
[[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "7" ]]
pass "off-host finalize success writes state and then prunes source retention to seven"
rm -f "$WRAPPER_STATE/last-success.json"
expect_failure "stale watchdog rejects a missing success record" \
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
python3 - "$WRAPPER_STATE/last-success.json" "$BUILD_SHA" <<'PYTHON'
import json
import sys
from datetime import datetime, timezone
with open(sys.argv[1], "w", encoding="utf-8") as handle:
json.dump({
"schemaVersion": 1,
"status": "complete",
"completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
"runId": "auto-synthetic",
"manifestSha256": "0" * 64,
"composeProject": "hyb-oa-current",
"buildSha": sys.argv[2],
}, handle)
PYTHON
cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
import json
import sys
with open(sys.argv[1], encoding="utf-8") as handle:
payload = json.load(handle)
payload["buildSha"] = "0" * 40
with open(sys.argv[1], "w", encoding="utf-8") as handle:
json.dump(payload, handle)
PYTHON
expect_failure "stale watchdog rejects a success record from another build" \
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
pass "stale watchdog accepts a fresh project/build-bound success record"
DRIFT_BUILD_SHA="0123456789abcdef0123456789abcdef01234567"
expect_failure "wrapper rejects an active runtime build that drifted from its installed contract" \
"${wrapper_env[@]}" BUILD_SHA="$DRIFT_BUILD_SHA" VITE_BUILD_SHA="${DRIFT_BUILD_SHA:0:12}" \
"$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null
FIRST_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")"
"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" remind-drill >/dev/null
SECOND_REMINDER="$(python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))["findings"][0]["detail"])' "$WRAPPER_STATE/last-run.json")"
[[ "$FIRST_REMINDER" == "$SECOND_REMINDER" && "$FIRST_REMINDER" == quarterly_shadow_restore_due_* ]]
pass "quarterly reminder finding key is stable within the same quarter"
expect_failure "receiver installer uninstall rejects an unsafe target path" env \
BACKUP_USER=hybbackup RECEIVER_TARGET=/ CONFIG_FILE=/etc/hyb-oa-test.conf \
"$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" uninstall
expect_failure "receiver installer rejects a system-managed backup root before mutation" env \
BACKUP_ROOT=/etc/hyb-oa-backups PUBLIC_KEY_FILE="$TEST_DIR/known-hosts" \
RECEIVER_SOURCE="$ROOT_DIR/scripts/receive-offhost-backup.sh" \
"$ROOT_DIR/deploy/trialrun/install-offhost-backup-receiver.sh" install
printf '[PASS] automatic off-host backup contract completed (%d checks)\n' "$PASS_COUNT"