文件
hyb-oa/scripts/qa/run-backup-transfer-incident-contract.sh

339 行
17 KiB
Bash
可执行文件

#!/usr/bin/env bash
set -eEuo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
INSPECTOR="$ROOT_DIR/scripts/qa/inspect-backup-transfer-incident.py"
TEST_ROOT="$(mktemp -d)"
PASS_COUNT=0
cleanup() {
if [[ -n "${TEST_ROOT:-}" && -d "$TEST_ROOT" && "$TEST_ROOT" == /tmp/* ]]; then
rm -rf -- "$TEST_ROOT"
fi
}
trap cleanup EXIT
pass() {
PASS_COUNT=$((PASS_COUNT + 1))
printf '[PASS] %s\n' "$1"
}
assert_line() {
local file="$1"
local expected="$2"
grep -Fxq -- "$expected" "$file" || {
printf '[FAIL] missing field: %s\n' "$expected" >&2
exit 1
}
}
make_batch() {
local target="$1"
local run_id="$2"
local db_bytes="${3:-1024}"
local storage_bytes="${4:-2048}"
mkdir -p "$target"
head -c "$db_bytes" /dev/zero >"$target/database.dump"
head -c "$storage_bytes" /dev/zero >"$target/storage.tar.gz"
python3 - "$target" "$run_id" <<'PY'
import hashlib
import json
import sys
from pathlib import Path
target = Path(sys.argv[1])
run_id = sys.argv[2]
def item(name):
data = (target / name).read_bytes()
return {"file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}
payload = {
"schemaVersion": 1,
"runId": run_id,
"createdAt": "2026-08-28T00:00:00Z",
"buildSha": "a" * 40,
"sourceImageDigest": None,
"status": "complete",
"artifacts": {"database": item("database.dump"), "storage": item("storage.tar.gz")},
}
(target / "manifest.json").write_text(json.dumps(payload) + "\n", encoding="utf-8")
PY
}
run_production() {
local fixture="$1"
local output="$2"
python3 -B "$INSPECTOR" --now-epoch 1787875200 production \
--backup-root "$fixture/backups" \
--state-dir "$fixture/state" \
--wrapper-script "$ROOT_DIR/scripts/run-automatic-backup.sh" \
--backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$output"
}
make_production_layout() {
local fixture="$1"
mkdir -p "$fixture/backups" "$fixture/state"
}
write_transfer_failure_state() {
local fixture="$1"
printf '%s\n' '{"findings":[{"detail":"phase_offhost_transfer_failed"}]}' >"$fixture/state/last-run.json"
}
PRODUCTION_VALID="$TEST_ROOT/production-valid"
make_production_layout "$PRODUCTION_VALID"
make_batch "$PRODUCTION_VALID/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
write_transfer_failure_state "$PRODUCTION_VALID"
printf 'rsync error: synthetic redacted fixture (code 23) at fixture.c(1)\n' >"$PRODUCTION_VALID/state/last-error.log"
run_production "$PRODUCTION_VALID" "$TEST_ROOT/production-valid.out"
assert_line "$TEST_ROOT/production-valid.out" "complete_local_recovery_point=yes"
assert_line "$TEST_ROOT/production-valid.out" "backup_risk_classification=production_single_point"
assert_line "$TEST_ROOT/production-valid.out" "transition_recovery_point_action=wait_natural"
assert_line "$TEST_ROOT/production-valid.out" "transfer_exit_code=23"
assert_line "$TEST_ROOT/production-valid.out" "transfer_failure_category=partial_transfer_error"
assert_line "$TEST_ROOT/production-valid.out" "transfer_duration_seconds=NOT_AVAILABLE"
pass "生产存在完整本地恢复点时识别为单点风险并保留未测时长"
PRODUCTION_MULTIPLE="$TEST_ROOT/production-multiple"
make_production_layout "$PRODUCTION_MULTIPLE"
make_batch "$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z"
make_batch "$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
printf 'X' | dd of="$PRODUCTION_MULTIPLE/backups/hyb-oa-backup-auto-20260827T000000Z/database.dump" bs=1 seek=0 conv=notrunc status=none
run_production "$PRODUCTION_MULTIPLE" "$TEST_ROOT/production-multiple.out"
assert_line "$TEST_ROOT/production-multiple.out" "complete_local_recovery_point=yes"
assert_line "$TEST_ROOT/production-multiple.out" "fully_verified_local_batch_count=1"
assert_line "$TEST_ROOT/production-multiple.out" "metadata_complete_local_batch_count=2"
assert_line "$TEST_ROOT/production-multiple.out" "older_metadata_batch_hash_status=NOT_RUN"
assert_line "$TEST_ROOT/production-multiple.out" "latest_local_candidate_hash_status=PASS"
pass "生产多个批次时只对最新元数据合格候选执行全量 SHA-256"
PRODUCTION_LATEST_HASH_FAIL="$TEST_ROOT/production-latest-hash-fail"
make_production_layout "$PRODUCTION_LATEST_HASH_FAIL"
make_batch "$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z"
make_batch "$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
printf 'X' | dd of="$PRODUCTION_LATEST_HASH_FAIL/backups/hyb-oa-backup-auto-20260828T000000Z/database.dump" bs=1 seek=0 conv=notrunc status=none
run_production "$PRODUCTION_LATEST_HASH_FAIL" "$TEST_ROOT/production-latest-hash-fail.out"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "complete_local_recovery_point=no"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "fully_verified_local_batch_count=0"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "metadata_complete_local_batch_count=2"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "older_metadata_batch_hash_status=NOT_RUN"
assert_line "$TEST_ROOT/production-latest-hash-fail.out" "latest_local_candidate_hash_status=FAIL"
pass "最新候选哈希失败时不回退扫描旧批次并保持无已证明恢复点"
PRODUCTION_EMPTY="$TEST_ROOT/production-empty"
make_production_layout "$PRODUCTION_EMPTY"
run_production "$PRODUCTION_EMPTY" "$TEST_ROOT/production-empty.out"
assert_line "$TEST_ROOT/production-empty.out" "complete_local_recovery_point=no"
assert_line "$TEST_ROOT/production-empty.out" "backup_risk_classification=no_proven_recovery_point"
assert_line "$TEST_ROOT/production-empty.out" "transition_recovery_point_action=request_manual_run"
pass "生产无完整本地恢复点时升级风险与过渡恢复点建议"
PRODUCTION_CORRUPT="$TEST_ROOT/production-corrupt"
make_production_layout "$PRODUCTION_CORRUPT"
make_batch "$PRODUCTION_CORRUPT/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
printf 'tamper' >>"$PRODUCTION_CORRUPT/backups/hyb-oa-backup-auto-20260828T000000Z/database.dump"
run_production "$PRODUCTION_CORRUPT" "$TEST_ROOT/production-corrupt.out"
assert_line "$TEST_ROOT/production-corrupt.out" "complete_local_recovery_point=no"
assert_line "$TEST_ROOT/production-corrupt.out" "invalid_local_batch_count=1"
pass "manifest 大小或 SHA 篡改不得冒充完整本地恢复点"
PRODUCTION_MALFORMED="$TEST_ROOT/production-malformed"
make_production_layout "$PRODUCTION_MALFORMED"
make_batch "$PRODUCTION_MALFORMED/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
printf '%s\n' '[]' >"$PRODUCTION_MALFORMED/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json"
run_production "$PRODUCTION_MALFORMED" "$TEST_ROOT/production-malformed.out"
assert_line "$TEST_ROOT/production-malformed.out" "complete_local_recovery_point=no"
assert_line "$TEST_ROOT/production-malformed.out" "invalid_local_batch_count=1"
pass "畸形 manifest 安全归类为无效批次且不输出 traceback"
PRODUCTION_UNREADABLE="$TEST_ROOT/production-unreadable"
make_production_layout "$PRODUCTION_UNREADABLE"
make_batch "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
chmod 000 "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json"
if [[ "$(id -u)" == 0 ]] && command -v runuser >/dev/null 2>&1 && id nobody >/dev/null 2>&1; then
chmod 0755 "$TEST_ROOT" "$PRODUCTION_UNREADABLE" "$PRODUCTION_UNREADABLE/backups" "$PRODUCTION_UNREADABLE/state" \
"$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z"
if runuser -u nobody -- python3 -B "$INSPECTOR" --now-epoch 1787875200 production \
--backup-root "$PRODUCTION_UNREADABLE/backups" \
--state-dir "$PRODUCTION_UNREADABLE/state" \
--wrapper-script "$ROOT_DIR/scripts/run-automatic-backup.sh" \
--backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$TEST_ROOT/production-unreadable.out"; then
printf '[FAIL] unreadable manifest unexpectedly passed\n' >&2
exit 1
fi
else
if run_production "$PRODUCTION_UNREADABLE" "$TEST_ROOT/production-unreadable.out"; then
printf '[FAIL] unreadable manifest unexpectedly passed\n' >&2
exit 1
fi
fi
chmod 0600 "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json"
assert_line "$TEST_ROOT/production-unreadable.out" "probe_category=access_denied"
pass "访问拒绝保持为 unknown/ACCESS_DENIED 而不是伪装成无有效备份"
assert_line "$TEST_ROOT/production-valid.out" "retained_after_transfer_failure=yes"
assert_line "$TEST_ROOT/production-valid.out" "pruned_after_offhost_success=yes"
assert_line "$TEST_ROOT/production-valid.out" "generation_temp_cleanup_on_failure=yes"
assert_line "$TEST_ROOT/production-valid.out" "cleanup_contract_source_verified=yes"
pass "生产脚本顺序机械证明失败保留、成功后 prune 与生成临时清理"
SOURCE_MUTATION="$TEST_ROOT/source-mutation"
cp "$ROOT_DIR/scripts/run-automatic-backup.sh" "$SOURCE_MUTATION"
python3 - "$SOURCE_MUTATION" <<'PY'
import sys
from pathlib import Path
path = Path(sys.argv[1])
text = path.read_text(encoding="utf-8")
transfer = text.index("capture_command offhost_transfer rsync")
text = text[:transfer] + 'prune_local_backups "$batch_name"\n ' + text[transfer:]
path.write_text(text, encoding="utf-8")
PY
python3 -B "$INSPECTOR" --now-epoch 1787875200 production \
--backup-root "$PRODUCTION_VALID/backups" \
--state-dir "$PRODUCTION_VALID/state" \
--wrapper-script "$SOURCE_MUTATION" \
--backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$TEST_ROOT/source-mutation.out"
assert_line "$TEST_ROOT/source-mutation.out" "retained_after_transfer_failure=no"
assert_line "$TEST_ROOT/source-mutation.out" "pruned_after_offhost_success=no"
pass "源码顺序漂移会关闭失败保留与成功后 prune 结论"
for code_category in \
23:partial_transfer_error \
30:transport_timeout \
12:protocol_stream_error \
11:file_io_error \
255:ssh_session_error; do
code="${code_category%%:*}"
category="${code_category#*:}"
fixture="$TEST_ROOT/rsync-$code"
make_production_layout "$fixture"
write_transfer_failure_state "$fixture"
printf 'rsync error: synthetic redacted fixture (code %s) at fixture.c(1)\n' "$code" >"$fixture/state/last-error.log"
run_production "$fixture" "$TEST_ROOT/rsync-$code.out"
assert_line "$TEST_ROOT/rsync-$code.out" "transfer_exit_code=$code"
assert_line "$TEST_ROOT/rsync-$code.out" "transfer_failure_category=$category"
done
pass "rsync 23/30/12/11/255 均映射为稳定脱敏类别"
PRODUCTION_UNKNOWN="$TEST_ROOT/production-unknown"
make_production_layout "$PRODUCTION_UNKNOWN"
write_transfer_failure_state "$PRODUCTION_UNKNOWN"
printf 'unstructured synthetic transport failure\n' >"$PRODUCTION_UNKNOWN/state/last-error.log"
run_production "$PRODUCTION_UNKNOWN" "$TEST_ROOT/production-unknown.out"
assert_line "$TEST_ROOT/production-unknown.out" "transfer_exit_code=NOT_AVAILABLE"
assert_line "$TEST_ROOT/production-unknown.out" "transfer_failure_category=not_available"
pass "原始 rsync 码不存在时安全失败为 NOT_AVAILABLE"
PRODUCTION_PHASE_MISMATCH="$TEST_ROOT/production-phase-mismatch"
make_production_layout "$PRODUCTION_PHASE_MISMATCH"
printf '%s\n' '{"findings":[{"detail":"phase_offhost_finalize_failed"}]}' >"$PRODUCTION_PHASE_MISMATCH/state/last-run.json"
printf 'rsync error: stale synthetic fixture (code 23) at fixture.c(1)\n' >"$PRODUCTION_PHASE_MISMATCH/state/last-error.log"
run_production "$PRODUCTION_PHASE_MISMATCH" "$TEST_ROOT/production-phase-mismatch.out"
assert_line "$TEST_ROOT/production-phase-mismatch.out" "last_backup_failure_phase=offhost_finalize"
assert_line "$TEST_ROOT/production-phase-mismatch.out" "transfer_exit_code=NOT_AVAILABLE"
assert_line "$TEST_ROOT/production-phase-mismatch.out" "transfer_exit_code_source=NOT_AVAILABLE"
pass "私有诊断只在安全摘要明确为 offhost_transfer 时绑定 rsync 原始码"
ARM_SIMILAR="$TEST_ROOT/arm-similar"
mkdir -p "$ARM_SIMILAR/inbox" "$ARM_SIMILAR/archive"
make_batch "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260827T000000Z.partial" "auto-20260827T000000Z" 1000 1000
make_batch "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z" 1040 1000
rm "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260827T000000Z.partial/storage.tar.gz"
rm "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260828T000000Z.partial/storage.tar.gz"
python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_SIMILAR/inbox" --archive-root "$ARM_SIMILAR/archive" >"$TEST_ROOT/arm-similar.out"
assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_count=2"
assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_stop_offset_class=similar"
assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_incomplete_contract_count=2"
pass "两个 partial 接近停止偏移按预先冻结阈值分类为 similar"
ARM_DIFFERENT="$TEST_ROOT/arm-different"
mkdir -p "$ARM_DIFFERENT/inbox" "$ARM_DIFFERENT/archive"
make_batch "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260827T000000Z.partial" "auto-20260827T000000Z" 1000 1000
make_batch "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z" 70000000 1000
rm "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260827T000000Z.partial/storage.tar.gz"
rm "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260828T000000Z.partial/storage.tar.gz"
python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_DIFFERENT/inbox" --archive-root "$ARM_DIFFERENT/archive" >"$TEST_ROOT/arm-different.out"
assert_line "$TEST_ROOT/arm-different.out" "arm_partial_stop_offset_class=different"
pass "两个 partial 明显不同停止偏移分类为 different"
python3 -B - "$INSPECTOR" <<'PY'
import importlib.util
import sys
spec = importlib.util.spec_from_file_location("backup_transfer_incident", sys.argv[1])
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
def sample(total_bytes):
return module.BatchResult(False, total_bytes, None, None, total_bytes)
cases = [
("现场量级的 2.68MB 与 0 字节必须不同", [sample(0), sample(2_682_168)], "different"),
("大文件差异同时落在 5% 与 64MiB 内才接近", [sample(1_000_000_000), sample(1_040_000_000)], "similar"),
("即使落在 64MiB 内但超过 5% 仍不同", [sample(10_000_000), sample(10_600_000)], "different"),
("即使落在 5% 内但超过 64MiB 仍不同", [sample(2_000_000_000), sample(2_080_000_000)], "different"),
]
for label, samples, expected in cases:
actual = module.stopping_class(samples)
if actual != expected:
raise SystemExit(f"{label}: expected {expected}, got {actual}")
PY
pass "停止偏移分类同时满足相对与绝对阈值并覆盖现场 2.68MB/0 字节回归"
ARM_COMPLETE="$TEST_ROOT/arm-complete"
mkdir -p "$ARM_COMPLETE/inbox" "$ARM_COMPLETE/archive"
make_batch "$ARM_COMPLETE/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z"
make_batch "$ARM_COMPLETE/archive/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z"
python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_COMPLETE/inbox" --archive-root "$ARM_COMPLETE/archive" >"$TEST_ROOT/arm-complete.out"
assert_line "$TEST_ROOT/arm-complete.out" "arm_partial_complete_contract_count=1"
assert_line "$TEST_ROOT/arm-complete.out" "arm_valid_atomic_archive_count=1"
pass "完整 partial 与正式原子归档分别按同一三文件身份和哈希合同验证"
PERMISSION_FIXTURE="$TEST_ROOT/permission"
mkdir -p "$PERMISSION_FIXTURE/inbox" "$PERMISSION_FIXTURE/archive"
chmod 000 "$PERMISSION_FIXTURE/inbox"
if command -v runuser >/dev/null 2>&1 && id nobody >/dev/null 2>&1 && [[ "$(id -u)" == 0 ]]; then
if runuser -u nobody -- python3 -B "$INSPECTOR" arm --inbox-root "$PERMISSION_FIXTURE/inbox" --archive-root "$PERMISSION_FIXTURE/archive" >"$TEST_ROOT/permission.out"; then
printf '[FAIL] permission denial unexpectedly passed\n' >&2
exit 1
fi
assert_line "$TEST_ROOT/permission.out" "probe_category=access_denied"
else
chmod 0700 "$PERMISSION_FIXTURE/inbox"
ln -s "$PERMISSION_FIXTURE/inbox" "$PERMISSION_FIXTURE/inbox-link"
if python3 -B "$INSPECTOR" arm --inbox-root "$PERMISSION_FIXTURE/inbox-link" --archive-root "$PERMISSION_FIXTURE/archive" >"$TEST_ROOT/permission.out"; then
printf '[FAIL] invalid root unexpectedly passed\n' >&2
exit 1
fi
assert_line "$TEST_ROOT/permission.out" "probe_category=root_invalid"
fi
chmod 0700 "$PERMISSION_FIXTURE/inbox"
pass "不可读或不安全根目录明确拒绝且不伪装为不存在"
if rg -n --fixed-strings \
-e "$TEST_ROOT" \
-e "auto-20260828T000000Z" \
-e "database.dump" \
-e "storage.tar.gz" \
-e "unstructured synthetic transport failure" \
"$TEST_ROOT"/*.out >/dev/null; then
printf '[FAIL] redacted output leaked fixture identity, path, artifact name, or diagnostic text\n' >&2
exit 1
fi
pass "stdout allowlist 不含路径、run id、产物名或原始诊断"
PYTHONPYCACHEPREFIX="$TEST_ROOT/pycache" python3 -m py_compile "$INSPECTOR"
bash -n "$ROOT_DIR/scripts/qa/run-backup-transfer-incident-contract.sh"
pass "Python 与 Bash 语法通过"
cleanup
[[ ! -e "$TEST_ROOT" ]]
TEST_ROOT=""
trap - EXIT
pass "合成 fixture 与 Python cache 零残留"
printf '[PASS] backup transfer incident contract: %s checks\n' "$PASS_COUNT"