运维(08-28-production-arm-backup-transfer-incident): 固化只读调查合同
这个提交包含在：
@@ -0,0 +1,505 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only, redacted inspection of an interrupted automatic-backup chain."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
BATCH_RE = re.compile(r"^hyb-oa-backup-(auto-(\d{8}T\d{6}Z))$")
|
||||
PARTIAL_RE = re.compile(r"^hyb-oa-backup-(auto-(\d{8}T\d{6}Z))\.partial$")
|
||||
TEMP_RE = re.compile(r"^\.hyb-oa-backup-.+\.tmp\..+$")
|
||||
EXPECTED_FILES = {"database.dump", "storage.tar.gz", "manifest.json"}
|
||||
ARTIFACTS = {"database": "database.dump", "storage": "storage.tar.gz"}
|
||||
RSYNC_CODE_RE = re.compile(r"rsync error:[^\r\n]*\(code (11|12|23|30|255)\)")
|
||||
RSYNC_CATEGORIES = {
|
||||
"11": "file_io_error",
|
||||
"12": "protocol_stream_error",
|
||||
"23": "partial_transfer_error",
|
||||
"30": "transport_timeout",
|
||||
"255": "ssh_session_error",
|
||||
}
|
||||
SIMILAR_RELATIVE_PPM = 50_000
|
||||
SIMILAR_ABSOLUTE_BYTES = 64 * 1024 * 1024
|
||||
SAFE_KEY_RE = re.compile(r"^[a-z0-9_]+$")
|
||||
SAFE_VALUE_RE = re.compile(r"^(?:[A-Za-z][A-Za-z0-9_]*|-?[0-9]+)$")
|
||||
|
||||
|
||||
class InspectionError(Exception):
|
||||
"""Stable, non-sensitive inspection failure."""
|
||||
|
||||
def __init__(self, category: str):
|
||||
super().__init__(category)
|
||||
self.category = category
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BatchResult:
|
||||
valid: bool
|
||||
total_bytes: int
|
||||
created_epoch: int | None
|
||||
expected_artifact_bytes: int | None
|
||||
received_artifact_bytes: int
|
||||
|
||||
|
||||
def emit(items: Iterable[tuple[str, object]]) -> None:
|
||||
rendered_items: list[tuple[str, str]] = []
|
||||
for key, value in items:
|
||||
if isinstance(value, bool):
|
||||
rendered = "yes" if value else "no"
|
||||
else:
|
||||
rendered = str(value)
|
||||
if not SAFE_KEY_RE.fullmatch(key) or not SAFE_VALUE_RE.fullmatch(rendered):
|
||||
print("probe_result=FAIL")
|
||||
print("probe_category=output_allowlist_violation")
|
||||
raise SystemExit(1)
|
||||
rendered_items.append((key, rendered))
|
||||
for key, rendered in rendered_items:
|
||||
print(f"{key}={rendered}")
|
||||
|
||||
|
||||
def safe_root(path: Path) -> Path:
|
||||
try:
|
||||
if not path.is_absolute() or path.is_symlink() or not path.is_dir():
|
||||
raise InspectionError("root_invalid")
|
||||
list(path.iterdir())
|
||||
except PermissionError as error:
|
||||
raise InspectionError("access_denied") from error
|
||||
except OSError as error:
|
||||
raise InspectionError("root_unreadable") from error
|
||||
return path
|
||||
|
||||
|
||||
def file_sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def tree_bytes(path: Path) -> int:
|
||||
total = 0
|
||||
try:
|
||||
for root, directories, files in os.walk(path, followlinks=False):
|
||||
directories[:] = [
|
||||
name for name in directories if not (Path(root) / name).is_symlink()
|
||||
]
|
||||
for name in files:
|
||||
item = Path(root) / name
|
||||
if item.is_symlink():
|
||||
continue
|
||||
total += item.stat().st_size
|
||||
except PermissionError as error:
|
||||
raise InspectionError("access_denied") from error
|
||||
except OSError as error:
|
||||
raise InspectionError("entry_unreadable") from error
|
||||
return total
|
||||
|
||||
|
||||
def run_epoch(stamp: str) -> int | None:
|
||||
try:
|
||||
parsed = datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc)
|
||||
except ValueError:
|
||||
return None
|
||||
return int(parsed.timestamp())
|
||||
|
||||
|
||||
def validate_batch(path: Path, expected_run_id: str, stamp: str) -> BatchResult:
|
||||
total = tree_bytes(path)
|
||||
received_artifact_bytes = 0
|
||||
expected_artifact_bytes: int | None = None
|
||||
created_epoch = run_epoch(stamp)
|
||||
if created_epoch is None:
|
||||
return BatchResult(False, total, None, None, 0)
|
||||
try:
|
||||
entries = list(path.iterdir())
|
||||
regular_names = {
|
||||
item.name
|
||||
for item in entries
|
||||
if not item.is_symlink() and stat.S_ISREG(item.stat().st_mode)
|
||||
}
|
||||
if len(entries) != 3 or regular_names != EXPECTED_FILES:
|
||||
return BatchResult(False, total, created_epoch, None, 0)
|
||||
manifest = json.loads((path / "manifest.json").read_text(encoding="utf-8"))
|
||||
if (
|
||||
manifest.get("schemaVersion") != 1
|
||||
or manifest.get("status") != "complete"
|
||||
or manifest.get("runId") != expected_run_id
|
||||
or not (manifest.get("buildSha") or manifest.get("sourceImageDigest"))
|
||||
):
|
||||
return BatchResult(False, total, created_epoch, None, 0)
|
||||
artifacts = manifest.get("artifacts") or {}
|
||||
expected_artifact_bytes = 0
|
||||
for key, expected_name in ARTIFACTS.items():
|
||||
item = artifacts.get(key) or {}
|
||||
target = path / expected_name
|
||||
if item.get("file") != expected_name or isinstance(item.get("bytes"), bool):
|
||||
return BatchResult(False, total, created_epoch, None, 0)
|
||||
expected_size = item.get("bytes")
|
||||
if not isinstance(expected_size, int) or expected_size < 0:
|
||||
return BatchResult(False, total, created_epoch, None, 0)
|
||||
expected_artifact_bytes += expected_size
|
||||
actual_size = target.stat().st_size
|
||||
received_artifact_bytes += actual_size
|
||||
if actual_size != expected_size or file_sha256(target) != item.get("sha256"):
|
||||
return BatchResult(
|
||||
False,
|
||||
total,
|
||||
created_epoch,
|
||||
expected_artifact_bytes,
|
||||
received_artifact_bytes,
|
||||
)
|
||||
return BatchResult(
|
||||
True,
|
||||
total,
|
||||
created_epoch,
|
||||
expected_artifact_bytes,
|
||||
received_artifact_bytes,
|
||||
)
|
||||
except PermissionError as error:
|
||||
raise InspectionError("access_denied") from error
|
||||
except (AttributeError, json.JSONDecodeError, OSError, TypeError, ValueError):
|
||||
return BatchResult(False, total, created_epoch, expected_artifact_bytes, received_artifact_bytes)
|
||||
|
||||
|
||||
def inspect_partial(path: Path, run_id: str, stamp: str) -> BatchResult:
|
||||
"""Measure a partial without requiring the complete three-file contract."""
|
||||
total = tree_bytes(path)
|
||||
received = 0
|
||||
expected: int | None = None
|
||||
try:
|
||||
manifest_path = path / "manifest.json"
|
||||
if manifest_path.is_file() and not manifest_path.is_symlink():
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
if manifest.get("runId") == run_id and manifest.get("schemaVersion") == 1:
|
||||
artifacts = manifest.get("artifacts") or {}
|
||||
sizes: list[int] = []
|
||||
for key, expected_name in ARTIFACTS.items():
|
||||
item = artifacts.get(key) or {}
|
||||
size = item.get("bytes")
|
||||
if (
|
||||
item.get("file") != expected_name
|
||||
or isinstance(size, bool)
|
||||
or not isinstance(size, int)
|
||||
or size < 0
|
||||
):
|
||||
sizes = []
|
||||
break
|
||||
sizes.append(size)
|
||||
target = path / expected_name
|
||||
if target.is_file() and not target.is_symlink():
|
||||
received += target.stat().st_size
|
||||
if len(sizes) == 2:
|
||||
expected = sum(sizes)
|
||||
validated = validate_batch(path, run_id, stamp)
|
||||
return BatchResult(validated.valid, total, run_epoch(stamp), expected, received)
|
||||
except PermissionError as error:
|
||||
raise InspectionError("access_denied") from error
|
||||
except (AttributeError, json.JSONDecodeError, OSError, TypeError, ValueError):
|
||||
return BatchResult(False, total, run_epoch(stamp), expected, received)
|
||||
|
||||
|
||||
def filesystem_fields(prefix: str, root: Path) -> list[tuple[str, object]]:
|
||||
values = os.statvfs(root)
|
||||
total = values.f_blocks * values.f_frsize
|
||||
available = values.f_bavail * values.f_frsize
|
||||
used_percent = 0 if total == 0 else ((total - available) * 100) // total
|
||||
return [
|
||||
(f"{prefix}_disk_used_percent", used_percent),
|
||||
(f"{prefix}_disk_free_gib", available // (1024**3)),
|
||||
]
|
||||
|
||||
|
||||
def source_contract(wrapper: Path, backup_script: Path) -> list[tuple[str, object]]:
|
||||
try:
|
||||
if (
|
||||
wrapper.is_symlink()
|
||||
or not wrapper.is_file()
|
||||
or backup_script.is_symlink()
|
||||
or not backup_script.is_file()
|
||||
):
|
||||
raise InspectionError("source_contract_invalid")
|
||||
wrapper_text = wrapper.read_text(encoding="utf-8")
|
||||
backup_text = backup_script.read_text(encoding="utf-8")
|
||||
except PermissionError as error:
|
||||
raise InspectionError("source_contract_access_denied") from error
|
||||
except OSError as error:
|
||||
raise InspectionError("source_contract_unreadable") from error
|
||||
|
||||
if "run_backup() {" not in wrapper_text or "\ncheck_stale() {" not in wrapper_text:
|
||||
raise InspectionError("source_contract_invalid")
|
||||
run_backup_body = wrapper_text.split("run_backup() {", 1)[1].split("\ncheck_stale() {", 1)[0]
|
||||
markers = {
|
||||
"local_publish": '[[ -d "$batch_dir" && -f "$batch_dir/manifest.json" ]]',
|
||||
"transfer": "capture_command offhost_transfer rsync",
|
||||
"finalize": "capture_command offhost_finalize ssh",
|
||||
"success": 'write_success_state "$run_id" "$manifest_sha"',
|
||||
"prune": 'prune_local_backups "$batch_name"',
|
||||
}
|
||||
positions = {key: run_backup_body.find(marker) for key, marker in markers.items()}
|
||||
all_present = all(position >= 0 for position in positions.values())
|
||||
retained = all_present and positions["local_publish"] < positions["transfer"] < positions["prune"]
|
||||
pruned = all_present and positions["finalize"] < positions["success"] < positions["prune"]
|
||||
temp_cleanup = all(
|
||||
marker in backup_text
|
||||
for marker in (
|
||||
"trap cleanup EXIT",
|
||||
'TEMP_DIR="$(mktemp -d',
|
||||
'rm -rf -- "$TEMP_DIR"',
|
||||
'mv "$TEMP_DIR" "$FINAL_DIR"',
|
||||
'TEMP_DIR=""',
|
||||
"trap - EXIT",
|
||||
)
|
||||
)
|
||||
return [
|
||||
("retained_after_transfer_failure", retained),
|
||||
("pruned_after_offhost_success", pruned),
|
||||
("generation_temp_cleanup_on_failure", temp_cleanup),
|
||||
("cleanup_contract_source_verified", all_present and temp_cleanup),
|
||||
]
|
||||
|
||||
|
||||
def last_backup_phase(state_dir: Path) -> str:
|
||||
state_file = state_dir / "last-run.json"
|
||||
if not state_file.exists():
|
||||
return "NOT_AVAILABLE"
|
||||
try:
|
||||
if state_file.is_symlink() or not state_file.is_file():
|
||||
return "STATE_INVALID"
|
||||
payload = json.loads(state_file.read_text(encoding="utf-8"))
|
||||
findings = payload.get("findings") or []
|
||||
detail = findings[0].get("detail") if len(findings) == 1 and isinstance(findings[0], dict) else None
|
||||
match = re.fullmatch(r"phase_([a-z_]+)_failed", detail or "")
|
||||
allowed = {
|
||||
"runtime_identity",
|
||||
"paired_backup",
|
||||
"offhost_transfer",
|
||||
"offhost_finalize",
|
||||
"success_state",
|
||||
"stale_check",
|
||||
"quarterly_reminder",
|
||||
}
|
||||
return match.group(1) if match and match.group(1) in allowed else "NOT_AVAILABLE"
|
||||
except PermissionError as error:
|
||||
raise InspectionError("diagnostic_access_denied") from error
|
||||
except (AttributeError, json.JSONDecodeError, OSError, TypeError):
|
||||
return "STATE_INVALID"
|
||||
|
||||
|
||||
def parse_transfer_diagnostic(state_dir: Path) -> list[tuple[str, object]]:
|
||||
phase = last_backup_phase(state_dir)
|
||||
diagnostic = state_dir / "last-error.log"
|
||||
if phase != "offhost_transfer" or not diagnostic.exists():
|
||||
code = "NOT_AVAILABLE"
|
||||
else:
|
||||
try:
|
||||
if diagnostic.is_symlink() or not diagnostic.is_file():
|
||||
raise InspectionError("diagnostic_invalid")
|
||||
text = diagnostic.read_text(encoding="utf-8", errors="replace")
|
||||
except PermissionError as error:
|
||||
raise InspectionError("diagnostic_access_denied") from error
|
||||
except OSError as error:
|
||||
raise InspectionError("diagnostic_unreadable") from error
|
||||
matches = RSYNC_CODE_RE.findall(text)
|
||||
code = matches[-1] if matches else "NOT_AVAILABLE"
|
||||
return [
|
||||
("last_backup_failure_phase", phase),
|
||||
("transfer_exit_code", code),
|
||||
("transfer_exit_code_source", "last_error_log_regex" if code != "NOT_AVAILABLE" else "NOT_AVAILABLE"),
|
||||
("transfer_failure_category", RSYNC_CATEGORIES.get(code, "not_available")),
|
||||
("transfer_started_epoch", "NOT_AVAILABLE"),
|
||||
("transfer_failed_epoch", "NOT_AVAILABLE"),
|
||||
("transfer_duration_seconds", "NOT_AVAILABLE"),
|
||||
("transfer_duration_reason", "wrapper_does_not_persist_transfer_clock"),
|
||||
]
|
||||
|
||||
|
||||
def production(args: argparse.Namespace) -> None:
|
||||
backup_root = safe_root(args.backup_root)
|
||||
state_dir = safe_root(args.state_dir)
|
||||
now = args.now_epoch
|
||||
valid: list[BatchResult] = []
|
||||
invalid_count = invalid_bytes = temp_count = temp_bytes = other_count = 0
|
||||
for entry in backup_root.iterdir():
|
||||
match = BATCH_RE.fullmatch(entry.name)
|
||||
if match and entry.is_dir() and not entry.is_symlink():
|
||||
result = validate_batch(entry, match.group(1), match.group(2))
|
||||
if result.valid and result.created_epoch is not None and result.created_epoch <= now:
|
||||
valid.append(result)
|
||||
else:
|
||||
invalid_count += 1
|
||||
invalid_bytes += result.total_bytes
|
||||
elif TEMP_RE.fullmatch(entry.name) and entry.is_dir() and not entry.is_symlink():
|
||||
temp_count += 1
|
||||
temp_bytes += tree_bytes(entry)
|
||||
else:
|
||||
other_count += 1
|
||||
|
||||
valid.sort(key=lambda item: item.created_epoch or -1)
|
||||
latest = valid[-1] if valid else None
|
||||
output: list[tuple[str, object]] = [
|
||||
("production_probe_result", "PASS"),
|
||||
("complete_local_recovery_point", bool(valid)),
|
||||
("backup_risk_classification", "production_single_point" if valid else "no_proven_recovery_point"),
|
||||
("complete_local_batch_count", len(valid)),
|
||||
("complete_local_batch_total_bytes", sum(item.total_bytes for item in valid)),
|
||||
(
|
||||
"latest_complete_local_batch_age_seconds",
|
||||
now - latest.created_epoch
|
||||
if latest and latest.created_epoch is not None
|
||||
else "NOT_AVAILABLE",
|
||||
),
|
||||
("latest_complete_local_batch_bytes", latest.total_bytes if latest else "NOT_AVAILABLE"),
|
||||
("invalid_local_batch_count", invalid_count),
|
||||
("invalid_local_batch_total_bytes", invalid_bytes),
|
||||
("generation_temp_count", temp_count),
|
||||
("generation_temp_total_bytes", temp_bytes),
|
||||
("unclassified_local_entry_count", other_count),
|
||||
("transition_recovery_point_action", "wait_natural" if valid else "request_manual_run"),
|
||||
]
|
||||
output.extend(filesystem_fields("production_backup", backup_root))
|
||||
output.extend(source_contract(args.wrapper_script, args.backup_script))
|
||||
output.extend(parse_transfer_diagnostic(state_dir))
|
||||
emit(output)
|
||||
|
||||
|
||||
def stopping_class(samples: list[BatchResult]) -> str:
|
||||
if len(samples) < 2:
|
||||
return "insufficient_samples"
|
||||
sizes = sorted(item.total_bytes for item in samples[-2:])
|
||||
difference = sizes[1] - sizes[0]
|
||||
relative_limit = sizes[1] * SIMILAR_RELATIVE_PPM // 1_000_000
|
||||
return "similar" if difference <= max(SIMILAR_ABSOLUTE_BYTES, relative_limit) else "different"
|
||||
|
||||
|
||||
def arm(args: argparse.Namespace) -> None:
|
||||
inbox_root = safe_root(args.inbox_root)
|
||||
archive_root = safe_root(args.archive_root)
|
||||
now = args.now_epoch
|
||||
partials: list[BatchResult] = []
|
||||
other_count = 0
|
||||
for entry in inbox_root.iterdir():
|
||||
match = PARTIAL_RE.fullmatch(entry.name)
|
||||
if match and entry.is_dir() and not entry.is_symlink():
|
||||
partials.append(inspect_partial(entry, match.group(1), match.group(2)))
|
||||
else:
|
||||
other_count += 1
|
||||
partials.sort(key=lambda item: item.created_epoch or -1)
|
||||
|
||||
archives: list[BatchResult] = []
|
||||
invalid_archive_count = 0
|
||||
for entry in archive_root.iterdir():
|
||||
match = BATCH_RE.fullmatch(entry.name)
|
||||
if match and entry.is_dir() and not entry.is_symlink():
|
||||
result = validate_batch(entry, match.group(1), match.group(2))
|
||||
if result.valid and result.created_epoch is not None and result.created_epoch <= now:
|
||||
archives.append(result)
|
||||
else:
|
||||
invalid_archive_count += 1
|
||||
archives.sort(key=lambda item: item.created_epoch or -1)
|
||||
|
||||
output: list[tuple[str, object]] = [
|
||||
("arm_probe_result", "PASS"),
|
||||
("arm_partial_count", len(partials)),
|
||||
("arm_partial_total_bytes", sum(item.total_bytes for item in partials)),
|
||||
("arm_partial_complete_contract_count", sum(1 for item in partials if item.valid)),
|
||||
("arm_partial_incomplete_contract_count", sum(1 for item in partials if not item.valid)),
|
||||
("arm_partial_stop_offset_class", stopping_class(partials)),
|
||||
("arm_partial_similarity_relative_ppm", SIMILAR_RELATIVE_PPM),
|
||||
("arm_partial_similarity_absolute_bytes", SIMILAR_ABSOLUTE_BYTES),
|
||||
("arm_inbox_other_entry_count", other_count),
|
||||
("arm_valid_atomic_archive_count", len(archives)),
|
||||
("arm_invalid_atomic_archive_count", invalid_archive_count),
|
||||
(
|
||||
"arm_latest_atomic_archive_age_seconds",
|
||||
now - archives[-1].created_epoch if archives else "NOT_AVAILABLE",
|
||||
),
|
||||
]
|
||||
for index, sample in enumerate(partials, 1):
|
||||
ratio = (
|
||||
min(sample.received_artifact_bytes, sample.expected_artifact_bytes)
|
||||
* 1_000_000
|
||||
// sample.expected_artifact_bytes
|
||||
if sample.expected_artifact_bytes and sample.expected_artifact_bytes > 0
|
||||
else "NOT_AVAILABLE"
|
||||
)
|
||||
output.extend(
|
||||
[
|
||||
(f"arm_partial_sample_{index}_total_bytes", sample.total_bytes),
|
||||
(
|
||||
f"arm_partial_sample_{index}_expected_artifact_bytes",
|
||||
sample.expected_artifact_bytes
|
||||
if sample.expected_artifact_bytes is not None
|
||||
else "NOT_AVAILABLE",
|
||||
),
|
||||
(f"arm_partial_sample_{index}_received_ratio_ppm", ratio),
|
||||
(
|
||||
f"arm_partial_sample_{index}_age_seconds",
|
||||
now - sample.created_epoch
|
||||
if sample.created_epoch is not None
|
||||
else "NOT_AVAILABLE",
|
||||
),
|
||||
(f"arm_partial_sample_{index}_complete_contract", sample.valid),
|
||||
]
|
||||
)
|
||||
output.extend(filesystem_fields("arm_inbox", inbox_root))
|
||||
output.extend(filesystem_fields("arm_archive", archive_root))
|
||||
emit(output)
|
||||
|
||||
|
||||
def absolute_path(value: str) -> Path:
|
||||
path = Path(value)
|
||||
if not path.is_absolute():
|
||||
raise argparse.ArgumentTypeError("an explicit absolute path is required")
|
||||
return path
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Emit redacted backup-chain incident fields")
|
||||
parser.add_argument("--now-epoch", type=int, default=int(datetime.now(timezone.utc).timestamp()))
|
||||
subparsers = parser.add_subparsers(dest="mode", required=True)
|
||||
production_parser = subparsers.add_parser("production")
|
||||
production_parser.add_argument("--backup-root", type=absolute_path, required=True)
|
||||
production_parser.add_argument("--state-dir", type=absolute_path, required=True)
|
||||
production_parser.add_argument("--wrapper-script", type=absolute_path, required=True)
|
||||
production_parser.add_argument("--backup-script", type=absolute_path, required=True)
|
||||
production_parser.set_defaults(handler=production)
|
||||
arm_parser = subparsers.add_parser("arm")
|
||||
arm_parser.add_argument("--inbox-root", type=absolute_path, required=True)
|
||||
arm_parser.add_argument("--archive-root", type=absolute_path, required=True)
|
||||
arm_parser.set_defaults(handler=arm)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = build_parser().parse_args()
|
||||
try:
|
||||
args.handler(args)
|
||||
except InspectionError as error:
|
||||
emit([("probe_result", "FAIL"), ("probe_category", error.category)])
|
||||
return 1
|
||||
except PermissionError:
|
||||
emit([("probe_result", "FAIL"), ("probe_category", "access_denied")])
|
||||
return 1
|
||||
except (OSError, TypeError, ValueError):
|
||||
emit([("probe_result", "FAIL"), ("probe_category", "inspection_io_error")])
|
||||
return 1
|
||||
except Exception:
|
||||
emit([("probe_result", "FAIL"), ("probe_category", "inspection_internal_error")])
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,287 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eEuo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
INSPECTOR="$ROOT_DIR/scripts/qa/inspect-backup-transfer-incident.py"
|
||||
TEST_ROOT="$(mktemp -d)"
|
||||
PASS_COUNT=0
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "${TEST_ROOT:-}" && -d "$TEST_ROOT" && "$TEST_ROOT" == /tmp/* ]]; then
|
||||
rm -rf -- "$TEST_ROOT"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
pass() {
|
||||
PASS_COUNT=$((PASS_COUNT + 1))
|
||||
printf '[PASS] %s\n' "$1"
|
||||
}
|
||||
|
||||
assert_line() {
|
||||
local file="$1"
|
||||
local expected="$2"
|
||||
grep -Fxq -- "$expected" "$file" || {
|
||||
printf '[FAIL] missing field: %s\n' "$expected" >&2
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
make_batch() {
|
||||
local target="$1"
|
||||
local run_id="$2"
|
||||
local db_bytes="${3:-1024}"
|
||||
local storage_bytes="${4:-2048}"
|
||||
mkdir -p "$target"
|
||||
head -c "$db_bytes" /dev/zero >"$target/database.dump"
|
||||
head -c "$storage_bytes" /dev/zero >"$target/storage.tar.gz"
|
||||
python3 - "$target" "$run_id" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
target = Path(sys.argv[1])
|
||||
run_id = sys.argv[2]
|
||||
|
||||
def item(name):
|
||||
data = (target / name).read_bytes()
|
||||
return {"file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}
|
||||
|
||||
payload = {
|
||||
"schemaVersion": 1,
|
||||
"runId": run_id,
|
||||
"createdAt": "2026-08-28T00:00:00Z",
|
||||
"buildSha": "a" * 40,
|
||||
"sourceImageDigest": None,
|
||||
"status": "complete",
|
||||
"artifacts": {"database": item("database.dump"), "storage": item("storage.tar.gz")},
|
||||
}
|
||||
(target / "manifest.json").write_text(json.dumps(payload) + "\n", encoding="utf-8")
|
||||
PY
|
||||
}
|
||||
|
||||
run_production() {
|
||||
local fixture="$1"
|
||||
local output="$2"
|
||||
python3 -B "$INSPECTOR" --now-epoch 1787875200 production \
|
||||
--backup-root "$fixture/backups" \
|
||||
--state-dir "$fixture/state" \
|
||||
--wrapper-script "$ROOT_DIR/scripts/run-automatic-backup.sh" \
|
||||
--backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$output"
|
||||
}
|
||||
|
||||
make_production_layout() {
|
||||
local fixture="$1"
|
||||
mkdir -p "$fixture/backups" "$fixture/state"
|
||||
}
|
||||
|
||||
write_transfer_failure_state() {
|
||||
local fixture="$1"
|
||||
printf '%s\n' '{"findings":[{"detail":"phase_offhost_transfer_failed"}]}' >"$fixture/state/last-run.json"
|
||||
}
|
||||
|
||||
PRODUCTION_VALID="$TEST_ROOT/production-valid"
|
||||
make_production_layout "$PRODUCTION_VALID"
|
||||
make_batch "$PRODUCTION_VALID/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
|
||||
write_transfer_failure_state "$PRODUCTION_VALID"
|
||||
printf 'rsync error: synthetic redacted fixture (code 23) at fixture.c(1)\n' >"$PRODUCTION_VALID/state/last-error.log"
|
||||
run_production "$PRODUCTION_VALID" "$TEST_ROOT/production-valid.out"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "complete_local_recovery_point=yes"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "backup_risk_classification=production_single_point"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "transition_recovery_point_action=wait_natural"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "transfer_exit_code=23"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "transfer_failure_category=partial_transfer_error"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "transfer_duration_seconds=NOT_AVAILABLE"
|
||||
pass "生产存在完整本地恢复点时识别为单点风险并保留未测时长"
|
||||
|
||||
PRODUCTION_EMPTY="$TEST_ROOT/production-empty"
|
||||
make_production_layout "$PRODUCTION_EMPTY"
|
||||
run_production "$PRODUCTION_EMPTY" "$TEST_ROOT/production-empty.out"
|
||||
assert_line "$TEST_ROOT/production-empty.out" "complete_local_recovery_point=no"
|
||||
assert_line "$TEST_ROOT/production-empty.out" "backup_risk_classification=no_proven_recovery_point"
|
||||
assert_line "$TEST_ROOT/production-empty.out" "transition_recovery_point_action=request_manual_run"
|
||||
pass "生产无完整本地恢复点时升级风险与过渡恢复点建议"
|
||||
|
||||
PRODUCTION_CORRUPT="$TEST_ROOT/production-corrupt"
|
||||
make_production_layout "$PRODUCTION_CORRUPT"
|
||||
make_batch "$PRODUCTION_CORRUPT/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
|
||||
printf 'tamper' >>"$PRODUCTION_CORRUPT/backups/hyb-oa-backup-auto-20260828T000000Z/database.dump"
|
||||
run_production "$PRODUCTION_CORRUPT" "$TEST_ROOT/production-corrupt.out"
|
||||
assert_line "$TEST_ROOT/production-corrupt.out" "complete_local_recovery_point=no"
|
||||
assert_line "$TEST_ROOT/production-corrupt.out" "invalid_local_batch_count=1"
|
||||
pass "manifest 大小或 SHA 篡改不得冒充完整本地恢复点"
|
||||
|
||||
PRODUCTION_MALFORMED="$TEST_ROOT/production-malformed"
|
||||
make_production_layout "$PRODUCTION_MALFORMED"
|
||||
make_batch "$PRODUCTION_MALFORMED/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
|
||||
printf '%s\n' '[]' >"$PRODUCTION_MALFORMED/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json"
|
||||
run_production "$PRODUCTION_MALFORMED" "$TEST_ROOT/production-malformed.out"
|
||||
assert_line "$TEST_ROOT/production-malformed.out" "complete_local_recovery_point=no"
|
||||
assert_line "$TEST_ROOT/production-malformed.out" "invalid_local_batch_count=1"
|
||||
pass "畸形 manifest 安全归类为无效批次且不输出 traceback"
|
||||
|
||||
PRODUCTION_UNREADABLE="$TEST_ROOT/production-unreadable"
|
||||
make_production_layout "$PRODUCTION_UNREADABLE"
|
||||
make_batch "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z" "auto-20260828T000000Z"
|
||||
chmod 000 "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json"
|
||||
if [[ "$(id -u)" == 0 ]] && command -v runuser >/dev/null 2>&1 && id nobody >/dev/null 2>&1; then
|
||||
chmod 0755 "$TEST_ROOT" "$PRODUCTION_UNREADABLE" "$PRODUCTION_UNREADABLE/backups" "$PRODUCTION_UNREADABLE/state" \
|
||||
"$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z"
|
||||
if runuser -u nobody -- python3 -B "$INSPECTOR" --now-epoch 1787875200 production \
|
||||
--backup-root "$PRODUCTION_UNREADABLE/backups" \
|
||||
--state-dir "$PRODUCTION_UNREADABLE/state" \
|
||||
--wrapper-script "$ROOT_DIR/scripts/run-automatic-backup.sh" \
|
||||
--backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$TEST_ROOT/production-unreadable.out"; then
|
||||
printf '[FAIL] unreadable manifest unexpectedly passed\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
if run_production "$PRODUCTION_UNREADABLE" "$TEST_ROOT/production-unreadable.out"; then
|
||||
printf '[FAIL] unreadable manifest unexpectedly passed\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
chmod 0600 "$PRODUCTION_UNREADABLE/backups/hyb-oa-backup-auto-20260828T000000Z/manifest.json"
|
||||
assert_line "$TEST_ROOT/production-unreadable.out" "probe_category=access_denied"
|
||||
pass "访问拒绝保持为 unknown/ACCESS_DENIED 而不是伪装成无有效备份"
|
||||
|
||||
assert_line "$TEST_ROOT/production-valid.out" "retained_after_transfer_failure=yes"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "pruned_after_offhost_success=yes"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "generation_temp_cleanup_on_failure=yes"
|
||||
assert_line "$TEST_ROOT/production-valid.out" "cleanup_contract_source_verified=yes"
|
||||
pass "生产脚本顺序机械证明失败保留、成功后 prune 与生成临时清理"
|
||||
|
||||
SOURCE_MUTATION="$TEST_ROOT/source-mutation"
|
||||
cp "$ROOT_DIR/scripts/run-automatic-backup.sh" "$SOURCE_MUTATION"
|
||||
python3 - "$SOURCE_MUTATION" <<'PY'
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
path = Path(sys.argv[1])
|
||||
text = path.read_text(encoding="utf-8")
|
||||
transfer = text.index("capture_command offhost_transfer rsync")
|
||||
text = text[:transfer] + 'prune_local_backups "$batch_name"\n ' + text[transfer:]
|
||||
path.write_text(text, encoding="utf-8")
|
||||
PY
|
||||
python3 -B "$INSPECTOR" --now-epoch 1787875200 production \
|
||||
--backup-root "$PRODUCTION_VALID/backups" \
|
||||
--state-dir "$PRODUCTION_VALID/state" \
|
||||
--wrapper-script "$SOURCE_MUTATION" \
|
||||
--backup-script "$ROOT_DIR/scripts/backup-all.sh" >"$TEST_ROOT/source-mutation.out"
|
||||
assert_line "$TEST_ROOT/source-mutation.out" "retained_after_transfer_failure=no"
|
||||
assert_line "$TEST_ROOT/source-mutation.out" "pruned_after_offhost_success=no"
|
||||
pass "源码顺序漂移会关闭失败保留与成功后 prune 结论"
|
||||
|
||||
for code_category in \
|
||||
23:partial_transfer_error \
|
||||
30:transport_timeout \
|
||||
12:protocol_stream_error \
|
||||
11:file_io_error \
|
||||
255:ssh_session_error; do
|
||||
code="${code_category%%:*}"
|
||||
category="${code_category#*:}"
|
||||
fixture="$TEST_ROOT/rsync-$code"
|
||||
make_production_layout "$fixture"
|
||||
write_transfer_failure_state "$fixture"
|
||||
printf 'rsync error: synthetic redacted fixture (code %s) at fixture.c(1)\n' "$code" >"$fixture/state/last-error.log"
|
||||
run_production "$fixture" "$TEST_ROOT/rsync-$code.out"
|
||||
assert_line "$TEST_ROOT/rsync-$code.out" "transfer_exit_code=$code"
|
||||
assert_line "$TEST_ROOT/rsync-$code.out" "transfer_failure_category=$category"
|
||||
done
|
||||
pass "rsync 23/30/12/11/255 均映射为稳定脱敏类别"
|
||||
|
||||
PRODUCTION_UNKNOWN="$TEST_ROOT/production-unknown"
|
||||
make_production_layout "$PRODUCTION_UNKNOWN"
|
||||
write_transfer_failure_state "$PRODUCTION_UNKNOWN"
|
||||
printf 'unstructured synthetic transport failure\n' >"$PRODUCTION_UNKNOWN/state/last-error.log"
|
||||
run_production "$PRODUCTION_UNKNOWN" "$TEST_ROOT/production-unknown.out"
|
||||
assert_line "$TEST_ROOT/production-unknown.out" "transfer_exit_code=NOT_AVAILABLE"
|
||||
assert_line "$TEST_ROOT/production-unknown.out" "transfer_failure_category=not_available"
|
||||
pass "原始 rsync 码不存在时安全失败为 NOT_AVAILABLE"
|
||||
|
||||
PRODUCTION_PHASE_MISMATCH="$TEST_ROOT/production-phase-mismatch"
|
||||
make_production_layout "$PRODUCTION_PHASE_MISMATCH"
|
||||
printf '%s\n' '{"findings":[{"detail":"phase_offhost_finalize_failed"}]}' >"$PRODUCTION_PHASE_MISMATCH/state/last-run.json"
|
||||
printf 'rsync error: stale synthetic fixture (code 23) at fixture.c(1)\n' >"$PRODUCTION_PHASE_MISMATCH/state/last-error.log"
|
||||
run_production "$PRODUCTION_PHASE_MISMATCH" "$TEST_ROOT/production-phase-mismatch.out"
|
||||
assert_line "$TEST_ROOT/production-phase-mismatch.out" "last_backup_failure_phase=offhost_finalize"
|
||||
assert_line "$TEST_ROOT/production-phase-mismatch.out" "transfer_exit_code=NOT_AVAILABLE"
|
||||
assert_line "$TEST_ROOT/production-phase-mismatch.out" "transfer_exit_code_source=NOT_AVAILABLE"
|
||||
pass "私有诊断只在安全摘要明确为 offhost_transfer 时绑定 rsync 原始码"
|
||||
|
||||
ARM_SIMILAR="$TEST_ROOT/arm-similar"
|
||||
mkdir -p "$ARM_SIMILAR/inbox" "$ARM_SIMILAR/archive"
|
||||
make_batch "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260827T000000Z.partial" "auto-20260827T000000Z" 1000 1000
|
||||
make_batch "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z" 1100 1000
|
||||
rm "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260827T000000Z.partial/storage.tar.gz"
|
||||
rm "$ARM_SIMILAR/inbox/hyb-oa-backup-auto-20260828T000000Z.partial/storage.tar.gz"
|
||||
python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_SIMILAR/inbox" --archive-root "$ARM_SIMILAR/archive" >"$TEST_ROOT/arm-similar.out"
|
||||
assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_count=2"
|
||||
assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_stop_offset_class=similar"
|
||||
assert_line "$TEST_ROOT/arm-similar.out" "arm_partial_incomplete_contract_count=2"
|
||||
pass "两个 partial 接近停止偏移按预先冻结阈值分类为 similar"
|
||||
|
||||
ARM_DIFFERENT="$TEST_ROOT/arm-different"
|
||||
mkdir -p "$ARM_DIFFERENT/inbox" "$ARM_DIFFERENT/archive"
|
||||
make_batch "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260827T000000Z.partial" "auto-20260827T000000Z" 1000 1000
|
||||
make_batch "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z" 70000000 1000
|
||||
rm "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260827T000000Z.partial/storage.tar.gz"
|
||||
rm "$ARM_DIFFERENT/inbox/hyb-oa-backup-auto-20260828T000000Z.partial/storage.tar.gz"
|
||||
python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_DIFFERENT/inbox" --archive-root "$ARM_DIFFERENT/archive" >"$TEST_ROOT/arm-different.out"
|
||||
assert_line "$TEST_ROOT/arm-different.out" "arm_partial_stop_offset_class=different"
|
||||
pass "两个 partial 明显不同停止偏移分类为 different"
|
||||
|
||||
ARM_COMPLETE="$TEST_ROOT/arm-complete"
|
||||
mkdir -p "$ARM_COMPLETE/inbox" "$ARM_COMPLETE/archive"
|
||||
make_batch "$ARM_COMPLETE/inbox/hyb-oa-backup-auto-20260828T000000Z.partial" "auto-20260828T000000Z"
|
||||
make_batch "$ARM_COMPLETE/archive/hyb-oa-backup-auto-20260827T000000Z" "auto-20260827T000000Z"
|
||||
python3 -B "$INSPECTOR" --now-epoch 1787875200 arm --inbox-root "$ARM_COMPLETE/inbox" --archive-root "$ARM_COMPLETE/archive" >"$TEST_ROOT/arm-complete.out"
|
||||
assert_line "$TEST_ROOT/arm-complete.out" "arm_partial_complete_contract_count=1"
|
||||
assert_line "$TEST_ROOT/arm-complete.out" "arm_valid_atomic_archive_count=1"
|
||||
pass "完整 partial 与正式原子归档分别按同一三文件身份和哈希合同验证"
|
||||
|
||||
PERMISSION_FIXTURE="$TEST_ROOT/permission"
|
||||
mkdir -p "$PERMISSION_FIXTURE/inbox" "$PERMISSION_FIXTURE/archive"
|
||||
chmod 000 "$PERMISSION_FIXTURE/inbox"
|
||||
if command -v runuser >/dev/null 2>&1 && id nobody >/dev/null 2>&1 && [[ "$(id -u)" == 0 ]]; then
|
||||
if runuser -u nobody -- python3 -B "$INSPECTOR" arm --inbox-root "$PERMISSION_FIXTURE/inbox" --archive-root "$PERMISSION_FIXTURE/archive" >"$TEST_ROOT/permission.out"; then
|
||||
printf '[FAIL] permission denial unexpectedly passed\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
assert_line "$TEST_ROOT/permission.out" "probe_category=access_denied"
|
||||
else
|
||||
chmod 0700 "$PERMISSION_FIXTURE/inbox"
|
||||
ln -s "$PERMISSION_FIXTURE/inbox" "$PERMISSION_FIXTURE/inbox-link"
|
||||
if python3 -B "$INSPECTOR" arm --inbox-root "$PERMISSION_FIXTURE/inbox-link" --archive-root "$PERMISSION_FIXTURE/archive" >"$TEST_ROOT/permission.out"; then
|
||||
printf '[FAIL] invalid root unexpectedly passed\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
assert_line "$TEST_ROOT/permission.out" "probe_category=root_invalid"
|
||||
fi
|
||||
chmod 0700 "$PERMISSION_FIXTURE/inbox"
|
||||
pass "不可读或不安全根目录明确拒绝且不伪装为不存在"
|
||||
|
||||
if rg -n --fixed-strings \
|
||||
-e "$TEST_ROOT" \
|
||||
-e "auto-20260828T000000Z" \
|
||||
-e "database.dump" \
|
||||
-e "storage.tar.gz" \
|
||||
-e "unstructured synthetic transport failure" \
|
||||
"$TEST_ROOT"/*.out >/dev/null; then
|
||||
printf '[FAIL] redacted output leaked fixture identity, path, artifact name, or diagnostic text\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
pass "stdout allowlist 不含路径、run id、产物名或原始诊断"
|
||||
|
||||
PYTHONPYCACHEPREFIX="$TEST_ROOT/pycache" python3 -m py_compile "$INSPECTOR"
|
||||
bash -n "$ROOT_DIR/scripts/qa/run-backup-transfer-incident-contract.sh"
|
||||
pass "Python 与 Bash 语法通过"
|
||||
|
||||
cleanup
|
||||
[[ ! -e "$TEST_ROOT" ]]
|
||||
TEST_ROOT=""
|
||||
trap - EXIT
|
||||
pass "合成 fixture 与 Python cache 零残留"
|
||||
|
||||
printf '[PASS] backup transfer incident contract: %s checks\n' "$PASS_COUNT"
|
||||
在新工单中引用
屏蔽一个用户