完善异机备份告警与同日恢复点取代（07-27-automatic-offhost-backup）

2026-07-27 13:54:16 +08:00
@@ -78,7 +78,7 @@ receiver() {
    "$ROOT_DIR/scripts/receive-offhost-backup.sh" "$@"
 }

-CURRENT_RUN_ID="auto-$(date -u +%Y%m%dT%H%M%SZ)"
+CURRENT_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z"
 CURRENT_BATCH="hyb-oa-backup-$CURRENT_RUN_ID"

 VALID_LAYOUT="$TEST_DIR/receiver-valid"
@@ -92,15 +92,41 @@ pass "receiver finalizes one complete synthetic batch"
 receiver "$VALID_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$VALID_MANIFEST_SHA" >/dev/null
 pass "receiver treats an identical repeated finalize as idempotent"

-SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000000Z"
-if [[ "$SECOND_RUN_ID" == "$CURRENT_RUN_ID" ]]; then
-  SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z"
-fi
+SECOND_RUN_ID="auto-$(date -u +%Y%m%d)T000001Z"
 SECOND_BATCH="hyb-oa-backup-$SECOND_RUN_ID"
 make_batch "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
 SECOND_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
-expect_failure "receiver rejects a different automatic batch for the same UTC day" \
-  receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA"
+receiver "$VALID_LAYOUT" finalize-test "$SECOND_RUN_ID" "$SECOND_MANIFEST_SHA" >/dev/null
+[[ ! -e "$VALID_LAYOUT/archive/$CURRENT_BATCH" && -d "$VALID_LAYOUT/archive/$SECOND_BATCH" ]]
+WEEKLY_BATCH="$(find "$VALID_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -print -quit)"
+[[ -n "$WEEKLY_BATCH" ]]
+[[ "$(stat -c %i "$WEEKLY_BATCH/database.dump")" == "$(stat -c %i "$VALID_LAYOUT/archive/$SECOND_BATCH/database.dump")" ]]
+pass "receiver atomically replaces a different batch for the same UTC day and rebuilds its weekly checkpoint"
+
+THIRD_RUN_ID="auto-$(date -u +%Y%m%d)T000002Z"
+THIRD_BATCH="hyb-oa-backup-$THIRD_RUN_ID"
+make_batch "$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial" "$THIRD_RUN_ID"
+printf 'corrupt candidate\n' >>"$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial/database.dump"
+THIRD_MANIFEST_SHA="$(sha256sum "$VALID_LAYOUT/inbox/${THIRD_BATCH}.partial/manifest.json" | awk '{print $1}')"
+expect_failure "receiver validates a same-day replacement before touching the current archive" \
+  receiver "$VALID_LAYOUT" finalize-test "$THIRD_RUN_ID" "$THIRD_MANIFEST_SHA"
+[[ -d "$VALID_LAYOUT/archive/$SECOND_BATCH" && ! -e "$VALID_LAYOUT/archive/$THIRD_BATCH" ]]
+pass "receiver preserves the current daily archive when a replacement candidate is corrupt"
+
+ROLLBACK_LAYOUT="$TEST_DIR/receiver-replacement-rollback"
+make_layout "$ROLLBACK_LAYOUT"
+make_batch "$ROLLBACK_LAYOUT/inbox/${CURRENT_BATCH}.partial" "$CURRENT_RUN_ID"
+ROLLBACK_FIRST_SHA="$(sha256sum "$ROLLBACK_LAYOUT/inbox/${CURRENT_BATCH}.partial/manifest.json" | awk '{print $1}')"
+receiver "$ROLLBACK_LAYOUT" finalize-test "$CURRENT_RUN_ID" "$ROLLBACK_FIRST_SHA" >/dev/null
+ROLLBACK_WEEKLY="$(find "$ROLLBACK_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -print -quit)"
+find "$ROLLBACK_LAYOUT/weekly" -mindepth 1 -maxdepth 1 -type d -exec rm -rf -- {} +
+printf 'invalid weekly target\n' >"$ROLLBACK_WEEKLY"
+make_batch "$ROLLBACK_LAYOUT/inbox/${SECOND_BATCH}.partial" "$SECOND_RUN_ID"
+ROLLBACK_SECOND_SHA="$(sha256sum "$ROLLBACK_LAYOUT/inbox/${SECOND_BATCH}.partial/manifest.json" | awk '{print $1}')"
+expect_failure "receiver rolls back the daily archive when weekly checkpoint publication fails" \
+  receiver "$ROLLBACK_LAYOUT" finalize-test "$SECOND_RUN_ID" "$ROLLBACK_SECOND_SHA"
+[[ -d "$ROLLBACK_LAYOUT/archive/$CURRENT_BATCH" && ! -e "$ROLLBACK_LAYOUT/archive/$SECOND_BATCH" ]]
+pass "receiver restores the prior daily point after a later replacement step fails"

 negative_receiver_case() {
  local label="$1"
@@ -225,7 +251,7 @@ if [[ "${1:-}" == "compose" ]]; then
  args=" $* "
  if [[ "$args" == *" ps -q server "* ]]; then echo server-id; exit 0; fi
  if [[ "$args" == *" ps -q db "* ]]; then echo db-id; exit 0; fi
-  if [[ "$args" == *" exec -T server "* ]]; then cat >>"$MOCK_ALERT_LOG"; exit 0; fi
+  if [[ "$args" == *" exec -T server "* ]]; then exit 99; fi
 fi
 if [[ "${1:-}" == "inspect" ]]; then
  template="${3:-}"
@@ -260,6 +286,7 @@ MOCK

 cat >"$MOCK_BIN/rsync" <<'MOCK'
 #!/usr/bin/env bash
+echo 'synthetic transfer diagnostic' >&2
 exit "${MOCK_RSYNC_EXIT:-1}"
 MOCK
 cat >"$MOCK_BIN/ssh" <<'MOCK'
@@ -267,6 +294,34 @@ cat >"$MOCK_BIN/ssh" <<'MOCK'
 exit "${MOCK_SSH_EXIT:-0}"
 MOCK
 chmod +x "$MOCK_BIN/docker" "$MOCK_BIN/rsync" "$MOCK_BIN/ssh"
+WECOM_TEST_HOST='qyapi.weixin.qq.com'
+WECOM_TEST_PATH='/cgi-bin/webhook/send'
+printf 'https://%s%s?key=%s\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
+  '00000000-0000-0000-0000-000000000000' >"$TEST_DIR/wecom-webhook"
+chmod 0600 "$TEST_DIR/wecom-webhook"
+ALERT_CAPTURE="$TEST_DIR/alert-capture.jsonl"
+cat >"$TEST_DIR/failure-summary.json" <<'JSON'
+{"issueId":"AUTO_BACKUP","status":"FAIL_AUTOMATIC_BACKUP","conclusion":"FAIL_AUTOMATIC_BACKUP","findingCount":1,"findings":[{"source":"automatic_backup","status":"FAIL_AUTOMATIC_BACKUP","severity":"P0","detail":"synthetic_failure"}]}
+JSON
+chmod 0600 "$TEST_DIR/failure-summary.json"
+chmod 0644 "$TEST_DIR/wecom-webhook"
+expect_failure "group alert helper rejects a webhook file that is not mode 0600" env \
+  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
+  python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook" \
+  --summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
+chmod 0600 "$TEST_DIR/wecom-webhook"
+ln -s "$TEST_DIR/wecom-webhook" "$TEST_DIR/wecom-webhook-link"
+expect_failure "group alert helper rejects a symbolic-link webhook file" env \
+  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
+  python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook-link" \
+  --summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"
+printf 'https://%s%s?key=%s&extra=forbidden\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
+  '00000000-0000-0000-0000-000000000000' >"$TEST_DIR/wecom-webhook-invalid"
+chmod 0600 "$TEST_DIR/wecom-webhook-invalid"
+expect_failure "group alert helper rejects a webhook URL with an unexpected query field" env \
+  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE" \
+  python3 "$ROOT_DIR/scripts/send-wecom-group-alert.py" --webhook-file "$TEST_DIR/wecom-webhook-invalid" \
+  --summary-file "$TEST_DIR/failure-summary.json" --state-file "$TEST_DIR/invalid-alert-state.json"

 wrapper_env=(
  env PATH="$MOCK_BIN:$PATH" REMOTE_DIR="$WRAPPER_ROOT"
@@ -275,7 +330,9 @@ wrapper_env=(
  STORAGE_HOST_DIR="$WRAPPER_STORAGE" BUILD_SHA="$BUILD_SHA" VITE_BUILD_SHA="$VITE_BUILD_SHA"
  AUTOMATIC_BACKUP_DIR="$WRAPPER_BACKUPS" AUTOMATIC_BACKUP_STATE_DIR="$WRAPPER_STATE"
  OFFHOST_TARGET="hybbackup@backup-alias" OFFHOST_SSH_KEY="$TEST_DIR/offhost-key"
-  OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_RECIPIENT_EMPLOYEE_NO="OPS001"
+  OFFHOST_KNOWN_HOSTS="$TEST_DIR/known-hosts" ALERT_WEBHOOK_FILE="$TEST_DIR/wecom-webhook"
+  ALERT_HELPER="$ROOT_DIR/scripts/send-wecom-group-alert.py"
+  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$ALERT_CAPTURE"
 )

 exec 8>"$WRAPPER_STATE/automatic-backup.lock"
@@ -305,6 +362,9 @@ assert payload["status"] == "FAIL_AUTOMATIC_BACKUP"
 assert not any(value in serialized for value in ("/tmp/", "postgresql://", "backup-alias", "fixture"))
 PYTHON
 pass "wrapper failure summary is category-only and redacted"
+[[ -f "$WRAPPER_STATE/last-error.log" && "$(stat -c %a "$WRAPPER_STATE/last-error.log")" == "600" ]]
+grep -Fq 'synthetic transfer diagnostic' "$WRAPPER_STATE/last-error.log"
+pass "wrapper retains the latest command diagnostic locally with mode 0600"

 find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' -exec rm -rf -- {} +
 for day in 01 02 03 04 05 06 07; do
@@ -315,6 +375,8 @@ done
 [[ -f "$WRAPPER_STATE/last-success.json" ]]
 [[ "$(find "$WRAPPER_BACKUPS" -mindepth 1 -maxdepth 1 -type d -name 'hyb-oa-backup-auto-*' | wc -l)" == "7" ]]
 pass "off-host finalize success writes state and then prunes source retention to seven"
+[[ ! -e "$WRAPPER_STATE/last-error.log" ]]
+pass "wrapper clears the retained diagnostic after a successful off-host run"

 rm -f "$WRAPPER_STATE/last-success.json"
 expect_failure "stale watchdog rejects a missing success record" \
@@ -329,7 +391,7 @@ with open(sys.argv[1], "w", encoding="utf-8") as handle:
        "schemaVersion": 1,
        "status": "complete",
        "completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
-        "runId": "auto-synthetic",
+        "runId": "auto-20260727T000000Z",
        "manifestSha256": "0" * 64,
        "composeProject": "hyb-oa-current",
        "buildSha": sys.argv[2],
@@ -345,12 +407,89 @@ payload["buildSha"] = "0" * 40
 with open(sys.argv[1], "w", encoding="utf-8") as handle:
    json.dump(payload, handle)
 PYTHON
-expect_failure "stale watchdog rejects a success record from another build" \
-  "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
+"${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
+python3 - "$WRAPPER_STATE/last-run.json" <<'PYTHON'
+import json
+import sys
+with open(sys.argv[1], encoding="utf-8") as handle:
+    payload = json.load(handle)
+assert payload["status"] == "PASS_WITH_FINDINGS"
+assert payload["findings"][0]["detail"] == "recovery_point_build_differs"
+PYTHON
+pass "stale watchdog downgrades a fresh recovery point from another build to one finding"
 mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
 "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale >/dev/null
 pass "stale watchdog accepts a fresh project/build-bound success record"

+cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
+python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
+import json
+import sys
+with open(sys.argv[1], encoding="utf-8") as handle:
+    payload = json.load(handle)
+payload["composeProject"] = "different-project"
+with open(sys.argv[1], "w", encoding="utf-8") as handle:
+    json.dump(payload, handle)
+PYTHON
+expect_failure "stale watchdog hard-fails a fresh record from another Compose project" \
+  "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
+mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
+cp "$WRAPPER_STATE/last-success.json" "$WRAPPER_STATE/last-success.valid.json"
+python3 - "$WRAPPER_STATE/last-success.json" <<'PYTHON'
+import json
+import sys
+with open(sys.argv[1], encoding="utf-8") as handle:
+    payload = json.load(handle)
+payload["manifestSha256"] = "invalid"
+with open(sys.argv[1], "w", encoding="utf-8") as handle:
+    json.dump(payload, handle)
+PYTHON
+expect_failure "stale watchdog hard-fails a malformed success record" \
+  "${wrapper_env[@]}" "$ROOT_DIR/scripts/run-automatic-backup.sh" check-stale
+mv "$WRAPPER_STATE/last-success.valid.json" "$WRAPPER_STATE/last-success.json"
+
+DEADMAN_LAYOUT="$TEST_DIR/deadman"
+mkdir -p "$DEADMAN_LAYOUT/archive" "$DEADMAN_LAYOUT/state"
+cp -a "$VALID_LAYOUT/archive/$SECOND_BATCH" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
+DEADMAN_CAPTURE="$TEST_DIR/deadman-alerts.jsonl"
+deadman_env=(env HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$DEADMAN_CAPTURE")
+"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
+  --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
+  --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
+[[ ! -e "$DEADMAN_CAPTURE" ]]
+pass "ARM dead-man accepts a fresh verified archive without sending a healthy baseline"
+DEADMAN_CORRUPT_STATE="$TEST_DIR/deadman-corrupt-state"
+DEADMAN_CORRUPT_CAPTURE="$TEST_DIR/deadman-corrupt-alerts.jsonl"
+mkdir "$DEADMAN_CORRUPT_STATE"
+chmod 0600 "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
+printf 'corruption\n' >>"$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
+expect_failure "ARM dead-man rejects a corrupt newest finalized archive" env \
+  HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$DEADMAN_CORRUPT_CAPTURE" \
+  python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
+    --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_CORRUPT_STATE" \
+    --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
+cp "$VALID_LAYOUT/archive/$SECOND_BATCH/database.dump" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
+chmod 0400 "$DEADMAN_LAYOUT/archive/$SECOND_BATCH/database.dump"
+pass "ARM dead-man detects archive corruption independently of source state"
+mv "$DEADMAN_LAYOUT/archive/$SECOND_BATCH" "$TEST_DIR/deadman-withheld-batch"
+expect_failure "ARM dead-man detects that the source no longer produces a finalized archive" \
+  "${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
+    --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
+    --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
+[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "1" ]]
+expect_failure "ARM dead-man suppresses an identical repeated failure alert" \
+  "${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
+    --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
+    --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py"
+[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "1" ]]
+pass "ARM dead-man keeps repeated identical failure alerts deduplicated"
+mv "$TEST_DIR/deadman-withheld-batch" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
+"${deadman_env[@]}" python3 "$ROOT_DIR/scripts/check-offhost-backup-deadman.py" \
+  --archive-root "$DEADMAN_LAYOUT/archive" --state-dir "$DEADMAN_LAYOUT/state" \
+  --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
+[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "2" ]]
+pass "ARM dead-man sends one recovery notification after finalized archives resume"
+
 DRIFT_BUILD_SHA="0123456789abcdef0123456789abcdef01234567"
 expect_failure "wrapper rejects an active runtime build that drifted from its installed contract" \
  "${wrapper_env[@]}" BUILD_SHA="$DRIFT_BUILD_SHA" VITE_BUILD_SHA="${DRIFT_BUILD_SHA:0:12}" \