完善异机备份告警与同日恢复点取代（07-27-automatic-offhost-backup）

scripts/run-automatic-backup.sh

@@ -15,9 +15,10 @@ AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}"
 OFFHOST_TARGET="${OFFHOST_TARGET:-}"
 OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}"
 OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}"
+ALERT_WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-}"
+ALERT_HELPER="${ALERT_HELPER:-$ROOT_DIR/scripts/send-wecom-group-alert.py}"
 AUTOMATIC_BACKUP_KEEP="${AUTOMATIC_BACKUP_KEEP:-7}"
 AUTOMATIC_BACKUP_MAX_AGE_SECONDS="${AUTOMATIC_BACKUP_MAX_AGE_SECONDS:-86400}"
-ALERT_RECIPIENT_EMPLOYEE_NO="${ALERT_RECIPIENT_EMPLOYEE_NO:-HYD000006}"
 MODE="${1:-run}"
 
 CURRENT_PHASE="preflight"
@@ -79,7 +80,12 @@ validate_common() {
     || fail "frontend build SHA must match the full build prefix"
   [[ "$AUTOMATIC_BACKUP_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "backup retention must be positive"
   [[ "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "stale threshold must be positive"
-  [[ "$ALERT_RECIPIENT_EMPLOYEE_NO" =~ ^[A-Za-z0-9_-]{1,80}$ ]] || fail "alert recipient id is invalid"
+  validate_absolute_path ALERT_WEBHOOK_FILE "$ALERT_WEBHOOK_FILE"
+  validate_absolute_path ALERT_HELPER "$ALERT_HELPER"
+  [[ -f "$ALERT_WEBHOOK_FILE" && ! -L "$ALERT_WEBHOOK_FILE" \
+    && "$(stat -c %a "$ALERT_WEBHOOK_FILE")" == "600" ]] \
+    || fail "alert webhook file must exist with mode 0600"
+  [[ -f "$ALERT_HELPER" && ! -L "$ALERT_HELPER" ]] || fail "alert helper is missing"
   if [[ "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR" \
     || "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR"/* \
     || "$AUTOMATIC_BACKUP_STATE_DIR" == "$AUTOMATIC_BACKUP_DIR"/* ]]; then
@@ -218,10 +224,25 @@ PYTHON
 send_summary_alert() {
   local state_name="$1"
   [[ -n "$SUMMARY_FILE" && -f "$SUMMARY_FILE" ]] || return 1
-  [[ -n "$SERVER_CONTAINER" ]] || resolve_active_runtime
-  compose exec -T server sh -lc \
-    "cd /app/server && WARCH3_OPS_ALERT_SEND=1 npm run send:wecom-ops-alert -- --summary-file=- --recipient-employee-no=${ALERT_RECIPIENT_EMPLOYEE_NO} --send --state-file=/app/artifacts/qa-flow/${state_name} --format=json" \
-    <"$SUMMARY_FILE" >/dev/null
+  python3 "$ALERT_HELPER" \
+    --webhook-file "$ALERT_WEBHOOK_FILE" \
+    --summary-file "$SUMMARY_FILE" \
+    --state-file "$AUTOMATIC_BACKUP_STATE_DIR/$state_name" >/dev/null
+}
+
+capture_command() {
+  local phase="$1"
+  shift
+  local temporary="$AUTOMATIC_BACKUP_STATE_DIR/.${phase}.diagnostic.$$"
+  : >"$temporary"
+  chmod 0600 "$temporary"
+  if "$@" >"$temporary" 2>&1; then
+    rm -f -- "$temporary"
+    return 0
+  fi
+  mv -f -- "$temporary" "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
+  chmod 0600 "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
+  return 1
 }
 
 on_error() {
@@ -320,11 +341,12 @@ print(f"{unquote(value.username)}\t{database_name}")
   [[ "$source_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "active server image digest is invalid"
 
   CURRENT_PHASE="paired_backup"
-  if ! DATABASE_URL="$database_url" DB_TOOL_MODE=docker DB_CONTAINER="$DB_CONTAINER" \
+  if ! capture_command paired_backup env \
+    DATABASE_URL="$database_url" DB_TOOL_MODE=docker DB_CONTAINER="$DB_CONTAINER" \
     POSTGRES_DB="$postgres_db" POSTGRES_USER="$postgres_user" POSTGRES_PASSWORD="$postgres_password" \
     STORAGE_DIR_HOST="$STORAGE_HOST_DIR" BACKUP_DIR="$AUTOMATIC_BACKUP_DIR" \
     BACKUP_RUN_ID="$run_id" BUILD_SHA="$BUILD_SHA" SOURCE_IMAGE_DIGEST="$source_image_digest" \
-    "$ROOT_DIR/scripts/backup-all.sh" >/dev/null 2>&1; then
+    "$ROOT_DIR/scripts/backup-all.sh"; then
     fail "paired backup command failed"
   fi
   [[ -d "$batch_dir" && -f "$batch_dir/manifest.json" ]] || fail "complete local batch was not published"
@@ -343,14 +365,14 @@ print(f"{unquote(value.username)}\t{database_name}")
     "$OFFHOST_SSH_KEY" "$OFFHOST_KNOWN_HOSTS"
 
   CURRENT_PHASE="offhost_transfer"
-  if ! rsync -a --partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= -e "$rsync_shell" \
-    "$batch_dir/" "$OFFHOST_TARGET:${batch_name}.partial/" >/dev/null 2>&1; then
+  if ! capture_command offhost_transfer rsync -a --partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= -e "$rsync_shell" \
+    "$batch_dir/" "$OFFHOST_TARGET:${batch_name}.partial/"; then
     fail "off-host transfer failed"
   fi
 
   CURRENT_PHASE="offhost_finalize"
-  if ! ssh "${ssh_options[@]}" "$OFFHOST_TARGET" \
-    "hyb-backup-finalize $run_id $manifest_sha" >/dev/null 2>&1; then
+  if ! capture_command offhost_finalize ssh "${ssh_options[@]}" "$OFFHOST_TARGET" \
+    "hyb-backup-finalize $run_id $manifest_sha"; then
     fail "off-host finalize failed"
   fi
 
@@ -359,6 +381,7 @@ print(f"{unquote(value.username)}\t{database_name}")
   prune_local_backups "$batch_name"
   write_summary "PASS_AUTOMATIC_BACKUP_COMPLETE" 0 "offhost_verified"
   send_summary_alert "automatic-backup-alert-state.json" || true
+  rm -f -- "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
   trap - ERR
   echo "[automatic-backup][PASS] paired batch finalized off-host"
 }
@@ -369,9 +392,11 @@ check_stale() {
   CURRENT_PHASE="stale_check"
   resolve_active_runtime
   local success_file="$AUTOMATIC_BACKUP_STATE_DIR/last-success.json"
-  if ! python3 - "$success_file" "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON'
+  local stale_result
+  if ! stale_result="$(python3 - "$success_file" "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON'
 import json
 import os
+import re
 import sys
 from datetime import datetime, timezone
 
@@ -382,15 +407,29 @@ with open(file_path, "r", encoding="utf-8") as handle:
     payload = json.load(handle)
 if payload.get("schemaVersion") != 1 or payload.get("status") != "complete":
     raise SystemExit(1)
-if payload.get("composeProject") != expected_project or payload.get("buildSha") != expected_build:
+if payload.get("composeProject") != expected_project:
+    raise SystemExit(1)
+if not re.fullmatch(r"auto-[0-9]{8}T[0-9]{6}Z", payload.get("runId", "")):
+    raise SystemExit(1)
+if not re.fullmatch(r"[0-9a-f]{64}", payload.get("manifestSha256", "")):
+    raise SystemExit(1)
+if not re.fullmatch(r"[0-9a-fA-F]{40,64}", payload.get("buildSha", "")):
     raise SystemExit(1)
 completed = datetime.fromisoformat(payload["completedAt"].replace("Z", "+00:00"))
 age = (datetime.now(timezone.utc) - completed).total_seconds()
 if age < 0 or age > int(max_age):
     raise SystemExit(1)
+print("BUILD_MISMATCH" if payload.get("buildSha") != expected_build else "FRESH")
 PYTHON
-  then
-    fail "latest off-host success is missing, invalid, mismatched, or stale"
+  )"; then
+    fail "latest off-host success is missing, invalid, project-mismatched, or stale"
+  fi
+  if [[ "$stale_result" == "BUILD_MISMATCH" ]]; then
+    write_summary "PASS_WITH_FINDINGS" 1 "recovery_point_build_differs"
+    send_summary_alert "automatic-backup-alert-state.json" || true
+    trap - ERR
+    echo "[automatic-backup][PASS_WITH_FINDINGS] off-host recovery point is fresh but build differs"
+    return 0
   fi
   write_summary "PASS_AUTOMATIC_BACKUP_FRESH" 0 "fresh_offhost_recovery_point"
   send_summary_alert "automatic-backup-alert-state.json" || true