完善异机备份告警与同日恢复点取代（07-27-automatic-offhost-backup）

scripts/receive-offhost-backup.sh

@@ -60,6 +60,7 @@ validate_layout() {
   [[ "$DAILY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "DAILY_KEEP must be positive"
   [[ "$WEEKLY_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "WEEKLY_KEEP must be positive"
   command -v "$PYTHON_BIN" >/dev/null 2>&1 || fail "python3 is required"
+  command -v flock >/dev/null 2>&1 || fail "flock is required"
 }
 
 validate_run_id() {
@@ -155,11 +156,44 @@ prune_named_directories() {
   done
 }
 
+publish_weekly_checkpoint() {
+  local archive_dir="$1"
+  local weekly_dir="$2"
+  local replace_existing="$3"
+  if [[ -e "$weekly_dir" && "$replace_existing" != "1" ]]; then
+    return 0
+  fi
+  local weekly_temp="${weekly_dir}.new.$$"
+  local weekly_rollback="${weekly_dir}.rollback.$$"
+  [[ ! -e "$weekly_temp" && ! -e "$weekly_rollback" ]] || return 1
+  if ! cp -al -- "$archive_dir" "$weekly_temp"; then
+    rm -rf -- "$weekly_temp"
+    return 1
+  fi
+  if [[ -e "$weekly_dir" ]]; then
+    [[ -d "$weekly_dir" && ! -L "$weekly_dir" ]] || return 1
+    mv -- "$weekly_dir" "$weekly_rollback" || return 1
+  fi
+  if ! mv -- "$weekly_temp" "$weekly_dir"; then
+    rm -rf -- "$weekly_temp"
+    if [[ -e "$weekly_rollback" ]]; then
+      mv -- "$weekly_rollback" "$weekly_dir"
+    fi
+    return 1
+  fi
+  if [[ -e "$weekly_rollback" ]]; then
+    rm -rf -- "$weekly_rollback"
+  fi
+}
+
 finalize_batch() {
   local run_id="$1"
   local expected_manifest_sha="$2"
   validate_run_id "$run_id"
   validate_layout
+  exec 9>"$BACKUP_ARCHIVE_DIR/.finalize.lock"
+  chmod 0600 "$BACKUP_ARCHIVE_DIR/.finalize.lock"
+  flock -n 9 || fail "another finalize operation is already running"
 
   local batch_name="hyb-oa-backup-$run_id"
   local partial_dir="$BACKUP_INBOX_DIR/${batch_name}.partial"
@@ -173,15 +207,32 @@ finalize_batch() {
 
   local run_day="${run_id#auto-}"
   run_day="${run_day%%T*}"
-  local existing_for_day
-  existing_for_day="$(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 -type d \
-    -name "hyb-oa-backup-auto-${run_day}T*Z" ! -name "$batch_name" -print -quit)"
-  [[ -z "$existing_for_day" ]] || fail "another automatic archive already exists for this UTC day"
+  local existing_for_day="" replacement_rollback=""
+  local -a existing_for_day_entries=()
+  mapfile -t existing_for_day_entries < <(find "$BACKUP_ARCHIVE_DIR" -mindepth 1 -maxdepth 1 -type d \
+    -name "hyb-oa-backup-auto-${run_day}T*Z" ! -name "$batch_name" -print)
+  [[ "${#existing_for_day_entries[@]}" -le 1 ]] || fail "daily archive invariant is already violated"
+  if [[ "${#existing_for_day_entries[@]}" -eq 1 ]]; then
+    existing_for_day="${existing_for_day_entries[0]}"
+  fi
 
   if [[ "$already_archived" == "0" ]]; then
     validate_batch "$partial_dir" "$run_id" "$expected_manifest_sha"
     chmod 0400 "$partial_dir/database.dump" "$partial_dir/storage.tar.gz" "$partial_dir/manifest.json"
-    mv -- "$partial_dir" "$archive_dir"
+    if [[ -n "$existing_for_day" ]]; then
+      [[ "$existing_for_day" =~ /hyb-oa-backup-auto-${run_day}T[0-9]{6}Z$ \
+        && -d "$existing_for_day" && ! -L "$existing_for_day" ]] \
+        || fail "existing daily archive is invalid"
+      replacement_rollback="$BACKUP_ARCHIVE_DIR/.daily-replacement-${run_day}.$$"
+      [[ ! -e "$replacement_rollback" ]] || fail "daily replacement rollback target already exists"
+      mv -- "$existing_for_day" "$replacement_rollback"
+    fi
+    if ! mv -- "$partial_dir" "$archive_dir"; then
+      if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
+        mv -- "$replacement_rollback" "$existing_for_day"
+      fi
+      fail "daily archive publication failed"
+    fi
   fi
 
   local iso_week
@@ -192,10 +243,17 @@ day = datetime.strptime(sys.argv[1], "%Y%m%d").date()
 iso = day.isocalendar()
 print(f"week-{iso.year:04d}-W{iso.week:02d}")
 PYTHON
-)"
+  )"
   local weekly_dir="$BACKUP_WEEKLY_DIR/$iso_week"
-  if [[ ! -e "$weekly_dir" ]]; then
-    cp -al -- "$archive_dir" "$weekly_dir"
+  if ! publish_weekly_checkpoint "$archive_dir" "$weekly_dir" "$([[ -n "$replacement_rollback" ]] && echo 1 || echo 0)"; then
+    if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
+      rm -rf -- "$archive_dir"
+      mv -- "$replacement_rollback" "$existing_for_day"
+    fi
+    fail "weekly checkpoint update failed"
+  fi
+  if [[ -n "$replacement_rollback" && -e "$replacement_rollback" ]]; then
+    rm -rf -- "$replacement_rollback"
   fi
 
   prune_named_directories "$BACKUP_ARCHIVE_DIR" "hyb-oa-backup-auto-" "$DAILY_KEEP"