478 行
18 KiB
Bash
可执行文件
478 行
18 KiB
Bash
可执行文件
#!/usr/bin/env bash
|
|
set -eEuo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
REMOTE_DIR="${REMOTE_DIR:-$ROOT_DIR}"
|
|
REMOTE_ENV_FILE="${REMOTE_ENV_FILE:-${REMOTE_DIR}/deploy/trialrun/.env}"
|
|
COMPOSE_FILE="${COMPOSE_FILE:-deploy/trialrun/docker-compose.yml}"
|
|
COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME:-}"
|
|
POSTGRES_DATA_DIR="${POSTGRES_DATA_DIR:-}"
|
|
STORAGE_HOST_DIR="${STORAGE_HOST_DIR:-}"
|
|
BUILD_SHA="${BUILD_SHA:-}"
|
|
VITE_BUILD_SHA="${VITE_BUILD_SHA:-}"
|
|
AUTOMATIC_BACKUP_DIR="${AUTOMATIC_BACKUP_DIR:-}"
|
|
AUTOMATIC_BACKUP_STATE_DIR="${AUTOMATIC_BACKUP_STATE_DIR:-}"
|
|
OFFHOST_TARGET="${OFFHOST_TARGET:-}"
|
|
OFFHOST_SSH_KEY="${OFFHOST_SSH_KEY:-}"
|
|
OFFHOST_KNOWN_HOSTS="${OFFHOST_KNOWN_HOSTS:-}"
|
|
ALERT_WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-}"
|
|
ALERT_HELPER="${ALERT_HELPER:-$ROOT_DIR/scripts/send-wecom-group-alert.py}"
|
|
AUTOMATIC_BACKUP_KEEP="${AUTOMATIC_BACKUP_KEEP:-7}"
|
|
AUTOMATIC_BACKUP_MAX_AGE_SECONDS="${AUTOMATIC_BACKUP_MAX_AGE_SECONDS:-86400}"
|
|
MODE="${1:-run}"
|
|
|
|
CURRENT_PHASE="preflight"
|
|
SERVER_CONTAINER=""
|
|
DB_CONTAINER=""
|
|
SUMMARY_FILE=""
|
|
COMPOSE_CMD=()
|
|
|
|
fail() {
|
|
echo "[automatic-backup][ERROR] $1" >&2
|
|
return 1
|
|
}
|
|
|
|
validate_absolute_path() {
|
|
local name="$1"
|
|
local value="$2"
|
|
if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then
|
|
fail "$name must be an explicit safe absolute path"
|
|
fi
|
|
if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \
|
|
|| "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then
|
|
fail "$name contains an unsafe path segment"
|
|
fi
|
|
}
|
|
|
|
validate_managed_directory_path() {
|
|
local name="$1"
|
|
local value="$2"
|
|
validate_absolute_path "$name" "$value"
|
|
[[ "$value" =~ ^/[^/]+/[^/]+ ]] \
|
|
|| fail "$name must be a dedicated path below a top-level directory"
|
|
case "$value" in
|
|
/bin/*|/boot/*|/dev/*|/etc/*|/lib/*|/lib64/*|/proc/*|/run/*|/sbin/*|/sys/*|/usr/*)
|
|
fail "$name must not be inside a system-managed directory"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
validate_common() {
|
|
local name value
|
|
for name in REMOTE_DIR REMOTE_ENV_FILE POSTGRES_DATA_DIR STORAGE_HOST_DIR \
|
|
AUTOMATIC_BACKUP_DIR AUTOMATIC_BACKUP_STATE_DIR; do
|
|
value="${!name:-}"
|
|
validate_absolute_path "$name" "$value"
|
|
done
|
|
validate_managed_directory_path AUTOMATIC_BACKUP_DIR "$AUTOMATIC_BACKUP_DIR"
|
|
validate_managed_directory_path AUTOMATIC_BACKUP_STATE_DIR "$AUTOMATIC_BACKUP_STATE_DIR"
|
|
[[ -d "$REMOTE_DIR" && -f "$REMOTE_ENV_FILE" ]] || fail "deployment root or env file is missing"
|
|
[[ -f "$POSTGRES_DATA_DIR/PG_VERSION" ]] || fail "active PGDATA marker is missing"
|
|
[[ -d "$STORAGE_HOST_DIR" ]] || fail "active storage directory is missing"
|
|
[[ "$COMPOSE_PROJECT_NAME" =~ ^[a-z0-9][a-z0-9_-]*$ ]] || fail "Compose project name is invalid"
|
|
[[ "$COMPOSE_FILE" =~ ^[A-Za-z0-9._/-]+$ && "$COMPOSE_FILE" != /* \
|
|
&& "/$COMPOSE_FILE/" != *"/../"* && "/$COMPOSE_FILE/" != *"/./"* ]] \
|
|
|| fail "Compose file must be a safe relative path"
|
|
[[ "$BUILD_SHA" =~ ^[0-9a-fA-F]{40,64}$ ]] || fail "full build SHA is invalid"
|
|
BUILD_SHA="${BUILD_SHA,,}"
|
|
VITE_BUILD_SHA="${VITE_BUILD_SHA,,}"
|
|
[[ "$VITE_BUILD_SHA" =~ ^[0-9a-f]{12}$ && "${BUILD_SHA:0:12}" == "$VITE_BUILD_SHA" ]] \
|
|
|| fail "frontend build SHA must match the full build prefix"
|
|
[[ "$AUTOMATIC_BACKUP_KEEP" =~ ^[1-9][0-9]*$ ]] || fail "backup retention must be positive"
|
|
[[ "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" =~ ^[1-9][0-9]*$ ]] || fail "stale threshold must be positive"
|
|
validate_absolute_path ALERT_WEBHOOK_FILE "$ALERT_WEBHOOK_FILE"
|
|
validate_absolute_path ALERT_HELPER "$ALERT_HELPER"
|
|
[[ -f "$ALERT_WEBHOOK_FILE" && ! -L "$ALERT_WEBHOOK_FILE" \
|
|
&& "$(stat -c %a "$ALERT_WEBHOOK_FILE")" == "600" ]] \
|
|
|| fail "alert webhook file must exist with mode 0600"
|
|
[[ -f "$ALERT_HELPER" && ! -L "$ALERT_HELPER" ]] || fail "alert helper is missing"
|
|
if [[ "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR" \
|
|
|| "$AUTOMATIC_BACKUP_DIR" == "$AUTOMATIC_BACKUP_STATE_DIR"/* \
|
|
|| "$AUTOMATIC_BACKUP_STATE_DIR" == "$AUTOMATIC_BACKUP_DIR"/* ]]; then
|
|
fail "backup and state directories must be independent"
|
|
fi
|
|
mkdir -p "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR"
|
|
[[ -d "$AUTOMATIC_BACKUP_DIR" && ! -L "$AUTOMATIC_BACKUP_DIR" \
|
|
&& -d "$AUTOMATIC_BACKUP_STATE_DIR" && ! -L "$AUTOMATIC_BACKUP_STATE_DIR" ]] \
|
|
|| fail "backup and state directories must not be symbolic links"
|
|
chmod 0700 "$AUTOMATIC_BACKUP_DIR" "$AUTOMATIC_BACKUP_STATE_DIR"
|
|
command -v python3 >/dev/null 2>&1 || fail "python3 is required"
|
|
}
|
|
|
|
validate_transfer_config() {
|
|
validate_absolute_path OFFHOST_SSH_KEY "$OFFHOST_SSH_KEY"
|
|
validate_absolute_path OFFHOST_KNOWN_HOSTS "$OFFHOST_KNOWN_HOSTS"
|
|
[[ -f "$OFFHOST_SSH_KEY" && ! -L "$OFFHOST_SSH_KEY" ]] || fail "off-host private key is missing"
|
|
[[ "$(stat -c %a "$OFFHOST_SSH_KEY")" == "600" ]] || fail "off-host private key mode must be 0600"
|
|
[[ -f "$OFFHOST_KNOWN_HOSTS" && -s "$OFFHOST_KNOWN_HOSTS" ]] || fail "pinned known-hosts file is missing"
|
|
[[ "$OFFHOST_TARGET" =~ ^[a-z_][a-z0-9_-]*@[A-Za-z0-9._-]+$ ]] \
|
|
|| fail "off-host target must use a safe user@alias value"
|
|
command -v ssh >/dev/null 2>&1 || fail "ssh is required"
|
|
command -v rsync >/dev/null 2>&1 || fail "rsync is required"
|
|
}
|
|
|
|
select_compose() {
|
|
if docker compose version >/dev/null 2>&1; then
|
|
COMPOSE_CMD=(docker compose)
|
|
elif command -v docker-compose >/dev/null 2>&1; then
|
|
COMPOSE_CMD=(docker-compose)
|
|
else
|
|
fail "Compose runtime is unavailable"
|
|
fi
|
|
}
|
|
|
|
compose() {
|
|
POSTGRES_DATA_DIR="$POSTGRES_DATA_DIR" \
|
|
STORAGE_HOST_DIR="$STORAGE_HOST_DIR" \
|
|
BUILD_SHA="$BUILD_SHA" \
|
|
VITE_BUILD_SHA="$VITE_BUILD_SHA" \
|
|
"${COMPOSE_CMD[@]}" -p "$COMPOSE_PROJECT_NAME" \
|
|
-f "$REMOTE_DIR/$COMPOSE_FILE" --env-file "$REMOTE_ENV_FILE" "$@"
|
|
}
|
|
|
|
container_env() {
|
|
local container_id="$1"
|
|
local key="$2"
|
|
docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$container_id" \
|
|
| sed -n "s/^${key}=//p" | head -n 1
|
|
}
|
|
|
|
mount_source() {
|
|
local container_id="$1"
|
|
local destination="$2"
|
|
docker inspect -f '{{json .Mounts}}' "$container_id" \
|
|
| python3 -c '
|
|
import json
|
|
import sys
|
|
|
|
destination = sys.argv[1]
|
|
mounts = json.load(sys.stdin)
|
|
matches = [mount.get("Source", "") for mount in mounts if mount.get("Destination") == destination]
|
|
if len(matches) != 1 or not matches[0]:
|
|
raise SystemExit(1)
|
|
print(matches[0])
|
|
' "$destination"
|
|
}
|
|
|
|
resolve_active_runtime() {
|
|
select_compose
|
|
local -a servers=() dbs=()
|
|
mapfile -t servers < <(compose ps -q server)
|
|
mapfile -t dbs < <(compose ps -q db)
|
|
[[ "${#servers[@]}" -eq 1 && -n "${servers[0]}" ]] || fail "expected exactly one active server container"
|
|
[[ "${#dbs[@]}" -eq 1 && -n "${dbs[0]}" ]] || fail "expected exactly one active database container"
|
|
SERVER_CONTAINER="${servers[0]}"
|
|
DB_CONTAINER="${dbs[0]}"
|
|
[[ "$(docker inspect -f '{{.State.Status}}' "$SERVER_CONTAINER")" == "running" ]] \
|
|
|| fail "server container is not running"
|
|
[[ "$(docker inspect -f '{{.State.Status}}' "$DB_CONTAINER")" == "running" ]] \
|
|
|| fail "database container is not running"
|
|
|
|
local actual_build storage_destination actual_storage actual_pgdata
|
|
actual_build="$(container_env "$SERVER_CONTAINER" BUILD_SHA)"
|
|
storage_destination="$(container_env "$SERVER_CONTAINER" STORAGE_DIR)"
|
|
[[ -n "$storage_destination" && "$storage_destination" == /* ]] || fail "server storage destination is invalid"
|
|
actual_storage="$(mount_source "$SERVER_CONTAINER" "$storage_destination")"
|
|
actual_pgdata="$(mount_source "$DB_CONTAINER" /var/lib/postgresql/data)"
|
|
[[ "$actual_build" == "$BUILD_SHA" ]] || fail "active server build does not match the installed backup contract"
|
|
[[ "$actual_storage" == "$STORAGE_HOST_DIR" ]] || fail "active storage mount does not match the installed backup contract"
|
|
[[ "$actual_pgdata" == "$POSTGRES_DATA_DIR" ]] || fail "active PGDATA mount does not match the installed backup contract"
|
|
}
|
|
|
|
write_summary() {
|
|
local status="$1"
|
|
local finding_count="$2"
|
|
local detail="$3"
|
|
local output_file="$AUTOMATIC_BACKUP_STATE_DIR/last-run.json"
|
|
local temp_file="$AUTOMATIC_BACKUP_STATE_DIR/.last-run.json.tmp.$$"
|
|
python3 - "$temp_file" "$status" "$finding_count" "$detail" "$VITE_BUILD_SHA" <<'PYTHON'
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
file_path, status, finding_count, detail, build_sha = sys.argv[1:]
|
|
finding_count = int(finding_count)
|
|
findings = []
|
|
if finding_count:
|
|
findings.append({
|
|
"source": "automatic_backup",
|
|
"status": status,
|
|
"severity": "P0",
|
|
"detail": detail,
|
|
})
|
|
payload = {
|
|
"issueId": "AUTO_BACKUP",
|
|
"status": status,
|
|
"conclusion": status,
|
|
"findingCount": finding_count,
|
|
"findings": findings,
|
|
"generatedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
"reportDir": "host-backup-state",
|
|
"buildSha": build_sha,
|
|
"privacy": "status/count/category only; no credentials, host addresses, absolute paths, file names, or payloads",
|
|
}
|
|
with open(file_path, "x", encoding="utf-8") as handle:
|
|
json.dump(payload, handle, ensure_ascii=True, indent=2)
|
|
handle.write("\n")
|
|
os.chmod(file_path, 0o600)
|
|
PYTHON
|
|
mv -f -- "$temp_file" "$output_file"
|
|
SUMMARY_FILE="$output_file"
|
|
}
|
|
|
|
send_summary_alert() {
|
|
local state_name="$1"
|
|
[[ -n "$SUMMARY_FILE" && -f "$SUMMARY_FILE" ]] || return 1
|
|
python3 "$ALERT_HELPER" \
|
|
--webhook-file "$ALERT_WEBHOOK_FILE" \
|
|
--summary-file "$SUMMARY_FILE" \
|
|
--state-file "$AUTOMATIC_BACKUP_STATE_DIR/$state_name" >/dev/null
|
|
}
|
|
|
|
capture_command() {
|
|
local phase="$1"
|
|
shift
|
|
local temporary="$AUTOMATIC_BACKUP_STATE_DIR/.${phase}.diagnostic.$$"
|
|
: >"$temporary"
|
|
chmod 0600 "$temporary"
|
|
if "$@" >"$temporary" 2>&1; then
|
|
rm -f -- "$temporary"
|
|
return 0
|
|
fi
|
|
mv -f -- "$temporary" "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
|
|
chmod 0600 "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
|
|
return 1
|
|
}
|
|
|
|
on_error() {
|
|
local exit_code="$?"
|
|
trap - ERR
|
|
set +e
|
|
write_summary "FAIL_AUTOMATIC_BACKUP" 1 "phase_${CURRENT_PHASE}_failed"
|
|
send_summary_alert "automatic-backup-alert-state.json" || true
|
|
echo "[automatic-backup][FAIL] phase=${CURRENT_PHASE}" >&2
|
|
exit "$exit_code"
|
|
}
|
|
|
|
prune_local_backups() {
|
|
local current_name="$1"
|
|
local -a entries=()
|
|
mapfile -t entries < <(find "$AUTOMATIC_BACKUP_DIR" -mindepth 1 -maxdepth 1 -type d \
|
|
-name 'hyb-oa-backup-auto-*' -printf '%f\n' | LC_ALL=C sort)
|
|
local remove_count=$((${#entries[@]} - AUTOMATIC_BACKUP_KEEP))
|
|
if (( remove_count <= 0 )); then
|
|
return 0
|
|
fi
|
|
local index entry target
|
|
for ((index = 0; index < remove_count; index += 1)); do
|
|
entry="${entries[$index]}"
|
|
[[ "$entry" != "$current_name" && "$entry" =~ ^hyb-oa-backup-auto-[0-9]{8}T[0-9]{6}Z$ ]] \
|
|
|| fail "local retention target failed safety validation"
|
|
target="$AUTOMATIC_BACKUP_DIR/$entry"
|
|
[[ -d "$target" && ! -L "$target" ]] || fail "local retention target is invalid"
|
|
rm -rf -- "$target"
|
|
done
|
|
}
|
|
|
|
write_success_state() {
|
|
local run_id="$1"
|
|
local manifest_sha="$2"
|
|
local temp_file="$AUTOMATIC_BACKUP_STATE_DIR/.last-success.json.tmp.$$"
|
|
python3 - "$temp_file" "$run_id" "$manifest_sha" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON'
|
|
import json
|
|
import os
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
file_path, run_id, manifest_sha, project, build_sha = sys.argv[1:]
|
|
payload = {
|
|
"schemaVersion": 1,
|
|
"status": "complete",
|
|
"completedAt": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
|
"runId": run_id,
|
|
"manifestSha256": manifest_sha,
|
|
"composeProject": project,
|
|
"buildSha": build_sha,
|
|
}
|
|
with open(file_path, "x", encoding="utf-8") as handle:
|
|
json.dump(payload, handle, ensure_ascii=True, indent=2)
|
|
handle.write("\n")
|
|
os.chmod(file_path, 0o600)
|
|
PYTHON
|
|
mv -f -- "$temp_file" "$AUTOMATIC_BACKUP_STATE_DIR/last-success.json"
|
|
}
|
|
|
|
run_backup() {
|
|
validate_common
|
|
validate_transfer_config
|
|
exec 9>"$AUTOMATIC_BACKUP_STATE_DIR/automatic-backup.lock"
|
|
flock -n 9 || fail "another automatic backup is already running"
|
|
trap on_error ERR
|
|
|
|
CURRENT_PHASE="runtime_identity"
|
|
resolve_active_runtime
|
|
|
|
local database_url postgres_db postgres_user postgres_password parsed_identity url_user url_db
|
|
database_url="$(container_env "$SERVER_CONTAINER" DATABASE_URL)"
|
|
postgres_db="$(container_env "$DB_CONTAINER" POSTGRES_DB)"
|
|
postgres_user="$(container_env "$DB_CONTAINER" POSTGRES_USER)"
|
|
postgres_password="$(container_env "$DB_CONTAINER" POSTGRES_PASSWORD)"
|
|
[[ -n "$database_url" && -n "$postgres_db" && -n "$postgres_user" && -n "$postgres_password" ]] \
|
|
|| fail "active database identity is incomplete"
|
|
parsed_identity="$(printf '%s' "$database_url" | python3 -c '
|
|
import sys
|
|
from urllib.parse import unquote, urlparse
|
|
value = urlparse(sys.stdin.read())
|
|
if value.scheme not in {"postgres", "postgresql"} or not value.username or not value.path.strip("/"):
|
|
raise SystemExit(1)
|
|
database_name = unquote(value.path.strip("/"))
|
|
print(f"{unquote(value.username)}\t{database_name}")
|
|
')"
|
|
IFS=$'\t' read -r url_user url_db <<<"$parsed_identity"
|
|
[[ "$url_user" == "$postgres_user" && "$url_db" == "$postgres_db" ]] \
|
|
|| fail "active server and database container identities do not match"
|
|
|
|
local run_id batch_name batch_dir source_image_digest manifest_sha
|
|
run_id="auto-$(date -u +%Y%m%dT%H%M%SZ)"
|
|
batch_name="hyb-oa-backup-$run_id"
|
|
batch_dir="$AUTOMATIC_BACKUP_DIR/$batch_name"
|
|
source_image_digest="$(docker inspect -f '{{.Image}}' "$SERVER_CONTAINER")"
|
|
[[ "$source_image_digest" =~ ^sha256:[0-9a-f]{64}$ ]] || fail "active server image digest is invalid"
|
|
|
|
CURRENT_PHASE="paired_backup"
|
|
if ! (
|
|
export DATABASE_URL="$database_url"
|
|
export DB_TOOL_MODE=docker DB_CONTAINER
|
|
export POSTGRES_DB="$postgres_db" POSTGRES_USER="$postgres_user"
|
|
export POSTGRES_PASSWORD="$postgres_password"
|
|
export STORAGE_DIR_HOST="$STORAGE_HOST_DIR" BACKUP_DIR="$AUTOMATIC_BACKUP_DIR"
|
|
export BACKUP_RUN_ID="$run_id" BUILD_SHA SOURCE_IMAGE_DIGEST="$source_image_digest"
|
|
capture_command paired_backup "$ROOT_DIR/scripts/backup-all.sh"
|
|
); then
|
|
fail "paired backup command failed"
|
|
fi
|
|
[[ -d "$batch_dir" && -f "$batch_dir/manifest.json" ]] || fail "complete local batch was not published"
|
|
manifest_sha="$(sha256sum "$batch_dir/manifest.json" | awk '{print $1}')"
|
|
|
|
local -a ssh_options=(
|
|
-i "$OFFHOST_SSH_KEY"
|
|
-o BatchMode=yes
|
|
-o IdentitiesOnly=yes
|
|
-o StrictHostKeyChecking=yes
|
|
-o "UserKnownHostsFile=$OFFHOST_KNOWN_HOSTS"
|
|
-o ConnectTimeout=15
|
|
)
|
|
local rsync_shell
|
|
printf -v rsync_shell 'ssh -i %q -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=%q -o ConnectTimeout=15' \
|
|
"$OFFHOST_SSH_KEY" "$OFFHOST_KNOWN_HOSTS"
|
|
|
|
CURRENT_PHASE="offhost_transfer"
|
|
if ! capture_command offhost_transfer rsync -a --partial --chmod=Du=rwx,Dgo=,Fu=rw,Fgo= -e "$rsync_shell" \
|
|
"$batch_dir/" "$OFFHOST_TARGET:${batch_name}.partial/"; then
|
|
fail "off-host transfer failed"
|
|
fi
|
|
|
|
CURRENT_PHASE="offhost_finalize"
|
|
if ! capture_command offhost_finalize ssh "${ssh_options[@]}" "$OFFHOST_TARGET" \
|
|
"hyb-backup-finalize $run_id $manifest_sha"; then
|
|
fail "off-host finalize failed"
|
|
fi
|
|
|
|
CURRENT_PHASE="success_state"
|
|
write_success_state "$run_id" "$manifest_sha"
|
|
prune_local_backups "$batch_name"
|
|
write_summary "PASS_AUTOMATIC_BACKUP_COMPLETE" 0 "offhost_verified"
|
|
send_summary_alert "automatic-backup-alert-state.json" || true
|
|
rm -f -- "$AUTOMATIC_BACKUP_STATE_DIR/last-error.log"
|
|
trap - ERR
|
|
echo "[automatic-backup][PASS] paired batch finalized off-host"
|
|
}
|
|
|
|
check_stale() {
|
|
validate_common
|
|
trap on_error ERR
|
|
CURRENT_PHASE="stale_check"
|
|
resolve_active_runtime
|
|
local success_file="$AUTOMATIC_BACKUP_STATE_DIR/last-success.json"
|
|
local stale_result
|
|
if ! stale_result="$(python3 - "$success_file" "$AUTOMATIC_BACKUP_MAX_AGE_SECONDS" "$COMPOSE_PROJECT_NAME" "$BUILD_SHA" <<'PYTHON'
|
|
import json
|
|
import os
|
|
import re
|
|
import sys
|
|
from datetime import datetime, timezone
|
|
|
|
file_path, max_age, expected_project, expected_build = sys.argv[1:]
|
|
if not os.path.isfile(file_path):
|
|
raise SystemExit(1)
|
|
with open(file_path, "r", encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
if payload.get("schemaVersion") != 1 or payload.get("status") != "complete":
|
|
raise SystemExit(1)
|
|
if payload.get("composeProject") != expected_project:
|
|
raise SystemExit(1)
|
|
if not re.fullmatch(r"auto-[0-9]{8}T[0-9]{6}Z", payload.get("runId", "")):
|
|
raise SystemExit(1)
|
|
if not re.fullmatch(r"[0-9a-f]{64}", payload.get("manifestSha256", "")):
|
|
raise SystemExit(1)
|
|
if not re.fullmatch(r"[0-9a-fA-F]{40,64}", payload.get("buildSha", "")):
|
|
raise SystemExit(1)
|
|
completed = datetime.fromisoformat(payload["completedAt"].replace("Z", "+00:00"))
|
|
age = (datetime.now(timezone.utc) - completed).total_seconds()
|
|
if age < 0 or age > int(max_age):
|
|
raise SystemExit(1)
|
|
print("BUILD_MISMATCH" if payload.get("buildSha") != expected_build else "FRESH")
|
|
PYTHON
|
|
)"; then
|
|
fail "latest off-host success is missing, invalid, project-mismatched, or stale"
|
|
fi
|
|
if [[ "$stale_result" == "BUILD_MISMATCH" ]]; then
|
|
write_summary "PASS_WITH_FINDINGS" 1 "recovery_point_build_differs"
|
|
send_summary_alert "automatic-backup-alert-state.json" || true
|
|
trap - ERR
|
|
echo "[automatic-backup][PASS_WITH_FINDINGS] off-host recovery point is fresh but build differs"
|
|
return 0
|
|
fi
|
|
write_summary "PASS_AUTOMATIC_BACKUP_FRESH" 0 "fresh_offhost_recovery_point"
|
|
send_summary_alert "automatic-backup-alert-state.json" || true
|
|
trap - ERR
|
|
echo "[automatic-backup][PASS] off-host recovery point is fresh"
|
|
}
|
|
|
|
remind_drill() {
|
|
validate_common
|
|
trap on_error ERR
|
|
CURRENT_PHASE="quarterly_reminder"
|
|
resolve_active_runtime
|
|
local quarter
|
|
quarter="$(python3 - <<'PYTHON'
|
|
from datetime import datetime, timezone
|
|
now = datetime.now(timezone.utc)
|
|
print(f"{now.year}-Q{((now.month - 1) // 3) + 1}")
|
|
PYTHON
|
|
)"
|
|
write_summary "PASS_WITH_FINDINGS" 1 "quarterly_shadow_restore_due_${quarter}"
|
|
send_summary_alert "automatic-backup-drill-alert-state.json"
|
|
trap - ERR
|
|
echo "[automatic-backup][PASS] quarterly restore reminder evaluated"
|
|
}
|
|
|
|
case "$MODE" in
|
|
run)
|
|
run_backup
|
|
;;
|
|
check-stale)
|
|
check_stale
|
|
;;
|
|
remind-drill)
|
|
remind_drill
|
|
;;
|
|
--help|-h)
|
|
echo "Usage: run-automatic-backup.sh [run|check-stale|remind-drill]"
|
|
;;
|
|
*)
|
|
fail "unsupported mode"
|
|
;;
|
|
esac
|