验证(08-23-isolated-backup-restore-drill): 固化隔离恢复runner合同
这个提交包含在：
@@ -0,0 +1,608 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
umask 077
|
||||
|
||||
CONFIG_FILE="@@RECEIVER_CONFIG_FILE@@"
|
||||
LIB_DIR="@@RESTORE_LIB_DIR@@"
|
||||
SELF_PATH="@@SELF_PATH@@"
|
||||
SUDOERS_PATH="@@SUDOERS_PATH@@"
|
||||
SUDOERS_MAIN_FILE="@@SUDOERS_MAIN_FILE@@"
|
||||
WORK_ROOT="@@WORK_ROOT@@"
|
||||
WORK_PREFIX="@@WORK_PREFIX@@"
|
||||
CONTAINER_PREFIX="@@CONTAINER_PREFIX@@"
|
||||
SHADOW_DB_USER="@@SHADOW_DB_USER@@"
|
||||
SHADOW_DB_NAME="@@SHADOW_DB_NAME@@"
|
||||
POSTGRES_IMAGE="@@POSTGRES_IMAGE@@"
|
||||
DEADMAN_TIMER_UNIT="@@DEADMAN_TIMER_UNIT@@"
|
||||
HEALTH_TIMER_UNIT="@@HEALTH_TIMER_UNIT@@"
|
||||
ARCHIVE_BATCH_PREFIX="@@ARCHIVE_BATCH_PREFIX@@"
|
||||
RUN_ID_PREFIX="@@RUN_ID_PREFIX@@"
|
||||
EXPECTED_DB_RESTORE_SHA="@@EXPECTED_DB_RESTORE_SHA@@"
|
||||
EXPECTED_STORAGE_RESTORE_SHA="@@EXPECTED_STORAGE_RESTORE_SHA@@"
|
||||
EXPECTED_ATTACHMENT_VALIDATOR_SHA="@@EXPECTED_ATTACHMENT_VALIDATOR_SHA@@"
|
||||
EXPECTED_BUILD_PREFIX="@@EXPECTED_BUILD_PREFIX@@"
|
||||
EXPECTED_MIGRATION_SET_SHA="@@EXPECTED_MIGRATION_SET_SHA@@"
|
||||
PHASE="bootstrap"
|
||||
WORK_DIR=""
|
||||
CONTAINER_NAME=""
|
||||
RUNTIME_CLEANED=0
|
||||
RESULT_STATUS="NO_GO"
|
||||
RPO_SECONDS=""
|
||||
RTO_SECONDS=""
|
||||
MIGRATION_SUCCESS=""
|
||||
MIGRATION_OTHER=""
|
||||
MIGRATION_SET_MATCH=""
|
||||
ARCHIVE_FILE_COUNT=""
|
||||
RESTORED_FILE_COUNT=""
|
||||
ATTACHMENT_TOTAL=""
|
||||
ATTACHMENT_MISMATCH=""
|
||||
TABLE_COUNTS=""
|
||||
SOURCE_RUN_ID=""
|
||||
DUMP_LIST_STATUS="NOT_RUN"
|
||||
TARGET_CONFIRMATION_STATUS="NOT_RUN"
|
||||
RESTORE_APPLY_STATUS="NOT_RUN"
|
||||
RESTORE_EXECUTION_PATH="NOT_RUN"
|
||||
DUMP_LIST_EXIT_CODE="NOT_RUN"
|
||||
RESTORE_SCRIPT_EXIT_CODE="NOT_RUN"
|
||||
PG_RESTORE_EXIT_CODE="NOT_RUN"
|
||||
PG_RESTORE_ERROR_COUNT="NOT_RUN"
|
||||
RESTORE_ERROR_CATEGORY="NOT_APPLICABLE"
|
||||
FAILURE_CATEGORY="none"
|
||||
DIAGNOSTICS_EMITTED=0
|
||||
|
||||
fail() {
|
||||
FAILURE_CATEGORY="$1"
|
||||
echo "[restore-drill][FAIL] phase=$PHASE category=$1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
require_command() {
|
||||
command -v "$1" >/dev/null 2>&1 || fail "required_command_missing"
|
||||
}
|
||||
|
||||
cleanup_runtime() {
|
||||
local cleanup_ok=1
|
||||
local container_suffix=""
|
||||
local work_suffix=""
|
||||
set +e
|
||||
if [[ -n "$CONTAINER_NAME" ]]; then
|
||||
container_suffix="${CONTAINER_NAME#"$CONTAINER_PREFIX"}"
|
||||
if [[ "$CONTAINER_NAME" == "$CONTAINER_PREFIX"* \
|
||||
&& "$container_suffix" =~ ^[0-9TZ-]+$ ]]; then
|
||||
docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||
docker inspect "$CONTAINER_NAME" >/dev/null 2>&1 && cleanup_ok=0
|
||||
else
|
||||
cleanup_ok=0
|
||||
fi
|
||||
fi
|
||||
if [[ -n "$WORK_DIR" ]]; then
|
||||
work_suffix="${WORK_DIR#"$WORK_ROOT/$WORK_PREFIX"}"
|
||||
if [[ "$WORK_DIR" == "$WORK_ROOT/$WORK_PREFIX"* \
|
||||
&& "$work_suffix" =~ ^[0-9TZ-]+$ \
|
||||
&& -d "$WORK_DIR" && ! -L "$WORK_DIR" ]]; then
|
||||
rm -rf -- "$WORK_DIR"
|
||||
[[ ! -e "$WORK_DIR" ]] || cleanup_ok=0
|
||||
elif [[ -e "$WORK_DIR" ]]; then
|
||||
cleanup_ok=0
|
||||
fi
|
||||
fi
|
||||
set -e
|
||||
[[ "$cleanup_ok" -eq 1 ]]
|
||||
}
|
||||
|
||||
cleanup_access() {
|
||||
local cleanup_ok=1
|
||||
set +e
|
||||
if [[ -f "$SUDOERS_PATH" && ! -L "$SUDOERS_PATH" \
|
||||
&& "$(stat -c %u "$SUDOERS_PATH" 2>/dev/null)" == "0" ]]; then
|
||||
rm -f -- "$SUDOERS_PATH" || cleanup_ok=0
|
||||
elif [[ -e "$SUDOERS_PATH" ]]; then
|
||||
cleanup_ok=0
|
||||
fi
|
||||
visudo -cf "$SUDOERS_MAIN_FILE" >/dev/null 2>&1 || cleanup_ok=0
|
||||
if [[ -d "$LIB_DIR" && ! -L "$LIB_DIR" \
|
||||
&& "$(stat -c %u "$LIB_DIR" 2>/dev/null)" == "0" ]]; then
|
||||
rm -rf -- "$LIB_DIR" || cleanup_ok=0
|
||||
elif [[ -e "$LIB_DIR" ]]; then
|
||||
cleanup_ok=0
|
||||
fi
|
||||
if [[ -f "$SELF_PATH" && ! -L "$SELF_PATH" \
|
||||
&& "$(stat -c %u "$SELF_PATH" 2>/dev/null)" == "0" ]]; then
|
||||
rm -f -- "$SELF_PATH" || cleanup_ok=0
|
||||
elif [[ -e "$SELF_PATH" ]]; then
|
||||
cleanup_ok=0
|
||||
fi
|
||||
set -e
|
||||
[[ "$cleanup_ok" -eq 1 ]]
|
||||
}
|
||||
|
||||
emit_restore_diagnostics() {
|
||||
if [[ "$DIAGNOSTICS_EMITTED" -eq 1 ]]; then
|
||||
return 0
|
||||
fi
|
||||
DIAGNOSTICS_EMITTED=1
|
||||
echo "failure_category=$FAILURE_CATEGORY"
|
||||
echo "dump_list=$DUMP_LIST_STATUS"
|
||||
echo "dump_list_exit_code=$DUMP_LIST_EXIT_CODE"
|
||||
echo "target_confirmation=$TARGET_CONFIRMATION_STATUS"
|
||||
echo "restore_execution_path=$RESTORE_EXECUTION_PATH"
|
||||
echo "restore_apply=$RESTORE_APPLY_STATUS"
|
||||
echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE"
|
||||
echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE"
|
||||
echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT"
|
||||
echo "restore_error_category=$RESTORE_ERROR_CATEGORY"
|
||||
}
|
||||
|
||||
finish() {
|
||||
local exit_code="$?"
|
||||
local runtime_cleanup="PASS"
|
||||
local access_cleanup="PASS"
|
||||
trap - EXIT HUP INT TERM
|
||||
# Emit only allowlisted stage diagnostics while the private runtime still
|
||||
# exists. Raw stderr remains mode 0600 and is removed by cleanup_runtime.
|
||||
emit_restore_diagnostics
|
||||
if [[ "$RUNTIME_CLEANED" -ne 1 ]]; then
|
||||
cleanup_runtime || runtime_cleanup="FAIL"
|
||||
fi
|
||||
cleanup_access || access_cleanup="FAIL"
|
||||
if [[ "$runtime_cleanup" != "PASS" || "$access_cleanup" != "PASS" ]]; then
|
||||
exit_code=1
|
||||
fi
|
||||
echo "runtime_cleanup=$runtime_cleanup"
|
||||
echo "temporary_access_cleanup=$access_cleanup"
|
||||
if [[ "$exit_code" -eq 0 ]]; then
|
||||
echo "result=$RESULT_STATUS"
|
||||
echo "rpo_seconds=$RPO_SECONDS"
|
||||
echo "data_restore_rto_seconds=$RTO_SECONDS"
|
||||
echo "migration_success=$MIGRATION_SUCCESS"
|
||||
echo "migration_other=$MIGRATION_OTHER"
|
||||
echo "migration_set_match=$MIGRATION_SET_MATCH"
|
||||
echo "archive_regular_files=$ARCHIVE_FILE_COUNT"
|
||||
echo "restored_regular_files=$RESTORED_FILE_COUNT"
|
||||
echo "attachment_reference_count=$ATTACHMENT_TOTAL"
|
||||
echo "attachment_mismatch_count=$ATTACHMENT_MISMATCH"
|
||||
printf '%s\n' "$TABLE_COUNTS"
|
||||
else
|
||||
echo "result=NO_GO"
|
||||
fi
|
||||
exit "$exit_code"
|
||||
}
|
||||
trap finish EXIT HUP INT TERM
|
||||
|
||||
for command_name in docker python3 sha256sum tar find stat df cp openssl visudo systemctl; do
|
||||
require_command "$command_name"
|
||||
done
|
||||
|
||||
PHASE="entry_contract"
|
||||
[[ "$(id -u)" -eq 0 ]] || fail "root_required"
|
||||
for absolute_path in "$CONFIG_FILE" "$LIB_DIR" "$SELF_PATH" "$SUDOERS_PATH" \
|
||||
"$SUDOERS_MAIN_FILE" "$WORK_ROOT"; do
|
||||
[[ "$absolute_path" == /* && "$absolute_path" != "/" ]] \
|
||||
|| fail "rendered_path_invalid"
|
||||
done
|
||||
[[ "$WORK_PREFIX" =~ ^[a-z0-9][a-z0-9-]*-$ \
|
||||
&& "$CONTAINER_PREFIX" =~ ^[a-z0-9][a-z0-9-]*-$ \
|
||||
&& "$SHADOW_DB_USER" =~ ^[a-z_][a-z0-9_]*$ \
|
||||
&& "$SHADOW_DB_NAME" =~ (^|_)(shadow|drill|restore_test)(_|$) \
|
||||
&& "$POSTGRES_IMAGE" =~ ^[a-z0-9._/-]+:[a-zA-Z0-9._-]+$ \
|
||||
&& "$DEADMAN_TIMER_UNIT" =~ ^[a-zA-Z0-9_.@-]+[.]timer$ \
|
||||
&& "$HEALTH_TIMER_UNIT" =~ ^[a-zA-Z0-9_.@-]+[.]timer$ \
|
||||
&& "$ARCHIVE_BATCH_PREFIX" =~ ^[a-z0-9][a-z0-9-]*-$ \
|
||||
&& "$RUN_ID_PREFIX" =~ ^[a-z0-9][a-z0-9-]*-$ ]] \
|
||||
|| fail "rendered_identity_invalid"
|
||||
[[ "$EXPECTED_DB_RESTORE_SHA" =~ ^[0-9a-f]{64}$ \
|
||||
&& "$EXPECTED_STORAGE_RESTORE_SHA" =~ ^[0-9a-f]{64}$ \
|
||||
&& "$EXPECTED_ATTACHMENT_VALIDATOR_SHA" =~ ^[0-9a-f]{64}$ \
|
||||
&& "$EXPECTED_BUILD_PREFIX" =~ ^[0-9a-f]{12}$ \
|
||||
&& "$EXPECTED_MIGRATION_SET_SHA" =~ ^[0-9a-f]{64}$ ]] \
|
||||
|| fail "rendered_digest_invalid"
|
||||
[[ -f "$CONFIG_FILE" && ! -L "$CONFIG_FILE" \
|
||||
&& "$(stat -c %u "$CONFIG_FILE")" == "0" ]] || fail "receiver_config_invalid"
|
||||
[[ -f "$LIB_DIR/restore-db.sh" && ! -L "$LIB_DIR/restore-db.sh" \
|
||||
&& -f "$LIB_DIR/restore-storage.sh" && ! -L "$LIB_DIR/restore-storage.sh" \
|
||||
&& -f "$LIB_DIR/validate-attachment-restore.py" \
|
||||
&& ! -L "$LIB_DIR/validate-attachment-restore.py" ]] \
|
||||
|| fail "restore_library_missing"
|
||||
[[ "$(sha256sum "$LIB_DIR/restore-db.sh" | awk '{print $1}')" == "$EXPECTED_DB_RESTORE_SHA" ]] \
|
||||
|| fail "db_restore_script_hash_mismatch"
|
||||
[[ "$(sha256sum "$LIB_DIR/restore-storage.sh" | awk '{print $1}')" == "$EXPECTED_STORAGE_RESTORE_SHA" ]] \
|
||||
|| fail "storage_restore_script_hash_mismatch"
|
||||
[[ "$(sha256sum "$LIB_DIR/validate-attachment-restore.py" | awk '{print $1}')" == "$EXPECTED_ATTACHMENT_VALIDATOR_SHA" ]] \
|
||||
|| fail "attachment_validator_hash_mismatch"
|
||||
|
||||
# Root-owned receiver configuration contains path-only assignments.
|
||||
# shellcheck disable=SC1090
|
||||
source "$CONFIG_FILE"
|
||||
BACKUP_ARCHIVE_DIR="${BACKUP_ARCHIVE_DIR:-}"
|
||||
[[ "$BACKUP_ARCHIVE_DIR" == /* && "$BACKUP_ARCHIVE_DIR" != "/" \
|
||||
&& -d "$BACKUP_ARCHIVE_DIR" && ! -L "$BACKUP_ARCHIVE_DIR" ]] \
|
||||
|| fail "archive_root_invalid"
|
||||
|
||||
PHASE="monitor_preflight"
|
||||
deadman_enabled_before="$(systemctl is-enabled "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)"
|
||||
deadman_active_before="$(systemctl is-active "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)"
|
||||
health_enabled_before="$(systemctl is-enabled "$HEALTH_TIMER_UNIT" 2>/dev/null || true)"
|
||||
health_active_before="$(systemctl is-active "$HEALTH_TIMER_UNIT" 2>/dev/null || true)"
|
||||
[[ "$deadman_enabled_before" == "enabled" && "$deadman_active_before" == "active" \
|
||||
&& "$health_enabled_before" == "enabled" && "$health_active_before" == "active" ]] \
|
||||
|| fail "monitor_preflight_not_ready"
|
||||
|
||||
validate_batch() {
|
||||
local batch_dir="$1"
|
||||
local identity_mode="$2"
|
||||
local expected_run_id="${3:-}"
|
||||
python3 - "$batch_dir" "$EXPECTED_BUILD_PREFIX" "$identity_mode" \
|
||||
"$expected_run_id" "$ARCHIVE_BATCH_PREFIX" "$RUN_ID_PREFIX" <<'PY'
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
|
||||
(
|
||||
batch_dir,
|
||||
build_prefix,
|
||||
identity_mode,
|
||||
expected_run_id,
|
||||
archive_batch_prefix,
|
||||
run_id_prefix,
|
||||
) = sys.argv[1:]
|
||||
if identity_mode not in {"source", "copy"}:
|
||||
raise SystemExit("identity_mode")
|
||||
if not os.path.isdir(batch_dir) or os.path.islink(batch_dir):
|
||||
raise SystemExit("batch_type")
|
||||
entries = set(os.listdir(batch_dir))
|
||||
expected = {"database.dump", "storage.tar.gz", "manifest.json"}
|
||||
if entries != expected:
|
||||
raise SystemExit("batch_files")
|
||||
for name in expected:
|
||||
path = os.path.join(batch_dir, name)
|
||||
mode = os.lstat(path).st_mode
|
||||
if not stat.S_ISREG(mode) or stat.S_ISLNK(mode):
|
||||
raise SystemExit("artifact_type")
|
||||
with open(os.path.join(batch_dir, "manifest.json"), encoding="utf-8") as handle:
|
||||
manifest = json.load(handle)
|
||||
run_id = manifest.get("runId")
|
||||
if manifest.get("schemaVersion") != 1 or manifest.get("status") != "complete":
|
||||
raise SystemExit("manifest_contract")
|
||||
run_id_pattern = re.escape(run_id_prefix) + r"[0-9]{8}T[0-9]{6}Z"
|
||||
if not isinstance(run_id, str) or not re.fullmatch(run_id_pattern, run_id):
|
||||
raise SystemExit("run_id")
|
||||
if identity_mode == "source":
|
||||
if expected_run_id:
|
||||
raise SystemExit("unexpected_expected_run_id")
|
||||
if os.path.basename(batch_dir) != archive_batch_prefix + run_id:
|
||||
raise SystemExit("source_batch_identity")
|
||||
else:
|
||||
if not re.fullmatch(run_id_pattern, expected_run_id):
|
||||
raise SystemExit("expected_run_id")
|
||||
if run_id != expected_run_id:
|
||||
raise SystemExit("copied_batch_source_identity")
|
||||
build = manifest.get("buildSha")
|
||||
if not isinstance(build, str) or not re.fullmatch(r"[0-9a-f]{40,64}", build):
|
||||
raise SystemExit("build_identity")
|
||||
if not build.startswith(build_prefix):
|
||||
raise SystemExit("build_mismatch")
|
||||
created = manifest.get("createdAt")
|
||||
try:
|
||||
created_at = dt.datetime.strptime(created, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=dt.timezone.utc)
|
||||
except Exception as exc:
|
||||
raise SystemExit("created_at") from exc
|
||||
artifacts = manifest.get("artifacts") or {}
|
||||
total_bytes = 0
|
||||
for key, name in (("database", "database.dump"), ("storage", "storage.tar.gz")):
|
||||
meta = artifacts.get(key) or {}
|
||||
if meta.get("file") != name or os.path.isabs(str(meta.get("file") or "")):
|
||||
raise SystemExit("artifact_name")
|
||||
path = os.path.join(batch_dir, name)
|
||||
size = os.path.getsize(path)
|
||||
if size != meta.get("bytes"):
|
||||
raise SystemExit("artifact_size")
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
if digest.hexdigest() != meta.get("sha256"):
|
||||
raise SystemExit("artifact_hash")
|
||||
total_bytes += size
|
||||
print(
|
||||
int(created_at.timestamp()),
|
||||
total_bytes,
|
||||
build[:8],
|
||||
int(bool(manifest.get("sourceImageDigest"))),
|
||||
run_id,
|
||||
)
|
||||
PY
|
||||
}
|
||||
|
||||
PHASE="select_latest_batch"
|
||||
selected_batch="$(python3 - "$BACKUP_ARCHIVE_DIR" \
|
||||
"$ARCHIVE_BATCH_PREFIX" "$RUN_ID_PREFIX" <<'PY'
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
root, archive_batch_prefix, run_id_prefix = sys.argv[1:]
|
||||
batch_pattern = re.escape(archive_batch_prefix + run_id_prefix) + r"[0-9]{8}T[0-9]{6}Z"
|
||||
candidates = []
|
||||
for entry in os.scandir(root):
|
||||
if entry.is_symlink() or not entry.is_dir(follow_symlinks=False):
|
||||
continue
|
||||
if not re.fullmatch(batch_pattern, entry.name):
|
||||
continue
|
||||
candidates.append((entry.name, entry.path))
|
||||
if not candidates:
|
||||
raise SystemExit(1)
|
||||
candidates.sort(key=lambda item: item[0])
|
||||
print(candidates[-1][1])
|
||||
PY
|
||||
)" || fail "latest_batch_not_found"
|
||||
|
||||
batch_metadata="$(validate_batch "$selected_batch" source "")" || fail "latest_batch_invalid"
|
||||
read -r created_epoch batch_total_bytes build_short source_digest_present SOURCE_RUN_ID <<<"$batch_metadata"
|
||||
drill_started_epoch="$(date -u +%s)"
|
||||
RPO_SECONDS="$((drill_started_epoch - created_epoch))"
|
||||
[[ "$RPO_SECONDS" -ge 0 && "$RPO_SECONDS" -le 86400 ]] || fail "latest_batch_outside_rpo"
|
||||
required_bytes="$((batch_total_bytes * 5 + 2147483648))"
|
||||
available_kib="$(df -Pk "$WORK_ROOT" | awk 'NR==2 {print $4}')"
|
||||
[[ "$available_kib" =~ ^[0-9]+$ ]] || fail "disk_capacity_unavailable"
|
||||
available_bytes="$((available_kib * 1024))"
|
||||
[[ "$available_bytes" -ge "$required_bytes" ]] || fail "disk_capacity_insufficient"
|
||||
|
||||
PHASE="create_isolated_runtime"
|
||||
run_id="$(date -u +%Y%m%dT%H%M%SZ)-$$-$RANDOM"
|
||||
[[ "$run_id" =~ ^[0-9TZ-]+$ ]] || fail "run_id_invalid"
|
||||
WORK_DIR="$WORK_ROOT/$WORK_PREFIX$run_id"
|
||||
CONTAINER_NAME="$CONTAINER_PREFIX$run_id"
|
||||
[[ ! -e "$WORK_DIR" ]] || fail "work_dir_exists"
|
||||
mkdir -m 0700 "$WORK_DIR"
|
||||
private_log="$WORK_DIR/private.log"
|
||||
: >"$private_log"
|
||||
chmod 0600 "$private_log"
|
||||
rto_started_epoch="$(date -u +%s)"
|
||||
cp -a -- "$selected_batch" "$WORK_DIR/batch"
|
||||
copied_metadata="$(validate_batch "$WORK_DIR/batch" copy "$SOURCE_RUN_ID")" \
|
||||
|| fail "copied_batch_invalid"
|
||||
[[ "$copied_metadata" == "$batch_metadata" ]] || fail "copied_batch_identity_mismatch"
|
||||
|
||||
env_file="$WORK_DIR/postgres.env"
|
||||
postgres_password="$(openssl rand -hex 24)"
|
||||
{
|
||||
echo "POSTGRES_USER=$SHADOW_DB_USER"
|
||||
echo "POSTGRES_PASSWORD=$postgres_password"
|
||||
echo "POSTGRES_DB=$SHADOW_DB_NAME"
|
||||
} >"$env_file"
|
||||
chmod 0600 "$env_file"
|
||||
printf '%s\n' '[stage=postgres_container_start]' >>"$private_log"
|
||||
docker run -d --rm --name "$CONTAINER_NAME" --network none --env-file "$env_file" \
|
||||
"$POSTGRES_IMAGE" >>"$private_log" 2>&1 || fail "postgres_container_start_failed"
|
||||
for _ in $(seq 1 60); do
|
||||
if docker exec "$CONTAINER_NAME" pg_isready -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
|
||||
>/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
docker exec "$CONTAINER_NAME" pg_isready -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
|
||||
>/dev/null 2>&1 || fail "postgres_not_ready"
|
||||
|
||||
PHASE="restore_database"
|
||||
DUMP_LIST_STATUS="FAIL"
|
||||
RESTORE_EXECUTION_PATH="DOCKER_STDIN"
|
||||
printf '%s\n' '[stage=dump_list]' >>"$private_log"
|
||||
set +e
|
||||
docker exec -i "$CONTAINER_NAME" pg_restore --list \
|
||||
<"$WORK_DIR/batch/database.dump" >>"$private_log" 2>&1
|
||||
DUMP_LIST_EXIT_CODE="$?"
|
||||
set -e
|
||||
if [[ "$DUMP_LIST_EXIT_CODE" -ne 0 ]]; then
|
||||
RESTORE_ERROR_CATEGORY="DUMP_LIST_NONZERO"
|
||||
fail "dump_list_failed"
|
||||
fi
|
||||
DUMP_LIST_STATUS="PASS"
|
||||
|
||||
TARGET_CONFIRMATION_STATUS="FAIL"
|
||||
printf '%s\n' '[stage=target_confirmation]' >>"$private_log"
|
||||
set +e
|
||||
actual_database="$(PGPASSWORD="$postgres_password" docker exec -e PGPASSWORD \
|
||||
"$CONTAINER_NAME" psql -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
|
||||
-Atqc 'SELECT current_database()' 2>>"$private_log")"
|
||||
target_confirmation_exit_code="$?"
|
||||
set -e
|
||||
if [[ "$target_confirmation_exit_code" -ne 0 \
|
||||
|| "$actual_database" != "$SHADOW_DB_NAME" ]]; then
|
||||
RESTORE_ERROR_CATEGORY="TARGET_CONFIRMATION_FAILED"
|
||||
fail "restore_target_confirmation_failed"
|
||||
fi
|
||||
TARGET_CONFIRMATION_STATUS="PASS"
|
||||
|
||||
real_docker_path="$(command -v docker)"
|
||||
docker_proxy_dir="$WORK_DIR/docker-proxy"
|
||||
docker_proxy="$docker_proxy_dir/docker"
|
||||
restore_diag_file="$WORK_DIR/restore-diag"
|
||||
mkdir -m 0700 "$docker_proxy_dir"
|
||||
cat >"$docker_proxy" <<'PROXY'
|
||||
#!/usr/bin/env bash
|
||||
set +e
|
||||
"$RESTORE_REAL_DOCKER" "$@"
|
||||
command_exit="$?"
|
||||
is_pg_restore=0
|
||||
is_list=0
|
||||
for argument in "$@"; do
|
||||
[[ "$argument" == "pg_restore" ]] && is_pg_restore=1
|
||||
[[ "$argument" == "--list" ]] && is_list=1
|
||||
done
|
||||
if [[ "$is_pg_restore" -eq 1 ]]; then
|
||||
if [[ "$is_list" -eq 1 ]]; then
|
||||
printf 'script_dump_list_exit_code=%s\n' "$command_exit" >>"$RESTORE_DIAG_FILE"
|
||||
else
|
||||
printf 'restore_apply_exit_code=%s\n' "$command_exit" >>"$RESTORE_DIAG_FILE"
|
||||
fi
|
||||
fi
|
||||
exit "$command_exit"
|
||||
PROXY
|
||||
chmod 0700 "$docker_proxy"
|
||||
: >"$restore_diag_file"
|
||||
chmod 0600 "$restore_diag_file"
|
||||
|
||||
RESTORE_APPLY_STATUS="FAIL"
|
||||
printf '%s\n' '[stage=restore_apply]' >>"$private_log"
|
||||
set +e
|
||||
RESTORE_REAL_DOCKER="$real_docker_path" \
|
||||
RESTORE_DIAG_FILE="$restore_diag_file" \
|
||||
PATH="$docker_proxy_dir:$PATH" \
|
||||
RESTORE_SCOPE=shadow \
|
||||
RESTORE_CONFIRM_DATABASE="$SHADOW_DB_NAME" \
|
||||
DB_TOOL_MODE=docker \
|
||||
DB_CONTAINER="$CONTAINER_NAME" \
|
||||
DATABASE_URL="postgresql://shadow.invalid/$SHADOW_DB_NAME" \
|
||||
POSTGRES_DB="$SHADOW_DB_NAME" \
|
||||
POSTGRES_USER="$SHADOW_DB_USER" \
|
||||
POSTGRES_PASSWORD="$postgres_password" \
|
||||
"$LIB_DIR/restore-db.sh" "$WORK_DIR/batch/database.dump" \
|
||||
>>"$private_log" 2>&1
|
||||
RESTORE_SCRIPT_EXIT_CODE="$?"
|
||||
set -e
|
||||
|
||||
script_dump_list_exit_code="$(awk -F= \
|
||||
'$1 == "script_dump_list_exit_code" {print $2}' "$restore_diag_file")"
|
||||
apply_exit_code="$(awk -F= \
|
||||
'$1 == "restore_apply_exit_code" {print $2}' "$restore_diag_file")"
|
||||
if [[ ! "$script_dump_list_exit_code" =~ ^[0-9]+$ \
|
||||
|| "$script_dump_list_exit_code" -ne 0 ]]; then
|
||||
RESTORE_ERROR_CATEGORY="SCRIPT_DUMP_LIST_NONZERO"
|
||||
fail "restore_script_list_failed"
|
||||
fi
|
||||
if [[ ! "$apply_exit_code" =~ ^[0-9]+$ ]]; then
|
||||
RESTORE_ERROR_CATEGORY="APPLY_EXIT_CODE_MISSING"
|
||||
fail "restore_diagnostic_missing"
|
||||
fi
|
||||
PG_RESTORE_EXIT_CODE="$apply_exit_code"
|
||||
if [[ "$PG_RESTORE_EXIT_CODE" -eq 0 ]]; then
|
||||
PG_RESTORE_ERROR_COUNT="0"
|
||||
else
|
||||
PG_RESTORE_ERROR_COUNT="$(sed -nE \
|
||||
's/^pg_restore: (warning: )?errors ignored on restore: ([0-9]+)$/\2/p' \
|
||||
"$private_log" | tail -n 1)"
|
||||
[[ "$PG_RESTORE_ERROR_COUNT" =~ ^[0-9]+$ ]] \
|
||||
|| PG_RESTORE_ERROR_COUNT="UNKNOWN"
|
||||
fi
|
||||
if [[ "$RESTORE_SCRIPT_EXIT_CODE" -ne 0 || "$PG_RESTORE_EXIT_CODE" -ne 0 ]]; then
|
||||
RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"
|
||||
fail "restore_apply_failed_after_list_pass"
|
||||
fi
|
||||
RESTORE_APPLY_STATUS="PASS"
|
||||
RESTORE_ERROR_CATEGORY="NONE"
|
||||
|
||||
PHASE="verify_database"
|
||||
migration_names="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \
|
||||
"$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
|
||||
-At <<'SQL' | LC_ALL=C sort
|
||||
SELECT migration_name
|
||||
FROM "_prisma_migrations"
|
||||
WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL
|
||||
ORDER BY migration_name;
|
||||
SQL
|
||||
)" || fail "migration_name_query_failed"
|
||||
[[ -n "$migration_names" ]] || fail "migration_set_empty"
|
||||
if printf '%s\n' "$migration_names" | grep -Ev '^[0-9]{14}_[a-z0-9_]+$' >/dev/null; then
|
||||
fail "migration_name_invalid"
|
||||
fi
|
||||
MIGRATION_SUCCESS="$(printf '%s\n' "$migration_names" | wc -l | tr -d ' ')"
|
||||
MIGRATION_OTHER="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \
|
||||
"$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
|
||||
-At <<'SQL'
|
||||
SELECT count(*)
|
||||
FROM "_prisma_migrations"
|
||||
WHERE finished_at IS NULL OR rolled_back_at IS NOT NULL;
|
||||
SQL
|
||||
)" || fail "migration_state_query_failed"
|
||||
[[ "$MIGRATION_OTHER" == "0" ]] || fail "migration_state_mismatch"
|
||||
migration_set_sha="$(printf '%s\n' "$migration_names" | sha256sum | awk '{print $1}')"
|
||||
[[ "$migration_set_sha" == "$EXPECTED_MIGRATION_SET_SHA" ]] \
|
||||
|| fail "migration_set_mismatch"
|
||||
MIGRATION_SET_MATCH="PASS"
|
||||
|
||||
TABLE_COUNTS="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \
|
||||
"$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
|
||||
-At -F = <<'SQL' | LC_ALL=C sort
|
||||
SELECT key, value FROM (VALUES
|
||||
('departments', (SELECT count(*)::bigint FROM departments)),
|
||||
('employees', (SELECT count(*)::bigint FROM employees)),
|
||||
('employee_roles', (SELECT count(*)::bigint FROM employee_roles)),
|
||||
('purchase_requests', (SELECT count(*)::bigint FROM purchase_requests)),
|
||||
('reimbursement_forms', (SELECT count(*)::bigint FROM reimbursement_forms)),
|
||||
('invoice_pool', (SELECT count(*)::bigint FROM invoice_pool)),
|
||||
('payment_batches', (SELECT count(*)::bigint FROM payment_batches)),
|
||||
('contract_registry', (SELECT count(*)::bigint FROM contract_registry)),
|
||||
('projects', (SELECT count(*)::bigint FROM projects)),
|
||||
('purchase_attachments', (SELECT count(*)::bigint FROM purchase_attachments)),
|
||||
('reimbursement_attachments', (SELECT count(*)::bigint FROM reimbursement_attachments)),
|
||||
('payment_batch_receipt_attachments', (SELECT count(*)::bigint FROM payment_batch_receipt_attachments))
|
||||
) AS counts(key, value);
|
||||
SQL
|
||||
)" || fail "table_count_query_failed"
|
||||
[[ "$(printf '%s\n' "$TABLE_COUNTS" | sed '/^$/d' | wc -l | tr -d ' ')" == "12" ]] \
|
||||
|| fail "table_count_key_mismatch"
|
||||
|
||||
PHASE="restore_storage"
|
||||
restored_storage="$WORK_DIR/restored-storage"
|
||||
printf '%s\n' '[stage=restore_storage]' >>"$private_log"
|
||||
if ! RESTORE_SCOPE=shadow "$LIB_DIR/restore-storage.sh" \
|
||||
"$WORK_DIR/batch/storage.tar.gz" "$restored_storage" \
|
||||
>>"$private_log" 2>&1; then
|
||||
fail "storage_restore_failed"
|
||||
fi
|
||||
ARCHIVE_FILE_COUNT="$(tar -tvzf "$WORK_DIR/batch/storage.tar.gz" \
|
||||
| awk 'substr($0,1,1)=="-" {count++} END {print count+0}')"
|
||||
RESTORED_FILE_COUNT="$(find "$restored_storage" -type f -printf '.' | wc -c | tr -d ' ')"
|
||||
[[ "$ARCHIVE_FILE_COUNT" == "$RESTORED_FILE_COUNT" ]] || fail "storage_file_count_mismatch"
|
||||
|
||||
PHASE="verify_attachment_references"
|
||||
attachment_result="$(PGPASSWORD="$postgres_password" docker exec -i -e PGPASSWORD \
|
||||
"$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 -U "$SHADOW_DB_USER" -d "$SHADOW_DB_NAME" \
|
||||
-At -F $'\t' <<'SQL' | python3 "$LIB_DIR/validate-attachment-restore.py" "$restored_storage"
|
||||
SELECT encode(convert_to(storage_path, 'UTF8'), 'base64'), file_size::text, coalesce(file_sha256, '') FROM invoice_pool
|
||||
UNION ALL
|
||||
SELECT encode(convert_to(storage_path, 'UTF8'), 'base64'), file_size::text, coalesce(file_sha256, '') FROM purchase_attachments
|
||||
UNION ALL
|
||||
SELECT encode(convert_to(storage_path, 'UTF8'), 'base64'), file_size::text, coalesce(file_sha256, '') FROM reimbursement_attachments
|
||||
UNION ALL
|
||||
SELECT encode(convert_to(storage_path, 'UTF8'), 'base64'), file_size::text, coalesce(file_sha256, '') FROM payment_batch_receipt_attachments
|
||||
UNION ALL
|
||||
SELECT encode(convert_to(attachment_storage_path, 'UTF8'), 'base64'), '', '' FROM leave_requests WHERE attachment_storage_path IS NOT NULL;
|
||||
SQL
|
||||
)" || fail "attachment_reference_query_failed"
|
||||
read -r ATTACHMENT_TOTAL ATTACHMENT_MISMATCH attachment_s3_count <<<"$attachment_result"
|
||||
[[ "$ATTACHMENT_TOTAL" =~ ^[0-9]+$ && "$ATTACHMENT_MISMATCH" == "0" \
|
||||
&& "$attachment_s3_count" == "0" ]] || fail "attachment_reference_mismatch"
|
||||
|
||||
PHASE="post_validation"
|
||||
copied_metadata_after="$(validate_batch "$WORK_DIR/batch" copy "$SOURCE_RUN_ID")" \
|
||||
|| fail "copied_batch_postcheck_failed"
|
||||
source_metadata_after="$(validate_batch "$selected_batch" source "")" \
|
||||
|| fail "source_archive_postcheck_failed"
|
||||
[[ "$copied_metadata_after" == "$batch_metadata" \
|
||||
&& "$source_metadata_after" == "$batch_metadata" ]] || fail "archive_identity_changed"
|
||||
rto_finished_epoch="$(date -u +%s)"
|
||||
RTO_SECONDS="$((rto_finished_epoch - rto_started_epoch))"
|
||||
|
||||
deadman_enabled_after="$(systemctl is-enabled "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)"
|
||||
deadman_active_after="$(systemctl is-active "$DEADMAN_TIMER_UNIT" 2>/dev/null || true)"
|
||||
health_enabled_after="$(systemctl is-enabled "$HEALTH_TIMER_UNIT" 2>/dev/null || true)"
|
||||
health_active_after="$(systemctl is-active "$HEALTH_TIMER_UNIT" 2>/dev/null || true)"
|
||||
[[ "$deadman_enabled_after" == "$deadman_enabled_before" \
|
||||
&& "$deadman_active_after" == "$deadman_active_before" \
|
||||
&& "$health_enabled_after" == "$health_enabled_before" \
|
||||
&& "$health_active_after" == "$health_active_before" ]] \
|
||||
|| fail "monitor_state_changed"
|
||||
|
||||
PHASE="runtime_cleanup"
|
||||
cleanup_runtime || fail "runtime_cleanup_failed"
|
||||
RUNTIME_CLEANED=1
|
||||
RESULT_STATUS="GO"
|
||||
PHASE="complete"
|
||||
exit 0
|
||||
@@ -0,0 +1,725 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
RUNNER_TEMPLATE="$ROOT_DIR/scripts/qa/offhost-shadow-restore-runner.template.sh"
|
||||
ATTACHMENT_VALIDATOR="$ROOT_DIR/scripts/qa/validate-offhost-shadow-restore-attachments.py"
|
||||
RESTORE_DB="$ROOT_DIR/scripts/restore-db.sh"
|
||||
RESTORE_STORAGE="$ROOT_DIR/scripts/restore-storage.sh"
|
||||
TEST_DIR="$(mktemp -d)"
|
||||
RUNNER="$TEST_DIR/rendered-runner.sh"
|
||||
PRIVATE_REPO_TEST_DIR="$ROOT_DIR/.restore-drill-private/contract-$$-$RANDOM"
|
||||
UNIGNORED_CONFIG="$ROOT_DIR/.restore-drill-private-contract-$$-$RANDOM.json"
|
||||
UNIGNORED_OUTPUT="$ROOT_DIR/.restore-drill-private-contract-$$-$RANDOM.sh"
|
||||
CONTAINER_NAME="$(basename "$0" .sh)-$RANDOM-$$"
|
||||
POSTGRES_USER="fixture_user_$RANDOM"
|
||||
POSTGRES_PASSWORD="$(openssl rand -hex 16)"
|
||||
POSTGRES_DB="fixture_restore_drill_$RANDOM"
|
||||
|
||||
cleanup() {
|
||||
[[ -z "$CONTAINER_NAME" ]] \
|
||||
|| docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
||||
rm -rf -- "$TEST_DIR"
|
||||
rm -rf -- "$PRIVATE_REPO_TEST_DIR"
|
||||
rm -f -- "$UNIGNORED_CONFIG" "$UNIGNORED_OUTPUT"
|
||||
}
|
||||
trap cleanup EXIT HUP INT TERM
|
||||
|
||||
fail() {
|
||||
echo "[runner-contract][FAIL] $1" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
pass() {
|
||||
echo "[runner-contract][PASS] $1"
|
||||
}
|
||||
|
||||
require_ignored_if_inside_repo() {
|
||||
local candidate_path="$1"
|
||||
local resolved_parent resolved_path relative_path
|
||||
resolved_parent="$(realpath -e "$(dirname "$candidate_path")")" \
|
||||
|| return 1
|
||||
resolved_path="$resolved_parent/$(basename "$candidate_path")"
|
||||
case "$resolved_path" in
|
||||
"$ROOT_DIR"/*)
|
||||
relative_path="${resolved_path#"$ROOT_DIR"/}"
|
||||
git -C "$ROOT_DIR" check-ignore -q --no-index -- "$relative_path" \
|
||||
|| return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
render_runner() {
|
||||
local config_file="$1"
|
||||
local output_file="$2"
|
||||
require_ignored_if_inside_repo "$config_file" || return 1
|
||||
require_ignored_if_inside_repo "$output_file" || return 1
|
||||
python3 - "$RUNNER_TEMPLATE" "$config_file" "$output_file" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
template_path, config_path, output_path = map(pathlib.Path, sys.argv[1:])
|
||||
template = template_path.read_text(encoding="utf-8")
|
||||
config = json.loads(config_path.read_text(encoding="utf-8"))
|
||||
tokens = set(re.findall(r"@@([A-Z0-9_]+)@@", template))
|
||||
if set(config) != tokens:
|
||||
raise SystemExit("render_config_keys")
|
||||
path_keys = {
|
||||
"RECEIVER_CONFIG_FILE",
|
||||
"RESTORE_LIB_DIR",
|
||||
"SELF_PATH",
|
||||
"SUDOERS_PATH",
|
||||
"SUDOERS_MAIN_FILE",
|
||||
"WORK_ROOT",
|
||||
}
|
||||
prefix_keys = {"WORK_PREFIX", "CONTAINER_PREFIX", "ARCHIVE_BATCH_PREFIX", "RUN_ID_PREFIX"}
|
||||
sha_keys = {
|
||||
"EXPECTED_DB_RESTORE_SHA",
|
||||
"EXPECTED_STORAGE_RESTORE_SHA",
|
||||
"EXPECTED_ATTACHMENT_VALIDATOR_SHA",
|
||||
"EXPECTED_MIGRATION_SET_SHA",
|
||||
}
|
||||
for key, value in config.items():
|
||||
if not isinstance(value, str) or not value or "\n" in value or "\r" in value:
|
||||
raise SystemExit("render_config_value")
|
||||
if key in path_keys:
|
||||
parsed = pathlib.PurePosixPath(value)
|
||||
if not parsed.is_absolute() or any(part in {"", ".", ".."} for part in parsed.parts[1:]):
|
||||
raise SystemExit("render_path_value")
|
||||
if not re.fullmatch(r"/[A-Za-z0-9_./-]+", value):
|
||||
raise SystemExit("render_path_charset")
|
||||
elif key in prefix_keys and not re.fullmatch(r"[a-z0-9][a-z0-9-]*-", value):
|
||||
raise SystemExit("render_prefix_value")
|
||||
elif key == "SHADOW_DB_USER" and not re.fullmatch(r"[a-z_][a-z0-9_]*", value):
|
||||
raise SystemExit("render_db_user")
|
||||
elif key == "SHADOW_DB_NAME" and (
|
||||
not re.fullmatch(r"[a-z_][a-z0-9_]*", value)
|
||||
or not re.search(r"(^|_)(shadow|drill|restore_test)(_|$)", value)
|
||||
):
|
||||
raise SystemExit("render_db_name")
|
||||
elif key == "POSTGRES_IMAGE" and not re.fullmatch(r"[a-z0-9._/-]+:[A-Za-z0-9._-]+", value):
|
||||
raise SystemExit("render_postgres_image")
|
||||
elif key in {"DEADMAN_TIMER_UNIT", "HEALTH_TIMER_UNIT"} and not re.fullmatch(
|
||||
r"[A-Za-z0-9_.@-]+\.timer", value
|
||||
):
|
||||
raise SystemExit("render_timer_unit")
|
||||
elif key in sha_keys and not re.fullmatch(r"[0-9a-f]{64}", value):
|
||||
raise SystemExit("render_sha")
|
||||
elif key == "EXPECTED_BUILD_PREFIX" and not re.fullmatch(r"[0-9a-f]{12}", value):
|
||||
raise SystemExit("render_build_prefix")
|
||||
template = template.replace(f"@@{key}@@", value)
|
||||
if re.search(r"@@[A-Z0-9_]+@@", template):
|
||||
raise SystemExit("render_placeholder_remaining")
|
||||
descriptor = os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o700)
|
||||
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
||||
handle.write(template)
|
||||
PY
|
||||
}
|
||||
|
||||
if [[ "${1:-}" == "render" ]]; then
|
||||
[[ "$#" -eq 3 ]] || fail "usage: $0 render <private-config.json> <output>"
|
||||
render_runner "$2" "$3" || fail "private render paths must be outside the repository or ignored"
|
||||
bash -n "$3"
|
||||
candidate_sha="$(sha256sum "$3" | awk '{print $1}')"
|
||||
echo "[runner-contract][PASS] rendered candidate syntax"
|
||||
echo "candidate_sha256=$candidate_sha"
|
||||
exit 0
|
||||
fi
|
||||
[[ "$#" -eq 0 ]] || fail "usage: $0 [render <private-config.json> <output>]"
|
||||
|
||||
db_restore_sha="$(sha256sum "$RESTORE_DB" | awk '{print $1}')"
|
||||
storage_restore_sha="$(sha256sum "$RESTORE_STORAGE" | awk '{print $1}')"
|
||||
attachment_validator_sha="$(sha256sum "$ATTACHMENT_VALIDATOR" | awk '{print $1}')"
|
||||
migration_fixture_names=$'20260101000000_fixture_one\n20260102000000_fixture_two'
|
||||
migration_fixture_sha="$(printf '%s\n' "$migration_fixture_names" | sha256sum | awk '{print $1}')"
|
||||
python3 - "$TEST_DIR/render-config.json" "$db_restore_sha" \
|
||||
"$storage_restore_sha" "$attachment_validator_sha" "$migration_fixture_sha" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
output, db_sha, storage_sha, validator_sha, migration_sha = sys.argv[1:]
|
||||
payload = {
|
||||
"RECEIVER_CONFIG_FILE": "/synthetic/receiver.conf",
|
||||
"RESTORE_LIB_DIR": "/synthetic/restore-lib",
|
||||
"SELF_PATH": "/synthetic/restore-entry",
|
||||
"SUDOERS_PATH": "/synthetic/restore-rule",
|
||||
"SUDOERS_MAIN_FILE": "/synthetic/sudoers",
|
||||
"WORK_ROOT": "/tmp",
|
||||
"WORK_PREFIX": "restore-contract-work-",
|
||||
"CONTAINER_PREFIX": "restore-contract-container-",
|
||||
"SHADOW_DB_USER": "fixture_user",
|
||||
"SHADOW_DB_NAME": "fixture_restore_drill",
|
||||
"POSTGRES_IMAGE": "postgres:16-alpine",
|
||||
"DEADMAN_TIMER_UNIT": "fixture-deadman.timer",
|
||||
"HEALTH_TIMER_UNIT": "fixture-health.timer",
|
||||
"ARCHIVE_BATCH_PREFIX": "fixture-backup-",
|
||||
"RUN_ID_PREFIX": "auto-",
|
||||
"EXPECTED_DB_RESTORE_SHA": db_sha,
|
||||
"EXPECTED_STORAGE_RESTORE_SHA": storage_sha,
|
||||
"EXPECTED_ATTACHMENT_VALIDATOR_SHA": validator_sha,
|
||||
"EXPECTED_BUILD_PREFIX": "a" * 12,
|
||||
"EXPECTED_MIGRATION_SET_SHA": migration_sha,
|
||||
}
|
||||
pathlib.Path(output).write_text(json.dumps(payload), encoding="utf-8")
|
||||
PY
|
||||
|
||||
cp "$TEST_DIR/render-config.json" "$UNIGNORED_CONFIG"
|
||||
if render_runner "$UNIGNORED_CONFIG" "$TEST_DIR/unignored-config-output.sh" \
|
||||
>/dev/null 2>&1; then
|
||||
fail "renderer accepted an unignored in-repository private config"
|
||||
fi
|
||||
[[ ! -e "$TEST_DIR/unignored-config-output.sh" ]] \
|
||||
|| fail "renderer published output for an unignored private config"
|
||||
pass "unignored in-repository private config fails before candidate creation"
|
||||
|
||||
if render_runner "$TEST_DIR/render-config.json" "$UNIGNORED_OUTPUT" \
|
||||
>/dev/null 2>&1; then
|
||||
fail "renderer accepted an unignored in-repository candidate path"
|
||||
fi
|
||||
[[ ! -e "$UNIGNORED_OUTPUT" ]] \
|
||||
|| fail "renderer published an unignored in-repository candidate"
|
||||
pass "unignored in-repository candidate path fails before creation"
|
||||
|
||||
mkdir -m 0700 -p "$PRIVATE_REPO_TEST_DIR"
|
||||
cp "$TEST_DIR/render-config.json" "$PRIVATE_REPO_TEST_DIR/render-config.json"
|
||||
render_runner "$PRIVATE_REPO_TEST_DIR/render-config.json" \
|
||||
"$PRIVATE_REPO_TEST_DIR/rendered-runner.sh"
|
||||
[[ -s "$PRIVATE_REPO_TEST_DIR/rendered-runner.sh" ]] \
|
||||
|| fail "ignored private render path did not produce a candidate"
|
||||
pass "ignored in-repository private config and candidate paths are accepted"
|
||||
|
||||
render_runner "$TEST_DIR/render-config.json" "$RUNNER"
|
||||
render_runner "$TEST_DIR/render-config.json" "$TEST_DIR/rendered-runner-repeat.sh"
|
||||
cmp -s "$RUNNER" "$TEST_DIR/rendered-runner-repeat.sh" \
|
||||
|| fail "identical private config did not render a deterministic candidate"
|
||||
pass "identical private config renders a deterministic candidate SHA-256"
|
||||
python3 - "$TEST_DIR/render-config.json" "$TEST_DIR/unsafe-render-config.json" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
source, target = map(pathlib.Path, sys.argv[1:])
|
||||
payload = json.loads(source.read_text(encoding="utf-8"))
|
||||
payload["SELF_PATH"] = '/synthetic/entry";unsafe-command'
|
||||
target.write_text(json.dumps(payload), encoding="utf-8")
|
||||
PY
|
||||
if render_runner "$TEST_DIR/unsafe-render-config.json" \
|
||||
"$TEST_DIR/unsafe-rendered-runner.sh" >/dev/null 2>&1; then
|
||||
fail "renderer accepted shell syntax in a runtime path"
|
||||
fi
|
||||
[[ ! -e "$TEST_DIR/unsafe-rendered-runner.sh" ]] \
|
||||
|| fail "renderer published an unsafe partial candidate"
|
||||
pass "renderer rejects shell syntax before publishing a candidate"
|
||||
|
||||
bash -n "$RUNNER" "$RUNNER_TEMPLATE" "$RESTORE_DB" "$RESTORE_STORAGE"
|
||||
python3 - "$ATTACHMENT_VALIDATOR" <<'PY'
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
compile(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"), sys.argv[1], "exec")
|
||||
PY
|
||||
if rg -n '([0-9]{1,3}[.]){3}[0-9]{1,3}' \
|
||||
"$RUNNER_TEMPLATE" "$ATTACHMENT_VALIDATOR" >/dev/null; then
|
||||
fail "tracked runner assets contain a site-specific value"
|
||||
fi
|
||||
python3 - "$0" <<'PY'
|
||||
import ipaddress
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||
for raw in re.findall(r"(?<![0-9])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?![0-9])", text):
|
||||
address = ipaddress.ip_address(raw)
|
||||
if not address.is_loopback:
|
||||
raise SystemExit("non_loopback_address_in_contract")
|
||||
PY
|
||||
local_account="${USER:-}"
|
||||
if [[ "$local_account" != "root" && "${#local_account}" -ge 5 ]] \
|
||||
&& rg -F "$local_account" "$RUNNER_TEMPLATE" "$ATTACHMENT_VALIDATOR" "$0" \
|
||||
>/dev/null; then
|
||||
fail "tracked runner assets contain the local account name"
|
||||
fi
|
||||
pass "tracked template renders one syntax-valid candidate without site-specific values"
|
||||
|
||||
python3 - "$RUNNER_TEMPLATE" <<'PY'
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||
truncating = [
|
||||
line.strip()
|
||||
for line in text.splitlines()
|
||||
if re.search(r"(?<!>)>\"\$private_log\"", line)
|
||||
]
|
||||
if truncating != [': >"$private_log"']:
|
||||
raise SystemExit("private_log_truncation_contract")
|
||||
required_markers = {
|
||||
"[stage=postgres_container_start]",
|
||||
"[stage=dump_list]",
|
||||
"[stage=target_confirmation]",
|
||||
"[stage=restore_apply]",
|
||||
"[stage=restore_storage]",
|
||||
}
|
||||
missing = {marker for marker in required_markers if marker not in text}
|
||||
if missing:
|
||||
raise SystemExit("private_log_stage_markers_missing")
|
||||
PY
|
||||
pass "private raw log is initialized once and appends fixed stage markers"
|
||||
|
||||
python3 - "$RUNNER" "$TEST_DIR/validate-batch.py" <<'PY'
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
runner_path, output_path = map(pathlib.Path, sys.argv[1:])
|
||||
text = runner_path.read_text(encoding="utf-8")
|
||||
function_text = text.split("validate_batch() {", 1)[1]
|
||||
match = re.search(r"<<'PY'\n(?P<body>.*?)\nPY\n\}", function_text, re.S)
|
||||
if not match:
|
||||
raise SystemExit("validate_batch_body_missing")
|
||||
output_path.write_text(match.group("body") + "\n", encoding="utf-8")
|
||||
PY
|
||||
|
||||
python3 - "$TEST_DIR/batches" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
run_id = "auto-20260825T000000Z"
|
||||
source = root / f"fixture-backup-{run_id}"
|
||||
source.mkdir(parents=True)
|
||||
(source / "database.dump").write_bytes(b"database-fixture")
|
||||
(source / "storage.tar.gz").write_bytes(b"storage-fixture")
|
||||
|
||||
def artifact(name):
|
||||
data = (source / name).read_bytes()
|
||||
return {"file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}
|
||||
|
||||
manifest = {
|
||||
"schemaVersion": 1,
|
||||
"status": "complete",
|
||||
"runId": run_id,
|
||||
"createdAt": "2026-08-25T00:00:00Z",
|
||||
"buildSha": "a" * 40,
|
||||
"sourceImageDigest": "sha256:" + "c" * 64,
|
||||
"artifacts": {
|
||||
"database": artifact("database.dump"),
|
||||
"storage": artifact("storage.tar.gz"),
|
||||
},
|
||||
}
|
||||
(source / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
shutil.copytree(source, root / "private-copy")
|
||||
shutil.copytree(source, root / "wrong-source-name")
|
||||
PY
|
||||
|
||||
source_batch="$TEST_DIR/batches/fixture-backup-auto-20260825T000000Z"
|
||||
copy_batch="$TEST_DIR/batches/private-copy"
|
||||
validator_args=("$(printf 'a%.0s' {1..12})" "fixture-backup-" "auto-")
|
||||
source_metadata="$(python3 "$TEST_DIR/validate-batch.py" "$source_batch" \
|
||||
"${validator_args[0]}" source '' "${validator_args[1]}" "${validator_args[2]}")"
|
||||
copy_metadata="$(python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \
|
||||
"${validator_args[0]}" copy auto-20260825T000000Z \
|
||||
"${validator_args[1]}" "${validator_args[2]}")"
|
||||
[[ "$source_metadata" == "$copy_metadata" ]] \
|
||||
|| fail "source and renamed copy metadata differ"
|
||||
pass "source identity and renamed private copy both pass with source binding"
|
||||
|
||||
if python3 "$TEST_DIR/validate-batch.py" \
|
||||
"$TEST_DIR/batches/wrong-source-name" "${validator_args[0]}" source '' \
|
||||
"${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then
|
||||
fail "source basename/run-id mismatch passed"
|
||||
fi
|
||||
pass "source basename/run-id mismatch fails closed"
|
||||
|
||||
printf 'tamper' >>"$copy_batch/storage.tar.gz"
|
||||
if python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \
|
||||
"${validator_args[0]}" copy auto-20260825T000000Z \
|
||||
"${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then
|
||||
fail "copied content tamper passed"
|
||||
fi
|
||||
pass "copied content tamper fails closed"
|
||||
|
||||
python3 - "$source_batch/storage.tar.gz" "$copy_batch/storage.tar.gz" \
|
||||
"$copy_batch/manifest.json" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
source_storage, copied_storage, path = map(pathlib.Path, sys.argv[1:])
|
||||
copied_storage.write_bytes(source_storage.read_bytes())
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
payload["runId"] = "auto-20260825T010000Z"
|
||||
path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
PY
|
||||
if python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \
|
||||
"${validator_args[0]}" copy auto-20260825T000000Z \
|
||||
"${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then
|
||||
fail "copy run-id different from verified source passed"
|
||||
fi
|
||||
pass "copy run-id different from verified source fails closed"
|
||||
|
||||
rendered_migration_sha="$(sed -nE \
|
||||
's/^EXPECTED_MIGRATION_SET_SHA="([0-9a-f]{64})"$/\1/p' "$RUNNER")"
|
||||
[[ "$rendered_migration_sha" == "$migration_fixture_sha" ]] \
|
||||
|| fail "rendered dynamic migration fingerprint mismatch"
|
||||
mutated_migration_sha="$(printf '%s\n%s\n' "$migration_fixture_names" \
|
||||
'20260103000000_fixture_three' | sha256sum | awk '{print $1}')"
|
||||
[[ "$mutated_migration_sha" != "$rendered_migration_sha" ]] \
|
||||
|| fail "migration set mutation was not detected"
|
||||
if rg -n 'EXPECTED_MIGRATIONS|migration[^\n]*==[[:space:]]*65' "$RUNNER" >/dev/null; then
|
||||
fail "runner contains a hard-coded migration count"
|
||||
fi
|
||||
pass "dynamic migration fingerprint accepts exact set and rejects mutation"
|
||||
|
||||
mkdir -p "$TEST_DIR/attachment-root/nested"
|
||||
printf 'attachment-fixture' >"$TEST_DIR/attachment-root/nested/file.bin"
|
||||
attachment_path_b64="$(printf 'nested/file.bin' | base64 -w0)"
|
||||
attachment_size="$(stat -c %s "$TEST_DIR/attachment-root/nested/file.bin")"
|
||||
attachment_sha="$(sha256sum "$TEST_DIR/attachment-root/nested/file.bin" | awk '{print $1}')"
|
||||
attachment_ok="$(printf '%s\t%s\t%s\n' "$attachment_path_b64" \
|
||||
"$attachment_size" "$attachment_sha" \
|
||||
| python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")"
|
||||
[[ "$attachment_ok" == "1 0 0" ]] \
|
||||
|| fail "attachment validator rejected a valid local reference"
|
||||
unsafe_path_b64="$(printf '../escape.bin' | base64 -w0)"
|
||||
attachment_unsafe="$(printf '%s\t\t\n' "$unsafe_path_b64" \
|
||||
| python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")"
|
||||
[[ "$attachment_unsafe" == "1 1 0" ]] \
|
||||
|| fail "attachment validator accepted path traversal"
|
||||
s3_path_b64="$(printf 's3:external-object' | base64 -w0)"
|
||||
attachment_s3="$(printf '%s\t\t\n' "$s3_path_b64" \
|
||||
| python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")"
|
||||
[[ "$attachment_s3" == "1 1 1" ]] \
|
||||
|| fail "attachment validator accepted an external object reference"
|
||||
pass "attachment validator accepts exact local hash and rejects traversal/S3"
|
||||
|
||||
python3 - "$RUNNER" <<'PY'
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")
|
||||
finish = re.search(r"\nfinish\(\) \{(?P<body>.*?)\n\}\ntrap finish", text, re.S)
|
||||
if not finish:
|
||||
raise SystemExit("finish function not found")
|
||||
body = finish.group("body")
|
||||
emit_position = body.find("\n emit_restore_diagnostics")
|
||||
cleanup_position = body.find("\n cleanup_runtime ||")
|
||||
if emit_position == -1:
|
||||
raise SystemExit("diagnostic emission missing")
|
||||
if cleanup_position == -1:
|
||||
raise SystemExit("runtime cleanup call missing")
|
||||
if emit_position > cleanup_position:
|
||||
raise SystemExit("diagnostics are emitted after cleanup")
|
||||
required = {
|
||||
'echo "dump_list=$DUMP_LIST_STATUS"',
|
||||
'echo "target_confirmation=$TARGET_CONFIRMATION_STATUS"',
|
||||
'echo "restore_execution_path=$RESTORE_EXECUTION_PATH"',
|
||||
'echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE"',
|
||||
'echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE"',
|
||||
'echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT"',
|
||||
'echo "restore_error_category=$RESTORE_ERROR_CATEGORY"',
|
||||
}
|
||||
missing = sorted(value for value in required if value not in text)
|
||||
if missing:
|
||||
raise SystemExit(f"diagnostic fields missing: {missing}")
|
||||
if 'RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"' not in text:
|
||||
raise SystemExit("non-zero apply classification missing")
|
||||
PY
|
||||
pass "diagnostics precede finally-equivalent cleanup and use allowlisted fields"
|
||||
|
||||
docker run -d --rm --name "$CONTAINER_NAME" -p 127.0.0.1::5432 \
|
||||
-e "POSTGRES_USER=$POSTGRES_USER" \
|
||||
-e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \
|
||||
-e "POSTGRES_DB=$POSTGRES_DB" \
|
||||
postgres:16-alpine >/dev/null
|
||||
|
||||
ready_streak=0
|
||||
for _ in $(seq 1 60); do
|
||||
if docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \
|
||||
-U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atqc 'SELECT 1' \
|
||||
>/dev/null 2>&1; then
|
||||
ready_streak="$((ready_streak + 1))"
|
||||
[[ "$ready_streak" -ge 2 ]] && break
|
||||
else
|
||||
ready_streak=0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
[[ "$ready_streak" -ge 2 ]] \
|
||||
|| fail "PostgreSQL 16 fixture did not become stably ready"
|
||||
|
||||
HOST_PORT="$(docker port "$CONTAINER_NAME" 5432/tcp \
|
||||
| awk -F: 'NR == 1 {print $NF}')"
|
||||
[[ "$HOST_PORT" =~ ^[0-9]+$ ]] \
|
||||
|| fail "loopback PostgreSQL 16 port unavailable"
|
||||
HOST_DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${HOST_PORT}/${POSTGRES_DB}"
|
||||
|
||||
docker exec -i "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \
|
||||
-U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' >/dev/null
|
||||
CREATE TABLE restore_contract_marker (value text NOT NULL);
|
||||
INSERT INTO restore_contract_marker(value) VALUES ('direct-real-script');
|
||||
SQL
|
||||
docker exec "$CONTAINER_NAME" pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
||||
--format=c --no-owner --no-acl >"$TEST_DIR/fixture.dump"
|
||||
docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \
|
||||
-U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
||||
-c 'DROP TABLE restore_contract_marker' >/dev/null
|
||||
|
||||
DOCKER_DATABASE_URL="postgresql://shadow.invalid/$POSTGRES_DB"
|
||||
|
||||
DB_TOOL_MODE=docker \
|
||||
DB_CONTAINER="$CONTAINER_NAME" \
|
||||
POSTGRES_DB="$POSTGRES_DB" \
|
||||
POSTGRES_USER="$POSTGRES_USER" \
|
||||
POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
||||
DATABASE_URL="$DOCKER_DATABASE_URL" \
|
||||
RESTORE_SCOPE=shadow \
|
||||
RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
|
||||
"$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null
|
||||
[[ "$(docker exec "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
||||
-Atqc 'SELECT value FROM restore_contract_marker')" \
|
||||
== "direct-real-script" ]] || fail "Docker stdin restore result mismatch"
|
||||
pass "real restore-db.sh Docker stdin path restores a PostgreSQL 16 custom dump"
|
||||
|
||||
head -c 32 "$TEST_DIR/fixture.dump" >"$TEST_DIR/corrupt.dump"
|
||||
if DB_TOOL_MODE=docker \
|
||||
DB_CONTAINER="$CONTAINER_NAME" \
|
||||
POSTGRES_DB="$POSTGRES_DB" \
|
||||
POSTGRES_USER="$POSTGRES_USER" \
|
||||
POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
||||
DATABASE_URL="$DOCKER_DATABASE_URL" \
|
||||
RESTORE_SCOPE=shadow \
|
||||
RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
|
||||
"$RESTORE_DB" "$TEST_DIR/corrupt.dump" >/dev/null 2>&1; then
|
||||
fail "corrupt custom dump passed list validation"
|
||||
fi
|
||||
pass "real restore-db.sh rejects a dump-list failure"
|
||||
|
||||
if DB_TOOL_MODE=docker \
|
||||
DB_CONTAINER="$CONTAINER_NAME" \
|
||||
POSTGRES_DB="$POSTGRES_DB" \
|
||||
POSTGRES_USER="$POSTGRES_USER" \
|
||||
POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
||||
DATABASE_URL="$DOCKER_DATABASE_URL" \
|
||||
RESTORE_SCOPE=shadow \
|
||||
RESTORE_CONFIRM_DATABASE=wrong_restore_drill \
|
||||
"$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null 2>&1; then
|
||||
fail "wrong target confirmation was accepted"
|
||||
fi
|
||||
pass "real restore-db.sh rejects target confirmation mismatch"
|
||||
|
||||
REAL_DOCKER="$(command -v docker)"
|
||||
mkdir -m 0700 "$TEST_DIR/proxy"
|
||||
cat >"$TEST_DIR/proxy/docker" <<'PROXY'
|
||||
#!/usr/bin/env bash
|
||||
set +e
|
||||
is_pg_restore=0
|
||||
is_list=0
|
||||
for argument in "$@"; do
|
||||
[[ "$argument" == "pg_restore" ]] && is_pg_restore=1
|
||||
[[ "$argument" == "--list" ]] && is_list=1
|
||||
done
|
||||
if [[ "$is_pg_restore" -eq 1 && "$is_list" -eq 0 ]]; then
|
||||
cat >/dev/null
|
||||
exit 23
|
||||
fi
|
||||
"$REAL_DOCKER" "$@"
|
||||
exit "$?"
|
||||
PROXY
|
||||
chmod 0700 "$TEST_DIR/proxy/docker"
|
||||
|
||||
set +e
|
||||
proxy_output="$(REAL_DOCKER="$REAL_DOCKER" \
|
||||
PATH="$TEST_DIR/proxy:$PATH" \
|
||||
DB_TOOL_MODE=docker \
|
||||
DB_CONTAINER="$CONTAINER_NAME" \
|
||||
POSTGRES_DB="$POSTGRES_DB" \
|
||||
POSTGRES_USER="$POSTGRES_USER" \
|
||||
POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
||||
DATABASE_URL="$DOCKER_DATABASE_URL" \
|
||||
RESTORE_SCOPE=shadow \
|
||||
RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
|
||||
"$RESTORE_DB" "$TEST_DIR/fixture.dump" 2>&1)"
|
||||
proxy_exit="$?"
|
||||
set -e
|
||||
[[ "$proxy_exit" -eq 23 ]] || fail "apply failure exit code was not preserved"
|
||||
[[ "$proxy_output" != *"$POSTGRES_PASSWORD"* \
|
||||
&& "$proxy_output" != *"$DOCKER_DATABASE_URL"* ]] \
|
||||
|| fail "apply failure output exposed fixture credentials"
|
||||
pass "real restore-db.sh preserves an injected apply exit and exposes no fixture secret"
|
||||
|
||||
mkdir -m 0700 "$TEST_DIR/host-tools"
|
||||
cat >"$TEST_DIR/host-tools/psql" <<'HOST_PSQL'
|
||||
#!/usr/bin/env bash
|
||||
exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \
|
||||
postgres:16-alpine psql "$@"
|
||||
HOST_PSQL
|
||||
cat >"$TEST_DIR/host-tools/pg_restore" <<'HOST_RESTORE'
|
||||
#!/usr/bin/env bash
|
||||
exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \
|
||||
-v "$HOST_TOOL_MOUNT:$HOST_TOOL_MOUNT:ro" \
|
||||
postgres:16-alpine pg_restore "$@"
|
||||
HOST_RESTORE
|
||||
chmod 0700 "$TEST_DIR/host-tools/psql" "$TEST_DIR/host-tools/pg_restore"
|
||||
export PGPASSWORD="$POSTGRES_PASSWORD" REAL_DOCKER HOST_TOOL_MOUNT="$TEST_DIR"
|
||||
for _ in $(seq 1 60); do
|
||||
if "$TEST_DIR/host-tools/psql" "$HOST_DATABASE_URL" \
|
||||
-Atqc 'SELECT 1' >/dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
"$TEST_DIR/host-tools/psql" "$HOST_DATABASE_URL" \
|
||||
-Atqc 'SELECT 1' >/dev/null 2>&1 \
|
||||
|| fail "host-path TCP fixture did not become ready"
|
||||
docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \
|
||||
-U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
||||
-c 'DROP TABLE restore_contract_marker' >/dev/null
|
||||
PATH="$TEST_DIR/host-tools:$PATH" \
|
||||
DB_TOOL_MODE=host \
|
||||
DATABASE_URL="$HOST_DATABASE_URL" \
|
||||
RESTORE_SCOPE=shadow \
|
||||
RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
|
||||
"$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null
|
||||
[[ "$(docker exec "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
||||
-Atqc 'SELECT value FROM restore_contract_marker')" == "direct-real-script" ]] \
|
||||
|| fail "host-file restore result mismatch"
|
||||
pass "real restore-db.sh host-file path restores through PostgreSQL 16 tools"
|
||||
|
||||
docker rm -f "$CONTAINER_NAME" >/dev/null
|
||||
if docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then
|
||||
fail "fixture container cleanup failed"
|
||||
fi
|
||||
CONTAINER_NAME=""
|
||||
|
||||
sed -n '1,/^trap finish EXIT HUP INT TERM$/p' "$RUNNER" \
|
||||
| sed '$d' >"$TEST_DIR/runner-finalization-functions.sh"
|
||||
|
||||
capture_finalization_case() {
|
||||
local output_file="$1"
|
||||
local failure_category="$2"
|
||||
local dump_status="$3"
|
||||
local dump_exit="$4"
|
||||
local target_status="$5"
|
||||
local apply_status="$6"
|
||||
local script_exit="$7"
|
||||
local pg_restore_exit="$8"
|
||||
local error_count="$9"
|
||||
local error_category="${10}"
|
||||
set +e
|
||||
(
|
||||
# shellcheck disable=SC1090
|
||||
source "$TEST_DIR/runner-finalization-functions.sh"
|
||||
cleanup_runtime() {
|
||||
echo "cleanup_marker=runtime"
|
||||
return 0
|
||||
}
|
||||
cleanup_access() {
|
||||
echo "cleanup_marker=access"
|
||||
return 0
|
||||
}
|
||||
FAILURE_CATEGORY="$failure_category"
|
||||
DUMP_LIST_STATUS="$dump_status"
|
||||
DUMP_LIST_EXIT_CODE="$dump_exit"
|
||||
TARGET_CONFIRMATION_STATUS="$target_status"
|
||||
RESTORE_EXECUTION_PATH="DOCKER_STDIN"
|
||||
RESTORE_APPLY_STATUS="$apply_status"
|
||||
RESTORE_SCRIPT_EXIT_CODE="$script_exit"
|
||||
PG_RESTORE_EXIT_CODE="$pg_restore_exit"
|
||||
PG_RESTORE_ERROR_COUNT="$error_count"
|
||||
RESTORE_ERROR_CATEGORY="$error_category"
|
||||
trap finish EXIT
|
||||
exit 23
|
||||
) >"$output_file" 2>&1
|
||||
local case_exit="$?"
|
||||
set -e
|
||||
[[ "$case_exit" -eq 23 ]] \
|
||||
|| fail "finalization did not preserve the injected exit code"
|
||||
}
|
||||
|
||||
capture_finalization_case \
|
||||
"$TEST_DIR/captured-dump-list-failure" \
|
||||
dump_list_failed FAIL 1 NOT_RUN NOT_RUN NOT_RUN NOT_RUN NOT_RUN \
|
||||
DUMP_LIST_NONZERO
|
||||
grep -Fxq 'dump_list=FAIL' "$TEST_DIR/captured-dump-list-failure" \
|
||||
|| fail "dump-list failure stage missing"
|
||||
grep -Fxq 'target_confirmation=NOT_RUN' \
|
||||
"$TEST_DIR/captured-dump-list-failure" \
|
||||
|| fail "target stage ran after dump-list failure"
|
||||
grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-dump-list-failure" \
|
||||
|| fail "apply stage ran after dump-list failure"
|
||||
pass "dump-list failure keeps target and apply stages NOT_RUN"
|
||||
|
||||
capture_finalization_case \
|
||||
"$TEST_DIR/captured-target-failure" \
|
||||
restore_target_confirmation_failed PASS 0 FAIL NOT_RUN NOT_RUN NOT_RUN \
|
||||
NOT_RUN TARGET_CONFIRMATION_FAILED
|
||||
grep -Fxq 'target_confirmation=FAIL' "$TEST_DIR/captured-target-failure" \
|
||||
|| fail "target-confirmation failure stage missing"
|
||||
grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-target-failure" \
|
||||
|| fail "apply stage ran after target-confirmation failure"
|
||||
pass "target-confirmation failure keeps apply stage NOT_RUN"
|
||||
|
||||
set +e
|
||||
(
|
||||
# shellcheck disable=SC1090
|
||||
source "$TEST_DIR/runner-finalization-functions.sh"
|
||||
cleanup_runtime() {
|
||||
echo "cleanup_marker=runtime"
|
||||
return 0
|
||||
}
|
||||
cleanup_access() {
|
||||
echo "cleanup_marker=access"
|
||||
return 0
|
||||
}
|
||||
FAILURE_CATEGORY="restore_apply_failed_after_list_pass"
|
||||
DUMP_LIST_STATUS="PASS"
|
||||
DUMP_LIST_EXIT_CODE="0"
|
||||
TARGET_CONFIRMATION_STATUS="PASS"
|
||||
RESTORE_EXECUTION_PATH="DOCKER_STDIN"
|
||||
RESTORE_APPLY_STATUS="FAIL"
|
||||
RESTORE_SCRIPT_EXIT_CODE="23"
|
||||
PG_RESTORE_EXIT_CODE="23"
|
||||
PG_RESTORE_ERROR_COUNT="UNKNOWN"
|
||||
RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"
|
||||
trap finish EXIT
|
||||
exit 23
|
||||
) >"$TEST_DIR/captured-finalization" 2>&1
|
||||
finalization_exit="$?"
|
||||
set -e
|
||||
[[ "$finalization_exit" -eq 23 ]] \
|
||||
|| fail "finalization did not preserve the injected exit code"
|
||||
diagnostic_line="$(grep -n '^restore_apply=FAIL$' \
|
||||
"$TEST_DIR/captured-finalization" | cut -d: -f1)"
|
||||
cleanup_line="$(grep -n '^cleanup_marker=runtime$' \
|
||||
"$TEST_DIR/captured-finalization" | cut -d: -f1)"
|
||||
[[ "$diagnostic_line" =~ ^[0-9]+$ && "$cleanup_line" =~ ^[0-9]+$ \
|
||||
&& "$diagnostic_line" -lt "$cleanup_line" ]] \
|
||||
|| fail "stage diagnostics were not emitted before teardown"
|
||||
grep -Fxq 'runtime_cleanup=PASS' "$TEST_DIR/captured-finalization" \
|
||||
|| fail "runtime teardown result was not retained"
|
||||
grep -Fxq 'temporary_access_cleanup=PASS' "$TEST_DIR/captured-finalization" \
|
||||
|| fail "access teardown result was not retained"
|
||||
grep -Fxq 'pg_restore_exit_code=23' "$TEST_DIR/captured-finalization" \
|
||||
|| fail "pg_restore exit code disappeared after teardown"
|
||||
pass "injected failure emits readable stages before finally-equivalent teardown"
|
||||
|
||||
if grep -Eiq 'postgresql://|SELECT[[:space:]]|/tmp/|local_fixture_password' \
|
||||
"$TEST_DIR/captured-dump-list-failure" \
|
||||
"$TEST_DIR/captured-target-failure" \
|
||||
"$TEST_DIR/captured-finalization"; then
|
||||
fail "allowlisted diagnostics exposed a path, credential or SQL"
|
||||
fi
|
||||
pass "captured stage diagnostics contain no path, credential or SQL"
|
||||
|
||||
echo "[runner-contract][PASS] all off-host shadow-restore runner contracts completed"
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/usr/bin/env python3
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if len(sys.argv) != 2:
|
||||
return 2
|
||||
|
||||
root = os.path.realpath(sys.argv[1])
|
||||
total = 0
|
||||
mismatch = 0
|
||||
s3_count = 0
|
||||
|
||||
for raw in sys.stdin:
|
||||
raw = raw.rstrip("\n")
|
||||
if not raw:
|
||||
continue
|
||||
parts = raw.split("\t")
|
||||
if len(parts) != 3:
|
||||
mismatch += 1
|
||||
continue
|
||||
|
||||
total += 1
|
||||
try:
|
||||
storage_path = base64.b64decode(parts[0], validate=True).decode("utf-8")
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
mismatch += 1
|
||||
continue
|
||||
|
||||
if storage_path.startswith("s3:"):
|
||||
s3_count += 1
|
||||
mismatch += 1
|
||||
continue
|
||||
|
||||
pure_path = pathlib.PurePosixPath(storage_path)
|
||||
if not storage_path or pure_path.is_absolute() or ".." in pure_path.parts:
|
||||
mismatch += 1
|
||||
continue
|
||||
|
||||
target = os.path.realpath(os.path.join(root, *pure_path.parts))
|
||||
if not target.startswith(root + os.sep) or not os.path.isfile(target):
|
||||
mismatch += 1
|
||||
continue
|
||||
|
||||
expected_size, expected_sha = parts[1], parts[2].lower()
|
||||
if expected_size and str(os.path.getsize(target)) != expected_size:
|
||||
mismatch += 1
|
||||
continue
|
||||
|
||||
if expected_sha:
|
||||
digest = hashlib.sha256()
|
||||
with open(target, "rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
if digest.hexdigest() != expected_sha:
|
||||
mismatch += 1
|
||||
|
||||
print(total, mismatch, s3_count)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
在新工单中引用
屏蔽一个用户