1036 行
40 KiB
Bash
可执行文件
1036 行
40 KiB
Bash
可执行文件
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
RUNNER_TEMPLATE="$ROOT_DIR/scripts/qa/offhost-shadow-restore-runner.template.sh"
|
|
ATTACHMENT_VALIDATOR="$ROOT_DIR/scripts/qa/validate-offhost-shadow-restore-attachments.py"
|
|
RESTORE_DB="$ROOT_DIR/scripts/restore-db.sh"
|
|
RESTORE_STORAGE="$ROOT_DIR/scripts/restore-storage.sh"
|
|
TEST_DIR="$(mktemp -d)"
|
|
RUNNER="$TEST_DIR/rendered-runner.sh"
|
|
PRIVATE_REPO_TEST_DIR="$ROOT_DIR/.restore-drill-private/contract-$$-$RANDOM"
|
|
UNIGNORED_CONFIG="$ROOT_DIR/.restore-drill-private-contract-$$-$RANDOM.json"
|
|
UNIGNORED_OUTPUT="$ROOT_DIR/.restore-drill-private-contract-$$-$RANDOM.sh"
|
|
CONTAINER_NAME="$(basename "$0" .sh)-$RANDOM-$$"
|
|
POSTGRES_USER="fixture_user_$RANDOM"
|
|
POSTGRES_PASSWORD="$(openssl rand -hex 16)"
|
|
POSTGRES_DB="fixture_restore_drill_$RANDOM"
|
|
|
|
cleanup() {
|
|
[[ -z "$CONTAINER_NAME" ]] \
|
|
|| docker rm -f "$CONTAINER_NAME" >/dev/null 2>&1 || true
|
|
rm -rf -- "$TEST_DIR"
|
|
rm -rf -- "$PRIVATE_REPO_TEST_DIR"
|
|
rm -f -- "$UNIGNORED_CONFIG" "$UNIGNORED_OUTPUT"
|
|
}
|
|
trap cleanup EXIT HUP INT TERM
|
|
|
|
fail() {
|
|
echo "[runner-contract][FAIL] $1" >&2
|
|
exit 1
|
|
}
|
|
|
|
pass() {
|
|
echo "[runner-contract][PASS] $1"
|
|
}
|
|
|
|
require_ignored_if_inside_repo() {
|
|
local candidate_path="$1"
|
|
local resolved_parent resolved_path relative_path
|
|
resolved_parent="$(realpath -e "$(dirname "$candidate_path")")" \
|
|
|| return 1
|
|
resolved_path="$resolved_parent/$(basename "$candidate_path")"
|
|
case "$resolved_path" in
|
|
"$ROOT_DIR"/*)
|
|
relative_path="${resolved_path#"$ROOT_DIR"/}"
|
|
git -C "$ROOT_DIR" check-ignore -q --no-index -- "$relative_path" \
|
|
|| return 1
|
|
;;
|
|
esac
|
|
}
|
|
|
|
render_runner() {
|
|
local config_file="$1"
|
|
local output_file="$2"
|
|
require_ignored_if_inside_repo "$config_file" || return 1
|
|
require_ignored_if_inside_repo "$output_file" || return 1
|
|
python3 - "$RUNNER_TEMPLATE" "$config_file" "$output_file" <<'PY'
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
template_path, config_path, output_path = map(pathlib.Path, sys.argv[1:])
|
|
template = template_path.read_text(encoding="utf-8")
|
|
config = json.loads(config_path.read_text(encoding="utf-8"))
|
|
tokens = set(re.findall(r"@@([A-Z0-9_]+)@@", template))
|
|
if set(config) != tokens:
|
|
raise SystemExit("render_config_keys")
|
|
path_keys = {
|
|
"RECEIVER_CONFIG_FILE",
|
|
"RESTORE_LIB_DIR",
|
|
"SELF_PATH",
|
|
"SUDOERS_PATH",
|
|
"SUDOERS_MAIN_FILE",
|
|
"WORK_ROOT",
|
|
}
|
|
prefix_keys = {"WORK_PREFIX", "CONTAINER_PREFIX", "ARCHIVE_BATCH_PREFIX", "RUN_ID_PREFIX"}
|
|
sha_keys = {
|
|
"EXPECTED_DB_RESTORE_SHA",
|
|
"EXPECTED_STORAGE_RESTORE_SHA",
|
|
"EXPECTED_ATTACHMENT_VALIDATOR_SHA",
|
|
"EXPECTED_MIGRATION_SET_SHA",
|
|
}
|
|
for key, value in config.items():
|
|
if not isinstance(value, str) or not value or "\n" in value or "\r" in value:
|
|
raise SystemExit("render_config_value")
|
|
if key in path_keys:
|
|
parsed = pathlib.PurePosixPath(value)
|
|
if not parsed.is_absolute() or any(part in {"", ".", ".."} for part in parsed.parts[1:]):
|
|
raise SystemExit("render_path_value")
|
|
if not re.fullmatch(r"/[A-Za-z0-9_./-]+", value):
|
|
raise SystemExit("render_path_charset")
|
|
elif key in prefix_keys and not re.fullmatch(r"[a-z0-9][a-z0-9-]*-", value):
|
|
raise SystemExit("render_prefix_value")
|
|
elif key == "SHADOW_DB_USER" and not re.fullmatch(r"[a-z_][a-z0-9_]*", value):
|
|
raise SystemExit("render_db_user")
|
|
elif key == "SHADOW_DB_NAME" and (
|
|
not re.fullmatch(r"[a-z_][a-z0-9_]*", value)
|
|
or not re.search(r"(^|_)(shadow|drill|restore_test)(_|$)", value)
|
|
):
|
|
raise SystemExit("render_db_name")
|
|
elif key == "POSTGRES_IMAGE" and not re.fullmatch(r"[a-z0-9._/-]+:[A-Za-z0-9._-]+", value):
|
|
raise SystemExit("render_postgres_image")
|
|
elif key in {"DEADMAN_TIMER_UNIT", "HEALTH_TIMER_UNIT"} and not re.fullmatch(
|
|
r"[A-Za-z0-9_.@-]+\.timer", value
|
|
):
|
|
raise SystemExit("render_timer_unit")
|
|
elif key in sha_keys and not re.fullmatch(r"[0-9a-f]{64}", value):
|
|
raise SystemExit("render_sha")
|
|
elif key == "EXPECTED_BUILD_PREFIX" and not re.fullmatch(r"[0-9a-f]{12}", value):
|
|
raise SystemExit("render_build_prefix")
|
|
template = template.replace(f"@@{key}@@", value)
|
|
if re.search(r"@@[A-Z0-9_]+@@", template):
|
|
raise SystemExit("render_placeholder_remaining")
|
|
descriptor = os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o700)
|
|
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
handle.write(template)
|
|
PY
|
|
}
|
|
|
|
if [[ "${1:-}" == "render" ]]; then
|
|
[[ "$#" -eq 3 ]] || fail "usage: $0 render <private-config.json> <output>"
|
|
render_runner "$2" "$3" || fail "private render paths must be outside the repository or ignored"
|
|
bash -n "$3"
|
|
candidate_sha="$(sha256sum "$3" | awk '{print $1}')"
|
|
echo "[runner-contract][PASS] rendered candidate syntax"
|
|
echo "candidate_sha256=$candidate_sha"
|
|
exit 0
|
|
fi
|
|
[[ "$#" -eq 0 ]] || fail "usage: $0 [render <private-config.json> <output>]"
|
|
|
|
db_restore_sha="$(sha256sum "$RESTORE_DB" | awk '{print $1}')"
|
|
storage_restore_sha="$(sha256sum "$RESTORE_STORAGE" | awk '{print $1}')"
|
|
attachment_validator_sha="$(sha256sum "$ATTACHMENT_VALIDATOR" | awk '{print $1}')"
|
|
migration_fixture_names=$'20260101000000_fixture_one\n20260102000000_fixture_two'
|
|
migration_fixture_sha="$(printf '%s\n' "$migration_fixture_names" | sha256sum | awk '{print $1}')"
|
|
python3 - "$TEST_DIR/render-config.json" "$db_restore_sha" \
|
|
"$storage_restore_sha" "$attachment_validator_sha" "$migration_fixture_sha" \
|
|
"$POSTGRES_USER" "$POSTGRES_DB" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
output, db_sha, storage_sha, validator_sha, migration_sha, db_user, db_name = sys.argv[1:]
|
|
payload = {
|
|
"RECEIVER_CONFIG_FILE": "/synthetic/receiver.conf",
|
|
"RESTORE_LIB_DIR": "/synthetic/restore-lib",
|
|
"SELF_PATH": "/synthetic/restore-entry",
|
|
"SUDOERS_PATH": "/synthetic/restore-rule",
|
|
"SUDOERS_MAIN_FILE": "/synthetic/sudoers",
|
|
"WORK_ROOT": "/tmp",
|
|
"WORK_PREFIX": "restore-contract-work-",
|
|
"CONTAINER_PREFIX": "restore-contract-container-",
|
|
"SHADOW_DB_USER": db_user,
|
|
"SHADOW_DB_NAME": db_name,
|
|
"POSTGRES_IMAGE": "postgres:16-alpine",
|
|
"DEADMAN_TIMER_UNIT": "fixture-deadman.timer",
|
|
"HEALTH_TIMER_UNIT": "fixture-health.timer",
|
|
"ARCHIVE_BATCH_PREFIX": "fixture-backup-",
|
|
"RUN_ID_PREFIX": "auto-",
|
|
"EXPECTED_DB_RESTORE_SHA": db_sha,
|
|
"EXPECTED_STORAGE_RESTORE_SHA": storage_sha,
|
|
"EXPECTED_ATTACHMENT_VALIDATOR_SHA": validator_sha,
|
|
"EXPECTED_BUILD_PREFIX": "a" * 12,
|
|
"EXPECTED_MIGRATION_SET_SHA": migration_sha,
|
|
}
|
|
pathlib.Path(output).write_text(json.dumps(payload), encoding="utf-8")
|
|
PY
|
|
|
|
cp "$TEST_DIR/render-config.json" "$UNIGNORED_CONFIG"
|
|
if render_runner "$UNIGNORED_CONFIG" "$TEST_DIR/unignored-config-output.sh" \
|
|
>/dev/null 2>&1; then
|
|
fail "renderer accepted an unignored in-repository private config"
|
|
fi
|
|
[[ ! -e "$TEST_DIR/unignored-config-output.sh" ]] \
|
|
|| fail "renderer published output for an unignored private config"
|
|
pass "unignored in-repository private config fails before candidate creation"
|
|
|
|
if render_runner "$TEST_DIR/render-config.json" "$UNIGNORED_OUTPUT" \
|
|
>/dev/null 2>&1; then
|
|
fail "renderer accepted an unignored in-repository candidate path"
|
|
fi
|
|
[[ ! -e "$UNIGNORED_OUTPUT" ]] \
|
|
|| fail "renderer published an unignored in-repository candidate"
|
|
pass "unignored in-repository candidate path fails before creation"
|
|
|
|
mkdir -m 0700 -p "$PRIVATE_REPO_TEST_DIR"
|
|
cp "$TEST_DIR/render-config.json" "$PRIVATE_REPO_TEST_DIR/render-config.json"
|
|
render_runner "$PRIVATE_REPO_TEST_DIR/render-config.json" \
|
|
"$PRIVATE_REPO_TEST_DIR/rendered-runner.sh"
|
|
[[ -s "$PRIVATE_REPO_TEST_DIR/rendered-runner.sh" ]] \
|
|
|| fail "ignored private render path did not produce a candidate"
|
|
pass "ignored in-repository private config and candidate paths are accepted"
|
|
|
|
render_runner "$TEST_DIR/render-config.json" "$RUNNER"
|
|
render_runner "$TEST_DIR/render-config.json" "$TEST_DIR/rendered-runner-repeat.sh"
|
|
cmp -s "$RUNNER" "$TEST_DIR/rendered-runner-repeat.sh" \
|
|
|| fail "identical private config did not render a deterministic candidate"
|
|
pass "identical private config renders a deterministic candidate SHA-256"
|
|
python3 - "$TEST_DIR/render-config.json" "$TEST_DIR/unsafe-render-config.json" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
source, target = map(pathlib.Path, sys.argv[1:])
|
|
payload = json.loads(source.read_text(encoding="utf-8"))
|
|
payload["SELF_PATH"] = '/synthetic/entry";unsafe-command'
|
|
target.write_text(json.dumps(payload), encoding="utf-8")
|
|
PY
|
|
if render_runner "$TEST_DIR/unsafe-render-config.json" \
|
|
"$TEST_DIR/unsafe-rendered-runner.sh" >/dev/null 2>&1; then
|
|
fail "renderer accepted shell syntax in a runtime path"
|
|
fi
|
|
[[ ! -e "$TEST_DIR/unsafe-rendered-runner.sh" ]] \
|
|
|| fail "renderer published an unsafe partial candidate"
|
|
pass "renderer rejects shell syntax before publishing a candidate"
|
|
|
|
bash -n "$RUNNER" "$RUNNER_TEMPLATE" "$RESTORE_DB" "$RESTORE_STORAGE"
|
|
python3 - "$ATTACHMENT_VALIDATOR" <<'PY'
|
|
import pathlib
|
|
import sys
|
|
|
|
compile(pathlib.Path(sys.argv[1]).read_text(encoding="utf-8"), sys.argv[1], "exec")
|
|
PY
|
|
python3 - "$RUNNER_TEMPLATE" "$ATTACHMENT_VALIDATOR" "$0" <<'PY'
|
|
import ipaddress
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
for source in sys.argv[1:]:
|
|
text = pathlib.Path(source).read_text(encoding="utf-8")
|
|
for raw in re.findall(r"(?<![0-9])(?:[0-9]{1,3}\.){3}[0-9]{1,3}(?![0-9])", text):
|
|
address = ipaddress.ip_address(raw)
|
|
if not address.is_loopback:
|
|
raise SystemExit("non_loopback_address_in_tracked_runner_asset")
|
|
PY
|
|
local_account="${USER:-}"
|
|
if [[ "$local_account" != "root" && "${#local_account}" -ge 5 ]] \
|
|
&& rg -F "$local_account" "$RUNNER_TEMPLATE" "$ATTACHMENT_VALIDATOR" "$0" \
|
|
>/dev/null; then
|
|
fail "tracked runner assets contain the local account name"
|
|
fi
|
|
pass "tracked template renders one syntax-valid candidate without site-specific values"
|
|
|
|
python3 - "$RUNNER_TEMPLATE" <<'PY'
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")
|
|
truncating = [
|
|
line.strip()
|
|
for line in text.splitlines()
|
|
if re.search(r"(?<!>)>\"\$private_log\"", line)
|
|
]
|
|
if truncating != [': >"$private_log"']:
|
|
raise SystemExit("private_log_truncation_contract")
|
|
required_markers = {
|
|
"[stage=postgres_container_start]",
|
|
"[stage=dump_list]",
|
|
"[stage=target_confirmation]",
|
|
"[stage=restore_apply]",
|
|
"[stage=restore_storage]",
|
|
}
|
|
missing = {marker for marker in required_markers if marker not in text}
|
|
if missing:
|
|
raise SystemExit("private_log_stage_markers_missing")
|
|
PY
|
|
pass "private raw log is initialized once and appends fixed stage markers"
|
|
|
|
python3 - "$RUNNER" "$TEST_DIR/confirm-postgres-target.sh" <<'PY'
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
runner_path, output_path = map(pathlib.Path, sys.argv[1:])
|
|
text = runner_path.read_text(encoding="utf-8")
|
|
match = re.search(
|
|
r"\n(confirm_postgres_target\(\) \{.*?\n\})\n\ncleanup_runtime\(\)",
|
|
text,
|
|
re.S,
|
|
)
|
|
if not match:
|
|
raise SystemExit("confirm_postgres_target_body_missing")
|
|
output_path.write_text(match.group(1) + "\n", encoding="utf-8")
|
|
PY
|
|
|
|
if rg -n 'pg_isready' "$RUNNER" >/dev/null; then
|
|
fail "socket-only pg_isready remains in rendered runner"
|
|
fi
|
|
pass "rendered runner exposes the single TCP target-confirmation implementation"
|
|
|
|
python3 - "$RUNNER" "$TEST_DIR/validate-batch.py" <<'PY'
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
runner_path, output_path = map(pathlib.Path, sys.argv[1:])
|
|
text = runner_path.read_text(encoding="utf-8")
|
|
function_text = text.split("validate_batch() {", 1)[1]
|
|
match = re.search(r"<<'PY'\n(?P<body>.*?)\nPY\n\}", function_text, re.S)
|
|
if not match:
|
|
raise SystemExit("validate_batch_body_missing")
|
|
output_path.write_text(match.group("body") + "\n", encoding="utf-8")
|
|
PY
|
|
|
|
python3 - "$TEST_DIR/batches" <<'PY'
|
|
import hashlib
|
|
import json
|
|
import pathlib
|
|
import shutil
|
|
import sys
|
|
|
|
root = pathlib.Path(sys.argv[1])
|
|
run_id = "auto-20260825T000000Z"
|
|
source = root / f"fixture-backup-{run_id}"
|
|
source.mkdir(parents=True)
|
|
(source / "database.dump").write_bytes(b"database-fixture")
|
|
(source / "storage.tar.gz").write_bytes(b"storage-fixture")
|
|
|
|
def artifact(name):
|
|
data = (source / name).read_bytes()
|
|
return {"file": name, "bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}
|
|
|
|
manifest = {
|
|
"schemaVersion": 1,
|
|
"status": "complete",
|
|
"runId": run_id,
|
|
"createdAt": "2026-08-25T00:00:00Z",
|
|
"buildSha": "a" * 40,
|
|
"sourceImageDigest": "sha256:" + "c" * 64,
|
|
"artifacts": {
|
|
"database": artifact("database.dump"),
|
|
"storage": artifact("storage.tar.gz"),
|
|
},
|
|
}
|
|
(source / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
|
shutil.copytree(source, root / "private-copy")
|
|
shutil.copytree(source, root / "wrong-source-name")
|
|
PY
|
|
|
|
source_batch="$TEST_DIR/batches/fixture-backup-auto-20260825T000000Z"
|
|
copy_batch="$TEST_DIR/batches/private-copy"
|
|
validator_args=("$(printf 'a%.0s' {1..12})" "fixture-backup-" "auto-")
|
|
source_metadata="$(python3 "$TEST_DIR/validate-batch.py" "$source_batch" \
|
|
"${validator_args[0]}" source '' "${validator_args[1]}" "${validator_args[2]}")"
|
|
copy_metadata="$(python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \
|
|
"${validator_args[0]}" copy auto-20260825T000000Z \
|
|
"${validator_args[1]}" "${validator_args[2]}")"
|
|
[[ "$source_metadata" == "$copy_metadata" ]] \
|
|
|| fail "source and renamed copy metadata differ"
|
|
pass "source identity and renamed private copy both pass with source binding"
|
|
|
|
if python3 "$TEST_DIR/validate-batch.py" \
|
|
"$TEST_DIR/batches/wrong-source-name" "${validator_args[0]}" source '' \
|
|
"${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then
|
|
fail "source basename/run-id mismatch passed"
|
|
fi
|
|
pass "source basename/run-id mismatch fails closed"
|
|
|
|
printf 'tamper' >>"$copy_batch/storage.tar.gz"
|
|
if python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \
|
|
"${validator_args[0]}" copy auto-20260825T000000Z \
|
|
"${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then
|
|
fail "copied content tamper passed"
|
|
fi
|
|
pass "copied content tamper fails closed"
|
|
|
|
python3 - "$source_batch/storage.tar.gz" "$copy_batch/storage.tar.gz" \
|
|
"$copy_batch/manifest.json" <<'PY'
|
|
import json
|
|
import pathlib
|
|
import sys
|
|
|
|
source_storage, copied_storage, path = map(pathlib.Path, sys.argv[1:])
|
|
copied_storage.write_bytes(source_storage.read_bytes())
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
payload["runId"] = "auto-20260825T010000Z"
|
|
path.write_text(json.dumps(payload), encoding="utf-8")
|
|
PY
|
|
if python3 "$TEST_DIR/validate-batch.py" "$copy_batch" \
|
|
"${validator_args[0]}" copy auto-20260825T000000Z \
|
|
"${validator_args[1]}" "${validator_args[2]}" >/dev/null 2>&1; then
|
|
fail "copy run-id different from verified source passed"
|
|
fi
|
|
pass "copy run-id different from verified source fails closed"
|
|
|
|
rendered_migration_sha="$(sed -nE \
|
|
's/^EXPECTED_MIGRATION_SET_SHA="([0-9a-f]{64})"$/\1/p' "$RUNNER")"
|
|
[[ "$rendered_migration_sha" == "$migration_fixture_sha" ]] \
|
|
|| fail "rendered dynamic migration fingerprint mismatch"
|
|
mutated_migration_sha="$(printf '%s\n%s\n' "$migration_fixture_names" \
|
|
'20260103000000_fixture_three' | sha256sum | awk '{print $1}')"
|
|
[[ "$mutated_migration_sha" != "$rendered_migration_sha" ]] \
|
|
|| fail "migration set mutation was not detected"
|
|
if rg -n 'EXPECTED_MIGRATIONS|migration[^\n]*==[[:space:]]*65' "$RUNNER" >/dev/null; then
|
|
fail "runner contains a hard-coded migration count"
|
|
fi
|
|
pass "dynamic migration fingerprint accepts exact set and rejects mutation"
|
|
|
|
mkdir -p "$TEST_DIR/attachment-root/nested"
|
|
printf 'attachment-fixture' >"$TEST_DIR/attachment-root/nested/file.bin"
|
|
attachment_path_b64="$(printf 'nested/file.bin' | base64 -w0)"
|
|
attachment_size="$(stat -c %s "$TEST_DIR/attachment-root/nested/file.bin")"
|
|
attachment_sha="$(sha256sum "$TEST_DIR/attachment-root/nested/file.bin" | awk '{print $1}')"
|
|
attachment_ok="$(printf '%s\t%s\t%s\n' "$attachment_path_b64" \
|
|
"$attachment_size" "$attachment_sha" \
|
|
| python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")"
|
|
[[ "$attachment_ok" == "1 0 0" ]] \
|
|
|| fail "attachment validator rejected a valid local reference"
|
|
unsafe_path_b64="$(printf '../escape.bin' | base64 -w0)"
|
|
attachment_unsafe="$(printf '%s\t\t\n' "$unsafe_path_b64" \
|
|
| python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")"
|
|
[[ "$attachment_unsafe" == "1 1 0" ]] \
|
|
|| fail "attachment validator accepted path traversal"
|
|
s3_path_b64="$(printf 's3:external-object' | base64 -w0)"
|
|
attachment_s3="$(printf '%s\t\t\n' "$s3_path_b64" \
|
|
| python3 "$ATTACHMENT_VALIDATOR" "$TEST_DIR/attachment-root")"
|
|
[[ "$attachment_s3" == "1 1 1" ]] \
|
|
|| fail "attachment validator accepted an external object reference"
|
|
pass "attachment validator accepts exact local hash and rejects traversal/S3"
|
|
|
|
python3 - "$RUNNER" <<'PY'
|
|
import pathlib
|
|
import re
|
|
import sys
|
|
|
|
text = pathlib.Path(sys.argv[1]).read_text(encoding="utf-8")
|
|
finish = re.search(r"\nfinish\(\) \{(?P<body>.*?)\n\}\ntrap finish", text, re.S)
|
|
if not finish:
|
|
raise SystemExit("finish function not found")
|
|
body = finish.group("body")
|
|
finalize_position = body.find("\n if ! rto_finalize; then")
|
|
emit_position = body.find("\n emit_restore_diagnostics")
|
|
cleanup_position = body.find("\n cleanup_runtime ||")
|
|
if finalize_position == -1:
|
|
raise SystemExit("RTO finalization missing")
|
|
if emit_position == -1:
|
|
raise SystemExit("diagnostic emission missing")
|
|
if cleanup_position == -1:
|
|
raise SystemExit("runtime cleanup call missing")
|
|
if not (finalize_position < emit_position < cleanup_position):
|
|
raise SystemExit("RTO/stage diagnostics are not finalized before cleanup")
|
|
required = {
|
|
'echo "dump_list=$DUMP_LIST_STATUS"',
|
|
'echo "target_confirmation=$TARGET_CONFIRMATION_STATUS"',
|
|
'echo "target_confirmation_exit_code=$TARGET_CONFIRMATION_EXIT_CODE"',
|
|
'echo "target_name_match=$TARGET_NAME_MATCH"',
|
|
'echo "target_confirmation_attempt_count=$TARGET_CONFIRMATION_ATTEMPT_COUNT"',
|
|
'echo "target_confirmation_success_streak=$TARGET_CONFIRMATION_SUCCESS_STREAK"',
|
|
'echo "restore_execution_path=$RESTORE_EXECUTION_PATH"',
|
|
'echo "restore_script_exit_code=$RESTORE_SCRIPT_EXIT_CODE"',
|
|
'echo "pg_restore_exit_code=$PG_RESTORE_EXIT_CODE"',
|
|
'echo "pg_restore_error_count=$PG_RESTORE_ERROR_COUNT"',
|
|
'echo "restore_error_category=$RESTORE_ERROR_CATEGORY"',
|
|
'echo "rto_batch_copy_seconds=$RTO_BATCH_COPY_SECONDS"',
|
|
'echo "rto_postgres_container_start_seconds=$RTO_POSTGRES_CONTAINER_START_SECONDS"',
|
|
'echo "rto_dump_list_seconds=$RTO_DUMP_LIST_SECONDS"',
|
|
'echo "rto_target_confirmation_seconds=$RTO_TARGET_CONFIRMATION_SECONDS"',
|
|
'echo "rto_restore_apply_seconds=$RTO_RESTORE_APPLY_SECONDS"',
|
|
'echo "rto_database_verify_seconds=$RTO_DATABASE_VERIFY_SECONDS"',
|
|
'echo "rto_storage_attachment_verify_seconds=$RTO_STORAGE_ATTACHMENT_VERIFY_SECONDS"',
|
|
'echo "rto_final_integrity_seconds=$RTO_FINAL_INTEGRITY_SECONDS"',
|
|
'echo "data_restore_rto_seconds=$RTO_SECONDS"',
|
|
}
|
|
missing = sorted(value for value in required if value not in text)
|
|
if missing:
|
|
raise SystemExit(f"diagnostic fields missing: {missing}")
|
|
if 'RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"' not in text:
|
|
raise SystemExit("non-zero apply classification missing")
|
|
transitions = [
|
|
"rto_begin batch_copy",
|
|
"rto_transition batch_copy postgres_container_start",
|
|
"rto_transition postgres_container_start dump_list",
|
|
"rto_transition dump_list target_confirmation",
|
|
"rto_transition target_confirmation restore_apply",
|
|
"rto_transition restore_apply database_verify",
|
|
"rto_transition database_verify storage_attachment_verify",
|
|
"rto_transition storage_attachment_verify final_integrity",
|
|
]
|
|
positions = [text.find(value) for value in transitions]
|
|
if any(position == -1 for position in positions) or positions != sorted(positions):
|
|
raise SystemExit("RTO stage transitions missing or out of order")
|
|
target_call_position = text.find("if ! confirm_postgres_target; then")
|
|
if not positions[3] < target_call_position < positions[4]:
|
|
raise SystemExit("TCP target confirmation is outside its RTO stage boundary")
|
|
if text.count('echo "data_restore_rto_seconds=$RTO_SECONDS"') != 1:
|
|
raise SystemExit("data restore RTO must be emitted exactly once")
|
|
PY
|
|
pass "eight-stage RTO and restore diagnostics precede cleanup with one vocabulary"
|
|
|
|
cat >"$TEST_DIR/mock-target-docker" <<'MOCK_DOCKER'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
has_tcp_loopback=0
|
|
previous=""
|
|
for argument in "$@"; do
|
|
if [[ "$previous" == "-h" && "$argument" == "127.0.0.1" ]]; then
|
|
has_tcp_loopback=1
|
|
fi
|
|
previous="$argument"
|
|
done
|
|
if [[ "$has_tcp_loopback" -ne 1 ]]; then
|
|
printf '%s\n' "$MOCK_EXPECTED_DATABASE"
|
|
exit 0
|
|
fi
|
|
index="$(cat "$MOCK_QUERY_STATE")"
|
|
index="$((index + 1))"
|
|
printf '%s\n' "$index" >"$MOCK_QUERY_STATE"
|
|
line="$(sed -n "${index}p" "$MOCK_QUERY_SEQUENCE")"
|
|
[[ -n "$line" ]] || exit 99
|
|
exit_code="${line%%|*}"
|
|
result="${line#*|}"
|
|
[[ "$exit_code" =~ ^[0-9]+$ ]] || exit 98
|
|
if [[ "$exit_code" -eq 0 ]]; then
|
|
printf '%s\n' "$result"
|
|
fi
|
|
exit "$exit_code"
|
|
MOCK_DOCKER
|
|
chmod 0700 "$TEST_DIR/mock-target-docker"
|
|
|
|
capture_target_confirmation_case() {
|
|
local label="$1"
|
|
local sequence="$2"
|
|
local output_file="$3"
|
|
local sequence_file="$TEST_DIR/$label.sequence"
|
|
local state_file="$TEST_DIR/$label.state"
|
|
printf '%s' "$sequence" >"$sequence_file"
|
|
printf '0\n' >"$state_file"
|
|
(
|
|
# shellcheck disable=SC1090
|
|
source "$TEST_DIR/confirm-postgres-target.sh"
|
|
docker() {
|
|
"$TEST_DIR/mock-target-docker" "$@"
|
|
}
|
|
sleep() {
|
|
:
|
|
}
|
|
export MOCK_EXPECTED_DATABASE="$POSTGRES_DB"
|
|
export MOCK_QUERY_SEQUENCE="$sequence_file"
|
|
export MOCK_QUERY_STATE="$state_file"
|
|
private_log="$TEST_DIR/$label.private.log"
|
|
: >"$private_log"
|
|
postgres_password="$POSTGRES_PASSWORD"
|
|
CONTAINER_NAME="fixture-container"
|
|
SHADOW_DB_USER="$POSTGRES_USER"
|
|
SHADOW_DB_NAME="$POSTGRES_DB"
|
|
if confirm_postgres_target; then
|
|
function_exit=0
|
|
else
|
|
function_exit="$?"
|
|
fi
|
|
echo "function_exit=$function_exit"
|
|
echo "target_confirmation_exit_code=$TARGET_CONFIRMATION_EXIT_CODE"
|
|
echo "target_name_match=$TARGET_NAME_MATCH"
|
|
echo "target_confirmation_attempt_count=$TARGET_CONFIRMATION_ATTEMPT_COUNT"
|
|
echo "target_confirmation_success_streak=$TARGET_CONFIRMATION_SUCCESS_STREAK"
|
|
echo "target_confirmation_failure_reason=$TARGET_CONFIRMATION_FAILURE_REASON"
|
|
) >"$output_file" 2>&1
|
|
}
|
|
|
|
capture_target_confirmation_case recovery \
|
|
"$(printf '1|\n0|%s\n0|%s\n' "$POSTGRES_DB" "$POSTGRES_DB")" \
|
|
"$TEST_DIR/readiness-recovery.out"
|
|
grep -Fxq 'function_exit=0' "$TEST_DIR/readiness-recovery.out" \
|
|
|| fail "TCP target confirmation did not recover after an initial query failure"
|
|
grep -Fxq 'target_confirmation_attempt_count=3' "$TEST_DIR/readiness-recovery.out" \
|
|
|| fail "TCP target confirmation recovery attempt count mismatch"
|
|
grep -Fxq 'target_confirmation_success_streak=2' "$TEST_DIR/readiness-recovery.out" \
|
|
|| fail "TCP target confirmation recovery streak mismatch"
|
|
pass "rendered TCP target confirmation recovers only after two consecutive successes"
|
|
|
|
capture_target_confirmation_case streak-reset \
|
|
"$(printf '0|%s\n1|\n0|%s\n0|%s\n' \
|
|
"$POSTGRES_DB" "$POSTGRES_DB" "$POSTGRES_DB")" \
|
|
"$TEST_DIR/readiness-streak-reset.out"
|
|
grep -Fxq 'function_exit=0' "$TEST_DIR/readiness-streak-reset.out" \
|
|
|| fail "TCP target confirmation did not recover after streak reset"
|
|
grep -Fxq 'target_confirmation_attempt_count=4' \
|
|
"$TEST_DIR/readiness-streak-reset.out" \
|
|
|| fail "query failure did not reset the consecutive-success streak"
|
|
pass "rendered TCP target confirmation resets its streak after a query failure"
|
|
|
|
capture_target_confirmation_case name-mismatch \
|
|
'0|wrong_restore_drill
|
|
' "$TEST_DIR/readiness-name-mismatch.out"
|
|
grep -Fxq 'function_exit=1' "$TEST_DIR/readiness-name-mismatch.out" \
|
|
|| fail "target-name mismatch did not fail"
|
|
grep -Fxq 'target_name_match=no' "$TEST_DIR/readiness-name-mismatch.out" \
|
|
|| fail "target-name mismatch diagnostic missing"
|
|
grep -Fxq 'target_confirmation_attempt_count=1' \
|
|
"$TEST_DIR/readiness-name-mismatch.out" \
|
|
|| fail "target-name mismatch was retried"
|
|
grep -Fxq 'target_confirmation_failure_reason=TARGET_NAME_MISMATCH' \
|
|
"$TEST_DIR/readiness-name-mismatch.out" \
|
|
|| fail "target-name mismatch category missing"
|
|
pass "rendered TCP target confirmation fails immediately on an exact-name mismatch"
|
|
|
|
query_failures=""
|
|
for _ in $(seq 1 60); do
|
|
query_failures+=$'1|\n'
|
|
done
|
|
capture_target_confirmation_case tcp-unavailable "$query_failures" \
|
|
"$TEST_DIR/readiness-tcp-unavailable.out"
|
|
grep -Fxq 'function_exit=1' "$TEST_DIR/readiness-tcp-unavailable.out" \
|
|
|| fail "socket-only mock passed without TCP loopback readiness"
|
|
grep -Fxq 'target_confirmation_attempt_count=60' \
|
|
"$TEST_DIR/readiness-tcp-unavailable.out" \
|
|
|| fail "permanent TCP failure did not stop at the bounded attempt count"
|
|
grep -Fxq 'target_confirmation_failure_reason=TARGET_QUERY_NONZERO' \
|
|
"$TEST_DIR/readiness-tcp-unavailable.out" \
|
|
|| fail "permanent TCP failure category missing"
|
|
pass "socket-only availability cannot bypass the rendered TCP readiness gate"
|
|
|
|
docker run -d --rm --name "$CONTAINER_NAME" -p 127.0.0.1::5432 \
|
|
-e "POSTGRES_USER=$POSTGRES_USER" \
|
|
-e "POSTGRES_PASSWORD=$POSTGRES_PASSWORD" \
|
|
-e "POSTGRES_DB=$POSTGRES_DB" \
|
|
postgres:16-alpine >/dev/null
|
|
|
|
private_log="$TEST_DIR/fixture-private.log"
|
|
: >"$private_log"
|
|
postgres_password="$POSTGRES_PASSWORD"
|
|
SHADOW_DB_USER="$POSTGRES_USER"
|
|
SHADOW_DB_NAME="$POSTGRES_DB"
|
|
# shellcheck disable=SC1090
|
|
source "$TEST_DIR/confirm-postgres-target.sh"
|
|
confirm_postgres_target \
|
|
|| fail "PostgreSQL 16 fixture did not pass rendered TCP target confirmation"
|
|
pass "real PostgreSQL 16 fixture readiness uses the rendered runner implementation"
|
|
|
|
HOST_PORT="$(docker port "$CONTAINER_NAME" 5432/tcp \
|
|
| awk -F: 'NR == 1 {print $NF}')"
|
|
[[ "$HOST_PORT" =~ ^[0-9]+$ ]] \
|
|
|| fail "loopback PostgreSQL 16 port unavailable"
|
|
HOST_DATABASE_URL="postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@127.0.0.1:${HOST_PORT}/${POSTGRES_DB}"
|
|
|
|
docker exec -i "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \
|
|
-U "$POSTGRES_USER" -d "$POSTGRES_DB" <<'SQL' >/dev/null
|
|
CREATE TABLE restore_contract_marker (value text NOT NULL);
|
|
INSERT INTO restore_contract_marker(value) VALUES ('direct-real-script');
|
|
SQL
|
|
docker exec "$CONTAINER_NAME" pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
|
--format=c --no-owner --no-acl >"$TEST_DIR/fixture.dump"
|
|
docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \
|
|
-U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
|
-c 'DROP TABLE restore_contract_marker' >/dev/null
|
|
|
|
DOCKER_DATABASE_URL="postgresql://shadow.invalid/$POSTGRES_DB"
|
|
|
|
DB_TOOL_MODE=docker \
|
|
DB_CONTAINER="$CONTAINER_NAME" \
|
|
POSTGRES_DB="$POSTGRES_DB" \
|
|
POSTGRES_USER="$POSTGRES_USER" \
|
|
POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
|
DATABASE_URL="$DOCKER_DATABASE_URL" \
|
|
RESTORE_SCOPE=shadow \
|
|
RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
|
|
"$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null
|
|
[[ "$(docker exec "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
|
-Atqc 'SELECT value FROM restore_contract_marker')" \
|
|
== "direct-real-script" ]] || fail "Docker stdin restore result mismatch"
|
|
pass "real restore-db.sh Docker stdin path restores a PostgreSQL 16 custom dump"
|
|
|
|
head -c 32 "$TEST_DIR/fixture.dump" >"$TEST_DIR/corrupt.dump"
|
|
if DB_TOOL_MODE=docker \
|
|
DB_CONTAINER="$CONTAINER_NAME" \
|
|
POSTGRES_DB="$POSTGRES_DB" \
|
|
POSTGRES_USER="$POSTGRES_USER" \
|
|
POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
|
DATABASE_URL="$DOCKER_DATABASE_URL" \
|
|
RESTORE_SCOPE=shadow \
|
|
RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
|
|
"$RESTORE_DB" "$TEST_DIR/corrupt.dump" >/dev/null 2>&1; then
|
|
fail "corrupt custom dump passed list validation"
|
|
fi
|
|
pass "real restore-db.sh rejects a dump-list failure"
|
|
|
|
if DB_TOOL_MODE=docker \
|
|
DB_CONTAINER="$CONTAINER_NAME" \
|
|
POSTGRES_DB="$POSTGRES_DB" \
|
|
POSTGRES_USER="$POSTGRES_USER" \
|
|
POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
|
DATABASE_URL="$DOCKER_DATABASE_URL" \
|
|
RESTORE_SCOPE=shadow \
|
|
RESTORE_CONFIRM_DATABASE=wrong_restore_drill \
|
|
"$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null 2>&1; then
|
|
fail "wrong target confirmation was accepted"
|
|
fi
|
|
pass "real restore-db.sh rejects target confirmation mismatch"
|
|
|
|
REAL_DOCKER="$(command -v docker)"
|
|
mkdir -m 0700 "$TEST_DIR/proxy"
|
|
cat >"$TEST_DIR/proxy/docker" <<'PROXY'
|
|
#!/usr/bin/env bash
|
|
set +e
|
|
is_pg_restore=0
|
|
is_list=0
|
|
for argument in "$@"; do
|
|
[[ "$argument" == "pg_restore" ]] && is_pg_restore=1
|
|
[[ "$argument" == "--list" ]] && is_list=1
|
|
done
|
|
if [[ "$is_pg_restore" -eq 1 && "$is_list" -eq 0 ]]; then
|
|
cat >/dev/null
|
|
exit 23
|
|
fi
|
|
"$REAL_DOCKER" "$@"
|
|
exit "$?"
|
|
PROXY
|
|
chmod 0700 "$TEST_DIR/proxy/docker"
|
|
|
|
set +e
|
|
proxy_output="$(REAL_DOCKER="$REAL_DOCKER" \
|
|
PATH="$TEST_DIR/proxy:$PATH" \
|
|
DB_TOOL_MODE=docker \
|
|
DB_CONTAINER="$CONTAINER_NAME" \
|
|
POSTGRES_DB="$POSTGRES_DB" \
|
|
POSTGRES_USER="$POSTGRES_USER" \
|
|
POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \
|
|
DATABASE_URL="$DOCKER_DATABASE_URL" \
|
|
RESTORE_SCOPE=shadow \
|
|
RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
|
|
"$RESTORE_DB" "$TEST_DIR/fixture.dump" 2>&1)"
|
|
proxy_exit="$?"
|
|
set -e
|
|
[[ "$proxy_exit" -eq 23 ]] || fail "apply failure exit code was not preserved"
|
|
[[ "$proxy_output" != *"$POSTGRES_PASSWORD"* \
|
|
&& "$proxy_output" != *"$DOCKER_DATABASE_URL"* ]] \
|
|
|| fail "apply failure output exposed fixture credentials"
|
|
pass "real restore-db.sh preserves an injected apply exit and exposes no fixture secret"
|
|
|
|
mkdir -m 0700 "$TEST_DIR/host-tools"
|
|
cat >"$TEST_DIR/host-tools/psql" <<'HOST_PSQL'
|
|
#!/usr/bin/env bash
|
|
exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \
|
|
postgres:16-alpine psql "$@"
|
|
HOST_PSQL
|
|
cat >"$TEST_DIR/host-tools/pg_restore" <<'HOST_RESTORE'
|
|
#!/usr/bin/env bash
|
|
exec "$REAL_DOCKER" run --rm --network host -e PGPASSWORD \
|
|
-v "$HOST_TOOL_MOUNT:$HOST_TOOL_MOUNT:ro" \
|
|
postgres:16-alpine pg_restore "$@"
|
|
HOST_RESTORE
|
|
chmod 0700 "$TEST_DIR/host-tools/psql" "$TEST_DIR/host-tools/pg_restore"
|
|
export PGPASSWORD="$POSTGRES_PASSWORD" REAL_DOCKER HOST_TOOL_MOUNT="$TEST_DIR"
|
|
docker exec "$CONTAINER_NAME" psql -v ON_ERROR_STOP=1 \
|
|
-U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
|
-c 'DROP TABLE restore_contract_marker' >/dev/null
|
|
PATH="$TEST_DIR/host-tools:$PATH" \
|
|
DB_TOOL_MODE=host \
|
|
DATABASE_URL="$HOST_DATABASE_URL" \
|
|
RESTORE_SCOPE=shadow \
|
|
RESTORE_CONFIRM_DATABASE="$POSTGRES_DB" \
|
|
"$RESTORE_DB" "$TEST_DIR/fixture.dump" >/dev/null
|
|
[[ "$(docker exec "$CONTAINER_NAME" psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \
|
|
-Atqc 'SELECT value FROM restore_contract_marker')" == "direct-real-script" ]] \
|
|
|| fail "host-file restore result mismatch"
|
|
pass "real restore-db.sh host-file path restores through PostgreSQL 16 tools"
|
|
|
|
docker rm -f "$CONTAINER_NAME" >/dev/null
|
|
if docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then
|
|
fail "fixture container cleanup failed"
|
|
fi
|
|
CONTAINER_NAME=""
|
|
|
|
sed -n '1,/^trap finish EXIT HUP INT TERM$/p' "$RUNNER" \
|
|
| sed '$d' >"$TEST_DIR/runner-finalization-functions.sh"
|
|
|
|
(
|
|
# shellcheck disable=SC1090
|
|
source "$TEST_DIR/runner-finalization-functions.sh"
|
|
RTO_TEST_NOW=100
|
|
rto_now() {
|
|
printf '%s\n' "$RTO_TEST_NOW"
|
|
}
|
|
rto_begin batch_copy
|
|
RTO_TEST_NOW=103
|
|
rto_transition batch_copy postgres_container_start
|
|
RTO_TEST_NOW=108
|
|
rto_transition postgres_container_start dump_list
|
|
RTO_TEST_NOW=110
|
|
rto_transition dump_list target_confirmation
|
|
RTO_TEST_NOW=121
|
|
rto_transition target_confirmation restore_apply
|
|
RTO_TEST_NOW=128
|
|
rto_transition restore_apply database_verify
|
|
RTO_TEST_NOW=141
|
|
rto_transition database_verify storage_attachment_verify
|
|
RTO_TEST_NOW=158
|
|
rto_transition storage_attachment_verify final_integrity
|
|
RTO_TEST_NOW=161
|
|
rto_finalize
|
|
emit_restore_diagnostics
|
|
) >"$TEST_DIR/captured-rto-success" 2>&1
|
|
for expected in \
|
|
'rto_batch_copy_seconds=3' \
|
|
'rto_postgres_container_start_seconds=5' \
|
|
'rto_dump_list_seconds=2' \
|
|
'rto_target_confirmation_seconds=11' \
|
|
'rto_restore_apply_seconds=7' \
|
|
'rto_database_verify_seconds=13' \
|
|
'rto_storage_attachment_verify_seconds=17' \
|
|
'rto_final_integrity_seconds=3' \
|
|
'data_restore_rto_seconds=61'; do
|
|
grep -Fxq "$expected" "$TEST_DIR/captured-rto-success" \
|
|
|| fail "successful eight-stage RTO total or boundary mismatch"
|
|
done
|
|
[[ "$(grep -c '^data_restore_rto_seconds=' "$TEST_DIR/captured-rto-success")" -eq 1 ]] \
|
|
|| fail "successful RTO total was not emitted exactly once"
|
|
pass "rendered runner measures eight contiguous stages with an exact total"
|
|
|
|
set +e
|
|
(
|
|
# shellcheck disable=SC1090
|
|
source "$TEST_DIR/runner-finalization-functions.sh"
|
|
cleanup_runtime() {
|
|
echo "cleanup_marker=runtime"
|
|
return 0
|
|
}
|
|
cleanup_access() {
|
|
echo "cleanup_marker=access"
|
|
return 0
|
|
}
|
|
RTO_BATCH_COPY_SECONDS="2"
|
|
RTO_POSTGRES_CONTAINER_START_SECONDS="4"
|
|
RTO_DUMP_LIST_SECONDS="1"
|
|
RTO_ACTIVE_STAGE="target_confirmation"
|
|
RTO_STAGE_STARTED_EPOCH="200"
|
|
RTO_TEST_NOW=209
|
|
rto_now() {
|
|
printf '%s\n' "$RTO_TEST_NOW"
|
|
}
|
|
FAILURE_CATEGORY="restore_target_confirmation_failed"
|
|
DUMP_LIST_STATUS="PASS"
|
|
DUMP_LIST_EXIT_CODE="0"
|
|
TARGET_CONFIRMATION_STATUS="FAIL"
|
|
TARGET_CONFIRMATION_EXIT_CODE="1"
|
|
TARGET_CONFIRMATION_ATTEMPT_COUNT="60"
|
|
RESTORE_EXECUTION_PATH="DOCKER_STDIN"
|
|
RESTORE_ERROR_CATEGORY="TARGET_QUERY_NONZERO"
|
|
trap finish EXIT
|
|
exit 23
|
|
) >"$TEST_DIR/captured-rto-target-failure" 2>&1
|
|
rto_target_failure_exit="$?"
|
|
set -e
|
|
[[ "$rto_target_failure_exit" -eq 23 ]] \
|
|
|| fail "target-confirmation timing failure did not preserve exit code"
|
|
for expected in \
|
|
'rto_batch_copy_seconds=2' \
|
|
'rto_postgres_container_start_seconds=4' \
|
|
'rto_dump_list_seconds=1' \
|
|
'rto_target_confirmation_seconds=9' \
|
|
'rto_restore_apply_seconds=NOT_RUN' \
|
|
'rto_database_verify_seconds=NOT_RUN' \
|
|
'rto_storage_attachment_verify_seconds=NOT_RUN' \
|
|
'rto_final_integrity_seconds=NOT_RUN' \
|
|
'data_restore_rto_seconds=16'; do
|
|
grep -Fxq "$expected" "$TEST_DIR/captured-rto-target-failure" \
|
|
|| fail "target-confirmation failure RTO boundary mismatch"
|
|
done
|
|
rto_line="$(grep -n '^rto_target_confirmation_seconds=9$' \
|
|
"$TEST_DIR/captured-rto-target-failure" | cut -d: -f1)"
|
|
rto_cleanup_line="$(grep -n '^cleanup_marker=runtime$' \
|
|
"$TEST_DIR/captured-rto-target-failure" | cut -d: -f1)"
|
|
[[ "$rto_line" =~ ^[0-9]+$ && "$rto_cleanup_line" =~ ^[0-9]+$ \
|
|
&& "$rto_line" -lt "$rto_cleanup_line" ]] \
|
|
|| fail "failed-stage RTO was not emitted before teardown"
|
|
pass "target-confirmation failure preserves elapsed stages and later NOT_RUN before cleanup"
|
|
|
|
capture_finalization_case() {
|
|
local output_file="$1"
|
|
local failure_category="$2"
|
|
local dump_status="$3"
|
|
local dump_exit="$4"
|
|
local target_status="$5"
|
|
local apply_status="$6"
|
|
local script_exit="$7"
|
|
local pg_restore_exit="$8"
|
|
local error_count="$9"
|
|
local error_category="${10}"
|
|
set +e
|
|
(
|
|
# shellcheck disable=SC1090
|
|
source "$TEST_DIR/runner-finalization-functions.sh"
|
|
cleanup_runtime() {
|
|
echo "cleanup_marker=runtime"
|
|
return 0
|
|
}
|
|
cleanup_access() {
|
|
echo "cleanup_marker=access"
|
|
return 0
|
|
}
|
|
FAILURE_CATEGORY="$failure_category"
|
|
DUMP_LIST_STATUS="$dump_status"
|
|
DUMP_LIST_EXIT_CODE="$dump_exit"
|
|
TARGET_CONFIRMATION_STATUS="$target_status"
|
|
case "$target_status" in
|
|
PASS)
|
|
TARGET_CONFIRMATION_EXIT_CODE="0"
|
|
TARGET_NAME_MATCH="yes"
|
|
TARGET_CONFIRMATION_ATTEMPT_COUNT="2"
|
|
TARGET_CONFIRMATION_SUCCESS_STREAK="2"
|
|
;;
|
|
FAIL)
|
|
TARGET_CONFIRMATION_EXIT_CODE="1"
|
|
TARGET_NAME_MATCH="NOT_RUN"
|
|
TARGET_CONFIRMATION_ATTEMPT_COUNT="60"
|
|
TARGET_CONFIRMATION_SUCCESS_STREAK="0"
|
|
;;
|
|
esac
|
|
RESTORE_EXECUTION_PATH="DOCKER_STDIN"
|
|
RESTORE_APPLY_STATUS="$apply_status"
|
|
RESTORE_SCRIPT_EXIT_CODE="$script_exit"
|
|
PG_RESTORE_EXIT_CODE="$pg_restore_exit"
|
|
PG_RESTORE_ERROR_COUNT="$error_count"
|
|
RESTORE_ERROR_CATEGORY="$error_category"
|
|
trap finish EXIT
|
|
exit 23
|
|
) >"$output_file" 2>&1
|
|
local case_exit="$?"
|
|
set -e
|
|
[[ "$case_exit" -eq 23 ]] \
|
|
|| fail "finalization did not preserve the injected exit code"
|
|
}
|
|
|
|
capture_finalization_case \
|
|
"$TEST_DIR/captured-dump-list-failure" \
|
|
dump_list_failed FAIL 1 NOT_RUN NOT_RUN NOT_RUN NOT_RUN NOT_RUN \
|
|
DUMP_LIST_NONZERO
|
|
grep -Fxq 'dump_list=FAIL' "$TEST_DIR/captured-dump-list-failure" \
|
|
|| fail "dump-list failure stage missing"
|
|
grep -Fxq 'target_confirmation=NOT_RUN' \
|
|
"$TEST_DIR/captured-dump-list-failure" \
|
|
|| fail "target stage ran after dump-list failure"
|
|
grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-dump-list-failure" \
|
|
|| fail "apply stage ran after dump-list failure"
|
|
grep -Fxq 'rto_batch_copy_seconds=NOT_RUN' \
|
|
"$TEST_DIR/captured-dump-list-failure" \
|
|
|| fail "pre-RTO failure unexpectedly reported a measured stage"
|
|
grep -Fxq 'data_restore_rto_seconds=NOT_RUN' \
|
|
"$TEST_DIR/captured-dump-list-failure" \
|
|
|| fail "pre-RTO failure unexpectedly reported a measured total"
|
|
pass "pre-RTO failure keeps timing, target and apply fields NOT_RUN"
|
|
|
|
capture_finalization_case \
|
|
"$TEST_DIR/captured-target-failure" \
|
|
restore_target_confirmation_failed PASS 0 FAIL NOT_RUN NOT_RUN NOT_RUN \
|
|
NOT_RUN TARGET_QUERY_NONZERO
|
|
grep -Fxq 'target_confirmation=FAIL' "$TEST_DIR/captured-target-failure" \
|
|
|| fail "target-confirmation failure stage missing"
|
|
grep -Fxq 'target_confirmation_exit_code=1' \
|
|
"$TEST_DIR/captured-target-failure" \
|
|
|| fail "target-confirmation failure exit code missing"
|
|
grep -Fxq 'target_confirmation_attempt_count=60' \
|
|
"$TEST_DIR/captured-target-failure" \
|
|
|| fail "target-confirmation attempt count missing"
|
|
grep -Fxq 'restore_apply=NOT_RUN' "$TEST_DIR/captured-target-failure" \
|
|
|| fail "apply stage ran after target-confirmation failure"
|
|
pass "target-confirmation failure keeps apply stage NOT_RUN"
|
|
|
|
set +e
|
|
(
|
|
# shellcheck disable=SC1090
|
|
source "$TEST_DIR/runner-finalization-functions.sh"
|
|
cleanup_runtime() {
|
|
echo "cleanup_marker=runtime"
|
|
return 0
|
|
}
|
|
cleanup_access() {
|
|
echo "cleanup_marker=access"
|
|
return 0
|
|
}
|
|
FAILURE_CATEGORY="restore_apply_failed_after_list_pass"
|
|
DUMP_LIST_STATUS="PASS"
|
|
DUMP_LIST_EXIT_CODE="0"
|
|
TARGET_CONFIRMATION_STATUS="PASS"
|
|
TARGET_CONFIRMATION_EXIT_CODE="0"
|
|
TARGET_NAME_MATCH="yes"
|
|
TARGET_CONFIRMATION_ATTEMPT_COUNT="2"
|
|
TARGET_CONFIRMATION_SUCCESS_STREAK="2"
|
|
RESTORE_EXECUTION_PATH="DOCKER_STDIN"
|
|
RESTORE_APPLY_STATUS="FAIL"
|
|
RESTORE_SCRIPT_EXIT_CODE="23"
|
|
PG_RESTORE_EXIT_CODE="23"
|
|
PG_RESTORE_ERROR_COUNT="UNKNOWN"
|
|
RESTORE_ERROR_CATEGORY="RESTORE_APPLY_NONZERO"
|
|
RTO_BATCH_COPY_SECONDS="2"
|
|
RTO_POSTGRES_CONTAINER_START_SECONDS="4"
|
|
RTO_DUMP_LIST_SECONDS="1"
|
|
RTO_TARGET_CONFIRMATION_SECONDS="9"
|
|
RTO_ACTIVE_STAGE="restore_apply"
|
|
RTO_STAGE_STARTED_EPOCH="300"
|
|
RTO_TEST_NOW=312
|
|
rto_now() {
|
|
printf '%s\n' "$RTO_TEST_NOW"
|
|
}
|
|
trap finish EXIT
|
|
exit 23
|
|
) >"$TEST_DIR/captured-finalization" 2>&1
|
|
finalization_exit="$?"
|
|
set -e
|
|
[[ "$finalization_exit" -eq 23 ]] \
|
|
|| fail "finalization did not preserve the injected exit code"
|
|
diagnostic_line="$(grep -n '^restore_apply=FAIL$' \
|
|
"$TEST_DIR/captured-finalization" | cut -d: -f1)"
|
|
cleanup_line="$(grep -n '^cleanup_marker=runtime$' \
|
|
"$TEST_DIR/captured-finalization" | cut -d: -f1)"
|
|
[[ "$diagnostic_line" =~ ^[0-9]+$ && "$cleanup_line" =~ ^[0-9]+$ \
|
|
&& "$diagnostic_line" -lt "$cleanup_line" ]] \
|
|
|| fail "stage diagnostics were not emitted before teardown"
|
|
grep -Fxq 'runtime_cleanup=PASS' "$TEST_DIR/captured-finalization" \
|
|
|| fail "runtime teardown result was not retained"
|
|
grep -Fxq 'temporary_access_cleanup=PASS' "$TEST_DIR/captured-finalization" \
|
|
|| fail "access teardown result was not retained"
|
|
grep -Fxq 'pg_restore_exit_code=23' "$TEST_DIR/captured-finalization" \
|
|
|| fail "pg_restore exit code disappeared after teardown"
|
|
grep -Fxq 'rto_restore_apply_seconds=12' "$TEST_DIR/captured-finalization" \
|
|
|| fail "restore-apply failure duration missing"
|
|
grep -Fxq 'rto_database_verify_seconds=NOT_RUN' "$TEST_DIR/captured-finalization" \
|
|
|| fail "database verification ran after restore-apply failure"
|
|
grep -Fxq 'data_restore_rto_seconds=28' "$TEST_DIR/captured-finalization" \
|
|
|| fail "restore-apply failure RTO total mismatch"
|
|
pass "injected restore-apply failure preserves diagnostics and RTO before teardown"
|
|
|
|
if grep -Eiq 'postgresql://|SELECT[[:space:]]|/tmp/|local_fixture_password' \
|
|
"$TEST_DIR/captured-dump-list-failure" \
|
|
"$TEST_DIR/captured-target-failure" \
|
|
"$TEST_DIR/captured-finalization"; then
|
|
fail "allowlisted diagnostics exposed a path, credential or SQL"
|
|
fi
|
|
pass "captured stage diagnostics contain no path, credential or SQL"
|
|
|
|
echo "[runner-contract][PASS] all off-host shadow-restore runner contracts completed"
|