wangsli/hyb-oa
1
0
派生 0
代码 工单 合并请求 工作流 8 软件包 项目 发布 百科 活动
文件
4a5cd8f2b3e04da24551ddf1ee99b4167aa76710
hyb-oa/scripts/qa/run-deployment-contract-refresh-contract.sh
T
wangsli和Claude Opus 5.5 1989f666ac 补齐手动快照模式回归测试与切换说明（08-28-production-arm-backup-transfer-incident） 
- 合同测试补齐：watchdog/drill 服务仍运行、timer 已 disable 但仍 active、
  断链模式文件三类场景，probe 与 refresh 两阶段均断言
- 三个源码变异在旧测试中全部放过、新测试全部拦住
- 切换说明补充超时处置需预先批准、安装前同步并核验源码、
  回滚时 Persistent timer 可能立即补跑并告警
- 报错分类细化（复核第 5 条）后置；未推送、未做远端切换

Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-06 10:51:45 +08:00

282 行
11 KiB
Bash
可执行文件
原始文件 Blame 文件历史

#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
CHECKER="$ROOT_DIR/deploy/trialrun/check-existing-deployment.sh"
HELPER="$ROOT_DIR/scripts/refresh-deployment-contract.sh"
LIBRARY="$ROOT_DIR/deploy/trialrun/deployment-contract-lib.sh"
INSTALLER="$ROOT_DIR/deploy/trialrun/install-deployment-contract-refresh.sh"
FIXTURE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/hyb-oa-contract-refresh.XXXXXX")"
trap 'rm -rf -- "$FIXTURE_ROOT"' EXIT
mkdir -p "$FIXTURE_ROOT/bin" "$FIXTURE_ROOT/etc/systemd/system" "$FIXTURE_ROOT/run/lock"
cat >"$FIXTURE_ROOT/bin/ssh" <<'SH'
#!/usr/bin/env bash
set -uo pipefail
target="${1:-}"
shift || true
mode="${FIXTURE_SSH_MODE:-pass}"
if [[ "$target" == "deploy" && "${1:-}" == "true" ]]; then
[[ "$mode" != "deploy_transport_fail" ]] || exit 255
exit 0
fi
if [[ "$target" == "deploy" && "$*" == *"systemctl show"* ]]; then
[[ "$mode" != "contract_fail" ]] || exit 1
printf '%s\n' 'REMOTE_DIR=/srv/app REMOTE_ENV_FILE=/srv/app/deploy/trialrun/.env COMPOSE_FILE=deploy/trialrun/docker-compose.yml COMPOSE_PROJECT_NAME=fixture POSTGRES_DATA_DIR=/srv/data/postgres STORAGE_HOST_DIR=/srv/data/storage BUILD_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'
exit 0
fi
if [[ "$target" == "deploy" ]]; then
if [[ "$mode" == "multi_fail" ]]; then
cat <<'OUT'
repository_root=FAIL
repository_writable=PASS
env_readable=FAIL
data_directories=PASS
runtime_unique=PASS
runtime_build=PASS
storage_mount=FAIL
pgdata_mount=PASS
backup_build=PASS
OUT
else
for item in repository_root repository_writable env_readable data_directories runtime_unique runtime_build storage_mount pgdata_mount backup_build; do
printf '%s=PASS\n' "$item"
done
fi
exit 0
fi
if [[ "$target" == "refresh" && "${1:-}" == "probe" ]]; then
case "$mode" in
root_transport_fail) exit 255 ;;
program_missing|multi_fail) exit 127 ;;
*) echo '[deployment-contract-refresh][PASS] probe'; exit 0 ;;
esac
fi
exit 1
SH
chmod 0755 "$FIXTURE_ROOT/bin/ssh"
run_checker() {
PATH="$FIXTURE_ROOT/bin:$PATH" DEPLOY_SERVER=deploy DEPLOY_REFRESH_SERVER=refresh \
FIXTURE_SSH_MODE="${1:-pass}" bash "$CHECKER"
}
pass_output="$(run_checker pass)"
[[ "$pass_output" == *"summary PASS=13 FAIL=0"* ]]
set +e
multi_output="$(run_checker multi_fail 2>&1)"
multi_status=$?
set -e
[[ "$multi_status" -ne 0 ]]
[[ "$multi_output" == *"repository_root FAIL"* \
&& "$multi_output" == *"env_readable FAIL"* \
&& "$multi_output" == *"storage_mount FAIL"* \
&& "$multi_output" == *"refresh_program FAIL"* \
&& "$multi_output" == *"summary PASS=9 FAIL=4"* ]]
[[ "$multi_output" != *"/srv/"* && "$multi_output" != *"aaaaaaaaaaaaaaaa"* ]]
set +e
root_output="$(run_checker root_transport_fail 2>&1)"
root_status=$?
set -e
[[ "$root_status" -ne 0 && "$root_output" == *"root_ssh_identity FAIL"* \
&& "$root_output" == *"refresh_program FAIL"* ]]
source "$LIBRARY"
[[ "$(deployment_contract_refresh_category 255 '')" == "root_identity_unavailable" ]]
[[ "$(deployment_contract_refresh_category 127 '')" == "refresh_program_unavailable" ]]
[[ "$(deployment_contract_refresh_category 42 '')" == "unit_build_not_refreshed" ]]
[[ "$(deployment_contract_refresh_category 43 '')" == "timer_not_active" ]]
[[ "$(deployment_contract_refresh_category 0 '[deployment-contract-refresh][PASS] refreshed')" == "success" ]]
for service in \
hyb-oa-automatic-backup-daily.service \
hyb-oa-automatic-backup-watchdog.service \
hyb-oa-automatic-backup-drill-reminder.service \
hyb-oa-warch3-ops.service; do
: >"$FIXTURE_ROOT/etc/systemd/system/$service"
done
cat >"$FIXTURE_ROOT/bin/docker" <<'SH'
#!/usr/bin/env bash
if [[ "${1:-}" == "ps" ]]; then
echo server-fixture
elif [[ "${1:-}" == "inspect" ]]; then
echo 'BUILD_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'
else
exit 1
fi
SH
chmod 0755 "$FIXTURE_ROOT/bin/docker"
cat >"$FIXTURE_ROOT/bin/systemctl" <<'SH'
#!/usr/bin/env bash
set -euo pipefail
root="${HYB_CONTRACT_REFRESH_TEST_ROOT:?}"
action="${1:-}"
case "$action" in
show)
unit="${2:-}"
property="${4:-}"
case "$property" in
LoadState) echo loaded ;;
FragmentPath) echo "$root/etc/systemd/system/$unit" ;;
Environment)
echo 'COMPOSE_PROJECT_NAME=fixture BUILD_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa VITE_BUILD_SHA=aaaaaaaaaaaa OLD_BUILD_SHA=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb OLD_VITE_BUILD_SHA=bbbbbbbbbbbb'
if [[ "${FIXTURE_IGNORE_DROPINS:-0}" != "1" ]]; then
dropin="$root/etc/systemd/system/${unit}.d/20-active-build.conf"
[[ ! -f "$dropin" ]] || sed -n 's/^Environment=//p' "$dropin" | paste -sd' ' -
fi
;;
*) exit 1 ;;
esac
;;
daemon-reload) exit 0 ;;
is-enabled)
unit="${2:-}"
if [[ "$unit" == hyb-oa-automatic-backup-* ]]; then
case "${FIXTURE_BACKUP_TIMERS:-running}" in
disabled|disabled_active) echo disabled; exit 1 ;;
mixed)
if [[ "$unit" == *watchdog* ]]; then echo enabled; else echo disabled; exit 1; fi
;;
*) echo enabled ;;
esac
else
echo enabled
fi
;;
is-active)
unit="${2:-}"
if [[ "$unit" == *.service ]]; then
case "$unit" in
hyb-oa-automatic-backup-daily.service) state="${FIXTURE_DAILY_SERVICE_STATE:-inactive}" ;;
hyb-oa-automatic-backup-watchdog.service) state="${FIXTURE_WATCHDOG_SERVICE_STATE:-inactive}" ;;
hyb-oa-automatic-backup-drill-reminder.service) state="${FIXTURE_DRILL_SERVICE_STATE:-inactive}" ;;
*) state=inactive ;;
esac
echo "$state"
[[ "$state" == "active" ]] || exit 3
exit 0
fi
if [[ "${FIXTURE_TIMER_INACTIVE:-0}" == "1" ]]; then echo inactive; exit 3; fi
if [[ "$unit" == hyb-oa-automatic-backup-* ]]; then
case "${FIXTURE_BACKUP_TIMERS:-running}" in
disabled) echo inactive; exit 3 ;;
disabled_active) echo active ;;
mixed)
if [[ "$unit" == *watchdog* ]]; then echo active; else echo inactive; exit 3; fi
;;
esac
elif [[ "${FIXTURE_OPS_TIMER_INACTIVE:-0}" == "1" ]]; then
echo inactive; exit 3
fi
echo active
;;
*) exit 1 ;;
esac
SH
chmod 0755 "$FIXTURE_ROOT/bin/systemctl"
probe_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" SSH_ORIGINAL_COMMAND=probe bash "$HELPER")"
[[ "$probe_output" == '[deployment-contract-refresh][PASS] probe' ]]
refresh_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" SSH_ORIGINAL_COMMAND=refresh bash "$HELPER")"
[[ "$refresh_output" == '[deployment-contract-refresh][PASS] refreshed' ]]
[[ "$(find "$FIXTURE_ROOT/etc/systemd/system" -name 20-active-build.conf -type f | wc -l)" -eq 4 ]]
set +e
HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_IGNORE_DROPINS=1 \
SSH_ORIGINAL_COMMAND=refresh bash "$HELPER" >/dev/null 2>&1
unit_status=$?
HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_TIMER_INACTIVE=1 \
SSH_ORIGINAL_COMMAND=refresh bash "$HELPER" >/dev/null 2>&1
timer_status=$?
HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" \
SSH_ORIGINAL_COMMAND='refresh extra' bash "$HELPER" >/dev/null 2>&1
unsupported_status=$?
set -e
[[ "$unit_status" -eq 42 && "$timer_status" -eq 43 && "$unsupported_status" -eq 64 ]]
run_refresh_status() {
local command="$1"
shift
set +e
env HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" "$@" \
SSH_ORIGINAL_COMMAND="$command" bash "$HELPER" >/dev/null 2>&1
local status=$?
set -e
printf '%s' "$status"
}
mode_file="$FIXTURE_ROOT/etc/hyb-oa-backup-mode"
# Automatic mode (no mode file): disabled backup timers stay fail-closed,
# both before (probe) and after (refresh) a release.
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 43 ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 43 ]]
[[ "$(run_refresh_status probe FIXTURE_TIMER_INACTIVE=1)" -eq 43 ]]
printf 'manual_snapshot\n' >"$mode_file"
manual_output="$(HYB_CONTRACT_REFRESH_TEST_ROOT="$FIXTURE_ROOT" FIXTURE_BACKUP_TIMERS=disabled \
SSH_ORIGINAL_COMMAND=refresh bash "$HELPER")"
[[ "$manual_output" == '[deployment-contract-refresh][PASS] refreshed' ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 0 ]]
# Manual mode must not silently accept running or partially running backup timers,
# and the pre-deploy probe must reject the mismatch before any release write.
[[ "$(run_refresh_status refresh)" -eq 45 ]]
[[ "$(run_refresh_status probe)" -eq 45 ]]
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=mixed)" -eq 45 ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=mixed)" -eq 45 ]]
# Disabled timers with an in-flight backup run are not a finished switch.
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=active)" -eq 45 ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=activating)" -eq 45 ]]
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_DAILY_SERVICE_STATE=failed)" -eq 0 ]]
# Each non-daily service can still be running independently of the daily job.
# disable without --now also leaves a timer active despite its disabled state.
for phase in probe refresh; do
for service_state in FIXTURE_WATCHDOG_SERVICE_STATE FIXTURE_DRILL_SERVICE_STATE; do
[[ "$(run_refresh_status "$phase" FIXTURE_BACKUP_TIMERS=disabled "$service_state=active")" -eq 45 ]]
done
[[ "$(run_refresh_status "$phase" FIXTURE_BACKUP_TIMERS=disabled_active)" -eq 45 ]]
done
# Non-backup ops timer is still required in manual mode.
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled FIXTURE_OPS_TIMER_INACTIVE=1)" -eq 43 ]]
printf 'manual\n' >"$mode_file"
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]]
[[ "$(run_refresh_status probe FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]]
printf 'manual_snapshot\nautomatic\n' >"$mode_file"
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]]
rm -f -- "$mode_file"
printf 'manual_snapshot\n' >"$FIXTURE_ROOT/backup-mode-target"
ln -s "$FIXTURE_ROOT/backup-mode-target" "$mode_file"
[[ "$(run_refresh_status refresh FIXTURE_BACKUP_TIMERS=disabled)" -eq 44 ]]
# A dangling link is invalid, not an absent file selecting automatic mode.
rm -f -- "$FIXTURE_ROOT/backup-mode-target"
[[ -L "$mode_file" && ! -e "$mode_file" ]]
[[ "$(run_refresh_status probe)" -eq 44 ]]
[[ "$(run_refresh_status refresh)" -eq 44 ]]
rm -f -- "$mode_file"
[[ "$(run_refresh_status refresh)" -eq 0 ]]
[[ "$(deployment_contract_refresh_category 44 '')" == "backup_mode_invalid" ]]
[[ "$(deployment_contract_refresh_category 45 '')" == "backup_timer_not_disabled" ]]
public_key_file="$FIXTURE_ROOT/refresh.pub"
key_blob='AAAAC3NzaC1lZDI1NTE5AAAAIFixturerefreshkeymaterial000000000000'
printf 'ssh-ed25519 %s fixture-refresh\n' "$key_blob" >"$public_key_file"
installer_output="$(HYB_CONTRACT_REFRESH_INSTALL_TEST_ROOT="$FIXTURE_ROOT" \
DEPLOY_REFRESH_PUBLIC_KEY_FILE="$public_key_file" bash "$INSTALLER" install)"
[[ "$installer_output" == '[deployment-contract-refresh-installer][PASS] forced command installed' ]]
authorized_keys="$FIXTURE_ROOT/root/.ssh/authorized_keys"
[[ "$(grep -Fc '# hyb-oa-managed-deployment-contract-refresh' "$authorized_keys")" -eq 1 ]]
printf 'ssh-ed25519 %s unrestricted-duplicate\n' "$key_blob" >>"$authorized_keys"
set +e
HYB_CONTRACT_REFRESH_INSTALL_TEST_ROOT="$FIXTURE_ROOT" \
DEPLOY_REFRESH_PUBLIC_KEY_FILE="$public_key_file" bash "$INSTALLER" install >/dev/null 2>&1
duplicate_status=$?
set -e
[[ "$duplicate_status" -ne 0 ]]
echo '[deployment-contract-refresh-contract][PASS] aggregate and forced-command contracts verified'
在新工单中引用 查看 Git Blame 复制永久链接