实现应用健康监控（07-27-app-health-monitoring）
这个提交包含在：
@@ -28,6 +28,7 @@ echo "=== [2/7] Governance + Sensitive Content ==="
|
||||
cd "$ROOT_DIR"
|
||||
node scripts/qa/check-trellis-governance.mjs
|
||||
node scripts/qa/check-sensitive-content.mjs
|
||||
bash scripts/qa/run-app-health-monitor-contract.sh
|
||||
bash -n scripts/qa/run-live-smoke-readiness.sh
|
||||
bash -n deploy/trialrun/deploy.sh
|
||||
)
|
||||
|
||||
+283
@@ -0,0 +1,283 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Probe the public OA readiness endpoint and drive a deduplicated alert."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import ssl
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SAFE_TOKEN = re.compile(r"^[A-Za-z0-9_-]{1,160}$")
|
||||
SAFE_HOST = re.compile(r"^[A-Za-z0-9.-]{1,253}$")
|
||||
MAX_RESPONSE_BYTES = 64 * 1024
|
||||
HEALTHY_STATUS = "PASS_APP_HEALTH_READY"
|
||||
FAILURE_STATUS = "FAIL_APP_HEALTH"
|
||||
|
||||
|
||||
class ProbeFailure(Exception):
|
||||
def __init__(self, category: str):
|
||||
super().__init__(category)
|
||||
self.category = category
|
||||
|
||||
|
||||
def fail(category: str) -> "NoReturn":
|
||||
print(f"[app-health][ERROR] {category}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def validate_regular_file(path: Path, label: str, private: bool = False) -> None:
|
||||
try:
|
||||
info = path.lstat()
|
||||
except FileNotFoundError:
|
||||
fail(f"{label}_missing")
|
||||
if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode):
|
||||
fail(f"{label}_invalid")
|
||||
if info.st_uid != os.geteuid():
|
||||
fail(f"{label}_owner_invalid")
|
||||
if private and stat.S_IMODE(info.st_mode) != 0o600:
|
||||
fail(f"{label}_mode_invalid")
|
||||
if not private and stat.S_IMODE(info.st_mode) & 0o022:
|
||||
fail(f"{label}_mode_invalid")
|
||||
|
||||
|
||||
def prepare_state_dir(path: Path) -> None:
|
||||
try:
|
||||
path.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
info = path.lstat()
|
||||
except OSError:
|
||||
fail("local_state_invalid")
|
||||
if (
|
||||
stat.S_ISLNK(info.st_mode)
|
||||
or not stat.S_ISDIR(info.st_mode)
|
||||
or info.st_uid != os.geteuid()
|
||||
or stat.S_IMODE(info.st_mode) != 0o700
|
||||
):
|
||||
fail("local_state_invalid")
|
||||
|
||||
|
||||
def validate_ready_url(value: str) -> str:
|
||||
try:
|
||||
parsed = urllib.parse.urlsplit(value)
|
||||
port = parsed.port
|
||||
except ValueError:
|
||||
fail("ready_url_invalid")
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or not parsed.hostname
|
||||
or not SAFE_HOST.fullmatch(parsed.hostname)
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or parsed.path != "/ready"
|
||||
or parsed.query
|
||||
or parsed.fragment
|
||||
or (port is not None and not 1 <= port <= 65535)
|
||||
):
|
||||
fail("ready_url_invalid")
|
||||
return value
|
||||
|
||||
|
||||
def load_probe_state(path: Path) -> dict[str, object]:
|
||||
if not path.exists():
|
||||
return {"schemaVersion": 1, "consecutiveFailures": 0, "lastCategory": None}
|
||||
validate_regular_file(path, "local_state", private=True)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
fail("local_state_invalid")
|
||||
if not isinstance(payload, dict):
|
||||
fail("local_state_invalid")
|
||||
count = payload.get("consecutiveFailures")
|
||||
category = payload.get("lastCategory")
|
||||
if (
|
||||
payload.get("schemaVersion") != 1
|
||||
or not isinstance(count, int)
|
||||
or isinstance(count, bool)
|
||||
or count < 0
|
||||
or (category is not None and (not isinstance(category, str) or not SAFE_TOKEN.fullmatch(category)))
|
||||
):
|
||||
fail("local_state_invalid")
|
||||
return payload
|
||||
|
||||
|
||||
def atomic_write(path: Path, payload: dict[str, object]) -> None:
|
||||
fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
|
||||
try:
|
||||
os.fchmod(fd, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle, ensure_ascii=True, indent=2)
|
||||
handle.write("\n")
|
||||
os.replace(temporary, path)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(temporary)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def classify_url_error(error: urllib.error.URLError) -> str:
|
||||
reason = error.reason
|
||||
if isinstance(reason, (ssl.SSLError, ssl.CertificateError)):
|
||||
return "tls_validation_failed"
|
||||
if isinstance(reason, (socket.timeout, TimeoutError)):
|
||||
return "request_timeout"
|
||||
return "dns_or_connect_failed"
|
||||
|
||||
|
||||
def probe(ready_url: str, timeout_seconds: float, ca_file: Path | None) -> None:
|
||||
if ca_file is not None:
|
||||
validate_regular_file(ca_file, "ca_file")
|
||||
try:
|
||||
context = ssl.create_default_context(cafile=str(ca_file) if ca_file else None)
|
||||
request = urllib.request.Request(
|
||||
ready_url,
|
||||
headers={"Accept": "application/json", "User-Agent": "hyb-oa-app-health/1"},
|
||||
method="GET",
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=timeout_seconds, context=context) as response:
|
||||
if not 200 <= response.status < 300:
|
||||
raise ProbeFailure("http_unavailable")
|
||||
content_length = response.headers.get("Content-Length")
|
||||
if content_length:
|
||||
try:
|
||||
if int(content_length) > MAX_RESPONSE_BYTES:
|
||||
raise ProbeFailure("response_too_large")
|
||||
except ValueError:
|
||||
raise ProbeFailure("response_contract_invalid") from None
|
||||
body = response.read(MAX_RESPONSE_BYTES + 1)
|
||||
except urllib.error.HTTPError:
|
||||
raise ProbeFailure("http_unavailable") from None
|
||||
except urllib.error.URLError as error:
|
||||
raise ProbeFailure(classify_url_error(error)) from None
|
||||
except (socket.timeout, TimeoutError):
|
||||
raise ProbeFailure("request_timeout") from None
|
||||
except (ssl.SSLError, ssl.CertificateError):
|
||||
raise ProbeFailure("tls_validation_failed") from None
|
||||
except (http.client.HTTPException, ConnectionError, OSError):
|
||||
raise ProbeFailure("dns_or_connect_failed") from None
|
||||
if len(body) > MAX_RESPONSE_BYTES:
|
||||
raise ProbeFailure("response_too_large")
|
||||
try:
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, json.JSONDecodeError):
|
||||
raise ProbeFailure("response_contract_invalid") from None
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict) or payload["data"].get("ok") is not True:
|
||||
raise ProbeFailure("response_contract_invalid")
|
||||
|
||||
|
||||
def write_summary(path: Path, status: str, category: str) -> None:
|
||||
finding = status == FAILURE_STATUS
|
||||
payload = {
|
||||
"issueId": "APP_HEALTH",
|
||||
"status": status,
|
||||
"conclusion": status,
|
||||
"findingCount": 1 if finding else 0,
|
||||
"findings": (
|
||||
[{"source": "public_ready", "status": status, "severity": "P0", "detail": category}]
|
||||
if finding
|
||||
else []
|
||||
),
|
||||
"privacy": "status/count/category only; no credentials, addresses, paths, or payloads",
|
||||
}
|
||||
atomic_write(path, payload)
|
||||
|
||||
|
||||
def evaluate_alert(alert_helper: Path, webhook_file: Path, summary_file: Path, alert_state_file: Path) -> None:
|
||||
validate_regular_file(alert_helper, "alert_helper")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(alert_helper),
|
||||
"--webhook-file",
|
||||
str(webhook_file),
|
||||
"--summary-file",
|
||||
str(summary_file),
|
||||
"--state-file",
|
||||
str(alert_state_file),
|
||||
],
|
||||
check=False,
|
||||
)
|
||||
except OSError:
|
||||
fail("alert_evaluation_failed")
|
||||
if result.returncode != 0:
|
||||
fail("alert_evaluation_failed")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--ready-url", required=True)
|
||||
parser.add_argument("--state-dir", required=True, type=Path)
|
||||
parser.add_argument("--webhook-file", required=True, type=Path)
|
||||
parser.add_argument("--alert-helper", required=True, type=Path)
|
||||
parser.add_argument("--failure-threshold", type=int, default=2)
|
||||
parser.add_argument("--timeout-seconds", type=float, default=10.0)
|
||||
parser.add_argument("--ca-file", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
ready_url = validate_ready_url(args.ready_url)
|
||||
if not 1 <= args.failure_threshold <= 10:
|
||||
fail("failure_threshold_invalid")
|
||||
if not 0.1 <= args.timeout_seconds <= 30:
|
||||
fail("timeout_invalid")
|
||||
prepare_state_dir(args.state_dir)
|
||||
probe_state_file = args.state_dir / "app-health-probe-state.json"
|
||||
summary_file = args.state_dir / "last-app-health.json"
|
||||
alert_state_file = args.state_dir / "app-health-alert-state.json"
|
||||
state = load_probe_state(probe_state_file)
|
||||
|
||||
try:
|
||||
probe(ready_url, args.timeout_seconds, args.ca_file)
|
||||
except ProbeFailure as error:
|
||||
failures = min(int(state["consecutiveFailures"]) + 1, args.failure_threshold)
|
||||
try:
|
||||
atomic_write(
|
||||
probe_state_file,
|
||||
{"schemaVersion": 1, "consecutiveFailures": failures, "lastCategory": error.category},
|
||||
)
|
||||
except OSError:
|
||||
fail("local_state_invalid")
|
||||
if failures >= args.failure_threshold:
|
||||
try:
|
||||
write_summary(summary_file, FAILURE_STATUS, error.category)
|
||||
except OSError:
|
||||
fail("local_state_invalid")
|
||||
evaluate_alert(args.alert_helper, args.webhook_file, summary_file, alert_state_file)
|
||||
print(
|
||||
f"[app-health][ERROR] health_check_failed category={error.category} consecutive={failures} alert=evaluated",
|
||||
file=sys.stderr,
|
||||
)
|
||||
else:
|
||||
print(
|
||||
f"[app-health][ERROR] health_check_failed category={error.category} consecutive={failures} alert=pending",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
|
||||
try:
|
||||
atomic_write(
|
||||
probe_state_file,
|
||||
{"schemaVersion": 1, "consecutiveFailures": 0, "lastCategory": None},
|
||||
)
|
||||
write_summary(summary_file, HEALTHY_STATUS, "public_ready_healthy")
|
||||
except OSError:
|
||||
fail("local_state_invalid")
|
||||
evaluate_alert(args.alert_helper, args.webhook_file, summary_file, alert_state_file)
|
||||
print("[app-health][PASS] public readiness is healthy")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,276 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
TEST_DIR="$(mktemp -d)"
|
||||
STUB_PID=""
|
||||
PASS_COUNT=0
|
||||
|
||||
cleanup() {
|
||||
if [[ -n "$STUB_PID" ]]; then
|
||||
kill "$STUB_PID" >/dev/null 2>&1 || true
|
||||
wait "$STUB_PID" >/dev/null 2>&1 || true
|
||||
fi
|
||||
rm -rf -- "$TEST_DIR"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
pass() {
|
||||
PASS_COUNT=$((PASS_COUNT + 1))
|
||||
echo "[PASS] $1"
|
||||
}
|
||||
|
||||
expect_failure() {
|
||||
local label="$1"
|
||||
shift
|
||||
if "$@"; then
|
||||
echo "[FAIL] expected failure: $label" >&2
|
||||
exit 1
|
||||
fi
|
||||
pass "$label"
|
||||
}
|
||||
|
||||
capture_lines() {
|
||||
local capture="$1"
|
||||
if [[ -f "$capture" ]]; then
|
||||
wc -l <"$capture" | tr -d ' '
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
}
|
||||
|
||||
new_state_dir() {
|
||||
local target="$1"
|
||||
mkdir -p "$target"
|
||||
chmod 0700 "$target"
|
||||
}
|
||||
|
||||
set_mode() {
|
||||
printf '%s\n' "$1" >"$TEST_DIR/mode"
|
||||
}
|
||||
|
||||
probe() {
|
||||
local state_dir="$1"
|
||||
local capture="$2"
|
||||
shift 2
|
||||
env HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$capture" \
|
||||
python3 "$ROOT_DIR/scripts/check-public-app-health.py" \
|
||||
--ready-url "https://localhost:${STUB_PORT}/ready" \
|
||||
--state-dir "$state_dir" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" \
|
||||
--alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \
|
||||
--failure-threshold 2 \
|
||||
--timeout-seconds 1 \
|
||||
--ca-file "$TEST_DIR/cert.pem" \
|
||||
"$@"
|
||||
}
|
||||
|
||||
openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 1 \
|
||||
-subj '/CN=localhost' -addext 'subjectAltName=DNS:localhost' \
|
||||
-keyout "$TEST_DIR/key.pem" -out "$TEST_DIR/cert.pem" >/dev/null 2>&1
|
||||
chmod 0600 "$TEST_DIR/key.pem" "$TEST_DIR/cert.pem"
|
||||
WECOM_TEST_HOST='qyapi.weixin.qq.com'
|
||||
WECOM_TEST_PATH='/cgi-bin/webhook/send'
|
||||
printf 'https://%s%s?key=%s\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \
|
||||
'01234567-89ab-cdef-0123-456789abcdef' >"$TEST_DIR/wecom-webhook"
|
||||
chmod 0600 "$TEST_DIR/wecom-webhook"
|
||||
printf '%s\n' healthy >"$TEST_DIR/mode"
|
||||
|
||||
python3 - "$TEST_DIR/cert.pem" "$TEST_DIR/key.pem" "$TEST_DIR/mode" "$TEST_DIR/port" <<'PY' >/dev/null 2>&1 &
|
||||
import http.server
|
||||
import json
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
cert_file, key_file, mode_file, port_file = map(Path, sys.argv[1:])
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
mode = mode_file.read_text(encoding="utf-8").strip()
|
||||
if self.path != "/ready":
|
||||
self.send_response(404)
|
||||
self.end_headers()
|
||||
return
|
||||
if mode == "timeout":
|
||||
time.sleep(2)
|
||||
if mode == "unavailable":
|
||||
status, body = 503, b'{"error":{"details":{"db":"do-not-leak"}}}'
|
||||
elif mode == "invalid-json":
|
||||
status, body = 200, b'not-json-private-payload'
|
||||
elif mode == "bad-contract":
|
||||
status, body = 200, json.dumps({"data": {"ok": False}}).encode()
|
||||
elif mode == "oversized":
|
||||
status, body = 200, b"x" * (64 * 1024 + 1)
|
||||
else:
|
||||
status, body = 200, json.dumps({"data": {"ok": True, "buildSha": "abcdef123456"}}).encode()
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
|
||||
server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
context.load_cert_chain(certfile=cert_file, keyfile=key_file)
|
||||
server.socket = context.wrap_socket(server.socket, server_side=True)
|
||||
port_file.write_text(str(server.server_port), encoding="utf-8")
|
||||
server.serve_forever()
|
||||
PY
|
||||
STUB_PID=$!
|
||||
|
||||
for _ in $(seq 1 50); do
|
||||
[[ -s "$TEST_DIR/port" ]] && break
|
||||
sleep 0.1
|
||||
done
|
||||
[[ -s "$TEST_DIR/port" ]]
|
||||
STUB_PORT="$(cat "$TEST_DIR/port")"
|
||||
|
||||
MAIN_STATE="$TEST_DIR/main-state"
|
||||
MAIN_CAPTURE="$TEST_DIR/main-capture.jsonl"
|
||||
new_state_dir "$MAIN_STATE"
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "0" ]]
|
||||
pass "healthy baseline writes state without sending an alert"
|
||||
|
||||
set_mode unavailable
|
||||
expect_failure "first failure stays pending" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "0" ]]
|
||||
expect_failure "second consecutive failure triggers alert evaluation" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "1" ]]
|
||||
expect_failure "identical continued failure remains non-zero" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "1" ]]
|
||||
pass "identical application failure is deduplicated"
|
||||
|
||||
set_mode invalid-json
|
||||
expect_failure "changed failure category remains non-zero" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "2" ]]
|
||||
pass "changed failure category sends one updated alert"
|
||||
|
||||
set_mode healthy
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]]
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]]
|
||||
pass "recovery sends exactly one notification"
|
||||
|
||||
TRANSIENT_STATE="$TEST_DIR/transient-state"
|
||||
TRANSIENT_CAPTURE="$TEST_DIR/transient-capture.jsonl"
|
||||
new_state_dir "$TRANSIENT_STATE"
|
||||
probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null
|
||||
set_mode unavailable
|
||||
expect_failure "single transient outage is recorded" probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null 2>&1
|
||||
set_mode healthy
|
||||
probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$TRANSIENT_CAPTURE")" == "0" ]]
|
||||
pass "single transient outage does not send failure or recovery noise"
|
||||
|
||||
for mode in bad-contract oversized timeout; do
|
||||
CASE_STATE="$TEST_DIR/${mode}-state"
|
||||
new_state_dir "$CASE_STATE"
|
||||
set_mode "$mode"
|
||||
expect_failure "$mode is rejected" probe "$CASE_STATE" "$TEST_DIR/${mode}-capture.jsonl" >/dev/null 2>&1
|
||||
python3 - "$CASE_STATE/app-health-probe-state.json" "$mode" <<'PY'
|
||||
import json, sys
|
||||
expected = {
|
||||
"bad-contract": "response_contract_invalid",
|
||||
"oversized": "response_too_large",
|
||||
"timeout": "request_timeout",
|
||||
}[sys.argv[2]]
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["lastCategory"] == expected, payload
|
||||
PY
|
||||
done
|
||||
|
||||
UNUSED_PORT="$(python3 - <<'PY'
|
||||
import socket
|
||||
sock = socket.socket()
|
||||
sock.bind(("127.0.0.1", 0))
|
||||
print(sock.getsockname()[1])
|
||||
sock.close()
|
||||
PY
|
||||
)"
|
||||
CONNECT_STATE="$TEST_DIR/connect-state"
|
||||
new_state_dir "$CONNECT_STATE"
|
||||
expect_failure "connection failure is rejected" env \
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/connect-capture.jsonl" \
|
||||
python3 "$ROOT_DIR/scripts/check-public-app-health.py" \
|
||||
--ready-url "https://localhost:${UNUSED_PORT}/ready" --state-dir "$CONNECT_STATE" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \
|
||||
--failure-threshold 2 --timeout-seconds 1 --ca-file "$TEST_DIR/cert.pem" >/dev/null 2>&1
|
||||
python3 - "$CONNECT_STATE/app-health-probe-state.json" <<'PY'
|
||||
import json, sys
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["lastCategory"] == "dns_or_connect_failed", payload
|
||||
PY
|
||||
pass "connection failure is stored as a stable category"
|
||||
|
||||
TLS_STATE="$TEST_DIR/tls-state"
|
||||
new_state_dir "$TLS_STATE"
|
||||
expect_failure "untrusted TLS certificate is rejected" env \
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/tls-capture.jsonl" \
|
||||
python3 "$ROOT_DIR/scripts/check-public-app-health.py" \
|
||||
--ready-url "https://localhost:${STUB_PORT}/ready" --state-dir "$TLS_STATE" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \
|
||||
--failure-threshold 2 --timeout-seconds 1 >/dev/null 2>&1
|
||||
python3 - "$TLS_STATE/app-health-probe-state.json" <<'PY'
|
||||
import json, sys
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["lastCategory"] == "tls_validation_failed", payload
|
||||
PY
|
||||
pass "TLS failure is stored as a stable category"
|
||||
|
||||
CORRUPT_STATE="$TEST_DIR/corrupt-state"
|
||||
new_state_dir "$CORRUPT_STATE"
|
||||
printf '%s\n' 'private-payload-must-not-appear' >"$CORRUPT_STATE/app-health-probe-state.json"
|
||||
chmod 0600 "$CORRUPT_STATE/app-health-probe-state.json"
|
||||
set_mode healthy
|
||||
expect_failure "corrupt local state fails closed" probe "$CORRUPT_STATE" "$TEST_DIR/corrupt-capture.jsonl" \
|
||||
>"$TEST_DIR/corrupt-stdout" 2>"$TEST_DIR/corrupt-stderr"
|
||||
grep -Fxq '[app-health][ERROR] local_state_invalid' "$TEST_DIR/corrupt-stderr"
|
||||
! grep -Fq 'private-payload' "$TEST_DIR/corrupt-stderr"
|
||||
! grep -Fq "$TEST_DIR" "$TEST_DIR/corrupt-stderr"
|
||||
pass "local state error output is stable and redacted"
|
||||
|
||||
python3 - "$MAIN_CAPTURE" <<'PY'
|
||||
import json, sys
|
||||
messages = [json.loads(line)["text"]["content"] for line in open(sys.argv[1], encoding="utf-8")]
|
||||
assert len(messages) == 3, messages
|
||||
assert "FAIL_APP_HEALTH" in messages[0]
|
||||
assert "http_unavailable" in messages[0]
|
||||
assert "response_contract_invalid" in messages[1]
|
||||
assert "PASS_APP_HEALTH_READY" in messages[2]
|
||||
for forbidden in ("do-not-leak", "localhost", "https://", "/tmp/", "not-json-private-payload"):
|
||||
assert all(forbidden not in message for message in messages), (forbidden, messages)
|
||||
PY
|
||||
pass "captured alerts contain only stable status and category text"
|
||||
|
||||
expect_failure "plain HTTP readiness URL is rejected" env \
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/http-capture.jsonl" \
|
||||
python3 "$ROOT_DIR/scripts/check-public-app-health.py" \
|
||||
--ready-url "http://localhost:${STUB_PORT}/ready" --state-dir "$TEST_DIR/http-state" \
|
||||
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \
|
||||
>/dev/null 2>&1
|
||||
|
||||
PYTHONPYCACHEPREFIX="$TEST_DIR/pycache" python3 -m py_compile \
|
||||
"$ROOT_DIR/scripts/check-public-app-health.py" "$ROOT_DIR/scripts/send-wecom-group-alert.py"
|
||||
bash -n "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
grep -Fq 'OnCalendar=*:0/5' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
systemd-analyze calendar '*:0/5' >/dev/null
|
||||
grep -Fq 'disabled application health monitor installed' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
grep -Fq 'install -o root -g root -m 0755 "$ALERT_HELPER_SOURCE" "$ALERT_HELPER_TARGET"' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
grep -Fq 'state and shared alert files retained' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
grep -Fq 'PROBE_TARGET must remain /usr/local/sbin/hyb-oa-public-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
grep -Fq 'ALERT_HELPER_TARGET must remain /usr/local/sbin/hyb-oa-wecom-group-alert' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
grep -Fq 'STATE_DIR must remain /var/lib/hyb-oa-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
grep -Fq 'SERVICE_NAME must remain hyb-oa-public-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
grep -Fq 'SYSTEMD_DIR must remain /etc/systemd/system' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh"
|
||||
pass "installer is five-minute, disabled-by-default, exact-target restricted, and preserves shared state"
|
||||
|
||||
printf '[PASS] public application health contract completed (%d checks)\n' "$PASS_COUNT"
|
||||
@@ -17,8 +17,10 @@ from pathlib import Path
|
||||
|
||||
SAFE_TOKEN = re.compile(r"^[A-Za-z0-9_-]{1,160}$")
|
||||
ALLOWED_STATUSES = {
|
||||
"FAIL_APP_HEALTH",
|
||||
"FAIL_AUTOMATIC_BACKUP",
|
||||
"FAIL_OFFHOST_DEADMAN",
|
||||
"PASS_APP_HEALTH_READY",
|
||||
"PASS_AUTOMATIC_BACKUP_COMPLETE",
|
||||
"PASS_AUTOMATIC_BACKUP_FRESH",
|
||||
"PASS_OFFHOST_DEADMAN_FRESH",
|
||||
|
||||
在新工单中引用
屏蔽一个用户