wangsli/hyb-oa
1
0
派生 0
代码 工单 合并请求 工作流 7 软件包 项目 发布 百科 活动

优化(wecom-public-health-alert-diagnostics-and-debounce): 增强公网健康告警诊断与防抖

浏览代码
这个提交包含在：
wangsli
2026-08-23 11:02:35 +08:00
父节点 986668a587
当前提交 a0f0483514
共修改 12 个文件，包含 935 行新增和 72 行删除
下载 Patch 文件 下载 Diff 文件 展开所有文件 折叠所有文件
scripts/qa/run-app-health-monitor-contract.sh
+139 -7
查看文件
@@ -137,6 +137,7 @@ MAIN_CAPTURE="$TEST_DIR/main-capture.jsonl"
new_state_dir "$MAIN_STATE"
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
[[ "$(capture_lines "$MAIN_CAPTURE")" == "0" ]]
[[ "$(stat -c %a "$MAIN_STATE/app-health-probe-state.json")" == "600" ]]
pass "healthy baseline writes state without sending an alert"
set_mode unavailable
@@ -155,14 +156,74 @@ pass "changed failure category sends one updated alert"
set_mode healthy
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
[[ "$(capture_lines "$MAIN_CAPTURE")" == "2" ]]
pass "first healthy sample keeps recovery pending"
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]]
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]]
pass "recovery sends exactly one notification"
pass "second healthy sample sends exactly one recovery notification"
set_mode invalid-json
expect_failure "first same-category recurrence stays pending" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]]
expect_failure "second same-category recurrence enters flapping" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
[[ "$(capture_lines "$MAIN_CAPTURE")" == "4" ]]
expect_failure "continued flapping failure remains deduplicated" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
[[ "$(capture_lines "$MAIN_CAPTURE")" == "4" ]]
set_mode healthy
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
[[ "$(capture_lines "$MAIN_CAPTURE")" == "4" ]]
python3 - "$MAIN_STATE/app-health-probe-state.json" <<'PY'
import json, os, sys, time
path = sys.argv[1]
payload = json.load(open(path, encoding="utf-8"))
assert payload["phase"] == "flapping", payload
payload["healthySince"] = int(time.time()) - 1800
with open(path, "w", encoding="utf-8") as handle:
json.dump(payload, handle)
handle.write("\n")
os.chmod(path, 0o600)
PY
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
[[ "$(capture_lines "$MAIN_CAPTURE")" == "4" ]]
python3 - "$MAIN_STATE/app-health-probe-state.json" <<'PY'
import json, sys
payload = json.load(open(sys.argv[1], encoding="utf-8"))
assert payload["phase"] == "healthy", payload
assert payload["flapTransitions"] == 0, payload
PY
pass "flapping sends once and exits silently after thirty healthy minutes"
DIFFERENT_STATE="$TEST_DIR/different-state"
DIFFERENT_CAPTURE="$TEST_DIR/different-capture.jsonl"
new_state_dir "$DIFFERENT_STATE"
set_mode healthy
probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null
set_mode unavailable
expect_failure "different-case first failure stays pending" probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null 2>&1
expect_failure "different-case second failure alerts" probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null 2>&1
set_mode healthy
probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null
probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null
[[ "$(capture_lines "$DIFFERENT_CAPTURE")" == "2" ]]
set_mode invalid-json
expect_failure "new-category recurrence first sample stays pending" probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null 2>&1
[[ "$(capture_lines "$DIFFERENT_CAPTURE")" == "2" ]]
expect_failure "new-category recurrence becomes a normal alert" probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null 2>&1
[[ "$(capture_lines "$DIFFERENT_CAPTURE")" == "3" ]]
python3 - "$DIFFERENT_STATE/app-health-probe-state.json" <<'PY'
import json, sys
payload = json.load(open(sys.argv[1], encoding="utf-8"))
assert payload["phase"] == "alerted", payload
assert payload["activeCategory"] == "response_contract_invalid", payload
PY
pass "different failure category is not swallowed by same-category flapping debounce"
TRANSIENT_STATE="$TEST_DIR/transient-state"
TRANSIENT_CAPTURE="$TEST_DIR/transient-capture.jsonl"
new_state_dir "$TRANSIENT_STATE"
set_mode healthy
probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null
set_mode unavailable
expect_failure "single transient outage is recorded" probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null 2>&1
@@ -184,10 +245,48 @@ expected = {
"timeout": "request_timeout",
}[sys.argv[2]]
payload = json.load(open(sys.argv[1], encoding="utf-8"))
assert payload["lastCategory"] == expected, payload
assert payload["activeCategory"] == expected, payload
PY
done
python3 - "$ROOT_DIR/scripts/check-public-app-health.py" <<'PY'
import importlib.util
import socket
import sys
import urllib.error
spec = importlib.util.spec_from_file_location("app_health", sys.argv[1])
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
assert module.classify_url_error(urllib.error.URLError(socket.gaierror(-2, "private-dns-text"))) == "dns_resolution_failed"
assert module.classify_url_error(urllib.error.URLError(ConnectionRefusedError())) == "tcp_connection_failed"
watch = module.new_probe_state()
watch.update({
"phase": "recovery_watch",
"lastRecoveredCategory": "http_unavailable",
"recoveredAt": 1000,
})
pending, candidate = module.transition_probe_state(watch, "http_unavailable", 4599, 2)
assert pending["phase"] == "failure_pending" and candidate is None, (pending, candidate)
flapping, candidate = module.transition_probe_state(pending, "http_unavailable", 4599, 2)
assert flapping["phase"] == "flapping", flapping
assert candidate == ("FAIL_APP_HEALTH_FLAPPING", "http_unavailable"), candidate
expired, candidate = module.transition_probe_state(watch, "http_unavailable", 4600, 2)
assert expired["phase"] == "failure_pending" and expired["recoveredAt"] is None, expired
alerted, candidate = module.transition_probe_state(expired, "http_unavailable", 4600, 2)
assert alerted["phase"] == "alerted", alerted
assert candidate == ("FAIL_APP_HEALTH", "http_unavailable"), candidate
flapping["healthySince"] = 5000
flapping["consecutiveFailures"] = 0
flapping["consecutiveSuccesses"] = 1
stable, candidate = module.transition_probe_state(flapping, None, 6800, 2)
assert stable["phase"] == "healthy" and candidate is None, (stable, candidate)
PY
pass "DNS/TCP classification and exact thirty/sixty-minute boundaries are deterministic"
UNUSED_PORT="$(python3 - <<'PY'
import socket
sock = socket.socket()
@@ -207,9 +306,9 @@ expect_failure "connection failure is rejected" env \
python3 - "$CONNECT_STATE/app-health-probe-state.json" <<'PY'
import json, sys
payload = json.load(open(sys.argv[1], encoding="utf-8"))
assert payload["lastCategory"] == "dns_or_connect_failed", payload
assert payload["activeCategory"] == "tcp_connection_failed", payload
PY
pass "connection failure is stored as a stable category"
pass "TCP connection failure is stored separately from DNS resolution failure"
TLS_STATE="$TEST_DIR/tls-state"
new_state_dir "$TLS_STATE"
@@ -222,10 +321,39 @@ expect_failure "untrusted TLS certificate is rejected" env \
python3 - "$TLS_STATE/app-health-probe-state.json" <<'PY'
import json, sys
payload = json.load(open(sys.argv[1], encoding="utf-8"))
assert payload["lastCategory"] == "tls_validation_failed", payload
assert payload["activeCategory"] == "tls_validation_failed", payload
PY
pass "TLS failure is stored as a stable category"
MIGRATION_STATE="$TEST_DIR/migration-state"
MIGRATION_CAPTURE="$TEST_DIR/migration-capture.jsonl"
new_state_dir "$MIGRATION_STATE"
printf '%s\n' '{"schemaVersion":1,"consecutiveFailures":0,"lastCategory":null}' \
>"$MIGRATION_STATE/app-health-probe-state.json"
chmod 0600 "$MIGRATION_STATE/app-health-probe-state.json"
set_mode healthy
probe "$MIGRATION_STATE" "$MIGRATION_CAPTURE" >/dev/null
python3 - "$MIGRATION_STATE/app-health-probe-state.json" <<'PY'
import json, sys
payload = json.load(open(sys.argv[1], encoding="utf-8"))
assert payload["schemaVersion"] == 2, payload
assert payload["phase"] == "healthy", payload
PY
[[ "$(capture_lines "$MIGRATION_CAPTURE")" == "0" ]]
pass "valid legacy state migrates without backfilling a notification"
INVALID_SCHEMA_STATE="$TEST_DIR/invalid-schema-state"
new_state_dir "$INVALID_SCHEMA_STATE"
printf '%s\n' '{"schemaVersion":2,"phase":"healthy","activeCategory":null,"lastRecoveredCategory":null,"consecutiveFailures":1000001,"consecutiveSuccesses":0,"recoveredAt":null,"healthySince":null,"flapTransitions":0}' \
>"$INVALID_SCHEMA_STATE/app-health-probe-state.json"
chmod 0600 "$INVALID_SCHEMA_STATE/app-health-probe-state.json"
set_mode healthy
expect_failure "out-of-range v2 state fails closed" probe "$INVALID_SCHEMA_STATE" \
"$TEST_DIR/invalid-schema-capture.jsonl" >"$TEST_DIR/invalid-schema-stdout" 2>"$TEST_DIR/invalid-schema-stderr"
grep -Fxq '[app-health][ERROR] local_state_invalid' "$TEST_DIR/invalid-schema-stderr"
! grep -Fq '1000001' "$TEST_DIR/invalid-schema-stderr"
pass "v2 state schema rejects inconsistent or out-of-range fields without disclosure"
CORRUPT_STATE="$TEST_DIR/corrupt-state"
new_state_dir "$CORRUPT_STATE"
printf '%s\n' 'private-payload-must-not-appear' >"$CORRUPT_STATE/app-health-probe-state.json"
@@ -241,15 +369,19 @@ pass "local state error output is stable and redacted"
python3 - "$MAIN_CAPTURE" <<'PY'
import json, sys
messages = [json.loads(line)["text"]["content"] for line in open(sys.argv[1], encoding="utf-8")]
assert len(messages) == 3, messages
assert len(messages) == 4, messages
assert "FAIL_APP_HEALTH" in messages[0]
assert "http_unavailable" in messages[0]
assert "response_contract_invalid" in messages[1]
assert "PASS_APP_HEALTH_READY" in messages[2]
assert "FAIL_APP_HEALTH_FLAPPING" in messages[3]
assert messages[0].startswith("OA 公网健康告警\n"), messages[0]
assert messages[2].startswith("OA 公网健康恢复\n"), messages[2]
assert messages[3].startswith("OA 公网健康链路反复抖动\n"), messages[3]
for forbidden in ("do-not-leak", "localhost", "https://", "/tmp/", "not-json-private-payload"):
assert all(forbidden not in message for message in messages), (forbidden, messages)
PY
pass "captured alerts contain only stable status and category text"
pass "captured application alerts use explicit source titles and stable redacted categories"
expect_failure "plain HTTP readiness URL is rejected" env \
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/http-capture.jsonl" \
scripts/qa/run-automatic-backup-contract.sh
+10
查看文件
@@ -701,6 +701,16 @@ mv "$TEST_DIR/deadman-withheld-batch" "$DEADMAN_LAYOUT/archive/$SECOND_BATCH"
--webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" >/dev/null
[[ "$(wc -l <"$DEADMAN_CAPTURE")" == "2" ]]
pass "ARM dead-man sends one recovery notification after finalized archives resume"
python3 - "$DEADMAN_CAPTURE" <<'PYTHON'
import json
import sys
messages = [json.loads(line)["text"]["content"] for line in open(sys.argv[1], encoding="utf-8")]
assert messages[0].startswith("内部 OA 运维告警\n"), messages
assert messages[1].startswith("内部 OA 运维恢复通知\n"), messages
assert all("OA 公网健康" not in message for message in messages), messages
PYTHON
pass "shared helper keeps dead-man alert and recovery titles unchanged"
DRIFT_BUILD_SHA="0123456789abcdef0123456789abcdef01234567"
expect_failure "wrapper rejects an active runtime build that drifted from its installed contract" \