优化(wecom-public-health-alert-diagnostics-and-debounce): 增强公网健康告警诊断与防抖
这个提交包含在：
@@ -137,6 +137,7 @@ MAIN_CAPTURE="$TEST_DIR/main-capture.jsonl"
|
||||
new_state_dir "$MAIN_STATE"
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "0" ]]
|
||||
[[ "$(stat -c %a "$MAIN_STATE/app-health-probe-state.json")" == "600" ]]
|
||||
pass "healthy baseline writes state without sending an alert"
|
||||
|
||||
set_mode unavailable
|
||||
@@ -155,14 +156,74 @@ pass "changed failure category sends one updated alert"
|
||||
|
||||
set_mode healthy
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "2" ]]
|
||||
pass "first healthy sample keeps recovery pending"
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]]
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]]
|
||||
pass "recovery sends exactly one notification"
|
||||
pass "second healthy sample sends exactly one recovery notification"
|
||||
|
||||
set_mode invalid-json
|
||||
expect_failure "first same-category recurrence stays pending" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]]
|
||||
expect_failure "second same-category recurrence enters flapping" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "4" ]]
|
||||
expect_failure "continued flapping failure remains deduplicated" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "4" ]]
|
||||
set_mode healthy
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "4" ]]
|
||||
python3 - "$MAIN_STATE/app-health-probe-state.json" <<'PY'
|
||||
import json, os, sys, time
|
||||
path = sys.argv[1]
|
||||
payload = json.load(open(path, encoding="utf-8"))
|
||||
assert payload["phase"] == "flapping", payload
|
||||
payload["healthySince"] = int(time.time()) - 1800
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
json.dump(payload, handle)
|
||||
handle.write("\n")
|
||||
os.chmod(path, 0o600)
|
||||
PY
|
||||
probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$MAIN_CAPTURE")" == "4" ]]
|
||||
python3 - "$MAIN_STATE/app-health-probe-state.json" <<'PY'
|
||||
import json, sys
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["phase"] == "healthy", payload
|
||||
assert payload["flapTransitions"] == 0, payload
|
||||
PY
|
||||
pass "flapping sends once and exits silently after thirty healthy minutes"
|
||||
|
||||
DIFFERENT_STATE="$TEST_DIR/different-state"
|
||||
DIFFERENT_CAPTURE="$TEST_DIR/different-capture.jsonl"
|
||||
new_state_dir "$DIFFERENT_STATE"
|
||||
set_mode healthy
|
||||
probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null
|
||||
set_mode unavailable
|
||||
expect_failure "different-case first failure stays pending" probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null 2>&1
|
||||
expect_failure "different-case second failure alerts" probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null 2>&1
|
||||
set_mode healthy
|
||||
probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null
|
||||
probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null
|
||||
[[ "$(capture_lines "$DIFFERENT_CAPTURE")" == "2" ]]
|
||||
set_mode invalid-json
|
||||
expect_failure "new-category recurrence first sample stays pending" probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$DIFFERENT_CAPTURE")" == "2" ]]
|
||||
expect_failure "new-category recurrence becomes a normal alert" probe "$DIFFERENT_STATE" "$DIFFERENT_CAPTURE" >/dev/null 2>&1
|
||||
[[ "$(capture_lines "$DIFFERENT_CAPTURE")" == "3" ]]
|
||||
python3 - "$DIFFERENT_STATE/app-health-probe-state.json" <<'PY'
|
||||
import json, sys
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["phase"] == "alerted", payload
|
||||
assert payload["activeCategory"] == "response_contract_invalid", payload
|
||||
PY
|
||||
pass "different failure category is not swallowed by same-category flapping debounce"
|
||||
|
||||
TRANSIENT_STATE="$TEST_DIR/transient-state"
|
||||
TRANSIENT_CAPTURE="$TEST_DIR/transient-capture.jsonl"
|
||||
new_state_dir "$TRANSIENT_STATE"
|
||||
set_mode healthy
|
||||
probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null
|
||||
set_mode unavailable
|
||||
expect_failure "single transient outage is recorded" probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null 2>&1
|
||||
@@ -184,10 +245,48 @@ expected = {
|
||||
"timeout": "request_timeout",
|
||||
}[sys.argv[2]]
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["lastCategory"] == expected, payload
|
||||
assert payload["activeCategory"] == expected, payload
|
||||
PY
|
||||
done
|
||||
|
||||
python3 - "$ROOT_DIR/scripts/check-public-app-health.py" <<'PY'
|
||||
import importlib.util
|
||||
import socket
|
||||
import sys
|
||||
import urllib.error
|
||||
|
||||
spec = importlib.util.spec_from_file_location("app_health", sys.argv[1])
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
assert module.classify_url_error(urllib.error.URLError(socket.gaierror(-2, "private-dns-text"))) == "dns_resolution_failed"
|
||||
assert module.classify_url_error(urllib.error.URLError(ConnectionRefusedError())) == "tcp_connection_failed"
|
||||
|
||||
watch = module.new_probe_state()
|
||||
watch.update({
|
||||
"phase": "recovery_watch",
|
||||
"lastRecoveredCategory": "http_unavailable",
|
||||
"recoveredAt": 1000,
|
||||
})
|
||||
pending, candidate = module.transition_probe_state(watch, "http_unavailable", 4599, 2)
|
||||
assert pending["phase"] == "failure_pending" and candidate is None, (pending, candidate)
|
||||
flapping, candidate = module.transition_probe_state(pending, "http_unavailable", 4599, 2)
|
||||
assert flapping["phase"] == "flapping", flapping
|
||||
assert candidate == ("FAIL_APP_HEALTH_FLAPPING", "http_unavailable"), candidate
|
||||
|
||||
expired, candidate = module.transition_probe_state(watch, "http_unavailable", 4600, 2)
|
||||
assert expired["phase"] == "failure_pending" and expired["recoveredAt"] is None, expired
|
||||
alerted, candidate = module.transition_probe_state(expired, "http_unavailable", 4600, 2)
|
||||
assert alerted["phase"] == "alerted", alerted
|
||||
assert candidate == ("FAIL_APP_HEALTH", "http_unavailable"), candidate
|
||||
|
||||
flapping["healthySince"] = 5000
|
||||
flapping["consecutiveFailures"] = 0
|
||||
flapping["consecutiveSuccesses"] = 1
|
||||
stable, candidate = module.transition_probe_state(flapping, None, 6800, 2)
|
||||
assert stable["phase"] == "healthy" and candidate is None, (stable, candidate)
|
||||
PY
|
||||
pass "DNS/TCP classification and exact thirty/sixty-minute boundaries are deterministic"
|
||||
|
||||
UNUSED_PORT="$(python3 - <<'PY'
|
||||
import socket
|
||||
sock = socket.socket()
|
||||
@@ -207,9 +306,9 @@ expect_failure "connection failure is rejected" env \
|
||||
python3 - "$CONNECT_STATE/app-health-probe-state.json" <<'PY'
|
||||
import json, sys
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["lastCategory"] == "dns_or_connect_failed", payload
|
||||
assert payload["activeCategory"] == "tcp_connection_failed", payload
|
||||
PY
|
||||
pass "connection failure is stored as a stable category"
|
||||
pass "TCP connection failure is stored separately from DNS resolution failure"
|
||||
|
||||
TLS_STATE="$TEST_DIR/tls-state"
|
||||
new_state_dir "$TLS_STATE"
|
||||
@@ -222,10 +321,39 @@ expect_failure "untrusted TLS certificate is rejected" env \
|
||||
python3 - "$TLS_STATE/app-health-probe-state.json" <<'PY'
|
||||
import json, sys
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["lastCategory"] == "tls_validation_failed", payload
|
||||
assert payload["activeCategory"] == "tls_validation_failed", payload
|
||||
PY
|
||||
pass "TLS failure is stored as a stable category"
|
||||
|
||||
MIGRATION_STATE="$TEST_DIR/migration-state"
|
||||
MIGRATION_CAPTURE="$TEST_DIR/migration-capture.jsonl"
|
||||
new_state_dir "$MIGRATION_STATE"
|
||||
printf '%s\n' '{"schemaVersion":1,"consecutiveFailures":0,"lastCategory":null}' \
|
||||
>"$MIGRATION_STATE/app-health-probe-state.json"
|
||||
chmod 0600 "$MIGRATION_STATE/app-health-probe-state.json"
|
||||
set_mode healthy
|
||||
probe "$MIGRATION_STATE" "$MIGRATION_CAPTURE" >/dev/null
|
||||
python3 - "$MIGRATION_STATE/app-health-probe-state.json" <<'PY'
|
||||
import json, sys
|
||||
payload = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert payload["schemaVersion"] == 2, payload
|
||||
assert payload["phase"] == "healthy", payload
|
||||
PY
|
||||
[[ "$(capture_lines "$MIGRATION_CAPTURE")" == "0" ]]
|
||||
pass "valid legacy state migrates without backfilling a notification"
|
||||
|
||||
INVALID_SCHEMA_STATE="$TEST_DIR/invalid-schema-state"
|
||||
new_state_dir "$INVALID_SCHEMA_STATE"
|
||||
printf '%s\n' '{"schemaVersion":2,"phase":"healthy","activeCategory":null,"lastRecoveredCategory":null,"consecutiveFailures":1000001,"consecutiveSuccesses":0,"recoveredAt":null,"healthySince":null,"flapTransitions":0}' \
|
||||
>"$INVALID_SCHEMA_STATE/app-health-probe-state.json"
|
||||
chmod 0600 "$INVALID_SCHEMA_STATE/app-health-probe-state.json"
|
||||
set_mode healthy
|
||||
expect_failure "out-of-range v2 state fails closed" probe "$INVALID_SCHEMA_STATE" \
|
||||
"$TEST_DIR/invalid-schema-capture.jsonl" >"$TEST_DIR/invalid-schema-stdout" 2>"$TEST_DIR/invalid-schema-stderr"
|
||||
grep -Fxq '[app-health][ERROR] local_state_invalid' "$TEST_DIR/invalid-schema-stderr"
|
||||
! grep -Fq '1000001' "$TEST_DIR/invalid-schema-stderr"
|
||||
pass "v2 state schema rejects inconsistent or out-of-range fields without disclosure"
|
||||
|
||||
CORRUPT_STATE="$TEST_DIR/corrupt-state"
|
||||
new_state_dir "$CORRUPT_STATE"
|
||||
printf '%s\n' 'private-payload-must-not-appear' >"$CORRUPT_STATE/app-health-probe-state.json"
|
||||
@@ -241,15 +369,19 @@ pass "local state error output is stable and redacted"
|
||||
python3 - "$MAIN_CAPTURE" <<'PY'
|
||||
import json, sys
|
||||
messages = [json.loads(line)["text"]["content"] for line in open(sys.argv[1], encoding="utf-8")]
|
||||
assert len(messages) == 3, messages
|
||||
assert len(messages) == 4, messages
|
||||
assert "FAIL_APP_HEALTH" in messages[0]
|
||||
assert "http_unavailable" in messages[0]
|
||||
assert "response_contract_invalid" in messages[1]
|
||||
assert "PASS_APP_HEALTH_READY" in messages[2]
|
||||
assert "FAIL_APP_HEALTH_FLAPPING" in messages[3]
|
||||
assert messages[0].startswith("OA 公网健康告警\n"), messages[0]
|
||||
assert messages[2].startswith("OA 公网健康恢复\n"), messages[2]
|
||||
assert messages[3].startswith("OA 公网健康链路反复抖动\n"), messages[3]
|
||||
for forbidden in ("do-not-leak", "localhost", "https://", "/tmp/", "not-json-private-payload"):
|
||||
assert all(forbidden not in message for message in messages), (forbidden, messages)
|
||||
PY
|
||||
pass "captured alerts contain only stable status and category text"
|
||||
pass "captured application alerts use explicit source titles and stable redacted categories"
|
||||
|
||||
expect_failure "plain HTTP readiness URL is rejected" env \
|
||||
HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/http-capture.jsonl" \
|
||||
|
||||
在新工单中引用
屏蔽一个用户