From bb5113c58be9c37a74d5fc8c4cc42c21a5d666c2 Mon Sep 17 00:00:00 2001 From: shunli Date: Mon, 27 Jul 2026 22:30:33 +0800 Subject: [PATCH] =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E5=BA=94=E7=94=A8=E5=81=A5?= =?UTF-8?q?=E5=BA=B7=E7=9B=91=E6=8E=A7=EF=BC=8807-27-app-health-monitoring?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .trellis/spec/hyb-oa/deployment-trialrun.md | 96 ++++++ .../07-27-app-health-monitoring/check.jsonl | 1 + .../07-27-app-health-monitoring/design.md | 91 ++++++ .../implement.jsonl | 1 + .../07-27-app-health-monitoring/implement.md | 47 +++ .../tasks/07-27-app-health-monitoring/prd.md | 69 +++++ .../07-27-app-health-monitoring/task.json | 26 ++ deploy/trialrun/checklist.md | 8 + .../install-offhost-app-health-monitor.sh | 158 ++++++++++ deploy/trialrun/runbook.md | 31 ++ docs/试运行与自动化验证手册.md | 21 ++ scripts/check-minimal.sh | 1 + scripts/check-public-app-health.py | 283 ++++++++++++++++++ scripts/qa/run-app-health-monitor-contract.sh | 276 +++++++++++++++++ scripts/send-wecom-group-alert.py | 2 + 15 files changed, 1111 insertions(+) create mode 100644 .trellis/tasks/07-27-app-health-monitoring/check.jsonl create mode 100644 .trellis/tasks/07-27-app-health-monitoring/design.md create mode 100644 .trellis/tasks/07-27-app-health-monitoring/implement.jsonl create mode 100644 .trellis/tasks/07-27-app-health-monitoring/implement.md create mode 100644 .trellis/tasks/07-27-app-health-monitoring/prd.md create mode 100644 .trellis/tasks/07-27-app-health-monitoring/task.json create mode 100755 deploy/trialrun/install-offhost-app-health-monitor.sh create mode 100755 scripts/check-public-app-health.py create mode 100755 scripts/qa/run-app-health-monitor-contract.sh diff --git a/.trellis/spec/hyb-oa/deployment-trialrun.md b/.trellis/spec/hyb-oa/deployment-trialrun.md index 55c28a3c..68618f25 100644 --- a/.trellis/spec/hyb-oa/deployment-trialrun.md +++ b/.trellis/spec/hyb-oa/deployment-trialrun.md @@ -1097,3 +1097,99 @@ bash scripts/run-automatic-backup.sh run # The Docker CLI inherits the value without placing it in argv. PGPASSWORD="$POSTGRES_PASSWORD" docker exec -e PGPASSWORD db pg_dump ... ``` + +## Scenario: Independent Public Application Health Monitor + +### 1. Scope / Trigger + +- Trigger: installing or changing the off-host public `/ready` probe, its ARM + timer, application-health alert states or the shared WeCom group-alert helper. +- Purpose: detect an unavailable OA host, edge/TLS path, application or database + readiness signal without adding a time-series platform or relying on traffic. + +### 2. Signatures + +- Probe: `scripts/check-public-app-health.py --ready-url --state-dir + --webhook-file --alert-helper + --failure-threshold 2 --timeout-seconds <1..30>`. +- Installer: `deploy/trialrun/install-offhost-app-health-monitor.sh + install|enable|uninstall`. +- Units: `hyb-oa-public-app-health.service` and + `hyb-oa-public-app-health.timer`; the timer uses `OnCalendar=*:0/5`. +- Local contract: `bash scripts/qa/run-app-health-monitor-contract.sh`. + +### 3. Contracts + +- Run the probe from the existing ARM host against an exact HTTPS `/ready` URL. + Success requires trusted TLS, HTTP 2xx, bounded JSON and `data.ok === true`. +- Count any stable failure category toward the same consecutive-failure + threshold. The first failure is pending; the second evaluates a failure + alert. Once alerting, a changed category may send a new alert, identical + failures are deduplicated, and one later success sends one recovery notice. +- Persist only schema version, count, stable category and the alert state in a + root-owned `0700` directory with atomic `0600` files. Never persist or send + the URL, resolved address, response payload, credential or raw exception. +- Application-health probe state and alert state are separate from backup and + dead-man state. The installer must deploy the helper revision that passed the + backup contract; an older installed helper may reject the new status tokens. +- `install` writes only the exact managed probe/helper/state/unit targets and + leaves the timer disabled. `enable`, ARM writes, real robot messages and a + controlled failure/recovery drill remain separately approved operations. +- `uninstall` removes only the exact application-health units and probe. It + retains the shared helper, webhook, state evidence and all backup components. + +### 4. Validation & Error Matrix + +- Non-HTTPS, userinfo, query, fragment or non-`/ready` URL -> reject before a + request with `ready_url_invalid`. +- DNS/connect, TLS, timeout, HTTP, oversized response or invalid JSON/contract + -> stable redacted category and non-zero oneshot exit. +- Invalid owner/mode/link/state or alert-helper failure -> fail closed with a + fixed local error; do not include the offending value. +- A first isolated failure followed by health -> no failure or recovery message. +- Any overridden managed target or service name -> installer stops before + `systemctl`, file replacement or removal. +- ARM itself or its network/robot path unavailable -> the monitor can be silent; + do not report this design as having an independent third-party dead-man. + +### 5. Good/Base/Bad Cases + +- Good: ARM observes two consecutive unavailable checks, sends one redacted + category alert, suppresses repeats and sends one recovery after health returns. +- Base: repository contracts pass while units remain disabled; report the code + as ready for an approved rollout, not as an active monitor. +- Bad: run the timer only on the OA host, send raw `/ready` errors, install and + enable in one unapproved step, or deploy the probe without its helper revision. + +### 6. Tests Required + +- The local HTTPS stub must assert healthy baseline, pending first failure, + second-failure alert, identical deduplication, changed-category alert, one + recovery, transient suppression, TLS/connect/timeout/HTTP/size/JSON failures, + corrupt private state, URL rejection and absence of sensitive output. +- Run Python compile and shell syntax checks, validate the five-minute calendar, + and assert disabled-by-default plus exact managed targets and retained state. +- Re-run `bash scripts/qa/run-automatic-backup-contract.sh` whenever the shared + helper allowlist or behavior changes. +- ARM install, enabled/active timer status and real failure/recovery messages are + external evidence and may not be inferred from the local contract. + +### 7. Wrong vs Correct + +#### Wrong + +```bash +# Local contract passed, so install and enable remotely and call it active. +bash deploy/trialrun/install-offhost-app-health-monitor.sh enable +``` + +#### Correct + +```bash +bash scripts/qa/run-app-health-monitor-contract.sh +bash scripts/qa/run-automatic-backup-contract.sh +# After separate ARM-write approval: install disabled, inspect, then seek +# enable/real-message/drill approval before claiming the monitor is active. + \ + bash deploy/trialrun/install-offhost-app-health-monitor.sh install +``` diff --git a/.trellis/tasks/07-27-app-health-monitoring/check.jsonl b/.trellis/tasks/07-27-app-health-monitoring/check.jsonl new file mode 100644 index 00000000..9dd3234a --- /dev/null +++ b/.trellis/tasks/07-27-app-health-monitoring/check.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/07-27-app-health-monitoring/design.md b/.trellis/tasks/07-27-app-health-monitoring/design.md new file mode 100644 index 00000000..e3118551 --- /dev/null +++ b/.trellis/tasks/07-27-app-health-monitoring/design.md @@ -0,0 +1,91 @@ +# 应用健康监控设计 + +## 1. Architecture and Boundaries + +首版由三个独立部分组成： + +1. 仓库内的 Python 探测器：从 ARM 发起公网 HTTPS `/ready` 请求，验证传输、状态码和最小 JSON 合同，维护连续失败计数，并生成只含状态/类别/计数的 0600 摘要文件。 +2. 现有 `scripts/send-wecom-group-alert.py`：扩展应用健康允许状态，继续负责 webhook 文件校验、脱敏消息、同类故障降噪和恢复通知。 +3. 独立 ARM systemd 安装器：复制探测器和本次已通过备份回归的共用告警 helper 修订，引用既有 root-owned 0600 webhook，写入 oneshot service/timer；`install` 默认只安装 disabled units，`enable` 才启用，`uninstall` 只删除应用健康组件并保留共用 helper 与状态证据。 + +不修改 Fastify `/ready`，不修改 Compose 拓扑，不引入 Prometheus/Alertmanager，不把应用监控并入备份 receiver 的生命周期。 + +## 2. Probe Contract + +### Input + +- 固定要求 `https` URL，拒绝 userinfo、fragment、非预期 query；默认目标为项目公开 OA 的 `/ready`。 +- 连接与读取均有短超时，单次 timer 不得长期占用。 +- 状态目录、摘要、探测状态和告警状态均由 root 持有，目录 0700、文件 0600，拒绝符号链接和宽权限输入。 + +### Success + +- TLS 证书链和主机名校验通过。 +- HTTP 2xx。 +- 响应是受大小限制的 JSON，且 `data.ok === true`。 +- 不要求固定 `buildSha`，避免正常发布触发误报；不持久化完整响应。 + +### Failure categories + +只输出稳定 token，不输出异常原文： + +- `dns_or_connect_failed` +- `tls_validation_failed` +- `request_timeout` +- `http_unavailable` +- `response_too_large` +- `response_contract_invalid` +- `local_state_invalid` +- `alert_evaluation_failed` + +HTTP 503 不解析并转发数据库错误详情，只归类为 `http_unavailable`。日志和企微消息不得包含地址解析结果、IP、请求/响应 payload、凭证或绝对路径。 + +## 3. Failure Confirmation and Alert State + +探测器维护独立 probe state：`schemaVersion`、`consecutiveFailures`、`lastCategory`。告警 helper 继续维护自己的 alert state。 + +状态转换： + +```text +healthy ──首次失败──> pending_failure（记录，非零退出，不发消息） +pending_failure ──再次失败──> alerting（调用 helper） +alerting ──相同失败──> helper 降噪 +alerting ──类别变化──> helper 发送新类别告警 +pending_failure ──成功──> healthy（不发送恢复消息） +alerting ──成功──> healthy（helper 发送一次恢复通知） +``` + +连续失败按“任意失败”累计，第二次使用当前类别告警；这样不会因 DNS、连接、HTTP 类别在同一事故中抖动而无限推迟。达到阈值后每次都调用 helper，由 helper 根据状态和类别决定是否发送。状态采用同目录临时文件 + 原子替换。 + +若真实消息发送失败，helper 不更新 alert state，下一次 timer 会重试。探测 service 保持非零退出，便于 `systemctl status`/journal 只读检查。 + +## 4. Systemd Design + +- service：`Type=oneshot`、`UMask=0077`、依赖 `network-online.target`，以 root 运行以读取现有 root-owned webhook；不加载 OA `.env`。 +- timer：每 5 分钟执行，`Persistent=true`，开机后补一次错过的执行；不启用随机延迟，以便最长发现时间可预测。 +- 安装器校验源文件、目标 URL、状态目录、webhook/helper 权限和依赖；`install` 与 `enable` 分离，避免未经真实告警批准就自动发送。 +- timer 位于 ARM，因此可发现 OA 主机整体不可达；ARM 自身宕机仍会导致探测失声，首版通过 runbook 中的定期 `systemctl list-timers` 检查记录该剩余风险，不新增第三方 dead-man。 + +## 5. Compatibility and Reuse + +- `send-wecom-group-alert.py` 仅新增 `FAIL_APP_HEALTH` 与 `PASS_APP_HEALTH_READY` 允许状态；现有 `FAIL_*`/健康恢复逻辑保持不变。 +- 现有自动备份契约测试必须继续通过，证明 allowlist 扩展未改变备份消息行为。 +- 应用监控使用独立 alert state 文件，不与自动备份或 WARCH3 的降噪状态互相覆盖。 + +## 6. Rollout and Rollback + +实现阶段只产出脚本、测试和现有运维文档更新。远端 rollout 是后续独立批准动作： + +1. 在非真实发送测试模式运行健康/失败/恢复状态机。 +2. ARM 执行安装但不启用，检查 unit 内容、文件权限和手动健康探测。 +3. 另获批准后启用 timer，并进行一次受控失败演练，确认只出现一条告警和一条恢复通知。 +4. 只读核对 timer 最近执行与下一次计划时间。 + +回滚时 disable/stop 并卸载应用健康 units/探测器；不触碰备份 receiver、归档、共用 webhook 和 helper。若 helper 已被其他能力共用，不在应用监控卸载中删除。 + +## 7. Deferred Items + +- 证书到期前 N 天预警。 +- 登录或报销等业务合成事务。 +- 5xx 比率、延迟分位数和趋势图。 +- ARM timer 自身的第三方 dead-man。 diff --git a/.trellis/tasks/07-27-app-health-monitoring/implement.jsonl b/.trellis/tasks/07-27-app-health-monitoring/implement.jsonl new file mode 100644 index 00000000..9dd3234a --- /dev/null +++ b/.trellis/tasks/07-27-app-health-monitoring/implement.jsonl @@ -0,0 +1 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} diff --git a/.trellis/tasks/07-27-app-health-monitoring/implement.md b/.trellis/tasks/07-27-app-health-monitoring/implement.md new file mode 100644 index 00000000..8673d11a --- /dev/null +++ b/.trellis/tasks/07-27-app-health-monitoring/implement.md @@ -0,0 +1,47 @@ +# 应用健康监控实施清单 + +## 1. Implementation + +- [x] 新增独立 Python 公网 `/ready` 探测器，完成严格 URL、TLS、HTTP、响应大小和 JSON 合同校验。 +- [x] 实现 0700/0600 本地状态合同、连续失败计数、原子写入和稳定脱敏类别。 +- [x] 扩展 `scripts/send-wecom-group-alert.py` 的应用健康状态 allowlist，保持既有降噪与恢复语义。 +- [x] 新增独立 ARM systemd 安装器，包含 `install`、`enable`、`uninstall`、输入/权限校验和 disabled-by-default 行为。 +- [x] 新增应用健康契约测试，使用本地 HTTPS/HTTP stub 或受控 mock 覆盖状态机、异常输入和隐私边界，不调用真实 webhook。 +- [x] 把新增测试接入最小验证入口；复跑现有自动备份契约，防止共用 helper 回归。 +- [x] 更新现有试运行与自动化验证手册、部署 runbook/checklist 中的安装、只读检查、受控演练和回滚步骤；不新建重复运维文档。 + +## 2. Expected Files + +- `scripts/`：应用健康探测器与共用企微告警 helper 小范围扩展。 +- `deploy/trialrun/`：独立 ARM 安装器及既有 runbook/checklist 更新。 +- `scripts/qa/`：应用健康契约测试与最小门禁接入。 +- `docs/试运行与自动化验证手册.md`：操作与验收边界更新。 + +若实现时发现需要修改 `/ready`、Compose、数据库、Caddy 配置或企业微信业务接口，必须退回规划并重新批准，不得顺手扩大范围。 + +## 3. Validation + +- [x] `python3 -m py_compile <新增探测器> scripts/send-wecom-group-alert.py` +- [x] `bash -n <新增安装器> scripts/qa/<新增契约测试>.sh` +- [x] `bash scripts/qa/<新增契约测试>.sh` +- [x] `bash scripts/qa/run-automatic-backup-contract.sh` +- [x] `node deploy/trialrun/validate-env-contract.js`（证明未引入部署 env 漂移） +- [x] `node scripts/qa/check-process-contract.mjs --baseline --json` +- [x] `node scripts/qa/audit-docs.mjs --check-registry` +- [x] `node scripts/qa/check-trellis-governance.mjs` +- [x] `python3 ./.trellis/scripts/task.py validate 07-27-app-health-monitoring` +- [x] `git diff --check` + +## 4. Review Gates + +- [x] 检查所有告警文本只包含状态、类别和恢复卡提示。 +- [x] 检查首次瞬时失败不发送、第二次持续失败发送、重复失败降噪、恢复只发送一次。 +- [x] 检查 helper 共用状态扩展不改变备份告警行为。 +- [x] 检查安装器默认不 enable、不发送真实消息，卸载不删除共用 helper/webhook 或备份能力。 +- [ ] 实际 ARM 安装、启用、真实告警演练和公网检查另行取得明确批准。 + +## 5. Rollback Points + +- 代码回滚：删除独立探测器/安装器/契约测试，恢复 helper allowlist 与现有文档段落。 +- 远端回滚：先 disable/stop 应用健康 timer，再运行该安装器的 uninstall；保留脱敏状态证据供复盘。 +- 不回滚或删除：自动备份 receiver、备份 dead-man、备份归档、共用 webhook 文件。 diff --git a/.trellis/tasks/07-27-app-health-monitoring/prd.md b/.trellis/tasks/07-27-app-health-monitoring/prd.md new file mode 100644 index 00000000..425fa7d1 --- /dev/null +++ b/.trellis/tasks/07-27-app-health-monitoring/prd.md @@ -0,0 +1,69 @@ +# 应用健康监控 + +## Goal + +为内部 OA 建立低维护、可降噪、可恢复通知的持续健康探测，使应用不可用、数据库不可达或公网 HTTPS 链路异常能够在无人访问时被及时发现，而不引入 Prometheus 等不符合 5–10 人团队维护成本的基础设施。 + +## Scope + +- 在现有 ARM 异机侧每 5 分钟从公网探测 OA `/ready`。 +- 新增独立探测脚本、独立 systemd 安装器和无需真实企微的契约测试。 +- 小范围扩展现有企微群机器人 helper，使其支持应用健康故障、降噪与恢复状态。 +- 更新现有试运行手册、部署 runbook/checklist；代码实现完成后，远端安装、启用和真实告警演练仍需单独批准。 + +## Background and Confirmed Facts + +- `server/src/app.js` 已提供 `/health`、`/ready` 和按配置启用的 `/metrics`；`/ready` 会查询数据库，失败时返回 503，成功响应包含脱敏的构建标识。 +- `deploy/trialrun/docker-compose.yml` 已有数据库与应用容器 healthcheck，但容器 healthcheck 本身不会向管理员发送通知。 +- `deploy/prometheus/alerts-slo.yml` 已定义可用性、延迟和 5xx 告警规则，但当前试运行 Compose 未部署 Prometheus/Alertmanager，因此规则没有持续执行者。 +- 现有部署、live smoke、WARCH3/WAUTO1 等脚本会调用公网 `/ready`，但它们是发布验证或每日业务/归档巡检，不是高频、独立的应用可用性看门狗。 +- `scripts/send-wecom-group-alert.py` 已实现 webhook 文件权限校验、脱敏状态消息、相同故障降噪和故障恢复通知；当前允许的状态类别仅覆盖备份/异机 dead-man，需要为应用健康状态显式扩展或抽取兼容合同。 +- `deploy/trialrun/install-warch3-ops-timer.sh` 和 `deploy/trialrun/install-automatic-backup-timer.sh` 提供了 systemd oneshot/timer、安装/卸载、路径校验和运行账户边界的可复用模式。 +- OA 主机上的 timer 无法在整机宕机、主机断网或 timer 自身停止时发出告警；异机 ARM 探测可覆盖这些故障，但属于外部运行配置，实际安装与启用需要另行明确批准。 + +## Requirements + +- R1：以公网 HTTPS `/ready` 为主要可用性信号，校验 HTTP 状态与响应中的 `data.ok=true`，不得记录完整响应或敏感 payload。 +- R2：探测失败时生成有限、稳定、脱敏的故障类别，至少区分连接/TLS/HTTP/响应契约异常；不得把域名解析结果、地址、凭证、绝对路径或原始日志发送到群内。 +- R3：相同故障连续发生时只发送一次告警；故障类别变化时允许重新告警；恢复后发送一次恢复通知。 +- R4：使用 systemd oneshot + timer 运行，提供安全的安装、启用、状态检查和卸载路径；探测进程不得写 OA 数据库或企业微信业务数据。 +- R5：告警复用现有企微群机器人安全入口及 0600 webhook 文件合同，不新增新的消息供应商或时序数据库。 +- R6：提供无需真实 webhook 的契约测试，覆盖健康、连续相同故障降噪、故障变化、恢复通知、异常状态文件和隐私输出。 +- R7：实际公网安装、systemd 变更、真实企微消息发送和远端配置不属于规划批准；执行前必须再次取得明确批准。 +- R8：首版探测固定运行在现有 ARM 异机侧，每 5 分钟执行一次；连续两次失败才发送故障告警，一次成功即恢复并在此前确有已发送故障时发送一次恢复通知。 + +## Acceptance Criteria + +- [x] AC1：定时探测可对健康公网 `/ready` 返回成功，并保留不含敏感信息的本地状态。 +- [x] AC2：连接失败、TLS 校验失败、非 2xx/503、非法 JSON 或 `data.ok!=true` 均进入稳定的脱敏失败类别并使 oneshot 非零退出。 +- [x] AC3：同一失败重复执行不会重复发送；失败类别变化会发送新告警；恢复后只发送一次恢复通知。 +- [ ] AC4：ARM timer 每 5 分钟执行，单次探测有明确连接/总超时；持续故障在约 10 分钟内告警；安装后可用只读命令核对 enabled/active 及最近执行结果。 +- [x] AC5：测试模式不访问真实企微，自动验证覆盖告警与恢复状态机、输入文件权限和消息脱敏。 +- [x] AC6：现有备份告警、WARCH3 巡检、部署脚本和 `/ready` 合同不被破坏。 +- [x] AC7：规划阶段不修改生产配置；若后续获批部署，公网独立探测和真实告警演练通过后才宣告监控可用。 + +本地实现已完成；AC4 保持未勾选并标记为 `EXTERNAL_GATED`，因为本任务未获 ARM 写入、timer 启用、真实企微消息或故障演练批准。仓库契约通过不代表监控已在运行。 + +## Out of Scope + +- 部署 Prometheus、Alertmanager、Grafana 或通用可观测性平台。 +- 业务登录、审批、报销等合成事务监控；首版只判断基础可用性。 +- 自动重启、自动回滚、数据库修复或流量切换。 +- TLS 证书提前到期提醒；首版通过标准 HTTPS 校验发现已经无效或不受信任的证书，提前到期预警留给后续独立评估。 +- 收集或长期存储访问日志、响应 payload、主机地址及其他敏感运行数据。 +- 本任务不处理组织架构、真实账号导入、发票图片附件或 OCR。 + +## Key Decisions + +- 探测运行在现有 ARM 异机侧，而非 OA 主机，以覆盖 OA 整机、主机网络和本机 timer 同时失效。 +- 每 5 分钟探测，连续两次失败确认持续故障；恢复只需一次成功。 +- 首版复用现有企微群机器人告警入口，不部署 Prometheus，也不执行自动恢复。 +- 应用健康安装器独立于备份 receiver 安装器，避免卸载或升级一个能力时意外影响另一个能力。 + +## Test Plan + +- 静态检查 Python 与 shell 语法。 +- 运行新增应用健康契约测试，覆盖健康、首次失败、持续失败、类别变化、重复降噪、恢复、异常状态和隐私边界。 +- 复跑自动备份契约，证明共用企微 helper 行为未回归。 +- 运行部署 env contract、流程基线、文档审计、Trellis 治理、task validate 与 `git diff --check`。 +- 远端阶段另获批准后，先安装 disabled units，再手工健康探测，最后执行一次受控失败/恢复演练并核对 timer 状态。 diff --git a/.trellis/tasks/07-27-app-health-monitoring/task.json b/.trellis/tasks/07-27-app-health-monitoring/task.json new file mode 100644 index 00000000..e4e73185 --- /dev/null +++ b/.trellis/tasks/07-27-app-health-monitoring/task.json @@ -0,0 +1,26 @@ +{ + "id": "app-health-monitoring", + "name": "app-health-monitoring", + "title": "应用健康监控", + "description": "在现有 ARM 异机侧持续探测 OA 公网就绪状态，并以脱敏、降噪、可恢复通知的企微告警及时发现服务中断。", + "status": "in_progress", + "dev_type": null, + "scope": "ARM off-host public readiness probe, deduplicated WeCom alerting, systemd timer installer, contract tests, and existing runbook updates", + "package": null, + "priority": "P2", + "creator": "shunli", + "assignee": "shunli", + "createdAt": "2026-07-27", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [], + "notes": "", + "meta": {} +} \ No newline at end of file diff --git a/deploy/trialrun/checklist.md b/deploy/trialrun/checklist.md index 874d211b..0e4a8c49 100644 --- a/deploy/trialrun/checklist.md +++ b/deploy/trialrun/checklist.md @@ -57,6 +57,14 @@ - [ ] **EXTERNAL_GATED**：已真正停止一次源端 timer 或故意破坏一次传输，ARM dead-man 仅产生一条脱敏企微告警；恢复 finalize 后收到一次恢复通知，同一失败重复运行已降噪 - [ ] 若调整过 `BACKUP_RUN_AT`，已人工确认调整后的次日 UTC 分桶和 26 小时窗口，未把一次预期空窗误判为长期故障 - [ ] 本季度已由第二名内部人员按 runbook 首部恢复卡在 ARM 隔离目标完成影子恢复走查；timer 未自动恢复生产 +- [ ] ARM 公网应用健康监控（未完成远端门禁前保持未勾选）： + - [ ] 本地 `bash scripts/qa/run-app-health-monitor-contract.sh` 通过，且自动备份告警契约回归通过 + - [ ] **EXTERNAL_GATED**：已批准 ARM systemd 写入，并确认复用 root-owned `0600` webhook；安装的是已通过自动备份回归的共用 helper 修订，不复制或输出 webhook 正文 + - [ ] `install-offhost-app-health-monitor.sh install` 只安装 disabled units；应用健康 state/alert state 与备份 dead-man 完全独立 + - [ ] 手工 oneshot 健康执行通过，输出不含地址、路径、响应 payload、凭证或原始异常 + - [ ] **EXTERNAL_GATED**：另获批准后执行 `enable`，timer 为 enabled/active，oneshot service 平时 inactive，下一次运行不超过 5 分钟 + - [ ] **EXTERNAL_GATED**：持续失败演练中第一次仅 pending、第二次只发一条、重复失败降噪、恢复只通知一次；未用“机器人能发消息”替代故障链路 + - [ ] 已记录剩余风险：ARM 自身失效会使探测失声，首版没有第三方 dead-man - [ ] 停机复制方案已冻结：旧 PostgreSQL 完全停止后复制到空新目录并保留所有权/权限/链接/扩展属性；旧目录原位保留且观察期内不再写入 ## B. 发布（Deploy） diff --git a/deploy/trialrun/install-offhost-app-health-monitor.sh b/deploy/trialrun/install-offhost-app-health-monitor.sh new file mode 100755 index 00000000..a54d394b --- /dev/null +++ b/deploy/trialrun/install-offhost-app-health-monitor.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -euo pipefail + +READY_URL="${READY_URL:-https://oa.hyinda.com/ready}" +PROBE_SOURCE="${PROBE_SOURCE:-}" +PROBE_TARGET="${PROBE_TARGET:-/usr/local/sbin/hyb-oa-public-app-health}" +ALERT_WEBHOOK_FILE="${ALERT_WEBHOOK_FILE:-}" +ALERT_HELPER_SOURCE="${ALERT_HELPER_SOURCE:-}" +ALERT_HELPER_TARGET="${ALERT_HELPER_TARGET:-/usr/local/sbin/hyb-oa-wecom-group-alert}" +STATE_DIR="${STATE_DIR:-/var/lib/hyb-oa-app-health}" +FAILURE_THRESHOLD="${FAILURE_THRESHOLD:-2}" +TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-10}" +SERVICE_NAME="${SERVICE_NAME:-hyb-oa-public-app-health}" +SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" +MODE="${1:-install}" + +fail() { + echo "[app-health-installer][ERROR] $1" >&2 + exit 1 +} + +need_root() { + [[ "$(id -u)" == "0" ]] || fail "root privileges are required" +} + +validate_absolute_path() { + local name="$1" + local value="$2" + if [[ -z "$value" || "$value" != /* || ! "$value" =~ ^/[A-Za-z0-9._/-]+$ ]]; then + fail "$name must be an explicit safe absolute path" + fi + if [[ "$value" == "/" || "$value" == */ || "$value" == *"//"* \ + || "/$value/" == *"/../"* || "/$value/" == *"/./"* ]]; then + fail "$name contains an unsafe path segment" + fi +} + +validate_managed_targets() { + [[ "$PROBE_TARGET" == "/usr/local/sbin/hyb-oa-public-app-health" ]] \ + || fail "PROBE_TARGET must remain /usr/local/sbin/hyb-oa-public-app-health" + [[ "$ALERT_HELPER_TARGET" == "/usr/local/sbin/hyb-oa-wecom-group-alert" ]] \ + || fail "ALERT_HELPER_TARGET must remain /usr/local/sbin/hyb-oa-wecom-group-alert" + [[ "$STATE_DIR" == "/var/lib/hyb-oa-app-health" ]] \ + || fail "STATE_DIR must remain /var/lib/hyb-oa-app-health" + [[ "$SYSTEMD_DIR" == "/etc/systemd/system" ]] \ + || fail "SYSTEMD_DIR must remain /etc/systemd/system" + [[ "$SERVICE_NAME" == "hyb-oa-public-app-health" ]] \ + || fail "SERVICE_NAME must remain hyb-oa-public-app-health" +} + +validate_inputs() { + local name value + for name in PROBE_SOURCE PROBE_TARGET ALERT_WEBHOOK_FILE ALERT_HELPER_SOURCE ALERT_HELPER_TARGET STATE_DIR SYSTEMD_DIR; do + value="${!name:-}" + validate_absolute_path "$name" "$value" + done + validate_managed_targets + [[ "$READY_URL" =~ ^https://[A-Za-z0-9.-]+(:([0-9]{1,5}))?/ready$ ]] \ + || fail "READY_URL must be an HTTPS /ready endpoint without query or fragment" + if [[ -n "${BASH_REMATCH[2]:-}" && ("${BASH_REMATCH[2]}" -lt 1 || "${BASH_REMATCH[2]}" -gt 65535) ]]; then + fail "READY_URL port is invalid" + fi + [[ "$FAILURE_THRESHOLD" == "2" ]] || fail "FAILURE_THRESHOLD must remain 2" + [[ "$TIMEOUT_SECONDS" =~ ^([1-9]|[12][0-9]|30)$ ]] || fail "TIMEOUT_SECONDS must be between 1 and 30" + [[ -f "$PROBE_SOURCE" && ! -L "$PROBE_SOURCE" ]] || fail "probe source is missing" + [[ -f "$ALERT_HELPER_SOURCE" && ! -L "$ALERT_HELPER_SOURCE" ]] || fail "alert helper source is missing" + [[ -f "$ALERT_WEBHOOK_FILE" && ! -L "$ALERT_WEBHOOK_FILE" \ + && "$(stat -c %a "$ALERT_WEBHOOK_FILE")" == "600" \ + && "$(stat -c %u "$ALERT_WEBHOOK_FILE")" == "0" ]] \ + || fail "alert webhook file must be root-owned with mode 0600" + [[ -d "$SYSTEMD_DIR" && ! -L "$SYSTEMD_DIR" ]] || fail "systemd directory is invalid" + command -v python3 >/dev/null 2>&1 || fail "python3 is required" + command -v systemctl >/dev/null 2>&1 || fail "systemctl is required" +} + +install_monitor() { + need_root + validate_inputs + systemctl disable --now "${SERVICE_NAME}.timer" >/dev/null 2>&1 || true + systemctl stop "${SERVICE_NAME}.service" >/dev/null 2>&1 || true + install -o root -g root -m 0755 "$PROBE_SOURCE" "$PROBE_TARGET" + install -o root -g root -m 0755 "$ALERT_HELPER_SOURCE" "$ALERT_HELPER_TARGET" + mkdir -p "$STATE_DIR" + [[ -d "$STATE_DIR" && ! -L "$STATE_DIR" ]] || fail "state directory is invalid" + chown root:root "$STATE_DIR" + chmod 0700 "$STATE_DIR" + + cat >"$SYSTEMD_DIR/${SERVICE_NAME}.service" <"$SYSTEMD_DIR/${SERVICE_NAME}.timer" </dev/null 2>&1 || true + systemctl stop "${SERVICE_NAME}.service" >/dev/null 2>&1 || true + rm -f -- "$SYSTEMD_DIR/${SERVICE_NAME}.timer" "$SYSTEMD_DIR/${SERVICE_NAME}.service" "$PROBE_TARGET" + systemctl daemon-reload + echo "[app-health-installer][PASS] monitor removed; state and shared alert files retained" +} + +case "$MODE" in + install) + install_monitor + ;; + enable) + enable_monitor + ;; + uninstall|--uninstall) + uninstall_monitor + ;; + --help|-h) + echo "Usage: bash install-offhost-app-health-monitor.sh [install|enable|uninstall]" + ;; + *) + fail "unsupported mode" + ;; +esac diff --git a/deploy/trialrun/runbook.md b/deploy/trialrun/runbook.md index 1281b259..974cca3b 100644 --- a/deploy/trialrun/runbook.md +++ b/deploy/trialrun/runbook.md @@ -340,6 +340,37 @@ RESTORE_SCOPE=shadow \ - 确认完整批次已经复制到异机/异盘，而不是仍只保存在应用主机。 - 准备独立 smoke 账号，避免使用老板、管理员或真实员工账号跑自动化。 +## 1.6 ARM 公网应用健康监控（启用仍为 EXTERNAL_GATED） + +应用健康监控与自动备份 dead-man 共用受控企微群机器人入口，但使用独立探测器、systemd units、probe state 和 alert state。它从 ARM 每 5 分钟访问公网 HTTPS `/ready`；连续两次失败才告警，一次成功即恢复。不得把它并入备份 receiver 的安装/卸载生命周期，也不得以本地测试通过代替 ARM 已启用证据。 + +先在仓库运行无真实消息合同： + +```bash +bash scripts/qa/run-app-health-monitor-contract.sh +``` + +获得 ARM systemd 写入批准后，使用受控配置中的绝对路径安装 disabled units。命令中的 webhook 正文不得作为参数、环境变量或聊天内容出现： + +```bash +PROBE_SOURCE= \ +ALERT_WEBHOOK_FILE= \ +ALERT_HELPER_SOURCE= \ +ALERT_HELPER_TARGET= \ + bash deploy/trialrun/install-offhost-app-health-monitor.sh install +``` + +安装后先人工执行一次 oneshot 并核对只含稳定状态/类别的输出；另获真实消息与 timer 启用批准后再执行同参数的 `enable`。检查 enabled/active、最近结果和下一次运行： + +```bash +systemctl is-enabled hyb-oa-public-app-health.timer +systemctl is-active hyb-oa-public-app-health.timer +systemctl list-timers hyb-oa-public-app-health.timer --no-pager +systemctl status hyb-oa-public-app-health.service --no-pager +``` + +真实验收需受控制造持续失败，确认首次失败不发、第二次失败只发一条、相同失败降噪、恢复只通知一次。只测试机器人能发送不算健康监控通过。卸载前先记录脱敏状态，随后执行安装器 `uninstall`；该操作只删除应用健康 units 和探测器，保留状态、共用 helper、webhook、备份 receiver、dead-man 和归档。 + ## 2. 升级流程（试运行） 建议版本化方式： - 每次发布打 tag，例如 `trialrun-2026-02-08-01` diff --git a/docs/试运行与自动化验证手册.md b/docs/试运行与自动化验证手册.md index 8bc34686..820b3328 100644 --- a/docs/试运行与自动化验证手册.md +++ b/docs/试运行与自动化验证手册.md @@ -152,6 +152,27 @@ cd client && E2E_BASE_URL=https://oa.hyinda.com npx playwright test tests/e2e/dg 无法登录、无法审批、无法提交核心单据、付款状态错误、权限越权、`/ready` 异常或冻结项被误开启时，结论应为 Pause，并另建或更新最小修复 Trellis task；不得用重置数据库或生产恢复临时绕过。 +### ARM 异机应用健康监控 + +`07-27-app-health-monitoring` 为公网 `/ready` 增加轻量异机探测：ARM 每 5 分钟经标准 HTTPS 校验访问一次；连续两次失败才发送一条脱敏企微群机器人告警，相同类别持续失败不重复发送，恢复后只通知一次。它覆盖 OA 应用、数据库就绪、OA 主机不可达、DNS/Caddy/TLS 已失效等基础故障，不替代登录、审批或报销等业务 smoke，也不执行自动重启或恢复。 + +当前仓库代码和本地契约通过，不等于 ARM 已安装或监控已生效。远端 `install`、`enable`、真实消息和故障/恢复演练均为 `EXTERNAL_GATED`，需分别取得批准。先运行本地无真实消息合同： + +```bash +bash scripts/qa/run-app-health-monitor-contract.sh +``` + +ARM 安装时必须复用现有 root-owned `0600` 机器人文件，并把本次已通过备份回归的共用告警 helper 修订安装到受控目标；应用健康使用独立 state/alert state，不得覆盖备份 dead-man 状态。`install` 默认停用 timer，只写入受控探测器、helper 修订和 units；人工核对 unit、权限与一次健康手动执行后，才可另行批准 `enable`。只读检查使用： + +```bash +systemctl is-enabled hyb-oa-public-app-health.timer +systemctl is-active hyb-oa-public-app-health.timer +systemctl list-timers hyb-oa-public-app-health.timer --no-pager +systemctl status hyb-oa-public-app-health.service --no-pager +``` + +受控演练必须证明：第一次失败仅进入 pending；第二次持续失败只发一条；重复失败降噪；恢复后只发一条。群消息只允许状态和稳定类别，不粘贴地址、路径、响应、凭证或 journal 原文。ARM 自身宕机仍会令该 timer 失声，首版保留为人工 timer 巡检风险，不据此宣称具备第三方 dead-man。 + 发布候选进入公网或小范围试运行前，先按 [`docs/trialrun/发布试运行准备清单.md`](trialrun/发布试运行准备清单.md) 完成证据整理、权限直连审计、部署预检、live smoke 和试运行分级。 系统应避免让已绑定用户反复登录。若手机企业微信能进入但仍要求登录，优先检查： diff --git a/scripts/check-minimal.sh b/scripts/check-minimal.sh index 5518ed4e..6ee238f2 100755 --- a/scripts/check-minimal.sh +++ b/scripts/check-minimal.sh @@ -28,6 +28,7 @@ echo "=== [2/7] Governance + Sensitive Content ===" cd "$ROOT_DIR" node scripts/qa/check-trellis-governance.mjs node scripts/qa/check-sensitive-content.mjs + bash scripts/qa/run-app-health-monitor-contract.sh bash -n scripts/qa/run-live-smoke-readiness.sh bash -n deploy/trialrun/deploy.sh ) diff --git a/scripts/check-public-app-health.py b/scripts/check-public-app-health.py new file mode 100755 index 00000000..2f08f80f --- /dev/null +++ b/scripts/check-public-app-health.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Probe the public OA readiness endpoint and drive a deduplicated alert.""" + +from __future__ import annotations + +import argparse +import http.client +import json +import os +import re +import socket +import ssl +import stat +import subprocess +import sys +import tempfile +import urllib.error +import urllib.parse +import urllib.request +from pathlib import Path + + +SAFE_TOKEN = re.compile(r"^[A-Za-z0-9_-]{1,160}$") +SAFE_HOST = re.compile(r"^[A-Za-z0-9.-]{1,253}$") +MAX_RESPONSE_BYTES = 64 * 1024 +HEALTHY_STATUS = "PASS_APP_HEALTH_READY" +FAILURE_STATUS = "FAIL_APP_HEALTH" + + +class ProbeFailure(Exception): + def __init__(self, category: str): + super().__init__(category) + self.category = category + + +def fail(category: str) -> "NoReturn": + print(f"[app-health][ERROR] {category}", file=sys.stderr) + raise SystemExit(1) + + +def validate_regular_file(path: Path, label: str, private: bool = False) -> None: + try: + info = path.lstat() + except FileNotFoundError: + fail(f"{label}_missing") + if stat.S_ISLNK(info.st_mode) or not stat.S_ISREG(info.st_mode): + fail(f"{label}_invalid") + if info.st_uid != os.geteuid(): + fail(f"{label}_owner_invalid") + if private and stat.S_IMODE(info.st_mode) != 0o600: + fail(f"{label}_mode_invalid") + if not private and stat.S_IMODE(info.st_mode) & 0o022: + fail(f"{label}_mode_invalid") + + +def prepare_state_dir(path: Path) -> None: + try: + path.mkdir(parents=True, exist_ok=True, mode=0o700) + info = path.lstat() + except OSError: + fail("local_state_invalid") + if ( + stat.S_ISLNK(info.st_mode) + or not stat.S_ISDIR(info.st_mode) + or info.st_uid != os.geteuid() + or stat.S_IMODE(info.st_mode) != 0o700 + ): + fail("local_state_invalid") + + +def validate_ready_url(value: str) -> str: + try: + parsed = urllib.parse.urlsplit(value) + port = parsed.port + except ValueError: + fail("ready_url_invalid") + if ( + parsed.scheme != "https" + or not parsed.hostname + or not SAFE_HOST.fullmatch(parsed.hostname) + or parsed.username is not None + or parsed.password is not None + or parsed.path != "/ready" + or parsed.query + or parsed.fragment + or (port is not None and not 1 <= port <= 65535) + ): + fail("ready_url_invalid") + return value + + +def load_probe_state(path: Path) -> dict[str, object]: + if not path.exists(): + return {"schemaVersion": 1, "consecutiveFailures": 0, "lastCategory": None} + validate_regular_file(path, "local_state", private=True) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + fail("local_state_invalid") + if not isinstance(payload, dict): + fail("local_state_invalid") + count = payload.get("consecutiveFailures") + category = payload.get("lastCategory") + if ( + payload.get("schemaVersion") != 1 + or not isinstance(count, int) + or isinstance(count, bool) + or count < 0 + or (category is not None and (not isinstance(category, str) or not SAFE_TOKEN.fullmatch(category))) + ): + fail("local_state_invalid") + return payload + + +def atomic_write(path: Path, payload: dict[str, object]) -> None: + fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=True, indent=2) + handle.write("\n") + os.replace(temporary, path) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + + +def classify_url_error(error: urllib.error.URLError) -> str: + reason = error.reason + if isinstance(reason, (ssl.SSLError, ssl.CertificateError)): + return "tls_validation_failed" + if isinstance(reason, (socket.timeout, TimeoutError)): + return "request_timeout" + return "dns_or_connect_failed" + + +def probe(ready_url: str, timeout_seconds: float, ca_file: Path | None) -> None: + if ca_file is not None: + validate_regular_file(ca_file, "ca_file") + try: + context = ssl.create_default_context(cafile=str(ca_file) if ca_file else None) + request = urllib.request.Request( + ready_url, + headers={"Accept": "application/json", "User-Agent": "hyb-oa-app-health/1"}, + method="GET", + ) + with urllib.request.urlopen(request, timeout=timeout_seconds, context=context) as response: + if not 200 <= response.status < 300: + raise ProbeFailure("http_unavailable") + content_length = response.headers.get("Content-Length") + if content_length: + try: + if int(content_length) > MAX_RESPONSE_BYTES: + raise ProbeFailure("response_too_large") + except ValueError: + raise ProbeFailure("response_contract_invalid") from None + body = response.read(MAX_RESPONSE_BYTES + 1) + except urllib.error.HTTPError: + raise ProbeFailure("http_unavailable") from None + except urllib.error.URLError as error: + raise ProbeFailure(classify_url_error(error)) from None + except (socket.timeout, TimeoutError): + raise ProbeFailure("request_timeout") from None + except (ssl.SSLError, ssl.CertificateError): + raise ProbeFailure("tls_validation_failed") from None + except (http.client.HTTPException, ConnectionError, OSError): + raise ProbeFailure("dns_or_connect_failed") from None + if len(body) > MAX_RESPONSE_BYTES: + raise ProbeFailure("response_too_large") + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise ProbeFailure("response_contract_invalid") from None + if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict) or payload["data"].get("ok") is not True: + raise ProbeFailure("response_contract_invalid") + + +def write_summary(path: Path, status: str, category: str) -> None: + finding = status == FAILURE_STATUS + payload = { + "issueId": "APP_HEALTH", + "status": status, + "conclusion": status, + "findingCount": 1 if finding else 0, + "findings": ( + [{"source": "public_ready", "status": status, "severity": "P0", "detail": category}] + if finding + else [] + ), + "privacy": "status/count/category only; no credentials, addresses, paths, or payloads", + } + atomic_write(path, payload) + + +def evaluate_alert(alert_helper: Path, webhook_file: Path, summary_file: Path, alert_state_file: Path) -> None: + validate_regular_file(alert_helper, "alert_helper") + try: + result = subprocess.run( + [ + sys.executable, + str(alert_helper), + "--webhook-file", + str(webhook_file), + "--summary-file", + str(summary_file), + "--state-file", + str(alert_state_file), + ], + check=False, + ) + except OSError: + fail("alert_evaluation_failed") + if result.returncode != 0: + fail("alert_evaluation_failed") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--ready-url", required=True) + parser.add_argument("--state-dir", required=True, type=Path) + parser.add_argument("--webhook-file", required=True, type=Path) + parser.add_argument("--alert-helper", required=True, type=Path) + parser.add_argument("--failure-threshold", type=int, default=2) + parser.add_argument("--timeout-seconds", type=float, default=10.0) + parser.add_argument("--ca-file", type=Path) + args = parser.parse_args() + + ready_url = validate_ready_url(args.ready_url) + if not 1 <= args.failure_threshold <= 10: + fail("failure_threshold_invalid") + if not 0.1 <= args.timeout_seconds <= 30: + fail("timeout_invalid") + prepare_state_dir(args.state_dir) + probe_state_file = args.state_dir / "app-health-probe-state.json" + summary_file = args.state_dir / "last-app-health.json" + alert_state_file = args.state_dir / "app-health-alert-state.json" + state = load_probe_state(probe_state_file) + + try: + probe(ready_url, args.timeout_seconds, args.ca_file) + except ProbeFailure as error: + failures = min(int(state["consecutiveFailures"]) + 1, args.failure_threshold) + try: + atomic_write( + probe_state_file, + {"schemaVersion": 1, "consecutiveFailures": failures, "lastCategory": error.category}, + ) + except OSError: + fail("local_state_invalid") + if failures >= args.failure_threshold: + try: + write_summary(summary_file, FAILURE_STATUS, error.category) + except OSError: + fail("local_state_invalid") + evaluate_alert(args.alert_helper, args.webhook_file, summary_file, alert_state_file) + print( + f"[app-health][ERROR] health_check_failed category={error.category} consecutive={failures} alert=evaluated", + file=sys.stderr, + ) + else: + print( + f"[app-health][ERROR] health_check_failed category={error.category} consecutive={failures} alert=pending", + file=sys.stderr, + ) + raise SystemExit(1) + + try: + atomic_write( + probe_state_file, + {"schemaVersion": 1, "consecutiveFailures": 0, "lastCategory": None}, + ) + write_summary(summary_file, HEALTHY_STATUS, "public_ready_healthy") + except OSError: + fail("local_state_invalid") + evaluate_alert(args.alert_helper, args.webhook_file, summary_file, alert_state_file) + print("[app-health][PASS] public readiness is healthy") + + +if __name__ == "__main__": + main() diff --git a/scripts/qa/run-app-health-monitor-contract.sh b/scripts/qa/run-app-health-monitor-contract.sh new file mode 100755 index 00000000..e717ab63 --- /dev/null +++ b/scripts/qa/run-app-health-monitor-contract.sh @@ -0,0 +1,276 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +TEST_DIR="$(mktemp -d)" +STUB_PID="" +PASS_COUNT=0 + +cleanup() { + if [[ -n "$STUB_PID" ]]; then + kill "$STUB_PID" >/dev/null 2>&1 || true + wait "$STUB_PID" >/dev/null 2>&1 || true + fi + rm -rf -- "$TEST_DIR" +} +trap cleanup EXIT + +pass() { + PASS_COUNT=$((PASS_COUNT + 1)) + echo "[PASS] $1" +} + +expect_failure() { + local label="$1" + shift + if "$@"; then + echo "[FAIL] expected failure: $label" >&2 + exit 1 + fi + pass "$label" +} + +capture_lines() { + local capture="$1" + if [[ -f "$capture" ]]; then + wc -l <"$capture" | tr -d ' ' + else + echo 0 + fi +} + +new_state_dir() { + local target="$1" + mkdir -p "$target" + chmod 0700 "$target" +} + +set_mode() { + printf '%s\n' "$1" >"$TEST_DIR/mode" +} + +probe() { + local state_dir="$1" + local capture="$2" + shift 2 + env HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$capture" \ + python3 "$ROOT_DIR/scripts/check-public-app-health.py" \ + --ready-url "https://localhost:${STUB_PORT}/ready" \ + --state-dir "$state_dir" \ + --webhook-file "$TEST_DIR/wecom-webhook" \ + --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \ + --failure-threshold 2 \ + --timeout-seconds 1 \ + --ca-file "$TEST_DIR/cert.pem" \ + "$@" +} + +openssl req -x509 -newkey rsa:2048 -nodes -sha256 -days 1 \ + -subj '/CN=localhost' -addext 'subjectAltName=DNS:localhost' \ + -keyout "$TEST_DIR/key.pem" -out "$TEST_DIR/cert.pem" >/dev/null 2>&1 +chmod 0600 "$TEST_DIR/key.pem" "$TEST_DIR/cert.pem" +WECOM_TEST_HOST='qyapi.weixin.qq.com' +WECOM_TEST_PATH='/cgi-bin/webhook/send' +printf 'https://%s%s?key=%s\n' "$WECOM_TEST_HOST" "$WECOM_TEST_PATH" \ + '01234567-89ab-cdef-0123-456789abcdef' >"$TEST_DIR/wecom-webhook" +chmod 0600 "$TEST_DIR/wecom-webhook" +printf '%s\n' healthy >"$TEST_DIR/mode" + +python3 - "$TEST_DIR/cert.pem" "$TEST_DIR/key.pem" "$TEST_DIR/mode" "$TEST_DIR/port" <<'PY' >/dev/null 2>&1 & +import http.server +import json +import ssl +import sys +import time +from pathlib import Path + +cert_file, key_file, mode_file, port_file = map(Path, sys.argv[1:]) + + +class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + mode = mode_file.read_text(encoding="utf-8").strip() + if self.path != "/ready": + self.send_response(404) + self.end_headers() + return + if mode == "timeout": + time.sleep(2) + if mode == "unavailable": + status, body = 503, b'{"error":{"details":{"db":"do-not-leak"}}}' + elif mode == "invalid-json": + status, body = 200, b'not-json-private-payload' + elif mode == "bad-contract": + status, body = 200, json.dumps({"data": {"ok": False}}).encode() + elif mode == "oversized": + status, body = 200, b"x" * (64 * 1024 + 1) + else: + status, body = 200, json.dumps({"data": {"ok": True, "buildSha": "abcdef123456"}}).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + +server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) +context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +context.load_cert_chain(certfile=cert_file, keyfile=key_file) +server.socket = context.wrap_socket(server.socket, server_side=True) +port_file.write_text(str(server.server_port), encoding="utf-8") +server.serve_forever() +PY +STUB_PID=$! + +for _ in $(seq 1 50); do + [[ -s "$TEST_DIR/port" ]] && break + sleep 0.1 +done +[[ -s "$TEST_DIR/port" ]] +STUB_PORT="$(cat "$TEST_DIR/port")" + +MAIN_STATE="$TEST_DIR/main-state" +MAIN_CAPTURE="$TEST_DIR/main-capture.jsonl" +new_state_dir "$MAIN_STATE" +probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null +[[ "$(capture_lines "$MAIN_CAPTURE")" == "0" ]] +pass "healthy baseline writes state without sending an alert" + +set_mode unavailable +expect_failure "first failure stays pending" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1 +[[ "$(capture_lines "$MAIN_CAPTURE")" == "0" ]] +expect_failure "second consecutive failure triggers alert evaluation" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1 +[[ "$(capture_lines "$MAIN_CAPTURE")" == "1" ]] +expect_failure "identical continued failure remains non-zero" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1 +[[ "$(capture_lines "$MAIN_CAPTURE")" == "1" ]] +pass "identical application failure is deduplicated" + +set_mode invalid-json +expect_failure "changed failure category remains non-zero" probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null 2>&1 +[[ "$(capture_lines "$MAIN_CAPTURE")" == "2" ]] +pass "changed failure category sends one updated alert" + +set_mode healthy +probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null +[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]] +probe "$MAIN_STATE" "$MAIN_CAPTURE" >/dev/null +[[ "$(capture_lines "$MAIN_CAPTURE")" == "3" ]] +pass "recovery sends exactly one notification" + +TRANSIENT_STATE="$TEST_DIR/transient-state" +TRANSIENT_CAPTURE="$TEST_DIR/transient-capture.jsonl" +new_state_dir "$TRANSIENT_STATE" +probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null +set_mode unavailable +expect_failure "single transient outage is recorded" probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null 2>&1 +set_mode healthy +probe "$TRANSIENT_STATE" "$TRANSIENT_CAPTURE" >/dev/null +[[ "$(capture_lines "$TRANSIENT_CAPTURE")" == "0" ]] +pass "single transient outage does not send failure or recovery noise" + +for mode in bad-contract oversized timeout; do + CASE_STATE="$TEST_DIR/${mode}-state" + new_state_dir "$CASE_STATE" + set_mode "$mode" + expect_failure "$mode is rejected" probe "$CASE_STATE" "$TEST_DIR/${mode}-capture.jsonl" >/dev/null 2>&1 + python3 - "$CASE_STATE/app-health-probe-state.json" "$mode" <<'PY' +import json, sys +expected = { + "bad-contract": "response_contract_invalid", + "oversized": "response_too_large", + "timeout": "request_timeout", +}[sys.argv[2]] +payload = json.load(open(sys.argv[1], encoding="utf-8")) +assert payload["lastCategory"] == expected, payload +PY +done + +UNUSED_PORT="$(python3 - <<'PY' +import socket +sock = socket.socket() +sock.bind(("127.0.0.1", 0)) +print(sock.getsockname()[1]) +sock.close() +PY +)" +CONNECT_STATE="$TEST_DIR/connect-state" +new_state_dir "$CONNECT_STATE" +expect_failure "connection failure is rejected" env \ + HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/connect-capture.jsonl" \ + python3 "$ROOT_DIR/scripts/check-public-app-health.py" \ + --ready-url "https://localhost:${UNUSED_PORT}/ready" --state-dir "$CONNECT_STATE" \ + --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \ + --failure-threshold 2 --timeout-seconds 1 --ca-file "$TEST_DIR/cert.pem" >/dev/null 2>&1 +python3 - "$CONNECT_STATE/app-health-probe-state.json" <<'PY' +import json, sys +payload = json.load(open(sys.argv[1], encoding="utf-8")) +assert payload["lastCategory"] == "dns_or_connect_failed", payload +PY +pass "connection failure is stored as a stable category" + +TLS_STATE="$TEST_DIR/tls-state" +new_state_dir "$TLS_STATE" +expect_failure "untrusted TLS certificate is rejected" env \ + HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/tls-capture.jsonl" \ + python3 "$ROOT_DIR/scripts/check-public-app-health.py" \ + --ready-url "https://localhost:${STUB_PORT}/ready" --state-dir "$TLS_STATE" \ + --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \ + --failure-threshold 2 --timeout-seconds 1 >/dev/null 2>&1 +python3 - "$TLS_STATE/app-health-probe-state.json" <<'PY' +import json, sys +payload = json.load(open(sys.argv[1], encoding="utf-8")) +assert payload["lastCategory"] == "tls_validation_failed", payload +PY +pass "TLS failure is stored as a stable category" + +CORRUPT_STATE="$TEST_DIR/corrupt-state" +new_state_dir "$CORRUPT_STATE" +printf '%s\n' 'private-payload-must-not-appear' >"$CORRUPT_STATE/app-health-probe-state.json" +chmod 0600 "$CORRUPT_STATE/app-health-probe-state.json" +set_mode healthy +expect_failure "corrupt local state fails closed" probe "$CORRUPT_STATE" "$TEST_DIR/corrupt-capture.jsonl" \ + >"$TEST_DIR/corrupt-stdout" 2>"$TEST_DIR/corrupt-stderr" +grep -Fxq '[app-health][ERROR] local_state_invalid' "$TEST_DIR/corrupt-stderr" +! grep -Fq 'private-payload' "$TEST_DIR/corrupt-stderr" +! grep -Fq "$TEST_DIR" "$TEST_DIR/corrupt-stderr" +pass "local state error output is stable and redacted" + +python3 - "$MAIN_CAPTURE" <<'PY' +import json, sys +messages = [json.loads(line)["text"]["content"] for line in open(sys.argv[1], encoding="utf-8")] +assert len(messages) == 3, messages +assert "FAIL_APP_HEALTH" in messages[0] +assert "http_unavailable" in messages[0] +assert "response_contract_invalid" in messages[1] +assert "PASS_APP_HEALTH_READY" in messages[2] +for forbidden in ("do-not-leak", "localhost", "https://", "/tmp/", "not-json-private-payload"): + assert all(forbidden not in message for message in messages), (forbidden, messages) +PY +pass "captured alerts contain only stable status and category text" + +expect_failure "plain HTTP readiness URL is rejected" env \ + HYB_WECOM_ALERT_TEST_MODE=1 HYB_WECOM_ALERT_TEST_CAPTURE="$TEST_DIR/http-capture.jsonl" \ + python3 "$ROOT_DIR/scripts/check-public-app-health.py" \ + --ready-url "http://localhost:${STUB_PORT}/ready" --state-dir "$TEST_DIR/http-state" \ + --webhook-file "$TEST_DIR/wecom-webhook" --alert-helper "$ROOT_DIR/scripts/send-wecom-group-alert.py" \ + >/dev/null 2>&1 + +PYTHONPYCACHEPREFIX="$TEST_DIR/pycache" python3 -m py_compile \ + "$ROOT_DIR/scripts/check-public-app-health.py" "$ROOT_DIR/scripts/send-wecom-group-alert.py" +bash -n "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +grep -Fq 'OnCalendar=*:0/5' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +systemd-analyze calendar '*:0/5' >/dev/null +grep -Fq 'disabled application health monitor installed' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +grep -Fq 'install -o root -g root -m 0755 "$ALERT_HELPER_SOURCE" "$ALERT_HELPER_TARGET"' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +grep -Fq 'state and shared alert files retained' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +grep -Fq 'PROBE_TARGET must remain /usr/local/sbin/hyb-oa-public-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +grep -Fq 'ALERT_HELPER_TARGET must remain /usr/local/sbin/hyb-oa-wecom-group-alert' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +grep -Fq 'STATE_DIR must remain /var/lib/hyb-oa-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +grep -Fq 'SERVICE_NAME must remain hyb-oa-public-app-health' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +grep -Fq 'SYSTEMD_DIR must remain /etc/systemd/system' "$ROOT_DIR/deploy/trialrun/install-offhost-app-health-monitor.sh" +pass "installer is five-minute, disabled-by-default, exact-target restricted, and preserves shared state" + +printf '[PASS] public application health contract completed (%d checks)\n' "$PASS_COUNT" diff --git a/scripts/send-wecom-group-alert.py b/scripts/send-wecom-group-alert.py index 8a5e647c..49ded119 100755 --- a/scripts/send-wecom-group-alert.py +++ b/scripts/send-wecom-group-alert.py @@ -17,8 +17,10 @@ from pathlib import Path SAFE_TOKEN = re.compile(r"^[A-Za-z0-9_-]{1,160}$") ALLOWED_STATUSES = { + "FAIL_APP_HEALTH", "FAIL_AUTOMATIC_BACKUP", "FAIL_OFFHOST_DEADMAN", + "PASS_APP_HEALTH_READY", "PASS_AUTOMATIC_BACKUP_COMPLETE", "PASS_AUTOMATIC_BACKUP_FRESH", "PASS_OFFHOST_DEADMAN_FRESH",