From f223bfb72fb2baef51199702c2def84c41af7aef Mon Sep 17 00:00:00 2001 From: shunli Date: Sat, 5 Sep 2026 21:02:58 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E9=9A=94=E7=A6=BB=E6=97=A2=E6=9C=89?= =?UTF-8?q?=E5=8F=91=E5=B8=83=E7=9A=84=E6=95=B0=E6=8D=AE=E5=BA=93=E5=AE=B9?= =?UTF-8?q?=E5=99=A8=E7=94=9F=E5=91=BD=E5=91=A8=E6=9C=9F=EF=BC=8809-05-exi?= =?UTF-8?q?sting-deploy-db-lifecycle-isolation=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .trellis/spec/hyb-oa/deployment-trialrun.md | 90 ++++++ .../check.jsonl | 3 + .../design.md | 45 +++ .../implement.jsonl | 3 + .../implement.md | 82 ++++++ .../prd.md | 60 ++++ .../task.json | 35 +++ deploy/trialrun/deploy.sh | 90 +++++- scripts/qa/run-release-gate.sh | 1 + .../unit/fixtures/deployment-command-fake.cjs | 156 +++++++++++ .../scripts/deployment-db-lifecycle.test.js | 264 ++++++++++++++++++ 11 files changed, 822 insertions(+), 7 deletions(-) create mode 100644 .trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/check.jsonl create mode 100644 .trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/design.md create mode 100644 .trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/implement.jsonl create mode 100644 .trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/implement.md create mode 100644 .trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/prd.md create mode 100644 .trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/task.json create mode 100644 server/tests/unit/fixtures/deployment-command-fake.cjs create mode 100644 server/tests/unit/scripts/deployment-db-lifecycle.test.js diff --git a/.trellis/spec/hyb-oa/deployment-trialrun.md b/.trellis/spec/hyb-oa/deployment-trialrun.md index e7600b9f..a1a19f55 100644 --- a/.trellis/spec/hyb-oa/deployment-trialrun.md +++ b/.trellis/spec/hyb-oa/deployment-trialrun.md @@ -19,6 +19,96 @@ operations require explicit approval and evidence. - Do not execute real payments, import bank statements, connect payment providers or write ERP without a separate approved Trellis task. +## Scenario: Existing Application Release Must Not Manage Database Containers + +### 1. Scope / Trigger + +- Applies to `deploy-existing.sh` → `deploy.sh` and direct `deploy.sh` execution. +- Separate application publication from database container lifecycle management; + this does not make migrations or seed operations read-only. + +### 2. Signatures + +- `DEPLOY_DATA_MODE=existing|initialize`; the existing wrapper pins `existing`. +- Only `initialize` may explicitly run Compose `up -d db`. +- One-shot init: `run --rm --no-deps -T --entrypoint sh init ...`. +- Application: explicit `build server`, then `up -d --no-build --no-deps server`; + optional worker and explicitly selected MinIO services also isolate dependencies. +- Database readiness: `timeout --signal=KILL 120 bash -s -- `; + poll every 2 seconds with no environment override for the 120-second deadline. + +### 3. Contracts + +- Existing releases may query database state but must not start, stop, restart, + recreate, replace or remove db, explicitly or via dependency propagation. + A project-wide lifecycle operation is not safe merely because it omits `db`. +- Resolve exactly one container through the approved Compose project/service. + Inspect its running state and Docker health, replacing the `service_healthy` + startup guarantee otherwise lost when dependency startup is disabled. +- The deadline encloses both polling and potentially stuck Compose/inspect calls. + Missing, stopped, unhealthy, ambiguous or uninspectable db must fail; only + `running|starting` waits, and only `running|healthy` proceeds. No automatic + initialize fallback, restart, permission repair or health-bypass option. +- Initialize starts db explicitly, then passes the same health gate before init. + Reuse the gate after init build, after init and after server build, before each + application switch; a healthy snapshot before a long build is not sufficient. + Prepare worker before switching server: resolve its existing container through + Compose, inspect Config.Image and verify that image reference, not just a retained + image ID whose tag may be gone. Build explicitly when no container/reference is + available, then check health and start with `--no-build --no-deps`. A cached image + without a container may incur an extra cached build. This assumes the current + build-only worker definition; future image overrides need their own compatibility + review, rather than treating a previous container reference as new configuration. + Preserve migration/seed command content and order, safe logs, identity/path + preflights, rollback window, build markers and backup-contract refresh. +- Missing timeout tooling or unsupported isolation options fail closed. Do not + retry a command without isolation to accommodate an older Compose executable. + +### 4. Validation & Error Matrix + +- Missing/invalid mode → nonzero before container operations. +- Missing timeout tool → `db_health_timeout_unavailable`, no db start or init. +- Missing/non-unique db → `db_container_missing_or_ambiguous`. +- Query failure → `db_health_query_failed`; invalid/unhealthy state → `db_not_healthy`. +- Deadline → `db_health_timeout`, nonzero; never continue init/application switch. +- Init failure → preserve failure status and stop before application switch. + +### 5. Good / Base / Bad Cases + +- Good: existing healthy db → isolated init → isolated application switch; + captured effective lifecycle targets never include db. +- Base: initialize → explicit db start → bounded healthy check → isolated init. +- Bad: remove only the explicit `up db` but leave init `run` or application `up` + free to start dependencies; grep for one `--no-deps` cannot prove safety. + +### 6. Tests Required + +- Execute the actual remote shell from current source in a private temporary + fixture; capture argv and resolve effective service targets from the real + Compose dependency graph. Assert execution reaches init and expected services. +- Cover V1/V2, both modes, worker/MinIO profiles and failure cases; mutate explicit + db startup and each init/server/worker dependency isolation independently. +- Change db health after build/init/server-switch stages and prove the next + guarded action never runs; verify image builds cannot follow the last health + check implicitly inside application `up`. +- Cover a retained worker container/image ID with a missing launch tag; rebuild + must happen before the server switch, not fail later at isolated worker startup. +- Test the real deadline mechanism with a shortened deadline only in a temporary + source copy, including a hung inspect; never shorten the production constant. +- Fake binaries and whitelisted PATH/env exist only in test subprocesses. Unknown + commands fail, real Docker/SSH/production credentials remain inaccessible. +- Run `server/tests/unit/scripts/deployment-db-lifecycle.test.js` in static gates + and retain existing `deployment-safety-contract.test.js` coverage. + +### 7. Wrong vs Correct + +- Wrong: database unavailable → application deploy silently runs `up db`. +- Correct: database unavailable → stop and request separate fault recovery. +- First authorized real release needs an operator, pre/post db container identity + comparison and separate application-image/tool recovery references. Keep the + old tool source as evidence, not a ready-to-run command sequence that reintroduces + unapproved db lifecycle operations or replays migrations/seeds. + ## Scenario: Exact Production Identity Bootstrap Before Migration Status ### 1. Scope / Trigger diff --git a/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/check.jsonl b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/check.jsonl new file mode 100644 index 00000000..31a8226f --- /dev/null +++ b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/check.jsonl @@ -0,0 +1,3 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} +{"file": ".trellis/spec/hyb-oa/deployment-trialrun.md", "reason": "既有部署身份、数据路径、迁移及备份验证边界"} +{"file": ".trellis/spec/hyb-oa/security-privacy.md", "reason": "合成测试与生产敏感信息保护边界"} diff --git a/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/design.md b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/design.md new file mode 100644 index 00000000..4a8d0a7e --- /dev/null +++ b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/design.md @@ -0,0 +1,45 @@ +# 既有环境发布的数据库容器隔离设计(待审) + +## 调用链与最小修改点 + +`production-identity-bootstrap.mjs` → `deploy-existing.sh`(固定 existing,身份与迁移预检)→ `deploy.sh`(本地校验/同步 → SSH 远端段 → Compose)。修改真正执行命令的远端段,而非仅在 wrapper 中增加一个无消费者的参数。直接调用 deploy.sh 也必须遵守相同模式约束。 + +## 容器执行合同 + +| 阶段 | existing | initialize | +| --- | --- | --- | +| 数据路径 | 维持 existing 检查,不补建 PGDATA | 维持现有初始化检查与目录准备 | +| db | 仅只读查询/就绪检查,不执行 up | 显式 up db 后有界等待健康 | +| init | build init;run --rm --no-deps,保留 -T、entrypoint 与原命令 | 同样隔离依赖,数据库已在上一阶段就绪 | +| server | build server → 健康检查 → up -d --no-build --no-deps server | 同样显式指定服务与隔离依赖 | +| worker(启用时) | 检查现有镜像,必要时先 build worker → 健康检查 → up -d --no-build --no-deps worker | 同左 | +| 可选 MinIO | 保留既有功能,显式限定服务;检查其有效依赖集合不得触达 db | 保留既有功能 | + +模式分支包围显式 db 操作本身;不以日志、注释或上游约定代替。不要在本任务新增“绕过隔离”的开关。已有 MinIO 初始化的依赖需要保留,测试按真实图判定,不将所有依赖无差别禁止。 + +## 就绪与失败 + +通过已确定的 Compose project/service 查询 db 容器并只读检查运行和就绪状态,不能用历史固定容器名。就绪检查采用总时限,沿用既有健康语义;同一检查函数复用于初始阶段、build init 后、init 后、build server 后、worker 准备前后,避免耗时阶段后沿用过期健康快照。existing db 不可用时停止,不运行 up/restart 作为恢复手段;initialize 创建后等待就绪。init 失败沿现有 set -e/受控日志传播,不切换应用。 + +只读 review 发现初稿单次提前检查不足以替代各阶段 service_healthy,现将 server build 与启动拆开,启动使用 --no-build 避免检查后的隐式构建。worker 不一律重建:在 server 切换之前通过 Compose 解析现有 worker 容器,读取其 Config.Image 并确认对应镜像引用可用;仅有旧镜像 ID 不足以证明标签存在。无容器或引用缺失时显式构建后检查健康;无容器但有缓存镜像的情况可能多一次命中缓存的构建,接受此有界兼容差异以消除隐式构建窗口。当前 Compose worker 没有显式 image 覆盖,本任务不改镜像配置、Compose 拓扑或 init 命令内容;未来修改镜像来源需单独验证所需引用与配置一致,不能以已有容器引用代表新配置。MinIO 显式选择 minio/minio_init 并隔离其他依赖,保留初始化容器自己的已有就绪重试。 + +此保护只隔离容器生命周期。迁移和 seed 会访问数据库内容;保留原有授权、预检和备份约束,不能声称 --no-deps 能代替数据库备份或迁移审查。 + +## 命令捕获与行为判定 + +复用现有 Vitest 临时目录、假二进制模式,新建专用测试,避免继续堆叠静态 contains 断言。优先以 fake SSH 在隔离目录执行生产脚本发送的原始远端 shell,或从当前源码提取唯一 REMOTE heredoc 执行并另外覆盖 wrapper 模式传递;不得手抄一份发布流程作为测试对象。 + +- fake 记录 JSON 参数数组,区分 compose 全局选项、子命令、服务参数与 init 容器内命令,不把 shell 命令中的词误判为服务。 +- 由仓库 Compose 服务依赖生成受测依赖图,不能硬编码一份与实际声明脱节的图。解析未知或含糊命令时失败;测试辅助依赖仅使用项目现有工具,必要新增依赖须再确认。 +- 对 up/run 等有依赖传播的操作计算有效服务集合;对 down 或无服务名的项目级生命周期操作按整个项目判定;stop/start/restart/rm/create/kill 等显式触达 db 的操作也拒绝。运行 `init --rm` 仅清理一次性 init 容器,不误判为删除 db。 +- existing 断言 db 生命周期事件列表为空,同时断言 init/server/预期 worker 出现,执行順序正确,入口成功完成目标阶段;initialize 断言 db 启动在 init 之前且健康检查生效。 +- 源码变异只作用于临时副本:恢复无条件 up db,移除 init/server/worker 各自隔离。每个变异须命中预期修改点并导致具体 db 违规测试失败,禁止 needle 未命中却算失败探针通过。 +- 拦截所有外部命令并清理继承敏感环境;未知外部调用直接失败,不允许落到真实 PATH 工具或网络。 + +## 兼容、复核与回滚 + +就绪循环的总时限固定 120 秒,轮询间隔 2 秒;用受限子进程强制截止,包含 Compose/inspect 调用卡住的情况,不仅限制循环次数。超时、查询错误、缺失/停止/不健康/无健康检查均失败退出,不继续 init。测试 fake 在专用 mkdtemp 目录,子进程传入白名单环境与独立 PATH,不更改 process.env、共享 shell 配置或生产文件。 + +首次真实执行前的工具恢复参考:保留 `05d2a3f` 的 `deploy/trialrun/deploy.sh`(可用 `git show 05d2a3f:deploy/trialrun/deploy.sh` 只读提取)。其原命令序列为 `up -d db` → 可选 MinIO → `build init` → `run --rm -T --entrypoint sh init ...migrate...seed` → `up -d --build server` → 可选 `up -d worker`。这是有风险的旧行为证据,不是获准执行的恢复脚本;禁止盲目复制整段运行。首次生产验证需人工在场,确认 Compose 身份、既有 db 健康、已知可用应用镜像及恢复指令;若新工具失败,先停并区分工具/应用/数据库故障,人工恢复应用也应显式隔离依赖,不重跑 init/seed 或自动启动 db。应用镜像回滚与部署工具恢复分别准备,执行均另行批准。 + +覆盖 V2 与 V1 fallback,不改 Compose 拓扑和生产配置。只读 reviewer 不写代码,重点查三条入口、健康时序与 fake 是否掩盖风险。缺独立复核能力时明确待复核,不以主会话自检替代。回滚不得无说明地恢复允许触碰 db 的旧发布路径;若防护使既有发布暂停,保持暂停并单独审查修复,不以紧急发布为由关闭保护。 diff --git a/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/implement.jsonl b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/implement.jsonl new file mode 100644 index 00000000..31a8226f --- /dev/null +++ b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/implement.jsonl @@ -0,0 +1,3 @@ +{"_example": "Fill with {\"file\": \"\", \"reason\": \"\"}. Put spec/research files only — no code paths. Run `python3 .trellis/scripts/get_context.py --mode packages` to list available specs. Delete this line once real entries are added."} +{"file": ".trellis/spec/hyb-oa/deployment-trialrun.md", "reason": "既有部署身份、数据路径、迁移及备份验证边界"} +{"file": ".trellis/spec/hyb-oa/security-privacy.md", "reason": "合成测试与生产敏感信息保护边界"} diff --git a/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/implement.md b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/implement.md new file mode 100644 index 00000000..bea6e874 --- /dev/null +++ b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/implement.md @@ -0,0 +1,82 @@ +# 执行清单与验证记录 + +## 授权与工作区 + +- 2026-09-05 用户已批准 PRD 并实施,task 已幂等 start,状态 in_progress;实施后先交独立 diff,不自动提交或部署。 +- 基线 HEAD:`05d2a3f`。已有旧 task 的 implement.md 未提交交接笔记必须原样保留,不纳入本 task 提交;Trellis 升级独立 worktree 不操作。 +- 当前无原生 update_plan,以本文件为执行清单,不另建计划系统。inline 主会话实施和检查,不派 implement/check 子代理;实施后的只读独立 reviewer 按用户要求执行。 +- deployment spec 体积超过自动上下文注入上限,task validate 会提示截断风险;主会话及后续只读 reviewer 必须按目标章节直接读取源文件,不以自动注入内容代表完整规约,不在本 task 顺手拆分 spec 或调整注入配置。 + +## 已完成规划核对 + +- [x] 核对 wrapper 固定 existing、远端模式校验与显式 db/间接依赖三个入口。 +- [x] 确认 server、worker、init 都在当前 Compose 文件依赖健康 db;MinIO profile 单独覆盖。 +- [x] 检查现有 deployment-safety-contract.test.js 的 fake PATH/临时目录及静态合同模式。 +- [x] 完成 PRD、设计、实施顺序和验收矩阵;只写当前 task 材料。 +- [x] 用户审阅本轮最终方案并批准实施;将健康等待明确为固定 120 秒,测试 PATH 仅作用于临时子进程,保留旧工具命令作为首次真实发布的人工恢复参考。 + +## 批准后实施顺序 + +1. 读取 trellis-before-dev 与 deployment/test/spec 相关约束,复核未变化基线;先用真实远端段构建命令捕获回归,证明原行为触发 db 违规。 +2. 将显式 db up 限制到 initialize;补 existing/initialize 的有界只读就绪检查,保持既有安全门禁。 +3. init/server/worker 隔离依赖,验证可选 profile 无 db 旁路;不改 migration/seed 内容。 +4. 执行正向矩阵与四类源码变异,验证具体违规服务与执行阶段;补非法模式、db 不可用、init 失败阻断。 +5. 将专用测试接入 run-release-gate.sh static,更新现有 deployment spec;不夹带其他门禁整改。 +6. 主会话 trellis-check,启动只读独立 review,修复当前范围内问题并复验;交付一个 task 的独立 diff 待审。 +7. 提交、push、生产执行、归档等待对应明确授权;不因 task 立项自动开展生产动作。 + +## 验证入口 + +- 专用测试路径:`server/tests/unit/scripts/deployment-db-lifecycle.test.js`;使用现有 Vitest,覆盖 V1/V2、模式、worker、MinIO、失败及变异矩阵。 +- 相关合同:`cd server && npx vitest run tests/unit/scripts/deployment-db-lifecycle.test.js tests/unit/scripts/deployment-safety-contract.test.js`。 +- Shell:`bash -n deploy/trialrun/deploy.sh deploy/trialrun/deploy-existing.sh scripts/qa/run-release-gate.sh`。 +- 既有 static 门禁、env contract、Trellis validate/governance、docs registry、process baseline、git diff --check。 +- 生产发布前后 db 容器身份比较仅为未来获批验收设计,本轮不 SSH、不运行真实 Docker,不执行迁移/seed,不测真实数据。 + +## 待交付停点 + +本轮停在本地实施与 diff 复核;不提交/push/部署/归档。财务 403 及其他债务不纳入此 task。 + +## 实施与验证过程(2026-09-05) + +- 真实远端段 baseline 红测:原部署脚本可执行到最终服务状态,但服务语义断言报 `database_lifecycle_forbidden:up,run,up`,Vitest 退出 1。首个 harness 调试因缺少白名单 chmod 提前退出,不作为缺陷复现;补齐后才得到上述真实违规。 +- 显式 db up 已移入 initialize 分支;existing 不修复缺失/停止/不健康 db。固定 120 秒强制 deadline 和 2 秒轮询覆盖状态查询卡住;init/server/worker 及显式选择的 MinIO 服务隔离依赖。 +- fake 来自测试专用临时目录,只有测试子进程使用白名单 PATH/env;DOCKER_HOST 指向临时不存在 socket,DOCKER_CONFIG 指向临时目录;没有继承真实凭据或 Docker 配置,不修改父进程 PATH。测试结束清理临时目录。 +- 捕获器解析参数及当前 Compose 图,区分一次性 init 删除和 db 删除;未知调用失败。四类变异作用于实际 REMOTE 源码的临时副本,先断言修改命中,再核对 forbidden db 事件;不修改仓库脚本做探针。 +- timeout 专项两项通过(持续 starting、inspect 卡住);仅临时副本缩短为 1 秒,实际执行 GNU timeout 验证强制截止,生产值保持 120。初轮 fake 未把被 SIGKILL 的子进程映射为 shell 的 137,导致两项失败;修正 fake 信号转码后重新通过,不改产品退出逻辑或放宽断言。 +- 既有部署安全与来源合同:`deployment-safety-contract.test.js` 24 项、`check-source-provenance.test.js` 7 项,共 31 项通过。均本地合成,不接触生产。 +- 首轮 static(当时 41 项新回归)通过,含 Prisma validate、client typecheck/build;后续 review 增量完成后再次复跑,不以此前通过替代最终版本验证。 +- 独立 reviewer 首轮发现 P2:单次健康检查早于耗时构建,可能在 db 随后变坏时仍进入 init/应用切换。已复用有界检查并把 server build 与切换拆开;worker 显式准备镜像后检查,以 --no-build 阻止检查后的隐式构建。补五个跨阶段健康漂移场景与 worker 有/无镜像测试,待最终验证和只读复核闭环。 +- 追加复核发现 P2:worker 关联镜像 ID 非空但启动标签可能已缺失。已改为容器 Config.Image 引用校验,标签缺失则先构建,且整个准备阶段前移到 server 切换之前;补该具体边界回归,不把 Docker image ID 与可用启动引用混为一谈。 + +## 最终本地验证与 diff 交付(2026-09-05) + +| 检查 | 实际结果 | +| --- | --- | +| `bash scripts/qa/run-release-gate.sh static` | PASS;最终版本 49 项专用回归全部通过,Prisma validate、client typecheck/build 通过。 | +| 部署安全与来源合同 | 两文件 31 项 PASS;与上行 49 项不重叠,共 80 个不同用例。 | +| 四类变异 | 源码中恢复无条件 db up、分别移除 init/server/worker 隔离,均执行到受测阶段并触发按 db 服务判定的违规断言;正常发布测试必须到达 init、应用及最终状态。 | +| 健康漂移与镜像 | 五类跨阶段健康变化、worker 现有/缺失镜像及标签缺失用例通过;真实 timeout 在临时缩短版的持续 starting/卡住 inspect 场景通过。生产每次仍固定 120 秒。 | +| 工程与治理 | shell syntax、JS/CJS syntax、Trellis validate/governance、docs registry、process baseline、敏感扫描、git diff --check 通过。task validate 的大 spec 注入截断警告保留并已要求直接按章节阅读。 | +| 只读独立 review | 两项 P2 均在代码和回归设计层面闭环,无新增高置信问题;reviewer 未写文件、未执行测试或生产操作,最终运行结果由主会话提供。review 已结束并关闭。 | +| 环境边界 | 测试只使用临时合成 env/路径/fake;未连接真实 Docker daemon、SSH、数据库或生产系统。仅查看本机 Compose 帮助/版本确认现有 CLI,不将其视为真实 V1/V2 发布验证。 | + +完整隔离 diff(包含未跟踪新增文件,排除旧 task 笔记):`artifacts/qa-flow/existing-deploy-db-lifecycle-isolation/review.diff`。 +旧工具原始命令序列:`artifacts/qa-flow/existing-deploy-db-lifecycle-isolation/old-tool-sequence-05d2a3f.txt`,只读参考、不可执行;稳定恢复边界见 design.md,不能照抄旧序列启动 db 或重放 init/seed。 + +AC1–AC7 已完成本地验收;AC8 的首次真实发布观察保持未执行,需单独批准并由人工盯住工具与应用两个恢复面。不提交、push、部署或归档,task 保持 in_progress 等待用户 diff 复核。旧错误可解释性 task 的交接笔记 diff 摘要在本轮前后保持一致,未纳入隔离 diff;Trellis 升级 worktree 未操作。 + +## 根因与防复发 + +- 根因属于跨层合同传播缺失和隐式假设:existing 模式只约束数据目录,未约束真正执行命令;原测试没有执行命令并计算依赖传播。 +- 仅移除显式 up db 会遗漏 run/up 依赖;仅加 --no-deps 又会丢失 service_healthy 时序;只做一次健康检查会被耗时阶段后的状态漂移击穿。 +- 预防已落地:实际模式分支、各关键阶段的同一有界健康函数、显式构建与隔离切换、真实源码变异、服务依赖图事件断言、static 门禁及现有 deployment spec。未扩展治理、账号、权限、状态机或其他债务范围。 + +## 用户复核与首次发布授权(2026-09-05) + +- 用户独立复核 PASS:将 initialize 条件改为 if true 后 49 项中 30 项失败;还原后 49/49 通过,md5 与备份一致且变异无残留。用户另行确认治理与 whitespace 通过、static 已接入。 +- 用户明确批准本 task 提交、push 和首次真实部署;先回报产品提交,再执行 push/受控部署。财务 403 排下一项,不与本次发布混做。 +- 部署前保留不可执行的旧工具命令序列,完成正式身份/迁移预检,捕获同一 Compose 项目 db 的 ID、Created、StartedAt、RestartCount、镜像和健康状态;发布期间持续查看阶段结果,完成后逐项对比,不能只用 /ready 代替 db 身份证据。 +- 健康超时或任何前置失败均停止,不调长 120 秒、不自行启动 db、不切换 initialize、不盲目重放旧命令或 init/seed。备份合同刷新、/ready 与 bundle marker 仍须通过。 +- 当前有旧 task 未提交交接笔记;它不属于本次提交。为满足部署来源 clean-worktree 门禁,将只对该文件临时 stash 保管,记录对象和摘要,发布及证据提交结束后原样恢复,不删除其他 stash 或改动。 +- 用户观测测试约 90 秒;保留全覆盖,不改抽样运行。如后续优化,仅合并冗余 fixture/减少重复准备,不删除独立行为断言和四类变异覆盖。 diff --git a/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/prd.md b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/prd.md new file mode 100644 index 00000000..ef592a3b --- /dev/null +++ b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/prd.md @@ -0,0 +1,60 @@ +# 既有环境应用发布隔离数据库容器生命周期 + +## Goal + +既有环境发布应用时,发布链路不得显式或经 Compose 依赖传播启动、停止、重建、替换或删除数据库容器。保留首次初始化的数据库启动能力,以及获批发布中的既有迁移、seed 和发布验证流程。保护对象是数据库容器生命周期,不将本任务宣称为数据库内容完全只读或所有数据丢失风险的解决方案。 + +## Background + +- 基线:`05d2a3f`。用户报告曾在应用发布时误重建生产数据库容器,因 PGDATA 为 bind mount 未丢失数据;本轮仅确认代码风险,不重演生产事件。 +- `deploy/trialrun/deploy-existing.sh:109` 起向 `deploy.sh` 固定传入 `DEPLOY_DATA_MODE=existing`。后者在 `:93` 校验模式,在 `:249` 区分数据目录检查,但关键容器操作未按模式隔离。 +- 三类入口:`deploy/trialrun/deploy.sh:317` 无条件 `up -d db`;`:326` 的 init `run` 未隔离依赖;`:329`、`:331` 的 server/worker `up` 未隔离依赖。`deploy/trialrun/docker-compose.yml:57`、`:79`、`:100` 分别声明三者依赖健康 db。 +- 因此只给 server 添加一个选项不足以解决问题,必须同时禁止显式 db 操作并隔离所有相关依赖启动入口。 + +## Requirements + +### R1 模式与操作边界 + +- `existing` 模式禁止显式及隐式数据库生命周期操作,包括全项目操作;不能在 db 缺失、停止或不健康时自动切换到 initialize 或尝试启动/修复 db。 +- `initialize` 保留显式启动 db 的既有能力;不得把此能力泄漏到 existing 发布。缺失或非法模式在任何容器变更前非零退出。 +- 数据库只读状态检查、就绪探测与容器内数据库迁移/seed 不属于容器生命周期变更;后两者仍须遵循既有发布授权与迁移门禁,本任务不改变其内容或顺序。 + +### R2 发布链路隔离与失败行为 + +- 同时覆盖入口 wrapper、远端执行段、init、server、启用的 worker,以及可选 profile;按实际服务依赖判定,不遗漏间接通路。 +- existing 在 init 或应用切换前确认目标项目 db 已存在、正在运行并就绪;失败则有界停止,输出安全原因,不隐式自愈。 +- initialize 启动 db 后须有界确认就绪,再进入 init;隔离依赖不能取消原来依赖健康状态提供的时序保障。 +- 每次就绪等待固定为 120 秒(不可由环境变量放宽),轮询间隔 2 秒;单次总时限包含卡住的状态查询,超时必须非零退出,不运行后续 init 或继续应用切换。缺失、停止、不健康或缺健康检查时直接失败。构建、init 之后重新检查,不能只用发布开始时的健康快照;应用构建与切换分开,健康检查之后不允许隐式构建。 +- 保留现有 Compose V2 与 `docker-compose` 兼容入口;不支持所需隔离能力时失败,不回退为不隔离的启动。 +- 保留身份、发布来源、数据挂载与路径、迁移及回滚窗口、受控日志、备份合同刷新、`/ready` 和公网版本等检查。不改生产配置、数据目录权限或部署身份。 + +## Test Plan + +### R3 本地可执行安全回归 + +- 在临时合成工作区运行实际发布 shell 路径,PATH 前置假的 docker/docker-compose,并截获 SSH、rsync 等外部入口,禁止接触真实 daemon、主机、网络服务、env 和数据目录。 +- 记录参数数组并解析操作、目标服务、项目及依赖传播;断言 existing 的数据库生命周期事件为零,不能只搜索选项字符串。全项目 up/down、直接 db 操作、依赖传播均纳入违规判定。 +- 测试必须证明 init 与预期应用服务实际执行,不能因脚本提前退出或 fake 总返回成功而得到“零 db 操作”的假通过。 +- initialize 应观察到允许的 db 启动与后续 init、应用执行;existing 缺失/停止/不健康 db、非法模式、init 失败应非零退出,且不切换应用或启动 db。 +- 正向矩阵覆盖 V2/V1、worker 开关及可选 MinIO profile;故意恢复无条件 `up db`,分别去掉 init/server/worker 的依赖隔离,测试均须发现 db 违规而失败。 +- 新回归进入既有本地 static 发布门禁,在任何生产执行前可运行;不为本任务重构整个部署测试体系。 + +## Scope And Authorization + +- 拟涉及:`deploy/trialrun/deploy.sh`、必要时 `deploy-existing.sh` 的模式传递合同、新增专用容器生命周期回归、现有 deployment safety 回归的相关断言、`scripts/qa/run-release-gate.sh` 的 static 接入、现有 deployment spec,以及本 task 文件。 + +## Out of Scope + +- 不改 Compose 服务拓扑、业务代码、schema/migration/seed 实现、付款及账号权限、IM routing、Trellis 升级、治理规则、目录权限、其他小债。财务 403 在本任务之后另开只读诊断。 +- 用户本轮只批准立 task 和规划。未经下一轮批准不实施;不启动生产发布、故障注入、数据修复或账号操作。一个 task、一个实施提交单元,实施后安排只读独立复核;提交、push、部署、归档各遵守明确授权边界。 + +## Acceptance Criteria + +- [x] AC1(R1):existing 实际执行捕获中没有显式或隐式 db 生命周期事件;无服务名的全项目操作同样受约束。 +- [x] AC2(R2):init、server、启用的 worker 均隔离依赖;init 的现有迁移/seed 命令内容和顺序保持,相关命令实际到达。 +- [x] AC3(R1/R2):initialize 可启动 db 并等待就绪;existing db 缺失/停止/不健康时失败,不启动或修复 db,不到达 init/应用切换;非法模式 fail closed。 +- [x] AC4(R3):四类真实源码变异(无条件 up db、init/server/worker 各自失去隔离)分别令捕获断言失败;正常模式矩阵通过,断言按服务生命周期语义而非字符串存在性成立。 +- [x] AC5(R3):测试完全本地合成,不访问真实 Docker、SSH、生产 env 或数据库;测试可经 static 门禁运行,fake 未知命令 fail closed,无提前退出假通过。 +- [x] AC6(R2):既有部署安全合同、身份/路径/迁移/日志/刷新/版本断言和 shell syntax 通过;未弱化其他安全约束。 +- [x] AC7:只读独立复核覆盖显式 db 操作、init 间接依赖、测试有效性与初始化兼容,问题闭环后才交实施 diff;同步现有 spec,未混入其他 task 或小债。 +- [ ] AC8:交付区分本地防护验证与生产执行;若未来单独批准生产验收,需比较同一项目 db 容器 ID/创建时间前后不变及应用版本更新、ready 通过,禁止在生产注入故障。本轮生产验证为未授权、不执行。 diff --git a/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/task.json b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/task.json new file mode 100644 index 00000000..786096bd --- /dev/null +++ b/.trellis/tasks/09-05-existing-deploy-db-lifecycle-isolation/task.json @@ -0,0 +1,35 @@ +{ + "id": "existing-deploy-db-lifecycle-isolation", + "name": "existing-deploy-db-lifecycle-isolation", + "title": "既有环境应用发布隔离数据库容器生命周期", + "description": "隔离 existing 应用发布中的显式及依赖传播数据库容器生命周期操作,保留 initialize 与既有迁移流程;当前仅批准规划。", + "status": "in_progress", + "dev_type": null, + "scope": null, + "package": null, + "priority": "P0", + "creator": "shunli", + "assignee": "shunli", + "createdAt": "2026-09-05", + "completedAt": null, + "branch": null, + "base_branch": "main", + "worktree_path": null, + "commit": null, + "pr_url": null, + "subtasks": [], + "children": [], + "parent": null, + "relatedFiles": [ + "deploy/trialrun/deploy.sh", + "deploy/trialrun/deploy-existing.sh", + "deploy/trialrun/docker-compose.yml", + "server/tests/unit/scripts/deployment-safety-contract.test.js", + "server/tests/unit/scripts/deployment-db-lifecycle.test.js", + "server/tests/unit/fixtures/deployment-command-fake.cjs", + "scripts/qa/run-release-gate.sh", + ".trellis/spec/hyb-oa/deployment-trialrun.md" + ], + "notes": "2026-09-05 用户已批准 PRD 和实施;重点:显式 up db 受模式约束、init 依赖隔离、按服务生命周期语义断言、固定120秒健康等待、fake不污染真实PATH、准备工具恢复参考。一个 task、一个实施提交单元、只读独立复核。实施后先发 diff,不自动提交、push、部署或归档。", + "meta": {} +} diff --git a/deploy/trialrun/deploy.sh b/deploy/trialrun/deploy.sh index f56d89c4..4792a777 100755 --- a/deploy/trialrun/deploy.sh +++ b/deploy/trialrun/deploy.sh @@ -248,7 +248,7 @@ cd "$REMOTE_DIR" if [[ "$DEPLOY_DATA_MODE" == "initialize" ]]; then mkdir -p "$POSTGRES_DATA_DIR" "$STORAGE_HOST_DIR" -else +elif [[ "$DEPLOY_DATA_MODE" == "existing" ]]; then if [[ ! -d "$POSTGRES_DATA_DIR" || ! -f "$POSTGRES_DATA_DIR/PG_VERSION" ]]; then echo "[ERROR] existing 模式要求 POSTGRES_DATA_DIR 已存在且包含 PG_VERSION" >&2 exit 1 @@ -257,6 +257,9 @@ else echo "[ERROR] existing 模式要求 STORAGE_HOST_DIR 已存在" >&2 exit 1 fi +else + echo "[ERROR] DEPLOY_DATA_MODE 必须显式设为 existing 或 initialize" >&2 + exit 1 fi if [[ "$ENABLE_MINIO" == "1" ]]; then mkdir -p deploy/trialrun/data/minio @@ -312,23 +315,96 @@ run_logged_deploy_stage() { echo ">>> 4.1 校验远端 compose 配置" "${compose_cmd[@]}" "${profile_args[@]}" config >/dev/null +if ! command -v timeout >/dev/null 2>&1; then + echo "[ERROR] db_health_timeout_unavailable:缺少有界就绪检查工具,停止发布" >&2 + exit 1 +fi -echo ">>> 4.2 启动基础服务" -"${compose_cmd[@]}" up -d db +wait_for_database_health() { + if timeout --signal=KILL 120 bash -s -- "${compose_cmd[@]}" <<'DB_HEALTH' +set -euo pipefail +while true; do + if ! database_id="$("$@" ps -a -q db 2>/dev/null)"; then + echo "[ERROR] db_health_query_failed:无法查询目标数据库容器,停止发布" + exit 1 + fi + if [[ ! "$database_id" =~ ^[a-f0-9]{12,64}$ ]]; then + echo "[ERROR] db_container_missing_or_ambiguous:目标数据库容器不存在或不唯一,停止发布" + exit 1 + fi + if ! database_health="$(docker inspect --type container --format '{{.State.Status}}|{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$database_id" 2>/dev/null)"; then + echo "[ERROR] db_health_query_failed:无法读取目标数据库健康状态,停止发布" + exit 1 + fi + case "$database_health" in + 'running|healthy') exit 0 ;; + 'running|starting') sleep 2 ;; + *) + echo "[ERROR] db_not_healthy:数据库未运行、不健康或缺少健康检查,停止发布;不会自动启动或修复" + exit 1 + ;; + esac +done +DB_HEALTH + then + echo "[PASS] 目标数据库健康检查通过;本次等待上限 120 秒" + else + local health_status=$? + if [[ "$health_status" == "124" || "$health_status" == "137" ]]; then + echo "[ERROR] db_health_timeout:数据库就绪检查超过 120 秒,停止发布;不执行后续 init 或应用切换" >&2 + fi + return 1 + fi +} + +echo ">>> 4.2 按部署模式检查基础服务" +if [[ "$DEPLOY_DATA_MODE" == "initialize" ]]; then + "${compose_cmd[@]}" up -d db +fi +wait_for_database_health if [[ "$ENABLE_MINIO" == "1" ]]; then - "${compose_cmd[@]}" "${profile_args[@]}" up -d minio minio_init + "${compose_cmd[@]}" "${profile_args[@]}" up -d --no-deps minio minio_init fi echo ">>> 4.3 构建最新 init 镜像" run_logged_deploy_stage "build-init" "${compose_cmd[@]}" build init echo ">>> 4.4 预检新配置并执行 init" -run_logged_deploy_stage "run-init" "${compose_cmd[@]}" run --rm -T --entrypoint sh init -lc 'cd /app/server && node -e '\''require("./src/config").getConfig()'\'' && npm run prisma:migrate:deploy && npm run seed:config && npm run seed:trialrun' +wait_for_database_health +run_logged_deploy_stage "run-init" "${compose_cmd[@]}" run --rm --no-deps -T --entrypoint sh init -lc 'cd /app/server && node -e '\''require("./src/config").getConfig()'\'' && npm run prisma:migrate:deploy && npm run seed:config && npm run seed:trialrun' echo ">>> 4.5 构建并切换应用服务" -run_logged_deploy_stage "build-switch-server" "${compose_cmd[@]}" "${profile_args[@]}" up -d --build server +wait_for_database_health +run_logged_deploy_stage "build-server" "${compose_cmd[@]}" "${profile_args[@]}" build server if [[ "$ENABLE_WORKER" == "1" ]]; then - "${compose_cmd[@]}" "${profile_args[@]}" up -d worker + wait_for_database_health + if ! worker_container_id="$("${compose_cmd[@]}" "${profile_args[@]}" ps -a -q worker)"; then + echo "[ERROR] worker_image_query_failed:无法检查 worker 容器,停止发布" >&2 + exit 1 + fi + worker_image_ready=0 + if [[ -n "$worker_container_id" ]]; then + if [[ ! "$worker_container_id" =~ ^[a-f0-9]{12,64}$ ]] || ! worker_image_reference="$(docker inspect --type container --format '{{.Config.Image}}' "$worker_container_id" 2>/dev/null)"; then + echo "[ERROR] worker_image_query_failed:worker 容器不唯一或镜像引用不可读,停止发布" >&2 + exit 1 + fi + if [[ ! "$worker_image_reference" =~ ^[A-Za-z0-9][A-Za-z0-9._/:@-]*$ ]]; then + echo "[ERROR] worker_image_query_failed:worker 镜像引用无效,停止发布" >&2 + exit 1 + fi + if docker image inspect "$worker_image_reference" >/dev/null 2>&1; then + worker_image_ready=1 + fi + fi + if [[ "$worker_image_ready" != "1" ]]; then + run_logged_deploy_stage "build-worker" "${compose_cmd[@]}" "${profile_args[@]}" build worker + fi +fi +wait_for_database_health +run_logged_deploy_stage "build-switch-server" "${compose_cmd[@]}" "${profile_args[@]}" up -d --no-build --no-deps server +if [[ "$ENABLE_WORKER" == "1" ]]; then + wait_for_database_health + "${compose_cmd[@]}" "${profile_args[@]}" up -d --no-build --no-deps worker fi echo ">>> 4.6 校验 Prisma migration status" diff --git a/scripts/qa/run-release-gate.sh b/scripts/qa/run-release-gate.sh index 07366a3b..e80228c0 100755 --- a/scripts/qa/run-release-gate.sh +++ b/scripts/qa/run-release-gate.sh @@ -61,6 +61,7 @@ stage_static() { node "$ROOT_DIR/deploy/trialrun/validate-env-contract.js" ( cd "$ROOT_DIR/server" + npm test -- tests/unit/scripts/deployment-db-lifecycle.test.js npm run prisma:validate ) ( diff --git a/server/tests/unit/fixtures/deployment-command-fake.cjs b/server/tests/unit/fixtures/deployment-command-fake.cjs new file mode 100644 index 00000000..d273acbf --- /dev/null +++ b/server/tests/unit/fixtures/deployment-command-fake.cjs @@ -0,0 +1,156 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); + +const GLOBAL_VALUES = new Set(['-p', '--project-name', '-f', '--file', '--env-file', '--profile']); +const OPTION_VALUES = new Set(['--entrypoint', '-e', '--env', '--tail', '--format', '--type', '--timeout', '-t']); +const SWITCHES = new Set(['-d', '--detach', '--build', '--no-build', '--no-deps', '--rm', '-T', '-q', '--quiet', '-a', '--all', '-f', '--force', '--no-trunc']); +const LIFECYCLE = new Set(['up', 'down', 'run', 'start', 'stop', 'restart', 'rm', 'create', 'kill', 'pause', 'unpause']); + +function parseCompose(argv) { + const args = [...argv]; + if (args[0] === 'compose') args.shift(); + const globals = {}; + while (args[0]?.startsWith('-')) { + const option = args.shift(); + if (!GLOBAL_VALUES.has(option) || !args.length) throw new Error('unknown_compose_global'); + globals[option] = args.shift(); + } + const operation = args.shift(); + const targets = []; + let noDeps = false; + while (args.length) { + const argument = args.shift(); + if (argument.startsWith('-')) { + if (OPTION_VALUES.has(argument)) { + if (!args.length) throw new Error('missing_option_value'); + args.shift(); + } else if (!SWITCHES.has(argument)) throw new Error('unknown_compose_option'); + if (argument === '--no-deps') noDeps = true; + } else { + targets.push(argument); + if (['run', 'exec', 'port'].includes(operation)) break; + } + } + return { operation, targets, noDeps, globals, containerArgs: args }; +} + +function lifecycleServices(call, services) { + if (!LIFECYCLE.has(call.operation)) return []; + const selected = call.operation === 'down' || !call.targets.length ? Object.keys(services) : call.targets; + const result = new Set(); + function visit(service) { + if (!Object.hasOwn(services, service)) throw new Error('unknown_compose_service'); + if (result.has(service)) return; + result.add(service); + if (['up', 'run'].includes(call.operation) && !call.noDeps) { + const dependencies = services[service].depends_on ?? {}; + for (const dependency of Array.isArray(dependencies) ? dependencies : Object.keys(dependencies)) visit(dependency); + } + } + selected.forEach(visit); + return [...result]; +} + +function main() { + const root = process.env.HYB_DEPLOYMENT_FIXTURE_ROOT; + if (!root || !path.basename(root).startsWith('hyb-db-lifecycle-') || path.dirname(process.argv[1]) !== path.join(root, 'bin')) { + throw new Error('fixture_only'); + } + const config = JSON.parse(fs.readFileSync(path.join(root, 'config.json'), 'utf8')); + const command = path.basename(process.argv[1]); + const args = process.argv.slice(2); + const event = { command, args }; + if (command === 'docker-compose' || (command === 'docker' && args[0] === 'compose')) { + event.compose = parseCompose(args); + event.lifecycle = lifecycleServices(event.compose, config.services); + } + fs.appendFileSync(path.join(root, 'trace.jsonl'), JSON.stringify(event) + '\n'); + if (event.compose && config.driftAfter === event.compose.operation + ':' + event.compose.targets.join()) { + fs.writeFileSync(path.join(root, 'health-drift'), '1'); + } + if (command === 'timeout') { + const child = spawnSync('/usr/bin/timeout', args, { env: process.env, stdio: 'inherit' }); + process.exit(child.status ?? (child.signal === 'SIGKILL' ? 137 : 99)); + } + if (command === 'sleep') return; + if (command === 'curl') { + if (args.join(' ') !== '-fsS http://127.0.0.1:31001/ready') throw new Error('unexpected_network_request'); + console.log(JSON.stringify({ data: { buildSha: '123456789abc' } })); + return; + } + if (command === 'docker' && args[0] === 'inspect') { + if (args.at(-1) === 'e'.repeat(64) && args.includes('{{.Config.Image}}')) { + console.log('synthetic-project-worker:latest'); + return; + } + if (args.at(-1) !== 'd'.repeat(64)) throw new Error('unexpected_container'); + if (config.health === 'inspect_error') process.exit(18); + const counterFile = path.join(root, 'inspect-count'); + const count = fs.existsSync(counterFile) ? Number(fs.readFileSync(counterFile, 'utf8')) : 0; + fs.writeFileSync(counterFile, String(count + 1)); + if (config.health === 'query_hang') { + setTimeout(() => {}, 20000); + return; + } + const states = { stopped: 'exited|healthy', unhealthy: 'running|unhealthy', no_health: 'running|missing', starting: 'running|starting' }; + const state = fs.existsSync(path.join(root, 'health-drift')) ? 'running|unhealthy' : config.health === 'warming' && count === 0 ? 'running|starting' : states[config.health] ?? 'running|healthy'; + console.log(state); + return; + } + if (command === 'docker' && args.join(' ') === 'image inspect synthetic-project-worker:latest') process.exit(config.workerTagPresent ? 0 : 1); + const call = event.compose; + if (!call) throw new Error('forbidden_external_command'); + if (call.operation === 'version') { + process.exit(command === 'docker' && config.engine === 'v1' ? 1 : 0); + } + if (call.globals['-p'] !== 'synthetic-project') throw new Error('wrong_compose_project'); + if (call.noDeps && config.unsupportedIsolation) process.exit(2); + if (LIFECYCLE.has(call.operation)) { + if (call.operation === 'up' && call.targets.includes('db')) fs.writeFileSync(path.join(root, 'db-started'), '1'); + if (call.operation === 'run') { + if (call.targets.join() !== 'init' || !call.containerArgs.join(' ').includes('npm run prisma:migrate:deploy && npm run seed:config && npm run seed:trialrun')) { + throw new Error('unexpected_init_command'); + } + if (config.initFails) process.exit(23); + } + return; + } + if (call.operation === 'ps') { + if (!call.targets.length) return; + if (call.targets.join() === 'worker') { + if (config.workerImagePresent) console.log('e'.repeat(64)); + return; + } + if (call.targets.join() !== 'db') throw new Error('unexpected_ps_service'); + if (config.health === 'query_error') process.exit(17); + if (config.health === 'missing' && !fs.existsSync(path.join(root, 'db-started'))) return; + console.log(config.health === 'multiple' ? 'd'.repeat(64) + '\n' + 'e'.repeat(64) : 'd'.repeat(64)); + return; + } + if (call.operation === 'port' && call.targets.join() === 'server' && call.containerArgs.join() === '3000') { + console.log('127.0.0.1:31001'); + return; + } + if (call.operation === 'config') return; + if (call.operation === 'build' && ['init', 'server', 'worker'].includes(call.targets.join())) return; + if (call.operation === 'logs' && call.targets.join() === 'server') return; + if (call.operation === 'exec' && call.targets.join() === 'server') { + const expected = ['sh -lc cd /app/server && npx prisma migrate status', 'node src/scripts/smoke-external-deps.js', 'node src/scripts/smoke-trialrun-runtime.js']; + if (expected.includes(call.containerArgs.join(' '))) return; + if (call.containerArgs[0] === 'node' && call.containerArgs[1] === '-e' && call.containerArgs[2]?.includes('EXPECTED_BUILD_SHA')) { + const input = JSON.parse(fs.readFileSync(0, 'utf8')); + if (input.data?.buildSha !== '123456789abc') throw new Error('wrong_ready_build'); + return; + } + } + throw new Error('unknown_fake_command'); +} + +module.exports = { parseCompose, lifecycleServices }; +if (require.main === module) { + try { main(); } catch (error) { + console.error('[deployment-fixture] ' + (/^[a-z_]+$/.test(error.message) ? error.message : 'unexpected_failure')); + process.exitCode = 97; + } +} diff --git a/server/tests/unit/scripts/deployment-db-lifecycle.test.js b/server/tests/unit/scripts/deployment-db-lifecycle.test.js new file mode 100644 index 00000000..6d97e62c --- /dev/null +++ b/server/tests/unit/scripts/deployment-db-lifecycle.test.js @@ -0,0 +1,264 @@ +const { execFile } = require('node:child_process'); +const { mkdtemp, mkdir, writeFile, readFile, chmod, symlink, rm } = require('node:fs/promises'); +const { promisify } = require('node:util'); +const os = require('node:os'); +const path = require('node:path'); +const YAML = require('yaml'); +const { parseCompose, lifecycleServices } = require('../fixtures/deployment-command-fake.cjs'); + +const execute = promisify(execFile); +const ROOT = path.resolve(__dirname, '../../../..'); +const DEPLOY = path.join(ROOT, 'deploy/trialrun/deploy.sh'); +const COMPOSE = path.join(ROOT, 'deploy/trialrun/docker-compose.yml'); +const FAKE = path.join(__dirname, '../fixtures/deployment-command-fake.cjs'); +const ORIGINAL_PATH = process.env.PATH; + +function remoteSection(source) { + const sections = [...source.matchAll(/<<'REMOTE'\n([\s\S]*?)\nREMOTE\n/g)]; + expect(sections).toHaveLength(1); + return sections[0][1]; +} + +function replaceOnce(source, before, after) { + expect(source.split(before)).toHaveLength(2); + return source.replace(before, after); +} + +function assertNoDatabaseLifecycle(trace) { + const violations = trace.filter(event => event.lifecycle?.includes('db')); + if (violations.length) throw new Error('database_lifecycle_forbidden:' + violations.map(event => event.compose.operation).join(',')); +} + +describe('database lifecycle isolation in the actual deployment remote script', () => { + let fixtureRoot; + let source; + let services; + + beforeEach(async () => { + fixtureRoot = await mkdtemp(path.join(os.tmpdir(), 'hyb-db-lifecycle-')); + source = await readFile(DEPLOY, 'utf8'); + services = YAML.parse(await readFile(COMPOSE, 'utf8')).services; + const bin = path.join(fixtureRoot, 'bin'); + await mkdir(bin); + const fakeSource = '#!' + process.execPath + '\n' + await readFile(FAKE, 'utf8'); + for (const command of ['docker', 'docker-compose', 'curl', 'timeout', 'sleep', 'ssh', 'rsync', 'node', 'git', 'sudo']) { + const file = path.join(bin, command); + await writeFile(file, fakeSource); + await chmod(file, 0o700); + } + for (const command of ['bash', 'mkdir', 'install', 'chmod', 'seq']) await symlink('/usr/bin/' + command, path.join(bin, command)); + await mkdir(path.join(fixtureRoot, 'app')); + await mkdir(path.join(fixtureRoot, 'storage')); + await mkdir(path.join(fixtureRoot, 'postgres')); + await writeFile(path.join(fixtureRoot, 'postgres/PG_VERSION'), '16\n'); + await writeFile(path.join(fixtureRoot, 'env'), 'APP_ORIGIN=https://example.invalid\n', { mode: 0o600 }); + }); + + afterEach(async () => { + expect(process.env.PATH).toBe(ORIGINAL_PATH); + await rm(fixtureRoot, { recursive: true, force: true }); + }); + + async function run({ mode = 'existing', worker = false, minio = false, engine = 'v2', health = 'healthy', initFails = false, unsupportedIsolation = false, driftAfter = '', workerImagePresent = true, workerTagPresent = true, script = remoteSection(source) } = {}) { + await writeFile(path.join(fixtureRoot, 'config.json'), JSON.stringify({ services, engine, health, initFails, unsupportedIsolation, driftAfter, workerImagePresent, workerTagPresent })); + await writeFile(path.join(fixtureRoot, 'trace.jsonl'), ''); + const scriptPath = path.join(fixtureRoot, 'remote.sh'); + await writeFile(scriptPath, script); + const env = { + PATH: path.join(fixtureRoot, 'bin'), HOME: fixtureRoot, + DOCKER_HOST: 'unix://' + path.join(fixtureRoot, 'nonexistent-docker.sock'), DOCKER_CONFIG: path.join(fixtureRoot, 'docker-config'), + HYB_DEPLOYMENT_FIXTURE_ROOT: fixtureRoot, + REMOTE_DIR: path.join(fixtureRoot, 'app'), REMOTE_ENV_FILE: path.join(fixtureRoot, 'env'), + POSTGRES_DATA_DIR: path.join(fixtureRoot, 'postgres'), STORAGE_HOST_DIR: path.join(fixtureRoot, 'storage'), + COMPOSE_PROJECT_NAME: 'synthetic-project', COMPOSE_FILE: 'compose.yml', + DEPLOY_DATA_MODE: mode, ENABLE_WORKER: worker ? '1' : '0', ENABLE_MINIO: minio ? '1' : '0', + DEPLOY_STATUS_OUTPUT_MODE: 'quiet', BUILD_SHA: '123456789abc'.padEnd(40, '0'), VITE_BUILD_SHA: '123456789abc', + }; + let result; + try { + result = { ...await execute('/usr/bin/bash', [scriptPath], { env, timeout: 15000 }), code: 0 }; + } catch (error) { + result = { code: error.code, stdout: error.stdout, stderr: error.stderr, killed: error.killed }; + } + const trace = (await readFile(path.join(fixtureRoot, 'trace.jsonl'), 'utf8')).trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + return { ...result, trace }; + } + + function assertCompleted(result, worker) { + expect(result.code, result.stdout + result.stderr).toBe(0); + const stages = result.trace.filter(event => ['run', 'up'].includes(event.compose?.operation)); + expect(stages.some(event => event.compose.operation === 'run' && event.compose.targets.join() === 'init')).toBe(true); + expect(stages.some(event => event.compose.operation === 'up' && event.compose.targets.join() === 'server')).toBe(true); + expect(stages.some(event => event.compose.operation === 'up' && event.compose.targets.join() === 'worker')).toBe(worker); + expect(result.stdout).toContain('4.10'); + } + + it('existing release never operates on db while reaching init and the application', async () => { + const result = await run(); + assertCompleted(result, false); + assertNoDatabaseLifecycle(result.trace); + }); + + for (const engine of ['v1', 'v2']) { + for (const mode of ['existing', 'initialize']) { + for (const worker of [false, true]) { + for (const minio of [false, true]) { + it(`${mode}/${engine}/worker=${worker}/minio=${minio} preserves the service boundary`, async () => { + const result = await run({ engine, mode, worker, minio }); + assertCompleted(result, worker); + const databaseEvents = result.trace.filter(event => event.lifecycle?.includes('db')); + if (mode === 'existing') assertNoDatabaseLifecycle(result.trace); + else { + expect(databaseEvents).toHaveLength(1); + expect(databaseEvents[0].compose).toMatchObject({ operation: 'up', targets: ['db'] }); + } + expect(result.trace.some(event => event.compose?.targets.join() === 'minio,minio_init')).toBe(minio); + const healthIndex = result.trace.findIndex(event => event.command === 'docker' && event.args[0] === 'inspect'); + const initIndex = result.trace.findIndex(event => event.compose?.operation === 'run'); + expect(healthIndex).toBeGreaterThan(-1); + expect(healthIndex).toBeLessThan(initIndex); + const deadline = result.trace.find(event => event.command === 'timeout'); + expect(deadline.args.slice(0, 2)).toEqual(['--signal=KILL', '120']); + }); + } + } + } + } + + it.each(['missing', 'stopped', 'unhealthy', 'no_health', 'multiple', 'query_error', 'inspect_error'])('existing %s database fails closed without init or application switching', async health => { + const result = await run({ health }); + expect(result.code).not.toBe(0); + assertNoDatabaseLifecycle(result.trace); + expect(result.trace.some(event => ['run', 'up'].includes(event.compose?.operation))).toBe(false); + }); + + it.each(['', 'unknown'])('invalid mode %s is rejected before container commands', async mode => { + const result = await run({ mode }); + expect(result.code).not.toBe(0); + expect(result.trace).toEqual([]); + }); + + it.each(['existing', 'initialize'])('missing deadline tooling stops %s before lifecycle operations', async mode => { + await rm(path.join(fixtureRoot, 'bin/timeout')); + const result = await run({ mode }); + expect(result.code).toBe(1); + expect(result.stderr).toContain('db_health_timeout_unavailable'); + expect(result.trace.some(event => event.lifecycle?.length)).toBe(false); + }); + + it('waits for starting health before init without starting db', async () => { + const result = await run({ health: 'warming' }); + assertCompleted(result, false); + assertNoDatabaseLifecycle(result.trace); + expect(result.trace.filter(event => event.args[0] === 'inspect').length).toBeGreaterThanOrEqual(2); + expect(result.trace.find(event => event.command === 'sleep').args).toEqual(['2']); + }); + + it('initialize starts a missing database before health and init', async () => { + const result = await run({ mode: 'initialize', health: 'missing' }); + assertCompleted(result, false); + const start = result.trace.findIndex(event => event.lifecycle?.includes('db')); + const health = result.trace.findIndex(event => event.args[0] === 'inspect'); + expect(start).toBeLessThan(health); + }); + + it.each(['starting', 'query_hang'])('the real timeout terminates %s health and prevents init', async health => { + const shortened = replaceOnce(remoteSection(source), 'timeout --signal=KILL 120 ', 'timeout --signal=KILL 1 '); + const result = await run({ health, script: shortened }); + expect(result.code).toBe(1); + expect(result.killed).not.toBe(true); + expect(result.stdout + result.stderr).toContain('db_health_timeout'); + assertNoDatabaseLifecycle(result.trace); + expect(result.trace.some(event => ['run', 'up'].includes(event.compose?.operation))).toBe(false); + }); + + it('init failure prevents server and worker switching', async () => { + const result = await run({ initFails: true, worker: true }); + expect(result.code).toBe(23); + assertNoDatabaseLifecycle(result.trace); + expect(result.trace.filter(event => event.compose?.operation === 'up')).toEqual([]); + }); + + it('unsupported isolation fails instead of retrying without protection', async () => { + const result = await run({ unsupportedIsolation: true }); + expect(result.code).toBe(2); + assertNoDatabaseLifecycle(result.trace); + expect(result.trace.filter(event => event.compose?.operation === 'run')).toHaveLength(1); + expect(result.trace.filter(event => event.compose?.operation === 'up')).toEqual([]); + }); + + it.each([ + ['build:init', 'run:init'], ['run:init', 'build:server'], ['build:server', 'up:server'], + ['up:server', 'up:worker'], ['build:worker', 'up:worker'], + ])('health drift after %s prevents %s without managing db', async (driftAfter, forbiddenStage) => { + const result = await run({ driftAfter, worker: true, workerImagePresent: false }); + expect(result.code).toBe(1); + expect(result.stdout).toContain('db_not_healthy'); + assertNoDatabaseLifecycle(result.trace); + const stages = result.trace.filter(event => event.compose).map(event => event.compose.operation + ':' + event.compose.targets.join()); + expect(stages).toContain(driftAfter); + expect(stages).not.toContain(forbiddenStage); + }); + + it.each([true, false])('worker image present=%s keeps build outside the guarded switch', async workerImagePresent => { + const result = await run({ worker: true, workerImagePresent }); + assertCompleted(result, true); + assertNoDatabaseLifecycle(result.trace); + const build = result.trace.findIndex(event => event.compose?.operation === 'build' && event.compose.targets.join() === 'worker'); + expect(build >= 0).toBe(!workerImagePresent); + for (const service of ['server', 'worker']) { + const start = result.trace.findIndex(event => event.compose?.operation === 'up' && event.compose.targets.join() === service); + expect(result.trace[start - 1].args[0]).toBe('inspect'); + expect(result.trace[start].args).toContain('--no-build'); + } + }); + + it('a worker container with a missing image tag is rebuilt before switching server', async () => { + const result = await run({ worker: true, workerImagePresent: true, workerTagPresent: false }); + assertCompleted(result, true); + assertNoDatabaseLifecycle(result.trace); + const build = result.trace.findIndex(event => event.compose?.operation === 'build' && event.compose.targets.join() === 'worker'); + const serverSwitch = result.trace.findIndex(event => event.compose?.operation === 'up' && event.compose.targets.join() === 'server'); + expect(build).toBeGreaterThan(-1); + expect(build).toBeLessThan(serverSwitch); + expect(result.trace.some(event => event.command === 'docker' && event.args[0] === 'image')).toBe(true); + }); + + it('the service analyzer rejects explicit, global and implicit database operations', () => { + for (const args of [['up', '-d', 'db'], ['down'], ['up', '-d'], ['restart', 'db'], ['rm', '-f', 'db'], ['run', '--rm', '-T', '--entrypoint', 'sh', 'init', '-lc', 'echo db'], ['up', '-d', '--build', 'server'], ['up', '-d', 'worker']]) { + expect(lifecycleServices(parseCompose(args), services)).toContain('db'); + } + expect(lifecycleServices(parseCompose(['run', '--no-deps', '--rm', '--entrypoint', 'sh', 'init', '-lc', 'echo db']), services)).toEqual(['init']); + expect(lifecycleServices(parseCompose(['exec', '-T', 'server', 'node', 'db']), services)).toEqual([]); + expect(() => parseCompose(['up', '--unknown', 'server'])).toThrow('unknown_compose_option'); + }); + + it.each(['db', 'init', 'server', 'worker'])('real-source mutation %s produces a forbidden database event', async service => { + let mutated = remoteSection(source); + if (service === 'db') mutated = replaceOnce(mutated, 'if [[ "$DEPLOY_DATA_MODE" == "initialize" ]]; then\n "${compose_cmd[@]}" up -d db\nfi', '"${compose_cmd[@]}" up -d db'); + else { + const lines = mutated.split('\n'); + const matches = lines.filter(line => line.includes('--no-deps') && (service === 'init' ? line.includes(' init -lc ') : line.endsWith(' ' + service))); + expect(matches).toHaveLength(1); + mutated = replaceOnce(mutated, matches[0], matches[0].replace('--no-deps ', '')); + } + const result = await run({ script: mutated, worker: true }); + assertCompleted(result, true); + expect(() => assertNoDatabaseLifecycle(result.trace)).toThrow('database_lifecycle_forbidden:'); + }); + + it('unknown external calls cannot escape the private fake PATH', async () => { + const result = await run({ script: 'set -e\nssh unexpected.invalid\n' }); + expect(result.code).toBe(97); + expect(result.trace).toEqual([{ command: 'ssh', args: ['unexpected.invalid'] }]); + }); + + it('the existing wrapper pins mode regardless of an inherited initialize value', async () => { + const wrapper = await readFile(path.join(ROOT, 'deploy/trialrun/deploy-existing.sh'), 'utf8'); + const invocations = [...wrapper.matchAll(/DEPLOY_SERVER="\$DEPLOY_SERVER" \\\nREMOTE_DIR="\$\{contract\[REMOTE_DIR\]\}"[\s\S]*?bash "\$ROOT_DIR\/deploy\/trialrun\/deploy\.sh"/g)]; + expect(invocations).toHaveLength(1); + const body = invocations[0][0].replace('bash "$ROOT_DIR/deploy/trialrun/deploy.sh"', '/usr/bin/printenv DEPLOY_DATA_MODE'); + const result = await execute('/usr/bin/bash', ['-c', 'declare -A contract=();\n' + body], { env: { ROOT_DIR: fixtureRoot, DEPLOY_DATA_MODE: 'initialize' } }); + expect(result.stdout.trim()).toBe('existing'); + }); +});