From fa326f92fae4bdfe4b59afb6851e559b4a3655fb Mon Sep 17 00:00:00 2001 From: shunli Date: Wed, 29 Jul 2026 08:28:42 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B6=E6=95=9B=E7=9C=9F=E5=AE=9EOCR?= =?UTF-8?q?=E6=A0=B7=E7=A5=A8=E9=AA=8C=E6=94=B6=E5=85=A5=E5=8F=A3=EF=BC=88?= =?UTF-8?q?07-28-travel-reimbursement-guide=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + .../spec/hyb-oa/reimbursement-invoice-ocr.md | 97 ++++ .../design.md | 8 + .../implement.md | 2 + scripts/qa/check-ocr-provider-samples.mjs | 505 ++++++------------ scripts/qa/run-p2-safe-regression.sh | 4 +- .../unit/scripts/ocr-provider-samples.test.js | 174 ++++++ 7 files changed, 464 insertions(+), 329 deletions(-) create mode 100644 server/tests/unit/scripts/ocr-provider-samples.test.js diff --git a/.gitignore b/.gitignore index d394f45e..ad9ad76b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,9 @@ client/playwright-report output/ artifacts/ +# Local sensitive business samples; never commit originals +/input/ + # Vitest coverage artifacts server/coverage server/.vitest diff --git a/.trellis/spec/hyb-oa/reimbursement-invoice-ocr.md b/.trellis/spec/hyb-oa/reimbursement-invoice-ocr.md index 1aca75d7..a0d531e2 100644 --- a/.trellis/spec/hyb-oa/reimbursement-invoice-ocr.md +++ b/.trellis/spec/hyb-oa/reimbursement-invoice-ocr.md @@ -251,6 +251,103 @@ const persisted = sanitizeOcrResultForPersistence("tencent", normalized); await prisma.ocrResult.create({ data: { result_json: persisted } }); ``` +## Scenario: Live OCR Sample Stdin Probe + +### 1. Scope / Trigger + +- Trigger: an authorized real PDF must prove Tencent credentials, network, + subtype normalization and required-field behavior without creating an OA + invoice or persisting the sample on the execution host. +- This probe is intentionally one-file and PDF-only. It does not retain the + historical manifest, directory scan, image evaluation or legacy VAT-only + command modes. + +### 2. Signatures + +- Command: + `node scripts/qa/check-ocr-provider-samples.mjs --alias=SAMPLE-PDF-`. +- Input: exactly one PDF on stdin, at most `OCR_PDF_MAX_BYTES` (7 MiB). +- Runtime env: `TENCENT_SECRET_ID`, `TENCENT_SECRET_KEY`, optional + `TENCENT_OCR_REGION` and `OCR_TIMEOUT_MS`; provider is fixed to `tencent`. +- Provider boundary: `runOcrForInvoice({ config, ocrType: "vat", + imageBase64, sourceMimeType: "application/pdf" })`. +- Output: one JSON object containing only `alias`, `provider`, `status`, + `transportMode`, `elapsedMs`, safe document category/subtype/count, + per-field hit booleans, missing field names and a bounded error code. + +### 3. Contracts + +- The caller selects one authorized PDF explicitly and streams it through + stdin. The probe never accepts a file path, directory, manifest, image or + batch list and never writes sample bytes to disk, OA storage or the database. +- Alias must match `SAMPLE-PDF-[A-Z0-9_-]+`; it replaces the original file name + in every output and evidence record. +- Validate the 7 MiB limit, PDF version header and terminal `%%EOF` before + constructing Base64 or invoking Tencent. One process performs at most one + provider call. +- Reuse `RecognizeGeneralInvoice`, `sanitizeOcrResultForPersistence` and the + production category-specific required-field rules. The probe must not carry + a parallel VAT-only field list. +- Never output recognized identifiers, dates, amounts, buyer/seller/passenger + names, provider messages, raw response or input bytes. Field presence is a + boolean; errors are bounded stable codes. +- A PASS proves the direct provider/normalization path only. It does not prove + OA upload, queue, persistence, duplicate detection, binding or reimbursement + submission. + +### 4. Validation & Error Matrix + +- Missing/invalid alias -> `sample_alias_invalid`; no provider call. +- Empty stdin -> `sample_empty`; no provider call. +- More than 7 MiB -> `sample_too_large`; stop reading and do not call provider. +- Invalid header or missing terminal EOF -> `sample_pdf_invalid`; no provider + call. +- Missing credentials or provider mismatch -> `ocr_credentials_missing` / + `ocr_provider_invalid`; no business values in output. +- Supported subtype with missing category-required fields -> `status=HOLD` + plus field names only. +- Tencent failure -> `status=HOLD` plus a bounded SDK error code; omit the + provider message and raw error object. + +### 5. Good/Base/Bad Cases + +- Good: stream one railway electronic ticket using an anonymous alias; receive + `electronic_train_ticket`, boolean field hits and no recognized values or + persisted sample. +- Base: stream one supported accommodation invoice and receive a PASS summary + for its VAT/full-VAT category. +- Bad: point the probe at a directory, auto-run every PDF, write a manifest with + real paths, accept images, print OCR values or retain the sample remotely. + +### 6. Tests Required + +- Unit tests inject synthetic provider results for VAT, full VAT and railway + categories and assert category-aware required fields. +- Negative tests assert non-PDF, truncated PDF and oversized input fail before + the provider mock is called. +- Privacy tests seed raw provider and recognizable normalized markers, then + assert serialized output contains only booleans and safe metadata. +- CLI test pipes one synthetic PDF over stdin with credentials absent and + asserts one parseable redacted error object with empty stderr. +- Before deployment run `node --check`, the focused tests, full backend tests, + sensitive-content scan, task validation and Trellis governance. + +### 7. Wrong vs Correct + +#### Wrong + +```bash +PLAYWRIGHT_OCR_LIVE_DIR= \ + node scripts/qa/check-ocr-provider-samples.mjs --provider=tencent --json +``` + +#### Correct + +```bash +node scripts/qa/check-ocr-provider-samples.mjs \ + --alias=SAMPLE-PDF-TRAIN < +``` + ## Scenario: Payment Batch Receipt Attention Count ### 1. Scope / Trigger diff --git a/.trellis/tasks/07-28-travel-reimbursement-guide/design.md b/.trellis/tasks/07-28-travel-reimbursement-guide/design.md index af85ea76..1e4d8a8f 100644 --- a/.trellis/tasks/07-28-travel-reimbursement-guide/design.md +++ b/.trellis/tasks/07-28-travel-reimbursement-guide/design.md @@ -44,6 +44,14 @@ 7. 只记录 provider、状态、尝试/完成证据和脱敏字段命中摘要；不记录票面、完整字段或原始 payload。 8. 正常清理或归档测试业务记录，再允许杨俊提交真实票据。 +### 授权样票验证工具边界 + +- 既有 `check-ocr-provider-samples.mjs` 必须与生产 adapter 使用同一 `RecognizeGeneralInvoice` 入口和按票种必填字段规则，不得继续引用已删除的 VAT-only 导出或把铁路电子客票套入 VAT 五字段门禁。 +- 真实 PDF 从本地受控、Git 忽略的样票目录按匿名别名选择，经 SSH 标准输入直接流入生产 server 容器内存；不得把原件、临时副本或 manifest 写入远端磁盘、OA 附件存储、数据库、仓库或报告。 +- 每个进程只接收一份 PDF，验证文件签名和 7MiB 上限后最多调用一次 `RecognizeGeneralInvoice`。住宿与铁路样票分两次执行，任一失败立即停止，不自动遍历目录中的其他文件。 +- 输出只允许匿名别名、provider、票种、安全传输模式、耗时、字段命中布尔值、缺字段名和稳定错误码；不得输出文件名、原始响应、识别值、人员/企业名称、票号、金额或票面内容。 +- live runner 的代码、合成测试和生产部署先通过，再单独取得两次真实 API 调用批准；工具可用不等于真实样票已验证。 + ## 文档口径 - `docs/员工快速上手.md` 增加一页式差旅报销步骤：火车票和住宿费拆明细，官方 PDF 绑定对应明细，两类费用均附支付截图，金额差异必须说明，提交后进入主管和财务流程。 diff --git a/.trellis/tasks/07-28-travel-reimbursement-guide/implement.md b/.trellis/tasks/07-28-travel-reimbursement-guide/implement.md index bb756013..01106402 100644 --- a/.trellis/tasks/07-28-travel-reimbursement-guide/implement.md +++ b/.trellis/tasks/07-28-travel-reimbursement-guide/implement.md @@ -44,6 +44,8 @@ - [x] 检查公网 `/ready` 和预期构建标识。 - [x] 在 server 内运行现有 external deps 和运行时 smoke，不输出 Secret 或原始响应。 - [x] 确认实际 provider 为 `tencent`，且只有一个 OCR job executor。 +- [x] 修正 live 样票脚本：复用 `RecognizeGeneralInvoice`、按票种必填字段和安全持久化白名单，支持单 PDF 标准输入且仅输出脱敏摘要。 +- [ ] 使用合成 PDF/provider 响应覆盖住宿、铁路、非 PDF、超限、缺字段和原始响应不输出，并完成本地门禁、提交和生产部署。 - [ ] 经单独样票批准后分别运行住宿电子普通发票和铁路电子客票 live OCR PDF 验收。 - [ ] 只保存脱敏状态、耗时、provider 和字段命中摘要；确认不存在 mock 调用证据。 - [ ] 按正常 API/业务生命周期清理或归档测试记录，并执行只读残留检查。 diff --git a/scripts/qa/check-ocr-provider-samples.mjs b/scripts/qa/check-ocr-provider-samples.mjs index f3a12bad..7e1b37cf 100644 --- a/scripts/qa/check-ocr-provider-samples.mjs +++ b/scripts/qa/check-ocr-provider-samples.mjs @@ -1,196 +1,59 @@ #!/usr/bin/env node import { createRequire } from "node:module"; -import fs from "node:fs"; import path from "node:path"; +import { pathToFileURL } from "node:url"; process.noDeprecation = true; const require = createRequire(import.meta.url); -const { runTencentVatOcr } = require("../../server/src/services/ocr/tencent"); +const { + OCR_PDF_MAX_BYTES, + buildMissingRequiredOcrError, + runOcrForInvoice, + sanitizeOcrResultForPersistence, +} = require("../../server/src/services/ocr"); -const SUPPORTED_PROVIDERS = new Set(["tencent"]); -const MIME_BY_EXT = new Map([ - [".pdf", "application/pdf"], - [".jpg", "image/jpeg"], - [".jpeg", "image/jpeg"], - [".png", "image/png"], -]); -const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png"]); -const PDF_EXTENSIONS = new Set([".pdf"]); -const REQUIRED_FIELDS = ["invoiceCode", "invoiceNumber", "invoiceDate", "totalAmount", "sellerName"]; -const OPTIONAL_FIELDS = ["buyerName", "amount", "taxAmount"]; +const PDF_MIME_TYPE = "application/pdf"; +const PDF_EOF_MARKER = Buffer.from("%%EOF", "ascii"); +const SAFE_ALIAS_PATTERN = /^SAMPLE-PDF-[A-Z0-9_-]{1,48}$/u; +const SAFE_TOKEN_PATTERN = /^[A-Za-z0-9_.-]{1,96}$/u; +const SAFE_FIELD_NAMES = [ + "invoiceCode", + "invoiceNumber", + "invoiceDate", + "amount", + "taxAmount", + "totalAmount", + "sellerName", + "buyerName", +]; + +class LiveOcrSampleError extends Error { + constructor(code) { + super(code); + this.name = "LiveOcrSampleError"; + this.code = code; + } +} function parseArgs(argv) { - const options = { - provider: process.env.OCR_PROVIDER_SAMPLE_PROVIDER || "tencent", - json: false, - strict: false, - manifest: process.env.OCR_PROVIDER_SAMPLE_MANIFEST || "", - imageEvaluation: process.env.OCR_IMAGE_OCR_EVALUATION === "1", - }; - + let alias = ""; for (const arg of argv) { - if (arg === "--json") { - options.json = true; - } else if (arg === "--strict") { - options.strict = true; - } else if (arg === "--image-evaluation") { - options.imageEvaluation = true; - } else if (arg.startsWith("--provider=")) { - options.provider = arg.slice("--provider=".length); - } else if (arg.startsWith("--manifest=")) { - options.manifest = arg.slice("--manifest=".length); + if (arg.startsWith("--alias=")) { + alias = arg.slice("--alias=".length).trim().toUpperCase(); + continue; } + throw new LiveOcrSampleError("unsupported_argument"); } - - return options; -} - -function splitEnvList(value) { - return String(value || "") - .split(/[\n,]/) - .map((item) => item.trim()) - .filter(Boolean); -} - -function resolveSamplePath(rawPath, baseDir) { - if (!rawPath || typeof rawPath !== "string") return ""; - return path.isAbsolute(rawPath) ? rawPath : path.resolve(baseDir, rawPath); -} - -function inferKind(filePath, declaredKind) { - const normalizedKind = String(declaredKind || "").trim().toLowerCase(); - if (["pdf", "pdf_vat", "vat_pdf"].includes(normalizedKind)) return "pdf"; - if (["image", "image_vat", "vat_image"].includes(normalizedKind)) return "image"; - - const ext = path.extname(filePath).toLowerCase(); - if (PDF_EXTENSIONS.has(ext)) return "pdf"; - if (IMAGE_EXTENSIONS.has(ext)) return "image"; - return "unknown"; -} - -function safeAlias(value, fallback) { - const alias = String(value || "").trim(); - if (!alias) return fallback; - if (/^SAMPLE-(PDF|IMG)-[A-Z0-9_-]+$/i.test(alias)) return alias.toUpperCase(); - return fallback; -} - -function pushSample(samples, source) { - const kind = inferKind(source.path, source.kind); - const fallbackPrefix = kind === "pdf" ? "SAMPLE-PDF" : kind === "image" ? "SAMPLE-IMG" : "SAMPLE-UNKNOWN"; - samples.push({ - alias: safeAlias(source.alias, `${fallbackPrefix}-${String(samples.length + 1).padStart(3, "0")}`), - filePath: source.path, - kind, - source: source.source, - }); -} - -function loadManifestSamples(manifestPath) { - if (!manifestPath) return { samples: [], error: null }; - if (!fs.existsSync(manifestPath)) { - return { samples: [], error: "manifest_not_found" }; + if (!SAFE_ALIAS_PATTERN.test(alias)) { + throw new LiveOcrSampleError("sample_alias_invalid"); } - - try { - const baseDir = path.dirname(manifestPath); - const parsed = JSON.parse(fs.readFileSync(manifestPath, "utf8")); - const items = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.samples) ? parsed.samples : []; - const samples = []; - - for (const item of items) { - const samplePath = resolveSamplePath(item?.path, baseDir); - if (!samplePath) continue; - pushSample(samples, { - alias: item?.alias, - path: samplePath, - kind: item?.kind, - source: "manifest", - }); - } - - return { samples, error: null }; - } catch { - return { samples: [], error: "manifest_parse_failed" }; - } -} - -function loadEnvSamples() { - const samples = []; - - const pdfFiles = [ - ...splitEnvList(process.env.PLAYWRIGHT_OCR_LIVE_FILE), - ...splitEnvList(process.env.LIVE_OCR_INVOICE_FILE), - ...splitEnvList(process.env.PLAYWRIGHT_OCR_LIVE_FILES), - ]; - - for (const item of pdfFiles) { - pushSample(samples, { path: item, kind: "pdf", source: "env_pdf" }); - } - - const pdfDir = process.env.PLAYWRIGHT_OCR_LIVE_DIR?.trim(); - if (pdfDir && fs.existsSync(pdfDir)) { - for (const entry of fs.readdirSync(pdfDir).sort((left, right) => left.localeCompare(right, "zh-CN"))) { - const filePath = path.join(pdfDir, entry); - if (fs.existsSync(filePath) && fs.statSync(filePath).isFile() && PDF_EXTENSIONS.has(path.extname(entry).toLowerCase())) { - pushSample(samples, { path: filePath, kind: "pdf", source: "env_pdf_dir" }); - } - } - } - - const imageFiles = [ - ...splitEnvList(process.env.PLAYWRIGHT_OCR_IMAGE_FILE), - ...splitEnvList(process.env.PLAYWRIGHT_OCR_IMAGE_FILES), - ...splitEnvList(process.env.LIVE_OCR_IMAGE_FILE), - ]; - - for (const item of imageFiles) { - pushSample(samples, { path: item, kind: "image", source: "env_image" }); - } - - const imageDirs = [ - process.env.PLAYWRIGHT_OCR_IMAGE_DIR?.trim(), - process.env.LIVE_OCR_IMAGE_DIR?.trim(), - ].filter(Boolean); - - for (const dir of imageDirs) { - if (!fs.existsSync(dir)) continue; - for (const entry of fs.readdirSync(dir).sort((left, right) => left.localeCompare(right, "zh-CN"))) { - const filePath = path.join(dir, entry); - if (fs.existsSync(filePath) && fs.statSync(filePath).isFile() && IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase())) { - pushSample(samples, { path: filePath, kind: "image", source: "env_image_dir" }); - } - } - } - - return samples; -} - -function hasExplicitImageEnv() { - return [ - "PLAYWRIGHT_OCR_IMAGE_FILE", - "PLAYWRIGHT_OCR_IMAGE_FILES", - "PLAYWRIGHT_OCR_IMAGE_DIR", - "LIVE_OCR_IMAGE_FILE", - "LIVE_OCR_IMAGE_DIR", - ].some((key) => Boolean(process.env[key]?.trim())); -} - -function dedupeSamples(samples) { - const seen = new Set(); - const result = []; - for (const sample of samples) { - const key = `${sample.alias}:${sample.filePath}`; - if (seen.has(key)) continue; - seen.add(key); - result.push(sample); - } - return result; + return { alias }; } function getTencentConfig() { return { + ocrProvider: "tencent", tencentSecretId: process.env.TENCENT_SECRET_ID || "", tencentSecretKey: process.env.TENCENT_SECRET_KEY || "", tencentOcrRegion: process.env.TENCENT_OCR_REGION || "ap-beijing", @@ -198,170 +61,158 @@ function getTencentConfig() { }; } -function hasTencentCredentials(config) { - return Boolean(config.tencentSecretId && config.tencentSecretKey); -} - -function summarizeFile(sample) { - try { - if (!fs.existsSync(sample.filePath)) { - return { ok: false, errorCode: "sample_not_found" }; - } - const stat = fs.statSync(sample.filePath); - if (!stat.isFile()) { - return { ok: false, errorCode: "sample_not_file" }; - } - const ext = path.extname(sample.filePath).toLowerCase(); - const mimeType = MIME_BY_EXT.get(ext); - if (!mimeType) { - return { ok: false, errorCode: "unsupported_sample_type" }; - } - return { - ok: true, - mimeType, - fileSizeBytes: stat.size, - transportMode: mimeType === "application/pdf" ? "pdf_native" : "image_native", - }; - } catch { - return { ok: false, errorCode: "sample_stat_failed" }; +function validateTencentConfig(config) { + if (!config?.tencentSecretId || !config?.tencentSecretKey) { + throw new LiveOcrSampleError("ocr_credentials_missing"); + } + if (config.ocrProvider !== "tencent") { + throw new LiveOcrSampleError("ocr_provider_invalid"); } } -function summarizeNormalizedResult(result) { - const fieldHits = {}; - for (const field of [...REQUIRED_FIELDS, ...OPTIONAL_FIELDS]) { - const value = result?.[field]; - fieldHits[field] = value != null && value !== ""; +async function readBoundedStdin(input, maxBytes = OCR_PDF_MAX_BYTES) { + const chunks = []; + let totalBytes = 0; + for await (const chunk of input) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + totalBytes += buffer.length; + if (totalBytes > maxBytes) { + throw new LiveOcrSampleError("sample_too_large"); + } + chunks.push(buffer); } - const missingRequiredFields = REQUIRED_FIELDS.filter((field) => !fieldHits[field]); + if (totalBytes === 0) { + throw new LiveOcrSampleError("sample_empty"); + } + return Buffer.concat(chunks, totalBytes); +} + +function validatePdfBuffer(buffer) { + if (!Buffer.isBuffer(buffer) || buffer.length === 0) { + throw new LiveOcrSampleError("sample_empty"); + } + if (buffer.length > OCR_PDF_MAX_BYTES) { + throw new LiveOcrSampleError("sample_too_large"); + } + + const header = buffer.toString("ascii", 0, Math.min(buffer.length, 16)); + if (!/^%PDF-(?:1\.[0-7]|2\.0)(?:\r\n|\r|\n)/u.test(header)) { + throw new LiveOcrSampleError("sample_pdf_invalid"); + } + + let lastContentIndex = buffer.length - 1; + while ( + lastContentIndex >= 0 && + [0x09, 0x0a, 0x0c, 0x0d, 0x20].includes(buffer[lastContentIndex]) + ) { + lastContentIndex -= 1; + } + const markerStart = lastContentIndex - PDF_EOF_MARKER.length + 1; + if ( + markerStart < 0 || + !buffer.subarray(markerStart, lastContentIndex + 1).equals(PDF_EOF_MARKER) + ) { + throw new LiveOcrSampleError("sample_pdf_invalid"); + } +} + +function safeToken(value) { + return typeof value === "string" && SAFE_TOKEN_PATTERN.test(value) ? value : null; +} + +function safeErrorCode(error) { + const code = typeof error?.code === "string" ? error.code : "provider_failed"; + return SAFE_TOKEN_PATTERN.test(code) ? code : "provider_failed"; +} + +function buildFieldHits(normalized) { + return Object.fromEntries( + SAFE_FIELD_NAMES.map((field) => [field, normalized?.[field] != null && normalized[field] !== ""]), + ); +} + +function buildErrorSummary(alias, error) { return { - fieldHits, - missingRequiredFields, - transportMode: result?.raw?.transportMode || null, + alias: SAFE_ALIAS_PATTERN.test(alias) ? alias : "SAMPLE-PDF-UNKNOWN", + provider: "tencent", + status: "HOLD", + transportMode: "pdf_native", + errorCode: safeErrorCode(error), }; } -async function checkSample(provider, config, sample) { - const fileSummary = summarizeFile(sample); - const base = { - alias: sample.alias, - kind: sample.kind, - source: sample.source, - status: "SKIP", - mimeType: fileSummary.mimeType || null, - fileSizeBytes: fileSummary.fileSizeBytes || null, - transportMode: fileSummary.transportMode || null, +async function runLiveSampleCheck({ + alias, + buffer, + config, + runOcr = runOcrForInvoice, + now = () => Date.now(), +}) { + if (!SAFE_ALIAS_PATTERN.test(alias)) { + throw new LiveOcrSampleError("sample_alias_invalid"); + } + validatePdfBuffer(buffer); + validateTencentConfig(config); + + const startedAt = now(); + const providerResult = await runOcr({ + config, + ocrType: "vat", + imageBase64: buffer.toString("base64"), + sourceMimeType: PDF_MIME_TYPE, + }); + const normalized = sanitizeOcrResultForPersistence("tencent", providerResult); + const qualityError = buildMissingRequiredOcrError("vat", normalized); + + return { + alias, + provider: "tencent", + status: qualityError ? "HOLD" : "PASS", + transportMode: normalized.transportMode === "pdf_native" ? "pdf_native" : null, + elapsedMs: Math.max(0, now() - startedAt), + documentCategory: safeToken(normalized.documentCategory), + documentSubtype: safeToken(normalized.documentSubtype), + documentCount: Number.isInteger(normalized.documentCount) ? normalized.documentCount : null, + fieldHits: buildFieldHits(normalized), + missingRequiredFields: qualityError?.startsWith("missing_required_fields:") + ? qualityError.slice("missing_required_fields:".length).split(",").filter(Boolean) + : [], + errorCode: qualityError && !qualityError.startsWith("missing_required_fields:") + ? safeToken(qualityError) || "invoice_quality_failed" + : null, }; - - if (!fileSummary.ok) { - return { ...base, status: "SKIP", errorCode: fileSummary.errorCode }; - } - - if (provider !== "tencent") { - return { ...base, status: "SKIP", errorCode: "unsupported_provider" }; - } - - if (!hasTencentCredentials(config)) { - return { ...base, status: "SKIP", errorCode: "missing_tencent_credentials" }; - } - - try { - const imageBase64 = fs.readFileSync(sample.filePath).toString("base64"); - const result = await runTencentVatOcr(config, { imageBase64, sourceMimeType: fileSummary.mimeType }); - const normalizedSummary = summarizeNormalizedResult(result); - const missingRequired = normalizedSummary.missingRequiredFields; - return { - ...base, - status: missingRequired.length ? "HOLD" : "PASS", - fieldHits: normalizedSummary.fieldHits, - missingRequiredFields: missingRequired, - transportMode: normalizedSummary.transportMode || base.transportMode, - }; - } catch (error) { - return { - ...base, - status: "HOLD", - errorCode: typeof error?.code === "string" ? error.code : "provider_failed", - transportMode: error?.transportMode || base.transportMode, - }; - } -} - -function buildDecision(results, manifestError, imageEvaluation) { - if (!imageEvaluation) return "FREEZE_BY_DEFAULT"; - if (manifestError) return "HOLD_FOR_SAMPLE"; - if (!results.length) return "HOLD_FOR_SAMPLE"; - const imageResults = results.filter((item) => item.kind === "image"); - if (!imageResults.length) return "HOLD_FOR_SAMPLE"; - if (imageResults.every((item) => item.status === "PASS")) return "GO_TO_IMPLEMENT"; - return "HOLD_FOR_SAMPLE"; -} - -function printHuman(summary) { - console.log(`[ocr-provider-samples] status=${summary.status} decision=${summary.decision}`); - console.log(`[ocr-provider-samples] provider=${summary.provider} imageEvaluation=${summary.imageEvaluation} samples=${summary.counts.total} pdf=${summary.counts.pdf} image=${summary.counts.image}`); - if (summary.manifestError) { - console.log(`[ocr-provider-samples] manifest=${summary.manifestError}`); - } - for (const result of summary.results) { - const error = result.errorCode ? ` error=${result.errorCode}` : ""; - console.log( - `[ocr-provider-samples] ${result.alias} kind=${result.kind} status=${result.status} transport=${result.transportMode || "-"} size=${result.fileSizeBytes ?? "-"}${error}`, - ); - } } async function main() { - const options = parseArgs(process.argv.slice(2)); - if (!SUPPORTED_PROVIDERS.has(options.provider)) { - throw new Error(`unsupported provider: ${options.provider}`); - } - - const manifest = loadManifestSamples(options.manifest); - const samples = dedupeSamples([...manifest.samples, ...loadEnvSamples()]); - const config = getTencentConfig(); - const results = []; - const imageEvaluation = options.imageEvaluation || Boolean(options.manifest) || hasExplicitImageEnv(); - - for (const sample of samples) { - results.push(await checkSample(options.provider, config, sample)); - } - - const decision = buildDecision(results, manifest.error, imageEvaluation); - const failed = results.some((item) => item.status === "HOLD"); - const skipped = !results.length || results.some((item) => item.status === "SKIP"); - const summary = { - checkedAt: new Date().toISOString(), - provider: options.provider, - status: failed ? "HOLD" : skipped ? "SKIP" : "PASS", - decision, - imageEvaluation, - manifestProvided: Boolean(options.manifest), - manifestError: manifest.error, - credentialReady: hasTencentCredentials(config), - counts: { - total: results.length, - pdf: results.filter((item) => item.kind === "pdf").length, - image: results.filter((item) => item.kind === "image").length, - pass: results.filter((item) => item.status === "PASS").length, - hold: results.filter((item) => item.status === "HOLD").length, - skip: results.filter((item) => item.status === "SKIP").length, - }, - results, - }; - - if (options.json) { - console.log(JSON.stringify(summary, null, 2)); - } else { - printHuman(summary); - } - - if (options.strict && summary.status !== "PASS") { - process.exit(1); + let alias = "SAMPLE-PDF-UNKNOWN"; + try { + ({ alias } = parseArgs(process.argv.slice(2))); + const buffer = await readBoundedStdin(process.stdin); + const summary = await runLiveSampleCheck({ + alias, + buffer, + config: getTencentConfig(), + }); + process.stdout.write(`${JSON.stringify(summary)}\n`); + if (summary.status !== "PASS") { + process.exitCode = 1; + } + } catch (error) { + process.stdout.write(`${JSON.stringify(buildErrorSummary(alias, error))}\n`); + process.exitCode = 1; } } -main().catch((error) => { - console.error(`[ocr-provider-samples][ERROR] ${error instanceof Error ? error.message : String(error)}`); - process.exit(1); -}); +const invokedPath = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : ""; +if (import.meta.url === invokedPath) { + await main(); +} + +export { + LiveOcrSampleError, + buildErrorSummary, + parseArgs, + readBoundedStdin, + runLiveSampleCheck, + validatePdfBuffer, +}; diff --git a/scripts/qa/run-p2-safe-regression.sh b/scripts/qa/run-p2-safe-regression.sh index d9bbe9f8..e475b694 100755 --- a/scripts/qa/run-p2-safe-regression.sh +++ b/scripts/qa/run-p2-safe-regression.sh @@ -74,7 +74,7 @@ load_default_env run_shell_step "backend-prisma" "cd '$SERVER_DIR' && npm run prisma:validate" run_shell_step "backend-node-check" "cd '$SERVER_DIR' && node --check src/routes/invoicePool.js src/jobs/ocrJob.js src/services/ocr/tencent.js src/routes/paymentBatches.js src/routes/exports.js src/routes/workflow.js src/routes/wecomAuth.js" -run_shell_step "backend-focused-tests" "cd '$SERVER_DIR' && npm test -- tests/unit/services/ocr.tencent.test.js tests/unit/jobs/ocrJob.test.js tests/unit/routes/invoicePool.upload.test.js tests/unit/routes/paymentBatches.reimbursement.test.js tests/unit/routes/exports.apOperations.test.js tests/unit/routes/wecomAuth.test.js tests/unit/routes/notifications.test.js" +run_shell_step "backend-focused-tests" "cd '$SERVER_DIR' && npm test -- tests/unit/services/ocr.tencent.test.js tests/unit/jobs/ocrJob.test.js tests/unit/routes/invoicePool.upload.test.js tests/unit/scripts/ocr-provider-samples.test.js tests/unit/routes/paymentBatches.reimbursement.test.js tests/unit/routes/exports.apOperations.test.js tests/unit/routes/wecomAuth.test.js tests/unit/routes/notifications.test.js" if [[ "${P2_SAFE_SKIP_BUILD:-0}" == "1" ]]; then echo "[p2-safe][SKIP] frontend-build P2_SAFE_SKIP_BUILD=1" | tee -a "$SUMMARY_TXT" @@ -96,4 +96,4 @@ else fi run_shell_step "wecom-rendering-matrix" "cd '$CLIENT_DIR' && E2E_BASE_URL='$PUBLIC_BASE_URL' npx playwright test tests/e2e/r6-wecom-mobile-rendering-hardening.spec.ts --workers=1 --reporter=list" -run_shell_step "ocr-freeze-default" "node '$ROOT_DIR/scripts/qa/check-ocr-provider-samples.mjs' --provider=tencent --json" +run_shell_step "ocr-live-runner-static" "node --check '$ROOT_DIR/scripts/qa/check-ocr-provider-samples.mjs'" diff --git a/server/tests/unit/scripts/ocr-provider-samples.test.js b/server/tests/unit/scripts/ocr-provider-samples.test.js new file mode 100644 index 00000000..e4001339 --- /dev/null +++ b/server/tests/unit/scripts/ocr-provider-samples.test.js @@ -0,0 +1,174 @@ +const path = require("node:path"); +const { pathToFileURL } = require("node:url"); +const { spawnSync } = require("node:child_process"); + +const ROOT_DIR = path.resolve(__dirname, "../../../.."); +const SCRIPT_PATH = path.join(ROOT_DIR, "scripts/qa/check-ocr-provider-samples.mjs"); +const OCR_PDF_MAX_BYTES = 7 * 1024 * 1024; + +function syntheticPdf(body = "synthetic") { + return Buffer.from(`%PDF-1.7\n${body}\n%%EOF\n`, "ascii"); +} + +function providerConfig() { + return { + ocrProvider: "tencent", + tencentSecretId: "synthetic-id", + tencentSecretKey: "synthetic-key", + tencentOcrRegion: "ap-beijing", + ocrTimeoutMs: 5000, + }; +} + +function normalizedResult(overrides = {}) { + return { + documentSubtype: "VatElectronicCommonInvoice", + documentCategory: "vat_invoice", + documentCount: 1, + invoiceCode: "synthetic-code", + invoiceNumber: "synthetic-number", + invoiceDate: "2026-01-01", + amount: "1.00", + taxAmount: "0.01", + totalAmount: "1.01", + sellerName: "synthetic-seller", + buyerName: "synthetic-buyer", + transportMode: "pdf_native", + ...overrides, + }; +} + +describe("check-ocr-provider-samples", () => { + let liveRunner; + + beforeAll(async () => { + liveRunner = await import(pathToFileURL(SCRIPT_PATH).href); + }); + + it.each([ + ["vat_invoice", normalizedResult()], + [ + "full_vat_invoice", + normalizedResult({ + documentSubtype: "VatElectronicInvoiceFull", + documentCategory: "full_vat_invoice", + invoiceCode: null, + }), + ], + [ + "electronic_train_ticket", + normalizedResult({ + documentSubtype: "ElectronicTrainTicketFull", + documentCategory: "electronic_train_ticket", + invoiceCode: null, + sellerName: null, + }), + ], + ])("accepts category-aware required fields for %s", async (_category, result) => { + const runOcr = vi.fn().mockResolvedValue(result); + + const summary = await liveRunner.runLiveSampleCheck({ + alias: "SAMPLE-PDF-TEST", + buffer: syntheticPdf(), + config: providerConfig(), + runOcr, + now: vi.fn().mockReturnValueOnce(100).mockReturnValueOnce(125), + }); + + expect(summary.status).toBe("PASS"); + expect(summary.elapsedMs).toBe(25); + expect(summary.documentCategory).toBe(result.documentCategory); + expect(summary.missingRequiredFields).toEqual([]); + expect(runOcr).toHaveBeenCalledTimes(1); + expect(runOcr.mock.calls[0][0]).toMatchObject({ + ocrType: "vat", + sourceMimeType: "application/pdf", + }); + }); + + it("holds a railway ticket when its buyer field is missing", async () => { + const summary = await liveRunner.runLiveSampleCheck({ + alias: "SAMPLE-PDF-TRAIN", + buffer: syntheticPdf(), + config: providerConfig(), + runOcr: vi.fn().mockResolvedValue( + normalizedResult({ + documentSubtype: "ElectronicTrainTicketFull", + documentCategory: "electronic_train_ticket", + invoiceCode: null, + sellerName: null, + buyerName: null, + }), + ), + }); + + expect(summary.status).toBe("HOLD"); + expect(summary.missingRequiredFields).toEqual(["buyerName"]); + }); + + it("never exposes provider raw fields or normalized business values", async () => { + const summary = await liveRunner.runLiveSampleCheck({ + alias: "SAMPLE-PDF-PRIVACY", + buffer: syntheticPdf(), + config: providerConfig(), + runOcr: vi.fn().mockResolvedValue( + normalizedResult({ + raw: { + passengerIdentity: "forbidden-identity-marker", + response: "forbidden-response-marker", + }, + }), + ), + }); + const serialized = JSON.stringify(summary); + + expect(serialized).not.toContain("forbidden-identity-marker"); + expect(serialized).not.toContain("forbidden-response-marker"); + expect(serialized).not.toContain("synthetic-number"); + expect(serialized).not.toContain("synthetic-seller"); + expect(summary.fieldHits.invoiceNumber).toBe(true); + expect(summary.fieldHits.sellerName).toBe(true); + }); + + it.each([ + ["non-pdf", Buffer.from("not a pdf", "ascii"), "sample_pdf_invalid"], + ["truncated", Buffer.from("%PDF-1.7\nmissing eof", "ascii"), "sample_pdf_invalid"], + ["oversized", Buffer.alloc(OCR_PDF_MAX_BYTES + 1, 0x20), "sample_too_large"], + ])("rejects %s before provider invocation", async (_caseName, buffer, errorCode) => { + const runOcr = vi.fn(); + + await expect( + liveRunner.runLiveSampleCheck({ + alias: "SAMPLE-PDF-TEST", + buffer, + config: providerConfig(), + runOcr, + }), + ).rejects.toMatchObject({ code: errorCode }); + expect(runOcr).not.toHaveBeenCalled(); + }); + + it("consumes one stdin PDF and returns only a safe config error without credentials", () => { + const result = spawnSync(process.execPath, [SCRIPT_PATH, "--alias=SAMPLE-PDF-STDIN"], { + cwd: ROOT_DIR, + env: { + ...process.env, + TENCENT_SECRET_ID: "", + TENCENT_SECRET_KEY: "", + }, + input: syntheticPdf(), + encoding: "utf8", + }); + const summary = JSON.parse(result.stdout.trim()); + + expect(result.status).toBe(1); + expect(result.stderr).toBe(""); + expect(summary).toEqual({ + alias: "SAMPLE-PDF-STDIN", + provider: "tencent", + status: "HOLD", + transportMode: "pdf_native", + errorCode: "ocr_credentials_missing", + }); + }); +});