收敛真实OCR样票验收入口（07-28-travel-reimbursement-guide）

scripts/qa/check-ocr-provider-samples.mjs

@@ -1,196 +1,59 @@
 #!/usr/bin/env node
 import { createRequire } from "node:module";
-import fs from "node:fs";
 import path from "node:path";
+import { pathToFileURL } from "node:url";
 
 process.noDeprecation = true;
 
 const require = createRequire(import.meta.url);
-const { runTencentVatOcr } = require("../../server/src/services/ocr/tencent");
+const {
+  OCR_PDF_MAX_BYTES,
+  buildMissingRequiredOcrError,
+  runOcrForInvoice,
+  sanitizeOcrResultForPersistence,
+} = require("../../server/src/services/ocr");
 
-const SUPPORTED_PROVIDERS = new Set(["tencent"]);
-const MIME_BY_EXT = new Map([
-  [".pdf", "application/pdf"],
-  [".jpg", "image/jpeg"],
-  [".jpeg", "image/jpeg"],
-  [".png", "image/png"],
-]);
-const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png"]);
-const PDF_EXTENSIONS = new Set([".pdf"]);
-const REQUIRED_FIELDS = ["invoiceCode", "invoiceNumber", "invoiceDate", "totalAmount", "sellerName"];
-const OPTIONAL_FIELDS = ["buyerName", "amount", "taxAmount"];
+const PDF_MIME_TYPE = "application/pdf";
+const PDF_EOF_MARKER = Buffer.from("%%EOF", "ascii");
+const SAFE_ALIAS_PATTERN = /^SAMPLE-PDF-[A-Z0-9_-]{1,48}$/u;
+const SAFE_TOKEN_PATTERN = /^[A-Za-z0-9_.-]{1,96}$/u;
+const SAFE_FIELD_NAMES = [
+  "invoiceCode",
+  "invoiceNumber",
+  "invoiceDate",
+  "amount",
+  "taxAmount",
+  "totalAmount",
+  "sellerName",
+  "buyerName",
+];
+
+class LiveOcrSampleError extends Error {
+  constructor(code) {
+    super(code);
+    this.name = "LiveOcrSampleError";
+    this.code = code;
+  }
+}
 
 function parseArgs(argv) {
-  const options = {
-    provider: process.env.OCR_PROVIDER_SAMPLE_PROVIDER || "tencent",
-    json: false,
-    strict: false,
-    manifest: process.env.OCR_PROVIDER_SAMPLE_MANIFEST || "",
-    imageEvaluation: process.env.OCR_IMAGE_OCR_EVALUATION === "1",
-  };
-
+  let alias = "";
   for (const arg of argv) {
-    if (arg === "--json") {
-      options.json = true;
-    } else if (arg === "--strict") {
-      options.strict = true;
-    } else if (arg === "--image-evaluation") {
-      options.imageEvaluation = true;
-    } else if (arg.startsWith("--provider=")) {
-      options.provider = arg.slice("--provider=".length);
-    } else if (arg.startsWith("--manifest=")) {
-      options.manifest = arg.slice("--manifest=".length);
+    if (arg.startsWith("--alias=")) {
+      alias = arg.slice("--alias=".length).trim().toUpperCase();
+      continue;
     }
+    throw new LiveOcrSampleError("unsupported_argument");
   }
-
-  return options;
-}
-
-function splitEnvList(value) {
-  return String(value || "")
-    .split(/[\n,]/)
-    .map((item) => item.trim())
-    .filter(Boolean);
-}
-
-function resolveSamplePath(rawPath, baseDir) {
-  if (!rawPath || typeof rawPath !== "string") return "";
-  return path.isAbsolute(rawPath) ? rawPath : path.resolve(baseDir, rawPath);
-}
-
-function inferKind(filePath, declaredKind) {
-  const normalizedKind = String(declaredKind || "").trim().toLowerCase();
-  if (["pdf", "pdf_vat", "vat_pdf"].includes(normalizedKind)) return "pdf";
-  if (["image", "image_vat", "vat_image"].includes(normalizedKind)) return "image";
-
-  const ext = path.extname(filePath).toLowerCase();
-  if (PDF_EXTENSIONS.has(ext)) return "pdf";
-  if (IMAGE_EXTENSIONS.has(ext)) return "image";
-  return "unknown";
-}
-
-function safeAlias(value, fallback) {
-  const alias = String(value || "").trim();
-  if (!alias) return fallback;
-  if (/^SAMPLE-(PDF|IMG)-[A-Z0-9_-]+$/i.test(alias)) return alias.toUpperCase();
-  return fallback;
-}
-
-function pushSample(samples, source) {
-  const kind = inferKind(source.path, source.kind);
-  const fallbackPrefix = kind === "pdf" ? "SAMPLE-PDF" : kind === "image" ? "SAMPLE-IMG" : "SAMPLE-UNKNOWN";
-  samples.push({
-    alias: safeAlias(source.alias, `${fallbackPrefix}-${String(samples.length + 1).padStart(3, "0")}`),
-    filePath: source.path,
-    kind,
-    source: source.source,
-  });
-}
-
-function loadManifestSamples(manifestPath) {
-  if (!manifestPath) return { samples: [], error: null };
-  if (!fs.existsSync(manifestPath)) {
-    return { samples: [], error: "manifest_not_found" };
+  if (!SAFE_ALIAS_PATTERN.test(alias)) {
+    throw new LiveOcrSampleError("sample_alias_invalid");
   }
-
-  try {
-    const baseDir = path.dirname(manifestPath);
-    const parsed = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
-    const items = Array.isArray(parsed) ? parsed : Array.isArray(parsed?.samples) ? parsed.samples : [];
-    const samples = [];
-
-    for (const item of items) {
-      const samplePath = resolveSamplePath(item?.path, baseDir);
-      if (!samplePath) continue;
-      pushSample(samples, {
-        alias: item?.alias,
-        path: samplePath,
-        kind: item?.kind,
-        source: "manifest",
-      });
-    }
-
-    return { samples, error: null };
-  } catch {
-    return { samples: [], error: "manifest_parse_failed" };
-  }
-}
-
-function loadEnvSamples() {
-  const samples = [];
-
-  const pdfFiles = [
-    ...splitEnvList(process.env.PLAYWRIGHT_OCR_LIVE_FILE),
-    ...splitEnvList(process.env.LIVE_OCR_INVOICE_FILE),
-    ...splitEnvList(process.env.PLAYWRIGHT_OCR_LIVE_FILES),
-  ];
-
-  for (const item of pdfFiles) {
-    pushSample(samples, { path: item, kind: "pdf", source: "env_pdf" });
-  }
-
-  const pdfDir = process.env.PLAYWRIGHT_OCR_LIVE_DIR?.trim();
-  if (pdfDir && fs.existsSync(pdfDir)) {
-    for (const entry of fs.readdirSync(pdfDir).sort((left, right) => left.localeCompare(right, "zh-CN"))) {
-      const filePath = path.join(pdfDir, entry);
-      if (fs.existsSync(filePath) && fs.statSync(filePath).isFile() && PDF_EXTENSIONS.has(path.extname(entry).toLowerCase())) {
-        pushSample(samples, { path: filePath, kind: "pdf", source: "env_pdf_dir" });
-      }
-    }
-  }
-
-  const imageFiles = [
-    ...splitEnvList(process.env.PLAYWRIGHT_OCR_IMAGE_FILE),
-    ...splitEnvList(process.env.PLAYWRIGHT_OCR_IMAGE_FILES),
-    ...splitEnvList(process.env.LIVE_OCR_IMAGE_FILE),
-  ];
-
-  for (const item of imageFiles) {
-    pushSample(samples, { path: item, kind: "image", source: "env_image" });
-  }
-
-  const imageDirs = [
-    process.env.PLAYWRIGHT_OCR_IMAGE_DIR?.trim(),
-    process.env.LIVE_OCR_IMAGE_DIR?.trim(),
-  ].filter(Boolean);
-
-  for (const dir of imageDirs) {
-    if (!fs.existsSync(dir)) continue;
-    for (const entry of fs.readdirSync(dir).sort((left, right) => left.localeCompare(right, "zh-CN"))) {
-      const filePath = path.join(dir, entry);
-      if (fs.existsSync(filePath) && fs.statSync(filePath).isFile() && IMAGE_EXTENSIONS.has(path.extname(entry).toLowerCase())) {
-        pushSample(samples, { path: filePath, kind: "image", source: "env_image_dir" });
-      }
-    }
-  }
-
-  return samples;
-}
-
-function hasExplicitImageEnv() {
-  return [
-    "PLAYWRIGHT_OCR_IMAGE_FILE",
-    "PLAYWRIGHT_OCR_IMAGE_FILES",
-    "PLAYWRIGHT_OCR_IMAGE_DIR",
-    "LIVE_OCR_IMAGE_FILE",
-    "LIVE_OCR_IMAGE_DIR",
-  ].some((key) => Boolean(process.env[key]?.trim()));
-}
-
-function dedupeSamples(samples) {
-  const seen = new Set();
-  const result = [];
-  for (const sample of samples) {
-    const key = `${sample.alias}:${sample.filePath}`;
-    if (seen.has(key)) continue;
-    seen.add(key);
-    result.push(sample);
-  }
-  return result;
+  return { alias };
 }
 
 function getTencentConfig() {
   return {
+    ocrProvider: "tencent",
     tencentSecretId: process.env.TENCENT_SECRET_ID || "",
     tencentSecretKey: process.env.TENCENT_SECRET_KEY || "",
     tencentOcrRegion: process.env.TENCENT_OCR_REGION || "ap-beijing",
@@ -198,170 +61,158 @@ function getTencentConfig() {
   };
 }
 
-function hasTencentCredentials(config) {
-  return Boolean(config.tencentSecretId && config.tencentSecretKey);
-}
-
-function summarizeFile(sample) {
-  try {
-    if (!fs.existsSync(sample.filePath)) {
-      return { ok: false, errorCode: "sample_not_found" };
-    }
-    const stat = fs.statSync(sample.filePath);
-    if (!stat.isFile()) {
-      return { ok: false, errorCode: "sample_not_file" };
-    }
-    const ext = path.extname(sample.filePath).toLowerCase();
-    const mimeType = MIME_BY_EXT.get(ext);
-    if (!mimeType) {
-      return { ok: false, errorCode: "unsupported_sample_type" };
-    }
-    return {
-      ok: true,
-      mimeType,
-      fileSizeBytes: stat.size,
-      transportMode: mimeType === "application/pdf" ? "pdf_native" : "image_native",
-    };
-  } catch {
-    return { ok: false, errorCode: "sample_stat_failed" };
+function validateTencentConfig(config) {
+  if (!config?.tencentSecretId || !config?.tencentSecretKey) {
+    throw new LiveOcrSampleError("ocr_credentials_missing");
+  }
+  if (config.ocrProvider !== "tencent") {
+    throw new LiveOcrSampleError("ocr_provider_invalid");
   }
 }
 
-function summarizeNormalizedResult(result) {
-  const fieldHits = {};
-  for (const field of [...REQUIRED_FIELDS, ...OPTIONAL_FIELDS]) {
-    const value = result?.[field];
-    fieldHits[field] = value != null && value !== "";
+async function readBoundedStdin(input, maxBytes = OCR_PDF_MAX_BYTES) {
+  const chunks = [];
+  let totalBytes = 0;
+  for await (const chunk of input) {
+    const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+    totalBytes += buffer.length;
+    if (totalBytes > maxBytes) {
+      throw new LiveOcrSampleError("sample_too_large");
+    }
+    chunks.push(buffer);
   }
-  const missingRequiredFields = REQUIRED_FIELDS.filter((field) => !fieldHits[field]);
+  if (totalBytes === 0) {
+    throw new LiveOcrSampleError("sample_empty");
+  }
+  return Buffer.concat(chunks, totalBytes);
+}
+
+function validatePdfBuffer(buffer) {
+  if (!Buffer.isBuffer(buffer) || buffer.length === 0) {
+    throw new LiveOcrSampleError("sample_empty");
+  }
+  if (buffer.length > OCR_PDF_MAX_BYTES) {
+    throw new LiveOcrSampleError("sample_too_large");
+  }
+
+  const header = buffer.toString("ascii", 0, Math.min(buffer.length, 16));
+  if (!/^%PDF-(?:1\.[0-7]|2\.0)(?:\r\n|\r|\n)/u.test(header)) {
+    throw new LiveOcrSampleError("sample_pdf_invalid");
+  }
+
+  let lastContentIndex = buffer.length - 1;
+  while (
+    lastContentIndex >= 0 &&
+    [0x09, 0x0a, 0x0c, 0x0d, 0x20].includes(buffer[lastContentIndex])
+  ) {
+    lastContentIndex -= 1;
+  }
+  const markerStart = lastContentIndex - PDF_EOF_MARKER.length + 1;
+  if (
+    markerStart < 0 ||
+    !buffer.subarray(markerStart, lastContentIndex + 1).equals(PDF_EOF_MARKER)
+  ) {
+    throw new LiveOcrSampleError("sample_pdf_invalid");
+  }
+}
+
+function safeToken(value) {
+  return typeof value === "string" && SAFE_TOKEN_PATTERN.test(value) ? value : null;
+}
+
+function safeErrorCode(error) {
+  const code = typeof error?.code === "string" ? error.code : "provider_failed";
+  return SAFE_TOKEN_PATTERN.test(code) ? code : "provider_failed";
+}
+
+function buildFieldHits(normalized) {
+  return Object.fromEntries(
+    SAFE_FIELD_NAMES.map((field) => [field, normalized?.[field] != null && normalized[field] !== ""]),
+  );
+}
+
+function buildErrorSummary(alias, error) {
   return {
-    fieldHits,
-    missingRequiredFields,
-    transportMode: result?.raw?.transportMode || null,
+    alias: SAFE_ALIAS_PATTERN.test(alias) ? alias : "SAMPLE-PDF-UNKNOWN",
+    provider: "tencent",
+    status: "HOLD",
+    transportMode: "pdf_native",
+    errorCode: safeErrorCode(error),
   };
 }
 
-async function checkSample(provider, config, sample) {
-  const fileSummary = summarizeFile(sample);
-  const base = {
-    alias: sample.alias,
-    kind: sample.kind,
-    source: sample.source,
-    status: "SKIP",
-    mimeType: fileSummary.mimeType || null,
-    fileSizeBytes: fileSummary.fileSizeBytes || null,
-    transportMode: fileSummary.transportMode || null,
+async function runLiveSampleCheck({
+  alias,
+  buffer,
+  config,
+  runOcr = runOcrForInvoice,
+  now = () => Date.now(),
+}) {
+  if (!SAFE_ALIAS_PATTERN.test(alias)) {
+    throw new LiveOcrSampleError("sample_alias_invalid");
+  }
+  validatePdfBuffer(buffer);
+  validateTencentConfig(config);
+
+  const startedAt = now();
+  const providerResult = await runOcr({
+    config,
+    ocrType: "vat",
+    imageBase64: buffer.toString("base64"),
+    sourceMimeType: PDF_MIME_TYPE,
+  });
+  const normalized = sanitizeOcrResultForPersistence("tencent", providerResult);
+  const qualityError = buildMissingRequiredOcrError("vat", normalized);
+
+  return {
+    alias,
+    provider: "tencent",
+    status: qualityError ? "HOLD" : "PASS",
+    transportMode: normalized.transportMode === "pdf_native" ? "pdf_native" : null,
+    elapsedMs: Math.max(0, now() - startedAt),
+    documentCategory: safeToken(normalized.documentCategory),
+    documentSubtype: safeToken(normalized.documentSubtype),
+    documentCount: Number.isInteger(normalized.documentCount) ? normalized.documentCount : null,
+    fieldHits: buildFieldHits(normalized),
+    missingRequiredFields: qualityError?.startsWith("missing_required_fields:")
+      ? qualityError.slice("missing_required_fields:".length).split(",").filter(Boolean)
+      : [],
+    errorCode: qualityError && !qualityError.startsWith("missing_required_fields:")
+      ? safeToken(qualityError) || "invoice_quality_failed"
+      : null,
   };
-
-  if (!fileSummary.ok) {
-    return { ...base, status: "SKIP", errorCode: fileSummary.errorCode };
-  }
-
-  if (provider !== "tencent") {
-    return { ...base, status: "SKIP", errorCode: "unsupported_provider" };
-  }
-
-  if (!hasTencentCredentials(config)) {
-    return { ...base, status: "SKIP", errorCode: "missing_tencent_credentials" };
-  }
-
-  try {
-    const imageBase64 = fs.readFileSync(sample.filePath).toString("base64");
-    const result = await runTencentVatOcr(config, { imageBase64, sourceMimeType: fileSummary.mimeType });
-    const normalizedSummary = summarizeNormalizedResult(result);
-    const missingRequired = normalizedSummary.missingRequiredFields;
-    return {
-      ...base,
-      status: missingRequired.length ? "HOLD" : "PASS",
-      fieldHits: normalizedSummary.fieldHits,
-      missingRequiredFields: missingRequired,
-      transportMode: normalizedSummary.transportMode || base.transportMode,
-    };
-  } catch (error) {
-    return {
-      ...base,
-      status: "HOLD",
-      errorCode: typeof error?.code === "string" ? error.code : "provider_failed",
-      transportMode: error?.transportMode || base.transportMode,
-    };
-  }
-}
-
-function buildDecision(results, manifestError, imageEvaluation) {
-  if (!imageEvaluation) return "FREEZE_BY_DEFAULT";
-  if (manifestError) return "HOLD_FOR_SAMPLE";
-  if (!results.length) return "HOLD_FOR_SAMPLE";
-  const imageResults = results.filter((item) => item.kind === "image");
-  if (!imageResults.length) return "HOLD_FOR_SAMPLE";
-  if (imageResults.every((item) => item.status === "PASS")) return "GO_TO_IMPLEMENT";
-  return "HOLD_FOR_SAMPLE";
-}
-
-function printHuman(summary) {
-  console.log(`[ocr-provider-samples] status=${summary.status} decision=${summary.decision}`);
-  console.log(`[ocr-provider-samples] provider=${summary.provider} imageEvaluation=${summary.imageEvaluation} samples=${summary.counts.total} pdf=${summary.counts.pdf} image=${summary.counts.image}`);
-  if (summary.manifestError) {
-    console.log(`[ocr-provider-samples] manifest=${summary.manifestError}`);
-  }
-  for (const result of summary.results) {
-    const error = result.errorCode ? ` error=${result.errorCode}` : "";
-    console.log(
-      `[ocr-provider-samples] ${result.alias} kind=${result.kind} status=${result.status} transport=${result.transportMode || "-"} size=${result.fileSizeBytes ?? "-"}${error}`,
-    );
-  }
 }
 
 async function main() {
-  const options = parseArgs(process.argv.slice(2));
-  if (!SUPPORTED_PROVIDERS.has(options.provider)) {
-    throw new Error(`unsupported provider: ${options.provider}`);
-  }
-
-  const manifest = loadManifestSamples(options.manifest);
-  const samples = dedupeSamples([...manifest.samples, ...loadEnvSamples()]);
-  const config = getTencentConfig();
-  const results = [];
-  const imageEvaluation = options.imageEvaluation || Boolean(options.manifest) || hasExplicitImageEnv();
-
-  for (const sample of samples) {
-    results.push(await checkSample(options.provider, config, sample));
-  }
-
-  const decision = buildDecision(results, manifest.error, imageEvaluation);
-  const failed = results.some((item) => item.status === "HOLD");
-  const skipped = !results.length || results.some((item) => item.status === "SKIP");
-  const summary = {
-    checkedAt: new Date().toISOString(),
-    provider: options.provider,
-    status: failed ? "HOLD" : skipped ? "SKIP" : "PASS",
-    decision,
-    imageEvaluation,
-    manifestProvided: Boolean(options.manifest),
-    manifestError: manifest.error,
-    credentialReady: hasTencentCredentials(config),
-    counts: {
-      total: results.length,
-      pdf: results.filter((item) => item.kind === "pdf").length,
-      image: results.filter((item) => item.kind === "image").length,
-      pass: results.filter((item) => item.status === "PASS").length,
-      hold: results.filter((item) => item.status === "HOLD").length,
-      skip: results.filter((item) => item.status === "SKIP").length,
-    },
-    results,
-  };
-
-  if (options.json) {
-    console.log(JSON.stringify(summary, null, 2));
-  } else {
-    printHuman(summary);
-  }
-
-  if (options.strict && summary.status !== "PASS") {
-    process.exit(1);
+  let alias = "SAMPLE-PDF-UNKNOWN";
+  try {
+    ({ alias } = parseArgs(process.argv.slice(2)));
+    const buffer = await readBoundedStdin(process.stdin);
+    const summary = await runLiveSampleCheck({
+      alias,
+      buffer,
+      config: getTencentConfig(),
+    });
+    process.stdout.write(`${JSON.stringify(summary)}\n`);
+    if (summary.status !== "PASS") {
+      process.exitCode = 1;
+    }
+  } catch (error) {
+    process.stdout.write(`${JSON.stringify(buildErrorSummary(alias, error))}\n`);
+    process.exitCode = 1;
   }
 }
 
-main().catch((error) => {
-  console.error(`[ocr-provider-samples][ERROR] ${error instanceof Error ? error.message : String(error)}`);
-  process.exit(1);
-});
+const invokedPath = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : "";
+if (import.meta.url === invokedPath) {
+  await main();
+}
+
+export {
+  LiveOcrSampleError,
+  buildErrorSummary,
+  parseArgs,
+  readBoundedStdin,
+  runLiveSampleCheck,
+  validatePdfBuffer,
+};

scripts/qa/run-p2-safe-regression.sh

@@ -74,7 +74,7 @@ load_default_env
 
 run_shell_step "backend-prisma" "cd '$SERVER_DIR' && npm run prisma:validate"
 run_shell_step "backend-node-check" "cd '$SERVER_DIR' && node --check src/routes/invoicePool.js src/jobs/ocrJob.js src/services/ocr/tencent.js src/routes/paymentBatches.js src/routes/exports.js src/routes/workflow.js src/routes/wecomAuth.js"
-run_shell_step "backend-focused-tests" "cd '$SERVER_DIR' && npm test -- tests/unit/services/ocr.tencent.test.js tests/unit/jobs/ocrJob.test.js tests/unit/routes/invoicePool.upload.test.js tests/unit/routes/paymentBatches.reimbursement.test.js tests/unit/routes/exports.apOperations.test.js tests/unit/routes/wecomAuth.test.js tests/unit/routes/notifications.test.js"
+run_shell_step "backend-focused-tests" "cd '$SERVER_DIR' && npm test -- tests/unit/services/ocr.tencent.test.js tests/unit/jobs/ocrJob.test.js tests/unit/routes/invoicePool.upload.test.js tests/unit/scripts/ocr-provider-samples.test.js tests/unit/routes/paymentBatches.reimbursement.test.js tests/unit/routes/exports.apOperations.test.js tests/unit/routes/wecomAuth.test.js tests/unit/routes/notifications.test.js"
 
 if [[ "${P2_SAFE_SKIP_BUILD:-0}" == "1" ]]; then
   echo "[p2-safe][SKIP] frontend-build P2_SAFE_SKIP_BUILD=1" | tee -a "$SUMMARY_TXT"
@@ -96,4 +96,4 @@ else
 fi
 
 run_shell_step "wecom-rendering-matrix" "cd '$CLIENT_DIR' && E2E_BASE_URL='$PUBLIC_BASE_URL' npx playwright test tests/e2e/r6-wecom-mobile-rendering-hardening.spec.ts --workers=1 --reporter=list"
-run_shell_step "ocr-freeze-default" "node '$ROOT_DIR/scripts/qa/check-ocr-provider-samples.mjs' --provider=tencent --json"
+run_shell_step "ocr-live-runner-static" "node --check '$ROOT_DIR/scripts/qa/check-ocr-provider-samples.mjs'"