From abba3371dd5ec33cad2af191918419e9d9fbea3f Mon Sep 17 00:00:00 2001 From: shunli Date: Tue, 5 May 2026 22:16:13 +0800 Subject: [PATCH] =?UTF-8?q?R92=20A2=20=E4=BF=AE=E5=A4=8D=E5=8F=91=E7=A5=A8?= =?UTF-8?q?=20OCR=20provider=20=E4=B8=8E=E4=BA=8C=E8=BF=9B=E5=88=B6?= =?UTF-8?q?=E5=86=99=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migration.sql | 1 + server/prisma/schema.prisma | 1 + server/src/jobs/ocrJob.js | 33 +++++++++--- server/src/routes/invoicePool.js | 5 +- server/src/services/ocr/mock.js | 37 ++++++++++--- server/tests/unit/jobs/ocrJob.test.js | 52 +++++++++++++++++++ .../unit/routes/invoicePool.upload.test.js | 1 + 7 files changed, 113 insertions(+), 17 deletions(-) create mode 100644 server/prisma/migrations/20260505211000_r92_ocr_provider_mock/migration.sql diff --git a/server/prisma/migrations/20260505211000_r92_ocr_provider_mock/migration.sql b/server/prisma/migrations/20260505211000_r92_ocr_provider_mock/migration.sql new file mode 100644 index 00000000..50d6dd1a --- /dev/null +++ b/server/prisma/migrations/20260505211000_r92_ocr_provider_mock/migration.sql @@ -0,0 +1 @@ +ALTER TYPE "OcrProvider" ADD VALUE IF NOT EXISTS 'mock'; diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma index 236d087c..b3c56ee7 100644 --- a/server/prisma/schema.prisma +++ b/server/prisma/schema.prisma @@ -160,6 +160,7 @@ enum InvoiceArchivedReason { enum OcrProvider { tencent + mock } enum OcrType { diff --git a/server/src/jobs/ocrJob.js b/server/src/jobs/ocrJob.js index ac7bc7fb..25b6e513 100644 --- a/server/src/jobs/ocrJob.js +++ b/server/src/jobs/ocrJob.js @@ -4,6 +4,16 @@ const { buildMissingRequiredOcrError, runOcrForInvoice } = require("../services/ const { readFileBuffer } = require("../services/storage"); const OCR_DUPLICATE_ERROR = "duplicate_invoice_code_number"; +const SUPPORTED_OCR_PROVIDERS = new Set(["tencent", "mock"]); + +function getEffectiveOcrProvider(config) { + const provider = config?.ocrProvider; + if (!SUPPORTED_OCR_PROVIDERS.has(provider)) { + throw new Error(`Unsupported OCR provider: ${provider}`); + } + + return provider; +} function trimErrorMessage(error) { const code = typeof error?.code === "string" ? error.code : null; @@ -59,7 +69,7 @@ async function claimPendingInvoices(prisma, config, workerId) { ocr_lock_owner = ${workerId} FROM candidates WHERE i.id = candidates.id - RETURNING i.id, i.file_sha256, i.ocr_type, i.storage_path, i.mime_type, i.ocr_attempts + RETURNING i.id, i.file_sha256, i.ocr_provider, i.ocr_type, i.storage_path, i.mime_type, i.ocr_attempts `; } @@ -89,10 +99,11 @@ async function findDuplicateInvoice(prisma, invoiceId, invoiceCode, invoiceNumbe } async function getOrCreateOcrResult(prisma, config, claimedInvoice, imageBase64) { + const ocrProvider = getEffectiveOcrProvider(config); const where = { file_sha256_ocr_provider_ocr_type: { file_sha256: claimedInvoice.file_sha256, - ocr_provider: "tencent", + ocr_provider: ocrProvider, ocr_type: claimedInvoice.ocr_type, }, }; @@ -123,7 +134,7 @@ async function getOrCreateOcrResult(prisma, config, claimedInvoice, imageBase64) const created = await prisma.ocrResult.create({ data: { file_sha256: claimedInvoice.file_sha256, - ocr_provider: "tencent", + ocr_provider: ocrProvider, ocr_type: claimedInvoice.ocr_type, result_json: normalized, }, @@ -160,12 +171,13 @@ async function getOrCreateOcrResult(prisma, config, claimedInvoice, imageBase64) } } -async function markInvoiceAsDuplicate(prisma, invoiceId, normalized, ocrResultId) { +async function markInvoiceAsDuplicate(prisma, config, invoiceId, normalized, ocrResultId) { const now = new Date(); await prisma.invoicePool.update({ where: { id: invoiceId }, data: { status: "ocr_failed", + ocr_provider: getEffectiveOcrProvider(config), ocr_result_id: ocrResultId, invoice_code: normalized.invoiceCode ?? null, invoice_number: normalized.invoiceNumber ?? null, @@ -184,12 +196,13 @@ async function markInvoiceAsDuplicate(prisma, invoiceId, normalized, ocrResultId }); } -async function markInvoiceAsAvailable(prisma, invoiceId, normalized, ocrResultId) { +async function markInvoiceAsAvailable(prisma, config, invoiceId, normalized, ocrResultId) { const now = new Date(); await prisma.invoicePool.update({ where: { id: invoiceId }, data: { status: "available", + ocr_provider: getEffectiveOcrProvider(config), ocr_result_id: ocrResultId, invoice_code: normalized.invoiceCode ?? null, invoice_number: normalized.invoiceNumber ?? null, @@ -208,12 +221,13 @@ async function markInvoiceAsAvailable(prisma, invoiceId, normalized, ocrResultId }); } -async function markInvoiceAsQualityFailed(prisma, invoiceId, normalized, ocrResultId, errorMessage) { +async function markInvoiceAsQualityFailed(prisma, config, invoiceId, normalized, ocrResultId, errorMessage) { const now = new Date(); await prisma.invoicePool.update({ where: { id: invoiceId }, data: { status: "ocr_failed", + ocr_provider: getEffectiveOcrProvider(config), ocr_result_id: ocrResultId, invoice_code: normalized.invoiceCode ?? null, invoice_number: normalized.invoiceNumber ?? null, @@ -241,6 +255,7 @@ async function markInvoiceAsFailed(prisma, claimedInvoice, errorMessage, config) where: { id: claimedInvoice.id }, data: { status: isTerminal ? "ocr_failed" : "pending_ocr", + ocr_provider: getEffectiveOcrProvider(config), ocr_attempts: nextAttempts, ocr_last_error: errorMessage, ocr_completed_at: isTerminal ? now : null, @@ -269,6 +284,7 @@ async function processClaimedInvoice(prisma, config, claimedInvoice, workerId, l if (qualityError) { await markInvoiceAsQualityFailed( prisma, + config, claimedInvoice.id, normalized, ocrResultId, @@ -295,7 +311,7 @@ async function processClaimedInvoice(prisma, config, claimedInvoice, workerId, l ); if (duplicate) { - await markInvoiceAsDuplicate(prisma, claimedInvoice.id, normalized, ocrResultId); + await markInvoiceAsDuplicate(prisma, config, claimedInvoice.id, normalized, ocrResultId); logger.warn( { invoiceId: claimedInvoice.id, @@ -308,7 +324,7 @@ async function processClaimedInvoice(prisma, config, claimedInvoice, workerId, l return { outcome: "duplicate" }; } - await markInvoiceAsAvailable(prisma, claimedInvoice.id, normalized, ocrResultId); + await markInvoiceAsAvailable(prisma, config, claimedInvoice.id, normalized, ocrResultId); return { outcome: "available" }; } catch (error) { const message = trimErrorMessage(error); @@ -427,6 +443,7 @@ function startOcrJob(fastify) { module.exports = { OCR_DUPLICATE_ERROR, claimPendingInvoices, + getEffectiveOcrProvider, markInvoiceAsQualityFailed, runOcrOnce, startOcrJob, diff --git a/server/src/routes/invoicePool.js b/server/src/routes/invoicePool.js index f8f6d202..47a75778 100644 --- a/server/src/routes/invoicePool.js +++ b/server/src/routes/invoicePool.js @@ -356,7 +356,8 @@ async function invoicePoolRoutes(fastify) { contentType: fileMimeType, }); - const status = isOcrEnabledForType(fastify.config, ocrType) ? "pending_ocr" : "available"; + const ocrEnabled = isOcrEnabledForType(fastify.config, ocrType); + const status = ocrEnabled ? "pending_ocr" : "available"; const invoice = await fastify.prisma.$transaction(async (tx) => { const now = new Date(); @@ -369,6 +370,7 @@ async function invoicePoolRoutes(fastify) { mime_type: fileMimeType, file_size: fileSize, file_sha256: fileSha256, + ocr_provider: ocrEnabled ? fastify.config.ocrProvider : undefined, ocr_type: ocrType, status, ocr_last_requested_at: status === "pending_ocr" ? now : null, @@ -1036,6 +1038,7 @@ async function invoicePoolRoutes(fastify) { where: { id: invoiceId }, data: { status: "pending_ocr", + ocr_provider: fastify.config.ocrProvider, ocr_last_requested_at: now, ocr_last_error: null, ocr_completed_at: null, diff --git a/server/src/services/ocr/mock.js b/server/src/services/ocr/mock.js index ed6a6069..cbe48d62 100644 --- a/server/src/services/ocr/mock.js +++ b/server/src/services/ocr/mock.js @@ -1,20 +1,39 @@ const crypto = require("node:crypto"); -function decodeMarker(imageBase64) { +function decodeImageBase64(imageBase64) { try { - return Buffer.from(imageBase64, "base64").toString("utf8"); + return Buffer.from(imageBase64, "base64"); } catch { - return ""; + return Buffer.alloc(0); } } -function buildInvoiceKey(markerText) { +function isTextLike(value) { + if (!value) { + return false; + } + + const sample = value.slice(0, 256); + const controlChars = sample.match(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g) ?? []; + return controlChars.length === 0; +} + +function decodeMarker(buffer) { + const text = buffer.toString("utf8"); + if (!isTextLike(text)) { + return ""; + } + + return text.replace(/[\u0000-\u001F\u007F]/g, " ").replace(/\s+/g, " ").trim(); +} + +function buildInvoiceKey(buffer, markerText) { const duplicateMatch = markerText.match(/MOCK_DUP:([A-Za-z0-9_-]+)/); if (duplicateMatch?.[1]) { return `dup:${duplicateMatch[1]}`; } - const hash = crypto.createHash("sha256").update(markerText || "mock-empty").digest("hex"); + const hash = crypto.createHash("sha256").update(buffer.length ? buffer : "mock-empty").digest("hex"); return `hash:${hash.slice(0, 20)}`; } @@ -40,12 +59,13 @@ function toAmount(seedHex, fallback) { } async function runMockOcr({ ocrType, imageBase64 }) { - const markerText = decodeMarker(imageBase64); + const buffer = decodeImageBase64(imageBase64); + const markerText = decodeMarker(buffer); if (markerText.includes("MOCK_OCR_FAIL")) { throw new Error("mock_ocr_failed"); } - const invoiceKey = buildInvoiceKey(markerText); + const invoiceKey = buildInvoiceKey(buffer, markerText); const hash = crypto.createHash("sha256").update(invoiceKey).digest("hex"); const invoiceCode = `MOCK${hash.slice(0, 8).toUpperCase()}`; const invoiceNumber = hash.slice(8, 16).toUpperCase(); @@ -62,7 +82,8 @@ async function runMockOcr({ ocrType, imageBase64 }) { raw: { provider: "mock", invoiceKey, - marker: markerText.slice(0, 128), + marker: markerText ? markerText.slice(0, 128) : null, + sourceSha256: crypto.createHash("sha256").update(buffer).digest("hex"), }, }; diff --git a/server/tests/unit/jobs/ocrJob.test.js b/server/tests/unit/jobs/ocrJob.test.js index 80adb93e..c9c314ac 100644 --- a/server/tests/unit/jobs/ocrJob.test.js +++ b/server/tests/unit/jobs/ocrJob.test.js @@ -8,8 +8,10 @@ function buildClaimedInvoice(overrides = {}) { return { id: "11111111-1111-4111-8111-111111111111", file_sha256: "sha256-1", + ocr_provider: "mock", ocr_type: "vat", storage_path: "20260428/invoice.txt", + mime_type: "text/plain", ocr_attempts: 0, ...overrides, }; @@ -74,10 +76,18 @@ describe("ocrJob quality gate", () => { where: { id: "11111111-1111-4111-8111-111111111111" }, data: expect.objectContaining({ status: "available", + ocr_provider: "mock", ocr_last_error: null, }), }), ); + expect(prisma.ocrResult.create).toHaveBeenCalledWith( + expect.objectContaining({ + data: expect.objectContaining({ + ocr_provider: "mock", + }), + }), + ); }); it("marks invoice ocr_failed when required VAT fields are missing", async () => { @@ -112,6 +122,7 @@ describe("ocrJob quality gate", () => { where: { id: "11111111-1111-4111-8111-111111111111" }, data: expect.objectContaining({ status: "ocr_failed", + ocr_provider: "mock", ocr_last_error: "missing_required_fields:invoiceCode,sellerName", }), }), @@ -124,4 +135,45 @@ describe("ocrJob quality gate", () => { "OCR invoice marked as failed for missing required fields", ); }); + + it("stores binary PDF mock OCR metadata without unsafe null marker text", async () => { + const pdfPayload = Buffer.from([0x25, 0x50, 0x44, 0x46, 0x00, 0xff, 0x10, 0x20, 0x41]); + await fs.writeFile(path.join(tempDir, "20260428/invoice.pdf"), pdfPayload); + + const prisma = buildPrisma([ + buildClaimedInvoice({ + storage_path: "20260428/invoice.pdf", + mime_type: "application/pdf", + file_sha256: "sha256-pdf-binary", + }), + ]); + const logger = { + warn: vi.fn(), + error: vi.fn(), + }; + + const result = await runOcrOnce( + prisma, + { + storageDir: tempDir, + ocrBatchSize: 1, + ocrLockTtlSeconds: 300, + ocrMaxAttempts: 3, + ocrProvider: "mock", + }, + logger, + "ocr-test-worker", + ); + + expect(result).toEqual({ claimed: 1, available: 1, duplicate: 0, failed: 0 }); + const createdPayload = prisma.ocrResult.create.mock.calls[0][0].data.result_json; + expect(createdPayload.raw).toEqual( + expect.objectContaining({ + provider: "mock", + marker: null, + sourceSha256: expect.any(String), + }), + ); + expect(JSON.stringify(createdPayload)).not.toContain("\\u0000"); + }); }); diff --git a/server/tests/unit/routes/invoicePool.upload.test.js b/server/tests/unit/routes/invoicePool.upload.test.js index f99f0ec3..2b6ca877 100644 --- a/server/tests/unit/routes/invoicePool.upload.test.js +++ b/server/tests/unit/routes/invoicePool.upload.test.js @@ -197,6 +197,7 @@ describe("invoicePool upload route", () => { expect.objectContaining({ data: expect.objectContaining({ ocr_type: "vat", + ocr_provider: "mock", status: "pending_ocr", }), }),