R73-A2 收敛 VAT OCR 归一化与质量门槛

这个提交包含在:
2026-04-28 21:13:21 +08:00
父节点 82fbd39114
当前提交 b7c018922c
共修改 9 个文件,包含 600 行新增和 14 行删除
+46 -1
查看文件
@@ -1,6 +1,6 @@
const cron = require("node-cron");
const { runOcrForInvoice } = require("../services/ocr");
const { buildMissingRequiredOcrError, runOcrForInvoice } = require("../services/ocr");
const { readFileBuffer } = require("../services/storage");
const OCR_DUPLICATE_ERROR = "duplicate_invoice_code_number";
@@ -201,6 +201,30 @@ async function markInvoiceAsAvailable(prisma, invoiceId, normalized, ocrResultId
});
}
async function markInvoiceAsQualityFailed(prisma, invoiceId, normalized, ocrResultId, errorMessage) {
const now = new Date();
await prisma.invoicePool.update({
where: { id: invoiceId },
data: {
status: "ocr_failed",
ocr_result_id: ocrResultId,
invoice_code: normalized.invoiceCode ?? null,
invoice_number: normalized.invoiceNumber ?? null,
invoice_date: normalized.invoiceDate ? new Date(toDateOnly(normalized.invoiceDate)) : null,
amount: toMoney(normalized.amount),
tax_amount: toMoney(normalized.taxAmount),
total_amount: toMoney(normalized.totalAmount),
seller_name: normalized.sellerName ?? null,
buyer_name: normalized.buyerName ?? null,
ocr_last_error: errorMessage,
ocr_completed_at: now,
ocr_attempts: { increment: 1 },
ocr_locked_at: null,
ocr_lock_owner: null,
},
});
}
async function markInvoiceAsFailed(prisma, claimedInvoice, errorMessage, config) {
const now = new Date();
const nextAttempts = Number(claimedInvoice.ocr_attempts || 0) + 1;
@@ -234,6 +258,26 @@ async function processClaimedInvoice(prisma, config, claimedInvoice, workerId, l
imageBase64,
);
const qualityError = buildMissingRequiredOcrError(claimedInvoice.ocr_type, normalized);
if (qualityError) {
await markInvoiceAsQualityFailed(
prisma,
claimedInvoice.id,
normalized,
ocrResultId,
qualityError,
);
logger.warn(
{
invoiceId: claimedInvoice.id,
ocrType: claimedInvoice.ocr_type,
qualityError,
},
"OCR invoice marked as failed for missing required fields",
);
return { outcome: "failed" };
}
const duplicate = await findDuplicateInvoice(
prisma,
claimedInvoice.id,
@@ -363,6 +407,7 @@ function startOcrJob(fastify) {
module.exports = {
OCR_DUPLICATE_ERROR,
claimPendingInvoices,
markInvoiceAsQualityFailed,
runOcrOnce,
startOcrJob,
};
+30 -4
查看文件
@@ -9,7 +9,7 @@ const { compactDateString } = require("../utils/date");
const { sha256Hex } = require("../utils/crypto");
const { toMoneyString } = require("../utils/workflow");
const { findActiveDelegationForDelegate } = require("../utils/delegation");
const { isOcrEnabledForType } = require("../services/ocr");
const { getMissingRequiredOcrFields, isOcrEnabledForType } = require("../services/ocr");
const { registerInvoiceVerificationRoutes } = require("./invoicePool/verification");
const { registerInvoicePoolReadRoutes } = require("./invoicePool/read");
const { normalizeInvoiceResponse } = require("./invoicePool/shared/mappers");
@@ -46,7 +46,7 @@ const invoiceEditBodySchema = z
invoiceNumber: z.string().trim().min(1).max(64).nullable().optional(),
invoiceDate: z
.string()
.regex(/^\\d{4}-\\d{2}-\\d{2}$/)
.regex(/^\d{4}-\d{2}-\d{2}$/)
.nullable()
.optional(),
sellerName: z.string().trim().min(1).max(256).nullable().optional(),
@@ -163,7 +163,7 @@ function normalizeMoneyInput(value) {
return null;
}
if (!/^\\d+(?:\\.\\d{1,2})?$/.test(input)) {
if (!/^\d+(?:\.\d{1,2})?$/.test(input)) {
return undefined;
}
@@ -186,7 +186,7 @@ function parseDateOnlyString(value) {
return null;
}
if (!/^\\d{4}-\\d{2}-\\d{2}$/.test(trimmed)) {
if (!/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) {
return undefined;
}
@@ -576,6 +576,32 @@ async function invoicePoolRoutes(fastify) {
if (input.status) {
if (input.status === "available" && current.status === "ocr_failed") {
const nextSnapshot = {
invoiceCode: "invoiceCode" in input ? input.invoiceCode : current.invoice_code,
invoiceNumber: "invoiceNumber" in input ? input.invoiceNumber : current.invoice_number,
invoiceDate:
"invoiceDate" in input
? input.invoiceDate
: current.invoice_date?.toISOString?.()?.slice(0, 10) ?? null,
totalAmount:
"totalAmount" in input
? normalizeMoneyInput(input.totalAmount)
: current.total_amount?.toString?.() ?? null,
sellerName: "sellerName" in input ? input.sellerName : current.seller_name,
};
const missingFields = getMissingRequiredOcrFields(current.ocr_type, nextSnapshot);
if (missingFields.length) {
throw new AppError({
code: "VALID_001",
message: "参数校验失败",
statusCode: 409,
details: {
ocr: "missing_required_fields",
missingFields,
},
});
}
updateData.status = "available";
updateData.ocr_last_error = null;
updateData.ocr_completed_at = new Date();
+27
查看文件
@@ -1,6 +1,10 @@
const { runTencentVatOcr } = require("./tencent");
const { runMockOcr } = require("./mock");
const REQUIRED_OCR_FIELDS_BY_TYPE = {
vat: ["invoiceCode", "invoiceNumber", "invoiceDate", "totalAmount", "sellerName"],
};
function isOcrEnabledForType(config, ocrType) {
if (config.ocrProvider === "mock") {
return ocrType === "vat";
@@ -9,6 +13,27 @@ function isOcrEnabledForType(config, ocrType) {
return config.ocrProvider === "tencent" && ocrType === "vat";
}
function getMissingRequiredOcrFields(ocrType, normalized) {
const requiredFields = REQUIRED_OCR_FIELDS_BY_TYPE[ocrType] ?? [];
if (!requiredFields.length) {
return [];
}
return requiredFields.filter((field) => {
const value = normalized?.[field];
return value == null || value === "";
});
}
function buildMissingRequiredOcrError(ocrType, normalized) {
const missingFields = getMissingRequiredOcrFields(ocrType, normalized);
if (!missingFields.length) {
return null;
}
return `missing_required_fields:${missingFields.join(",")}`;
}
async function runOcrForInvoice({ config, ocrType, imageBase64 }) {
if (config.ocrProvider === "mock") {
return runMockOcr({ ocrType, imageBase64 });
@@ -26,6 +51,8 @@ async function runOcrForInvoice({ config, ocrType, imageBase64 }) {
}
module.exports = {
buildMissingRequiredOcrError,
getMissingRequiredOcrFields,
isOcrEnabledForType,
runOcrForInvoice,
};
+21 -1
查看文件
@@ -18,6 +18,18 @@ function buildInvoiceKey(markerText) {
return `hash:${hash.slice(0, 20)}`;
}
function parseMissingFields(markerText) {
const match = markerText.match(/MOCK_OCR_MISSING:([A-Za-z0-9_,-]+)/);
if (!match?.[1]) {
return [];
}
return match[1]
.split(",")
.map((field) => field.trim())
.filter(Boolean);
}
function toAmount(seedHex, fallback) {
const value = Number.parseInt(seedHex.slice(0, 8), 16);
if (!Number.isFinite(value)) {
@@ -38,7 +50,7 @@ async function runMockOcr({ ocrType, imageBase64 }) {
const invoiceCode = `MOCK${hash.slice(0, 8).toUpperCase()}`;
const invoiceNumber = hash.slice(8, 16).toUpperCase();
return {
const result = {
invoiceCode,
invoiceNumber,
invoiceDate: "2026-02-08",
@@ -53,6 +65,14 @@ async function runMockOcr({ ocrType, imageBase64 }) {
marker: markerText.slice(0, 128),
},
};
for (const field of parseMissingFields(markerText)) {
if (Object.prototype.hasOwnProperty.call(result, field)) {
result[field] = null;
}
}
return result;
}
module.exports = {
+42 -6
查看文件
@@ -46,12 +46,41 @@ function toNumberString(value) {
return num.toFixed(2);
}
function addNumberStrings(...values) {
const normalized = values.map((value) => toNumberString(value));
if (normalized.some((value) => value == null)) {
return null;
}
const total = normalized.reduce((sum, value) => sum + Number(value), 0);
return total.toFixed(2);
}
function subtractNumberStrings(minuend, subtrahend) {
const normalizedMinuend = toNumberString(minuend);
const normalizedSubtrahend = toNumberString(subtrahend);
if (normalizedMinuend == null || normalizedSubtrahend == null) {
return null;
}
return (Number(normalizedMinuend) - Number(normalizedSubtrahend)).toFixed(2);
}
function normalizeIdentifier(value) {
if (typeof value !== "string") {
return null;
}
const normalized = value.replace(/\s+/g, "").trim();
return normalized || null;
}
function normalizeString(value) {
if (typeof value !== "string") {
return null;
}
const normalized = value.trim();
const normalized = value.replace(/\s+/g, " ").trim();
return normalized || null;
}
@@ -76,13 +105,19 @@ function createTencentOcrClient(config) {
}
function normalizeVatInvoiceResult(result) {
const amount = toNumberString(result?.AmountWithoutTax);
const taxAmount = toNumberString(result?.TaxAmount);
const totalAmountFromSource = toNumberString(result?.TotalAmount ?? result?.AmountInFiguers);
const derivedAmount = amount ?? subtractNumberStrings(totalAmountFromSource, taxAmount);
const derivedTotalAmount = totalAmountFromSource ?? addNumberStrings(derivedAmount, taxAmount);
return {
invoiceCode: normalizeString(result?.InvoiceCode),
invoiceNumber: normalizeString(result?.InvoiceNum),
invoiceCode: normalizeIdentifier(result?.InvoiceCode),
invoiceNumber: normalizeIdentifier(result?.InvoiceNum),
invoiceDate: normalizeDate(result?.InvoiceDate),
amount: toNumberString(result?.AmountWithoutTax),
taxAmount: toNumberString(result?.TaxAmount),
totalAmount: toNumberString(result?.TotalAmount ?? result?.AmountInFiguers),
amount: derivedAmount,
taxAmount,
totalAmount: derivedTotalAmount,
sellerName: normalizeString(result?.SellerName),
buyerName: normalizeString(result?.BuyerName),
raw: result ?? {},
@@ -99,5 +134,6 @@ async function runTencentVatOcr(config, imageBase64) {
}
module.exports = {
normalizeVatInvoiceResult,
runTencentVatOcr,
};
+127
查看文件
@@ -0,0 +1,127 @@
const fs = require("node:fs/promises");
const os = require("node:os");
const path = require("node:path");
const { runOcrOnce } = require("../../../src/jobs/ocrJob");
function buildClaimedInvoice(overrides = {}) {
return {
id: "11111111-1111-4111-8111-111111111111",
file_sha256: "sha256-1",
ocr_type: "vat",
storage_path: "20260428/invoice.txt",
ocr_attempts: 0,
...overrides,
};
}
function buildPrisma(claimedInvoices) {
return {
$queryRaw: vi.fn(async () => claimedInvoices),
ocrResult: {
findUnique: vi.fn(async () => null),
create: vi.fn(async () => ({ id: "ocr-result-1" })),
},
invoicePool: {
findFirst: vi.fn(async () => null),
update: vi.fn(async () => ({ id: "invoice-1" })),
},
};
}
describe("ocrJob quality gate", () => {
let tempDir = "";
beforeEach(async () => {
vi.clearAllMocks();
tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "hyb-oa-r73-ocr-job-"));
await fs.mkdir(path.join(tempDir, "20260428"), { recursive: true });
});
afterEach(async () => {
await fs.rm(tempDir, { recursive: true, force: true });
});
it("marks invoice available when OCR returns all required VAT fields", async () => {
await fs.writeFile(
path.join(tempDir, "20260428/invoice.txt"),
"MOCK_SUCCESS_R73",
"utf8",
);
const prisma = buildPrisma([buildClaimedInvoice()]);
const logger = {
warn: vi.fn(),
error: vi.fn(),
};
const result = await runOcrOnce(
prisma,
{
storageDir: tempDir,
ocrBatchSize: 1,
ocrLockTtlSeconds: 300,
ocrMaxAttempts: 3,
ocrProvider: "mock",
},
logger,
"ocr-test-worker",
);
expect(result).toEqual({ claimed: 1, available: 1, duplicate: 0, failed: 0 });
expect(prisma.invoicePool.update).toHaveBeenCalledWith(
expect.objectContaining({
where: { id: "11111111-1111-4111-8111-111111111111" },
data: expect.objectContaining({
status: "available",
ocr_last_error: null,
}),
}),
);
});
it("marks invoice ocr_failed when required VAT fields are missing", async () => {
await fs.writeFile(
path.join(tempDir, "20260428/invoice.txt"),
"MOCK_OCR_MISSING:invoiceCode,sellerName",
"utf8",
);
const prisma = buildPrisma([buildClaimedInvoice()]);
const logger = {
warn: vi.fn(),
error: vi.fn(),
};
const result = await runOcrOnce(
prisma,
{
storageDir: tempDir,
ocrBatchSize: 1,
ocrLockTtlSeconds: 300,
ocrMaxAttempts: 3,
ocrProvider: "mock",
},
logger,
"ocr-test-worker",
);
expect(result).toEqual({ claimed: 1, available: 0, duplicate: 0, failed: 1 });
expect(prisma.invoicePool.update).toHaveBeenCalledWith(
expect.objectContaining({
where: { id: "11111111-1111-4111-8111-111111111111" },
data: expect.objectContaining({
status: "ocr_failed",
ocr_last_error: "missing_required_fields:invoiceCode,sellerName",
}),
}),
);
expect(logger.warn).toHaveBeenCalledWith(
expect.objectContaining({
invoiceId: "11111111-1111-4111-8111-111111111111",
qualityError: "missing_required_fields:invoiceCode,sellerName",
}),
"OCR invoice marked as failed for missing required fields",
);
});
});
@@ -0,0 +1,212 @@
const Fastify = require("fastify");
const { AppError } = require("../../../src/utils/AppError");
const invoicePoolRoutes = require("../../../src/routes/invoicePool");
const INVOICE_ID = "11111111-1111-4111-8111-111111111111";
const OWNER_ID = "22222222-2222-4222-8222-222222222222";
function buildInvoice(overrides = {}) {
return {
id: INVOICE_ID,
status: "ocr_failed",
archived_reason: null,
archived_at: null,
owner_employee_id: OWNER_ID,
uploaded_by_employee_id: OWNER_ID,
original_name: "invoice.png",
mime_type: "image/png",
file_size: 1024,
file_sha256: "sha-1",
remarks: null,
ocr_type: "vat",
ocr_provider: "mock",
ocr_attempts: 1,
ocr_last_requested_at: new Date("2026-04-28T11:00:00.000Z"),
ocr_last_error: "missing_required_fields:invoiceCode,sellerName",
ocr_completed_at: new Date("2026-04-28T11:00:05.000Z"),
invoice_code: null,
invoice_number: "251F7425",
invoice_date: new Date("2026-02-08T00:00:00.000Z"),
seller_name: null,
buyer_name: "Mock Buyer",
amount: "107.69",
tax_amount: "400.71",
total_amount: "282.53",
linked_reimbursement_id: null,
linked_item_id: null,
travel_order_id: null,
is_duplicate_override: false,
override_ref_form_number: null,
override_reason: null,
overridden_by: null,
created_at: new Date("2026-04-28T11:00:00.000Z"),
updated_at: new Date("2026-04-28T11:00:05.000Z"),
verification_records: [],
...overrides,
};
}
function buildQueryRawHandler(state) {
return vi.fn(async (strings) => {
const sql = Array.isArray(strings) ? strings.join("?") : String(strings);
if (sql.includes('FROM "invoice_pool"')) {
return state.invoice ? [{ id: state.invoice.id }] : [];
}
return [];
});
}
function buildPrisma(invoiceOverrides = {}) {
const state = {
invoice: buildInvoice(invoiceOverrides),
};
const prisma = {
invoicePool: {
findUnique: vi.fn(async () => state.invoice),
update: vi.fn(async ({ data }) => {
state.invoice = {
...state.invoice,
...data,
updated_at: new Date("2026-04-28T11:05:00.000Z"),
};
return state.invoice;
}),
},
auditLog: {
create: vi.fn().mockResolvedValue({ id: "audit-1" }),
},
$queryRaw: buildQueryRawHandler(state),
$transaction: vi.fn().mockImplementation(async (callback) => callback(prisma)),
};
return { prisma, state };
}
async function buildApp(invoiceOverrides = {}) {
const { prisma, state } = buildPrisma(invoiceOverrides);
const app = Fastify();
app.decorate("prisma", prisma);
app.decorate("config", {
storageDir: "/tmp",
appOrigin: "http://localhost:5173",
maxUploadSizeBytes: 10 * 1024 * 1024,
invoiceArchiveUnusedDays: 30,
ocrProvider: "mock",
});
app.decorate("authenticate", async (request) => {
if (!request.headers.authorization) {
throw new AppError({ code: "AUTH_001", message: "未登录", statusCode: 401 });
}
request.employee = {
id: OWNER_ID,
permissionCodes: ["invoice.upload"],
roleCodes: ["employee"],
};
});
app.decorate(
"hasPermission",
(employee, permissionCode) =>
(employee?.permissionCodes ?? []).includes("*") ||
(employee?.permissionCodes ?? []).includes(permissionCode),
);
app.decorate("requirePermission", (permissionCode) => async (request) => {
await app.authenticate(request);
if (!app.hasPermission(request.employee, permissionCode)) {
throw new AppError({ code: "PERM_001", message: "无权限", statusCode: 403 });
}
});
app.setErrorHandler((error, request, reply) => {
if (error instanceof AppError) {
return reply.status(error.statusCode).send({
requestId: request.id,
error: {
code: error.code,
message: error.message,
details: error.details ?? {},
},
});
}
return reply.status(500).send({
requestId: request.id,
error: { code: "SYS_001", message: "服务器内部错误", details: {} },
});
});
await app.register(invoicePoolRoutes, { prefix: "/api" });
return { app, prisma, state };
}
describe("PATCH /api/invoice-pool/:id", () => {
it("accepts valid invoiceDate and allows marking OCR-failed VAT invoice as available", async () => {
const { app, state } = await buildApp();
try {
const response = await app.inject({
method: "PATCH",
url: `/api/invoice-pool/${INVOICE_ID}`,
headers: { authorization: "Bearer test" },
payload: {
status: "available",
invoiceCode: "044003204111",
invoiceNumber: "251F7425",
invoiceDate: "2026-02-08",
sellerName: "北京悦海堂餐饮管理有限公司",
buyerName: "Mock Buyer",
amount: "107.69",
taxAmount: "400.71",
totalAmount: "282.53",
},
});
expect(response.statusCode).toBe(200);
expect(response.json().data).toMatchObject({
id: INVOICE_ID,
status: "available",
invoiceCode: "044003204111",
invoiceDate: "2026-02-08",
sellerName: "北京悦海堂餐饮管理有限公司",
});
expect(state.invoice.status).toBe("available");
expect(state.invoice.invoice_date.toISOString().slice(0, 10)).toBe("2026-02-08");
expect(state.invoice.ocr_last_error).toBeNull();
} finally {
await app.close();
}
});
it("returns missing OCR required fields when marking available without required data", async () => {
const { app } = await buildApp();
try {
const response = await app.inject({
method: "PATCH",
url: `/api/invoice-pool/${INVOICE_ID}`,
headers: { authorization: "Bearer test" },
payload: {
status: "available",
invoiceNumber: "251F7425",
invoiceDate: "2026-02-08",
buyerName: "Mock Buyer",
amount: "107.69",
taxAmount: "400.71",
totalAmount: "282.53",
},
});
expect(response.statusCode).toBe(409);
expect(response.json().error).toMatchObject({
code: "VALID_001",
details: {
ocr: "missing_required_fields",
missingFields: ["invoiceCode", "sellerName"],
},
});
} finally {
await app.close();
}
});
});
@@ -36,7 +36,7 @@ function buildMultipartRequest({ filename, contentType, content, fields = [] })
};
}
async function buildApp() {
async function buildApp({ ocrProvider = "off" } = {}) {
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "hyb-oa-r71-invoice-upload-"));
const invoicePoolCreate = vi.fn(async ({ data }) => ({
id: "11111111-1111-4111-8111-111111111111",
@@ -62,7 +62,7 @@ async function buildApp() {
storageDir: tempDir,
maxUploadSizeBytes: 5 * 1024 * 1024,
invoiceArchiveUnusedDays: 30,
ocrProvider: "off",
ocrProvider,
});
app.decorate("authenticate", async (request) => {
if (!request.headers.authorization) {
@@ -164,4 +164,46 @@ describe("invoicePool upload route", () => {
await fs.rm(tempDir, { recursive: true, force: true });
}
});
it("puts VAT uploads into pending_ocr when OCR provider is enabled", async () => {
const { app, prisma, tempDir } = await buildApp({ ocrProvider: "mock" });
try {
const requestPayload = buildMultipartRequest({
filename: "invoice.png",
contentType: "image/png",
content: "mock-image",
fields: [["ocr_type", "vat"]],
});
const response = await app.inject({
method: "POST",
url: "/api/invoice-pool/upload",
headers: {
authorization: "Bearer test",
...requestPayload.headers,
},
payload: requestPayload.payload,
});
expect(response.statusCode).toBe(200);
expect(response.json().data).toEqual(
expect.objectContaining({
invoicePoolId: "11111111-1111-4111-8111-111111111111",
status: "pending_ocr",
}),
);
expect(prisma.invoicePool.create).toHaveBeenCalledWith(
expect.objectContaining({
data: expect.objectContaining({
ocr_type: "vat",
status: "pending_ocr",
}),
}),
);
} finally {
await app.close();
await fs.rm(tempDir, { recursive: true, force: true });
}
});
});
@@ -0,0 +1,51 @@
const {
normalizeVatInvoiceResult,
} = require("../../../src/services/ocr/tencent");
describe("tencent VAT OCR normalization", () => {
it("normalizes identifiers, date and amounts from Tencent result", () => {
const normalized = normalizeVatInvoiceResult({
InvoiceCode: " 0440 0320 4111 ",
InvoiceNum: " 1234 5678 ",
InvoiceDate: "2026年4月28日",
AmountWithoutTax: "1,488.50",
TaxAmount: "194.50",
AmountInFiguers: "¥1683.00",
SellerName: " 北京悦海堂餐饮管理有限公司 ",
BuyerName: " 华印公司 ",
});
expect(normalized).toEqual(
expect.objectContaining({
invoiceCode: "044003204111",
invoiceNumber: "12345678",
invoiceDate: "2026-04-28",
amount: "1488.50",
taxAmount: "194.50",
totalAmount: "1683.00",
sellerName: "北京悦海堂餐饮管理有限公司",
buyerName: "华印公司",
}),
);
});
it("derives amount from total and tax when Tencent omits AmountWithoutTax", () => {
const result = normalizeVatInvoiceResult({
InvoiceCode: "044003204111",
InvoiceNum: "12345678",
InvoiceDate: "2026/04/28",
TaxAmount: 194.5,
TotalAmount: 1683,
SellerName: "北京悦海堂餐饮管理有限公司",
BuyerName: "华印公司",
});
expect(result).toEqual(
expect.objectContaining({
amount: "1488.50",
taxAmount: "194.50",
totalAmount: "1683.00",
}),
);
});
});