488 行
14 KiB
JavaScript
488 行
14 KiB
JavaScript
const cron = require("node-cron");
|
|
|
|
const { buildMissingRequiredOcrError, runOcrForInvoice } = require("../services/ocr");
|
|
const { publishInvoiceUpdated } = require("../services/events");
|
|
const { resolveInvoiceDedupBlocker } = require("../services/invoiceGovernance");
|
|
const { readFileBuffer } = require("../services/storage");
|
|
|
|
const OCR_DUPLICATE_ERROR = "duplicate_invoice_code_number";
|
|
const SUPPORTED_OCR_PROVIDERS = new Set(["tencent", "mock"]);
|
|
|
|
function getEffectiveOcrProvider(config) {
|
|
const provider = config?.ocrProvider;
|
|
if (!SUPPORTED_OCR_PROVIDERS.has(provider)) {
|
|
throw new Error(`Unsupported OCR provider: ${provider}`);
|
|
}
|
|
|
|
return provider;
|
|
}
|
|
|
|
function trimErrorMessage(error) {
|
|
const code = typeof error?.code === "string" ? error.code : null;
|
|
const transportMode = typeof error?.transportMode === "string" ? error.transportMode : null;
|
|
const baseMessage = error?.message ?? "ocr_failed";
|
|
const message =
|
|
transportMode || code
|
|
? `[${transportMode ?? "unknown"}][${code ?? "ocr_failed"}] ${baseMessage}`
|
|
: baseMessage;
|
|
if (message.length > 500) {
|
|
return message.slice(0, 500);
|
|
}
|
|
return message;
|
|
}
|
|
|
|
function toDateOnly(value) {
|
|
if (!value) {
|
|
return null;
|
|
}
|
|
|
|
return `${value}T00:00:00.000Z`;
|
|
}
|
|
|
|
function toMoney(value) {
|
|
if (value == null) {
|
|
return null;
|
|
}
|
|
|
|
const num = Number(value);
|
|
if (!Number.isFinite(num)) {
|
|
return null;
|
|
}
|
|
|
|
return num.toFixed(2);
|
|
}
|
|
|
|
async function claimPendingInvoices(prisma, config, workerId) {
|
|
const batchSize = Math.max(1, Number(config.ocrBatchSize || 10));
|
|
const lockTtlSeconds = Math.max(60, Number(config.ocrLockTtlSeconds || 300));
|
|
|
|
return prisma.$queryRaw`
|
|
WITH candidates AS (
|
|
SELECT id
|
|
FROM "invoice_pool"
|
|
WHERE status = 'pending_ocr'
|
|
AND (ocr_locked_at IS NULL OR ocr_locked_at < NOW() - make_interval(secs => ${lockTtlSeconds}))
|
|
ORDER BY created_at ASC
|
|
LIMIT ${batchSize}
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE "invoice_pool" AS i
|
|
SET ocr_locked_at = NOW(),
|
|
ocr_lock_owner = ${workerId}
|
|
FROM candidates
|
|
WHERE i.id = candidates.id
|
|
RETURNING i.id, i.owner_employee_id, i.uploaded_by_employee_id, i.file_sha256, i.ocr_provider, i.ocr_type, i.storage_path, i.mime_type, i.ocr_attempts
|
|
`;
|
|
}
|
|
|
|
async function claimPendingInvoiceById(prisma, config, workerId, invoiceId) {
|
|
const lockTtlSeconds = Math.max(60, Number(config.ocrLockTtlSeconds || 300));
|
|
|
|
return prisma.$queryRaw`
|
|
WITH candidates AS (
|
|
SELECT id
|
|
FROM "invoice_pool"
|
|
WHERE id = ${invoiceId}::uuid
|
|
AND status = 'pending_ocr'
|
|
AND (ocr_locked_at IS NULL OR ocr_locked_at < NOW() - make_interval(secs => ${lockTtlSeconds}))
|
|
ORDER BY created_at ASC
|
|
LIMIT 1
|
|
FOR UPDATE SKIP LOCKED
|
|
)
|
|
UPDATE "invoice_pool" AS i
|
|
SET ocr_locked_at = NOW(),
|
|
ocr_lock_owner = ${workerId}
|
|
FROM candidates
|
|
WHERE i.id = candidates.id
|
|
RETURNING i.id, i.owner_employee_id, i.uploaded_by_employee_id, i.file_sha256, i.ocr_provider, i.ocr_type, i.storage_path, i.mime_type, i.ocr_attempts
|
|
`;
|
|
}
|
|
|
|
async function getOrCreateOcrResult(prisma, config, claimedInvoice, imageBase64) {
|
|
const ocrProvider = getEffectiveOcrProvider(config);
|
|
const where = {
|
|
file_sha256_ocr_provider_ocr_type: {
|
|
file_sha256: claimedInvoice.file_sha256,
|
|
ocr_provider: ocrProvider,
|
|
ocr_type: claimedInvoice.ocr_type,
|
|
},
|
|
};
|
|
|
|
const cached = await prisma.ocrResult.findUnique({
|
|
where,
|
|
select: {
|
|
id: true,
|
|
result_json: true,
|
|
},
|
|
});
|
|
|
|
if (cached) {
|
|
return {
|
|
ocrResultId: cached.id,
|
|
normalized: cached.result_json,
|
|
};
|
|
}
|
|
|
|
const normalized = await runOcrForInvoice({
|
|
config,
|
|
ocrType: claimedInvoice.ocr_type,
|
|
imageBase64,
|
|
sourceMimeType: claimedInvoice.mime_type,
|
|
});
|
|
|
|
try {
|
|
const created = await prisma.ocrResult.create({
|
|
data: {
|
|
file_sha256: claimedInvoice.file_sha256,
|
|
ocr_provider: ocrProvider,
|
|
ocr_type: claimedInvoice.ocr_type,
|
|
result_json: normalized,
|
|
},
|
|
select: {
|
|
id: true,
|
|
},
|
|
});
|
|
|
|
return {
|
|
ocrResultId: created.id,
|
|
normalized,
|
|
};
|
|
} catch (error) {
|
|
if (error?.code !== "P2002") {
|
|
throw error;
|
|
}
|
|
|
|
const raced = await prisma.ocrResult.findUnique({
|
|
where,
|
|
select: {
|
|
id: true,
|
|
result_json: true,
|
|
},
|
|
});
|
|
|
|
if (!raced) {
|
|
throw error;
|
|
}
|
|
|
|
return {
|
|
ocrResultId: raced.id,
|
|
normalized: raced.result_json,
|
|
};
|
|
}
|
|
}
|
|
|
|
async function markInvoiceAsDuplicate(prisma, config, invoiceId, normalized, ocrResultId) {
|
|
const now = new Date();
|
|
await prisma.invoicePool.update({
|
|
where: { id: invoiceId },
|
|
data: {
|
|
status: "ocr_failed",
|
|
ocr_provider: getEffectiveOcrProvider(config),
|
|
ocr_result_id: ocrResultId,
|
|
invoice_code: normalized.invoiceCode ?? null,
|
|
invoice_number: normalized.invoiceNumber ?? null,
|
|
invoice_date: normalized.invoiceDate ? new Date(toDateOnly(normalized.invoiceDate)) : null,
|
|
amount: toMoney(normalized.amount),
|
|
tax_amount: toMoney(normalized.taxAmount),
|
|
total_amount: toMoney(normalized.totalAmount),
|
|
seller_name: normalized.sellerName ?? null,
|
|
buyer_name: normalized.buyerName ?? null,
|
|
ocr_last_error: OCR_DUPLICATE_ERROR,
|
|
ocr_completed_at: now,
|
|
ocr_attempts: { increment: 1 },
|
|
ocr_locked_at: null,
|
|
ocr_lock_owner: null,
|
|
},
|
|
});
|
|
}
|
|
|
|
async function markInvoiceAsAvailable(prisma, config, invoiceId, normalized, ocrResultId) {
|
|
const now = new Date();
|
|
await prisma.invoicePool.update({
|
|
where: { id: invoiceId },
|
|
data: {
|
|
status: "available",
|
|
ocr_provider: getEffectiveOcrProvider(config),
|
|
ocr_result_id: ocrResultId,
|
|
invoice_code: normalized.invoiceCode ?? null,
|
|
invoice_number: normalized.invoiceNumber ?? null,
|
|
invoice_date: normalized.invoiceDate ? new Date(toDateOnly(normalized.invoiceDate)) : null,
|
|
amount: toMoney(normalized.amount),
|
|
tax_amount: toMoney(normalized.taxAmount),
|
|
total_amount: toMoney(normalized.totalAmount),
|
|
seller_name: normalized.sellerName ?? null,
|
|
buyer_name: normalized.buyerName ?? null,
|
|
ocr_last_error: null,
|
|
ocr_completed_at: now,
|
|
ocr_attempts: { increment: 1 },
|
|
ocr_locked_at: null,
|
|
ocr_lock_owner: null,
|
|
},
|
|
});
|
|
}
|
|
|
|
async function markInvoiceAsQualityFailed(prisma, config, invoiceId, normalized, ocrResultId, errorMessage) {
|
|
const now = new Date();
|
|
await prisma.invoicePool.update({
|
|
where: { id: invoiceId },
|
|
data: {
|
|
status: "ocr_failed",
|
|
ocr_provider: getEffectiveOcrProvider(config),
|
|
ocr_result_id: ocrResultId,
|
|
invoice_code: normalized.invoiceCode ?? null,
|
|
invoice_number: normalized.invoiceNumber ?? null,
|
|
invoice_date: normalized.invoiceDate ? new Date(toDateOnly(normalized.invoiceDate)) : null,
|
|
amount: toMoney(normalized.amount),
|
|
tax_amount: toMoney(normalized.taxAmount),
|
|
total_amount: toMoney(normalized.totalAmount),
|
|
seller_name: normalized.sellerName ?? null,
|
|
buyer_name: normalized.buyerName ?? null,
|
|
ocr_last_error: errorMessage,
|
|
ocr_completed_at: now,
|
|
ocr_attempts: { increment: 1 },
|
|
ocr_locked_at: null,
|
|
ocr_lock_owner: null,
|
|
},
|
|
});
|
|
}
|
|
|
|
async function markInvoiceAsFailed(prisma, claimedInvoice, errorMessage, config) {
|
|
const now = new Date();
|
|
const nextAttempts = Number(claimedInvoice.ocr_attempts || 0) + 1;
|
|
const isTerminal = nextAttempts >= config.ocrMaxAttempts;
|
|
const nextStatus = isTerminal ? "ocr_failed" : "pending_ocr";
|
|
|
|
await prisma.invoicePool.update({
|
|
where: { id: claimedInvoice.id },
|
|
data: {
|
|
status: nextStatus,
|
|
ocr_provider: getEffectiveOcrProvider(config),
|
|
ocr_attempts: nextAttempts,
|
|
ocr_last_error: errorMessage,
|
|
ocr_completed_at: isTerminal ? now : null,
|
|
ocr_locked_at: null,
|
|
ocr_lock_owner: null,
|
|
},
|
|
});
|
|
|
|
return nextStatus;
|
|
}
|
|
|
|
async function processClaimedInvoice(prisma, config, claimedInvoice, workerId, logger) {
|
|
try {
|
|
const fileBuffer = await readFileBuffer({
|
|
config,
|
|
storagePath: claimedInvoice.storage_path,
|
|
});
|
|
const imageBase64 = fileBuffer.toString("base64");
|
|
|
|
const { ocrResultId, normalized } = await getOrCreateOcrResult(
|
|
prisma,
|
|
config,
|
|
claimedInvoice,
|
|
imageBase64,
|
|
);
|
|
|
|
const qualityError = buildMissingRequiredOcrError(claimedInvoice.ocr_type, normalized);
|
|
if (qualityError) {
|
|
await markInvoiceAsQualityFailed(
|
|
prisma,
|
|
config,
|
|
claimedInvoice.id,
|
|
normalized,
|
|
ocrResultId,
|
|
qualityError,
|
|
);
|
|
logger.warn(
|
|
{
|
|
invoiceId: claimedInvoice.id,
|
|
ocrType: claimedInvoice.ocr_type,
|
|
mimeType: claimedInvoice.mime_type,
|
|
transportMode: normalized?.transportMode ?? normalized?.raw?.transportMode ?? null,
|
|
qualityError,
|
|
},
|
|
"OCR invoice marked as failed for missing required fields",
|
|
);
|
|
return { outcome: "failed", status: "ocr_failed" };
|
|
}
|
|
|
|
const duplicate = await resolveInvoiceDedupBlocker(
|
|
prisma,
|
|
{
|
|
excludeInvoiceId: claimedInvoice.id,
|
|
fileSha256: claimedInvoice.file_sha256,
|
|
invoiceCode: normalized.invoiceCode,
|
|
invoiceNumber: normalized.invoiceNumber,
|
|
ownerEmployeeId: claimedInvoice.owner_employee_id,
|
|
},
|
|
);
|
|
|
|
if (duplicate) {
|
|
await markInvoiceAsDuplicate(prisma, config, claimedInvoice.id, normalized, ocrResultId);
|
|
logger.warn(
|
|
{
|
|
invoiceId: claimedInvoice.id,
|
|
duplicateInvoiceId: duplicate.conflict.id,
|
|
duplicateReason: duplicate.reason,
|
|
mimeType: claimedInvoice.mime_type,
|
|
transportMode: normalized?.transportMode ?? normalized?.raw?.transportMode ?? null,
|
|
},
|
|
"OCR invoice marked as duplicate",
|
|
);
|
|
return { outcome: "duplicate", status: "ocr_failed" };
|
|
}
|
|
|
|
await markInvoiceAsAvailable(prisma, config, claimedInvoice.id, normalized, ocrResultId);
|
|
return { outcome: "available", status: "available" };
|
|
} catch (error) {
|
|
const message = trimErrorMessage(error);
|
|
const status = await markInvoiceAsFailed(prisma, claimedInvoice, message, config);
|
|
logger.error(
|
|
{
|
|
err: error,
|
|
invoiceId: claimedInvoice.id,
|
|
mimeType: claimedInvoice.mime_type,
|
|
workerId,
|
|
transportMode: error?.transportMode ?? null,
|
|
},
|
|
"OCR processing failed",
|
|
);
|
|
return { outcome: "failed", status };
|
|
}
|
|
}
|
|
|
|
async function processClaimedInvoices(prisma, config, logger, workerId, claimed, options = {}) {
|
|
if (!claimed.length) {
|
|
return { claimed: 0, available: 0, duplicate: 0, failed: 0 };
|
|
}
|
|
|
|
const stats = { claimed: claimed.length, available: 0, duplicate: 0, failed: 0 };
|
|
|
|
for (const item of claimed) {
|
|
const result = await processClaimedInvoice(prisma, config, item, workerId, logger);
|
|
if (result?.outcome === "available") {
|
|
stats.available += 1;
|
|
} else if (result?.outcome === "duplicate") {
|
|
stats.duplicate += 1;
|
|
} else if (result?.outcome === "failed") {
|
|
stats.failed += 1;
|
|
}
|
|
|
|
if (typeof options.onInvoiceProcessed === "function") {
|
|
await options.onInvoiceProcessed({
|
|
invoiceId: item.id,
|
|
status: result?.status ?? item.status ?? null,
|
|
outcome: result?.outcome ?? null,
|
|
ownerEmployeeId: item.owner_employee_id ?? null,
|
|
uploadedByEmployeeId: item.uploaded_by_employee_id ?? null,
|
|
});
|
|
}
|
|
}
|
|
|
|
return stats;
|
|
}
|
|
|
|
async function runOcrOnce(prisma, config, logger, workerId = `ocr:${process.pid}`, options = {}) {
|
|
const claimed = await claimPendingInvoices(prisma, config, workerId);
|
|
return processClaimedInvoices(prisma, config, logger, workerId, claimed, options);
|
|
}
|
|
|
|
async function runOcrForInvoiceId(prisma, config, logger, invoiceId, workerId = `ocr:${process.pid}`, options = {}) {
|
|
const claimed = await claimPendingInvoiceById(prisma, config, workerId, invoiceId);
|
|
return processClaimedInvoices(prisma, config, logger, workerId, claimed, options);
|
|
}
|
|
|
|
function startOcrJob(fastify) {
|
|
if (!["tencent", "mock"].includes(fastify.config.ocrProvider)) {
|
|
fastify.log.info({ provider: fastify.config.ocrProvider }, "OCR job disabled");
|
|
return null;
|
|
}
|
|
|
|
if (!fastify.config.ocrCronEnabled) {
|
|
fastify.log.info({ provider: fastify.config.ocrProvider }, "OCR cron disabled");
|
|
return null;
|
|
}
|
|
|
|
let running = false;
|
|
const workerId = `ocr-worker:${process.pid}`;
|
|
const onInvoiceProcessed = (event) =>
|
|
publishInvoiceUpdated(fastify, {
|
|
invoiceId: event.invoiceId,
|
|
status: event.status,
|
|
audienceEmployeeIds: [event.ownerEmployeeId, event.uploadedByEmployeeId],
|
|
});
|
|
|
|
const task = cron.schedule(fastify.config.ocrPollCron, async () => {
|
|
if (running) {
|
|
return;
|
|
}
|
|
|
|
running = true;
|
|
try {
|
|
const result = await runOcrOnce(fastify.prisma, fastify.config, fastify.log, workerId, {
|
|
onInvoiceProcessed,
|
|
});
|
|
fastify.metrics?.jobRunsTotal?.inc?.({
|
|
job: "ocr.tick",
|
|
driver: "cron",
|
|
outcome: "success",
|
|
});
|
|
if (result.available > 0) {
|
|
fastify.metrics?.jobProcessedTotal?.inc?.(
|
|
{ job: "ocr.tick", driver: "cron", kind: "available" },
|
|
result.available,
|
|
);
|
|
}
|
|
if (result.duplicate > 0) {
|
|
fastify.metrics?.jobProcessedTotal?.inc?.(
|
|
{ job: "ocr.tick", driver: "cron", kind: "duplicate" },
|
|
result.duplicate,
|
|
);
|
|
}
|
|
if (result.failed > 0) {
|
|
fastify.metrics?.jobProcessedTotal?.inc?.(
|
|
{ job: "ocr.tick", driver: "cron", kind: "failed" },
|
|
result.failed,
|
|
);
|
|
}
|
|
if (result.claimed > 0) {
|
|
fastify.log.info(
|
|
{
|
|
workerId,
|
|
...result,
|
|
},
|
|
"OCR job tick processed",
|
|
);
|
|
}
|
|
} catch (error) {
|
|
fastify.log.error({ err: error }, "OCR job tick failed");
|
|
fastify.metrics?.jobRunsTotal?.inc?.({
|
|
job: "ocr.tick",
|
|
driver: "cron",
|
|
outcome: "failure",
|
|
});
|
|
} finally {
|
|
running = false;
|
|
}
|
|
});
|
|
|
|
fastify.addHook("onClose", async () => {
|
|
task.stop();
|
|
task.destroy();
|
|
});
|
|
|
|
fastify.log.info({ cron: fastify.config.ocrPollCron }, "OCR job scheduled");
|
|
return task;
|
|
}
|
|
|
|
module.exports = {
|
|
OCR_DUPLICATE_ERROR,
|
|
claimPendingInvoiceById,
|
|
claimPendingInvoices,
|
|
getEffectiveOcrProvider,
|
|
markInvoiceAsQualityFailed,
|
|
runOcrForInvoiceId,
|
|
runOcrOnce,
|
|
startOcrJob,
|
|
};
|