实现报销材料合并预览与差旅概要（07-27-yang-jun-reimbursement-readiness）
这个提交包含在：
@@ -0,0 +1,100 @@
|
||||
const crypto = require("node:crypto");
|
||||
const { PDFDocument } = require("pdf-lib");
|
||||
const sharp = require("sharp");
|
||||
|
||||
const FIXED_PDF_DATE = new Date("2000-01-01T00:00:00.000Z");
|
||||
const PDF_CREATOR = "hyb-oa";
|
||||
const PDF_PRODUCER = "hyb-oa reimbursement material merger";
|
||||
|
||||
function materialCollectionHash(materials, mergerVersion) {
|
||||
const hash = crypto.createHash("sha256");
|
||||
hash.update(String(mergerVersion));
|
||||
for (const material of materials) {
|
||||
hash.update("\0");
|
||||
hash.update([
|
||||
material.materialType,
|
||||
material.id,
|
||||
material.fileSha256 ?? "",
|
||||
material.mimeType,
|
||||
String(material.fileSize),
|
||||
].join("\u001f"));
|
||||
}
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
async function normalizeImageForPdf(source, limits) {
|
||||
const image = sharp(source, { failOn: "error", limitInputPixels: limits.maxImagePixels }).rotate();
|
||||
const metadata = await image.metadata();
|
||||
if (!metadata.width || !metadata.height || metadata.width * metadata.height > limits.maxImagePixels) {
|
||||
throw new Error("material_preview_image_invalid");
|
||||
}
|
||||
const normalized = await image
|
||||
.flatten({ background: "#ffffff" })
|
||||
.jpeg({ quality: 90, chromaSubsampling: "4:4:4", mozjpeg: false })
|
||||
.toBuffer();
|
||||
return { buffer: normalized, width: metadata.width, height: metadata.height };
|
||||
}
|
||||
|
||||
function fitPage(width, height) {
|
||||
const maxSide = 842;
|
||||
const scale = Math.min(1, maxSide / Math.max(width, height));
|
||||
return { width: Math.max(1, width * scale), height: Math.max(1, height * scale) };
|
||||
}
|
||||
|
||||
async function mergeMaterialsToPdf(materials, limits) {
|
||||
const output = await PDFDocument.create();
|
||||
output.setCreationDate(FIXED_PDF_DATE);
|
||||
output.setModificationDate(FIXED_PDF_DATE);
|
||||
output.setCreator(PDF_CREATOR);
|
||||
output.setProducer(PDF_PRODUCER);
|
||||
output.setTitle("Reimbursement Material Preview");
|
||||
output.setSubject("Derived preview; originals retained separately");
|
||||
|
||||
let pageCount = 0;
|
||||
for (const material of materials) {
|
||||
const source = Buffer.from(material.source);
|
||||
if (material.mimeType === "application/pdf") {
|
||||
let input;
|
||||
try {
|
||||
input = await PDFDocument.load(source, { updateMetadata: false });
|
||||
} catch {
|
||||
throw new Error("material_preview_pdf_invalid");
|
||||
}
|
||||
const indices = input.getPageIndices();
|
||||
if (!indices.length || pageCount + indices.length > limits.maxPages) {
|
||||
throw new Error(indices.length ? "material_preview_page_limit" : "material_preview_pdf_empty");
|
||||
}
|
||||
const pages = await output.copyPages(input, indices);
|
||||
pages.forEach((page) => output.addPage(page));
|
||||
pageCount += pages.length;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pageCount + 1 > limits.maxPages) throw new Error("material_preview_page_limit");
|
||||
let normalized;
|
||||
try {
|
||||
normalized = await normalizeImageForPdf(source, limits);
|
||||
} catch (error) {
|
||||
if (error?.message === "material_preview_image_invalid") throw error;
|
||||
throw new Error("material_preview_image_invalid");
|
||||
}
|
||||
const embedded = await output.embedJpg(normalized.buffer);
|
||||
const size = fitPage(normalized.width, normalized.height);
|
||||
const page = output.addPage([size.width, size.height]);
|
||||
page.drawImage(embedded, { x: 0, y: 0, width: size.width, height: size.height });
|
||||
pageCount += 1;
|
||||
}
|
||||
|
||||
if (!pageCount) throw new Error("material_preview_empty");
|
||||
const bytes = await output.save({
|
||||
useObjectStreams: true,
|
||||
addDefaultPage: false,
|
||||
objectsPerTick: 50,
|
||||
updateFieldAppearances: false,
|
||||
});
|
||||
const buffer = Buffer.from(bytes);
|
||||
if (buffer.length > limits.maxOutputBytes) throw new Error("material_preview_output_size_limit");
|
||||
return { buffer, pageCount };
|
||||
}
|
||||
|
||||
module.exports = { materialCollectionHash, mergeMaterialsToPdf };
|
||||
@@ -1,23 +1,17 @@
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs/promises");
|
||||
const os = require("node:os");
|
||||
const path = require("node:path");
|
||||
const { promisify } = require("node:util");
|
||||
const { execFile } = require("node:child_process");
|
||||
const sharp = require("sharp");
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const { Worker } = require("node:worker_threads");
|
||||
const { materialCollectionHash } = require("./reimbursementMaterialPdf");
|
||||
|
||||
const MATERIAL_PREVIEW_MERGER_VERSION = "pdf-v1";
|
||||
const MATERIAL_PREVIEW_LIMITS = Object.freeze({
|
||||
maxMaterials: 20,
|
||||
maxSourceBytes: 50 * 1024 * 1024,
|
||||
maxPages: 40,
|
||||
maxPageWidth: 1800,
|
||||
maxPagePixels: 40_000_000,
|
||||
maxOutputBytes: 40 * 1024 * 1024,
|
||||
maxCacheBytes: 80 * 1024 * 1024,
|
||||
rasterDpi: 144,
|
||||
rasterTimeoutMs: 20_000,
|
||||
maxImagePixels: 40_000_000,
|
||||
maxOutputBytes: 50 * 1024 * 1024,
|
||||
maxCacheBytes: 100 * 1024 * 1024,
|
||||
mergeTimeoutMs: 20_000,
|
||||
});
|
||||
|
||||
class MaterialPreviewError extends Error {
|
||||
@@ -28,183 +22,137 @@ class MaterialPreviewError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
function pageNumberFromName(fileName) {
|
||||
const matched = fileName.match(/-(\d+)\.png$/i);
|
||||
return matched ? Number(matched[1]) : Number.MAX_SAFE_INTEGER;
|
||||
}
|
||||
|
||||
async function normalizePreviewImage(buffer, limits = MATERIAL_PREVIEW_LIMITS) {
|
||||
try {
|
||||
return await sharp(buffer, { failOn: "error", limitInputPixels: limits.maxPagePixels })
|
||||
.rotate()
|
||||
.resize({ width: limits.maxPageWidth, withoutEnlargement: true })
|
||||
.png({ compressionLevel: 8, adaptiveFiltering: true })
|
||||
.toBuffer();
|
||||
} catch {
|
||||
throw new MaterialPreviewError("material_preview_image_invalid");
|
||||
}
|
||||
}
|
||||
|
||||
async function rasterizePdfWithPoppler(buffer, limits = MATERIAL_PREVIEW_LIMITS) {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "hyb-oa-material-preview-"));
|
||||
const sourcePath = path.join(tempDir, "source.pdf");
|
||||
const outputPrefix = path.join(tempDir, "page");
|
||||
|
||||
try {
|
||||
await fs.writeFile(sourcePath, buffer, { flag: "wx" });
|
||||
try {
|
||||
await execFileAsync(
|
||||
"pdftoppm",
|
||||
[
|
||||
"-png",
|
||||
"-r",
|
||||
String(limits.rasterDpi),
|
||||
"-f",
|
||||
"1",
|
||||
"-l",
|
||||
String(limits.maxPages + 1),
|
||||
"-scale-to-x",
|
||||
String(limits.maxPageWidth),
|
||||
"-scale-to-y",
|
||||
"-1",
|
||||
sourcePath,
|
||||
outputPrefix,
|
||||
],
|
||||
{
|
||||
timeout: limits.rasterTimeoutMs,
|
||||
maxBuffer: 1024 * 1024,
|
||||
windowsHide: true,
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") {
|
||||
throw new MaterialPreviewError("material_preview_rasterizer_unavailable");
|
||||
function runMergeWorker(materials, limits) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const worker = new Worker(path.join(__dirname, "reimbursementMaterialPreviewWorker.js"), {
|
||||
workerData: { materials, limits },
|
||||
});
|
||||
const timeout = setTimeout(() => {
|
||||
worker.terminate().catch(() => {});
|
||||
reject(new MaterialPreviewError("material_preview_merge_timeout"));
|
||||
}, limits.mergeTimeoutMs);
|
||||
timeout.unref?.();
|
||||
worker.once("message", (message) => {
|
||||
clearTimeout(timeout);
|
||||
if (!message?.ok) {
|
||||
reject(new MaterialPreviewError(message?.code || "material_preview_merge_failed"));
|
||||
return;
|
||||
}
|
||||
if (error?.killed || error?.signal) {
|
||||
throw new MaterialPreviewError("material_preview_raster_timeout");
|
||||
resolve({ buffer: Buffer.from(message.bytes), pageCount: message.pageCount });
|
||||
});
|
||||
worker.once("error", () => {
|
||||
clearTimeout(timeout);
|
||||
reject(new MaterialPreviewError("material_preview_merge_failed"));
|
||||
});
|
||||
worker.once("exit", (code) => {
|
||||
if (code !== 0) {
|
||||
clearTimeout(timeout);
|
||||
reject(new MaterialPreviewError("material_preview_merge_failed"));
|
||||
}
|
||||
throw new MaterialPreviewError("material_preview_pdf_invalid");
|
||||
}
|
||||
|
||||
const fileNames = (await fs.readdir(tempDir))
|
||||
.filter((fileName) => /^page-\d+\.png$/i.test(fileName))
|
||||
.sort((left, right) => pageNumberFromName(left) - pageNumberFromName(right));
|
||||
if (!fileNames.length) {
|
||||
throw new MaterialPreviewError("material_preview_pdf_empty");
|
||||
}
|
||||
|
||||
const pages = [];
|
||||
for (const fileName of fileNames) {
|
||||
pages.push(await normalizePreviewImage(await fs.readFile(path.join(tempDir, fileName)), limits));
|
||||
}
|
||||
return pages;
|
||||
} finally {
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function createReimbursementMaterialPreviewStore(options = {}) {
|
||||
const limits = { ...MATERIAL_PREVIEW_LIMITS, ...(options.limits ?? {}) };
|
||||
const rasterizePdf = options.rasterizePdf ?? ((buffer) => rasterizePdfWithPoppler(buffer, limits));
|
||||
const normalizeImage = options.normalizeImage ?? ((buffer) => normalizePreviewImage(buffer, limits));
|
||||
const mergePdf = options.mergePdf ?? ((materials) => runMergeWorker(materials, limits));
|
||||
const now = options.now ?? (() => Date.now());
|
||||
const randomUUID = options.randomUUID ?? (() => crypto.randomUUID());
|
||||
const bundles = new Map();
|
||||
const grants = new Map();
|
||||
const artifacts = new Map();
|
||||
const pendingArtifacts = new Map();
|
||||
let cachedBytes = 0;
|
||||
|
||||
function deleteBundle(bundleId) {
|
||||
const bundle = bundles.get(bundleId);
|
||||
if (!bundle) return;
|
||||
cachedBytes -= bundle.outputBytes;
|
||||
bundles.delete(bundleId);
|
||||
const grant = grants.get(bundleId);
|
||||
if (!grant) return;
|
||||
grants.delete(bundleId);
|
||||
const stillUsed = [...grants.values()].some((candidate) => candidate.cacheKey === grant.cacheKey);
|
||||
if (!stillUsed) {
|
||||
const artifact = artifacts.get(grant.cacheKey);
|
||||
if (artifact) cachedBytes -= artifact.buffer.length;
|
||||
artifacts.delete(grant.cacheKey);
|
||||
}
|
||||
}
|
||||
|
||||
function pruneExpired() {
|
||||
const current = now();
|
||||
for (const [bundleId, bundle] of bundles) {
|
||||
if (bundle.expiresAt <= current) deleteBundle(bundleId);
|
||||
for (const [bundleId, grant] of grants) {
|
||||
if (grant.expiresAt <= current) deleteBundle(bundleId);
|
||||
}
|
||||
}
|
||||
|
||||
async function createBundle({ actorId, reimbursementId, materials, expiresAt, readSource }) {
|
||||
pruneExpired();
|
||||
if (!Array.isArray(materials) || materials.length === 0) {
|
||||
throw new MaterialPreviewError("material_preview_empty");
|
||||
}
|
||||
if (materials.length > limits.maxMaterials) {
|
||||
throw new MaterialPreviewError("material_preview_material_limit");
|
||||
}
|
||||
const sourceBytes = materials.reduce((sum, material) => sum + Number(material.fileSize ?? 0), 0);
|
||||
if (!Number.isSafeInteger(sourceBytes) || sourceBytes <= 0 || sourceBytes > limits.maxSourceBytes) {
|
||||
if (!Array.isArray(materials) || !materials.length) throw new MaterialPreviewError("material_preview_empty");
|
||||
if (materials.length > limits.maxMaterials) throw new MaterialPreviewError("material_preview_material_limit");
|
||||
const declaredBytes = materials.reduce((sum, material) => sum + Number(material.fileSize ?? 0), 0);
|
||||
if (!Number.isSafeInteger(declaredBytes) || declaredBytes <= 0 || declaredBytes > limits.maxSourceBytes) {
|
||||
throw new MaterialPreviewError("material_preview_source_size_limit");
|
||||
}
|
||||
|
||||
const pages = [];
|
||||
let outputBytes = 0;
|
||||
let actualSourceBytes = 0;
|
||||
for (const material of materials) {
|
||||
const source = await readSource(material);
|
||||
if (!Buffer.isBuffer(source) || source.length === 0) {
|
||||
throw new MaterialPreviewError("material_preview_source_missing");
|
||||
const collectionHash = materialCollectionHash(materials, MATERIAL_PREVIEW_MERGER_VERSION);
|
||||
const cacheKey = `${reimbursementId}:${collectionHash}:${MATERIAL_PREVIEW_MERGER_VERSION}`;
|
||||
let artifact = artifacts.get(cacheKey);
|
||||
if (!artifact) {
|
||||
let pending = pendingArtifacts.get(cacheKey);
|
||||
if (!pending) pending = (async () => {
|
||||
const loaded = [];
|
||||
let actualBytes = 0;
|
||||
for (const material of materials) {
|
||||
const source = await readSource(material);
|
||||
if (!Buffer.isBuffer(source) || !source.length) throw new MaterialPreviewError("material_preview_source_missing");
|
||||
actualBytes += source.length;
|
||||
if (actualBytes > limits.maxSourceBytes) throw new MaterialPreviewError("material_preview_source_size_limit");
|
||||
loaded.push({ ...material, source });
|
||||
}
|
||||
actualSourceBytes += source.length;
|
||||
if (actualSourceBytes > limits.maxSourceBytes) {
|
||||
throw new MaterialPreviewError("material_preview_source_size_limit");
|
||||
let generated;
|
||||
try {
|
||||
generated = await mergePdf(loaded);
|
||||
} catch (error) {
|
||||
if (error instanceof MaterialPreviewError) throw error;
|
||||
throw new MaterialPreviewError(error?.message || "material_preview_merge_failed");
|
||||
}
|
||||
const renderedPages = material.mimeType === "application/pdf"
|
||||
? await rasterizePdf(source)
|
||||
: [await normalizeImage(source)];
|
||||
for (let index = 0; index < renderedPages.length; index += 1) {
|
||||
if (pages.length >= limits.maxPages) {
|
||||
throw new MaterialPreviewError("material_preview_page_limit");
|
||||
if (!Buffer.isBuffer(generated?.buffer) || !generated.buffer.length) {
|
||||
throw new MaterialPreviewError("material_preview_merge_failed");
|
||||
}
|
||||
if (generated.buffer.length > limits.maxOutputBytes) throw new MaterialPreviewError("material_preview_output_size_limit");
|
||||
if (cachedBytes + generated.buffer.length > limits.maxCacheBytes) {
|
||||
pruneExpired();
|
||||
if (cachedBytes + generated.buffer.length > limits.maxCacheBytes) {
|
||||
throw new MaterialPreviewError("material_preview_cache_limit");
|
||||
}
|
||||
const image = renderedPages[index];
|
||||
outputBytes += image.length;
|
||||
if (outputBytes > limits.maxOutputBytes) {
|
||||
throw new MaterialPreviewError("material_preview_output_size_limit");
|
||||
}
|
||||
pages.push({
|
||||
buffer: image,
|
||||
materialId: material.id,
|
||||
materialType: material.materialType,
|
||||
materialIndex: material.materialIndex,
|
||||
pageNumber: index + 1,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (cachedBytes + outputBytes > limits.maxCacheBytes) {
|
||||
pruneExpired();
|
||||
if (cachedBytes + outputBytes > limits.maxCacheBytes) {
|
||||
throw new MaterialPreviewError("material_preview_cache_limit");
|
||||
}
|
||||
artifacts.set(cacheKey, generated);
|
||||
cachedBytes += generated.buffer.length;
|
||||
return generated;
|
||||
})().finally(() => pendingArtifacts.delete(cacheKey));
|
||||
pendingArtifacts.set(cacheKey, pending);
|
||||
artifact = await pending;
|
||||
}
|
||||
|
||||
const bundleId = randomUUID();
|
||||
const bundle = { actorId, reimbursementId, expiresAt, outputBytes, pages };
|
||||
bundles.set(bundleId, bundle);
|
||||
cachedBytes += outputBytes;
|
||||
const bundle = { actorId, reimbursementId, expiresAt, cacheKey, collectionHash, pageCount: artifact.pageCount };
|
||||
grants.set(bundleId, bundle);
|
||||
const timer = setTimeout(() => deleteBundle(bundleId), Math.max(1, expiresAt - now()));
|
||||
timer.unref?.();
|
||||
return { bundleId, bundle };
|
||||
return { bundleId, bundle, fileSize: artifact.buffer.length };
|
||||
}
|
||||
|
||||
function getPage({ bundleId, pageIndex }) {
|
||||
function getBundle({ bundleId }) {
|
||||
pruneExpired();
|
||||
const bundle = bundles.get(bundleId);
|
||||
if (!bundle || !Number.isInteger(pageIndex) || pageIndex < 0 || pageIndex >= bundle.pages.length) {
|
||||
return null;
|
||||
}
|
||||
return { bundle, page: bundle.pages[pageIndex] };
|
||||
const bundle = grants.get(bundleId);
|
||||
const artifact = bundle ? artifacts.get(bundle.cacheKey) : null;
|
||||
return bundle && artifact ? { bundle, artifact } : null;
|
||||
}
|
||||
|
||||
return { createBundle, getPage, deleteBundle };
|
||||
return { createBundle, getBundle, deleteBundle };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
MATERIAL_PREVIEW_LIMITS,
|
||||
MATERIAL_PREVIEW_MERGER_VERSION,
|
||||
MaterialPreviewError,
|
||||
createReimbursementMaterialPreviewStore,
|
||||
normalizePreviewImage,
|
||||
rasterizePdfWithPoppler,
|
||||
runMergeWorker,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
const { parentPort, workerData } = require("node:worker_threads");
|
||||
const { mergeMaterialsToPdf } = require("./reimbursementMaterialPdf");
|
||||
|
||||
mergeMaterialsToPdf(workerData.materials, workerData.limits)
|
||||
.then(({ buffer, pageCount }) => {
|
||||
const bytes = Uint8Array.from(buffer);
|
||||
parentPort.postMessage({ ok: true, bytes, pageCount }, [bytes.buffer]);
|
||||
})
|
||||
.catch((error) => {
|
||||
parentPort.postMessage({ ok: false, code: error?.message || "material_preview_merge_failed" });
|
||||
});
|
||||
在新工单中引用
屏蔽一个用户