feat(server): MinIO/S3 对象存储兼容(F21)
- 新增 storage 抽象:local 与 s3: 前缀兼容(不改 schema) - invoice-pool 上传/预览/ocr 读取适配 S3/MinIO - trialrun compose 增加 minio profile + env 示例与 runbook 指引 - 更新 README 与 v3 CSV 状态推进
这个提交包含在:
@@ -50,6 +50,13 @@ function getConfig() {
|
||||
|
||||
appTimezone: process.env.APP_TIMEZONE ?? "Asia/Shanghai",
|
||||
storageDir: path.resolve(serverRoot, storageDir),
|
||||
storageDriver: process.env.STORAGE_DRIVER ?? "local",
|
||||
s3Endpoint: process.env.S3_ENDPOINT ?? "",
|
||||
s3Region: process.env.S3_REGION ?? "us-east-1",
|
||||
s3Bucket: process.env.S3_BUCKET ?? "",
|
||||
s3AccessKeyId: process.env.S3_ACCESS_KEY_ID ?? "",
|
||||
s3SecretAccessKey: process.env.S3_SECRET_ACCESS_KEY ?? "",
|
||||
s3ForcePathStyle: process.env.S3_FORCE_PATH_STYLE === "true",
|
||||
serveClientDist: process.env.SERVE_CLIENT_DIST === "true",
|
||||
clientDistDir: path.resolve(serverRoot, clientDistDir),
|
||||
signedUrlTtlSeconds: Number(process.env.SIGNED_URL_TTL_SECONDS ?? 600),
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
const fs = require("node:fs/promises");
|
||||
const path = require("node:path");
|
||||
const cron = require("node-cron");
|
||||
|
||||
const { runOcrForInvoice } = require("../services/ocr");
|
||||
const { readFileBuffer } = require("../services/storage");
|
||||
|
||||
const OCR_DUPLICATE_ERROR = "duplicate_invoice_code_number";
|
||||
|
||||
@@ -222,8 +221,10 @@ async function markInvoiceAsFailed(prisma, claimedInvoice, errorMessage, config)
|
||||
|
||||
async function processClaimedInvoice(prisma, config, claimedInvoice, workerId, logger) {
|
||||
try {
|
||||
const absolutePath = path.resolve(config.storageDir, claimedInvoice.storage_path);
|
||||
const fileBuffer = await fs.readFile(absolutePath);
|
||||
const fileBuffer = await readFileBuffer({
|
||||
config,
|
||||
storagePath: claimedInvoice.storage_path,
|
||||
});
|
||||
const imageBase64 = fileBuffer.toString("base64");
|
||||
|
||||
const { ocrResultId, normalized } = await getOrCreateOcrResult(
|
||||
|
||||
+28
-23
@@ -1,6 +1,4 @@
|
||||
const crypto = require("node:crypto");
|
||||
const fs = require("node:fs/promises");
|
||||
const fsSync = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const { z } = require("zod");
|
||||
@@ -13,6 +11,7 @@ const { signPayload, verifyToken } = require("../utils/signedUrl");
|
||||
const { toMoneyString } = require("../utils/workflow");
|
||||
const { findActiveDelegationForDelegate } = require("../utils/delegation");
|
||||
const { isOcrEnabledForType } = require("../services/ocr");
|
||||
const { openReadStream, StorageNotFoundError, writeBuffer } = require("../services/storage");
|
||||
|
||||
const MIME_ALLOWLIST = new Set([
|
||||
"application/pdf",
|
||||
@@ -565,11 +564,13 @@ async function invoicePoolRoutes(fastify) {
|
||||
const fileSha256 = sha256Hex(fileBuffer);
|
||||
const datePrefix = compactDateString(fastify.config.appTimezone);
|
||||
const extension = path.extname(filePart.filename ?? "").toLowerCase();
|
||||
const storagePath = path.join(datePrefix, `${crypto.randomUUID()}${extension}`);
|
||||
const absoluteStoragePath = path.resolve(fastify.config.storageDir, storagePath);
|
||||
|
||||
await fs.mkdir(path.dirname(absoluteStoragePath), { recursive: true });
|
||||
await fs.writeFile(absoluteStoragePath, fileBuffer);
|
||||
const objectKey = path.posix.join(datePrefix, `${crypto.randomUUID()}${extension}`);
|
||||
const storagePath = await writeBuffer({
|
||||
config: fastify.config,
|
||||
key: objectKey,
|
||||
buffer: fileBuffer,
|
||||
contentType: filePart.mimetype,
|
||||
});
|
||||
|
||||
const status = isOcrEnabledForType(fastify.config, ocrType) ? "pending_ocr" : "available";
|
||||
|
||||
@@ -1918,23 +1919,27 @@ async function invoicePoolRoutes(fastify) {
|
||||
});
|
||||
}
|
||||
|
||||
const storageRoot = path.resolve(fastify.config.storageDir);
|
||||
const absolutePath = path.resolve(storageRoot, invoice.storage_path);
|
||||
if (!absolutePath.startsWith(`${storageRoot}${path.sep}`)) {
|
||||
throw new AppError({
|
||||
code: "PERM_001",
|
||||
message: "无权限",
|
||||
statusCode: 403,
|
||||
});
|
||||
}
|
||||
|
||||
let stream = null;
|
||||
try {
|
||||
await fs.access(absolutePath);
|
||||
} catch {
|
||||
const opened = await openReadStream({
|
||||
config: fastify.config,
|
||||
storagePath: invoice.storage_path,
|
||||
});
|
||||
stream = opened.stream;
|
||||
} catch (error) {
|
||||
if (error instanceof StorageNotFoundError) {
|
||||
throw new AppError({
|
||||
code: "SYS_404",
|
||||
message: "Not Found",
|
||||
statusCode: 404,
|
||||
});
|
||||
}
|
||||
|
||||
throw new AppError({
|
||||
code: "SYS_404",
|
||||
message: "Not Found",
|
||||
statusCode: 404,
|
||||
code: "SYS_001",
|
||||
message: "服务器内部错误",
|
||||
statusCode: 500,
|
||||
details: {},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1960,7 +1965,7 @@ async function invoicePoolRoutes(fastify) {
|
||||
"content-disposition",
|
||||
`inline; filename=\"${sanitizeFilename(invoice.original_name)}\"`,
|
||||
);
|
||||
return reply.send(fsSync.createReadStream(absolutePath));
|
||||
return reply.send(stream);
|
||||
});
|
||||
|
||||
fastify.post(
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
const fs = require("node:fs/promises");
|
||||
const fsSync = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const { S3Client, PutObjectCommand, GetObjectCommand } = require("@aws-sdk/client-s3");
|
||||
|
||||
class StorageNotFoundError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = "StorageNotFoundError";
|
||||
}
|
||||
}
|
||||
|
||||
class StorageConfigError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = "StorageConfigError";
|
||||
}
|
||||
}
|
||||
|
||||
function parseStoragePath(storagePath) {
|
||||
const raw = typeof storagePath === "string" ? storagePath : "";
|
||||
if (raw.startsWith("s3:")) {
|
||||
return { driver: "s3", key: raw.slice("s3:".length) };
|
||||
}
|
||||
return { driver: "local", key: raw };
|
||||
}
|
||||
|
||||
function ensureSafeLocalPath(storageRoot, relativeKey) {
|
||||
const root = path.resolve(storageRoot);
|
||||
const absolutePath = path.resolve(root, relativeKey);
|
||||
if (!absolutePath.startsWith(`${root}${path.sep}`)) {
|
||||
throw new StorageConfigError("Invalid storage_path (path traversal detected)");
|
||||
}
|
||||
return absolutePath;
|
||||
}
|
||||
|
||||
function resolveS3Config(config) {
|
||||
if (!config?.s3Bucket) {
|
||||
throw new StorageConfigError("Missing S3_BUCKET for s3 storage");
|
||||
}
|
||||
|
||||
return {
|
||||
bucket: config.s3Bucket,
|
||||
region: config.s3Region || "us-east-1",
|
||||
endpoint: config.s3Endpoint || undefined,
|
||||
credentials:
|
||||
config.s3AccessKeyId && config.s3SecretAccessKey
|
||||
? {
|
||||
accessKeyId: config.s3AccessKeyId,
|
||||
secretAccessKey: config.s3SecretAccessKey,
|
||||
}
|
||||
: undefined,
|
||||
forcePathStyle: Boolean(config.s3ForcePathStyle),
|
||||
};
|
||||
}
|
||||
|
||||
let cachedS3Client = null;
|
||||
let cachedS3ClientKey = null;
|
||||
|
||||
function getS3Client(config) {
|
||||
const s3 = resolveS3Config(config);
|
||||
const cacheKey = JSON.stringify({
|
||||
bucket: s3.bucket,
|
||||
region: s3.region,
|
||||
endpoint: s3.endpoint ?? "",
|
||||
accessKeyId: s3.credentials?.accessKeyId ?? "",
|
||||
forcePathStyle: s3.forcePathStyle,
|
||||
});
|
||||
|
||||
if (cachedS3Client && cacheKey === cachedS3ClientKey) {
|
||||
return cachedS3Client;
|
||||
}
|
||||
|
||||
cachedS3ClientKey = cacheKey;
|
||||
cachedS3Client = new S3Client({
|
||||
region: s3.region,
|
||||
endpoint: s3.endpoint,
|
||||
credentials: s3.credentials,
|
||||
forcePathStyle: s3.forcePathStyle,
|
||||
});
|
||||
return cachedS3Client;
|
||||
}
|
||||
|
||||
async function streamToBuffer(stream) {
|
||||
const chunks = [];
|
||||
for await (const chunk of stream) {
|
||||
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
||||
}
|
||||
return Buffer.concat(chunks);
|
||||
}
|
||||
|
||||
async function writeBuffer({ config, key, buffer, contentType }) {
|
||||
if (!Buffer.isBuffer(buffer) || buffer.length === 0) {
|
||||
throw new StorageConfigError("Invalid buffer");
|
||||
}
|
||||
|
||||
const driver = config?.storageDriver ?? "local";
|
||||
if (driver === "s3") {
|
||||
const s3 = resolveS3Config(config);
|
||||
const client = getS3Client(config);
|
||||
|
||||
await client.send(
|
||||
new PutObjectCommand({
|
||||
Bucket: s3.bucket,
|
||||
Key: key,
|
||||
Body: buffer,
|
||||
ContentType: contentType || "application/octet-stream",
|
||||
}),
|
||||
);
|
||||
|
||||
// Use a lightweight prefix to support mixed local+s3 records without schema changes.
|
||||
return `s3:${key}`;
|
||||
}
|
||||
|
||||
const absolutePath = ensureSafeLocalPath(config.storageDir, key);
|
||||
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
|
||||
await fs.writeFile(absolutePath, buffer);
|
||||
return key;
|
||||
}
|
||||
|
||||
async function readFileBuffer({ config, storagePath }) {
|
||||
const parsed = parseStoragePath(storagePath);
|
||||
if (!parsed.key) {
|
||||
throw new StorageNotFoundError("missing_storage_path");
|
||||
}
|
||||
|
||||
if (parsed.driver === "s3") {
|
||||
const s3 = resolveS3Config(config);
|
||||
const client = getS3Client(config);
|
||||
|
||||
try {
|
||||
const result = await client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: s3.bucket,
|
||||
Key: parsed.key,
|
||||
}),
|
||||
);
|
||||
if (!result?.Body) {
|
||||
throw new StorageNotFoundError("s3_body_missing");
|
||||
}
|
||||
return streamToBuffer(result.Body);
|
||||
} catch (error) {
|
||||
const code = error?.$metadata?.httpStatusCode;
|
||||
if (code === 404 || error?.name === "NoSuchKey") {
|
||||
throw new StorageNotFoundError("s3_not_found");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const absolutePath = ensureSafeLocalPath(config.storageDir, parsed.key);
|
||||
try {
|
||||
return await fs.readFile(absolutePath);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") {
|
||||
throw new StorageNotFoundError("local_not_found");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function openReadStream({ config, storagePath }) {
|
||||
const parsed = parseStoragePath(storagePath);
|
||||
if (!parsed.key) {
|
||||
throw new StorageNotFoundError("missing_storage_path");
|
||||
}
|
||||
|
||||
if (parsed.driver === "s3") {
|
||||
const s3 = resolveS3Config(config);
|
||||
const client = getS3Client(config);
|
||||
|
||||
try {
|
||||
const result = await client.send(
|
||||
new GetObjectCommand({
|
||||
Bucket: s3.bucket,
|
||||
Key: parsed.key,
|
||||
}),
|
||||
);
|
||||
|
||||
if (!result?.Body) {
|
||||
throw new StorageNotFoundError("s3_body_missing");
|
||||
}
|
||||
|
||||
return { stream: result.Body };
|
||||
} catch (error) {
|
||||
const code = error?.$metadata?.httpStatusCode;
|
||||
if (code === 404 || error?.name === "NoSuchKey") {
|
||||
throw new StorageNotFoundError("s3_not_found");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
const absolutePath = ensureSafeLocalPath(config.storageDir, parsed.key);
|
||||
try {
|
||||
await fs.access(absolutePath);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") {
|
||||
throw new StorageNotFoundError("local_not_found");
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
|
||||
return { stream: fsSync.createReadStream(absolutePath) };
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
StorageConfigError,
|
||||
StorageNotFoundError,
|
||||
openReadStream,
|
||||
parseStoragePath,
|
||||
readFileBuffer,
|
||||
writeBuffer,
|
||||
};
|
||||
|
||||
在新工单中引用
屏蔽一个用户