#!/usr/bin/env bash set -euo pipefail ERP_SYNC_TIMEOUT_MS="${ERP_SYNC_TIMEOUT_MS:-300}" MOCK_ERP_PORT="${MOCK_ERP_PORT:-$((3900 + RANDOM % 500))}" if ! command -v node >/dev/null 2>&1; then echo "[ERROR] node not found" >&2 exit 1 fi if ! command -v curl >/dev/null 2>&1; then echo "[ERROR] curl not found" >&2 exit 1 fi if ! command -v jq >/dev/null 2>&1; then echo "[ERROR] jq not found" >&2 exit 1 fi ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" if [[ -z "${DATABASE_URL:-}" && -f ".env" ]]; then set -a # shellcheck disable=SC1091 source .env set +a fi if [[ -z "${DATABASE_URL:-}" ]]; then echo "[ERROR] DATABASE_URL is required (export it or provide server/.env)" >&2 exit 1 fi TMP_DIR="$(mktemp -d)" RUN_SUFFIX="$(date +%s)-$RANDOM" MOCK_SERVER_PID="" cleanup() { if [[ -n "$MOCK_SERVER_PID" ]] && kill -0 "$MOCK_SERVER_PID" >/dev/null 2>&1; then kill "$MOCK_SERVER_PID" >/dev/null 2>&1 || true wait "$MOCK_SERVER_PID" >/dev/null 2>&1 || true fi rm -rf "$TMP_DIR" } trap cleanup EXIT start_mock_erp_server() { local server_file="$TMP_DIR/mock-erp-server.js" cat >"$server_file" <<'NODE' const http = require("node:http"); const { URL } = require("node:url"); const port = Number(process.argv[2] || 3900); function sendJson(res, status, payload) { res.statusCode = status; res.setHeader("content-type", "application/json; charset=utf-8"); res.end(JSON.stringify(payload)); } const server = http.createServer(async (req, res) => { const url = new URL(req.url, `http://${req.headers.host || `127.0.0.1:${port}`}`); if (req.method === "GET" && url.pathname === "/health") { return sendJson(res, 200, { ok: true, mode: "health" }); } if (req.method === "POST" && url.pathname === "/erp/reimbursements/paid") { const mode = url.searchParams.get("mode") || "success"; const chunks = []; for await (const chunk of req) { chunks.push(chunk); } let parsedBody = null; try { parsedBody = JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}"); } catch { parsedBody = null; } if (mode === "timeout") { setTimeout(() => { sendJson(res, 200, { ok: true, scenario: "timeout", externalId: `ERP-TIMEOUT-${Date.now()}`, reimbursementId: parsedBody?.reimbursementId || null, }); }, 1500); return; } if (mode === "client") { return sendJson(res, 422, { ok: false, scenario: "client", code: "invalid_payload", message: "mock client error", }); } if (mode === "server") { return sendJson(res, 503, { ok: false, scenario: "server", code: "service_unavailable", message: "mock server error", }); } return sendJson(res, 200, { ok: true, scenario: "success", externalId: `ERP-${parsedBody?.formNumber || parsedBody?.reimbursementId || Date.now()}`, reimbursementId: parsedBody?.reimbursementId || null, }); } sendJson(res, 404, { ok: false, message: "not_found" }); }); server.listen(port, "127.0.0.1", () => { process.stdout.write(`mock-erp-listening:${port}\n`); }); NODE node "$server_file" "$MOCK_ERP_PORT" >"$TMP_DIR/mock-erp.log" 2>&1 & MOCK_SERVER_PID="$!" for _ in $(seq 1 30); do if curl -sS "http://127.0.0.1:$MOCK_ERP_PORT/health" >/dev/null 2>&1; then return 0 fi sleep 0.2 done echo "[ERROR] mock ERP server failed to start" >&2 cat "$TMP_DIR/mock-erp.log" >&2 || true exit 1 } create_paid_reimbursement_and_enqueue() { local scenario="$1" DATABASE_URL="$DATABASE_URL" RUN_SUFFIX="$RUN_SUFFIX" SCENARIO="$scenario" node <<'NODE' require("dotenv/config"); const { createPrismaClient } = require("./src/prismaClient"); const { getConfig } = require("./src/config"); const { buildErpPayload } = require("./src/services/integrations/erp"); const { enqueueIntegrationEvent } = require("./src/services/integrations/outbox"); (async () => { const runSuffix = process.env.RUN_SUFFIX; const scenario = process.env.SCENARIO; const config = getConfig(); const { prisma, pool } = createPrismaClient(); await prisma.$connect(); try { const owner = await prisma.employee.findFirst({ where: { email: { in: ["employee@hyinda.com", "finance_mgr@hyinda.com"], }, }, select: { id: true, department_id: true }, orderBy: { created_at: "asc" }, }); if (!owner) { throw new Error("employee/finance_mgr not found"); } const reimbursement = await prisma.reimbursementForm.create({ data: { form_number: `BX-P10-${scenario}-${runSuffix}`.slice(0, 60), submitter_id: owner.id, owner_id: owner.id, department_id: owner.department_id, reimbursement_date: new Date("2026-02-10T00:00:00.000Z"), total_amount: "100.00", status: "paid", payment_voucher_no: `PV-${scenario}-${runSuffix}`.slice(0, 60), payment_date: new Date("2026-02-10T00:00:00.000Z"), paid_at: new Date(), created_by: owner.id, updated_by: owner.id, }, select: { id: true, form_number: true, owner_id: true, submitter_id: true, total_amount: true, payment_voucher_no: true, payment_date: true, paid_at: true, }, }); const payload = buildErpPayload({ reimbursement }); const enqueued = await enqueueIntegrationEvent(prisma, { eventType: "erp.reimbursement.paid", payload, dedupeKey: reimbursement.id, maxAttempts: config.integrationOutboxMaxAttempts, actorId: owner.id, }); process.stdout.write( JSON.stringify({ scenario, reimbursementId: reimbursement.id, outboxId: enqueued.id, duplicated: enqueued.duplicated, }), ); } finally { await prisma.$disconnect(); await pool.end(); } })().catch((error) => { console.error(error); process.exit(1); }); NODE } query_outbox_state() { local outbox_id="$1" DATABASE_URL="$DATABASE_URL" OUTBOX_ID="$outbox_id" node <<'NODE' require("dotenv/config"); const { createPrismaClient } = require("./src/prismaClient"); (async () => { const outboxId = process.env.OUTBOX_ID; const { prisma, pool } = createPrismaClient(); await prisma.$connect(); try { const row = await prisma.integrationOutbox.findUnique({ where: { id: outboxId }, select: { id: true, status: true, attempts: true, next_retry_at: true, dead_letter_at: true, last_error: true, }, }); if (!row) { throw new Error(`outbox not found: ${outboxId}`); } process.stdout.write( JSON.stringify({ id: row.id, status: row.status, attempts: row.attempts, nextRetryAt: row.next_retry_at ? row.next_retry_at.toISOString() : null, deadLetterAt: row.dead_letter_at ? row.dead_letter_at.toISOString() : null, lastError: row.last_error, }), ); } finally { await prisma.$disconnect(); await pool.end(); } })().catch((error) => { console.error(error); process.exit(1); }); NODE } run_outbox_once() { local endpoint="$1" ERP_SYNC_ENABLED=true \ ERP_SYNC_PROVIDER=http \ ERP_SYNC_ENDPOINT="$endpoint" \ ERP_SYNC_TIMEOUT_MS="$ERP_SYNC_TIMEOUT_MS" \ ERP_SYNC_AUTH_TOKEN="" \ npm run --silent job:integration-outbox-once >"$TMP_DIR/outbox-once.log" } set_erp_flag() { local mode="$1" DATABASE_URL="$DATABASE_URL" MODE="$mode" node <<'NODE' require("dotenv/config"); const { createPrismaClient } = require("./src/prismaClient"); (async () => { const mode = process.env.MODE; const { prisma, pool } = createPrismaClient(); await prisma.$connect(); try { await prisma.featureFlag.upsert({ where: { flag_key: "erp_sync" }, create: { flag_key: "erp_sync", override_mode: mode, rollout_type: "global", department_ids: [], note: "regression-integrations", }, update: { override_mode: mode, rollout_type: "global", department_ids: [], note: "regression-integrations", version: { increment: 1 }, }, }); } finally { await prisma.$disconnect(); await pool.end(); } })().catch((error) => { console.error(error); process.exit(1); }); NODE } echo "[0/6] Ensure ERP feature flag force_on" set_erp_flag "force_on" assert_mode_semantics() { local mode="$1" local state_json="$2" case "$mode" in success) jq -e '.status == "succeeded" and .nextRetryAt == null and .deadLetterAt == null' <<<"$state_json" >/dev/null ;; client) jq -e '.status == "dead_letter" and .attempts == 1 and .deadLetterAt != null and .nextRetryAt == null' <<<"$state_json" >/dev/null jq -e '.lastError | tostring | test("422|erp_sync_http_4xx")' <<<"$state_json" >/dev/null ;; server) jq -e '.status == "failed" and .attempts == 1 and .deadLetterAt == null and .nextRetryAt != null' <<<"$state_json" >/dev/null jq -e '.lastError | tostring | test("503|erp_sync_http_5xx")' <<<"$state_json" >/dev/null ;; timeout) jq -e '.status == "failed" and .attempts == 1 and .deadLetterAt == null and .nextRetryAt != null' <<<"$state_json" >/dev/null jq -e '.lastError | tostring | test("erp_sync_timeout|timeout")' <<<"$state_json" >/dev/null ;; *) echo "[ERROR] unsupported mode assertion: $mode" >&2 exit 1 ;; esac } run_mode_case() { local mode="$1" local created_json created_json="$(create_paid_reimbursement_and_enqueue "$mode")" local outbox_id outbox_id="$(jq -er '.outboxId' <<<"$created_json")" local endpoint="http://127.0.0.1:$MOCK_ERP_PORT/erp/reimbursements/paid?mode=$mode" local state_json="" local status="" for _ in $(seq 1 8); do run_outbox_once "$endpoint" state_json="$(query_outbox_state "$outbox_id")" status="$(jq -r '.status' <<<"$state_json")" if [[ "$status" != "pending" && "$status" != "processing" ]]; then break fi sleep 0.2 done if [[ "$status" == "pending" || "$status" == "processing" ]]; then echo "[ERROR] mode=$mode did not finish processing" >&2 echo "$state_json" >&2 exit 1 fi if ! assert_mode_semantics "$mode" "$state_json"; then echo "[ERROR] mode=$mode semantics assertion failed" >&2 echo "$state_json" >&2 exit 1 fi echo "[OK] mode=$mode outbox_status=$status" } echo "[1/6] Start embedded mock ERP server" start_mock_erp_server echo "[2/6] ERP success path => succeeded" run_mode_case "success" echo "[3/6] ERP 4xx path => dead_letter" run_mode_case "client" echo "[4/6] ERP 5xx path => failed(retryable)" run_mode_case "server" echo "[5/6] ERP timeout path => failed(retryable)" run_mode_case "timeout" echo "[6/6] Done" echo "[OK] Integrations regression passed" echo "[post] Reset ERP feature flag force_off" set_erp_flag "force_off"