#!/usr/bin/env node import { readFileSync } from "node:fs"; import path from "node:path"; import { pathToFileURL } from "node:url"; const WITNESS_TAG_PATTERN = /@local-candidate:[a-z0-9-]+/g; const FAILED_STATUSES = new Set(["failed", "timedOut", "interrupted"]); function collectSpecs(suites, output = [], inheritedFile = "") { for (const suite of Array.isArray(suites) ? suites : []) { const suiteFile = suite?.file ?? inheritedFile; for (const spec of Array.isArray(suite?.specs) ? suite.specs : []) { output.push({ ...spec, __sourceFile: spec?.file ?? suiteFile }); } collectSpecs(suite?.suites, output, suiteFile); } return output; } function tagsForSpec(spec) { const candidates = [ ...(String(spec?.title ?? "").match(WITNESS_TAG_PATTERN) ?? []), ...(Array.isArray(spec?.tags) ? spec.tags : []).flatMap((tag) => String(tag).match(WITNESS_TAG_PATTERN) ?? []), ]; return [...new Set(candidates)]; } function finalStatus(test) { const results = Array.isArray(test?.results) ? test.results : []; return results.at(-1)?.status ?? test?.status ?? "missing"; } function safeDiagnosticText(value, maxLength = 160) { return String(value ?? "") .replace(/[\r\n\t]+/gu, " ") .trim() .slice(0, maxLength); } function safeSpecDiagnostic(spec, status) { const line = Number(spec?.line); return { file: path.basename(safeDiagnosticText(spec?.__sourceFile ?? spec?.file, 240)) || "unknown-spec", line: Number.isSafeInteger(line) && line > 0 ? line : null, title: safeDiagnosticText(spec?.title) || "unknown-test", status, }; } function normalizeWitnessSpecFile(file) { const normalized = String(file ?? "").replaceAll("\\", "/"); if (normalized.startsWith("client/tests/e2e/")) return normalized; if (normalized.startsWith("tests/e2e/")) return `client/${normalized}`; const marker = "/client/tests/e2e/"; const markerIndex = normalized.lastIndexOf(marker); if (markerIndex >= 0) return normalized.slice(markerIndex + 1); if (/^[^/]+\.spec\.[cm]?[jt]sx?$/u.test(normalized)) return normalized; return ""; } function witnessSpecMatches(observedFile, expectedFile) { if (!observedFile || !expectedFile) return false; if (observedFile.includes("/")) return observedFile === expectedFile; return observedFile === path.basename(expectedFile); } function duplicates(values) { const counts = new Map(); for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1); return [...counts.entries()].filter(([, count]) => count > 1).map(([value]) => value).sort(); } export function verifyLocalCandidateResults(manifest, results, { expectedTagCount = 10 } = {}) { const flows = Array.isArray(manifest?.localCandidateFlows) ? manifest.localCandidateFlows : []; const expectedTags = flows.map((flow) => flow?.witnessTag).filter((tag) => typeof tag === "string"); const expectedTagSet = new Set(expectedTags); const expectedSpecByTag = new Map( flows.map((flow) => [flow?.witnessTag, normalizeWitnessSpecFile(flow?.witnessSpec)]), ); const manifestDuplicateTags = duplicates(expectedTags); const manifestInvalidTags = flows .filter((flow) => flow?.witnessTag !== `@local-candidate:${flow?.bundle ?? ""}`) .map((flow) => String(flow?.witnessTag ?? "")) .sort(); const observed = []; const allTests = []; const multiTagTests = []; const wrongWitnessSpecs = []; for (const spec of collectSpecs(results?.suites)) { const tags = tagsForSpec(spec); const specFile = normalizeWitnessSpecFile(spec?.__sourceFile ?? spec?.file); const tests = Array.isArray(spec?.tests) ? spec.tests : []; for (const test of tests) { const status = finalStatus(test); allTests.push(safeSpecDiagnostic(spec, status)); if (tags.length > 1) multiTagTests.push(safeSpecDiagnostic(spec, status)); for (const tag of tags) { observed.push({ tag, status }); const expectedSpec = expectedSpecByTag.get(tag); if (!witnessSpecMatches(specFile, expectedSpec)) { wrongWitnessSpecs.push({ tag, expectedFile: path.basename(expectedSpec || "unknown-spec"), observedFile: path.basename(specFile || "unknown-spec"), }); } } } } const observedTags = observed.map((item) => item.tag); const duplicateTags = duplicates(observedTags); const unknownTags = [...new Set(observedTags.filter((tag) => !expectedTagSet.has(tag)))].sort(); const missingTags = expectedTags.filter((tag) => !observedTags.includes(tag)).sort(); const skippedTags = [...new Set(observed.filter((item) => item.status === "skipped").map((item) => item.tag))].sort(); const failedTags = [...new Set(observed.filter((item) => item.status !== "passed" && item.status !== "skipped").map((item) => item.tag))].sort(); const passedTags = [...new Set(observed.filter((item) => item.status === "passed").map((item) => item.tag))].sort(); const suiteFailedTestCount = allTests.filter((item) => FAILED_STATUSES.has(item.status)).length; const suiteSkippedTestCount = allTests.filter((item) => item.status === "skipped").length; const suiteUnexpectedCount = Number.isFinite(results?.stats?.unexpected) ? Number(results.stats.unexpected) : 0; const suiteReportedSkippedCount = Number.isFinite(results?.stats?.skipped) ? Number(results.stats.skipped) : 0; const resultErrorCount = Array.isArray(results?.errors) ? results.errors.length : 0; const failedTests = allTests.filter((item) => FAILED_STATUSES.has(item.status)).slice(0, 20); const skippedTests = allTests.filter((item) => item.status === "skipped").slice(0, 20); const issues = []; if (expectedTags.length !== expectedTagCount) issues.push("unexpected_expected_tag_count"); if (manifestDuplicateTags.length) issues.push("manifest_duplicate_tag"); if (manifestInvalidTags.length) issues.push("manifest_invalid_tag"); if (missingTags.length) issues.push("missing_tag_result"); if (duplicateTags.length) issues.push("duplicate_tag_result"); if (unknownTags.length) issues.push("unknown_tag_result"); if (multiTagTests.length) issues.push("multi_tag_test_result"); if (wrongWitnessSpecs.length) issues.push("wrong_witness_spec_result"); if (skippedTags.length) issues.push("skipped_tag_result"); if (failedTags.length) issues.push("failed_tag_result"); if (passedTags.length !== expectedTags.length) issues.push("incomplete_passed_tag_set"); if (suiteFailedTestCount > 0 || suiteUnexpectedCount > 0 || resultErrorCount > 0) issues.push("playwright_suite_failed"); if (suiteSkippedTestCount > 0 || suiteReportedSkippedCount > 0) issues.push("playwright_suite_skipped"); return { ok: issues.length === 0, issues, expectedTagCount: expectedTags.length, passedTagCount: passedTags.length, skippedTagCount: skippedTags.length, failedTagCount: failedTags.length, missingTags, duplicateTags, unknownTags, skippedTags, failedTags, manifestDuplicateTags, manifestInvalidTags, multiTagTests, wrongWitnessSpecs, totalTestResultCount: allTests.length, suiteFailedTestCount, suiteSkippedTestCount, suiteUnexpectedCount, suiteReportedSkippedCount, resultErrorCount, failedTests, skippedTests, }; } function readJson(file) { return JSON.parse(readFileSync(file, "utf8")); } function parseArgs(argv = process.argv.slice(2)) { const manifestIndex = argv.indexOf("--manifest"); const resultsIndex = argv.indexOf("--results"); if (manifestIndex < 0 || resultsIndex < 0 || !argv[manifestIndex + 1] || !argv[resultsIndex + 1]) { throw new Error("missing_arguments"); } return { manifestPath: path.resolve(argv[manifestIndex + 1]), resultsPath: path.resolve(argv[resultsIndex + 1]), }; } function main() { try { const args = parseArgs(); const summary = verifyLocalCandidateResults(readJson(args.manifestPath), readJson(args.resultsPath)); console.log(JSON.stringify(summary, null, 2)); if (!summary.ok) process.exitCode = 1; } catch { console.log(JSON.stringify({ ok: false, issues: ["invalid_or_missing_result_json"] }, null, 2)); process.exitCode = 1; } } if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { main(); }