200 行
8.1 KiB
JavaScript
200 行
8.1 KiB
JavaScript
#!/usr/bin/env node
|
|
import { readFileSync } from "node:fs";
|
|
import path from "node:path";
|
|
import { pathToFileURL } from "node:url";
|
|
|
|
const WITNESS_TAG_PATTERN = /@local-candidate:[a-z0-9-]+/g;
|
|
const FAILED_STATUSES = new Set(["failed", "timedOut", "interrupted"]);
|
|
|
|
function collectSpecs(suites, output = [], inheritedFile = "") {
|
|
for (const suite of Array.isArray(suites) ? suites : []) {
|
|
const suiteFile = suite?.file ?? inheritedFile;
|
|
for (const spec of Array.isArray(suite?.specs) ? suite.specs : []) {
|
|
output.push({ ...spec, __sourceFile: spec?.file ?? suiteFile });
|
|
}
|
|
collectSpecs(suite?.suites, output, suiteFile);
|
|
}
|
|
return output;
|
|
}
|
|
|
|
function tagsForSpec(spec) {
|
|
const candidates = [
|
|
...(String(spec?.title ?? "").match(WITNESS_TAG_PATTERN) ?? []),
|
|
...(Array.isArray(spec?.tags) ? spec.tags : []).flatMap((tag) => String(tag).match(WITNESS_TAG_PATTERN) ?? []),
|
|
];
|
|
return [...new Set(candidates)];
|
|
}
|
|
|
|
function finalStatus(test) {
|
|
const results = Array.isArray(test?.results) ? test.results : [];
|
|
return results.at(-1)?.status ?? test?.status ?? "missing";
|
|
}
|
|
|
|
function safeDiagnosticText(value, maxLength = 160) {
|
|
return String(value ?? "")
|
|
.replace(/[\r\n\t]+/gu, " ")
|
|
.trim()
|
|
.slice(0, maxLength);
|
|
}
|
|
|
|
function safeSpecDiagnostic(spec, status) {
|
|
const line = Number(spec?.line);
|
|
return {
|
|
file: path.basename(safeDiagnosticText(spec?.__sourceFile ?? spec?.file, 240)) || "unknown-spec",
|
|
line: Number.isSafeInteger(line) && line > 0 ? line : null,
|
|
title: safeDiagnosticText(spec?.title) || "unknown-test",
|
|
status,
|
|
};
|
|
}
|
|
|
|
function normalizeWitnessSpecFile(file) {
|
|
const normalized = String(file ?? "").replaceAll("\\", "/");
|
|
if (normalized.startsWith("client/tests/e2e/")) return normalized;
|
|
if (normalized.startsWith("tests/e2e/")) return `client/${normalized}`;
|
|
const marker = "/client/tests/e2e/";
|
|
const markerIndex = normalized.lastIndexOf(marker);
|
|
if (markerIndex >= 0) return normalized.slice(markerIndex + 1);
|
|
if (/^[^/]+\.spec\.[cm]?[jt]sx?$/u.test(normalized)) return normalized;
|
|
return "";
|
|
}
|
|
|
|
function witnessSpecMatches(observedFile, expectedFile) {
|
|
if (!observedFile || !expectedFile) return false;
|
|
if (observedFile.includes("/")) return observedFile === expectedFile;
|
|
return observedFile === path.basename(expectedFile);
|
|
}
|
|
|
|
function duplicates(values) {
|
|
const counts = new Map();
|
|
for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1);
|
|
return [...counts.entries()].filter(([, count]) => count > 1).map(([value]) => value).sort();
|
|
}
|
|
|
|
export function verifyLocalCandidateResults(manifest, results, { expectedTagCount = 10 } = {}) {
|
|
const flows = Array.isArray(manifest?.localCandidateFlows) ? manifest.localCandidateFlows : [];
|
|
const expectedTags = flows.map((flow) => flow?.witnessTag).filter((tag) => typeof tag === "string");
|
|
const expectedTagSet = new Set(expectedTags);
|
|
const expectedSpecByTag = new Map(
|
|
flows.map((flow) => [flow?.witnessTag, normalizeWitnessSpecFile(flow?.witnessSpec)]),
|
|
);
|
|
const manifestDuplicateTags = duplicates(expectedTags);
|
|
const manifestInvalidTags = flows
|
|
.filter((flow) => flow?.witnessTag !== `@local-candidate:${flow?.bundle ?? ""}`)
|
|
.map((flow) => String(flow?.witnessTag ?? ""))
|
|
.sort();
|
|
|
|
const observed = [];
|
|
const allTests = [];
|
|
const multiTagTests = [];
|
|
const wrongWitnessSpecs = [];
|
|
for (const spec of collectSpecs(results?.suites)) {
|
|
const tags = tagsForSpec(spec);
|
|
const specFile = normalizeWitnessSpecFile(spec?.__sourceFile ?? spec?.file);
|
|
const tests = Array.isArray(spec?.tests) ? spec.tests : [];
|
|
for (const test of tests) {
|
|
const status = finalStatus(test);
|
|
allTests.push(safeSpecDiagnostic(spec, status));
|
|
if (tags.length > 1) multiTagTests.push(safeSpecDiagnostic(spec, status));
|
|
for (const tag of tags) {
|
|
observed.push({ tag, status });
|
|
const expectedSpec = expectedSpecByTag.get(tag);
|
|
if (!witnessSpecMatches(specFile, expectedSpec)) {
|
|
wrongWitnessSpecs.push({
|
|
tag,
|
|
expectedFile: path.basename(expectedSpec || "unknown-spec"),
|
|
observedFile: path.basename(specFile || "unknown-spec"),
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
const observedTags = observed.map((item) => item.tag);
|
|
const duplicateTags = duplicates(observedTags);
|
|
const unknownTags = [...new Set(observedTags.filter((tag) => !expectedTagSet.has(tag)))].sort();
|
|
const missingTags = expectedTags.filter((tag) => !observedTags.includes(tag)).sort();
|
|
const skippedTags = [...new Set(observed.filter((item) => item.status === "skipped").map((item) => item.tag))].sort();
|
|
const failedTags = [...new Set(observed.filter((item) => item.status !== "passed" && item.status !== "skipped").map((item) => item.tag))].sort();
|
|
const passedTags = [...new Set(observed.filter((item) => item.status === "passed").map((item) => item.tag))].sort();
|
|
const suiteFailedTestCount = allTests.filter((item) => FAILED_STATUSES.has(item.status)).length;
|
|
const suiteSkippedTestCount = allTests.filter((item) => item.status === "skipped").length;
|
|
const suiteUnexpectedCount = Number.isFinite(results?.stats?.unexpected) ? Number(results.stats.unexpected) : 0;
|
|
const suiteReportedSkippedCount = Number.isFinite(results?.stats?.skipped) ? Number(results.stats.skipped) : 0;
|
|
const resultErrorCount = Array.isArray(results?.errors) ? results.errors.length : 0;
|
|
const failedTests = allTests.filter((item) => FAILED_STATUSES.has(item.status)).slice(0, 20);
|
|
const skippedTests = allTests.filter((item) => item.status === "skipped").slice(0, 20);
|
|
|
|
const issues = [];
|
|
if (expectedTags.length !== expectedTagCount) issues.push("unexpected_expected_tag_count");
|
|
if (manifestDuplicateTags.length) issues.push("manifest_duplicate_tag");
|
|
if (manifestInvalidTags.length) issues.push("manifest_invalid_tag");
|
|
if (missingTags.length) issues.push("missing_tag_result");
|
|
if (duplicateTags.length) issues.push("duplicate_tag_result");
|
|
if (unknownTags.length) issues.push("unknown_tag_result");
|
|
if (multiTagTests.length) issues.push("multi_tag_test_result");
|
|
if (wrongWitnessSpecs.length) issues.push("wrong_witness_spec_result");
|
|
if (skippedTags.length) issues.push("skipped_tag_result");
|
|
if (failedTags.length) issues.push("failed_tag_result");
|
|
if (passedTags.length !== expectedTags.length) issues.push("incomplete_passed_tag_set");
|
|
if (suiteFailedTestCount > 0 || suiteUnexpectedCount > 0 || resultErrorCount > 0) issues.push("playwright_suite_failed");
|
|
if (suiteSkippedTestCount > 0 || suiteReportedSkippedCount > 0) issues.push("playwright_suite_skipped");
|
|
|
|
return {
|
|
ok: issues.length === 0,
|
|
issues,
|
|
expectedTagCount: expectedTags.length,
|
|
passedTagCount: passedTags.length,
|
|
skippedTagCount: skippedTags.length,
|
|
failedTagCount: failedTags.length,
|
|
missingTags,
|
|
duplicateTags,
|
|
unknownTags,
|
|
skippedTags,
|
|
failedTags,
|
|
manifestDuplicateTags,
|
|
manifestInvalidTags,
|
|
multiTagTests,
|
|
wrongWitnessSpecs,
|
|
totalTestResultCount: allTests.length,
|
|
suiteFailedTestCount,
|
|
suiteSkippedTestCount,
|
|
suiteUnexpectedCount,
|
|
suiteReportedSkippedCount,
|
|
resultErrorCount,
|
|
failedTests,
|
|
skippedTests,
|
|
};
|
|
}
|
|
|
|
function readJson(file) {
|
|
return JSON.parse(readFileSync(file, "utf8"));
|
|
}
|
|
|
|
function parseArgs(argv = process.argv.slice(2)) {
|
|
const manifestIndex = argv.indexOf("--manifest");
|
|
const resultsIndex = argv.indexOf("--results");
|
|
if (manifestIndex < 0 || resultsIndex < 0 || !argv[manifestIndex + 1] || !argv[resultsIndex + 1]) {
|
|
throw new Error("missing_arguments");
|
|
}
|
|
return {
|
|
manifestPath: path.resolve(argv[manifestIndex + 1]),
|
|
resultsPath: path.resolve(argv[resultsIndex + 1]),
|
|
};
|
|
}
|
|
|
|
function main() {
|
|
try {
|
|
const args = parseArgs();
|
|
const summary = verifyLocalCandidateResults(readJson(args.manifestPath), readJson(args.resultsPath));
|
|
console.log(JSON.stringify(summary, null, 2));
|
|
if (!summary.ok) process.exitCode = 1;
|
|
} catch {
|
|
console.log(JSON.stringify({ ok: false, issues: ["invalid_or_missing_result_json"] }, null, 2));
|
|
process.exitCode = 1;
|
|
}
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) {
|
|
main();
|
|
}
|