Files

357 lines
12 KiB
TypeScript

// Renders a Markdown rollup from the per-shard Vally JUnit results: groups by shard,
// collapses per-trial testcases to one stimulus, applies the threshold, and lists failing
// scenarios. Presentation only — never changes pass/fail.
import fs from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { globFiles } from "./lib/glob.ts";
// Maps a JUnit file path back to its shard and job attempt. Result artifacts download into
// folders named `eval-result-<shardName>-<attempt>` (the attempt suffix keeps "Rerun failed
// jobs" from colliding on the artifact name); everything below that is the shard's JUnit.
export function getShardArtifact(filePath) {
const segments = filePath.split(/[\\/]/).filter(Boolean);
for (const segment of segments) {
if (segment.startsWith("eval-result-")) {
const raw = segment.replace(/^eval-result-/, "");
// Strip a trailing `-<digits>` job-attempt suffix (absent in older single-attempt runs).
const match = raw.match(/^(.*)-(\d+)$/);
if (match) {
return { shardName: match[1], attempt: Number(match[2]) };
}
return { shardName: raw, attempt: 1 };
}
}
// Fallback: nearest ancestor dir that isn't a Vally timestamp folder.
for (let i = segments.length - 2; i >= 0; i--) {
if (!/^\d{4}-\d{2}-\d{2}T/.test(segments[i])) {
return { shardName: segments[i], attempt: 1 };
}
}
return { shardName: "unknown", attempt: 1 };
}
// Back-compat: return just the shard name (attempt suffix stripped).
export function getShardName(filePath) {
return getShardArtifact(filePath).shardName;
}
// Strips Vally's ' (trial N)' suffix so every trial of a stimulus collapses to one stimulus.
export function getStimulusName(name) {
if (!name || !name.trim()) {
return "(unnamed scenario)";
}
return name.replace(/\s*\(trial\s+\d+\)\s*$/, "").trim();
}
// Formats a 0..1 ratio as a percentage, dropping a trailing '.0' so whole numbers read as
// '100%' while fractional rates keep one decimal ('93.5%').
export function formatPct(ratio) {
const value = ratio * 100;
if (Math.abs(value - Math.round(value)) < 0.05) {
return `${Math.round(value)}%`;
}
return `${value.toFixed(1)}%`;
}
function getAttr(attrs, name) {
const match = attrs.match(new RegExp(`\\b${name}=["']([^"']*)["']`));
return match ? match[1] : undefined;
}
// Minimal dependency-free JUnit parse: testsuite (optional threshold property) > testcase
// with optional <failure>/<error>/<skipped> children. `\b` avoids matching `testsuites`.
// NOTE: coupled to Vally's JUnit output shape; if that XML changes, update these regexes
// (or swap in a real XML parser).
function parseSuites(content) {
const suites = [];
const suiteRe = /<testsuite\b([^>]*?)(\/>|>([\s\S]*?)<\/testsuite>)/g;
let suiteMatch;
while ((suiteMatch = suiteRe.exec(content))) {
const inner = suiteMatch[2].startsWith("/>") ? "" : suiteMatch[3];
let threshold;
const thresholdMatch = inner.match(
/<property\b[^>]*\bname=["']threshold["'][^>]*\bvalue=["']([^"']+)["']/
);
if (thresholdMatch) {
const parsed = Number(thresholdMatch[1]);
if (!Number.isNaN(parsed)) {
threshold = parsed;
}
}
const testcases = [];
const caseRe = /<testcase\b([^>]*?)(\/>|>([\s\S]*?)<\/testcase>)/g;
let caseMatch;
while ((caseMatch = caseRe.exec(inner))) {
const attrs = caseMatch[1];
const body = caseMatch[2].startsWith("/>") ? "" : caseMatch[3];
const timeStr = getAttr(attrs, "time");
testcases.push({
name: getAttr(attrs, "name") ?? "",
time: timeStr ? Number(timeStr) || 0 : 0,
failure: /<failure\b/.test(body) || /<error\b/.test(body),
skipped: /<skipped\b/.test(body),
});
}
suites.push({ threshold, testcases });
}
return suites;
}
/**
* Reads every JUnit XML under `resultsRoot`, grouped by shard, and returns an object keyed
* by shard name with aggregated totals and failing-scenario names.
*
* @param {string} resultsRoot Folder containing the downloaded per-shard result artifacts.
* @returns {Record<string, {shardName: string, total: number, failed: number, skipped: number, durationS: number, failures: string[]}>}
*/
export function getEvalSummary(resultsRoot) {
const root = path.resolve(resultsRoot);
const xmlFiles = globFiles(root, "**/*.xml");
const shards = {};
// A "Rerun failed jobs" retry publishes a higher-attempt artifact for the same shard.
// Resolve each file's shard + attempt up front, then only aggregate the highest attempt
// per shard so a retry supersedes the earlier one instead of double-counting.
const parsedFiles = xmlFiles.map((xmlFile) => ({ xmlFile, ...getShardArtifact(xmlFile) }));
const maxAttempt = {};
for (const { shardName, attempt } of parsedFiles) {
maxAttempt[shardName] = Math.max(maxAttempt[shardName] ?? 0, attempt);
}
for (const { xmlFile, shardName, attempt } of parsedFiles) {
if (attempt !== maxAttempt[shardName]) {
continue; // superseded by a later rerun attempt of this shard
}
if (!shards[shardName]) {
shards[shardName] = {
shardName,
total: 0,
failed: 0,
skipped: 0,
durationS: 0,
failures: [],
// stimulus name -> { trials, passed, skipped, threshold }
stimuli: new Map(),
};
}
const shard = shards[shardName];
const content = fs.readFileSync(xmlFile, "utf8");
// Read each <testsuite>'s threshold and aggregate its trials back up to the stimulus,
// mirroring exactly what `vally eval` gates on.
for (const suite of parseSuites(content)) {
const threshold = suite.threshold ?? 0.8;
for (const testcase of suite.testcases) {
const stimulus = getStimulusName(testcase.name);
if (!shard.stimuli.has(stimulus)) {
shard.stimuli.set(stimulus, { trials: 0, passed: 0, skipped: 0, threshold });
}
const entry = shard.stimuli.get(stimulus);
entry.threshold = threshold;
shard.durationS += testcase.time;
if (testcase.skipped) {
entry.skipped++;
} else {
entry.trials++;
if (!testcase.failure) {
entry.passed++;
}
}
}
}
}
// Collapse each stimulus's trials into one pass/fail, once per shard after all XML is read.
for (const shardName of Object.keys(shards)) {
const shard = shards[shardName];
for (const [stimulus, entry] of shard.stimuli) {
shard.total++;
if (entry.trials === 0) {
shard.skipped++;
continue;
}
const passRate = entry.passed / entry.trials;
// 1e-9 epsilon guards float rounding so 4/5 = 0.8 is not dropped below an 0.8 gate.
if (passRate + 1e-9 < entry.threshold) {
shard.failed++;
shard.failures.push(`${stimulus} (${entry.passed}/${entry.trials} runs passed)`);
}
}
}
return shards;
}
/**
* Renders the Markdown summary for the given shard map.
*
* @param {Record<string, object>} shards Output of getEvalSummary.
* @returns {string} Markdown.
*/
export function formatEvalSummaryMarkdown(shards) {
const all = Object.values(shards);
const shardCount = all.length;
let totalTests = 0;
let totalFailed = 0;
let totalSkipped = 0;
for (const shard of all) {
totalTests += shard.total;
totalFailed += shard.failed;
totalSkipped += shard.skipped;
}
const totalPassed = totalTests - totalFailed - totalSkipped;
const lines = [];
// A run that parsed zero testcases is NOT a pass — surface it as a loud NO RESULTS state.
let overall;
let overallIcon;
if (totalTests === 0) {
overall = "NO RESULTS";
overallIcon = "⚠️";
} else if (totalFailed === 0) {
overall = "PASSED";
overallIcon = "✅";
} else {
overall = "FAILED";
overallIcon = "❌";
}
// Pass rate is measured over scenarios that actually ran (skips excluded).
const nonSkipped = totalPassed + totalFailed;
const overallRatio = nonSkipped > 0 ? totalPassed / nonSkipped : 0;
lines.push(`## ${overallIcon} Vally eval results — ${overall}`);
lines.push("");
if (totalTests === 0) {
lines.push(`No scenarios were found across ${shardCount} shard(s).`);
lines.push("");
lines.push("> ⚠️ No eval testcases were found in the downloaded results. This usually means the");
lines.push("> eval shards did not publish JUnit — the shard jobs failed before running, or the");
lines.push("> `eval-result-*` artifacts were empty. Check the Eval stage shard logs.");
return lines.join("\n") + "\n";
}
// Glanceable one-liner.
const scenarioWord = totalTests === 1 ? "scenario" : "scenarios";
const shardWord = shardCount === 1 ? "shard" : "shards";
const tallies = [`✅ **${totalPassed} passed**`];
if (totalFailed > 0) {
tallies.push(`❌ **${totalFailed} failed**`);
}
if (totalSkipped > 0) {
tallies.push(`⏭️ ${totalSkipped} skipped`);
}
tallies.push(`**${formatPct(overallRatio)}** pass rate`);
lines.push(
`**${totalTests} ${scenarioWord}** across **${shardCount} ${shardWord}** — ${tallies.join(" · ")}`
);
lines.push("");
// Red shards first (then alphabetical) so a reader's eye lands on failures.
const ordered = [...all].sort((a, b) => {
const aClean = a.failed === 0 ? 1 : 0;
const bClean = b.failed === 0 ? 1 : 0;
if (aClean !== bClean) {
return aClean - bClean;
}
return a.shardName.localeCompare(b.shardName);
});
lines.push("| Shard | Result | Pass rate | Passed | Failed | Skipped | Time (s) |");
lines.push("| --- | :---: | ---: | ---: | ---: | ---: | ---: |");
for (const shard of ordered) {
const passed = shard.total - shard.failed - shard.skipped;
const icon = shard.failed === 0 ? "✅" : "❌";
const ran = passed + shard.failed;
const shardPct = ran > 0 ? formatPct(passed / ran) : "—";
lines.push(
`| ${shard.shardName} | ${icon} | ${shardPct} | ${passed} | ${shard.failed} | ${shard.skipped} | ${shard.durationS.toFixed(1)} |`
);
}
// Totals row.
const totalIcon = totalFailed === 0 ? "✅" : "❌";
let totalDuration = 0;
for (const shard of all) {
totalDuration += shard.durationS;
}
lines.push(
`| **Total** | ${totalIcon} | **${formatPct(overallRatio)}** | **${totalPassed}** | **${totalFailed}** | **${totalSkipped}** | **${totalDuration.toFixed(1)}** |`
);
if (totalFailed > 0) {
const failWord = totalFailed === 1 ? "scenario" : "scenarios";
lines.push("");
lines.push(`<details open><summary><strong>❌ ${totalFailed} failing ${failWord}</strong></summary>`);
lines.push("");
for (const shard of ordered) {
if (shard.failed === 0) {
continue;
}
lines.push(`- **${shard.shardName}**`);
for (const name of shard.failures) {
lines.push(` - ❌ ${name}`);
}
}
lines.push("");
lines.push("</details>");
}
return lines.join("\n") + "\n";
}
// ----- CLI -----
function parseArgs(argv) {
const options = { outputPath: "eval-summary.md" };
for (let i = 0; i < argv.length; i++) {
const next = () => argv[++i];
switch (argv[i]) {
case "--results-root":
options.resultsRoot = next();
break;
case "--output-path":
options.outputPath = next();
break;
default:
throw new Error(`Unknown argument: ${argv[i]}`);
}
}
if (!options.resultsRoot) {
throw new Error("Missing required argument: --results-root");
}
return options;
}
function main(argv) {
const options = parseArgs(argv);
const shards = getEvalSummary(options.resultsRoot);
const markdown = formatEvalSummaryMarkdown(shards);
fs.writeFileSync(options.outputPath, markdown, "utf8");
console.log(markdown);
if (process.env.TF_BUILD) {
console.log(`##vso[task.uploadsummary]${path.resolve(options.outputPath)}`);
}
return shards;
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
try {
main(process.argv.slice(2));
} catch (error) {
console.error(error.message);
process.exit(1);
}
}