mirror of
https://github.com/hanzoai/azure-sdk-for-js.git
synced 2026-10-30 19:51:51 +00:00
Sync eng/common directory with azure-sdk-tools for PR https://github.com/Azure/azure-sdk-tools/pull/16330 See [eng/common workflow](https://github.com/Azure/azure-sdk-tools/blob/main/eng/common/README.md#workflow) --------- Co-authored-by: helen229 <[email protected]>
131 lines
4.4 KiB
TypeScript
131 lines
4.4 KiB
TypeScript
// Runs one Vally eval shard and gates on the eval verdict (from results.jsonl), not the
|
|
// `vally` exit code, since vally can exit non-zero on a teardown flake after a pass.
|
|
|
|
import { spawnSync } from "node:child_process";
|
|
import path from "node:path";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
import { getVallyShardVerdict } from "./lib/verdict.ts";
|
|
|
|
// The pinned Vally CLI is installed next to this script (package.json + node_modules live here),
|
|
// so the npm --prefix is just this file's own directory.
|
|
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
|
|
/**
|
|
* Executes the shard and returns the process exit code (0 pass, 1 fail). Side effects
|
|
* (logging + Azure Pipelines `##vso` issues) go to the console; the verdict drives the code.
|
|
*
|
|
* @param {object} options
|
|
* @param {string} options.evalArgs Whitespace-separated `-e <file>` args, exactly as the matrix emits them.
|
|
* @param {string} options.shardName Shard name (log messages + result folder).
|
|
* @param {string} options.outputDir `--output-dir` Vally writes results into.
|
|
* @param {number} [options.threshold] Pass-rate gate forwarded to `vally eval --threshold`.
|
|
* @returns {number} 0 when the shard verdict passes, 1 otherwise.
|
|
*/
|
|
export function runShard({ evalArgs, shardName, outputDir, threshold = 0.8 }) {
|
|
const evalArgList = evalArgs.split(/\s+/).filter(Boolean);
|
|
const thresholdArg = String(threshold);
|
|
|
|
console.log(
|
|
`Running: vally eval ${evalArgs} --junit --threshold ${thresholdArg} --output-dir "${outputDir}"`
|
|
);
|
|
|
|
// Do NOT abort on a non-zero exit — the verdict below is authoritative.
|
|
// Uses spawnSync (not the vendored execFile helper) on purpose: the shard needs vally's output
|
|
// streamed live to the log (stdio: "inherit") and must keep going on a non-zero exit, whereas
|
|
// the exec helper captures output and rejects on failure.
|
|
const proc = spawnSync(
|
|
"npm",
|
|
[
|
|
"exec",
|
|
"--no",
|
|
"--prefix",
|
|
SCRIPT_DIR,
|
|
"--",
|
|
"vally",
|
|
"eval",
|
|
...evalArgList,
|
|
"--junit",
|
|
"--threshold",
|
|
thresholdArg,
|
|
"--output-dir",
|
|
outputDir,
|
|
],
|
|
{ stdio: "inherit", shell: process.platform === "win32" }
|
|
);
|
|
const vallyExit = proc.status ?? 1;
|
|
|
|
const verdict = getVallyShardVerdict({ resultsDir: outputDir, threshold });
|
|
for (const line of verdict.lines) {
|
|
console.log(` ${line}`);
|
|
}
|
|
|
|
if (!verdict.found) {
|
|
console.log(
|
|
`##vso[task.logissue type=error]Shard '${shardName}' produced no usable verdict (vally exit ${vallyExit}). Treating as failure.`
|
|
);
|
|
return 1;
|
|
}
|
|
|
|
if (verdict.passed) {
|
|
if (verdict.hadExecutionErrors) {
|
|
// Post-run teardown noise, not a mid-eval failure — verdict already passed; just log.
|
|
console.log(
|
|
`Shard '${shardName}' passed the pass-rate threshold; vally flagged execution errors (post-run teardown noise, not blocking).`
|
|
);
|
|
}
|
|
if (vallyExit !== 0) {
|
|
// vally's teardown can exit non-zero after the verdict is written — log, don't fail.
|
|
console.log(
|
|
`vally exited ${vallyExit} during post-run shutdown; shard '${shardName}' is PASSED per results.jsonl (exit code ignored).`
|
|
);
|
|
}
|
|
console.log(`##[section]Shard '${shardName}' PASSED (verdict from results.jsonl).`);
|
|
return 0;
|
|
}
|
|
|
|
console.log(
|
|
`##vso[task.logissue type=error]Shard '${shardName}' FAILED - one or more evals are below the pass-rate threshold.`
|
|
);
|
|
return 1;
|
|
}
|
|
|
|
// ----- CLI -----
|
|
|
|
function parseArgs(argv) {
|
|
const options = { threshold: 0.8 };
|
|
for (let i = 0; i < argv.length; i++) {
|
|
const next = () => argv[++i];
|
|
switch (argv[i]) {
|
|
case "--eval-args":
|
|
options.evalArgs = next();
|
|
break;
|
|
case "--shard-name":
|
|
options.shardName = next();
|
|
break;
|
|
case "--output-dir":
|
|
options.outputDir = next();
|
|
break;
|
|
case "--threshold":
|
|
options.threshold = Number(next());
|
|
break;
|
|
default:
|
|
throw new Error(`Unknown argument: ${argv[i]}`);
|
|
}
|
|
}
|
|
for (const required of ["evalArgs", "shardName", "outputDir"]) {
|
|
if (!options[required]) {
|
|
throw new Error(`Missing required argument for ${required}.`);
|
|
}
|
|
}
|
|
return options;
|
|
}
|
|
|
|
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
try {
|
|
process.exit(runShard(parseArgs(process.argv.slice(2))));
|
|
} catch (error) {
|
|
console.error(error.message);
|
|
process.exit(1);
|
|
}
|
|
}
|