mirror of
https://github.com/hanzoai/dev-desktop.git
synced 2026-11-20 01:07:23 +00:00
149 lines
5.1 KiB
TypeScript
149 lines
5.1 KiB
TypeScript
// @effect-diagnostics nodeBuiltinImport:off - Host-side video analysis shells out to ffmpeg.
|
|
|
|
import * as NodeChildProcess from "node:child_process";
|
|
import * as NodeFSP from "node:fs/promises";
|
|
|
|
import { check, type Check, type Rect } from "./checks.ts";
|
|
import { run } from "./device.ts";
|
|
|
|
/** Pixel width of an image or video, to map snapshot coordinates onto its frames. */
|
|
async function pixelWidth(path: string): Promise<number> {
|
|
const { stdout } = await run("ffprobe", [
|
|
"-v",
|
|
"error",
|
|
"-select_streams",
|
|
"v:0",
|
|
"-show_entries",
|
|
"stream=width",
|
|
"-of",
|
|
"csv=p=0",
|
|
path,
|
|
]);
|
|
return Number(stdout.trim());
|
|
}
|
|
|
|
/** Downsampled grayscale crops of one region, one per video frame (or one for a PNG). */
|
|
async function grayFrames(input: string, region: Rect, scale: number): Promise<Uint8Array[]> {
|
|
const even = (value: number) => Math.max(2, Math.floor(value / 2) * 2);
|
|
const [x, y, width, height] = [
|
|
Math.round(region.x * scale),
|
|
Math.round(region.y * scale),
|
|
even(region.width * scale),
|
|
even(region.height * scale),
|
|
];
|
|
const [outWidth, outHeight] = [even(region.width / 3), even(region.height / 3)];
|
|
const bytes = await new Promise<Buffer>((resolve, reject) => {
|
|
const child = NodeChildProcess.spawn(
|
|
"ffmpeg",
|
|
[
|
|
"-v",
|
|
"error",
|
|
"-i",
|
|
input,
|
|
// One output frame per recorded frame, so indices match select=eq(n,…) for evidence.
|
|
"-fps_mode",
|
|
"passthrough",
|
|
"-vf",
|
|
`crop=${width}:${height}:${x}:${y},scale=${outWidth}:${outHeight},format=gray`,
|
|
"-f",
|
|
"rawvideo",
|
|
"-",
|
|
],
|
|
{ stdio: ["ignore", "pipe", "pipe"] },
|
|
);
|
|
const chunks: Buffer[] = [];
|
|
let stderr = "";
|
|
child.stdout.on("data", (chunk: Buffer) => chunks.push(chunk));
|
|
child.stderr.on("data", (chunk) => (stderr += chunk));
|
|
child.on("error", reject);
|
|
child.on("close", (code) =>
|
|
code === 0 ? resolve(Buffer.concat(chunks)) : reject(new Error(`ffmpeg failed: ${stderr}`)),
|
|
);
|
|
});
|
|
const size = outWidth * outHeight;
|
|
return Array.from({ length: Math.floor(bytes.length / size) }, (_, index) =>
|
|
bytes.subarray(index * size, (index + 1) * size),
|
|
);
|
|
}
|
|
|
|
const distance = (a: Uint8Array, b: Uint8Array) => {
|
|
let sum = 0;
|
|
for (let index = 0; index < a.length; index++) sum += Math.abs(a[index]! - b[index]!);
|
|
return sum / a.length;
|
|
};
|
|
|
|
/** How far a region has moved from its origin toward its destination look, 0 to 1. */
|
|
export interface FrameProgress {
|
|
readonly nav: number;
|
|
readonly chrome: number;
|
|
}
|
|
|
|
/**
|
|
* A frame lags when the top of the screen is well into the transition while the tab bar
|
|
* region still looks like the origin: tabs lingering over an entering thread, or a returned
|
|
* list drawn before its tabs.
|
|
*/
|
|
const chromeLags = (frame: FrameProgress) => frame.nav > 0.3 && frame.chrome < 0.4;
|
|
// One lagging frame is capture jitter; a regression lasts several.
|
|
const LAG_FRAME_LIMIT = 2;
|
|
|
|
const progress = (origin: number, destination: number) =>
|
|
origin + destination === 0 ? 0 : origin / (origin + destination);
|
|
|
|
/**
|
|
* Checks that the tab bar moves with the screen during a recorded transition. The first and
|
|
* last frames are the before and after looks, so nothing is compared across capture sources.
|
|
*/
|
|
export async function lagCheck(
|
|
name: string,
|
|
input: {
|
|
readonly video: string;
|
|
readonly nav: Rect;
|
|
readonly chrome: Rect;
|
|
/** Snapshot units across the screen, to scale regions onto the video's pixels. */
|
|
readonly viewportWidth: number;
|
|
},
|
|
evidence: string,
|
|
): Promise<{ check: Check; frame?: string }> {
|
|
// A recording the device failed to export is a gap in evidence, not a lag.
|
|
if (!(await NodeFSP.stat(input.video).catch(() => null)))
|
|
return { check: check(name, true, "no recording; not checked") };
|
|
const scale = (await pixelWidth(input.video)) / input.viewportWidth;
|
|
const [navFrames, chromeFrames] = await Promise.all([
|
|
grayFrames(input.video, input.nav, scale),
|
|
grayFrames(input.video, input.chrome, scale),
|
|
]);
|
|
const [navOrigin, navDestination] = [navFrames[0], navFrames.at(-1)];
|
|
const [chromeOrigin, chromeDestination] = [chromeFrames[0], chromeFrames.at(-1)];
|
|
if (!navOrigin || !navDestination || !chromeOrigin || !chromeDestination)
|
|
throw new Error(`Could not read frames of ${input.video}.`);
|
|
const lagging = navFrames.flatMap((nav, index) => {
|
|
const frame = {
|
|
nav: progress(distance(nav, navOrigin), distance(nav, navDestination)),
|
|
chrome: progress(
|
|
distance(chromeFrames[index]!, chromeOrigin),
|
|
distance(chromeFrames[index]!, chromeDestination),
|
|
),
|
|
};
|
|
return chromeLags(frame) ? [index] : [];
|
|
});
|
|
if (lagging.length < LAG_FRAME_LIMIT)
|
|
return { check: check(name, true, `${lagging.length} lagging video frames`) };
|
|
await run("ffmpeg", [
|
|
"-v",
|
|
"error",
|
|
"-y",
|
|
"-i",
|
|
input.video,
|
|
"-vf",
|
|
`select=eq(n\\,${lagging[0]})`,
|
|
"-frames:v",
|
|
"1",
|
|
evidence,
|
|
]);
|
|
return {
|
|
check: check(name, false, `${lagging.length} video frames (first #${lagging[0]})`),
|
|
frame: evidence,
|
|
};
|
|
}
|