mirror of
https://github.com/hanzoai/frames.git
synced 2026-09-24 00:08:03 +00:00
feat(cli): add remove-background command for transparent video
Adds `hyperframes remove-background` — a local-AI subcommand that mattes a video or image with the u2net_human_seg ONNX model and emits a transparent WebM (VP9-alpha), ProRes 4444 .mov, or RGBA PNG. Drops directly into any composition's <video> tag — no green screen, no API keys, no upload. Auto-picks the fastest available execution provider via onnxruntime-node: CoreML on Apple Silicon, CUDA when HYPERFRAMES_CUDA=1, CPU otherwise. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
This commit is contained in:
James
@@ -21,7 +21,7 @@
|
||||
},
|
||||
"packages/cli": {
|
||||
"name": "@hyperframes/cli",
|
||||
"version": "0.4.31",
|
||||
"version": "0.4.42",
|
||||
"bin": {
|
||||
"hyperframes": "./dist/cli.js",
|
||||
},
|
||||
@@ -35,10 +35,12 @@
|
||||
"giget": "^3.2.0",
|
||||
"hono": "^4.0.0",
|
||||
"mime-types": "^3.0.2",
|
||||
"onnxruntime-node": "^1.20.0",
|
||||
"open": "^10.0.0",
|
||||
"postcss": "^8.5.8",
|
||||
"prettier": "^3.8.1",
|
||||
"puppeteer-core": "^24.39.1",
|
||||
"sharp": "^0.34.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@clack/prompts": "^1.1.0",
|
||||
@@ -62,7 +64,7 @@
|
||||
},
|
||||
"packages/core": {
|
||||
"name": "@hyperframes/core",
|
||||
"version": "0.4.31",
|
||||
"version": "0.4.42",
|
||||
"dependencies": {
|
||||
"@chenglou/pretext": "^0.0.5",
|
||||
"postcss": "^8.5.8",
|
||||
@@ -89,7 +91,7 @@
|
||||
},
|
||||
"packages/engine": {
|
||||
"name": "@hyperframes/engine",
|
||||
"version": "0.4.31",
|
||||
"version": "0.4.42",
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.13.0",
|
||||
"@hyperframes/core": "workspace:^",
|
||||
@@ -107,7 +109,7 @@
|
||||
},
|
||||
"packages/player": {
|
||||
"name": "@hyperframes/player",
|
||||
"version": "0.4.31",
|
||||
"version": "0.4.42",
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.1.0",
|
||||
"gsap": "^3.12.5",
|
||||
@@ -119,7 +121,7 @@
|
||||
},
|
||||
"packages/producer": {
|
||||
"name": "@hyperframes/producer",
|
||||
"version": "0.4.31",
|
||||
"version": "0.4.42",
|
||||
"dependencies": {
|
||||
"@fontsource/archivo-black": "^5.2.8",
|
||||
"@fontsource/eb-garamond": "^5.2.7",
|
||||
@@ -158,7 +160,7 @@
|
||||
},
|
||||
"packages/shader-transitions": {
|
||||
"name": "@hyperframes/shader-transitions",
|
||||
"version": "0.4.31",
|
||||
"version": "0.4.42",
|
||||
"dependencies": {
|
||||
"html2canvas": "^1.4.1",
|
||||
},
|
||||
@@ -170,7 +172,7 @@
|
||||
},
|
||||
"packages/studio": {
|
||||
"name": "@hyperframes/studio",
|
||||
"version": "0.4.31",
|
||||
"version": "0.4.42",
|
||||
"dependencies": {
|
||||
"@codemirror/autocomplete": "^6.20.1",
|
||||
"@codemirror/commands": "^6.10.3",
|
||||
@@ -454,6 +456,56 @@
|
||||
|
||||
"@hyperframes/studio": ["@hyperframes/studio@workspace:packages/studio"],
|
||||
|
||||
"@img/colour": ["@img/[email protected]", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="],
|
||||
|
||||
"@img/sharp-darwin-arm64": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="],
|
||||
|
||||
"@img/sharp-darwin-x64": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="],
|
||||
|
||||
"@img/sharp-libvips-darwin-arm64": ["@img/[email protected]", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="],
|
||||
|
||||
"@img/sharp-libvips-darwin-x64": ["@img/[email protected]", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="],
|
||||
|
||||
"@img/sharp-libvips-linux-arm": ["@img/[email protected]", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="],
|
||||
|
||||
"@img/sharp-libvips-linux-arm64": ["@img/[email protected]", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="],
|
||||
|
||||
"@img/sharp-libvips-linux-ppc64": ["@img/[email protected]", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="],
|
||||
|
||||
"@img/sharp-libvips-linux-riscv64": ["@img/[email protected]", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="],
|
||||
|
||||
"@img/sharp-libvips-linux-s390x": ["@img/[email protected]", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="],
|
||||
|
||||
"@img/sharp-libvips-linux-x64": ["@img/[email protected]", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="],
|
||||
|
||||
"@img/sharp-libvips-linuxmusl-arm64": ["@img/[email protected]", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="],
|
||||
|
||||
"@img/sharp-libvips-linuxmusl-x64": ["@img/[email protected]", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="],
|
||||
|
||||
"@img/sharp-linux-arm": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="],
|
||||
|
||||
"@img/sharp-linux-arm64": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="],
|
||||
|
||||
"@img/sharp-linux-ppc64": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="],
|
||||
|
||||
"@img/sharp-linux-riscv64": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="],
|
||||
|
||||
"@img/sharp-linux-s390x": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="],
|
||||
|
||||
"@img/sharp-linux-x64": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="],
|
||||
|
||||
"@img/sharp-linuxmusl-arm64": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="],
|
||||
|
||||
"@img/sharp-linuxmusl-x64": ["@img/[email protected]", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="],
|
||||
|
||||
"@img/sharp-wasm32": ["@img/[email protected]", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="],
|
||||
|
||||
"@img/sharp-win32-arm64": ["@img/[email protected]", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="],
|
||||
|
||||
"@img/sharp-win32-ia32": ["@img/[email protected]", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="],
|
||||
|
||||
"@img/sharp-win32-x64": ["@img/[email protected]", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="],
|
||||
|
||||
"@isaacs/cliui": ["@isaacs/[email protected]", "", { "dependencies": { "string-width": "5.1.2", "string-width-cjs": "npm:[email protected]", "strip-ansi": "7.2.0", "strip-ansi-cjs": "npm:[email protected]", "wrap-ansi": "8.1.0", "wrap-ansi-cjs": "npm:[email protected]" } }, "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA=="],
|
||||
|
||||
"@istanbuljs/schema": ["@istanbuljs/[email protected]", "", {}, "sha512-ZXRY4jNvVgSVQ8DL3LTcakaAtXwTVUxE81hslsyD2AtoXW/wVob10HkOJ1X/pAlcI7D+2YoZKg5do8G/w6RYgA=="],
|
||||
@@ -956,10 +1008,16 @@
|
||||
|
||||
"default-browser-id": ["[email protected]", "", {}, "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q=="],
|
||||
|
||||
"define-data-property": ["[email protected]", "", { "dependencies": { "es-define-property": "^1.0.0", "es-errors": "^1.3.0", "gopd": "^1.0.1" } }, "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A=="],
|
||||
|
||||
"define-lazy-prop": ["[email protected]", "", {}, "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg=="],
|
||||
|
||||
"define-properties": ["[email protected]", "", { "dependencies": { "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" } }, "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg=="],
|
||||
|
||||
"degenerator": ["[email protected]", "", { "dependencies": { "ast-types": "0.13.4", "escodegen": "2.1.0", "esprima": "4.0.1" } }, "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ=="],
|
||||
|
||||
"detect-libc": ["[email protected]", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="],
|
||||
|
||||
"devtools-protocol": ["[email protected]", "", {}, "sha512-nv7iKtNZQshSW2hKzYNr46nM/Cfh5SEvE2oV0/SEGgc9XupIY5ggf84Cz8eJIkBce7S3bmTAauFD6aysMpnqsQ=="],
|
||||
|
||||
"didyoumean": ["[email protected]", "", {}, "sha512-gxtyfqMg7GKyhQmb056K7M3xszy/myH8w+B4RT+QXBQsvAOdc3XymqDDPHx1BgPgsdAA5SIifona89YtRATDzw=="],
|
||||
@@ -992,12 +1050,18 @@
|
||||
|
||||
"error-ex": ["[email protected]", "", { "dependencies": { "is-arrayish": "0.2.1" } }, "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ=="],
|
||||
|
||||
"es-define-property": ["[email protected]", "", {}, "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g=="],
|
||||
|
||||
"es-errors": ["[email protected]", "", {}, "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw=="],
|
||||
|
||||
"es-module-lexer": ["[email protected]", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="],
|
||||
|
||||
"esbuild": ["[email protected]", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="],
|
||||
|
||||
"escalade": ["[email protected]", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="],
|
||||
|
||||
"escape-string-regexp": ["[email protected]", "", {}, "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA=="],
|
||||
|
||||
"escodegen": ["[email protected]", "", { "dependencies": { "esprima": "4.0.1", "estraverse": "5.3.0", "esutils": "2.0.3" }, "optionalDependencies": { "source-map": "0.6.1" }, "bin": { "esgenerate": "bin/esgenerate.js", "escodegen": "bin/escodegen.js" } }, "sha512-2NlIDTwUWJN0mRPQOdtQBzbUHvdGY2P1VXSyU83Q3xKxM7WHX2Ql8dKq782Q9TgQUNOLEzEYu9bzLNj1q88I5w=="],
|
||||
|
||||
"esprima": ["[email protected]", "", { "bin": { "esparse": "./bin/esparse.js", "esvalidate": "./bin/esvalidate.js" } }, "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A=="],
|
||||
@@ -1074,18 +1138,26 @@
|
||||
|
||||
"glob-parent": ["[email protected]", "", { "dependencies": { "is-glob": "4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="],
|
||||
|
||||
"global-agent": ["[email protected]", "", { "dependencies": { "globalthis": "^1.0.2", "matcher": "^4.0.0", "semver": "^7.3.5", "serialize-error": "^8.1.0" } }, "sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g=="],
|
||||
|
||||
"global-directory": ["[email protected]", "", { "dependencies": { "ini": "4.1.1" } }, "sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q=="],
|
||||
|
||||
"globalthis": ["[email protected]", "", { "dependencies": { "define-properties": "^1.2.1", "gopd": "^1.0.1" } }, "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ=="],
|
||||
|
||||
"google-auth-library": ["[email protected]", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.1.4", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-e27Z6EThmVNNvtYASwQxose/G57rkRuaRbQyxM2bvYLLX/GqWZ5chWq2EBoUchJbCc57eC9ArzO5wMsEmWftCw=="],
|
||||
|
||||
"google-logging-utils": ["[email protected]", "", {}, "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA=="],
|
||||
|
||||
"gopd": ["[email protected]", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="],
|
||||
|
||||
"gsap": ["[email protected]", "", {}, "sha512-dMW4CWBTUK1AEEDeZc1g4xpPGIrSf9fJF960qbTZmN/QwZIWY5wgliS6JWl9/25fpTGJrMRtSjGtOmPnfjZB+A=="],
|
||||
|
||||
"happy-dom": ["[email protected]", "", { "dependencies": { "@types/node": ">=20.0.0", "@types/whatwg-mimetype": "^3.0.2", "@types/ws": "^8.18.1", "entities": "^7.0.1", "whatwg-mimetype": "^3.0.0", "ws": "^8.18.3" } }, "sha512-GZZ9mKe8r646NUAf/zemnGbjYh4Bt8/MqASJY+pSm5ZDtc3YQox+4gsLI7yi1hba6o+eCsGxpHn5+iEVn31/FQ=="],
|
||||
|
||||
"has-flag": ["[email protected]", "", {}, "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ=="],
|
||||
|
||||
"has-property-descriptors": ["[email protected]", "", { "dependencies": { "es-define-property": "^1.0.0" } }, "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg=="],
|
||||
|
||||
"hasown": ["[email protected]", "", { "dependencies": { "function-bind": "1.1.2" } }, "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ=="],
|
||||
|
||||
"hono": ["[email protected]", "", {}, "sha512-VJCEvtrezO1IAR+kqEYnxUOoStaQPGrCmX3j4wDTNOcD1uRPFpGlwQUIW8niPuvHXaTUxeOUl5MMDGrl+tmO9A=="],
|
||||
@@ -1232,6 +1304,8 @@
|
||||
|
||||
"make-dir": ["[email protected]", "", { "dependencies": { "semver": "7.7.4" } }, "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw=="],
|
||||
|
||||
"matcher": ["[email protected]", "", { "dependencies": { "escape-string-regexp": "^4.0.0" } }, "sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ=="],
|
||||
|
||||
"mdn-data": ["[email protected]", "", {}, "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ=="],
|
||||
|
||||
"meow": ["[email protected]", "", {}, "sha512-pxQJQzB6djGPXh08dacEloMFopsOqGVRKFPYvPOt9XDZ1HasbgDZA74CJGreSU4G3Ak7EFJGoiH2auq+yXISgA=="],
|
||||
@@ -1282,8 +1356,14 @@
|
||||
|
||||
"object-hash": ["[email protected]", "", {}, "sha512-RSn9F68PjH9HqtltsSnqYC1XXoWe9Bju5+213R98cNGttag9q9yAOTzdbsqvIa7aNm5WffBZFpWYr2aWrklWAw=="],
|
||||
|
||||
"object-keys": ["[email protected]", "", {}, "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA=="],
|
||||
|
||||
"once": ["[email protected]", "", { "dependencies": { "wrappy": "1.0.2" } }, "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w=="],
|
||||
|
||||
"onnxruntime-common": ["[email protected]", "", {}, "sha512-kKvYQFdos4LWJqhZ+nmKu3NT8NXzw8I5x9fNUKe1rNKcPfNKnYXUtW7JBpcKFsvLtrJashRgVYSbFap4cHxvNg=="],
|
||||
|
||||
"onnxruntime-node": ["[email protected]", "", { "dependencies": { "adm-zip": "^0.5.16", "global-agent": "^4.1.3", "onnxruntime-common": "1.25.1" }, "os": [ "linux", "win32", "darwin", ] }, "sha512-N0M58CGTiTsLkPpx9bxmRFi24GT6r67Qei/GrBEIiDyntcYdXU5vQZp112ypydG9vEKRFgbgUYQJnEi+jll8dg=="],
|
||||
|
||||
"open": ["[email protected]", "", { "dependencies": { "default-browser": "5.5.0", "define-lazy-prop": "3.0.0", "is-inside-container": "1.0.0", "wsl-utils": "0.1.0" } }, "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA=="],
|
||||
|
||||
"oxc-parser": ["[email protected]", "", { "dependencies": { "@oxc-project/types": "0.120.0" }, "optionalDependencies": { "@oxc-parser/binding-android-arm-eabi": "0.120.0", "@oxc-parser/binding-android-arm64": "0.120.0", "@oxc-parser/binding-darwin-arm64": "0.120.0", "@oxc-parser/binding-darwin-x64": "0.120.0", "@oxc-parser/binding-freebsd-x64": "0.120.0", "@oxc-parser/binding-linux-arm-gnueabihf": "0.120.0", "@oxc-parser/binding-linux-arm-musleabihf": "0.120.0", "@oxc-parser/binding-linux-arm64-gnu": "0.120.0", "@oxc-parser/binding-linux-arm64-musl": "0.120.0", "@oxc-parser/binding-linux-ppc64-gnu": "0.120.0", "@oxc-parser/binding-linux-riscv64-gnu": "0.120.0", "@oxc-parser/binding-linux-riscv64-musl": "0.120.0", "@oxc-parser/binding-linux-s390x-gnu": "0.120.0", "@oxc-parser/binding-linux-x64-gnu": "0.120.0", "@oxc-parser/binding-linux-x64-musl": "0.120.0", "@oxc-parser/binding-openharmony-arm64": "0.120.0", "@oxc-parser/binding-wasm32-wasi": "0.120.0", "@oxc-parser/binding-win32-arm64-msvc": "0.120.0", "@oxc-parser/binding-win32-ia32-msvc": "0.120.0", "@oxc-parser/binding-win32-x64-msvc": "0.120.0" } }, "sha512-WyPWZlcIm+Fkte63FGfgFB8mAAk33aH9h5N9lphXVOHSXEBFFsmYdOBedVKly363aWABjZdaj/m9lBfEY4wt+w=="],
|
||||
@@ -1404,6 +1484,10 @@
|
||||
|
||||
"semver": ["[email protected]", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="],
|
||||
|
||||
"serialize-error": ["[email protected]", "", { "dependencies": { "type-fest": "^0.20.2" } }, "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ=="],
|
||||
|
||||
"sharp": ["[email protected]", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="],
|
||||
|
||||
"shebang-command": ["[email protected]", "", { "dependencies": { "shebang-regex": "3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
|
||||
|
||||
"shebang-regex": ["[email protected]", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
|
||||
@@ -1508,6 +1592,8 @@
|
||||
|
||||
"tsx": ["[email protected]", "", { "dependencies": { "esbuild": "0.27.4", "get-tsconfig": "4.13.6" }, "optionalDependencies": { "fsevents": "2.3.3" }, "bin": { "tsx": "dist/cli.mjs" } }, "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw=="],
|
||||
|
||||
"type-fest": ["[email protected]", "", {}, "sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ=="],
|
||||
|
||||
"typed-query-selector": ["[email protected]", "", {}, "sha512-uzR+FzI8qrUEIu96oaeBJmd9E7CFEiQ3goA5qCVgc4s5llSubcfGHq9yUstZx/k4s9dXHVKsE35YWoFyvEqEHA=="],
|
||||
|
||||
"typescript": ["[email protected]", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
@@ -76,6 +76,7 @@
|
||||
"guides/hyperframes-vs-remotion",
|
||||
"guides/gsap-animation",
|
||||
"guides/rendering",
|
||||
"guides/remove-background",
|
||||
"guides/hdr",
|
||||
"guides/performance",
|
||||
"guides/timeline-editing",
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
---
|
||||
title: Remove Background (transparent video)
|
||||
description: "Remove the background from a video or image and drop it into any composition as a transparent overlay."
|
||||
---
|
||||
|
||||
Background removal — also called *matting* in VFX — separates a foreground subject (typically a person) from its background. The output is a video with an alpha channel: fully transparent where the background was, opaque where the subject is. Drop it into any HyperFrames composition as a `<video>` tag and the subject floats over whatever you put behind them.
|
||||
|
||||
The CLI ships a built-in `remove-background` command that runs locally — no API keys, no cloud upload, no green screen.
|
||||
|
||||
## Quick Start
|
||||
|
||||
<Steps>
|
||||
<Step title="Verify ffmpeg is installed">
|
||||
The pipeline needs `ffmpeg` and `ffprobe` for decode + encode. Most systems already have them; if not:
|
||||
|
||||
```bash Terminal
|
||||
# macOS
|
||||
brew install ffmpeg
|
||||
|
||||
# Ubuntu / Debian
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
|
||||
Confirm with `npx hyperframes doctor` — both should be green.
|
||||
</Step>
|
||||
<Step title="Remove the background from your video">
|
||||
```bash Terminal
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.webm
|
||||
```
|
||||
|
||||
On the first run, the CLI downloads ~168 MB of model weights to `~/.cache/hyperframes/background-removal/models/`. Subsequent runs reuse the cache.
|
||||
|
||||
Output:
|
||||
|
||||
```
|
||||
◇ Removed background from 240 frames in 38.4s (6.3 fps, CoreML) → ./transparent.webm
|
||||
```
|
||||
</Step>
|
||||
<Step title="Drop it into a composition">
|
||||
The output is a standard VP9-with-alpha WebM. Chrome's `<video>` element decodes the alpha plane natively — no special player needed:
|
||||
|
||||
```html composition.html
|
||||
<div class="scene">
|
||||
<!-- background layer -->
|
||||
<img src="city.jpg" class="bg" />
|
||||
|
||||
<!-- transparent avatar floats on top -->
|
||||
<video src="transparent.webm" autoplay muted loop playsinline></video>
|
||||
</div>
|
||||
```
|
||||
|
||||
Render the composition with the usual `hyperframes render`.
|
||||
</Step>
|
||||
</Steps>
|
||||
|
||||
## How it works
|
||||
|
||||
The pipeline runs four stages, all locally:
|
||||
|
||||
```
|
||||
ffmpeg decode → u²-net_human_seg inference → alpha composite → ffmpeg encode
|
||||
(raw RGB) (320×320 mask, then upsampled) (VP9-alpha)
|
||||
```
|
||||
|
||||
The model is **u²-net_human_seg** (MIT license, ~168 MB ONNX). It runs through `onnxruntime-node` with the best-available execution provider on your machine: CoreML on Apple Silicon, CUDA on NVIDIA, CPU otherwise.
|
||||
|
||||
The output is encoded with the exact ffmpeg flags Chrome's `<video>` element needs to decode alpha — `-pix_fmt yuva420p` plus the `alpha_mode=1` metadata tag. Get those wrong and the alpha plane is silently discarded by browsers.
|
||||
|
||||
## Output formats
|
||||
|
||||
| Extension | Codec | When to use | Size (4s @ 1080p) |
|
||||
|-----------|-------|-------------|-------------------|
|
||||
| `.webm` (default) | VP9 with alpha | Drop into `<video>` for HTML5-native transparent playback | ~1 MB |
|
||||
| `.mov` | ProRes 4444 with alpha | Editing round-trip in Premiere / Resolve / Final Cut | ~50 MB |
|
||||
| `.png` | PNG with alpha | Single-image cutout (only when the input is also a single image) | varies |
|
||||
|
||||
```bash Terminal
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.webm # web playback
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.mov # editing
|
||||
npx hyperframes remove-background portrait.jpg -o cutout.png # still image
|
||||
```
|
||||
|
||||
## Performance
|
||||
|
||||
Real-world numbers from the [matting eval](https://www.heygenverse.com/a/0dd5a431-1832-4858-862d-de7fb7d02654), running u²-net_human_seg on a 4-second 1080p clip:
|
||||
|
||||
| Platform | Provider | ms/frame | 30-second clip |
|
||||
|----------|----------|----------|----------------|
|
||||
| Apple Silicon (M2 Pro / M3 / M4) | CoreML | ~263 | ~2 min |
|
||||
| NVIDIA GPU (T4, A10, RTX) | CUDA | ~80–150 | ~30–60 s |
|
||||
| Linux x86 | CPU | ~1100 | ~16 min |
|
||||
| macOS Intel | CPU | ~900 | ~13 min |
|
||||
|
||||
Matting is offline preprocessing — you run it once per asset and reuse the output. CPU-only is slow but always works; if you reuse the same avatar repeatedly, run it once on a faster machine and check the transparent output into your project.
|
||||
|
||||
## Picking a device explicitly
|
||||
|
||||
`--device auto` is the default and right for almost everyone. The flag exists for two cases:
|
||||
|
||||
- **Force CPU on a GPU box** when you want to keep the GPU free for other work, or are debugging an EP-specific issue:
|
||||
|
||||
```bash Terminal
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu
|
||||
```
|
||||
|
||||
- **Opt into CUDA** by setting `HYPERFRAMES_CUDA=1` and providing a GPU-enabled `onnxruntime-node` build (the bundled build is CPU + CoreML only, to keep the install small for the 99% of users who don't have a GPU):
|
||||
|
||||
```bash Terminal
|
||||
HYPERFRAMES_CUDA=1 npx hyperframes remove-background avatar.mp4 -o transparent.webm --device cuda
|
||||
```
|
||||
|
||||
Run `npx hyperframes remove-background --info` to see what providers are detected on your machine and which one `auto` would pick.
|
||||
|
||||
## Using the transparent video in a composition
|
||||
|
||||
The transparent WebM behaves like any other video element. The two patterns you'll use most:
|
||||
|
||||
**Avatar over a background image:**
|
||||
|
||||
```html
|
||||
<div style="position: relative; width: 1920px; height: 1080px;">
|
||||
<img src="background.jpg" style="position: absolute; inset: 0;" />
|
||||
<video
|
||||
src="transparent.webm"
|
||||
autoplay
|
||||
muted
|
||||
loop
|
||||
playsinline
|
||||
style="position: absolute; right: 80px; bottom: 0; height: 90%;"
|
||||
></video>
|
||||
</div>
|
||||
```
|
||||
|
||||
**Avatar over a HyperFrames scene:**
|
||||
|
||||
```html
|
||||
<!-- scene contents (text, animations, etc.) -->
|
||||
<div class="title-card">Welcome</div>
|
||||
|
||||
<!-- avatar layered on top -->
|
||||
<video src="transparent.webm" autoplay muted loop playsinline class="avatar"></video>
|
||||
```
|
||||
|
||||
The avatar inherits the composition's frame rate and timeline — it plays through once during the scene's duration, so match the source clip length to the scene length when possible. If the scene is longer than the clip, `loop` handles it.
|
||||
|
||||
<Tip>
|
||||
When rendering a composition that contains a `<video>` element, the renderer reads the source via ffmpeg internally. Transparent WebMs are decoded with the alpha plane preserved.
|
||||
</Tip>
|
||||
|
||||
## What u²-net_human_seg is and isn't good for
|
||||
|
||||
The model is purpose-built for **portrait / human matting**. It excels when:
|
||||
|
||||
- ✅ The subject is a person, head-and-shoulders or full-body
|
||||
- ✅ The framing is reasonably stable (not a wide handheld shot)
|
||||
- ✅ The background contrasts with the subject
|
||||
|
||||
It struggles or fails on:
|
||||
|
||||
- ❌ Non-human subjects (products, animals, objects). The model will return a mostly-empty mask.
|
||||
- ❌ Very fine hair detail on a busy background. The 320×320 inference resolution means hair tips get softened — fine for most use cases, but compositors notice.
|
||||
- ❌ Frame-to-frame temporal consistency. Each frame is processed independently, so static backgrounds with moving subjects can show subtle edge flicker. For most web playback this is invisible; for high-end VFX it may matter.
|
||||
- ❌ Live streams or real-time capture. The pipeline is batch-only.
|
||||
|
||||
If your use case hits one of these, see the alternatives below.
|
||||
|
||||
## Alternatives — when the built-in command isn't the right tool
|
||||
|
||||
The CLI ships **one model on purpose** — the one that's MIT-licensed, runs everywhere, and produces production-quality output for HeyGen-style avatar workflows. The list below leads with **free, open-source tools** that pair naturally with HyperFrames. Each entry calls out the actual catch — license, install effort, hardware needs — so you can pick the right one for your situation. Full benchmarks are in the [matting eval](https://www.heygenverse.com/a/0dd5a431-1832-4858-862d-de7fb7d02654).
|
||||
|
||||
### Free, open-source CLIs and libraries
|
||||
|
||||
These all run locally with no account, no upload, no watermark.
|
||||
|
||||
| Tool | When to use it | Catch |
|
||||
|------|----------------|-------|
|
||||
| [`rembg`](https://github.com/danielgatis/rembg) (Python, MIT) | You need a different subject type — `isnet-general-use` for objects/animals/products, `birefnet-portrait` for a quality ceiling on hair, `silueta` for a tiny ~40 MB footprint. Same family as our default model, more variety. | Requires Python + `pip install rembg`. Some bundled models (`birefnet-*`) need ~4 GB RAM and are CPU-only |
|
||||
| [BiRefNet](https://github.com/ZhengPeng7/BiRefNet) (PyTorch, MIT) | Highest-fidelity portrait mattes available — visibly better hair edges than u²-net | Heavy (~4 GB inference RAM), slow on CPU, broken on Apple CoreML at the time of the eval |
|
||||
| [Robust Video Matting (RVM)](https://github.com/PeterL1n/RobustVideoMatting) (PyTorch, **GPL-3.0**) | The only widely-available model with **temporal consistency** built in — no edge flicker on moving subjects. Best choice when you're matting a long talking-head clip and frame-to-frame stability matters | GPL-3.0 license is incompatible with most commercial / proprietary codebases. Read your repo's license before using |
|
||||
| [Backgroundremover](https://github.com/nadermx/backgroundremover) (Python, MIT) | Simple `pip install` wrapper around u²-net; nice if you want a Python API instead of our Node CLI | Same model family as ours, no quality difference — pick whichever fits your stack |
|
||||
| [ComfyUI](https://github.com/comfyanonymous/ComfyUI) (open-source, GPL-3.0 core) | Custom workflows: chain a segmentation model + alpha refinement + temporal smoothing. The right tool for tricky cases (multiple subjects, hair against a similar background, sports footage) | Setup is involved (Python, models, node graph). Worth it for repeat specialty work |
|
||||
|
||||
After running any of these externally, encode the output as a HyperFrames-compatible transparent WebM with:
|
||||
|
||||
```bash Terminal
|
||||
ffmpeg -i frames-%04d.png -c:v libvpx-vp9 \
|
||||
-pix_fmt yuva420p \
|
||||
-metadata:s:v:0 alpha_mode=1 \
|
||||
-auto-alt-ref 0 -b:v 0 -crf 30 \
|
||||
transparent.webm
|
||||
```
|
||||
|
||||
### Free desktop / GUI tools
|
||||
|
||||
| Tool | When to use it | Catch |
|
||||
|------|----------------|-------|
|
||||
| [DaVinci Resolve — Magic Mask](https://www.blackmagicdesign.com/products/davinciresolve) | You're already editing in Resolve, want a brush-based UI with manual refinement, and need to round-trip the alpha into a larger edit | macOS / Windows / Linux desktop install. The free tier covers Magic Mask; paid Studio version unlocks higher resolutions on some features |
|
||||
| [Backgroundremover.app](https://backgroundremover.app) (web) | One-off image cutout, no signup, no watermark | Single images only, not video. Free tier is hosted but the underlying tool is the same `rembg` model family |
|
||||
| [PhotoRoom Background Remover](https://www.photoroom.com/tools/background-remover) (web) | Quick one-off image, polished UI, no signup | Single images only, e-commerce-tuned model |
|
||||
|
||||
### Web SaaS tools (free tiers, with strings)
|
||||
|
||||
| Tool | When to use it | Catch |
|
||||
|------|----------------|-------|
|
||||
| [unscreen.com](https://www.unscreen.com) | Quick one-off video, no install, drag-and-drop | **Free tier is watermarked and capped at short clips** (~10s preview). Paid removes both. Run by the team behind remove.bg |
|
||||
| [RunwayML — Green Screen](https://runwayml.com) | Polished UI with brush refinement and time-aware tracking; the closest a SaaS gets to professional roto | Free tier exists but is credit-limited; serious use is a subscription |
|
||||
| [Kapwing — Background Remover](https://www.kapwing.com/tools/remove-video-background) | Browser-based, integrates with their video editor | Free tier is watermarked; paid removes it |
|
||||
|
||||
### How to choose
|
||||
|
||||
- **Avatars / portraits, web playback, MIT-clean** → use the built-in `hyperframes remove-background` (this is what it's tuned for).
|
||||
- **Non-human subject** (product, animal, object) → `rembg` with `isnet-general-use`.
|
||||
- **Maximum portrait quality, especially hair** → `BiRefNet` via Python.
|
||||
- **Long video where edge flicker would be visible**, GPL is OK → `RVM`.
|
||||
- **One-off marketing clip, no install** → DaVinci Resolve (free) for video, Backgroundremover.app for a still image.
|
||||
- **Specialty case the off-the-shelf models can't handle** → ComfyUI with a custom graph.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Model download fails or hangs
|
||||
|
||||
The weights live on GitHub Releases (rembg's `v0.0.0` release, ~168 MB). If your network blocks GitHub or the download is interrupted:
|
||||
|
||||
```bash Terminal
|
||||
# Manually download and drop into the cache
|
||||
mkdir -p ~/.cache/hyperframes/background-removal/models
|
||||
curl -L -o ~/.cache/hyperframes/background-removal/models/u2net_human_seg.onnx \
|
||||
https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2net_human_seg.onnx
|
||||
```
|
||||
|
||||
Subsequent `remove-background` runs skip the download and use your local copy.
|
||||
|
||||
### "ffmpeg and ffprobe are required"
|
||||
|
||||
The pipeline shells out to ffmpeg for decode + encode. Install via `brew install ffmpeg` on macOS or `sudo apt install ffmpeg` on Debian/Ubuntu. Verify with `npx hyperframes doctor`.
|
||||
|
||||
### The output WebM looks fully opaque in the browser
|
||||
|
||||
Chrome only reads the alpha plane when the WebM is encoded as `yuva420p` with the `alpha_mode=1` metadata tag. The CLI sets both. If you re-encode the output yourself (e.g. with another ffmpeg invocation), preserve those flags:
|
||||
|
||||
```bash Terminal
|
||||
ffmpeg -i in.webm -c:v libvpx-vp9 \
|
||||
-pix_fmt yuva420p \
|
||||
-metadata:s:v:0 alpha_mode=1 \
|
||||
-auto-alt-ref 0 \
|
||||
out.webm
|
||||
```
|
||||
|
||||
To verify a WebM has alpha, extract the first frame and inspect:
|
||||
|
||||
```bash Terminal
|
||||
ffmpeg -y -c:v libvpx-vp9 -i out.webm -frames:v 1 -pix_fmt rgba -update 1 frame0.png
|
||||
```
|
||||
|
||||
The decoded `frame0.png` should be RGBA and have non-trivial alpha values.
|
||||
|
||||
### CoreML is "available" but inference fails to start
|
||||
|
||||
The pipeline auto-falls-back to CPU if CoreML fails to bind, with a warning. If you want to skip the CoreML attempt entirely, force CPU:
|
||||
|
||||
```bash Terminal
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu
|
||||
```
|
||||
|
||||
### The alpha mask has rough or jagged edges
|
||||
|
||||
That usually means the source frame is high-contrast against a similar-toned background and the model's 320×320 inference resolution is showing through. Two paths forward:
|
||||
|
||||
1. Re-frame or re-shoot to give the subject a more contrasting background.
|
||||
2. Try `birefnet-portrait` via `rembg` (see [Other open-source models](#other-open-source-models)) — it's higher quality at hair edges but slower and heavier.
|
||||
|
||||
## Reference
|
||||
|
||||
- CLI: [`hyperframes remove-background`](/packages/cli#remove-background)
|
||||
- Eval: [Matting eval — v7](https://www.heygenverse.com/a/0dd5a431-1832-4858-862d-de7fb7d02654)
|
||||
- Source model: [danielgatis/rembg](https://github.com/danielgatis/rembg)
|
||||
- ONNX runtime: [`onnxruntime-node`](https://www.npmjs.com/package/onnxruntime-node)
|
||||
@@ -341,6 +341,53 @@ This is suppressed in CI environments, non-TTY shells, and when `HYPERFRAMES_NO_
|
||||
<Tip>
|
||||
Combine `tts` with `transcribe` to generate narration and word-level timestamps for captions in a single workflow: generate the audio with `tts`, then transcribe the output with `transcribe` to get word-level timing.
|
||||
</Tip>
|
||||
|
||||
### `remove-background`
|
||||
|
||||
Remove the background from a video or image using a local AI model. The output is transparent media you can drop into any composition's `<video>` or `<img>` element — no green screen required.
|
||||
|
||||
```bash
|
||||
# Default: VP9-with-alpha WebM (HTML5-native, ~1 MB / 4s @ 1080p)
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.webm
|
||||
|
||||
# ProRes 4444 .mov for editing round-trip
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.mov
|
||||
|
||||
# Single image → transparent PNG
|
||||
npx hyperframes remove-background portrait.jpg -o cutout.png
|
||||
|
||||
# Force CPU on a machine that has CoreML or CUDA
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu
|
||||
|
||||
# Inspect detected providers without rendering
|
||||
npx hyperframes remove-background --info
|
||||
```
|
||||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `--output, -o` | Output path. Format inferred from extension: `.webm` (default), `.mov`, `.png` |
|
||||
| `--device` | Execution provider: `auto` (default), `cpu`, `coreml`, `cuda` |
|
||||
| `--info` | Print detected execution providers and exit (no render) |
|
||||
| `--json` | Output result as JSON |
|
||||
|
||||
The model is `u2net_human_seg` (MIT, ~168 MB ONNX). Weights download to `~/.cache/hyperframes/background-removal/models/` on first run and are reused thereafter. Peak inference RAM is ~1.5 GB.
|
||||
|
||||
`--device auto` picks CoreML on Apple Silicon, CUDA when available, and CPU otherwise. The CLI bundles the CPU build of `onnxruntime-node`; for CUDA, set `HYPERFRAMES_CUDA=1` and provide a GPU-enabled `onnxruntime-node` build.
|
||||
|
||||
Output formats:
|
||||
|
||||
| Format | Use case | Size (4s @ 1080p) |
|
||||
|--------|----------|-------------------|
|
||||
| `.webm` (VP9 alpha) | Drop into `<video>` for HTML5-native transparent playback | ~1 MB |
|
||||
| `.mov` (ProRes 4444) | Editing round-trip in Premiere / Resolve / DaVinci | ~50 MB |
|
||||
| `.png` | Single-image cutout | varies |
|
||||
|
||||
<Tip>
|
||||
The `<video>` element in Chrome only respects the alpha plane when the WebM is encoded as `yuva420p` with the `alpha_mode=1` metadata tag. The CLI sets both automatically — if you re-encode the output yourself, preserve those flags.
|
||||
</Tip>
|
||||
|
||||
See the [Remove Background guide](/guides/remove-background) for the full workflow — using transparent videos in compositions, performance per platform, limitations of `u2net_human_seg`, and free alternative tools when this model isn't the right fit.
|
||||
|
||||
### `capture`
|
||||
|
||||
Capture a website — extract screenshots, design tokens, fonts, assets, and animations for video production:
|
||||
|
||||
@@ -33,10 +33,12 @@
|
||||
"giget": "^3.2.0",
|
||||
"hono": "^4.0.0",
|
||||
"mime-types": "^3.0.2",
|
||||
"onnxruntime-node": "^1.20.0",
|
||||
"open": "^10.0.0",
|
||||
"postcss": "^8.5.8",
|
||||
"prettier": "^3.8.1",
|
||||
"puppeteer-core": "^24.39.1"
|
||||
"puppeteer-core": "^24.39.1",
|
||||
"sharp": "^0.34.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@clack/prompts": "^1.1.0",
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
/**
|
||||
* u2net_human_seg inference: RGB frame → RGBA frame (alpha = human mask).
|
||||
*
|
||||
* Pre/postprocessing matches rembg's u2net session
|
||||
* (https://github.com/danielgatis/rembg/blob/main/rembg/sessions/u2net.py)
|
||||
* so output should be pixel-equivalent to `rembg new_session("u2net_human_seg")`.
|
||||
*/
|
||||
import type { InferenceSession, Tensor } from "onnxruntime-node";
|
||||
import type sharpType from "sharp";
|
||||
import { ensureModel, selectProviders, type Device, type ModelId } from "./manager.js";
|
||||
|
||||
const INPUT_SIZE = 320;
|
||||
const INPUT_PLANE = INPUT_SIZE * INPUT_SIZE;
|
||||
const MEAN = [0.485, 0.456, 0.406] as const;
|
||||
const STD = [1.0, 1.0, 1.0] as const;
|
||||
|
||||
type Sharp = typeof sharpType;
|
||||
interface OrtModule {
|
||||
InferenceSession: typeof InferenceSession;
|
||||
Tensor: typeof Tensor;
|
||||
}
|
||||
|
||||
export interface Session {
|
||||
/** Run inference on one RGB frame, return RGBA bytes (H*W*4). */
|
||||
process(rgb: Buffer, width: number, height: number): Promise<Buffer>;
|
||||
/** ORT EP that was actually selected. */
|
||||
provider: string;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export interface CreateSessionOptions {
|
||||
model?: ModelId;
|
||||
device?: Device;
|
||||
onProgress?: (message: string) => void;
|
||||
}
|
||||
|
||||
export async function createSession(options: CreateSessionOptions = {}): Promise<Session> {
|
||||
const ort = (await import("onnxruntime-node")) as unknown as OrtModule;
|
||||
const sharpMod = await import("sharp");
|
||||
const sharp = sharpMod.default as Sharp;
|
||||
|
||||
const choice = selectProviders(options.device ?? "auto");
|
||||
const path = await ensureModel(options.model, { onProgress: options.onProgress });
|
||||
|
||||
options.onProgress?.(`Loading model on ${choice.label}...`);
|
||||
|
||||
const tryCreate = (providers: string[]) =>
|
||||
ort.InferenceSession.create(path, {
|
||||
executionProviders: providers,
|
||||
graphOptimizationLevel: "all",
|
||||
});
|
||||
|
||||
let session: InferenceSession;
|
||||
let providerUsed = choice.label;
|
||||
try {
|
||||
session = await tryCreate(choice.providers);
|
||||
} catch (err) {
|
||||
if (choice.providers[0] === "cpu") throw err;
|
||||
options.onProgress?.(
|
||||
`${choice.label} provider failed (${(err as Error).message}); falling back to CPU.`,
|
||||
);
|
||||
session = await tryCreate(["cpu"]);
|
||||
providerUsed = "CPU";
|
||||
}
|
||||
|
||||
const inputName = session.inputNames[0];
|
||||
const outputName = session.outputNames[0];
|
||||
if (!inputName || !outputName) {
|
||||
throw new Error("ONNX session is missing input or output bindings");
|
||||
}
|
||||
|
||||
// Pre-allocated per-frame buffers reused across every process() call.
|
||||
// At 1080p this saves ~9 MB of allocations per frame. rgbaBuf is sized
|
||||
// lazily on the first call (we don't know W/H until then).
|
||||
const inputData = new Float32Array(3 * INPUT_PLANE);
|
||||
const maskBuf = Buffer.allocUnsafe(INPUT_PLANE);
|
||||
let rgbaBuf: Buffer | null = null;
|
||||
|
||||
return {
|
||||
provider: providerUsed,
|
||||
async process(rgb, width, height) {
|
||||
const tensor = await preprocess(sharp, ort, rgb, width, height, inputData);
|
||||
const outputs = await session.run({ [inputName]: tensor });
|
||||
const output = outputs[outputName];
|
||||
if (!output) throw new Error(`Model did not return output '${outputName}'`);
|
||||
const expectedBytes = width * height * 4;
|
||||
if (!rgbaBuf || rgbaBuf.length !== expectedBytes) {
|
||||
rgbaBuf = Buffer.allocUnsafe(expectedBytes);
|
||||
}
|
||||
return await postprocess(sharp, output, rgb, width, height, maskBuf, rgbaBuf);
|
||||
},
|
||||
async close() {
|
||||
await session.release();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function preprocess(
|
||||
sharp: Sharp,
|
||||
ort: OrtModule,
|
||||
rgb: Buffer,
|
||||
width: number,
|
||||
height: number,
|
||||
inputData: Float32Array,
|
||||
): Promise<Tensor> {
|
||||
const resized = await sharp(rgb, { raw: { width, height, channels: 3 } })
|
||||
.resize(INPUT_SIZE, INPUT_SIZE, { kernel: "lanczos3", fit: "fill" })
|
||||
.raw()
|
||||
.toBuffer();
|
||||
|
||||
// rembg's normalize divides by `np.max(im_ary)` (NOT 255). Match exactly so
|
||||
// we hit the same operating point as the model's training distribution.
|
||||
let maxPixel = 0;
|
||||
for (let i = 0; i < resized.length; i++) {
|
||||
if (resized[i]! > maxPixel) maxPixel = resized[i]!;
|
||||
}
|
||||
if (maxPixel === 0) maxPixel = 1;
|
||||
|
||||
for (let y = 0; y < INPUT_SIZE; y++) {
|
||||
for (let x = 0; x < INPUT_SIZE; x++) {
|
||||
const src = (y * INPUT_SIZE + x) * 3;
|
||||
const dst = y * INPUT_SIZE + x;
|
||||
inputData[dst] = (resized[src]! / maxPixel - MEAN[0]) / STD[0];
|
||||
inputData[INPUT_PLANE + dst] = (resized[src + 1]! / maxPixel - MEAN[1]) / STD[1];
|
||||
inputData[2 * INPUT_PLANE + dst] = (resized[src + 2]! / maxPixel - MEAN[2]) / STD[2];
|
||||
}
|
||||
}
|
||||
|
||||
return new ort.Tensor("float32", inputData, [1, 3, INPUT_SIZE, INPUT_SIZE]);
|
||||
}
|
||||
|
||||
async function postprocess(
|
||||
sharp: Sharp,
|
||||
output: Tensor,
|
||||
rgb: Buffer,
|
||||
width: number,
|
||||
height: number,
|
||||
maskBuf: Buffer,
|
||||
rgbaBuf: Buffer,
|
||||
): Promise<Buffer> {
|
||||
const raw = output.data as Float32Array;
|
||||
|
||||
let lo = Infinity;
|
||||
let hi = -Infinity;
|
||||
for (let i = 0; i < INPUT_PLANE; i++) {
|
||||
const v = raw[i]!;
|
||||
if (v < lo) lo = v;
|
||||
if (v > hi) hi = v;
|
||||
}
|
||||
const range = hi - lo || 1;
|
||||
|
||||
for (let i = 0; i < INPUT_PLANE; i++) {
|
||||
const norm = (raw[i]! - lo) / range;
|
||||
maskBuf[i] = Math.max(0, Math.min(255, Math.round(norm * 255)));
|
||||
}
|
||||
|
||||
// lanczos3 keeps soft edges; nearest leaves visible jaggies on hair.
|
||||
const fullMask = await sharp(maskBuf, {
|
||||
raw: { width: INPUT_SIZE, height: INPUT_SIZE, channels: 1 },
|
||||
})
|
||||
.resize(width, height, { kernel: "lanczos3", fit: "fill" })
|
||||
.raw()
|
||||
.toBuffer();
|
||||
|
||||
for (let i = 0; i < width * height; i++) {
|
||||
rgbaBuf[i * 4] = rgb[i * 3]!;
|
||||
rgbaBuf[i * 4 + 1] = rgb[i * 3 + 1]!;
|
||||
rgbaBuf[i * 4 + 2] = rgb[i * 3 + 2]!;
|
||||
rgbaBuf[i * 4 + 3] = fullMask[i]!;
|
||||
}
|
||||
return rgbaBuf;
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
describe("background-removal/manager — selectProviders", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
delete process.env["HYPERFRAMES_CUDA"];
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it("returns CPU explicitly when --device cpu", async () => {
|
||||
vi.doMock("node:os", () => ({
|
||||
platform: () => "darwin",
|
||||
arch: () => "arm64",
|
||||
homedir: () => "/tmp",
|
||||
}));
|
||||
const { selectProviders } = await import("./manager.js");
|
||||
const choice = selectProviders("cpu");
|
||||
expect(choice.providers).toEqual(["cpu"]);
|
||||
expect(choice.label).toBe("CPU");
|
||||
});
|
||||
|
||||
it("auto picks CoreML on darwin-arm64", async () => {
|
||||
vi.doMock("node:os", () => ({
|
||||
platform: () => "darwin",
|
||||
arch: () => "arm64",
|
||||
homedir: () => "/tmp",
|
||||
}));
|
||||
const { selectProviders } = await import("./manager.js");
|
||||
const choice = selectProviders("auto");
|
||||
expect(choice.providers).toEqual(["coreml", "cpu"]);
|
||||
expect(choice.label).toBe("CoreML");
|
||||
});
|
||||
|
||||
it("auto falls back to CPU on linux without HYPERFRAMES_CUDA", async () => {
|
||||
vi.doMock("node:os", () => ({
|
||||
platform: () => "linux",
|
||||
arch: () => "x64",
|
||||
homedir: () => "/tmp",
|
||||
}));
|
||||
const { selectProviders } = await import("./manager.js");
|
||||
const choice = selectProviders("auto");
|
||||
expect(choice.providers).toEqual(["cpu"]);
|
||||
expect(choice.label).toBe("CPU");
|
||||
});
|
||||
|
||||
it("auto picks CUDA on linux when HYPERFRAMES_CUDA=1", async () => {
|
||||
process.env["HYPERFRAMES_CUDA"] = "1";
|
||||
vi.doMock("node:os", () => ({
|
||||
platform: () => "linux",
|
||||
arch: () => "x64",
|
||||
homedir: () => "/tmp",
|
||||
}));
|
||||
const { selectProviders } = await import("./manager.js");
|
||||
const choice = selectProviders("auto");
|
||||
expect(choice.providers).toEqual(["cuda", "cpu"]);
|
||||
expect(choice.label).toBe("CUDA");
|
||||
});
|
||||
|
||||
it("--device coreml on linux throws", async () => {
|
||||
vi.doMock("node:os", () => ({
|
||||
platform: () => "linux",
|
||||
arch: () => "x64",
|
||||
homedir: () => "/tmp",
|
||||
}));
|
||||
const { selectProviders } = await import("./manager.js");
|
||||
expect(() => selectProviders("coreml")).toThrow(/CoreML execution provider not available/);
|
||||
});
|
||||
|
||||
it("--device cuda without env var throws", async () => {
|
||||
vi.doMock("node:os", () => ({
|
||||
platform: () => "linux",
|
||||
arch: () => "x64",
|
||||
homedir: () => "/tmp",
|
||||
}));
|
||||
const { selectProviders } = await import("./manager.js");
|
||||
expect(() => selectProviders("cuda")).toThrow(/CUDA execution provider not available/);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,96 @@
|
||||
import { existsSync, mkdirSync } from "node:fs";
|
||||
import { homedir, platform, arch } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { downloadFile } from "../utils/download.js";
|
||||
|
||||
export const MODELS_DIR = join(homedir(), ".cache", "hyperframes", "background-removal", "models");
|
||||
|
||||
export const DEFAULT_MODEL = "u2net_human_seg" as const;
|
||||
export type ModelId = typeof DEFAULT_MODEL;
|
||||
|
||||
const MODEL_URLS: Record<ModelId, string> = {
|
||||
u2net_human_seg:
|
||||
"https://github.com/danielgatis/rembg/releases/download/v0.0.0/u2net_human_seg.onnx",
|
||||
};
|
||||
|
||||
export const MODEL_MEMORY_MB: Record<ModelId, number> = {
|
||||
u2net_human_seg: 1500,
|
||||
};
|
||||
|
||||
export const DEVICES = ["auto", "cpu", "coreml", "cuda"] as const;
|
||||
export type Device = (typeof DEVICES)[number];
|
||||
|
||||
export function isDevice(value: unknown): value is Device {
|
||||
return typeof value === "string" && (DEVICES as readonly string[]).includes(value);
|
||||
}
|
||||
|
||||
export interface ProviderChoice {
|
||||
providers: string[];
|
||||
label: "CoreML" | "CUDA" | "CPU";
|
||||
}
|
||||
|
||||
export function selectProviders(device: Device = "auto"): ProviderChoice {
|
||||
if (device === "cpu") return { providers: ["cpu"], label: "CPU" };
|
||||
|
||||
const available = listAvailableProviders();
|
||||
const hasCoreML = available.includes("coreml");
|
||||
const hasCUDA = available.includes("cuda");
|
||||
|
||||
if (device === "coreml") {
|
||||
if (!hasCoreML) {
|
||||
throw new Error(
|
||||
"CoreML execution provider not available. Install onnxruntime-node on Apple Silicon, or use --device cpu.",
|
||||
);
|
||||
}
|
||||
return { providers: ["coreml", "cpu"], label: "CoreML" };
|
||||
}
|
||||
if (device === "cuda") {
|
||||
if (!hasCUDA) {
|
||||
throw new Error(
|
||||
"CUDA execution provider not available. Use --device cpu or install an onnxruntime-node build with CUDA support.",
|
||||
);
|
||||
}
|
||||
return { providers: ["cuda", "cpu"], label: "CUDA" };
|
||||
}
|
||||
|
||||
if (hasCoreML && platform() === "darwin" && arch() === "arm64") {
|
||||
return { providers: ["coreml", "cpu"], label: "CoreML" };
|
||||
}
|
||||
if (hasCUDA) return { providers: ["cuda", "cpu"], label: "CUDA" };
|
||||
return { providers: ["cpu"], label: "CPU" };
|
||||
}
|
||||
|
||||
let _cachedProviders: string[] | undefined;
|
||||
export function listAvailableProviders(): string[] {
|
||||
if (_cachedProviders) return _cachedProviders;
|
||||
|
||||
// The npm onnxruntime-node ships with CPU on every platform and bundles the
|
||||
// CoreML EP only on darwin-arm64. CUDA is opt-in via a separate gpu build —
|
||||
// gate behind an env var so we don't try to bind to a missing EP.
|
||||
const out: string[] = ["cpu"];
|
||||
if (platform() === "darwin" && arch() === "arm64") out.push("coreml");
|
||||
if (process.env["HYPERFRAMES_CUDA"] === "1") out.push("cuda");
|
||||
_cachedProviders = out;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function modelPath(model: ModelId = DEFAULT_MODEL): string {
|
||||
return join(MODELS_DIR, `${model}.onnx`);
|
||||
}
|
||||
|
||||
export async function ensureModel(
|
||||
model: ModelId = DEFAULT_MODEL,
|
||||
options?: { onProgress?: (message: string) => void },
|
||||
): Promise<string> {
|
||||
const dest = modelPath(model);
|
||||
if (existsSync(dest)) return dest;
|
||||
|
||||
mkdirSync(MODELS_DIR, { recursive: true });
|
||||
options?.onProgress?.(`Downloading ${model} weights (~168 MB)...`);
|
||||
await downloadFile(MODEL_URLS[model], dest);
|
||||
|
||||
if (!existsSync(dest)) {
|
||||
throw new Error(`Model download failed: ${model}`);
|
||||
}
|
||||
return dest;
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { inferOutputFormat, inferInputKind, buildEncoderArgs } from "./pipeline.js";
|
||||
|
||||
describe("background-removal/pipeline — inferOutputFormat", () => {
|
||||
it("maps .webm → webm", () => {
|
||||
expect(inferOutputFormat("/tmp/out.webm")).toBe("webm");
|
||||
});
|
||||
it("maps .mov → mov", () => {
|
||||
expect(inferOutputFormat("/tmp/out.mov")).toBe("mov");
|
||||
});
|
||||
it("maps .png → png", () => {
|
||||
expect(inferOutputFormat("/tmp/out.png")).toBe("png");
|
||||
});
|
||||
it("rejects unknown extensions", () => {
|
||||
expect(() => inferOutputFormat("/tmp/out.mp4")).toThrow(/Unsupported output extension/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("background-removal/pipeline — inferInputKind", () => {
|
||||
it("recognizes mp4/mov/webm/mkv/avi as video", () => {
|
||||
for (const ext of [".mp4", ".mov", ".webm", ".mkv", ".avi"]) {
|
||||
expect(inferInputKind(`/tmp/clip${ext}`)).toBe("video");
|
||||
}
|
||||
});
|
||||
it("recognizes jpg/png/webp as image", () => {
|
||||
for (const ext of [".jpg", ".jpeg", ".png", ".webp"]) {
|
||||
expect(inferInputKind(`/tmp/img${ext}`)).toBe("image");
|
||||
}
|
||||
});
|
||||
it("rejects unknown extensions", () => {
|
||||
expect(() => inferInputKind("/tmp/file.gif")).toThrow(/Unsupported input/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("background-removal/pipeline — buildEncoderArgs", () => {
|
||||
it("webm preset emits VP9 + alpha_mode metadata", () => {
|
||||
const args = buildEncoderArgs("webm", 1920, 1080, 30, "/tmp/out.webm");
|
||||
expect(args).toContain("libvpx-vp9");
|
||||
expect(args).toContain("yuva420p");
|
||||
// The alpha_mode metadata must be present; without it Chrome ignores the alpha plane.
|
||||
const idx = args.indexOf("-metadata:s:v:0");
|
||||
expect(idx).toBeGreaterThan(-1);
|
||||
expect(args[idx + 1]).toBe("alpha_mode=1");
|
||||
expect(args[args.length - 1]).toBe("/tmp/out.webm");
|
||||
});
|
||||
|
||||
it("mov preset emits ProRes 4444 + yuva444p10le", () => {
|
||||
const args = buildEncoderArgs("mov", 1920, 1080, 30, "/tmp/out.mov");
|
||||
expect(args).toContain("prores_ks");
|
||||
expect(args).toContain("4444");
|
||||
expect(args).toContain("yuva444p10le");
|
||||
});
|
||||
|
||||
it("png preset emits a single RGBA frame", () => {
|
||||
const args = buildEncoderArgs("png", 1920, 1080, 30, "/tmp/out.png");
|
||||
expect(args).toContain("-frames:v");
|
||||
expect(args).toContain("rgba");
|
||||
});
|
||||
|
||||
it("threads input dimensions and fps into raw video header", () => {
|
||||
const args = buildEncoderArgs("webm", 640, 480, 24, "/tmp/o.webm");
|
||||
const sIdx = args.indexOf("-s");
|
||||
expect(args[sIdx + 1]).toBe("640x480");
|
||||
const rIdx = args.indexOf("-r");
|
||||
expect(args[rIdx + 1]).toBe("24");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,324 @@
|
||||
/**
|
||||
* Background-removal rendering pipeline.
|
||||
*
|
||||
* Decode source frames via ffmpeg → run inference per frame → encode the RGBA
|
||||
* stream via a second ffmpeg process. Output formats:
|
||||
* .webm → VP9 with alpha (HTML5-native, ~1 MB / 4s @ 1080p)
|
||||
* .mov → ProRes 4444 with alpha (editing round-trip)
|
||||
* .png → single RGBA still (only when input is also a single image)
|
||||
*
|
||||
* The encode flags for VP9-with-alpha mirror the `chunkEncoder.ts` pattern in
|
||||
* @hyperframes/engine — `-pix_fmt yuva420p` plus the
|
||||
* `-metadata:s:v:0 alpha_mode=1` tag are what make Chrome's `<video>` element
|
||||
* decode the alpha plane.
|
||||
*/
|
||||
import { spawn } from "node:child_process";
|
||||
import { extname } from "node:path";
|
||||
import { hasFFmpeg, hasFFprobe } from "../whisper/manager.js";
|
||||
import { createSession, type Session } from "./inference.js";
|
||||
import { type Device, type ModelId } from "./manager.js";
|
||||
|
||||
export type OutputFormat = "webm" | "mov" | "png";
|
||||
|
||||
export interface RenderOptions {
|
||||
inputPath: string;
|
||||
outputPath: string;
|
||||
device?: Device;
|
||||
model?: ModelId;
|
||||
onProgress?: (event: ProgressEvent) => void;
|
||||
}
|
||||
|
||||
export type ProgressEvent =
|
||||
| { kind: "info"; message: string }
|
||||
| { kind: "metadata"; width: number; height: number; fps: number; frameCount: number }
|
||||
| { kind: "frame"; index: number; total: number; avgMsPerFrame: number };
|
||||
|
||||
export interface RenderResult {
|
||||
outputPath: string;
|
||||
framesProcessed: number;
|
||||
durationSeconds: number;
|
||||
avgMsPerFrame: number;
|
||||
provider: string;
|
||||
format: OutputFormat;
|
||||
}
|
||||
|
||||
const VIDEO_EXTENSIONS = new Set([".mp4", ".mov", ".webm", ".mkv", ".avi"]);
|
||||
const IMAGE_EXTENSIONS = new Set([".jpg", ".jpeg", ".png", ".webp"]);
|
||||
|
||||
interface MediaInfo {
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
frameCount: number;
|
||||
}
|
||||
|
||||
export function inferOutputFormat(outputPath: string): OutputFormat {
|
||||
const ext = extname(outputPath).toLowerCase();
|
||||
if (ext === ".webm") return "webm";
|
||||
if (ext === ".mov") return "mov";
|
||||
if (ext === ".png") return "png";
|
||||
throw new Error(
|
||||
`Unsupported output extension: ${ext}. Use .webm (VP9 alpha), .mov (ProRes 4444), or .png.`,
|
||||
);
|
||||
}
|
||||
|
||||
export function inferInputKind(inputPath: string): "video" | "image" {
|
||||
const ext = extname(inputPath).toLowerCase();
|
||||
if (VIDEO_EXTENSIONS.has(ext)) return "video";
|
||||
if (IMAGE_EXTENSIONS.has(ext)) return "image";
|
||||
throw new Error(
|
||||
`Unsupported input: ${ext}. Use a video (mp4/mov/webm/mkv/avi) or image (jpg/png/webp).`,
|
||||
);
|
||||
}
|
||||
|
||||
interface EngineMetadata {
|
||||
width: number;
|
||||
height: number;
|
||||
fps: number;
|
||||
durationSeconds: number;
|
||||
}
|
||||
|
||||
async function probeMedia(inputPath: string): Promise<MediaInfo> {
|
||||
const isImage = inferInputKind(inputPath) === "image";
|
||||
const engine = (await import("@hyperframes/engine")) as {
|
||||
extractMediaMetadata: (path: string) => Promise<EngineMetadata>;
|
||||
};
|
||||
const meta = await engine.extractMediaMetadata(inputPath);
|
||||
|
||||
if (isImage) {
|
||||
return { width: meta.width, height: meta.height, fps: 0, frameCount: 1 };
|
||||
}
|
||||
|
||||
const fps = meta.fps || 30;
|
||||
const frameCount = meta.durationSeconds ? Math.round(meta.durationSeconds * fps) : 0;
|
||||
return { width: meta.width, height: meta.height, fps, frameCount };
|
||||
}
|
||||
|
||||
export function buildEncoderArgs(
|
||||
format: OutputFormat,
|
||||
width: number,
|
||||
height: number,
|
||||
fps: number,
|
||||
outputPath: string,
|
||||
): string[] {
|
||||
const base = [
|
||||
"-y",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-pix_fmt",
|
||||
"rgba",
|
||||
"-s",
|
||||
`${width}x${height}`,
|
||||
"-r",
|
||||
String(fps || 30),
|
||||
"-i",
|
||||
"-",
|
||||
];
|
||||
|
||||
if (format === "webm") {
|
||||
return [
|
||||
...base,
|
||||
"-c:v",
|
||||
"libvpx-vp9",
|
||||
"-b:v",
|
||||
"0",
|
||||
"-crf",
|
||||
"30",
|
||||
"-deadline",
|
||||
"good",
|
||||
"-row-mt",
|
||||
"1",
|
||||
"-auto-alt-ref",
|
||||
"0",
|
||||
"-pix_fmt",
|
||||
"yuva420p",
|
||||
"-metadata:s:v:0",
|
||||
"alpha_mode=1",
|
||||
"-an",
|
||||
outputPath,
|
||||
];
|
||||
}
|
||||
if (format === "mov") {
|
||||
return [
|
||||
...base,
|
||||
"-c:v",
|
||||
"prores_ks",
|
||||
"-profile:v",
|
||||
"4444",
|
||||
"-vendor",
|
||||
"apl0",
|
||||
"-pix_fmt",
|
||||
"yuva444p10le",
|
||||
"-an",
|
||||
outputPath,
|
||||
];
|
||||
}
|
||||
return [...base, "-frames:v", "1", "-pix_fmt", "rgba", "-update", "1", outputPath];
|
||||
}
|
||||
|
||||
async function* readFrames(
|
||||
stream: NodeJS.ReadableStream,
|
||||
frameBytes: number,
|
||||
): AsyncGenerator<Buffer> {
|
||||
let buffered: Buffer = Buffer.alloc(0);
|
||||
for await (const chunk of stream) {
|
||||
buffered =
|
||||
buffered.length === 0 ? (chunk as Buffer) : Buffer.concat([buffered, chunk as Buffer]);
|
||||
while (buffered.length >= frameBytes) {
|
||||
// Copy because the next concat would clobber the underlying memory.
|
||||
yield Buffer.from(buffered.subarray(0, frameBytes));
|
||||
buffered = buffered.subarray(frameBytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function render(options: RenderOptions): Promise<RenderResult> {
|
||||
if (!hasFFmpeg() || !hasFFprobe()) {
|
||||
throw new Error("ffmpeg and ffprobe are required. Install: brew install ffmpeg");
|
||||
}
|
||||
|
||||
const format = inferOutputFormat(options.outputPath);
|
||||
const inputKind = inferInputKind(options.inputPath);
|
||||
|
||||
if (inputKind === "image" && format !== "png") {
|
||||
throw new Error(
|
||||
`Image input requires a .png output (got ${extname(options.outputPath)}). Use a video input for .webm/.mov.`,
|
||||
);
|
||||
}
|
||||
if (inputKind === "video" && format === "png") {
|
||||
throw new Error(
|
||||
`Video input requires a .webm or .mov output (got .png). Use an image input for .png.`,
|
||||
);
|
||||
}
|
||||
|
||||
const media = await probeMedia(options.inputPath);
|
||||
|
||||
options.onProgress?.({
|
||||
kind: "metadata",
|
||||
width: media.width,
|
||||
height: media.height,
|
||||
fps: media.fps,
|
||||
frameCount: media.frameCount,
|
||||
});
|
||||
|
||||
const session = await createSession({
|
||||
model: options.model,
|
||||
device: options.device,
|
||||
onProgress: (msg) => options.onProgress?.({ kind: "info", message: msg }),
|
||||
});
|
||||
|
||||
try {
|
||||
const start = Date.now();
|
||||
const framesProcessed = await runPipeline(options, session, media, format);
|
||||
const durationSeconds = (Date.now() - start) / 1000;
|
||||
const avgMsPerFrame = framesProcessed ? (durationSeconds * 1000) / framesProcessed : 0;
|
||||
|
||||
return {
|
||||
outputPath: options.outputPath,
|
||||
framesProcessed,
|
||||
durationSeconds,
|
||||
avgMsPerFrame,
|
||||
provider: session.provider,
|
||||
format,
|
||||
};
|
||||
} finally {
|
||||
await session.close();
|
||||
}
|
||||
}
|
||||
|
||||
const RECENT_WINDOW = 30;
|
||||
|
||||
async function runPipeline(
|
||||
options: RenderOptions,
|
||||
session: Session,
|
||||
media: MediaInfo,
|
||||
format: OutputFormat,
|
||||
): Promise<number> {
|
||||
const { inputPath, outputPath } = options;
|
||||
const { width, height, fps, frameCount } = media;
|
||||
const frameBytes = width * height * 3;
|
||||
|
||||
const decoder = spawn(
|
||||
"ffmpeg",
|
||||
["-loglevel", "error", "-i", inputPath, "-f", "rawvideo", "-pix_fmt", "rgb24", "-an", "-"],
|
||||
{ stdio: ["ignore", "pipe", "pipe"] },
|
||||
);
|
||||
|
||||
let decoderStderr = "";
|
||||
decoder.stderr?.on("data", (d: Buffer) => {
|
||||
decoderStderr += d.toString();
|
||||
});
|
||||
const decoderExit = waitForExit(decoder, "ffmpeg decoder", () => decoderStderr);
|
||||
|
||||
const encoder = spawn("ffmpeg", buildEncoderArgs(format, width, height, fps || 30, outputPath), {
|
||||
stdio: ["pipe", "ignore", "pipe"],
|
||||
});
|
||||
let encoderStderr = "";
|
||||
encoder.stderr?.on("data", (d: Buffer) => {
|
||||
encoderStderr += d.toString();
|
||||
});
|
||||
const encoderExit = waitForExit(encoder, "ffmpeg encoder", () => encoderStderr);
|
||||
|
||||
let processed = 0;
|
||||
const total = frameCount;
|
||||
|
||||
// Running average over the last RECENT_WINDOW frames.
|
||||
const recentMs = new Array<number>(RECENT_WINDOW).fill(0);
|
||||
let recentSum = 0;
|
||||
let recentSlot = 0;
|
||||
let recentCount = 0;
|
||||
|
||||
try {
|
||||
for await (const rgb of readFrames(decoder.stdout!, frameBytes)) {
|
||||
const t0 = Date.now();
|
||||
const rgba = await session.process(rgb, width, height);
|
||||
const elapsed = Date.now() - t0;
|
||||
|
||||
recentSum += elapsed - recentMs[recentSlot]!;
|
||||
recentMs[recentSlot] = elapsed;
|
||||
recentSlot = (recentSlot + 1) % RECENT_WINDOW;
|
||||
if (recentCount < RECENT_WINDOW) recentCount++;
|
||||
|
||||
if (!encoder.stdin!.write(rgba)) {
|
||||
await new Promise<void>((resolve) => encoder.stdin!.once("drain", () => resolve()));
|
||||
}
|
||||
|
||||
processed++;
|
||||
options.onProgress?.({
|
||||
kind: "frame",
|
||||
index: processed,
|
||||
total,
|
||||
avgMsPerFrame: recentSum / recentCount,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
decoder.kill("SIGKILL");
|
||||
encoder.kill("SIGKILL");
|
||||
throw err;
|
||||
}
|
||||
|
||||
encoder.stdin!.end();
|
||||
await Promise.all([decoderExit, encoderExit]);
|
||||
|
||||
if (processed === 0) {
|
||||
throw new Error(
|
||||
`No frames produced from ${inputPath}. Decoder stderr:\n${decoderStderr.slice(-400)}`,
|
||||
);
|
||||
}
|
||||
|
||||
return processed;
|
||||
}
|
||||
|
||||
function waitForExit(
|
||||
proc: ReturnType<typeof spawn>,
|
||||
label: string,
|
||||
getStderr: () => string,
|
||||
): Promise<void> {
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
proc.on("error", reject);
|
||||
proc.on("exit", (code) => {
|
||||
if (code === 0 || code === null) resolve();
|
||||
else reject(new Error(`${label} exited with code ${code}: ${getStderr().slice(-400)}`));
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -38,6 +38,7 @@ const subCommands = {
|
||||
compositions: () => import("./commands/compositions.js").then((m) => m.default),
|
||||
benchmark: () => import("./commands/benchmark.js").then((m) => m.default),
|
||||
browser: () => import("./commands/browser.js").then((m) => m.default),
|
||||
"remove-background": () => import("./commands/remove-background.js").then((m) => m.default),
|
||||
transcribe: () => import("./commands/transcribe.js").then((m) => m.default),
|
||||
tts: () => import("./commands/tts.js").then((m) => m.default),
|
||||
docs: () => import("./commands/docs.js").then((m) => m.default),
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
import { defineCommand } from "citty";
|
||||
import { resolve } from "node:path";
|
||||
import { existsSync } from "node:fs";
|
||||
import * as clack from "@clack/prompts";
|
||||
import { c } from "../ui/colors.js";
|
||||
import { isDevice, DEVICES } from "../background-removal/manager.js";
|
||||
import type { Example } from "./_examples.js";
|
||||
|
||||
export const examples: Example[] = [
|
||||
[
|
||||
"Remove background from a video, output transparent VP9 WebM (default)",
|
||||
"hyperframes remove-background avatar.mp4 -o transparent.webm",
|
||||
],
|
||||
[
|
||||
"Output ProRes 4444 .mov for editing round-trip",
|
||||
"hyperframes remove-background avatar.mp4 -o transparent.mov",
|
||||
],
|
||||
[
|
||||
"Remove background from a single image, output transparent PNG",
|
||||
"hyperframes remove-background portrait.jpg -o cutout.png",
|
||||
],
|
||||
[
|
||||
"Force CPU (skip CoreML/CUDA)",
|
||||
"hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu",
|
||||
],
|
||||
["Show detected providers without rendering", "hyperframes remove-background --info"],
|
||||
];
|
||||
|
||||
export default defineCommand({
|
||||
meta: {
|
||||
name: "remove-background",
|
||||
description:
|
||||
"Remove background from a video or image using a local AI model — outputs transparent WebM, ProRes 4444, or PNG",
|
||||
},
|
||||
args: {
|
||||
input: {
|
||||
type: "positional",
|
||||
description: "Source video (.mp4/.mov/.webm/.mkv) or image (.jpg/.png/.webp)",
|
||||
required: false,
|
||||
},
|
||||
output: {
|
||||
type: "string",
|
||||
description: "Output path. Format inferred from extension: .webm (default), .mov, .png",
|
||||
alias: "o",
|
||||
},
|
||||
device: {
|
||||
type: "string",
|
||||
description: `Execution provider: ${DEVICES.join(", ")}`,
|
||||
default: "auto",
|
||||
},
|
||||
info: {
|
||||
type: "boolean",
|
||||
description: "Print detected execution providers and exit (no render)",
|
||||
default: false,
|
||||
},
|
||||
json: {
|
||||
type: "boolean",
|
||||
description: "Output result as JSON",
|
||||
default: false,
|
||||
},
|
||||
},
|
||||
async run({ args }) {
|
||||
if (args.info) {
|
||||
return showInfo(args.json);
|
||||
}
|
||||
if (!args.input) {
|
||||
console.error(
|
||||
c.error(
|
||||
"Input file is required. Run `hyperframes remove-background --info` for providers.",
|
||||
),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
if (!args.output) {
|
||||
console.error(c.error("--output (-o) is required. Use a .webm, .mov, or .png path."));
|
||||
process.exit(1);
|
||||
}
|
||||
if (!isDevice(args.device)) {
|
||||
console.error(
|
||||
c.error(`Invalid --device '${String(args.device)}'. Use: ${DEVICES.join(", ")}.`),
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const inputPath = resolve(args.input);
|
||||
const outputPath = resolve(args.output);
|
||||
|
||||
const { render } = await import("../background-removal/pipeline.js");
|
||||
|
||||
const spin = args.json ? null : clack.spinner();
|
||||
spin?.start("Preparing background-removal pipeline...");
|
||||
|
||||
try {
|
||||
const result = await render({
|
||||
inputPath,
|
||||
outputPath,
|
||||
device: args.device,
|
||||
onProgress: (event) => {
|
||||
if (event.kind === "info") {
|
||||
spin?.message(event.message);
|
||||
} else if (event.kind === "metadata") {
|
||||
const dims = `${event.width}×${event.height}`;
|
||||
const frames = event.frameCount ? ` · ${event.frameCount} frames` : "";
|
||||
spin?.message(`Source ${dims} @ ${event.fps.toFixed(0)}fps${frames}`);
|
||||
} else if (event.kind === "frame") {
|
||||
const pct = event.total ? ` (${Math.floor((100 * event.index) / event.total)}%)` : "";
|
||||
spin?.message(
|
||||
`Frame ${event.index}${event.total ? `/${event.total}` : ""}${pct} — ${Math.round(event.avgMsPerFrame)}ms/frame avg`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
if (args.json) {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
ok: true,
|
||||
outputPath: result.outputPath,
|
||||
framesProcessed: result.framesProcessed,
|
||||
durationSeconds: Number(result.durationSeconds.toFixed(2)),
|
||||
avgMsPerFrame: Number(result.avgMsPerFrame.toFixed(1)),
|
||||
provider: result.provider,
|
||||
format: result.format,
|
||||
}),
|
||||
);
|
||||
} else {
|
||||
const fpsThroughput = result.durationSeconds
|
||||
? (result.framesProcessed / result.durationSeconds).toFixed(1)
|
||||
: "n/a";
|
||||
spin?.stop(
|
||||
c.success(
|
||||
`Removed background from ${c.accent(String(result.framesProcessed))} frames in ${result.durationSeconds.toFixed(1)}s (${fpsThroughput} fps, ${c.accent(result.provider)}) → ${c.accent(result.outputPath)}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
if (args.json) {
|
||||
console.log(JSON.stringify({ ok: false, error: message }));
|
||||
} else {
|
||||
spin?.stop(c.error(`Background removal failed: ${message}`));
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
async function showInfo(json: boolean): Promise<void> {
|
||||
const { selectProviders, listAvailableProviders, DEFAULT_MODEL, MODEL_MEMORY_MB, modelPath } =
|
||||
await import("../background-removal/manager.js");
|
||||
|
||||
const providers = listAvailableProviders();
|
||||
const auto = selectProviders("auto");
|
||||
const cached = existsSync(modelPath());
|
||||
|
||||
if (json) {
|
||||
console.log(
|
||||
JSON.stringify({
|
||||
defaultModel: DEFAULT_MODEL,
|
||||
modelCached: cached,
|
||||
modelPath: modelPath(),
|
||||
peakMemoryMb: MODEL_MEMORY_MB[DEFAULT_MODEL],
|
||||
availableProviders: providers,
|
||||
autoProvider: auto.label,
|
||||
}),
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(c.bold("hyperframes remove-background — system info"));
|
||||
console.log("");
|
||||
console.log(` ${c.dim("Default model:")} ${c.accent(DEFAULT_MODEL)}`);
|
||||
console.log(` ${c.dim("Peak memory:")} ~${MODEL_MEMORY_MB[DEFAULT_MODEL]} MB`);
|
||||
console.log(
|
||||
` ${c.dim("Weights cached:")} ${cached ? c.success("yes") : c.dim("no (will download on first run)")}`,
|
||||
);
|
||||
console.log(` ${c.dim("Cache path:")} ${modelPath()}`);
|
||||
console.log("");
|
||||
console.log(` ${c.dim("Available providers:")} ${providers.join(", ")}`);
|
||||
console.log(` ${c.dim("Auto-selected:")} ${c.accent(auto.label)}`);
|
||||
}
|
||||
@@ -60,6 +60,7 @@ const GROUPS: Group[] = [
|
||||
"Transcribe audio/video to word-level timestamps, or import an existing transcript",
|
||||
],
|
||||
["tts", "Generate speech audio from text using a local AI model (Kokoro-82M)"],
|
||||
["remove-background", "Remove background from a video or image to produce transparent media"],
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -101,7 +102,7 @@ const STATIC_EXAMPLES: Record<string, Example[]> = {
|
||||
|
||||
// ── Render root help ───────────────────────────────────────────────────────
|
||||
function renderRootHelp(): string {
|
||||
const NAME_COL = 16;
|
||||
const NAME_COL = 19;
|
||||
const CMD_COL = 46;
|
||||
const lines: string[] = [];
|
||||
|
||||
|
||||
@@ -205,8 +205,16 @@ export async function ensureModel(
|
||||
}
|
||||
|
||||
export function hasFFmpeg(): boolean {
|
||||
return hasBinary("ffmpeg");
|
||||
}
|
||||
|
||||
export function hasFFprobe(): boolean {
|
||||
return hasBinary("ffprobe");
|
||||
}
|
||||
|
||||
function hasBinary(name: string): boolean {
|
||||
try {
|
||||
execFileSync("ffmpeg", ["-version"], { stdio: "ignore", timeout: 5000 });
|
||||
execFileSync(name, ["-version"], { stdio: "ignore", timeout: 5000 });
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
name: hyperframes-cli
|
||||
description: HyperFrames CLI tool — hyperframes init, lint, inspect, preview, render, transcribe, tts, doctor, browser, info, upgrade, compositions, docs, benchmark. Use when scaffolding a project, linting, validating, inspecting visual layout in compositions, previewing in the studio, rendering to video, transcribing audio, generating TTS, or troubleshooting the HyperFrames environment.
|
||||
description: HyperFrames CLI tool — hyperframes init, lint, inspect, preview, render, transcribe, tts, remove-background, doctor, browser, info, upgrade, compositions, docs, benchmark. Use when scaffolding a project, linting, validating, inspecting visual layout in compositions, previewing in the studio, rendering to video, transcribing audio, generating TTS, removing the background from an avatar video for transparent overlays, or troubleshooting the HyperFrames environment.
|
||||
---
|
||||
|
||||
# HyperFrames CLI
|
||||
@@ -133,6 +133,20 @@ npx hyperframes tts script.txt --voice bf_emma
|
||||
npx hyperframes tts --list # show all voices
|
||||
```
|
||||
|
||||
## Background Removal (transparent video)
|
||||
|
||||
Remove the background from a video or image so it can be used as a transparent overlay in a composition (e.g. an avatar floating on a background).
|
||||
|
||||
```bash
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.webm # default: VP9 alpha WebM
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.mov # ProRes 4444 for editing
|
||||
npx hyperframes remove-background portrait.jpg -o cutout.png # single-image cutout
|
||||
npx hyperframes remove-background avatar.mp4 -o transparent.webm --device cpu
|
||||
npx hyperframes remove-background --info # detected providers
|
||||
```
|
||||
|
||||
Uses `u2net_human_seg` (MIT). First run downloads ~168 MB of weights to `~/.cache/hyperframes/background-removal/models/` and reuses them after. Drop the resulting `.webm` into a composition with `<video src="transparent.webm" autoplay muted loop>` — Chrome decodes the alpha natively.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
```bash
|
||||
|
||||
Reference in New Issue
Block a user