mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
587 lines
17 KiB
Python
Executable File
587 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
Build script for hanzo Python wheels.
|
|
|
|
Auto-detects platform, architecture, and available accelerators.
|
|
Builds appropriate wheels based on the detected environment.
|
|
|
|
Usage:
|
|
python scripts/build_wheels.py --list # Show buildable packages
|
|
python scripts/build_wheels.py --all # Build all supported
|
|
python scripts/build_wheels.py -p hanzo hanzo-cuda
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from enum import Enum
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
# ============================================================================
|
|
# Constants and Configuration
|
|
# ============================================================================
|
|
|
|
SCRIPT_DIR = Path(__file__).parent.resolve()
|
|
REPO_ROOT = SCRIPT_DIR.parent
|
|
PYPROJECT_PATH = REPO_ROOT / "hanzo-pyo3" / "pyproject.toml"
|
|
CARGO_MANIFEST = REPO_ROOT / "hanzo-pyo3" / "Cargo.toml"
|
|
DOCKERFILE_PATH = REPO_ROOT / "Dockerfile.manylinux"
|
|
|
|
PACKAGE_NAMES = [
|
|
"hanzo",
|
|
"hanzo-cuda",
|
|
"hanzo-metal",
|
|
"hanzo-accelerate",
|
|
"hanzo-mkl",
|
|
]
|
|
|
|
TRUTHY = {"1", "true", "yes", "on"}
|
|
|
|
|
|
class OS(Enum):
|
|
LINUX = "linux"
|
|
DARWIN = "darwin"
|
|
WINDOWS = "windows"
|
|
|
|
|
|
class Arch(Enum):
|
|
X86_64 = "x86_64"
|
|
AARCH64 = "aarch64"
|
|
|
|
|
|
@dataclass
|
|
class Platform:
|
|
os: OS
|
|
arch: Arch
|
|
has_cuda: bool
|
|
has_metal: bool
|
|
|
|
|
|
@dataclass
|
|
class PackageConfig:
|
|
name: str
|
|
features: list[str]
|
|
supported_os: list[OS]
|
|
supported_arch: list[Arch]
|
|
requires_accelerator: Optional[str] # "cuda", "metal", or None
|
|
|
|
|
|
# ============================================================================
|
|
# Platform Detection
|
|
# ============================================================================
|
|
|
|
|
|
def detect_platform() -> Platform:
|
|
"""Auto-detect OS, architecture, and available accelerators."""
|
|
# Detect OS
|
|
system = platform.system().lower()
|
|
if system == "linux":
|
|
os_type = OS.LINUX
|
|
elif system == "darwin":
|
|
os_type = OS.DARWIN
|
|
elif system == "windows":
|
|
os_type = OS.WINDOWS
|
|
else:
|
|
raise RuntimeError(f"Unsupported OS: {system}")
|
|
|
|
# Detect architecture
|
|
machine = platform.machine().lower()
|
|
if machine in ("x86_64", "amd64"):
|
|
arch = Arch.X86_64
|
|
elif machine in ("aarch64", "arm64"):
|
|
arch = Arch.AARCH64
|
|
else:
|
|
raise RuntimeError(f"Unsupported architecture: {machine}")
|
|
|
|
# Detect CUDA
|
|
has_cuda = _detect_cuda()
|
|
|
|
# Detect Metal (macOS aarch64 only)
|
|
has_metal = os_type == OS.DARWIN and arch == Arch.AARCH64
|
|
|
|
return Platform(os=os_type, arch=arch, has_cuda=has_cuda, has_metal=has_metal)
|
|
|
|
|
|
def _detect_cuda() -> bool:
|
|
"""Check if CUDA is available."""
|
|
# Check for nvidia-smi
|
|
if shutil.which("nvidia-smi"):
|
|
try:
|
|
result = subprocess.run(
|
|
["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=5,
|
|
)
|
|
if result.returncode == 0 and len(result.stdout.strip()) > 0:
|
|
return True
|
|
except (subprocess.TimeoutExpired, FileNotFoundError):
|
|
pass
|
|
|
|
# Check for CUDA library paths
|
|
cuda_paths = [
|
|
"/usr/local/cuda",
|
|
"/opt/cuda",
|
|
os.environ.get("CUDA_HOME", ""),
|
|
os.environ.get("CUDA_PATH", ""),
|
|
]
|
|
return any(Path(p).exists() for p in cuda_paths if p)
|
|
|
|
|
|
def _cuda_version() -> Optional[tuple[int, int]]:
|
|
"""Detect the CUDA toolkit (major, minor) from nvcc, if available."""
|
|
nvcc = shutil.which("nvcc")
|
|
if not nvcc:
|
|
for home in (os.environ.get("CUDA_HOME"), os.environ.get("CUDA_PATH"), "/usr/local/cuda"):
|
|
cand = Path(home) / "bin" / "nvcc" if home else None
|
|
if cand and cand.exists():
|
|
nvcc = str(cand)
|
|
break
|
|
if not nvcc:
|
|
return None
|
|
try:
|
|
out = subprocess.run([nvcc, "--version"], capture_output=True, text=True, timeout=5)
|
|
m = re.search(r"release (\d+)\.(\d+)", out.stdout)
|
|
if m:
|
|
return (int(m.group(1)), int(m.group(2)))
|
|
except (subprocess.SubprocessError, FileNotFoundError, OSError):
|
|
pass
|
|
return None
|
|
|
|
|
|
def _env_truthy(name: str) -> bool:
|
|
return os.environ.get(name, "").lower() in TRUTHY
|
|
|
|
|
|
def _detect_nccl() -> bool:
|
|
if platform.system().lower() != "linux":
|
|
return False
|
|
|
|
for home in (
|
|
os.environ.get("NCCL_ROOT"),
|
|
os.environ.get("NCCL_HOME"),
|
|
os.environ.get("CUDA_HOME"),
|
|
os.environ.get("CUDA_PATH"),
|
|
"/usr/local/cuda",
|
|
):
|
|
if not home:
|
|
continue
|
|
root = Path(home)
|
|
for subdir in ("lib", "lib64", "lib/x86_64-linux-gnu"):
|
|
if any((root / subdir).glob("libnccl.so*")):
|
|
return True
|
|
|
|
for libdir in (
|
|
"/usr/lib/x86_64-linux-gnu",
|
|
"/usr/lib/aarch64-linux-gnu",
|
|
"/usr/local/lib",
|
|
"/usr/local/lib64",
|
|
"/usr/lib64",
|
|
):
|
|
if any(Path(libdir).glob("libnccl.so*")):
|
|
return True
|
|
|
|
if shutil.which("ldconfig"):
|
|
try:
|
|
result = subprocess.run(
|
|
["ldconfig", "-p"], capture_output=True, text=True, timeout=5
|
|
)
|
|
return result.returncode == 0 and "libnccl.so" in result.stdout
|
|
except (subprocess.SubprocessError, FileNotFoundError, OSError):
|
|
pass
|
|
|
|
return False
|
|
|
|
|
|
def _cuda_features() -> list[str]:
|
|
"""Features for the CUDA wheel; NCCL is enabled by default when available."""
|
|
features = ["cuda"]
|
|
if (
|
|
platform.system().lower() == "linux"
|
|
and not _env_truthy("HANZO_BUILD_NO_NCCL")
|
|
and (_detect_nccl() or _env_truthy("HANZO_BUILD_NCCL"))
|
|
):
|
|
features.append("nccl")
|
|
version = _cuda_version()
|
|
if version is not None and version >= (13, 1):
|
|
features.append("cutile")
|
|
return features
|
|
|
|
|
|
# ============================================================================
|
|
# Package Configuration
|
|
# ============================================================================
|
|
|
|
|
|
def get_package_configs() -> dict[str, PackageConfig]:
|
|
"""Define the build configuration for each package."""
|
|
return {
|
|
"hanzo": PackageConfig(
|
|
name="hanzo",
|
|
features=[], # Features determined by platform
|
|
supported_os=[OS.LINUX, OS.DARWIN, OS.WINDOWS],
|
|
supported_arch=[Arch.X86_64, Arch.AARCH64],
|
|
requires_accelerator=None,
|
|
),
|
|
"hanzo-cuda": PackageConfig(
|
|
name="hanzo-cuda",
|
|
features=["cuda"],
|
|
supported_os=[OS.LINUX, OS.WINDOWS],
|
|
supported_arch=[Arch.X86_64, Arch.AARCH64],
|
|
requires_accelerator="cuda",
|
|
),
|
|
"hanzo-metal": PackageConfig(
|
|
name="hanzo-metal",
|
|
features=["metal"],
|
|
supported_os=[OS.DARWIN],
|
|
supported_arch=[Arch.AARCH64],
|
|
requires_accelerator="metal",
|
|
),
|
|
"hanzo-accelerate": PackageConfig(
|
|
name="hanzo-accelerate",
|
|
features=["accelerate"],
|
|
supported_os=[OS.DARWIN],
|
|
supported_arch=[Arch.AARCH64],
|
|
requires_accelerator=None, # Accelerate is always available on macOS
|
|
),
|
|
"hanzo-mkl": PackageConfig(
|
|
name="hanzo-mkl",
|
|
features=["mkl"],
|
|
supported_os=[OS.LINUX, OS.WINDOWS],
|
|
supported_arch=[Arch.X86_64],
|
|
requires_accelerator=None,
|
|
),
|
|
}
|
|
|
|
|
|
def get_features_for_base_package(plat: Platform) -> list[str]:
|
|
"""Get features for the 'hanzo' base package based on platform."""
|
|
if plat.os == OS.DARWIN and plat.arch == Arch.AARCH64:
|
|
return ["metal"] # macOS aarch64: Metal
|
|
elif plat.arch == Arch.X86_64:
|
|
return ["mkl"] # x86_64: MKL
|
|
else:
|
|
return [] # aarch64 Linux: CPU-only
|
|
|
|
|
|
def get_buildable_packages(
|
|
configs: dict[str, PackageConfig], plat: Platform
|
|
) -> list[str]:
|
|
"""Get list of packages that can be built on the current platform."""
|
|
buildable = []
|
|
|
|
for name, cfg in configs.items():
|
|
# Check OS and arch support
|
|
if plat.os not in cfg.supported_os:
|
|
continue
|
|
if plat.arch not in cfg.supported_arch:
|
|
continue
|
|
|
|
# Check accelerator requirements
|
|
if cfg.requires_accelerator == "cuda" and not plat.has_cuda:
|
|
continue
|
|
if cfg.requires_accelerator == "metal" and not plat.has_metal:
|
|
continue
|
|
|
|
buildable.append(name)
|
|
|
|
return buildable
|
|
|
|
|
|
# ============================================================================
|
|
# PyProject.toml Modification
|
|
# ============================================================================
|
|
|
|
|
|
def modify_pyproject_name(name: str) -> None:
|
|
"""Modify project.name in pyproject.toml."""
|
|
content = PYPROJECT_PATH.read_text()
|
|
|
|
# Use regex to replace the name field
|
|
new_content = re.sub(
|
|
r'^name\s*=\s*"[^"]*"',
|
|
f'name = "{name}"',
|
|
content,
|
|
flags=re.MULTILINE,
|
|
)
|
|
|
|
PYPROJECT_PATH.write_text(new_content)
|
|
print(f" Set project.name to '{name}'")
|
|
|
|
|
|
def restore_pyproject_name() -> None:
|
|
"""Restore project.name to default 'hanzo'."""
|
|
modify_pyproject_name("hanzo")
|
|
|
|
|
|
# ============================================================================
|
|
# Build Functions
|
|
# ============================================================================
|
|
|
|
|
|
def build_wheel(
|
|
package_config: PackageConfig,
|
|
plat: Platform,
|
|
output_dir: Path,
|
|
) -> Path:
|
|
"""Build a wheel for the given package configuration."""
|
|
# Determine features
|
|
if package_config.name == "hanzo":
|
|
features = get_features_for_base_package(plat)
|
|
else:
|
|
features = package_config.features
|
|
|
|
# Create output directory
|
|
package_output = output_dir / package_config.name
|
|
package_output.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Modify pyproject.toml
|
|
modify_pyproject_name(package_config.name)
|
|
|
|
try:
|
|
# Use Docker manylinux ONLY for CPU-only builds on Linux (no features)
|
|
# CUDA, MKL, and other accelerator builds use native maturin
|
|
if plat.os == OS.LINUX and not features:
|
|
_build_with_docker(features, package_output, plat)
|
|
else:
|
|
_build_with_maturin(features, package_output, plat)
|
|
finally:
|
|
# Always restore pyproject.toml
|
|
restore_pyproject_name()
|
|
|
|
return package_output
|
|
|
|
|
|
def _build_with_maturin(features: list[str], output_dir: Path, plat: Platform) -> None:
|
|
"""Build using native maturin."""
|
|
cmd = [
|
|
"maturin",
|
|
"build",
|
|
"--release",
|
|
"--strip",
|
|
"-o",
|
|
str(output_dir),
|
|
"-m",
|
|
str(CARGO_MANIFEST),
|
|
"--interpreter",
|
|
"python3.10",
|
|
]
|
|
|
|
if features:
|
|
cmd.extend(["--features", ",".join(features)])
|
|
|
|
# Skip auditwheel for CUDA builds - don't bundle CUDA shared libraries
|
|
# Users are expected to have CUDA installed on their system
|
|
# Note: MKL is statically linked (mkl-static-lp64-iomp) so doesn't need this
|
|
if "cuda" in features:
|
|
cmd.extend(["--auditwheel", "skip"])
|
|
|
|
env = os.environ.copy()
|
|
|
|
# macOS-specific settings for Metal builds
|
|
if plat.os == OS.DARWIN and "metal" in features:
|
|
env["MACOSX_DEPLOYMENT_TARGET"] = "15.0"
|
|
print(" Setting MACOSX_DEPLOYMENT_TARGET=15.0 for Metal build")
|
|
|
|
print(f" Running: {' '.join(cmd)}")
|
|
subprocess.run(cmd, check=True, env=env, cwd=REPO_ROOT)
|
|
|
|
|
|
def _build_with_docker(features: list[str], output_dir: Path, plat: Platform) -> None:
|
|
"""Build using Docker manylinux container."""
|
|
# Build docker image if needed
|
|
print(" Building Docker image...")
|
|
subprocess.run(
|
|
[
|
|
"docker",
|
|
"build",
|
|
"-t",
|
|
"hanzo-wheelmaker:latest",
|
|
"-f",
|
|
"Dockerfile.manylinux",
|
|
".",
|
|
],
|
|
cwd=REPO_ROOT,
|
|
check=True,
|
|
)
|
|
|
|
# Construct maturin command for inside container
|
|
maturin_args = [
|
|
"build",
|
|
"--release",
|
|
"--strip",
|
|
"-o",
|
|
f"/io/wheels/{output_dir.name}",
|
|
"-m",
|
|
"hanzo-pyo3/Cargo.toml",
|
|
"--interpreter",
|
|
"python3.10",
|
|
]
|
|
|
|
if features:
|
|
maturin_args.extend(["--features", ",".join(features)])
|
|
|
|
# Docker command
|
|
docker_cmd = [
|
|
"docker",
|
|
"run",
|
|
"--rm",
|
|
"-v",
|
|
f"{REPO_ROOT}:/io",
|
|
"-e",
|
|
"RUSTFLAGS=-C target-cpu=generic",
|
|
]
|
|
|
|
docker_cmd.extend(["hanzo-wheelmaker:latest"] + maturin_args)
|
|
|
|
print(" Running Docker build with RUSTFLAGS=-C target-cpu=generic")
|
|
print(f" Maturin args: {' '.join(maturin_args)}")
|
|
subprocess.run(docker_cmd, check=True)
|
|
|
|
# Fix ownership of target/ directory (Docker creates files as root)
|
|
import getpass
|
|
|
|
user = getpass.getuser()
|
|
print(" Fixing ownership of target/ directory...")
|
|
subprocess.run(
|
|
["sudo", "chown", "-R", f"{user}:{user}", "target/"], cwd=REPO_ROOT, check=False
|
|
)
|
|
|
|
# Move wheels from repo wheels/ to output_dir
|
|
docker_wheels_dir = REPO_ROOT / "wheels" / output_dir.name
|
|
if docker_wheels_dir.exists() and docker_wheels_dir != output_dir:
|
|
for whl in docker_wheels_dir.glob("*.whl"):
|
|
dest = output_dir / whl.name
|
|
shutil.move(str(whl), str(dest))
|
|
print(f" Moved {whl.name} to {output_dir}")
|
|
|
|
|
|
# ============================================================================
|
|
# Main Entry Point
|
|
# ============================================================================
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(
|
|
description="Build hanzo Python wheels",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
# Build all packages supported on current platform
|
|
python scripts/build_wheels.py --all
|
|
|
|
# Build specific packages
|
|
python scripts/build_wheels.py --packages hanzo hanzo-cuda
|
|
|
|
# Specify output directory
|
|
python scripts/build_wheels.py --all -o ./dist
|
|
|
|
# List what can be built on this platform
|
|
python scripts/build_wheels.py --list
|
|
""",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--packages",
|
|
"-p",
|
|
nargs="+",
|
|
choices=PACKAGE_NAMES,
|
|
help="Packages to build",
|
|
)
|
|
parser.add_argument(
|
|
"--all",
|
|
"-a",
|
|
action="store_true",
|
|
help="Build all packages supported on current platform",
|
|
)
|
|
parser.add_argument(
|
|
"--output",
|
|
"-o",
|
|
type=Path,
|
|
default=Path("wheels"),
|
|
help="Output directory for wheels (default: ./wheels)",
|
|
)
|
|
parser.add_argument(
|
|
"--list",
|
|
"-l",
|
|
action="store_true",
|
|
help="List packages that can be built on current platform",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
# Detect platform
|
|
plat = detect_platform()
|
|
print(f"Detected platform: {plat.os.value}/{plat.arch.value}")
|
|
print(f" CUDA available: {plat.has_cuda}")
|
|
print(f" Metal available: {plat.has_metal}")
|
|
|
|
# Get package configs
|
|
configs = get_package_configs()
|
|
|
|
# Filter to packages buildable on this platform
|
|
buildable = get_buildable_packages(configs, plat)
|
|
|
|
if args.list:
|
|
print("\nPackages buildable on this platform:")
|
|
for name in buildable:
|
|
cfg = configs[name]
|
|
features = (
|
|
cfg.features
|
|
if cfg.name != "hanzo"
|
|
else get_features_for_base_package(plat)
|
|
)
|
|
print(f" - {name} (features: {features or 'none'})")
|
|
return 0
|
|
|
|
# Determine which packages to build
|
|
if args.packages:
|
|
to_build = args.packages
|
|
elif args.all:
|
|
to_build = buildable
|
|
else:
|
|
parser.error(
|
|
"Specify --packages or --all, or use --list to see available packages"
|
|
)
|
|
return 1
|
|
|
|
# Validate packages
|
|
for pkg in to_build:
|
|
if pkg not in buildable:
|
|
print(f"Error: {pkg} cannot be built on this platform", file=sys.stderr)
|
|
print(f"Buildable packages: {buildable}", file=sys.stderr)
|
|
return 1
|
|
|
|
# Build each package
|
|
args.output.mkdir(parents=True, exist_ok=True)
|
|
|
|
for pkg_name in to_build:
|
|
print(f"\n{'=' * 60}")
|
|
print(f"Building {pkg_name}")
|
|
print(f"{'=' * 60}")
|
|
|
|
build_wheel(configs[pkg_name], plat, args.output)
|
|
|
|
print(f"\n{'=' * 60}")
|
|
print("All wheels built successfully!")
|
|
print(f"Output directory: {args.output.absolute()}")
|
|
|
|
# List built wheels
|
|
print("\nBuilt wheels:")
|
|
for whl in args.output.rglob("*.whl"):
|
|
print(f" {whl.relative_to(args.output)}")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|