mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Every mention of another inference engine is rewritten to say what our code does, or the math it computes. Reference values are "the reference value from the published config" or "the recorded reference output", line citations into another engine's source become the rule itself, comparisons with other engines are dropped, and asides about other engines' bugs go. Identifiers follow: - CUDA: hanzo-paged-attn's `namespace vllm` is `pagedattn`, hanzo-engine's MoE helpers `moe_dtype` and `moe_detail`, the rotary kernels' `hanzo_rotary`; the shuffle/ldg macros are HANZO_*. - GPTQ: the shuffled-layout path (`reconstruct_exllama`, `use_exllama`) is `reconstruct_shuffled` / `is_shuffled`, on the kernel and the FFI side. - Tests: `*_matches_vllm` and friends name what they compare against (the reference fixture, the tile rule). - MiniCPM-o's vision-LLM embedding is `vlm_embedding`; the rotary custom op is `rotary-embedding`. - hanzo-bench's board scores ours against a "rival" whose name comes from the manifest: `rival_<tag>.json`, `--rival*` manifest flags, `rival t/s` columns; dossier.sh and spec_decode.sh take `--rival-bench`. - install.ps1 installs this repository's hanzo-cli, as install.sh does, with HANZO_INSTALL_NCCL / HANZO_INSTALL_NO_NCCL. - bench_http.py reads only the hanzo: metric families. Kept, because they are what they are: LICENSE, LICENSE-APACHE, NOTICE and tools/upstream-map.sh; each ported file's copyright/licence/provenance header; Cargo.toml dependency names; data identifiers (Hugging Face repo IDs, fixture file names and fixture metadata keys the tests read, on-disk paths, executable names the GPU and engine guards match on a host); model and format names. LLM.md says the rule once, under "One name: Hanzo Engine". Co-authored-by: Hanzo Dev <[email protected]>
368 lines
11 KiB
PowerShell
368 lines
11 KiB
PowerShell
#Requires -Version 5.1
|
|
# hanzo Installation Script for Windows
|
|
# Automatic hardware detection and feature configuration
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
# Color output functions
|
|
function Write-Info { Write-Host "info: $args" -ForegroundColor Blue }
|
|
function Write-Success { Write-Host "success: $args" -ForegroundColor Green }
|
|
function Write-Warn { Write-Host "warning: $args" -ForegroundColor Yellow }
|
|
function Write-Err { Write-Host "error: $args" -ForegroundColor Red; exit 1 }
|
|
|
|
# Banner
|
|
function Show-Banner {
|
|
Write-Host "Hanzo Engine" -NoNewline
|
|
Write-Host " - fast, flexible LLM inference." -ForegroundColor Blue
|
|
Write-Host ""
|
|
}
|
|
|
|
# Minimum required Rust version (from Cargo.toml rust-version)
|
|
$RequiredRustVersion = "1.88"
|
|
$HanzoRepoUrl = "https://github.com/hanzoai/engine"
|
|
$HanzoBranch = "main"
|
|
$HanzoCliPackage = "hanzo-cli"
|
|
|
|
# Check if Rust is installed
|
|
function Test-Rust {
|
|
try {
|
|
$null = Get-Command cargo -ErrorAction Stop
|
|
return $true
|
|
} catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
# Get installed Rust version (major.minor)
|
|
function Get-RustVersion {
|
|
try {
|
|
$output = & rustc --version 2>$null
|
|
if ($output -match 'rustc (\d+\.\d+)') {
|
|
return $matches[1]
|
|
}
|
|
} catch {}
|
|
return $null
|
|
}
|
|
|
|
# Compare two version strings (returns $true if $v1 >= $v2)
|
|
function Test-VersionGte {
|
|
param([string]$v1, [string]$v2)
|
|
|
|
$v1Parts = $v1 -split '\.'
|
|
$v2Parts = $v2 -split '\.'
|
|
|
|
$v1Major = [int]$v1Parts[0]
|
|
$v1Minor = [int]$v1Parts[1]
|
|
$v2Major = [int]$v2Parts[0]
|
|
$v2Minor = [int]$v2Parts[1]
|
|
|
|
if ($v1Major -gt $v2Major) {
|
|
return $true
|
|
} elseif ($v1Major -eq $v2Major -and $v1Minor -ge $v2Minor) {
|
|
return $true
|
|
}
|
|
return $false
|
|
}
|
|
|
|
# Install Rust via rustup
|
|
function Install-Rust {
|
|
Write-Info "Installing Rust via rustup..."
|
|
$rustupInit = "$env:TEMP\rustup-init.exe"
|
|
|
|
try {
|
|
Invoke-WebRequest -Uri "https://win.rustup.rs/x86_64" -OutFile $rustupInit -UseBasicParsing
|
|
& $rustupInit -y
|
|
|
|
# Add cargo to PATH for current session
|
|
$env:PATH = "$env:USERPROFILE\.cargo\bin;$env:PATH"
|
|
|
|
Write-Success "Rust installed successfully"
|
|
} catch {
|
|
Write-Err "Failed to install Rust: $_"
|
|
}
|
|
}
|
|
|
|
# Update Rust to latest version
|
|
function Update-Rust {
|
|
Write-Info "Updating Rust to latest version..."
|
|
try {
|
|
& rustup update stable
|
|
Write-Success "Rust updated successfully"
|
|
} catch {
|
|
Write-Err "Failed to update Rust: $_"
|
|
}
|
|
}
|
|
|
|
# Get CUDA compute capability
|
|
function Get-CudaComputeCap {
|
|
try {
|
|
$nvidiaSmi = Get-Command nvidia-smi -ErrorAction Stop
|
|
$output = & nvidia-smi --query-gpu=compute_cap --format=csv,noheader 2>$null
|
|
if ($output) {
|
|
$cc = ($output -split "`n")[0].Trim() -replace '\.',''
|
|
return $cc
|
|
}
|
|
} catch {}
|
|
return $null
|
|
}
|
|
|
|
# CUDA toolkit version as major*100+minor (e.g. 13.1 -> 1301). $null if nvcc is unavailable.
|
|
function Get-CudaVersionCode {
|
|
try {
|
|
$null = Get-Command nvcc -ErrorAction Stop
|
|
$output = & nvcc --version 2>$null | Out-String
|
|
if ($output -match "release (\d+)\.(\d+)") {
|
|
return [int]$Matches[1] * 100 + [int]$Matches[2]
|
|
}
|
|
} catch {}
|
|
return $null
|
|
}
|
|
|
|
# Check if MKL is installed
|
|
function Test-MKL {
|
|
if ($env:MKLROOT -and (Test-Path $env:MKLROOT)) {
|
|
return $true
|
|
}
|
|
|
|
$mklPaths = @(
|
|
"C:\Program Files (x86)\Intel\oneAPI\mkl\latest",
|
|
"C:\Program Files\Intel\oneAPI\mkl\latest",
|
|
"$env:USERPROFILE\intel\oneapi\mkl\latest"
|
|
)
|
|
|
|
foreach ($path in $mklPaths) {
|
|
if (Test-Path $path) {
|
|
return $true
|
|
}
|
|
}
|
|
return $false
|
|
}
|
|
|
|
# Check if CPU is Intel
|
|
function Test-IntelCpu {
|
|
try {
|
|
$cpu = Get-CimInstance -ClassName Win32_Processor | Select-Object -First 1
|
|
return $cpu.Manufacturer -match "Intel" -or $cpu.Name -match "Intel"
|
|
} catch {
|
|
return $false
|
|
}
|
|
}
|
|
|
|
# Check if cuDNN is installed
|
|
function Test-CuDNN {
|
|
# Check common cuDNN library paths on Windows
|
|
$cudnnPaths = @(
|
|
"$env:CUDA_PATH\bin\cudnn*.dll",
|
|
"C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*\bin\cudnn*.dll",
|
|
"C:\Program Files\NVIDIA\CUDNN\*\bin\cudnn*.dll"
|
|
"C:\Program Files\NVIDIA\CUDNN\*\bin\*\x64\cudnn*.dll"
|
|
)
|
|
|
|
foreach ($pattern in $cudnnPaths) {
|
|
if (Get-Item $pattern -ErrorAction SilentlyContinue) {
|
|
return $true
|
|
}
|
|
}
|
|
return $false
|
|
}
|
|
|
|
# Check if NCCL is installed
|
|
function Test-NCCL {
|
|
$ncclPaths = @(
|
|
"$env:NCCL_ROOT\bin\nccl*.dll",
|
|
"$env:NCCL_ROOT\lib\nccl*.lib",
|
|
"$env:NCCL_HOME\bin\nccl*.dll",
|
|
"$env:NCCL_HOME\lib\nccl*.lib",
|
|
"$env:CUDA_PATH\bin\nccl*.dll",
|
|
"$env:CUDA_PATH\lib\x64\nccl*.lib",
|
|
"C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*\bin\nccl*.dll",
|
|
"C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\*\lib\x64\nccl*.lib"
|
|
)
|
|
|
|
foreach ($pattern in $ncclPaths) {
|
|
if (Get-Item $pattern -ErrorAction SilentlyContinue) {
|
|
return $true
|
|
}
|
|
}
|
|
return $false
|
|
}
|
|
|
|
# Build feature string based on detected hardware
|
|
function Get-Features {
|
|
$features = @()
|
|
|
|
# Check for CUDA
|
|
$cudaCC = Get-CudaComputeCap
|
|
if ($cudaCC) {
|
|
$features += "cuda"
|
|
|
|
$ccMajor = $cudaCC.Substring(0, 1)
|
|
$ccMinor = if ($cudaCC.Length -gt 1) { $cudaCC.Substring(1) } else { "0" }
|
|
Write-Info "CUDA detected (compute capability: $ccMajor.$ccMinor)"
|
|
|
|
if ($env:HANZO_INSTALL_NO_NCCL -eq "1") {
|
|
Write-Info "HANZO_INSTALL_NO_NCCL=1 set - skipping nccl"
|
|
} elseif (Test-NCCL) {
|
|
$features += "nccl"
|
|
Write-Info "NCCL detected - enabling nccl for CUDA multi-GPU tensor parallelism"
|
|
} elseif ($env:HANZO_INSTALL_NCCL -eq "1") {
|
|
$features += "nccl"
|
|
Write-Warn "HANZO_INSTALL_NCCL=1 set but NCCL was not detected; the build may fail unless NCCL is on the linker path"
|
|
} else {
|
|
Write-Warn "NCCL not found - skipping nccl. Install NCCL or set HANZO_INSTALL_NCCL=1 to force it; NCCL is the preferred CUDA multi-GPU path."
|
|
}
|
|
|
|
# Check for cuDNN
|
|
if (Test-CuDNN) {
|
|
$features += "cudnn"
|
|
Write-Info "cuDNN detected - enabling cudnn"
|
|
} else {
|
|
Write-Info "cuDNN not found - skipping cudnn feature"
|
|
}
|
|
|
|
# Add flash attention based on compute capability
|
|
if ($cudaCC -eq "90") {
|
|
$features += "flash-attn-v3"
|
|
Write-Info "Hopper GPU detected - enabling flash-attn-v3"
|
|
} elseif ([int]$cudaCC -ge 80) {
|
|
$features += "flash-attn"
|
|
Write-Info "Ampere+ GPU detected - enabling flash-attn"
|
|
}
|
|
|
|
# cuTile: optimized CUDA kernels. Needs CUDA >= 13.1 (its JIT tool tileiras ships with 13.1+); runs on Ampere (80-89) or Blackwell+ (>=100), not Hopper (90-99).
|
|
$cudaVer = Get-CudaVersionCode
|
|
$ccNum = [int]$cudaCC
|
|
if ($cudaVer -and $cudaVer -ge 1301 -and ((($ccNum -ge 80) -and ($ccNum -lt 90)) -or ($ccNum -ge 100))) {
|
|
$features += "cutile"
|
|
Write-Info "CUDA >= 13.1 and supported arch - enabling cutile (optimized kernels)"
|
|
}
|
|
} else {
|
|
Write-Info "No NVIDIA GPU detected"
|
|
}
|
|
|
|
# Check for MKL on Intel
|
|
if ((Test-IntelCpu) -and (Test-MKL)) {
|
|
$features += "mkl"
|
|
Write-Info "Intel MKL detected - enabling mkl"
|
|
}
|
|
|
|
return $features -join " "
|
|
}
|
|
|
|
# Install the engine from the repository. There is no registry release of
|
|
# $HanzoCliPackage, so the repository is the only place the binary exists.
|
|
# HANZOAI_VERSION pins a release tag; HANZOAI_INSTALL_DIR redirects the bin dir.
|
|
function Install-Hanzo {
|
|
param([string]$Features)
|
|
|
|
$cargoArgs = @("install", "--git", $HanzoRepoUrl, "--locked", $HanzoCliPackage)
|
|
if ($env:HANZOAI_VERSION) {
|
|
$cargoArgs += @("--tag", $env:HANZOAI_VERSION)
|
|
Write-Info "Pinning release tag $($env:HANZOAI_VERSION)"
|
|
} else {
|
|
$cargoArgs += @("--branch", $HanzoBranch)
|
|
}
|
|
# cargo --root R writes R\bin, but callers name a BIN dir, so trim one \bin.
|
|
if ($env:HANZOAI_INSTALL_DIR) {
|
|
$root = $env:HANZOAI_INSTALL_DIR.TrimEnd('\', '/')
|
|
if ($root -match '[\\/]bin$') { $root = $root.Substring(0, $root.Length - 4) }
|
|
$cargoArgs += @("--root", $root)
|
|
Write-Info "Installing into $root\bin"
|
|
}
|
|
if ($Features) {
|
|
$cargoArgs += @("--features", $Features)
|
|
Write-Info "Installing $HanzoCliPackage with features: $Features"
|
|
} else {
|
|
Write-Info "Installing $HanzoCliPackage with default features"
|
|
}
|
|
& cargo @cargoArgs
|
|
|
|
if ($LASTEXITCODE -ne 0) {
|
|
Write-Err "Installation failed"
|
|
}
|
|
}
|
|
|
|
# Main installation flow
|
|
function Main {
|
|
Show-Banner
|
|
|
|
Write-Info "Detected OS: Windows"
|
|
|
|
# Check for Rust
|
|
if (Test-Rust) {
|
|
$rustVersionFull = & rustc --version 2>$null
|
|
$rustVersion = Get-RustVersion
|
|
Write-Info "Rust is installed: $rustVersionFull"
|
|
|
|
# Check if version meets minimum requirement
|
|
if ($rustVersion -and -not (Test-VersionGte $rustVersion $RequiredRustVersion)) {
|
|
Write-Warn "Rust $rustVersion is below the required version $RequiredRustVersion"
|
|
Write-Host ""
|
|
$response = Read-Host "Would you like to update Rust now? [Y/n]"
|
|
if ($response -match "^[Nn]") {
|
|
Write-Err "Rust $RequiredRustVersion or newer is required to install hanzo"
|
|
}
|
|
Update-Rust
|
|
# Re-check version after update
|
|
$rustVersion = Get-RustVersion
|
|
if (-not (Test-VersionGte $rustVersion $RequiredRustVersion)) {
|
|
Write-Err "Failed to update Rust to required version $RequiredRustVersion"
|
|
}
|
|
}
|
|
} else {
|
|
Write-Warn "Rust is not installed"
|
|
Write-Host ""
|
|
$response = Read-Host "Would you like to install Rust now? [Y/n]"
|
|
if ($response -match "^[Nn]") {
|
|
Write-Err "Rust is required to install hanzo"
|
|
}
|
|
Install-Rust
|
|
}
|
|
|
|
Write-Host ""
|
|
Write-Info "Detecting hardware capabilities..."
|
|
|
|
# Build features
|
|
$features = Get-Features
|
|
|
|
Write-Host ""
|
|
Write-Host "Installation Summary" -ForegroundColor White
|
|
Write-Host "====================" -ForegroundColor White
|
|
if ($features) {
|
|
Write-Host "Features: " -NoNewline
|
|
Write-Host $features -ForegroundColor Green
|
|
} else {
|
|
Write-Host "Features: " -NoNewline
|
|
Write-Host "(none - CPU only)" -ForegroundColor Yellow
|
|
}
|
|
Write-Host ""
|
|
|
|
# Confirm installation
|
|
$response = Read-Host "Proceed with installation? [Y/n]"
|
|
if ($response -match "^[Nn]") {
|
|
Write-Info "Installation cancelled"
|
|
exit 0
|
|
}
|
|
|
|
Write-Host ""
|
|
Install-Hanzo -Features $features
|
|
|
|
Write-Host ""
|
|
Write-Success "hanzo installed successfully!"
|
|
Write-Host ""
|
|
Write-Host "Quick Start" -ForegroundColor White
|
|
Write-Host "===========" -ForegroundColor White
|
|
Write-Host ""
|
|
Write-Host " hanzo run -m Qwen/Qwen3-4B"
|
|
Write-Host ""
|
|
Write-Host " hanzo serve --agent -m google/gemma-4-E4B-it"
|
|
Write-Host ""
|
|
Write-Host "For more information, visit: https://github.com/hanzoai/engine"
|
|
Write-Host ""
|
|
Write-Host "Note: " -ForegroundColor Yellow -NoNewline
|
|
Write-Host "Restart your terminal to use the 'hanzo' command."
|
|
}
|
|
|
|
# Run main
|
|
Main
|