mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
Replace all remaining references to the upstream project/author/framework brands with hanzo / hanzoai across source, comments, docs, examples, and build scripts. - Project spellings mistral.rs / mistral-rs / mistralrs -> hanzo (and the hanzo-* crate/binary names in Dockerfiles, Makefile, install.ps1, .gitignore). - Author EricLBuehler / EricBuehler / EricB / Eric Buehler -> hanzoai / Hanzo AI (incl. Cargo `authors` and LICENSE copyright). - URLs: github.com/EricLBuehler/mistral.rs -> github.com/hanzoai/engine; ericlbuehler.github.io/mistral.rs -> hanzoai.github.io/engine; github.com/huggingface/candle -> github.com/hanzoai/ml; docs.rs/candle-* -> docs.rs/hanzo-ml; ghcr.io image -> ghcr.io/hanzoai/engine. - Env var prefix MISTRALRS_ -> HANZO_ (producers and consumers kept in sync, incl. the build-script-injected HANZO_GIT_REVISION). - ROCm static lib libmistralrsrocm.a / mistralrsrocm -> libhanzorocm.a / hanzorocm. - localStorage keys mistralrs_settings -> hanzo_settings. Deliberately preserved: real model architectures (struct Mistral, models/mistral.rs file, Mixtral/Llama/Gemma/Qwen/Phi), real model-org HF repo-ids (mistralai/, meta-llama/, google/, Qwen/, microsoft/), the `use hanzo_ml as candle` code aliases and `candle::` identifiers, and the upstream-author HF data repo-id suffix in "hanzoai/mistralrs_tests". Both `cargo check` and `cargo check --features vulkan` pass with 0 errors. Co-authored-by: Hanzo Dev <[email protected]>
216 lines
6.5 KiB
Python
216 lines
6.5 KiB
Python
"""
|
|
Example demonstrating multi-model usage with hanzo Python bindings.
|
|
|
|
This example shows how to:
|
|
1. Load a model using Runner
|
|
2. List available models
|
|
3. Use model_id parameter to target specific models
|
|
4. Manage default model selection
|
|
5. Model loading/unloading operations
|
|
|
|
Models used:
|
|
- Multimodal: google/gemma-4-E4B-it (MultimodalArchitecture.Gemma4)
|
|
- Text: Qwen/Qwen3-4B (Architecture.Qwen3)
|
|
"""
|
|
|
|
from hanzo import (
|
|
Runner,
|
|
Which,
|
|
ChatCompletionRequest,
|
|
Architecture,
|
|
MultimodalArchitecture,
|
|
)
|
|
|
|
|
|
# Example 1: Using Runner with model_id parameter for multi-model operations
|
|
def example_runner_with_model_id():
|
|
"""Demonstrate using Runner with model_id in requests."""
|
|
|
|
# Create a runner with Gemma 4 E4B multimodal model
|
|
runner = Runner(
|
|
which=Which.MultimodalPlain(
|
|
model_id="google/gemma-4-E4B-it",
|
|
arch=MultimodalArchitecture.Gemma4,
|
|
),
|
|
in_situ_quant="Q4K",
|
|
)
|
|
|
|
# List available models
|
|
model_ids = runner.list_models()
|
|
print("Available models:", model_ids)
|
|
|
|
# Get default model
|
|
default_model = runner.get_default_model_id()
|
|
print(f"Default model: {default_model}")
|
|
|
|
# Send request with specific model_id
|
|
messages = [{"role": "user", "content": "Hello, how are you?"}]
|
|
request = ChatCompletionRequest(messages=messages, model="default")
|
|
|
|
if model_ids:
|
|
# Request to specific model
|
|
response = runner.send_chat_completion_request(
|
|
request=request, model_id=model_ids[0]
|
|
)
|
|
print(f"Response from {model_ids[0]}:", response.choices[0].message.content)
|
|
|
|
# Request without model_id (uses default)
|
|
response = runner.send_chat_completion_request(request=request)
|
|
print("Response from default model:", response.choices[0].message.content)
|
|
|
|
|
|
# Example 2: Model management operations
|
|
def example_model_management():
|
|
"""Demonstrate model management operations."""
|
|
|
|
runner = Runner(
|
|
which=Which.Plain(
|
|
model_id="Qwen/Qwen3-4B",
|
|
arch=Architecture.Qwen3,
|
|
),
|
|
in_situ_quant="Q4K",
|
|
)
|
|
|
|
# List models with their status
|
|
print("Initial models with status:", runner.list_models_with_status())
|
|
|
|
# Get default model
|
|
current_default = runner.get_default_model_id()
|
|
print(f"Current default model: {current_default}")
|
|
|
|
# Check if a model is loaded
|
|
model_ids = runner.list_models()
|
|
if model_ids:
|
|
is_loaded = runner.is_model_loaded(model_ids[0])
|
|
print(f"Is {model_ids[0]} loaded? {is_loaded}")
|
|
|
|
# In a multi-model setup, you could change the default
|
|
if model_ids and len(model_ids) > 1:
|
|
runner.set_default_model_id(model_ids[1])
|
|
print(f"Changed default model to: {model_ids[1]}")
|
|
|
|
|
|
# Example 3: Model unloading and reloading
|
|
def example_unload_reload():
|
|
"""Demonstrate model unloading and reloading."""
|
|
|
|
runner = Runner(
|
|
which=Which.MultimodalPlain(
|
|
model_id="google/gemma-4-E4B-it",
|
|
arch=MultimodalArchitecture.Gemma4,
|
|
),
|
|
in_situ_quant="Q4K",
|
|
)
|
|
|
|
model_ids = runner.list_models()
|
|
if not model_ids:
|
|
print("No models loaded")
|
|
return
|
|
|
|
model_id = model_ids[0]
|
|
print(f"Initial status: {runner.list_models_with_status()}")
|
|
|
|
# Unload the model to free memory
|
|
# Note: This preserves the model configuration for later reload
|
|
print(f"Unloading model: {model_id}")
|
|
runner.unload_model(model_id)
|
|
|
|
# Check status after unload
|
|
print(f"Status after unload: {runner.list_models_with_status()}")
|
|
print(f"Is {model_id} loaded? {runner.is_model_loaded(model_id)}")
|
|
|
|
# Reload the model when needed
|
|
print(f"Reloading model: {model_id}")
|
|
runner.reload_model(model_id)
|
|
|
|
# Check status after reload
|
|
print(f"Status after reload: {runner.list_models_with_status()}")
|
|
|
|
|
|
# Example 4: Streaming with specific models
|
|
def example_streaming_with_models():
|
|
"""Demonstrate streaming responses from specific models."""
|
|
|
|
runner = Runner(
|
|
which=Which.Plain(
|
|
model_id="Qwen/Qwen3-4B",
|
|
arch=Architecture.Qwen3,
|
|
),
|
|
in_situ_quant="Q4K",
|
|
)
|
|
|
|
messages = [{"role": "user", "content": "Tell me a short story"}]
|
|
request = ChatCompletionRequest(messages=messages, model="default", stream=True)
|
|
|
|
model_ids = runner.list_models()
|
|
if model_ids:
|
|
# Stream from specific model
|
|
stream = runner.send_chat_completion_request(
|
|
request=request, model_id=model_ids[0]
|
|
)
|
|
|
|
print(f"Streaming from {model_ids[0]}:")
|
|
for chunk in stream:
|
|
if chunk.choices[0].delta.content:
|
|
print(chunk.choices[0].delta.content, end="", flush=True)
|
|
print() # New line after streaming
|
|
|
|
|
|
# Example 5: Multi-model setup with multimodal and text models
|
|
def example_multi_model_setup():
|
|
"""
|
|
Example showing a real multi-model setup with multimodal and text models.
|
|
|
|
This example loads:
|
|
- Multimodal model: google/gemma-4-E4B-it
|
|
- Text model: Qwen/Qwen3-4B
|
|
"""
|
|
# Load a multimodal model first
|
|
runner = Runner(
|
|
which=Which.MultimodalPlain(
|
|
model_id="google/gemma-4-E4B-it",
|
|
arch=MultimodalArchitecture.Gemma4,
|
|
),
|
|
in_situ_quant="Q4K",
|
|
)
|
|
|
|
print("Initial models:", runner.list_models())
|
|
|
|
# Add a text model dynamically (if add_model is available)
|
|
# runner.add_model(
|
|
# model_id="qwen",
|
|
# which=Which.Plain(
|
|
# model_id="Qwen/Qwen3-4B",
|
|
# arch=Architecture.Qwen3,
|
|
# ),
|
|
# in_situ_quant="Q4K",
|
|
# )
|
|
# print("After add_model:", runner.list_models())
|
|
|
|
# Send a request to gemma
|
|
messages = [{"role": "user", "content": "What is 2 + 2?"}]
|
|
request = ChatCompletionRequest(messages=messages, model="default")
|
|
response = runner.send_chat_completion_request(request)
|
|
print(f"Gemma response: {response.choices[0].message.content}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
print("=== Multi-Model Example 1: Runner with model_id ===")
|
|
example_runner_with_model_id()
|
|
print("\n" + "=" * 50 + "\n")
|
|
|
|
print("=== Multi-Model Example 2: Model Management ===")
|
|
example_model_management()
|
|
print("\n" + "=" * 50 + "\n")
|
|
|
|
print("=== Multi-Model Example 3: Unload/Reload ===")
|
|
example_unload_reload()
|
|
print("\n" + "=" * 50 + "\n")
|
|
|
|
print("=== Multi-Model Example 4: Streaming ===")
|
|
example_streaming_with_models()
|
|
print("\n" + "=" * 50 + "\n")
|
|
|
|
print("=== Multi-Model Example 5: Multi-Model Setup ===")
|
|
example_multi_model_setup()
|