mirror of
https://github.com/hanzoai/engine.git
synced 2026-11-20 14:17:52 +00:00
* Add cli * Add quantize command * Works * Not interactive mode for quantize * Not prefix cache for quantize * Fix scheduler off by one bug * Add config toml * Add ui, doctor, tune * Add static files * Better message * Fix max batch size default * Remove dead code * Format * Store git rev * Fix a few bugs with ui upload * Clippy * Better error on quantized file missing * Better error on quantized file missing * Better error * Tweak doctor cehcks * Format * Add cli docs * Better tue and doctor * Tweaks * Handle max context * Remove concurrent * Add more subcomands * Remove prune * Fix command * Fix handling of bench prompt * Fix prompt timings * Add version and help * Clarify interactive * Cleaner * Show latency * Show toml * Fix default * Also log for toml * Better toml docs * Format for device layres * Change from >=75 to >=80 for FA * Add install scripts * Update all docs * Apply prefix in server too * Redo readme * Fix default port * Fix image tokens * Mark old crates as deprecated and warn * Mention cli * Finalize readme * Add doctor * Fix install script * Fix curl * Add nccl feature * Tweak readme * Fixes for install scripts * Fix confusing HF error with auto, fix timings for prompt * Switch serve models * Unify hf paths + token resolution * Add config link, tweak quants docs * Update typos * Fix block metadata bug * Fix tagline * All models extract logitcs before lmhead * Show guide to quickstart or restart * Add banner * Tweaks * More tweaks * Contributing and credits * Simplify python docs * Add mdbook, deploy workflow * Add banner * Fix deploy * Dark theme to ayu * Add python sdk to main docs * Add python sdk to main docs * Link to crates.io * Default run * Update to point at readme * Remove dead configs * Update version, ring feature too * Fix images rendering, better docs for features * Fix version bump * Clippy * Dont doc mistralrs-cli * Fix link warn
68 lines
1.8 KiB
Python
68 lines
1.8 KiB
Python
import requests
|
|
import httpx
|
|
import textwrap
|
|
import json
|
|
|
|
|
|
def log_response(response: httpx.Response):
|
|
request = response.request
|
|
print(f"Request: {request.method} {request.url}")
|
|
print(" Headers:")
|
|
for key, value in request.headers.items():
|
|
if key.lower() == "authorization":
|
|
value = "[...]"
|
|
if key.lower() == "cookie":
|
|
value = value.split("=")[0] + "=..."
|
|
print(f" {key}: {value}")
|
|
print(" Body:")
|
|
try:
|
|
request_body = json.loads(request.content)
|
|
print(textwrap.indent(json.dumps(request_body, indent=2), " "))
|
|
except json.JSONDecodeError:
|
|
print(textwrap.indent(request.content.decode(), " "))
|
|
print(f"Response: status_code={response.status_code}")
|
|
print(" Headers:")
|
|
for key, value in response.headers.items():
|
|
if key.lower() == "set-cookie":
|
|
value = value.split("=")[0] + "=..."
|
|
print(f" {key}: {value}")
|
|
|
|
|
|
BASE_URL = "http://localhost:1234/v1"
|
|
|
|
# Enable this to log requests and responses
|
|
# openai.http_client = httpx.Client(
|
|
# event_hooks={"request": [print], "response": [log_response]}
|
|
# )
|
|
|
|
FILENAME = "picture.jpg"
|
|
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
}
|
|
|
|
payload = {
|
|
"model": "phi3v",
|
|
"messages": [
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {
|
|
"url": FILENAME,
|
|
},
|
|
},
|
|
{
|
|
"type": "text",
|
|
"text": "What is shown in this image? Write a detailed response analyzing the scene.",
|
|
},
|
|
],
|
|
}
|
|
],
|
|
"max_tokens": 300,
|
|
}
|
|
|
|
response = requests.post(f"{BASE_URL}/chat/completions", headers=headers, json=payload)
|
|
print(response.json())
|