Files
engine/examples/server/phi3v_local_img.py
Eric BuehlerandGitHub 5537355ce6 v0.7.0 preparation: mistralrs-cli, revamped documentation, readme update (#1848)
* Add cli

* Add quantize command

* Works

* Not interactive mode for quantize

* Not prefix cache for quantize

* Fix scheduler off by one bug

* Add config toml

* Add ui, doctor, tune

* Add static files

* Better message

* Fix max batch size default

* Remove dead code

* Format

* Store git rev

* Fix a few bugs with ui upload

* Clippy

* Better error on quantized file missing

* Better error on quantized file missing

* Better error

* Tweak doctor cehcks

* Format

* Add cli docs

* Better tue and doctor

* Tweaks

* Handle max context

* Remove concurrent

* Add more subcomands

* Remove prune

* Fix command

* Fix handling of bench prompt

* Fix prompt timings

* Add version and help

* Clarify interactive

* Cleaner

* Show latency

* Show toml

* Fix default

* Also log for toml

* Better toml docs

* Format for device layres

* Change from >=75 to >=80 for FA

* Add install scripts

* Update all docs

* Apply prefix in server too

* Redo readme

* Fix default port

* Fix image tokens

* Mark old crates as deprecated and warn

* Mention cli

* Finalize readme

* Add doctor

* Fix install script

* Fix curl

* Add nccl feature

* Tweak readme

* Fixes for install scripts

* Fix confusing HF error with auto, fix timings for prompt

* Switch serve models

* Unify hf paths + token resolution

* Add config link, tweak quants docs

* Update typos

* Fix block metadata bug

* Fix tagline

* All models extract logitcs before lmhead

* Show guide to quickstart or restart

* Add banner

* Tweaks

* More tweaks

* Contributing and credits

* Simplify python docs

* Add mdbook, deploy workflow

* Add banner

* Fix deploy

* Dark theme to ayu

* Add python sdk to main docs

* Add python sdk to main docs

* Link to crates.io

* Default run

* Update to point at readme

* Remove dead configs

* Update version, ring feature too

* Fix images rendering, better docs for features

* Fix version bump

* Clippy

* Dont doc mistralrs-cli

* Fix link warn
2026-01-27 15:58:40 -05:00

68 lines
1.8 KiB
Python

import requests
import httpx
import textwrap
import json
def log_response(response: httpx.Response):
request = response.request
print(f"Request: {request.method} {request.url}")
print(" Headers:")
for key, value in request.headers.items():
if key.lower() == "authorization":
value = "[...]"
if key.lower() == "cookie":
value = value.split("=")[0] + "=..."
print(f" {key}: {value}")
print(" Body:")
try:
request_body = json.loads(request.content)
print(textwrap.indent(json.dumps(request_body, indent=2), " "))
except json.JSONDecodeError:
print(textwrap.indent(request.content.decode(), " "))
print(f"Response: status_code={response.status_code}")
print(" Headers:")
for key, value in response.headers.items():
if key.lower() == "set-cookie":
value = value.split("=")[0] + "=..."
print(f" {key}: {value}")
BASE_URL = "http://localhost:1234/v1"
# Enable this to log requests and responses
# openai.http_client = httpx.Client(
# event_hooks={"request": [print], "response": [log_response]}
# )
FILENAME = "picture.jpg"
headers = {
"Content-Type": "application/json",
}
payload = {
"model": "phi3v",
"messages": [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": FILENAME,
},
},
{
"type": "text",
"text": "What is shown in this image? Write a detailed response analyzing the scene.",
},
],
}
],
"max_tokens": 300,
}
response = requests.post(f"{BASE_URL}/chat/completions", headers=headers, json=payload)
print(response.json())