mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
Eric BuehlerandGitHub
4cd824d9e4
* Add initial * Loading for text model * Add ple embeddings * Add altup, laurel block * Update rmsnorm * Add mlp * Update attn norm application * Currently no kv shared * Wire it up * It runs * Fix bf16 * Fix scaled embd * Fixes for mean * tmp * Attn confirmed * Fix target_magnitude * Add shared kv * Ok it works * Remove npy * Fix streaming * Remove warnings * Remove paged attn * Refactor rope * Add immediate isq * Add vision & mproj * Update image processor * Vision merge runs, not correct * Remove * Add mobilenet v5 * Add multimodal vision embedding * Fix load * runs * Fix gamma * Works but just not vision tower * It works!! * Tweak * Fix warnings * Move vision tower * Fix warn * Update cache manager things * Refactor * Add audio model, it loads * Add audio processing * It runs at least * tmp * A bit better * Audio works!!!! * Fused attn in vision * Clippy * Update audio runner * Optimized audio model * Remove unused things * Fix inputs processor bug * Remove comments * Clippy * Small optimizations * Format * Correctly register modalities * Add docs * Update readme * Runs there * Fixed padding from https://github.com/Blaizzy/mlx-vlm/pull/410 * Add better checks * Fix sdpa n_kv_groups * Vision encoder works! * Rotate image * Clippy * Fix cuda loading * Updated device mapper * Fix overflow * Fix dtype errors * Refactor image/audio embeddings * Fix metal * Fix dtype mismatch * Audio processing fixes * Audio processing fixes * Works * Audio is good * Fix boi/eoi too * Embed the chat templates * Better embedding accuracy in non f32 * More f32 * Support bf16 on metal * Add more ISQ * Fixed device map * Clippy * Gemma3n no paged attn * Fix saturating sub * Faster rmsnorm * Use sdpa for vision model * Fix ple bug * Fix name * Fix multiaudio * Add matformer config loading * Add docs * Add support for matformer in auto device mapper * Update docs * Typos * Tweak * Tweak * Fix multidevice * Fix gemma3n text model auto device map * Fix dims3 * Fix auto devic emap vision * Non-metal keeps PLE on cpu * Complete merge * Vision dtype f16 -> f32 * Fix metal nm device * Fix uqff * Typos * Reference uqff * Fix tests
64 lines
1.8 KiB
Python
64 lines
1.8 KiB
Python
from openai import OpenAI
|
|
import httpx
|
|
import textwrap
|
|
import json
|
|
|
|
|
|
def log_response(response: httpx.Response):
|
|
request = response.request
|
|
print(f"Request: {request.method} {request.url}")
|
|
print(" Headers:")
|
|
for key, value in request.headers.items():
|
|
if key.lower() == "authorization":
|
|
value = "[...]"
|
|
if key.lower() == "cookie":
|
|
value = value.split("=")[0] + "=..."
|
|
print(f" {key}: {value}")
|
|
print(" Body:")
|
|
try:
|
|
request_body = json.loads(request.content)
|
|
print(textwrap.indent(json.dumps(request_body, indent=2), " "))
|
|
except json.JSONDecodeError:
|
|
print(textwrap.indent(request.content.decode(), " "))
|
|
print(f"Response: status_code={response.status_code}")
|
|
print(" Headers:")
|
|
for key, value in response.headers.items():
|
|
if key.lower() == "set-cookie":
|
|
value = value.split("=")[0] + "=..."
|
|
print(f" {key}: {value}")
|
|
|
|
|
|
client = OpenAI(api_key="foobar", base_url="http://localhost:1234/v1/")
|
|
|
|
# Enable this to log requests and responses
|
|
# client._client = httpx.Client(
|
|
# event_hooks={"request": [print], "response": [log_response]}
|
|
# )
|
|
|
|
completion = client.chat.completions.create(
|
|
model="ignore",
|
|
messages=[
|
|
{
|
|
"role": "user",
|
|
"content": [
|
|
{
|
|
"type": "image_url",
|
|
"image_url": {
|
|
"url": "https://upload.wikimedia.org/wikipedia/commons/f/fd/Pink_flower.jpg"
|
|
},
|
|
},
|
|
{
|
|
"type": "text",
|
|
"text": "Please describe this image in detail.",
|
|
},
|
|
],
|
|
},
|
|
],
|
|
max_tokens=256,
|
|
frequency_penalty=1.0,
|
|
top_p=0.1,
|
|
temperature=0,
|
|
)
|
|
resp = completion.choices[0].message.content
|
|
print(resp)
|