mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
c3be7b6be4
* Add model * Update loader * Fix llama config usage * Docs * Fix config no_rope_layers * Fix tie_word_embeddings default * Add chat template * Embed the chat templates * Fix embedding template * enable_thinking default true * Update examples * XML tools for smollm3 * Add smollm3 docs * Fix openai examples * Clippy --------- Co-authored-by: Eric Buehler <[email protected]>
50 lines
1.0 KiB
Python
50 lines
1.0 KiB
Python
import openai
|
|
from datetime import datetime
|
|
|
|
Runs = 4
|
|
|
|
ENDPOINT = "http://localhost:1234/v1/"
|
|
|
|
|
|
def request(stream: bool):
|
|
client = openai.Client(api_key="foobar", base_url=ENDPOINT)
|
|
return client.chat.completions.create(
|
|
model="default",
|
|
messages=[
|
|
{
|
|
"role": "user",
|
|
"content": "What is the meaning of life? Write a long story.",
|
|
}
|
|
],
|
|
stream=stream,
|
|
max_tokens=400,
|
|
temperature=0.0,
|
|
)
|
|
|
|
|
|
def run():
|
|
for run in range(Runs):
|
|
print("\nCompletion: ")
|
|
print("=" * 15)
|
|
|
|
now = datetime.now()
|
|
request(stream=False)
|
|
finished = datetime.now()
|
|
|
|
print(f"Duration: {finished - now}")
|
|
|
|
print("\nStreaming: ")
|
|
print("=" * 15)
|
|
|
|
now = datetime.now()
|
|
stream = request(stream=True)
|
|
for _message in stream:
|
|
pass
|
|
finished = datetime.now()
|
|
|
|
print(f"Duration: {finished - now}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|