mirror of
https://github.com/zenlm/zen-coder.git
synced 2026-11-10 21:30:01 +00:00
- Rename Qwen2.5-Coder-* examples to zen-coder-* (consistent Zen branding) - Add LLM.md knowledge base (symlinked as CLAUDE.md/AGENTS.md/etc.) - Update .gitignore: exclude model weights and AI symlink files - Add training configs, inference scripts, test suite - Add MLX README, model card, Makefile for common operations
36 lines
1.3 KiB
Python
36 lines
1.3 KiB
Python
from transformers import AutoTokenizer, AutoModelForCausalLM
|
|
from transformers import TextIteratorStreamer
|
|
from threading import Thread
|
|
|
|
device = "cuda" # the device to load the model onto
|
|
|
|
# Now you do not need to add "trust_remote_code=True"
|
|
tokenizer = AutoTokenizer.from_pretrained("zenlm/zen-coder")
|
|
model = AutoModelForCausalLM.from_pretrained("zenlm/zen-coder", device_map="auto").eval()
|
|
|
|
# Instead of using model.chat(), we directly use model.generate()
|
|
# But you need to use tokenizer.apply_chat_template() to format your inputs as shown below
|
|
prompt = "write a quick sort algorithm."
|
|
messages = [
|
|
{"role": "system", "content": "You are Zen. You are a helpful assistant."},
|
|
{"role": "user", "content": prompt}
|
|
]
|
|
text = tokenizer.apply_chat_template(
|
|
messages,
|
|
tokenize=False,
|
|
add_generation_prompt=True
|
|
)
|
|
model_inputs = tokenizer([text], return_tensors="pt").to(device)
|
|
|
|
streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
|
|
|
|
generation_kwargs = dict(inputs=model_inputs.input_ids, streamer=streamer, max_new_tokens=2048)
|
|
thread = Thread(target=model.generate, kwargs=generation_kwargs)
|
|
|
|
thread.start()
|
|
generated_text = ""
|
|
for new_text in streamer:
|
|
generated_text += new_text
|
|
print(new_text, end="")
|
|
print(generated_text)
|