Files
zen-coder/examples/zen-coder-instruct-stream.py
zeekay 89da15d070 refactor: zen-coder branding - rename examples, add LLM.md and training configs
- Rename Qwen2.5-Coder-* examples to zen-coder-* (consistent Zen branding)
- Add LLM.md knowledge base (symlinked as CLAUDE.md/AGENTS.md/etc.)
- Update .gitignore: exclude model weights and AI symlink files
- Add training configs, inference scripts, test suite
- Add MLX README, model card, Makefile for common operations
2026-02-27 19:06:12 -08:00

36 lines
1.3 KiB
Python

from transformers import AutoTokenizer, AutoModelForCausalLM
from transformers import TextIteratorStreamer
from threading import Thread
device = "cuda" # the device to load the model onto
# Now you do not need to add "trust_remote_code=True"
tokenizer = AutoTokenizer.from_pretrained("zenlm/zen-coder")
model = AutoModelForCausalLM.from_pretrained("zenlm/zen-coder", device_map="auto").eval()
# Instead of using model.chat(), we directly use model.generate()
# But you need to use tokenizer.apply_chat_template() to format your inputs as shown below
prompt = "write a quick sort algorithm."
messages = [
{"role": "system", "content": "You are Zen. You are a helpful assistant."},
{"role": "user", "content": prompt}
]
text = tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True
)
model_inputs = tokenizer([text], return_tensors="pt").to(device)
streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True)
generation_kwargs = dict(inputs=model_inputs.input_ids, streamer=streamer, max_new_tokens=2048)
thread = Thread(target=model.generate, kwargs=generation_kwargs)
thread.start()
generated_text = ""
for new_text in streamer:
generated_text += new_text
print(new_text, end="")
print(generated_text)