mirror of
https://github.com/zenlm/zen-coder.git
synced 2026-11-21 11:29:28 +00:00
- Rename Qwen2.5-Coder-* examples to zen-coder-* (consistent Zen branding) - Add LLM.md knowledge base (symlinked as CLAUDE.md/AGENTS.md/etc.) - Update .gitignore: exclude model weights and AI symlink files - Add training configs, inference scripts, test suite - Add MLX README, model card, Makefile for common operations
52 lines
1.3 KiB
YAML
52 lines
1.3 KiB
YAML
# AutoTrain Config: Devstral Small 2 (24B) - Apache 2.0 License
|
|
# Best coding model for fine-tuning - 256K context
|
|
# Run: autotrain llm --config devstral_small_24b_config.yaml
|
|
|
|
task: llm-sft
|
|
log: tensorboard
|
|
|
|
# Model - Devstral Small 2 (24B, Apache 2.0)
|
|
base_model: mistralai/Devstral-Small-2-24B-Instruct-2512
|
|
|
|
# Dataset
|
|
data_path: hanzoai/zen-agentic-dataset
|
|
train_split: train
|
|
valid_split: valid
|
|
chat_template: mistral # Mistral chat format
|
|
|
|
# Column mapping
|
|
column_mapping:
|
|
text_column: messages
|
|
|
|
# Training - optimized for 24B model
|
|
learning_rate: 1e-5 # Lower LR for larger model
|
|
num_epochs: 2
|
|
batch_size: 1 # Smaller batch for 24B
|
|
gradient_accumulation: 8 # Effective batch = 8
|
|
block_size: 8192 # Devstral supports 256K, use 8K for training
|
|
warmup_ratio: 0.05
|
|
weight_decay: 0.01
|
|
max_grad_norm: 1.0
|
|
|
|
# LoRA - essential for 24B model
|
|
peft: true
|
|
quantization: int4 # QLoRA - required for 24B on consumer GPUs
|
|
lora_r: 32 # Lower rank for larger model
|
|
lora_alpha: 64
|
|
lora_dropout: 0.05
|
|
target_modules: all-linear
|
|
|
|
# Optimization
|
|
mixed_precision: bf16
|
|
gradient_checkpointing: true
|
|
|
|
# Output
|
|
project_name: zen-coder-devstral-small-24b
|
|
push_to_hub: false
|
|
# hub_model_id: zenlm/zen-coder-devstral-24b-instruct
|
|
|
|
# Memory requirements:
|
|
# - Full precision: ~96GB VRAM
|
|
# - QLoRA (int4): ~24GB VRAM (fits on RTX 4090, A100)
|
|
# - For M3 Max 64GB: Use MLX instead of AutoTrain
|