mirror of
https://github.com/zenlm/zen4-ultra-trainer.git
synced 2026-10-30 19:51:37 +00:00
QLoRA fine-tuning for Kimi K2.5 (1.04T MoE) with gate/router unfreezing. Standard abliteration fails on MoE — backprop through router instead. - train_zen4_ultra.py: SFT + DPO training modes - merge_and_upload.py: Merge LoRA adapters into base - generate_compliance_data.py: Generate compliance training data - app.py: HuggingFace Spaces Gradio app - data/: Identity training data (736 examples)
178 lines
6.5 KiB
Python
178 lines
6.5 KiB
Python
"""
|
|
Zen4 Ultra Training — HuggingFace Spaces
|
|
|
|
QLoRA fine-tuning for Kimi K2.5 (1.04T MoE) with gate/router unfreezing.
|
|
Standard abliteration fails on MoE — this uses backprop through the router instead.
|
|
|
|
Requires: A100 80GB x4 Space
|
|
"""
|
|
|
|
import os
|
|
import gradio as gr
|
|
import torch
|
|
from transformers import (
|
|
AutoModelForCausalLM, AutoTokenizer, TrainingArguments,
|
|
Trainer, BitsAndBytesConfig, DataCollatorForLanguageModeling,
|
|
)
|
|
from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
|
|
from datasets import load_dataset
|
|
from huggingface_hub import HfApi
|
|
|
|
BASE_MODEL = "moonshotai/Kimi-K2.5"
|
|
|
|
# K2.5 target modules (DeepseekV3 architecture)
|
|
TARGET_MODULES = [
|
|
"q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj",
|
|
"shared_experts.gate_proj", "shared_experts.up_proj", "shared_experts.down_proj",
|
|
]
|
|
|
|
DATASETS = {
|
|
"zen4-identity": "zenlm/zen-identity",
|
|
"dolphin-r1": "cognitivecomputations/dolphin-r1",
|
|
"hermes-2": "NousResearch/hermes-2",
|
|
"local (data/train.jsonl)": "./data/train.jsonl",
|
|
}
|
|
|
|
|
|
def train(dataset_key, lr, epochs, batch_size, lora_rank, unfreeze_gate, push_repo, progress=gr.Progress()):
|
|
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
if device == "cpu":
|
|
return "No GPU available. Use a GPU-enabled Space (A100 x4 recommended)."
|
|
|
|
progress(0.05, desc="Loading tokenizer...")
|
|
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
|
|
if tokenizer.pad_token is None:
|
|
tokenizer.pad_token = tokenizer.eos_token
|
|
|
|
progress(0.1, desc=f"Loading {BASE_MODEL} (INT4)...")
|
|
bnb_config = BitsAndBytesConfig(
|
|
load_in_4bit=True, bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_use_double_quant=True,
|
|
)
|
|
model = AutoModelForCausalLM.from_pretrained(
|
|
BASE_MODEL, quantization_config=bnb_config,
|
|
device_map="auto", trust_remote_code=True, torch_dtype=torch.bfloat16,
|
|
)
|
|
model = prepare_model_for_kbit_training(model)
|
|
|
|
progress(0.3, desc="Applying LoRA + gate unfreeze...")
|
|
lora_config = LoraConfig(
|
|
r=lora_rank, lora_alpha=lora_rank * 2, lora_dropout=0.05,
|
|
target_modules=TARGET_MODULES, bias="none", task_type="CAUSAL_LM",
|
|
)
|
|
model = get_peft_model(model, lora_config)
|
|
|
|
if unfreeze_gate:
|
|
gate_count = 0
|
|
for name, param in model.named_parameters():
|
|
if ".gate.weight" in name and "gate_proj" not in name:
|
|
param.requires_grad = True
|
|
gate_count += 1
|
|
status = f"LoRA applied. Unfroze {gate_count} gate/router layers.\n"
|
|
else:
|
|
status = "LoRA applied (gate frozen).\n"
|
|
|
|
trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
|
|
total = sum(p.numel() for p in model.parameters())
|
|
status += f"Trainable: {trainable:,} / {total:,} ({100*trainable/total:.2f}%)\n"
|
|
|
|
progress(0.4, desc="Loading dataset...")
|
|
ds_path = DATASETS.get(dataset_key, dataset_key)
|
|
if ds_path.endswith(".jsonl"):
|
|
dataset = load_dataset("json", data_files=ds_path, split="train")
|
|
else:
|
|
dataset = load_dataset(ds_path, split="train")
|
|
|
|
# Auto-format
|
|
cols = dataset.column_names
|
|
if "messages" in cols:
|
|
def fmt(ex):
|
|
return {"text": tokenizer.apply_chat_template(ex["messages"], tokenize=False)}
|
|
dataset = dataset.map(fmt)
|
|
elif "text" in cols:
|
|
pass
|
|
elif "instruction" in cols:
|
|
def fmt(ex):
|
|
return {"text": f"<|im_start|>user\n{ex['instruction']}<|im_end|>\n<|im_start|>assistant\n{ex['output']}<|im_end|>"}
|
|
dataset = dataset.map(fmt)
|
|
|
|
def tokenize(examples):
|
|
return tokenizer(examples["text"], truncation=True, max_length=4096, padding="max_length")
|
|
|
|
tokenized = dataset.map(tokenize, batched=True, remove_columns=dataset.column_names)
|
|
status += f"Dataset: {len(tokenized)} examples\n"
|
|
|
|
progress(0.5, desc="Training...")
|
|
output_dir = "./zen4-ultra-lora"
|
|
training_args = TrainingArguments(
|
|
output_dir=output_dir, num_train_epochs=epochs,
|
|
per_device_train_batch_size=batch_size, gradient_accumulation_steps=16,
|
|
learning_rate=lr, warmup_ratio=0.03, logging_steps=1, save_steps=50,
|
|
bf16=True, optim="paged_adamw_8bit", gradient_checkpointing=True,
|
|
report_to="none",
|
|
)
|
|
|
|
trainer = Trainer(
|
|
model=model, args=training_args, train_dataset=tokenized,
|
|
data_collator=DataCollatorForLanguageModeling(tokenizer, mlm=False),
|
|
)
|
|
trainer.train()
|
|
|
|
progress(0.9, desc="Saving...")
|
|
model.save_pretrained(output_dir)
|
|
tokenizer.save_pretrained(output_dir)
|
|
|
|
if push_repo:
|
|
progress(0.95, desc=f"Pushing to {push_repo}...")
|
|
api = HfApi()
|
|
api.upload_folder(folder_path=output_dir, repo_id=push_repo, repo_type="model",
|
|
commit_message="zen4-ultra LoRA adapters")
|
|
status += f"Pushed to https://huggingface.co/{push_repo}\n"
|
|
|
|
status += "Done!"
|
|
return status
|
|
|
|
|
|
with gr.Blocks(title="Zen4 Ultra Training") as demo:
|
|
gr.Markdown("""
|
|
# Zen4 Ultra Training
|
|
|
|
QLoRA fine-tuning for Kimi K2.5 (1.04T MoE, 32B active).
|
|
|
|
**Key innovation**: Unfreezes MoE gate/router weights alongside LoRA on attention + shared experts.
|
|
Standard abliteration fails on MoE because refusal is encoded in expert routing — this approach
|
|
uses backpropagation to modify the router directly.
|
|
|
|
| Component | Target |
|
|
|-----------|--------|
|
|
| Attention | q_a_proj, q_b_proj, kv_a/b_proj, o_proj |
|
|
| Shared Experts | gate_proj, up_proj, down_proj |
|
|
| Router/Gate | Unfrozen (nn.Parameter, not LoRA) |
|
|
|
|
Requires **4x A100 80GB** Space.
|
|
""")
|
|
|
|
with gr.Row():
|
|
dataset_select = gr.Dropdown(
|
|
choices=list(DATASETS.keys()), value="zen4-identity", label="Dataset"
|
|
)
|
|
push_repo = gr.Textbox(value="zenlm/zen4-ultra", label="Push to (blank=skip)")
|
|
|
|
with gr.Row():
|
|
lr = gr.Slider(1e-6, 1e-3, value=2e-5, label="Learning Rate")
|
|
epochs = gr.Slider(1, 5, value=2, step=1, label="Epochs")
|
|
|
|
with gr.Row():
|
|
batch = gr.Slider(1, 4, value=1, step=1, label="Batch Size")
|
|
rank = gr.Slider(8, 64, value=32, step=8, label="LoRA Rank")
|
|
|
|
unfreeze = gr.Checkbox(value=True, label="Unfreeze gate/router weights (recommended)")
|
|
train_btn = gr.Button("Train", variant="primary")
|
|
output = gr.Textbox(label="Status", lines=10)
|
|
|
|
train_btn.click(train, [dataset_select, lr, epochs, batch, rank, unfreeze, push_repo], output)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
demo.launch()
|