mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
21 lines
396 B
TOML
21 lines
396 B
TOML
[model]
|
|
model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
|
arch = "mistral"
|
|
|
|
[anymoe]
|
|
dataset_json = "examples/amoe.json"
|
|
prefix = "model.layers"
|
|
mlp = "mlp"
|
|
model_ids = ["typeof/zephyr-7b-beta-lora"]
|
|
|
|
[anymoe.config]
|
|
hidden_size = 4096
|
|
epochs = 25
|
|
gate_model_id = "saved_gate"
|
|
loss_csv_path = "loss.csv"
|
|
|
|
[anymoe.config.expert_type.lora_adapter]
|
|
rank = 64
|
|
alpha = 16
|
|
target_modules = ["gate_proj"]
|