mirror of
https://github.com/zenlm/engine.git
synced 2026-10-30 15:00:05 +00:00
18 lines
395 B
TOML
18 lines
395 B
TOML
[model]
|
|
model_id = "mistralai/Mistral-7B-Instruct-v0.1"
|
|
arch = "mistral"
|
|
|
|
[anymoe]
|
|
dataset_json = "examples/amoe.json"
|
|
prefix = "model.layers"
|
|
mlp = "mlp"
|
|
model_ids = ["HuggingFaceH4/zephyr-7b-beta"]
|
|
layers = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
|
|
|
|
[anymoe.config]
|
|
hidden_size = 4096
|
|
epochs = 25
|
|
expert_type = "fine_tuned"
|
|
gate_model_id = "saved_gate"
|
|
loss_csv_path = "loss.csv"
|