mirror of
https://github.com/hanzoai/open-instruct.git
synced 2026-11-03 02:13:07 +00:00
Jacob MorrisonandGitHub
3d0039fb47
Adding support for OLMoE SFT + DPO w/ and w/o load balancing loss
27 lines
834 B
YAML
27 lines
834 B
YAML
model_name_or_path: allenai/OLMoE-1B-7B-0924
|
|
model_revision: main
|
|
tokenizer_name: allenai/OLMoE-1B-7B-0924
|
|
use_slow_tokenizer: true
|
|
dataset_mixer:
|
|
allenai/tulu-v3-mix-preview-4096-OLMoE: 1.0
|
|
ai2-adapt-dev/daring-anteater-specialized: 1.0
|
|
max_seq_length: 4096 # Note, reduced from 8192 to fit on one GPU with DeepSpeed Stage3
|
|
preprocessing_num_workers: 128
|
|
per_device_train_batch_size: 2 # note, this is set up for 8 GPUs
|
|
gradient_accumulation_steps: 2 # effective batch size 128 with 4 nodes
|
|
learning_rate: 2.0e-05
|
|
lr_scheduler_type: linear
|
|
warmup_ratio: 0.03
|
|
weight_decay: 0.0
|
|
num_train_epochs: 2
|
|
output_dir: /output/
|
|
with_tracking: true
|
|
report_to:
|
|
- wandb
|
|
logging_steps: 1
|
|
dataset_mix_dir: /output/
|
|
checkpointing_steps: 1000
|
|
keep_last_n_checkpoints: 1
|
|
load_balancing_loss: true # TODO: set to false
|
|
load_balancing_weight: 0.5
|