mirror of
https://github.com/hanzoai/open-instruct.git
synced 2026-11-03 02:13:07 +00:00
fdd1091954
* up * init mixer * init * add early and clearer check * up * tests * style * up * install from requirements * up * force packaging install * up * fiddle * fiddle * Add safety data to data mixer yaml files (#197) * add yaml files for safety data * update max_seq_length * update max_seq_length * fixes * style * fixes * runs * Datamix safety: Tulu2Mix data added (#203) * add yaml files for safety data * update max_seq_length * update max_seq_length * update data paths --------- Co-authored-by: Nathan Lambert <[email protected]> --------- Co-authored-by: Nouha Dziri <[email protected]>
28 lines
1.1 KiB
YAML
28 lines
1.1 KiB
YAML
# Note, the exact model was trained on TPUs in a different repo
|
|
# Example file for mixing, use exact tulu 2 mixture for replication allenai/tulu-v2-sft-mixture
|
|
model_name_or_path: allenai/OLMo-1.7-7B-hf
|
|
model_revision: main
|
|
use_flash_attn: true
|
|
tokenizer_name: allenai/OLMo-1.7-7B-hf
|
|
use_slow_tokenizer: true
|
|
dataset_mixer:
|
|
allenai/wildjailbreak: 262000
|
|
/net/nfs.cirrascale/mosaic/oe-safety-datasets/vanilla_harmful_v.03/sampled_data/gpt-3.5-turbo_synthetic_harm_v03.jsonl: 21876
|
|
/net/nfs.cirrascale/mosaic/oe-safety-datasets/wildchat_lmsys_sexual/gpt4_lmsys_wildchat_dedup_50ksampled.jsonl: 16888
|
|
allenai/tulu-v2-sft-mixture: 326154
|
|
max_seq_length: 4096 # Note, reduced from 8192 to fit on one GPU with DeepSpeed Stage3
|
|
preprocessing_num_workers: 128
|
|
per_device_train_batch_size: 1 # note, this is set up for 8 GPUs
|
|
gradient_accumulation_steps: 16 # effective batch size 128 for tulu 2
|
|
learning_rate: 2.0e-05
|
|
lr_scheduler_type: linear
|
|
warmup_ratio: 0.03
|
|
weight_decay: 0.0
|
|
num_train_epochs: 3
|
|
output_dir: output/OLMo-1.7-7B-safe-remix
|
|
with_tracking: true
|
|
report_to:
|
|
- wandb
|
|
logging_steps: 1
|
|
checkpointing_steps: epoch
|