mirror of
https://github.com/hanzoai/open-instruct.git
synced 2026-10-30 18:26:32 +00:00
36 lines
1.2 KiB
Bash
36 lines
1.2 KiB
Bash
python open_instruct/ppo_vllm_thread_ray_gtrl.py \
|
|
--dataset_mixer '{"ai2-adapt-dev/gsm8k_math_ifeval_ground_truth_mixed": 1.0}' \
|
|
--dataset_train_splits train \
|
|
--dataset_eval_mixer '{"ai2-adapt-dev/gsm8k_math_ground_truth": 1.0}' \
|
|
--dataset_eval_splits test \
|
|
--max_token_length 2048 \
|
|
--max_prompt_token_length 2048 \
|
|
--response_length 2048 \
|
|
--model_name_or_path allenai/Llama-3.1-Tulu-3-8B-DPO \
|
|
--reward_model_path allenai/Llama-3.1-Tulu-3-8B-RM \
|
|
--non_stop_penalty \
|
|
--stop_token eos \
|
|
--temperature 1.0 \
|
|
--ground_truths_key ground_truth \
|
|
--chat_template tulu \
|
|
--sft_messages_key messages \
|
|
--learning_rate 3e-7 \
|
|
--total_episodes 10000000 \
|
|
--penalty_reward_value -10.0 \
|
|
--deepspeed_stage 3 \
|
|
--per_device_train_batch_size 2 \
|
|
--local_rollout_forward_batch_size 2 \
|
|
--local_mini_batch_size 32 \
|
|
--local_rollout_batch_size 32 \
|
|
--actor_num_gpus_per_node 7 \
|
|
--vllm_tensor_parallel_size 1 \
|
|
--beta 0.05 \
|
|
--apply_verifiable_reward true \
|
|
--output_dir output/rlvr_8b \
|
|
--seed 3 \
|
|
--num_evals 3 \
|
|
--save_freq 100 \
|
|
--reward_model_multiplier 0.0 \
|
|
--gradient_checkpointing \
|
|
--with_tracking
|