mirror of
https://github.com/hanzoai/open-instruct.git
synced 2026-10-30 18:26:32 +00:00
74069db498
* Reorganize the data preparation scripts for tulu v1 and v2. * Minor improvement * Remove open_platypus_commercial subset from Daring-Anteater * Use hard-coded examples repo. * Fix some bugs. * Add OpenMathInstruct. * Add a few more v3.5.x SFT mix ablations for the cleaner datasets. * More experiments on mixes. * help merge * prep for merge * reapply changes * fix naming --------- Co-authored-by: Nathan Lambert <[email protected]>
168 lines
4.9 KiB
Bash
Executable File
168 lines
4.9 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
echo "Converting Aya dataset..."
|
|
python -m scripts.data.sft.aya \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/aya
|
|
|
|
echo "Converting Coconot dataset..."
|
|
python -m scripts.data.sft.coconot \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--local_save_dir ./data/sft/coconot
|
|
|
|
echo "Converting CodeFeedback Filtered Instructions dataset..."
|
|
python -m scripts.data.sft.codefeedback_mix \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/codefeedback_filtered_instructions
|
|
|
|
echo "Converting Daring Anteater dataset..."
|
|
python -m scripts.data.sft.daring_anteater \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--remove_subsets open_platypus_commercial \
|
|
--local_save_dir ./data/sft/daring_anteater
|
|
|
|
echo "Converting Evol CodeAlpaca dataset..."
|
|
python -m scripts.data.sft.evol_codealpaca \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/evol_codealpaca
|
|
|
|
echo "Converting Flan V2 dataset..."
|
|
python -m scripts.data.sft.flan \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/flan
|
|
|
|
echo "Converting LIMA dataset..."
|
|
python -m scripts.data.sft.lima \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/lima
|
|
|
|
echo "Converting LMSYS Chat 1M dataset..."
|
|
python -m scripts.data.sft.lmsys_chat \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--model_name_regex="gpt-4" \
|
|
--local_save_dir ./data/sft/lmsys_chat
|
|
|
|
echo "Converting Metamath dataset..."
|
|
python -m scripts.data.sft.metamath \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/metamath
|
|
|
|
echo "Converting No Robots dataset..."
|
|
python -m scripts.data.sft.no_robots \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--local_save_dir ./data/sft/no_robots
|
|
|
|
echo "Converting NuminaMath dataset..."
|
|
python -m scripts.data.sft.numinamath \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/numinamath
|
|
|
|
echo "Converting Open Assistant v1 and v2 dataset..."
|
|
python -m scripts.data.sft.open_assistant \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--top_k=1 \
|
|
--local_save_dir ./data/sft/open_assistant
|
|
|
|
echo "Converting OpenMathInstruct-2 dataset..."
|
|
python -m scripts.data.sft.open_math_instruct \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/open_math_instruct
|
|
|
|
echo "Converting SciRiff dataset..."
|
|
python -m scripts.data.sft.sciriff \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/sciriff
|
|
|
|
echo "Converting ShareGPT dataset..."
|
|
python -m scripts.data.sft.sharegpt \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/sharegpt
|
|
|
|
echo "Converting Slim Orca dataset..."
|
|
python -m scripts.data.sft.slim_orca \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/slim_orca
|
|
|
|
echo "Converting TableGPT dataset..."
|
|
python -m scripts.data.sft.table_gpt \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/table_gpt
|
|
|
|
echo "Converting WebInstruct dataset..."
|
|
python -m scripts.data.sft.web_instruct \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/web_instruct
|
|
|
|
echo "Converting WildChat dataset..."
|
|
python -m scripts.data.sft.wildchat \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/wildchat
|
|
|
|
echo "Converting WizardLM dataset..."
|
|
python -m scripts.data.sft.wizardlm \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--apply_keyword_filters \
|
|
--apply_empty_message_filters \
|
|
--local_save_dir ./data/sft/wizardlm
|
|
|
|
echo "Converting Tulu Hard Coded dataset..."
|
|
python -m scripts.data.sft.tulu_hard_coded \
|
|
--push_to_hub \
|
|
--hf_entity ai2-adapt-dev \
|
|
--repeat_n=10 \
|
|
--local_save_dir ./data/sft/tulu_hard_coded
|
|
|