mirror of
https://github.com/hanzoai/open-instruct.git
synced 2026-10-30 18:26:32 +00:00
74069db498
* Reorganize the data preparation scripts for tulu v1 and v2. * Minor improvement * Remove open_platypus_commercial subset from Daring-Anteater * Use hard-coded examples repo. * Fix some bugs. * Add OpenMathInstruct. * Add a few more v3.5.x SFT mix ablations for the cleaner datasets. * More experiments on mixes. * help merge * prep for merge * reapply changes * fix naming --------- Co-authored-by: Nathan Lambert <[email protected]>
57 lines
2.1 KiB
Bash
Executable File
57 lines
2.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# Run with scripts/data/get_statistics_tulu_v3.sh
|
|
|
|
# List of datasets
|
|
datasets=(
|
|
ai2-adapt-dev/oasst1_converted
|
|
ai2-adapt-dev/flan_v2_converted
|
|
ai2-adapt-dev/tulu_hard_coded_repeated_10
|
|
ai2-adapt-dev/no_robots_converted
|
|
ai2-adapt-dev/tulu_v3.9_wildchat_100k
|
|
ai2-adapt-dev/personahub_math_v5_regen_149960
|
|
allenai/tulu-3-sft-personas-math-grade
|
|
ai2-adapt-dev/tulu_v3.9_open_math_2_gsm8k_50k
|
|
ai2-adapt-dev/numinamath_tir_math_decontaminated
|
|
ai2-adapt-dev/tulu_v3.9_personahub_math_interm_algebra_20k
|
|
ai2-adapt-dev/personahub_code_v2_34999
|
|
ai2-adapt-dev/evol_codealpaca_heval_decontaminated
|
|
ai2-adapt-dev/personahub_ifdata_manual_seed_v3_29980
|
|
ai2-adapt-dev/coconot_converted
|
|
ai2-adapt-dev/tulu_v3.9_wildjailbreak_decontaminated_50k
|
|
ai2-adapt-dev/tulu_v3.9_synthetic_finalresp_wildguardmixtrain_decontaminated_50k
|
|
ai2-adapt-dev/tulu_v3.9_sciriff_10k
|
|
ai2-adapt-dev/tulu_v3.9_table_gpt_5k
|
|
ai2-adapt-dev/tulu_v3.9_aya_100k
|
|
)
|
|
|
|
# For every dataset, get the statistics if the output directory doesn't exist
|
|
for dataset in "${datasets[@]}"; do
|
|
output_file="data/processed/${dataset}_statistics.json"
|
|
|
|
if [ ! -f "$output_file" ]; then
|
|
echo "Getting statistics for $dataset..."
|
|
python scripts/data/get_statistics.py --data_path ${dataset} --save_path ${output_file}
|
|
else
|
|
echo "Statistics for $dataset already exist. Skipping..."
|
|
fi
|
|
done
|
|
|
|
# list of preference datasets
|
|
datasets_pref=(
|
|
allenai/ultrafeedback_binarized_cleaned_train
|
|
ai2-adapt-dev/DaringAnteater-prefs-RM-filter
|
|
ai2-adapt-dev/WildChat-prefs-280824
|
|
ai2-adapt-dev/Llama-3.1-if_taxonomy_tulu
|
|
)
|
|
|
|
# For every dataset, get the statistics if the output directory doesn't exist
|
|
for dataset in "${datasets_pref[@]}"; do
|
|
output_file="data/processed/${dataset}_statistics.json"
|
|
|
|
if [ ! -f "$output_file" ]; then
|
|
echo "Getting statistics for $dataset..."
|
|
python scripts/data/get_statistics.py --data_path ${dataset} --save_path ${output_file} --messages_key chosen
|
|
else
|
|
echo "Statistics for $dataset already exist. Skipping..."
|
|
fi
|
|
done |