synth/games/tasks/dyck_language_reasoning_errors/run.sh

#!/bin/bash

# Dyck Language Reasoning Errors 游戏自动化运行脚本

# 设置颜色输出
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
RED='\033[0;31m'
NC='\033[0m' # No Color

# 设置脚本目录
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/../../.." &> /dev/null && pwd)"

# 默认参数
NUM_QUESTIONS=1000  # 修改为1000条
MODEL_NAME=""
AID="11025" # 默认应用ID

# 打印标题
echo -e "${YELLOW}=========================================${NC}"
echo -e "${YELLOW}Dyck Language Reasoning Errors Automated Running${NC}"
echo -e "${YELLOW}=========================================${NC}"

# 命令行参数解析仅用于覆盖默认值
while [[ $# -gt 0 ]]; do
    case $1 in
        --model|-m)
            MODEL_NAME="$2"
            shift 2
            ;;
        --aid|-a)
            AID="$2"
            shift 2
            ;;
        --help|-h)
            echo -e "${BLUE}Usage:${NC}"
            echo -e "  ./run.sh [options]"
            echo -e ""
            echo -e "${BLUE}Options:${NC}"
            echo -e "  --model, -m <model>      the model name used (default: abab8)"
            echo -e "  --aid, -a <ID>           the application ID (default: 11025)"
            echo -e "  --help, -h               show help information"
            exit 0
            ;;
        *)
            echo -e "${RED}Error: unknown parameter $1${NC}"
            exit 1
            ;;
    esac
done

echo -e "${BLUE}Configuration information:${NC}"
echo -e "- Game name: dyck_language_reasoning_errors"
echo -e "- Number of questions: ${NUM_QUESTIONS} (per parameter set)"
echo -e "- Model name: ${MODEL_NAME}"
echo -e "- Application ID: ${AID}"

# 设置工作目录
cd "${ROOT_DIR}"
WORKSPACE_DIR=$(pwd)

# 构建数据文件路径
GAME_NAME="dyck_language_reasoning_errors"
GAME_DIR="${WORKSPACE_DIR}/games/tasks/${GAME_NAME}"
DATA_DIR="${GAME_DIR}/data"

# 确保数据目录和评估目录存在
mkdir -p "${DATA_DIR}"
mkdir -p "${GAME_DIR}/eval"

# 定义要运行的参数组合
declare -a PARAM_SETS=(
    "n_types=2 total_length=10 n_errors=1"
    "n_types=2 total_length=15 n_errors=1"
    "n_types=3 total_length=10 n_errors=1"
    "n_types=3 total_length=15 n_errors=1"
)

# 对每组参数运行测试
for PARAMS in "${PARAM_SETS[@]}"; do
    # 解析参数
    N_TYPES=$(echo $PARAMS | grep -o "n_types=[0-9]*" | cut -d= -f2)
    TOTAL_LENGTH=$(echo $PARAMS | grep -o "total_length=[0-9]*" | cut -d= -f2)
    N_ERRORS=$(echo $PARAMS | grep -o "n_errors=[0-9]*" | cut -d= -f2)
    
    echo -e "\n${YELLOW}=========================================${NC}"
    echo -e "${YELLOW}Processing parameter set: n_types=${N_TYPES}, total_length=${TOTAL_LENGTH}, n_errors=${N_ERRORS}${NC}"
    echo -e "${YELLOW}=========================================${NC}"
    
    # 生成目录路径
    PARAM_DIR="${DATA_DIR}/num_of_data_${NUM_QUESTIONS}/n_types_${N_TYPES}/total_length_${TOTAL_LENGTH}/n_errors_${N_ERRORS}"
    mkdir -p "${PARAM_DIR}"
    
    # 生成数据文件名
    DATA_FILE="${PARAM_DIR}/${GAME_NAME}_${NUM_QUESTIONS}.jsonl"
    
    # 第一步：生成数据
    echo -e "${YELLOW}============================================${NC}"
    echo -e "${YELLOW}Step 1: Generate test data${NC}"
    echo -e "${YELLOW}Game: ${GAME_NAME}${NC}"
    echo -e "${YELLOW}Number of questions: ${NUM_QUESTIONS}${NC}"
    echo -e "${YELLOW}Number of types: ${N_TYPES}${NC}"
    echo -e "${YELLOW}Sequence length: ${TOTAL_LENGTH}${NC}"
    echo -e "${YELLOW}Number of errors: ${N_ERRORS}${NC}"
    echo -e "${YELLOW}Data file: ${DATA_FILE}${NC}"
    echo -e "${YELLOW}============================================${NC}"
    
    # 执行数据生成命令
    python -m games.tasks.${GAME_NAME}.scripts.${GAME_NAME} --num_of_data ${NUM_QUESTIONS} --n_types ${N_TYPES} --total_length ${TOTAL_LENGTH} --n_errors ${N_ERRORS}
    
    # 检查数据文件是否生成成功
    if [ ! -f "${DATA_FILE}" ]; then
        echo -e "${RED}Error: Data file not generated: ${DATA_FILE}${NC}"
        continue
    fi
    

done

echo -e "\n${GREEN}============================================${NC}"
echo -e "${GREEN}All parameter set tests completed!${NC}"
echo -e "${GREEN}============================================${NC}"