Files
2024-09-25 18:04:29 +08:00

323 lines
12 KiB
Python

import argparse
import json
import multiprocessing
import os
import pickle
import threading
import time
from collections import Counter, defaultdict
from concurrent.futures import ProcessPoolExecutor, as_completed
from datetime import datetime
from typing import Any, Dict, List, Tuple
from warnings import warn
import numpy as np
from termcolor import cprint
from tqdm import tqdm
from evalplus.data import (
get_human_eval_plus,
get_human_eval_plus_hash,
get_mbpp_plus,
get_mbpp_plus_hash,
load_solutions,
)
from evalplus.data.mbpp import mbpp_serialize_inputs
# from evalplus.data.utils import CACHE_DIR
CACHE_DIR = "/home/data/xixi.yjx/eval_cache/evalplus"
from evalplus.eval import (
PASS,
compatible_eval_result,
estimate_pass_at_k,
untrusted_check,
)
from evalplus.eval._special_oracle import MBPP_OUTPUT_NOT_NONE_TASKS
from evalplus.gen.util import trusted_exec
# 1st item: the status
# 2nd item (optional): the detailed pass/fail boolean for each input
Result = Tuple[str, List[bool]]
def get_groundtruth(problems, hashcode, tasks_only_output_not_none):
cache_file = os.path.join(CACHE_DIR, f"{hashcode}.pkl")
if os.path.exists(cache_file):
print(f"Load from ground-truth from {cache_file}")
with open(cache_file, "rb") as f:
return pickle.load(f)
os.makedirs(CACHE_DIR, exist_ok=True)
print("Computing expected output...")
tbegin = time.time()
expected_output = {}
for task_id, problem in problems.items():
oracle = {}
oracle["base"], oracle["base_time"] = trusted_exec(
problem["prompt"] + problem["canonical_solution"],
problem["base_input"],
problem["entry_point"],
record_time=True,
output_not_none=problem["entry_point"] in tasks_only_output_not_none,
)
oracle["plus"], oracle["plus_time"] = trusted_exec(
problem["prompt"] + problem["canonical_solution"],
problem["plus_input"],
problem["entry_point"],
record_time=True,
output_not_none=problem["entry_point"] in tasks_only_output_not_none,
)
expected_output[task_id] = oracle
print(f"Expected outputs computed in {time.time() - tbegin:.2f}s")
with open(cache_file, "wb") as f:
pickle.dump(expected_output, f)
return expected_output
def check_correctness(
dataset: str,
completion_id: int,
problem: Dict[str, Any],
solution: str,
expected_output: Dict[str, List],
base_only=False,
fast_check=False,
identifier=None,
min_time_limit: float = 0.1,
gt_time_limit_factor: float = 2.0,
) -> Dict[str, Result]: # {...}, "base" | "plus" -> (status, details)
ret = {
"completion_id": completion_id,
"task_id": problem["task_id"],
"_identifier": identifier,
"solution": solution,
}
ret["base"] = untrusted_check(
dataset,
solution,
problem["base_input"],
problem["entry_point"],
expected=expected_output["base"],
atol=problem["atol"],
ref_time=expected_output["base_time"],
fast_check=fast_check,
min_time_limit=min_time_limit,
gt_time_limit_factor=gt_time_limit_factor,
)
if not base_only:
ret["plus"] = untrusted_check(
dataset,
solution,
problem["plus_input"],
problem["entry_point"],
expected=expected_output["plus"],
atol=problem["atol"],
ref_time=expected_output["plus_time"],
fast_check=fast_check,
min_time_limit=min_time_limit,
gt_time_limit_factor=gt_time_limit_factor,
)
return ret
def evaluate(flags):
if flags.parallel is None:
n_workers = max(1, multiprocessing.cpu_count() // 2)
else:
n_workers = flags.parallel
if os.path.isdir(flags.samples):
result_path = os.path.join(flags.samples, "eval_results.json")
else:
assert flags.samples.endswith(".jsonl")
result_path = flags.samples.replace(".jsonl", "_eval_results.json")
if os.path.isfile(result_path) and not flags.i_just_wanna_run:
print(f"Load from previous results from {result_path}")
with open(result_path, "r") as f:
results = json.load(f)
results = compatible_eval_result(results)
else:
if flags.dataset == "humaneval":
problems = get_human_eval_plus(mini=flags.mini, noextreme=flags.noextreme, version=flags.version)
dataset_hash = get_human_eval_plus_hash(mini=flags.mini, noextreme=flags.noextreme, version=flags.version)
expected_output = get_groundtruth(problems, dataset_hash, [])
elif flags.dataset == "mbpp":
problems = get_mbpp_plus(mini=flags.mini, noextreme=flags.noextreme, version=flags.version)
dataset_hash = get_mbpp_plus_hash(mini=flags.mini, noextreme=flags.noextreme, version=flags.version)
expected_output = get_groundtruth(
problems,
dataset_hash,
MBPP_OUTPUT_NOT_NONE_TASKS,
)
results = {
"date": datetime.now().strftime("%Y-%m-%d %H:%M"),
"hash": dataset_hash,
"eval": {},
}
with ProcessPoolExecutor(max_workers=n_workers) as executor:
futures = []
completion_id = Counter()
n_samples = 0
eval_results = defaultdict(list) # task_id ->
remainings = set()
print("Reading samples...")
for sample in tqdm(load_solutions(flags.samples)):
task_id = sample["task_id"]
if task_id not in problems:
warn(f"Task {task_id} is found in the samples but not found in the dataset")
continue
solution = sample["solution"] if "solution" in sample else problems[task_id]["prompt"] + sample["completion"]
remainings.add(sample["_identifier"])
args = (
flags.dataset,
completion_id[task_id],
problems[task_id],
solution,
expected_output[task_id],
flags.base_only,
not flags.test_details, # fast_check
sample["_identifier"],
flags.min_time_limit,
flags.gt_time_limit_factor,
)
futures.append(executor.submit(check_correctness, *args))
completion_id[task_id] += 1
n_samples += 1
assert n_samples == len(remainings), "Missing problems in unfinished"
assert len(completion_id) == len(problems), "Missing problems in samples"
def stucking_checker():
while remainings:
last_size = len(remainings)
time.sleep(20)
if last_size != len(remainings) or len(remainings) == 0:
continue
# Potential stucking
warn("No samples had finished testing in the last 20s")
warn(f"{len(remainings)} samples to be tested: {remainings}")
threading.Thread(target=stucking_checker).start()
for future in tqdm(as_completed(futures), total=n_samples):
result = future.result()
remainings.remove(result["_identifier"])
eval_results[result["task_id"]].append(result)
# sort the results for each problem by completion_id
for task_id, task_results in eval_results.items():
task_results.sort(key=lambda x: x["completion_id"])
results["eval"][task_id] = []
for res in task_results:
def get_failed_tests(stat, details, inputs) -> List[Any]:
if stat == PASS or not details:
return []
if flags.test_details:
return [inputs[i] for i in range(len(details)) if not details[i]]
# else => simply return the only and the last fail test
return [inputs[len(details) - 1]]
base_stat, base_details = res["base"]
base_fail_tests = get_failed_tests(base_stat, base_details, problems[task_id]["base_input"])
# initialize plus tests
plus_stat = None
plus_fail_tests = []
# with plus tests
if not flags.base_only:
plus_stat, plus_details = res["plus"]
plus_fail_tests = get_failed_tests(plus_stat, plus_details, problems[task_id]["plus_input"])
if flags.dataset == "mbpp":
base_fail_tests = mbpp_serialize_inputs(task_id, base_fail_tests)
plus_fail_tests = mbpp_serialize_inputs(task_id, plus_fail_tests)
results["eval"][task_id].append(
{
"task_id": task_id,
"solution": res["solution"],
"base_status": base_stat,
"plus_status": plus_stat,
"base_fail_tests": base_fail_tests,
"plus_fail_tests": plus_fail_tests,
}
)
# Calculate pass@k.
total = np.array([len(r) for r in results["eval"].values()])
base_correct = []
new_correct = []
for res in results["eval"].values():
bc = sum([r["base_status"] == PASS for r in res])
base_correct.append(bc)
if not flags.base_only:
new_correct.append(sum([res[i]["base_status"] == res[i]["plus_status"] == PASS for i in range(len(res))]))
base_correct = np.array(base_correct)
pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, base_correct, k).mean() for k in [1, 10, 100] if total.min() >= k}
cprint(f"{flags.dataset} (base tests)", "red")
for k, v in pass_at_k.items():
cprint(f"{k}:\t{v:.3f}", "red")
if new_correct:
cprint(f"{flags.dataset}+ (base + extra tests)", "green")
pass_at_k = {f"pass@{k}": estimate_pass_at_k(total, np.array(new_correct), k).mean() for k in [1, 10, 100] if (total >= k).all()}
for k, v in pass_at_k.items():
cprint(f"{k}:\t{v:.3f}", "green")
# save results
if os.path.isfile(result_path) and flags.i_just_wanna_run:
decision = ""
while decision.lower() not in ["y", "n"]:
print(f"{result_path} already exists. Press [Y/N] to overwrite or exit...")
decision = input()
if decision.lower() == "y":
# mv the file to a backup
new_path = result_path + ".bak"
while os.path.isfile(new_path):
new_path += ".bak"
os.rename(result_path, new_path)
print(f"Backup {result_path} to {new_path}")
if not os.path.isfile(result_path):
with open(result_path, "w") as f:
json.dump(results, f)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--dataset", required=True, type=str, choices=["humaneval", "mbpp"])
parser.add_argument("--samples", required=True, type=str)
parser.add_argument("--base-only", action="store_true")
parser.add_argument("--parallel", default=None, type=int)
parser.add_argument("--i-just-wanna-run", action="store_true")
parser.add_argument("--test-details", action="store_true")
parser.add_argument("--min-time-limit", default=1, type=float)
parser.add_argument("--gt-time-limit-factor", default=4.0, type=float)
parser.add_argument("--mini", action="store_true")
parser.add_argument("--noextreme", action="store_true", help="Omit extreme test inputs")
parser.add_argument("--version", default="default", type=str, help="Version of the dataset")
args = parser.parse_args()
evaluate(args)
if __name__ == "__main__":
main()