diff --git a/src/lighteval/metrics/dynamic_metrics.py b/src/lighteval/metrics/dynamic_metrics.py index 577934e9..ee32dfb4 100644 --- a/src/lighteval/metrics/dynamic_metrics.py +++ b/src/lighteval/metrics/dynamic_metrics.py @@ -246,7 +246,8 @@ def sample_level_fn(golds: list[str], predictions: list[str], formatted_doc: Doc # Assert on empty gold and warn on empty pred if any(len(g) == 0 for g in extracted_golds): - raise ValueError(f"No gold targets found for at least one gold. Gold: {golds}, Pred: {predictions}") + logger.warning(f"We did not manage to extract a gold in the correct format. Gold: {golds}") + extracted_golds = [[gold] for gold in golds] if all(len(p) == 0 for p in extracted_predictions): logger.warning( diff --git a/src/lighteval/models/litellm_model.py b/src/lighteval/models/litellm_model.py index ccb10c47..84006178 100644 --- a/src/lighteval/models/litellm_model.py +++ b/src/lighteval/models/litellm_model.py @@ -141,9 +141,6 @@ def __call_api(self, prompt, return_logits, max_new_tokens, num_samples, stop_se response = litellm.completion(**kwargs) - print(response) - print(kwargs) - # If response is empty, retry without caching (maybe the error is recoverable and solved with a retry) if response.choices[0].message.content is None: kwargs["caching"] = False diff --git a/src/lighteval/tasks/extended/olympiade_bench/main.py b/src/lighteval/tasks/extended/olympiade_bench/main.py index a648ed57..f4004bbb 100644 --- a/src/lighteval/tasks/extended/olympiade_bench/main.py +++ b/src/lighteval/tasks/extended/olympiade_bench/main.py @@ -31,14 +31,104 @@ from lighteval.utils.language import Language +chinese_answer_type_dict = {"Numerical": "数值", "Expression": "表达式", "Equation": "方程", "Interval": "区间"} +english_answer_type_dict = { + "Numerical": "a numerical value", + "Expression": "an expression", + "Equation": "an equation", + "Interval": "an interval", +} + + +def get_single_answer_type_text(answer_type, is_chinese): + if "-" in answer_type: # No need now + answer_type = answer_type[: answer_type.find("-")] + for t in ["Numerical", "Expression", "Equation", "Interval"]: + if t in answer_type: + if is_chinese: + return chinese_answer_type_dict[t] + else: + return english_answer_type_dict[t] + exit(f"Error parsing answer type {answer_type}!") + + +def get_answer_type_text(answer_type, is_chinese, multiple_answer): + if ( + ("Need_human_evaluate" in answer_type) or ("Tuple" in answer_type) + ): # 'Tuple' has various meanings in different context, such as position or values of a series of variable, so it may lead to confusion to directly use 'tuple' in the prompt. + full_answer_text = "" + else: + if not multiple_answer: + answer_text = get_single_answer_type_text(answer_type, is_chinese) + if is_chinese: + full_answer_text = f"，答案类型为{answer_text}" + else: + full_answer_text = f"The answer of The problem should be {answer_text}. " + else: + if "," not in answer_type: # Same answer type for all answers + answer_text = get_single_answer_type_text(answer_type, is_chinese) + if is_chinese: + full_answer_text = f"，题目有多个答案，答案类型均为{answer_text}" + else: + full_answer_text = f"The problem has multiple answers, each of them should be {answer_text}. " + else: + answer_types = answer_type.split(",") + answer_types = [get_single_answer_type_text(t, is_chinese) for t in answer_types] + if len(set(answer_types)) == 1: + answer_text = answer_types[0] + if is_chinese: + full_answer_text = f"，题目有多个答案，答案类型均为{answer_text}" + else: + full_answer_text = f"The problem has multiple answers, each of them should be {answer_text}. " + else: + if is_chinese: + answer_text = "、".join(answer_types) + full_answer_text = f"，题目有多个答案，答案类型分别为{answer_text}" + else: + answer_text = ", ".join(answer_types) + full_answer_text = ( + f"The problem has multiple answers, with the answers in order being {answer_text}. " + ) + return full_answer_text + + # Very specific task where there are no precise outputs but instead we test if the format obeys rules def olympiad_bench_prompt(line, task_name: str = None): + is_math = "Math" in line["subject"] + subject = "Math" if is_math else "Physics" + + is_theorem_proving = "TP" in task_name + unit = line["unit"] + is_multiple_answer = line["is_multiple_answer"] + + if is_theorem_proving: + instruction = f"The following is a theorem proving problem from an International {subject} competition. Please use logical reasoning and common theorems to prove the proposition in the problem according to the given requirements. Please use LaTeX format to represent the variables and formulas used in the proof." + else: + if is_multiple_answer: + multiple_answer_text = "\\boxed{multiple answers connected with commas}" + else: + multiple_answer_text = "\\boxed{answer}" + unit_text = "" + if unit: + multiple_answer_text += "(unit)" + unit_text = ", note that the unit of the answer should not be included in \\boxed{}" + + answer_type_text = get_answer_type_text( + line["answer_type"], is_chinese=False, multiple_answer=is_multiple_answer + ) + + instruction = f'The following is an open-ended problem from an International {subject} competition. {answer_type_text}Please calculate the answer according to the given requirements and the information provided. Please use LaTeX format to represent the variables and formulas used in the solution process and results. Please end your solution with "So the final answer is {multiple_answer_text}." and give the result explicitly{unit_text}.' + + # instruction += f"\nYou are an AI assistant. Please answer the following {subject} competition questions as required." + + choice = line["final_answer"] + return Doc( task_name=task_name, - query=line["question"], - choices=[line["final_answer"]], + query=instruction + "\n" + line["question"], + choices=[choice], gold_index=0, - instruction="", + instruction=instruction, specific={}, ) @@ -54,7 +144,7 @@ def olympiad_bench_prompt(line, task_name: str = None): # * COMP: Competition problems # * CEE: Chinese College Entrance Exam problems -question_type = ["OE", "TP"] +question_type = ["OE"] # "TP"] multimodality = ["TO"] # MM subject = ["physics", "maths"] language = ["en"] # "zh"] @@ -69,6 +159,29 @@ def olympiad_bench_prompt(line, task_name: str = None): for src in source: olympiad_bench_subsets.append(f"{qt}_{mm}_{sub}_{lang}_{src}") +available_subsets = [ + "OE_MM_maths_en_COMP", + "OE_MM_maths_zh_CEE", + "OE_MM_maths_zh_COMP", + "OE_MM_physics_en_COMP", + "OE_MM_physics_zh_CEE", + "OE_TO_maths_en_COMP", + "OE_TO_maths_zh_CEE", + "OE_TO_maths_zh_COMP", + "OE_TO_physics_en_COMP", + "OE_TO_physics_zh_CEE", + "TP_MM_maths_en_COMP", + "TP_MM_maths_zh_CEE", + "TP_MM_maths_zh_COMP", + "TP_MM_physics_en_COMP", + "TP_TO_maths_en_COMP", + "TP_TO_maths_zh_CEE", + "TP_TO_maths_zh_COMP", + "TP_TO_physics_en_COMP", +] + +olympiad_bench_subsets = set(olympiad_bench_subsets).intersection(available_subsets) + extraction_targets = [ExprExtractionConfig(), LatexExtractionConfig()] metric = multilingual_extractive_match_metric( @@ -77,23 +190,28 @@ def olympiad_bench_prompt(line, task_name: str = None): pred_extraction_target=extraction_targets, precision=6, ) -# We create the task config -olympiad_bench = LightevalTaskConfig( - name="olympiad_bench", - prompt_function=olympiad_bench_prompt, - suite=["extended"], - hf_repo="Hothan/OlympiadBench", - hf_subset=olympiad_bench_subsets[0], - metric=[metric], - hf_avail_splits=["train"], - evaluation_splits=["train"], - few_shots_split="train", - few_shots_select="random_sampling", - generation_size=2048, - stop_sequence=[], # no stop sequence, will use eot token - version="1.0", -) -# print(olympiad_bench) +task_configs = [] + +for subset in olympiad_bench_subsets: + # We create the task config + task_configs.append( + LightevalTaskConfig( + name="olympiad_bench:" + subset, + prompt_function=olympiad_bench_prompt, + suite=["extended"], + hf_repo="Hothan/OlympiadBench", + hf_subset=subset, + metric=[metric], + hf_avail_splits=["train"], + evaluation_splits=["train"], + few_shots_split="train", + few_shots_select="random_sampling", + generation_size=2048, + stop_sequence=[], # no stop sequence, will use eot token + version=1, + ) + ) + -TASKS_TABLE = [olympiad_bench] +TASKS_TABLE = task_configs