Remove the eos token override in the Default Config Task (#54)

huggingface · Feb 27, 2024 · 449817f · 449817f
1 parent 589e6b0
commit 449817f
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 6 deletions.
diff --git a/src/lighteval/tasks/lighteval_task.py b/src/lighteval/tasks/lighteval_task.py
@@ -83,8 +83,6 @@ def __post_init__(self):
             self.hf_avail_splits = ["train", "validation", "test"]
         if self.evaluation_splits is None:
             self.evaluation_splits = ["validation"]
-        if self.stop_sequence is None:
-            self.stop_sequence = ["\n"]
 
         # Convert list to tuple for hashing
         self.metric = tuple(self.metric)

diff --git a/tasks_examples/custom_tasks/custom_evaluation_tasks.py b/tasks_examples/custom_tasks/custom_evaluation_tasks.py
@@ -25,20 +25,23 @@
         hf_repo="hellaswag",
         hf_subset="default",
         metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+        stop_sequence=["\n"],
     ),
     LightevalTaskConfig(
         name="winogrande",
         prompt_function="winogrande",
         hf_repo="winogrande",
         hf_subset="winogrande_xl",
         metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+        stop_sequence=["\n"],
     ),
     LightevalTaskConfig(
         name="piqa",
         prompt_function="piqa_harness",
         hf_repo="piqa",
         hf_subset="plain_text",
         metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+        stop_sequence=["\n"],
     ),
     LightevalTaskConfig(
         name="siqa",
@@ -47,13 +50,15 @@
         hf_subset="default",
         hf_avail_splits=["train", "validation"],
         metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+        stop_sequence=["\n"],
     ),
     LightevalTaskConfig(
         name="openbookqa",
         prompt_function="openbookqa",
         hf_repo="openbookqa",
         hf_subset="main",
         metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+        stop_sequence=["\n"],
     ),
     LightevalTaskConfig(
         name="arc:easy",
@@ -63,6 +68,7 @@
         evaluation_splits=["test"],
         generation_size=1,
         metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+        stop_sequence=["\n"],
     ),
     LightevalTaskConfig(
         name="arc:challenge",
@@ -72,13 +78,15 @@
         evaluation_splits=["test"],
         generation_size=1,
         metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+        stop_sequence=["\n"],
     ),
     LightevalTaskConfig(
         name="commonsense_qa",
         prompt_function="commonsense_qa_prompt",
         hf_repo="commonsense_qa",
         hf_subset="default",
         metric=["loglikelihood_acc", "loglikelihood_acc_norm_nospace"],
+        stop_sequence=["\n"],
     ),
 ]
 
@@ -176,6 +184,7 @@ def natural_questions_prompt(line, task_name: str = None):
         hf_repo="super_glue",
         hf_subset="boolq",
         metric=["target_perplexity"],
+        stop_sequence=["\n"],
     ),
     LightevalTaskConfig(
         name="quac",
@@ -236,9 +245,9 @@ def __init__(
             few_shots_select=few_shots_select,
             suite=suite,
             generation_size=generation_size,
-            stop_sequence=stop_sequence,
             output_regex=output_regex,
             frozen=frozen,
+            stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]),
         )
 
 
@@ -303,7 +312,7 @@ def __init__(
             few_shots_select=few_shots_select,
             suite=suite,
             generation_size=generation_size,
-            stop_sequence=stop_sequence,
+            stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]),
             output_regex=output_regex,
             frozen=frozen,
         )
@@ -445,7 +454,7 @@ def __init__(
             few_shots_select=few_shots_select,
             suite=suite,
             generation_size=generation_size,
-            stop_sequence=stop_sequence,
+            stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]),
             output_regex=output_regex,
             frozen=frozen,
         )
@@ -537,7 +546,7 @@ def __init__(
             few_shots_select=few_shots_select,
             suite=suite,
             generation_size=generation_size,
-            stop_sequence=stop_sequence,
+            stop_sequence=(stop_sequence if stop_sequence is not None else ["\n"]),
             output_regex=output_regex,
             frozen=frozen,
         )