Merge remote-tracking branch 'origin/main' into math_extraction

huggingface · Jan 28, 2025 · 9d2553e · 9d2553e
2 parents 8a161c1 + cb075a5
commit 9d2553e
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 4 deletions.
diff --git a/src/lighteval/logging/evaluation_tracker.py b/src/lighteval/logging/evaluation_tracker.py
@@ -34,7 +34,6 @@
 import torch
 from datasets import Dataset, load_dataset
 from datasets.utils.metadata import MetadataConfigs
-from fsspec import url_to_fs
 from huggingface_hub import DatasetCard, DatasetCardData, HfApi, HFSummaryWriter, hf_hub_url
 
 from lighteval.logging.info_loggers import (
@@ -53,6 +52,11 @@
 if is_nanotron_available():
     from nanotron.config import GeneralArgs  # type: ignore
 
+try:
+    from fsspec import url_to_fs
+except ImportError:
+    from fsspec.core import url_to_fs
+
 
 class EnhancedJSONEncoder(json.JSONEncoder):
     """

diff --git a/src/lighteval/tasks/default_tasks.py b/src/lighteval/tasks/default_tasks.py
@@ -6210,7 +6210,7 @@
     evaluation_splits=["validation"],
     few_shots_split=None,
     few_shots_select=None,
-    generation_size=-1,
+    generation_size=1,
     metric=[
         Metrics.exact_match,
         Metrics.quasi_exact_match,

diff --git a/src/lighteval/tasks/templates/utils/translation_literals.py b/src/lighteval/tasks/templates/utils/translation_literals.py
@@ -980,7 +980,7 @@ def __getattribute__(self, name: str) -> str:
         language=Language.UKRAINIAN,
         question_word="питання",
         answer="відповідь",
-        confirmation_word="вірно",
+        confirmation_word="правильно",
         yes="так",
         no="ні",
         also="також",
@@ -998,6 +998,7 @@ def __getattribute__(self, name: str) -> str:
         sentence_space=" ",
         colon=":",
         semicolon=";",
+        indices=["А", "Б", "В", "Г", "Д", "Е"],
     ),
     Language.URDU: TranslationLiterals(
         language=Language.URDU,

diff --git a/tests/metrics/test_extractive_match.py b/tests/metrics/test_extractive_match.py
@@ -969,7 +969,6 @@ def test_math_extraction_edge_cases(gold, pred, expected):
             r"Thus, the answer is $x \in \boxed{(2,12) \cup (12,102)}$",
             1,
         ),
-        (r"$204$", r"$24+108=204$", 1),
     ],
 )
 def test_math_extraction_additional_cases(gold, pred, expected):
-Original file line number
+Diff line change
@@ Expand Up / @@ -969,7 +969,6 @@ def test_math_extraction_edge_cases(gold, pred, expected): @@
                 r"Thus, the answer is $x \in \boxed{(2,12) \cup (12,102)}$",
 ,
             ),
-            (r"$204$", r"$24+108=204$", 1),
         ],
     )
     def test_math_extraction_additional_cases(gold, pred, expected):
@@ Expand Down @@