removed the use of a GenerationConfig object, as it's got lots of params set by default which slow down generations

Author: clefourrier

src/lighteval/models/model_input.py

@@ -107,14 +107,17 @@ def to_transformers_dict(self) -> dict:
         """Selects relevant generation and sampling parameters for transformers models.
         Doc: https://huggingface.co/docs/transformers/v4.46.3/en/main_classes/text_generation#transformers.GenerationConfig
 
+        Note: We actually don't use the GenerationConfig object itself because it has a huge number of parameters automatically
+        initialized, to a config which slows down evals insanely.
+
         Returns:
             dict: The parameters to create a transformers.GenerationConfig in the model config.
         """
         # Task specific sampling params to set in model: do_sample, num_return_sequences, num_beans
         args = {
             "max_new_tokens": self.max_new_tokens,
             "min_new_tokens": self.min_new_tokens,
-            "early_stopping": self.early_stopping,
+            "early_stopping": self.early_stopping or False,
             "stop_strings": self.stop_tokens,
             "temperature": self.temperature,
             "top_k": self.top_k,
@@ -125,8 +128,6 @@ def to_transformers_dict(self) -> dict:
             "output_scores": True,
             "return_dict_in_generate": True,
         }
-        # Even though we only use the dict representation of the GenerationConfig
-        # we still create the object as it uses validation steps
         return {k: v for k, v in args.items() if v is not None}
 
     def to_tgi_inferenceendpoint_dict(self) -> dict:

src/lighteval/models/transformers/transformers_model.py

@@ -36,7 +36,6 @@
     AutoModelForCausalLM,
     AutoTokenizer,
     BitsAndBytesConfig,
-    GenerationConfig,
     GPTQConfig,
     PretrainedConfig,
 )
@@ -656,7 +655,7 @@ def greedy_until_multi_turn(  # noqa: C901
                 ]
             )
 
-            generation_config = GenerationConfig.from_dict(self.generation_config_dict)
+            generation_config = self.generation_config_dict.copy()
             generation_config.update(
                 {
                     "max_new_tokens": max_generated_tokens,
@@ -669,7 +668,7 @@ def greedy_until_multi_turn(  # noqa: C901
             )
 
             model_outputs: GenerateOutput = self.model.generate(
-                **model_inputs, stopping_criteria=stopping_criteria, generation_config=generation_config
+                **model_inputs, stopping_criteria=stopping_criteria, **generation_config
             )
             model_outputs = model_outputs.sequences[0, model_inputs["input_ids"].size(1) :]
             model_generations = [model_outputs]
@@ -699,7 +698,7 @@ def greedy_until_multi_turn(  # noqa: C901
                     ]
                 )
 
-                generation_config = GenerationConfig.from_dict(self.generation_config_dict)
+                generation_config = self.generation_config_dict.copy()
                 generation_config.update(
                     {
                         "max_new_tokens": max_generated_tokens,
@@ -715,7 +714,7 @@ def greedy_until_multi_turn(  # noqa: C901
                     input_ids=model_inputs["input_ids"],
                     attention_mask=model_inputs["attention_mask"],
                     stopping_criteria=stopping_criteria,
-                    generation_config=generation_config,
+                    **generation_config,
                 )
                 model_outputs = model_outputs.sequences[0, model_inputs["input_ids"].size(1) :]
                 model_generations.append(model_outputs)
@@ -896,7 +895,7 @@ def _generate(
         stopping_criteria = stop_sequences_criteria(self.tokenizer, stop_sequences=stop_tokens, batch=batch)
         batch_size, _ = batch.input_ids.shape
 
-        generation_config = GenerationConfig.from_dict(self.generation_config_dict)
+        generation_config = self.generation_config_dict.copy()
         generation_config.update(
             max_new_tokens=max_new_tokens,
             pad_token_id=self.tokenizer.pad_token_id if self.tokenizer.pad_token_id else self.tokenizer.eos_token_id,
@@ -912,7 +911,7 @@ def _generate(
             input_ids=batch.input_ids,
             attention_mask=batch.input_mask,
             stopping_criteria=stopping_criteria,
-            generation_config=generation_config,
+            **generation_config,
         )
         generations = outputs.sequences[:, batch.input_ids.size(1) :]
         generations = torch.reshape(generations, (batch_size, num_samples, -1))