From 9653896d627fc2a2668168016966eb63f7b91344 Mon Sep 17 00:00:00 2001
From: Erez Schwartz <erezs@ai21.com>
Date: Mon, 20 Jan 2025 18:07:58 +0200
Subject: [PATCH 1/3] fix to simple_evaluate method

---
 lmms_eval/evaluator.py | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py
index c499c375..99f70e28 100755
--- a/lmms_eval/evaluator.py
+++ b/lmms_eval/evaluator.py
@@ -170,14 +170,20 @@ def simple_evaluate(
 
     task_dict = get_task_dict(tasks, task_manager)
 
-    ModelClass = get_model(model)
-    lm = ModelClass.create_from_arg_string(
-        model_args,
-        {
-            "batch_size": batch_size,
-            "device": device,
-        },
-    )
+    if isinstance(model, str):
+        if model_args is None:
+            model_args = ""
+        lm = lmms_eval.models.get_model(model).create_from_arg_string(
+            model_args,
+            {
+                "batch_size": batch_size,
+                "max_batch_size": max_batch_size,
+                "device": device,
+            },
+        )
+    elif isinstance(model, lmms_eval.model.lmms):
+        #   use existing model instance
+        lm = model
 
     # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.
     # (setting of num_fewshot ; bypassing metric calculation ; setting fewshot seed)

From 0e96abcf75049cdb9ed602eeda52cc4c559df998 Mon Sep 17 00:00:00 2001
From: Erez Schwartz <erezs@ai21.com>
Date: Mon, 20 Jan 2025 18:12:55 +0200
Subject: [PATCH 2/3] fix to simple_evaluate method

---
 lmms_eval/evaluator.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py
index 99f70e28..00b70c0c 100755
--- a/lmms_eval/evaluator.py
+++ b/lmms_eval/evaluator.py
@@ -182,7 +182,6 @@ def simple_evaluate(
             },
         )
     elif isinstance(model, lmms_eval.model.lmms):
-        #   use existing model instance
         lm = model
 
     # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.

From 3e7ce82578607b8d5bb70c40ca82c4aac17b15f0 Mon Sep 17 00:00:00 2001
From: Erez Schwartz <erezs@ai21.com>
Date: Mon, 20 Jan 2025 18:21:42 +0200
Subject: [PATCH 3/3] fix

---
 lmms_eval/evaluator.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lmms_eval/evaluator.py b/lmms_eval/evaluator.py
index 00b70c0c..8776eed1 100755
--- a/lmms_eval/evaluator.py
+++ b/lmms_eval/evaluator.py
@@ -181,7 +181,7 @@ def simple_evaluate(
                 "device": device,
             },
         )
-    elif isinstance(model, lmms_eval.model.lmms):
+    elif isinstance(model, lmms_eval.api.model.lmms):
         lm = model
 
     # helper function to recursively apply config overrides to leaf subtasks, skipping their constituent groups.