create baselines

.github/workflows/nvidia_workflow.yml

@@ -28,6 +28,10 @@ jobs:
     - name: Create input files
       shell: bash
       run: |
+
+        # install jq
+        apt update && apt install -y jq
+
         # Extract the payload content without printing it
         apt-get update && apt-get install -y jq
         PAYLOAD=$(jq -r '.inputs.payload' $GITHUB_EVENT_PATH)

examples/eval.py

@@ -345,7 +345,7 @@ def main():
             if mode == "benchmark":
                 return run_benchmarking(logger, pool, tests)
 
-            if mode == "leaderboard":
+            if mode == "leaderboard" or mode == "milestone":
                 # warmup
                 run_single_benchmark(pool, tests[0], False, 100, 1e7)
                 logger.log("benchmark-count", len(tests))

examples/matmul_py/pytorch_ref.py

@@ -0,0 +1,8 @@
+#!POPCORN leaderboard matmul_py
+
+from task import input_t, output_t
+
+
+def custom_kernel(data: input_t) -> output_t:
+    a, b = data
+    return a @ b

examples/matmul_py/reference.py

@@ -20,7 +20,7 @@ def check_implementation(data: input_t, output: output_t) -> str:
     reasons = verbose_allclose(output, expected)
     if len(reasons) > 0:
         # TODO better processing of reasons
-        return "mismatch found! custom implementation doesn't match reference.: " + reasons[0]
+        return False, "mismatch found! custom implementation doesn't match reference.: " + reasons[0]
 
-    return ''
+    return True, ''
 

examples/matmul_py/task.yml

@@ -6,6 +6,20 @@ files:
   - {"name": "utils.py", "source": "../utils.py"}
   - {"name": "reference.py", "source": "reference.py"}
   - {"name": "eval.py", "source": "../eval.py"}
+  - {"name": "pytorch_ref.py", "source": "pytorch_ref.py"}
+  - {"name": "torch_mm_ref.py", "source": "torch_mm_ref.py"}
+
+milestones:
+  - {
+      "milestone_name": "pytorch reference", 
+      "filename": "pytorch_ref.py", 
+      "description": "PyTorch reference implementation as a performance baseline for matmul"
+    }
+  - {
+      "milestone_name": "torch.mm reference", 
+      "filename": "torch_mm_ref.py", 
+      "description": "torch.mm reference implementation as a performance baseline for matmul"
+    }
 
 lang: "py"
 

examples/matmul_py/torch_mm_ref.py

@@ -0,0 +1,9 @@
+#!POPCORN leaderboard matmul_py
+
+from task import input_t, output_t
+import torch
+
+
+def custom_kernel(data: input_t) -> output_t:
+    a, b = data
+    return torch.mm(a, b)

src/discord-cluster-manager/api/main.py

@@ -356,7 +356,11 @@ async def run_submission(  # noqa: C901
                 all_leaderboards = [lb["name"] for lb in db.get_leaderboards()]
                 if leaderboard_name not in all_leaderboards:
                     raise HTTPException(
-                        status_code=404, detail=f"Leaderboard '{leaderboard_name}' not found."
+                        status_code=404,
+                        detail=(
+                            f"Leaderboard '{leaderboard_name}' not found "
+                            "when trying to run submission."
+                        )
                     )
                 else:
                     raise HTTPException(

src/discord-cluster-manager/api/utils.py

@@ -155,7 +155,7 @@ async def _run_submission(
     submission: SubmissionRequest, user_info: dict, mode: SubmissionMode, bot
 ):
     try:
-        req = prepare_submission(submission, bot.leaderboard_db)
+        req = prepare_submission(submission, bot.leaderboard_db, mode)
     except Exception as e:
         raise HTTPException(status_code=400, detail=str(e)) from e