Merge pull request #728 from egraphs-good/oflatt-nightly-hotfix

Nightly hotfix- missing suites in data

infra/profile.py

@@ -9,8 +9,24 @@
 
 import concurrent.futures
 
-NUM_WARMUP_SAMPLES = 50
-SAMPLES_PER_BENCHMARK_AND_TREATMENT = 200
+
+# testing mode takes much fewer samples than the real eval in the paper
+IS_TESTING_MODE = True
+
+def num_warmup_samples():
+  if IS_TESTING_MODE:
+    return 2
+  return 50
+
+def num_samples():
+  if IS_TESTING_MODE:
+    return 100
+  return 1000
+
+
+def average(lst):
+  return sum(lst) / len(lst)
+
 
 treatments = [
   "rvsdg-round-trip-to-executable",
@@ -138,6 +154,13 @@ def optimize(benchmark):
   return res
 
 
+def take_sample(cmd, benchmark):
+  result = subprocess.run(cmd, capture_output=True, shell=True)
+
+  if result.returncode != 0:
+    raise Exception(f'Error running {benchmark.name} with {benchmark.treatment}: {result.stderr}')
+  return int(result.stderr)
+
 
 def bench(benchmark):
   print(f'[{benchmark.index}/{benchmark.total}] Benchmarking {benchmark.name} with {benchmark.treatment}', flush=True)
@@ -155,27 +178,30 @@ def bench(benchmark):
     else:
       # hyperfine command for measuring time, unused in favor of cycles
       # cmd = f'hyperfine --style none --warmup 1 --max-runs 2 --export-json /dev/stdout "{profile_dir}/{benchmark.treatment}{" " + args if len(args) > 0 else ""}"'
+
+      args_str = " " + args if len(args) > 0 else ""
+      cmd = f'{profile_dir}/{benchmark.treatment}{args_str}'
       num_samples_so_far = 0
+      warmup_cycles = []
       resulting_num_cycles = []
+
+      # take some warmup cycles
+      while len(warmup_cycles) < num_warmup_samples():
+        warmup_cycles.append(take_sample(cmd, benchmark))
+
+      num_to_run = num_samples()
+      print(f'Running {num_to_run} samples for {benchmark.name} with {benchmark.treatment}', flush=True)
+
       while True:
-        args_str = " " + args if len(args) > 0 else ""
-        cmd = f'{profile_dir}/{benchmark.treatment}{args_str}'
-        result = subprocess.run(cmd, capture_output=True, shell=True)
-
-        if result.returncode != 0:
-          raise Exception(f'Error running {benchmark.name} with {benchmark.treatment}: {result.stderr}')
-        res_cycles = int(result.stderr)
-        resulting_num_cycles.append(res_cycles)
+        resulting_num_cycles.append(take_sample(cmd, benchmark))
 
         num_samples_so_far += 1
         # if we have run for at least 1 second and we have at least 2 samples, stop
         #if time.time() - time_start > time_per_benchmark and len(resulting_num_cycles) >= 2:
          # break
-        if num_samples_so_far >= SAMPLES_PER_BENCHMARK_AND_TREATMENT + NUM_WARMUP_SAMPLES:
+        if num_samples_so_far >= num_to_run:
           break
-      # throw away the first NUM_WARMUP_SAMPLES samples
-      resulting_num_cycles = resulting_num_cycles[NUM_WARMUP_SAMPLES:]
-
+
       return (f'{profile_dir}/{benchmark.treatment}', resulting_num_cycles)
 
 # Run modes that we expect to output llvm IR
@@ -192,14 +218,30 @@ def should_have_llvm_ir(runMethod):
     "llvm-eggcc-O3-O0",
   ]
 
+# go up in directory until hitting "passing" folder
+def get_suite(path):
+  while True:
+    # if we can't go up anymore, return unknown
+    if os.path.dirname(path) == path:
+      return "unknown"
+
+    oldpath = path
+    path = os.path.dirname(path)
+    print(os.path.basename(path))
+    if os.path.basename(path) == "passing":
+      return os.path.basename(oldpath)
+
+
 # aggregate all profile info into a single json array.
 def aggregate(compile_data, bench_times, paths):
     res = []
 
     for path in sorted(compile_data.keys()):
       name = path.split("/")[-2]
       runMethod = path.split("/")[-1]
-      result = {"runMethod": runMethod, "benchmark": name, "cycles": bench_times[path], "path": paths[name]}
+      suite = get_suite(paths[name])
+      result = {"runMethod": runMethod, "benchmark": name, "cycles": bench_times[path], "path": paths[name], "suite": suite}
+
 
       # add compile time info
       for key in compile_data[path]: