fix(py): update script to collect crab's results

scripts/get_exper_brunch_stat.py

@@ -8,6 +8,8 @@ def read_brunchstat_from_log(log_file_name):
   line = log_file.readline()
   data = defaultdict(list)
   cur_test = None
+  first = False
+  second = False
   while line:
     # look for next test
     new_test = re.search("Test: ", line)
@@ -19,6 +21,20 @@ def read_brunchstat_from_log(log_file_name):
     elif line.startswith("BRUNCH_STAT"):
       stat = line.split()
       # stat_name = " ".join(stat[1:-1])
+      if stat[1] == "crab.isderef.not.solve":
+        first = True
+      if stat[1] == "crab.isderef.solve":
+        if not first:
+          data[cur_test].append(0)
+          first = True
+        second = True
+      if stat[1] == "BMC":
+        if not first:
+          data[cur_test].append(0)
+        if not second:
+          data[cur_test].append(0)
+        first = False
+        second = False
       stat_num = stat[-1]
       data[cur_test].append(stat_num)
     line = log_file.readline()
@@ -34,9 +50,10 @@ def write_brunchstat_into_csv(data, out_file):
       "result",
       "bmc.circ_sz",
       "bmc.dag_sz",
+      "crab.isderef.not.solve",
+      "crab.isderef.solve",
       "bmc_time",
       "bmc_solve_time",
-      "cda_time", "gssa_time",
       "opsem_assert_time",
       "opsem_simp_time",
       "seahorn_total_time"])

scripts/get_exper_res.py

@@ -30,6 +30,7 @@ def get_output_level():
 def make_new_cmake_conf():
     use_klee = "ON" if args.klee else "OFF"
     use_bleeding_edge = "ON" if args.bleed_edge else "OFF"
+    use_crab = "ON" if args.usecrab else "OFF"
     if args.smack_parser:
         use_smack = "ON" if args.smack else "OFF"
         smack_enable_no_mem_split = "ON" if args.mem_no_split else "OFF"
@@ -39,7 +40,8 @@ def make_new_cmake_conf():
         smack_args = ""
     return f'cmake -DSEA_LINK=llvm-link-10 -DCMAKE_C_COMPILER=clang-10\
     -DCMAKE_CXX_COMPILER=clang++-10 -DSEA_ENABLE_KLEE={use_klee} {smack_args}\
-    -DSEA_WITH_BLEEDING_EDGE={use_bleeding_edge} -DSEAHORN_ROOT={SEAHORN_ROOT} ../ -GNinja'
+    -DSEA_WITH_BLEEDING_EDGE={use_bleeding_edge} -DSEA_ENABLE_CRAB={use_crab}\
+    -DSEAHORN_ROOT={SEAHORN_ROOT} ../ -GNinja'
 
 
 def read_data_from_xml(res_data):
@@ -174,6 +176,7 @@ def main():
         description='Present flags to decide which tool will be tested.')
     parser.add_argument('--seahorn', action='store_true', default=True)
     parser.add_argument('--klee', action='store_true', default=False)
+    parser.add_argument('--usecrab', action='store_true', default=False)
     parser.add_argument('--bleed_edge', action='store_true', default=False)
     parser.add_argument('--debug', action='store_true', default=False)
     parser.add_argument('--timeout', type=int, default=2000,