Intro refs. fixed

Author: limo1996

python_wrapper/bsub_script_scaling.sh

@@ -30,13 +30,17 @@ do
     done
 done
 
+# remove the two unsat formulas
+rm uuf50-01.cnf
+rm uuf50-02.cnf
+
 for nodes in ${num_nodes}
 do
     for file in ${FILES}
     do
         for run in $(seq 1 $num_runs)
         do
-            timeout "$timeout"s mpirun -np "$nodes" ../stealing_parallel_main "$file" > "$file".out
+            timeout "$timeout"s mpirun -np "$nodes" ../stealing_main "$file" > "$file".out
         done
     done
 done

python_wrapper/cnf_difficulty.py

@@ -0,0 +1,28 @@
+import subprocess
+import os
+
+PROGRAM = '../sequential_main -cerr-level 1'
+FOLDER = '../cnfs/benchmark_formulas'
+
+
+def main():
+    files = sorted(['../cnfs/benchmark_formulas/{}'.format(f)
+                    for f in os.listdir(FOLDER)
+                    if os.path.isfile(os.path.join(FOLDER, f))
+                    and f.endswith('.cnf')])
+    for f in files:
+        diff = run(f)
+        print('Formula {} needs {} dpll branches'.format(f, diff))
+
+
+def run(cnf):
+    command = PROGRAM + " " + cnf
+    p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+    p.wait()
+    stderr = [line.decode('utf-8').strip() for line in iter(p.stderr.readline, b'')]
+    return len(stderr)
+
+
+if __name__ == '__main__':
+    main()

python_wrapper/euler.py

@@ -81,9 +81,7 @@ def run_test(self, basepath='.', script='bsub_script.sh'):
         os.remove('{}/timeout.txt'.format(basepath))
         os.remove('{}/overall_runtime_minutes.txt'.format(basepath))
         print('once the job is finished, run the following command to download the results file:')
-        print('scp -o PreferredAuthentications=password -o PubkeyAuthentication=no {0}@euler.ethz.ch:{1}/{2}/time_measurements.tar .'.format(self.nethz_username, self.euler_folder_name, self.folder))
-        print('scp -o PreferredAuthentications=password -o PubkeyAuthentication=no {0}@euler.ethz.ch:{1}/{2}/wait_measurements.tar .'.format(self.nethz_username, self.euler_folder_name, self.folder))
-        print('scp -o PreferredAuthentications=password -o PubkeyAuthentication=no {0}@euler.ethz.ch:{1}/{2}/comm_measurements.tar .'.format(self.nethz_username, self.euler_folder_name, self.folder))
+        print('scp -o PreferredAuthentications=password -o PubkeyAuthentication=no {0}@euler.ethz.ch:{1}/{2}/measurements.tar .'.format(self.nethz_username, self.euler_folder_name, self.folder))
 
     def run_test_for_num_nodes(self, basepath, script='bsub_script.sh'):
         tar = self.package_tar(basepath, script)

python_wrapper/experiments/dpll_scaling.py

@@ -2,15 +2,18 @@
 import sys
 
 import click
-import numpy as np
 import matplotlib.pyplot as plt
+import numpy as np
+from scipy.interpolate import spline
 
 from abstract_experiment import AbstractExperiment
 from utils import get_netz_username, parse_into_dict, conf_95_mean
 
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from euler import EulerTester
 
+NAME_MAP = {'parallel': 'Parallel', 'stealing': 'Work Stealing'}
+
 
 class DpllScaling(AbstractExperiment):
     def __init__(self):
@@ -20,7 +23,7 @@ def __init__(self):
     def run_experiment(self):
         test_folder = 'benchmark_formulas'
         # number of cores
-        num_nodes = [2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 32, 40, 48]
+        num_nodes = [2, 3, 4, 5, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32, 36, 40, 44, 48]
         # number of runs per formula
         num_runs = 10
         # timeout per formula in seconds
@@ -36,70 +39,162 @@ def process_euler_experiment(self):
         self.data = parse_into_dict('measurements.tar')
 
     def plot(self):
-        self._runtime_plot()
-        self._speedup_plot()
+        for p in ['parallel', 'stealing']:
+            self._runtime_plot(p)
+            self._speedup_plot(p)
+        files = ['uf50-01', 'par8-1-c', 'ais6', 'flat75-4', 'par8-4-', 'anomaly']
+        colors = ['teal', 'firebrick', 'darkorange', 'royalblue',
+                  'palevioletred', 'slateblue']
+        offsets = [-0.25, 0.25, 0, 0.125, -0.125,
+                   -0.25, 0.25, 0, 0.125, -0.125,
+                   -0.25, 0.25, 0, 0.125, -0.125]
+        markers = ['o', 'd', 's', '^', 'x', 'D']
+        figure = plt.figure()
+        ax = figure.add_subplot(1, 1, 1)
+        for f in range(len(files)):
+            self._plot_runtime(self.data[files[f]], 'parallel', ax,
+                               color=colors[f], offset=offsets[f])
+        #plt.show()
+        for p in ['parallel', 'stealing']:
+            figure = plt.figure()
+            ax = figure.add_subplot(1, 1, 1)
+            for f in range(len(files)):
+                self._plot_speedup(self.data[files[f]], p, ax,
+                                   color=colors[f], marker=markers[f],
+                                   offset=offsets[f], plotline=(f == 0),
+                                   legend=files[f])
+            plt.title('Speedup with {} Version'.format(NAME_MAP[p]))
+            plt.xlabel('# cores')
+            plt.ylabel('speedup')
+            handles, labels = ax.get_legend_handles_labels()
+            h = handles.pop(0)
+            handles.append(h)
+            l = labels.pop(0)
+            labels.append(l)
+            plt.legend(handles, labels)
+            plt.tight_layout(pad=0)
+            plt.savefig('../../report/figures/dpll_scaling_{}.pdf'.format(p),
+                        format='pdf')
+            #plt.show()
+            figure = plt.figure()
+            ax = figure.add_subplot(1, 1, 1)
+            for f in range(len(files)):
+                self._plot_waiting(self.data[files[f]], p, ax,
+                                   color=colors[f], marker=markers[f],
+                                   offset=offsets[f], label=files[f])
+            plt.title('Overall waiting time per cnf {} Version'.format(
+                NAME_MAP[p]))
+            plt.xlabel('# cores')
+            plt.ylabel('overall waiting time [ms]')
+            plt.legend()
+            plt.tight_layout(pad=0)
+            plt.savefig('../../report/figures/dpll_waiting_{}.pdf'.format(p),
+                        format='pdf')
+            #plt.show()
 
-    def _runtime_plot(self):
+    def _runtime_plot(self, parallel_key):
         for f, v in self.data.items():
-            xs = []
-            ys = []
-            lowers = []
-            uppers = []
-            if 'seq' in v:
-                xs.append(1)
-                mean = np.mean(np.array(v['seq']['time']).flatten())
-                ys.append(mean)
-                lower, upper = conf_95_mean(np.array(v['seq']['time']).flatten())
-                lowers.append(mean - lower)
-                uppers.append(upper - mean)
-            for cores, v_ in v['parallel'].items():
-                xs.append(int(cores))
-                mean = np.mean(np.array(v_['time']).flatten())
-                ys.append(mean)
-                lower, upper = conf_95_mean(np.array(v_['time']).flatten())
-                lowers.append(mean - lower)
-                uppers.append(upper - mean)
-            ys = [y for _, y in sorted(zip(xs, ys))]
-            xs = sorted(xs)
-            plt.figure()
-            plt.errorbar(xs, ys, yerr=[lowers, uppers], fmt='o',
-                         markerfacecolor='none')
+            figure = plt.figure().add_subplot(1, 1, 1)
+            self._plot_runtime(v, parallel_key, figure)
             plt.title('{}.cnf'.format(f))
-            plt.ylim(0, max(ys) + 500)
             plt.xlabel('# cores')
-            plt.ylabel('avg runtime [ms]')
-            plt.savefig('figures/{}.png'.format(f))
+            plt.ylabel('avg total waiting time [ms]')
+            plt.savefig('figures/{}_{}.png'.format(f, parallel_key))
 
-    def _speedup_plot(self):
+    def _plot_runtime(self, data, parallel_key, figure, color='teal', offset=0.0):
+        if parallel_key not in data:
+            return
+        xs, ys, lowers, uppers = [], [], [], []
+        if 'seq' in data:
+            xs.append(1 + offset)
+            mean = np.mean(np.array(data['seq']['time']).flatten())
+            ys.append(mean)
+            lower, upper = conf_95_mean(np.array(data['seq']['time']).flatten())
+            lowers.append(mean - lower)
+            uppers.append(upper - mean)
+        for cores, v_ in data[parallel_key].items():
+            xs.append(int(cores))
+            mean = np.mean(np.array(v_['time']).flatten())
+            ys.append(mean)
+            lower, upper = conf_95_mean(np.array(v_['time']).flatten())
+            lowers.append(mean - lower)
+            uppers.append(upper - mean)
+        ys = [y for _, y in sorted(zip(xs, ys))]
+        xs = sorted(xs)
+        figure.errorbar(xs, ys, yerr=[lowers, uppers], fmt='o',
+                        markerfacecolor='none', color=color)
+        xinter = np.linspace(min(xs), max(xs), 300)
+        yinter = spline(xs, ys, xinter)
+        plt.plot(xinter, yinter, '-', alpha=0.5, linewidth=0.5,
+                 color=color)
+
+    def _speedup_plot(self, parallel_key):
         for f, v in self.data.items():
-            xs = []
-            ys = []
-            lowers = []
-            uppers = []
-            if 'seq' in v:
-                sequential = np.array(v['seq']['time']).flatten()
-                for cores, v_ in v['parallel'].items():
-                    xs.append(int(cores))
-                    parallel = []
-                    for times in v_['time']:
-                        parallel.append(np.mean(times))
-                    speedup = sequential / parallel
-                    mean_speedup = np.mean(speedup)
-                    ys.append(mean_speedup)
-                    lower, upper = conf_95_mean(speedup)
-                    lowers.append(mean_speedup - lower)
-                    uppers.append(upper - mean_speedup)
-                ys = [y for _, y in sorted(zip(xs, ys))]
-                xs = sorted(xs)
-                plt.figure()
-                plt.errorbar(xs, ys, yerr=[lowers, uppers], fmt='o',
-                             markerfacecolor='none')
-                plt.plot([0, max(xs)], [0, max(xs)], '--')
-                plt.title('{}.cnf'.format(f))
-                plt.xlabel('# cores')
-                plt.ylabel('speedup')
-                plt.savefig('figures/{}_speedup.png'.format(f))
+            figure = plt.figure().add_subplot(1, 1, 1)
+            self._plot_speedup(v, parallel_key, figure)
+            plt.title('{}.cnf'.format(f))
+            plt.xlabel('# cores')
+            plt.ylabel('speedup')
+            plt.savefig('figures/{}_speedup_{}.png'.format(f, parallel_key))
+
+    def _plot_speedup(self, data, parallel_key, figure, color='teal',
+                      marker='o', offset=0.0, plotline=True, legend=''):
+        if parallel_key not in data:
+            return
+        xs, ys, lowers, uppers = [], [], [], []
+        if 'seq' in data:
+            sequential = np.array(data['seq']['time']).flatten()
+            for cores, v_ in data[parallel_key].items():
+                xs.append(int(cores))
+                parallel = []
+                for times in v_['time']:
+                    parallel.append(np.mean(times))
+                speedup = sequential / parallel
+                mean_speedup = np.mean(speedup)
+                ys.append(mean_speedup)
+                lower, upper = conf_95_mean(speedup)
+                lowers.append(mean_speedup - lower)
+                uppers.append(upper - mean_speedup)
+            ys = [y for _, y in sorted(zip(xs, ys))]
+            xs = sorted(xs)
+            xs = list(map(lambda x: float(x) + offset, xs))
+            if legend == '':
+                figure.errorbar(xs, ys, yerr=[lowers, uppers], fmt=marker,
+                                markerfacecolor='none', color=color)
+            else:
+                figure.errorbar(xs, ys, yerr=[lowers, uppers], fmt=marker,
+                                markerfacecolor='none', color=color,
+                                label=legend)
+            #xinter = np.linspace(min(xs), max(xs), 300)
+            #yinter = spline(xs, ys, xinter)
+            #plt.plot(xinter, yinter, '-', alpha=0.5, linewidth=0.5,
+            #         color=color)
+            plt.plot(xs, ys, '-', alpha=0.5, linewidth=0.5, color=color)
+            if plotline:
+                plt.plot([0, max(xs)], [0, max(xs)], '-', color='black',
+                         label='Linear Speedup')
 
+    def _plot_waiting(self, data, parallel_key, figure, color='teal',
+                      offset=0.0, label='', marker='o'):
+        if parallel_key not in data:
+            return
+        xs, ys, lowers, uppers = [], [], [], []
+        for cores, v_ in data[parallel_key].items():
+            xs.append(int(cores) + offset)
+            sums = list(map(lambda wt: np.sum(np.array(wt[1:])),
+                             v_['wait']))
+            mean = np.mean(sums)
+            ys.append(mean)
+            lower, upper = conf_95_mean(sums)
+            lowers.append(mean - lower)
+            uppers.append(upper - mean)
+        ys = [y for _, y in sorted(zip(xs, ys))]
+        lowers = [l for _, l in sorted(zip(xs, lowers))]
+        uppers = [u for _, u in sorted(zip(xs, uppers))]
+        xs = sorted(xs)
+        figure.errorbar(xs, ys, yerr=[lowers, uppers], fmt=marker,
+                        markerfacecolor='none', color=color, label=label)
+        plt.plot(xs, ys, '-', alpha=0.5, linewidth=0.5, color=color)
 
 @click.command()
 @click.option('--rerun/--no-rerun', default=False,
@@ -110,7 +205,7 @@ def main(rerun, process):
     e = DpllScaling()
     if rerun:
         e.run_experiment()
-    if process:
+    elif process:
         e.process_euler_experiment()
         e.results_to_json()
     else:

python_wrapper/experiments/results/DpllScaling.json

@@ -126,10 +126,15 @@ def delete_files_in_folder(folder, file_extension):
 
 
 def conf_95_mean(measurements):
-    return stats.t.interval(0.95,
+    lower, upper = stats.t.interval(0.95,
                             len(measurements) - 1,
                             loc=np.mean(measurements),
                             scale=stats.sem(measurements))
+    if lower != lower:  # check if nan
+        lower = np.mean(measurements)
+    if upper != upper:  # check if nan
+        upper = np.mean(measurements)
+    return lower, upper
 
 
 def get_netz_username():