Merge pull request #2 from egraphs-good/benchmark

Add comparison benchmark results and benchmarking file

Author: kirstenmg

.gitignore

@@ -0,0 +1,7 @@
+*.class
+jd-cli
+jd-cli.jar
+jd-cli.bat
+tmp/
+glpk-5.0/
+optimized/

Dockerfile

@@ -3,17 +3,24 @@ FROM ubuntu:14.04.2
 
 # Install dependencies: Java 6, build essential (to build glpk), wget (to download jd-cli)
 RUN apt-get update && \
-    apt-get -y install openjdk-6-jdk build-essential wget
+    apt-get -y install openjdk-6-jdk build-essential wget git
 
 # Download the repository
-# FIXME: we need to make the repo public for this to work
-# RUN git clone [email protected]:egraphs-good/peggy-comparison.git
+RUN git clone https://github.com/egraphs-good/peggy-comparison.git
 
-# # Install GLPK. We just need the glpsol executable.
-# RUN wget "https://ftp.gnu.org/gnu/glpk/glpk-5.0.tar.gz" && \
-#     unzip glpk-5.0.tar.gz && cd glpk-5.0 && \
-#     ./configure && make install && \
-#     mv examples/glpsol /usr/bin/glpsol && cd glpk-5.0 && rm -rf glpk-5.0
 
-# # Install jd-cli for command line decompiling. We need an old version to work with Java 6.
-# RUN wget "https://github.com/intoolswetrust/jd-cli/releases/download/jd-cmd-0.9.2.Final/jd-cli-0.9.2-dist.tar.gz" && tar -xzvf jd-cli-0.9.2-dist.tar.gz 
+# Install GLPK. We just need the glpsol executable.
+RUN wget "https://ftp.gnu.org/gnu/glpk/glpk-5.0.tar.gz" && \
+    tar -xzvf glpk-5.0.tar.gz && cd glpk-5.0 && \
+    ./configure && make install && \
+    mv examples/glpsol /usr/bin/glpsol && \
+    cd .. && rm -rf glpk-5.0 && rm glpk-5.0.tar.gz
+
+# Install jd-cli for command line decompiling. We need an old version to work with Java 6.
+RUN wget "https://github.com/intoolswetrust/jd-cli/releases/download/jd-cmd-0.9.2.Final/jd-cli-0.9.2-dist.tar.gz" && \
+    tar -xzvf jd-cli-0.9.2-dist.tar.gz && \
+    mv jd-cli peggy-comparison && mv jd-cli.jar peggy-comparison && \
+    rm jd-cli-0.9.2-dist.tar.gz && rm jd-cli.bat
+
+# Everything we need is in peggy-comparison
+WORKDIR /peggy-comparison

Foo.java

@@ -16,3 +16,8 @@ Now run the image.
 ```
 docker run -it peggy
 ```
+
+# Comparison benchmark
+This benchmark is qualitative, based on analysis of the optimized output of peggy on certain small example files. These files are in the `benchmark` directory.
+
+Decompiled output for each of the benchmark Java files is stored in `results`. If you would like to re-generate these files, run `python3 benchmark_peggy.py`. In a Docker container, this took 24 min user time, 41 min total time.

README.md

@@ -0,0 +1,19 @@
+public class UnswitchedLoop {
+    // This causes the encoding of output CFG to never terminate
+    public int expected(int n) {
+        int j = 0;
+        if (n < 0) {
+            for (int i = 0; i < n; i++) {
+                System.out.println(i);
+                j = 2;
+                j++;
+            }
+        } else {
+            for (int i = 0; i < n; i++) {
+                System.out.println(i);
+                j++;
+            }
+        }
+        return j;
+    }
+}

benchmark/failing/UnswitchedLoop.java

@@ -0,0 +1,29 @@
+public class BranchHoisting {
+    public int original(int n) {
+        int y = 0;
+        int x = 0;
+        while (y < 500) {
+            if (n == 0) {
+               x = y * 2;
+            } else {
+                x = y * 3;
+            }
+            y++;
+        }
+        return x;
+    }
+
+    public int expected(int n) {
+        int y = 0;
+        int x = 0;
+        while (y < 500) y++;
+
+        if (n == 0) {
+            x = y * 2;
+        } else {
+            x = y * 3;
+        }
+
+        return x;
+    }
+}

benchmark/passing/BranchHoisting.java

@@ -0,0 +1,15 @@
+public class ConditionalConstantFolding {
+    public int original(int x) {
+        if (x == 5) {
+            return 4 * x;
+        } else if (x == 4) {
+            return 5 * x;
+        } else {
+            return 20;
+        }
+    }
+
+    public int expected() {
+        return 20;
+    }
+}

benchmark/passing/ConditionalConstantFolding.java

@@ -0,0 +1,7 @@
+public class ConstantFold {
+    public int original() {
+        int j = 1 + 1;
+        int k = j * 3;
+        return k - 10;
+    }
+}

benchmark/passing/ConstantFold.java

@@ -0,0 +1,11 @@
+public class DeadLoopDeletion {
+    public int original() {
+        int j = 3;
+        for (int i = 0; i < 4; i++) {
+            j++;
+        }
+        j = 2;
+        return j;
+    }
+    
+}

benchmark/passing/DeadLoopDeletion.java

@@ -0,0 +1,13 @@
+public class IfTrue {
+    public int original(int x) {
+        if (true) {
+            return x;
+        } else {
+            return x - 1;
+        }
+    }
+
+    public int expected(int x) {
+        return x;
+    }
+}