Add test that parses fuzzer_stats (google#17)

PR google#15 added new statistics outputed to the out/fuzzer_stats file. Add tests for it.

- create a .travis/ directory in which we can put helper scripts for travis
- create .travis/check_fuzzer_stats.sh to parse out/fuzzer_stats and check for expected key:value pairs.
- run several jobs to test for different environment variables ( AFL_EXIT_WHEN_DONE, AFL_BENCH_JUST_ONE, AFL_BENCH_UNTIL_CRASH, and manual stopping)

Author: neuracr

.travis.yml

@@ -1,7 +1,10 @@
 language: c
 
 env:
-  - AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_NO_UI=1
+  - AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_NO_UI=1 AFL_STOP_MANUALLY=1
+  - AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_NO_UI=1 AFL_EXIT_WHEN_DONE=1 
+ # TODO: test AFL_BENCH_UNTIL_CRASH once we have a target that crashes
+  - AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1 AFL_NO_UI=1 AFL_BENCH_JUST_ONE=1
 
 before_install:
   - sudo apt update
@@ -13,10 +16,18 @@ script:
   - ./afl-gcc ./test-instr.c -o test-instr-gcc
   - mkdir seeds
   - echo "" > seeds/nil_seed
-  - timeout --preserve-status 5s ./afl-fuzz -i seeds -o out/ -- ./test-instr-gcc
+  - if [ -z "$AFL_STOP_MANUALLY" ]; 
+    then ./afl-fuzz -i seeds -o out/ -- ./test-instr-gcc; 
+    else timeout --preserve-status 5s ./afl-fuzz -i seeds -o out/ -- ./test-instr-gcc;
+    fi
+  - .travis/check_fuzzer_stats.sh -o out -k peak_rss_mb -v 1450 -p 500
   - rm -r out/*
   - ./afl-clang ./test-instr.c -o test-instr-clang
-  - timeout --preserve-status 5s ./afl-fuzz -i seeds -o out/ -- ./test-instr-clang
+  - if [ -z "$AFL_STOP_MANUALLY" ]; 
+    then ./afl-fuzz -i seeds -o out/ -- ./test-instr-clang; 
+    else timeout --preserve-status 5s ./afl-fuzz -i seeds -o out/ -- ./test-instr-clang;
+    fi
+  - .travis/check_fuzzer_stats.sh -o out -k peak_rss_mb -v 1450 -p 500
   - make clean
   - CC=clang CXX=clang++ make
   - cd llvm_mode
@@ -26,9 +37,18 @@ script:
   - cd ..
   - rm -r out/*
   - ./afl-clang-fast ./test-instr.c -o test-instr-clang-fast
-  - timeout --preserve-status 5s ./afl-fuzz -i seeds -o out/ -- ./test-instr-clang-fast
+  - if [ -z "$AFL_STOP_MANUALLY" ]; 
+    then ./afl-fuzz -i seeds -o out/ -- ./test-instr-clang; 
+    else timeout --preserve-status 5s ./afl-fuzz -i seeds -o out/ -- ./test-instr-clang-fast;
+    fi
+  - .travis/check_fuzzer_stats.sh -o out -k peak_rss_mb -v 1450 -p 500
   - cd qemu_mode
   - ./build_qemu_support.sh
   - cd ..
   - gcc ./test-instr.c -o test-no-instr
-  - timeout --preserve-status 5s ./afl-fuzz -Q -i seeds -o out/ -- ./test-no-instr
+  - if [ -z "$AFL_STOP_MANUALLY" ]; 
+    then ./afl-fuzz -Q -i seeds -o out/ -- ./test-no-instr; 
+    else timeout --preserve-status 5s ./afl-fuzz -Q -i seeds -o out/ -- ./test-no-instr;
+    fi
+  - .travis/check_fuzzer_stats.sh -o out -k peak_rss_mb -v 12000 -p 9000
+

.travis/check_fuzzer_stats.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+usage() { 
+  echo "Usage: $0 -o <out_dir> -k <key> -v <value> [-p <precision>]" 1>&2;
+  echo " " 1>&2;
+  echo "Checks if a key:value appears in the fuzzer_stats report" 1>&2;
+  echo " " 1>&2;
+  echo -n "If \"value\" is numeric and \"precision\" is defined, checks if the stat " 1>&2;
+  echo "printed by afl is value+/-precision." 1>&2;
+  exit 1; }
+
+while getopts "o:k:v:p:" opt;  do
+  case "${opt}" in
+      o)
+        o=${OPTARG}
+        ;;
+      k)
+        k=${OPTARG}
+        ;;
+      v)
+        v=${OPTARG}
+        ;;
+      p)
+        p=${OPTARG}
+        ;;
+      *)
+        usage
+        ;;
+  esac
+done
+
+if [ -z $o ] || [ -z $k ] || [ -z $v ]; then usage; fi
+
+# xargs to trim the surrounding whitespaces
+stat_v=$( grep $k "$o"/fuzzer_stats | cut -d ":" -f 2 | xargs )
+v=$( echo "$v" | xargs )
+
+if [ -z stat_v ];
+  then echo "ERROR: key $k not found in fuzzer_stats." 1>&2
+  exit 1
+fi
+
+re_percent='^[0-9]+([.][0-9]+)?\%$'
+# if the argument is a number in percentage, get rid of the %
+if [[  "$v" =~ $re_percent ]]; then v=${v: :-1}; fi
+if [[  "$stat_v" =~ $re_percent ]]; then stat_v=${stat_v: :-1}; fi
+
+re_numeric='^[0-9]+([.][0-9]+)?$'
+# if the argument is not a number, we check for strict equality
+if (! [[  "$v" =~ $re_numeric ]]) || (! [[  "$stat_v" =~ $re ]]);
+  then if [ "$v" != "$stat_v" ];
+    then echo "ERROR: \"$k:$stat_v\" (should be $v)." 1>&2
+    exit 2;
+  fi
+# checks if the stat reported by afl is in the range
+elif [ "$stat_v" -lt $(( v - p )) ] || [ "$stat_v" -gt $(( v + p )) ];
+  then echo "ERROR: key $k:$stat_v is out of correct range." 1>&2
+  exit 3;
+fi
+echo "OK: key $k:$stat_v" 1>&2
+