Benchmarks for Aderyn and individual detectors (#290)

Author: TilakMaddy

Cargo.lock

@@ -0,0 +1 @@
+aderyn-report-for-bench.md

aderyn_driver/.gitignore

@@ -10,4 +10,11 @@ license = "MIT"
 
 [dependencies]
 aderyn_core = { path  = "../aderyn_core", version = "0.0.21" }
-rayon = "1.8.0"
+rayon = "1.8.0"
+
+[dev-dependencies]
+criterion = "0.5.1"
+
+[[bench]]
+name = "detectors_benchmarks"
+harness = false

aderyn_driver/Cargo.toml

@@ -0,0 +1,45 @@
+use std::path::PathBuf;
+
+use aderyn_driver::{
+    detector::get_all_issue_detectors,
+    driver::{drive, Args},
+};
+
+use criterion::{criterion_group, criterion_main, Criterion};
+
+fn bench_individual_detectors_on_contract_playground(c: &mut Criterion) {
+    let root_path = PathBuf::from("../tests/contract-playground");
+    let (_, context) = aderyn_driver::with_project_root_at(&root_path, &None, &None);
+
+    for mut detector in get_all_issue_detectors() {
+        c.bench_function(detector.name().as_str(), |b| {
+            b.iter(|| {
+                let _ = detector.detect(&context).unwrap();
+            })
+        });
+    }
+}
+
+fn bench_aderyn_on_contract_playground(c: &mut Criterion) {
+    let root_path = PathBuf::from("../tests/contract-playground");
+
+    c.bench_function("aderyn", |b| {
+        b.iter(|| {
+            drive(Args {
+                root: root_path.to_string_lossy().to_string(),
+                output: String::from("aderyn-report-for-bench.md"),
+                no_snippets: false,
+                exclude: None,
+                scope: None,
+                stdout: false,
+            });
+        });
+    });
+}
+
+criterion_group!(
+    detectors,                                         // Group name is the first argument
+    bench_aderyn_on_contract_playground,               // Group 1
+    bench_individual_detectors_on_contract_playground, // Group 2
+);
+criterion_main!(detectors);

aderyn_driver/benches/detectors_benchmarks.rs

@@ -7,3 +7,4 @@ pub use aderyn_core::ast as core_ast;
 pub use aderyn_core::context;
 pub use aderyn_core::detect as detection_modules;
 pub use aderyn_core::detect::detector;
+pub use process_foundry::with_project_root_at;

aderyn_driver/src/lib.rs

@@ -0,0 +1 @@
+{"group_id":"aderyn","function_id":null,"value_str":null,"throughput":null,"full_id":"aderyn","directory_name":"aderyn","title":"aderyn"}

benchmarks/aderyn/base/benchmark.json

@@ -0,0 +1 @@
+{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":110012715.45975,"upper_bound":119876058.76},"point_estimate":114670170.35,"standard_error":2522559.824001567},"median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":103040968.0,"upper_bound":114958188.0},"point_estimate":107556753.5,"standard_error":3978876.4603039916},"median_abs_dev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":14318521.333095431,"upper_bound":23442213.25071752},"point_estimate":17426259.924522042,"standard_error":2391538.6143476125},"slope":null,"std_dev":{"confidence_interval":{"confidence_level":0.95,"lower_bound":18201039.33954221,"upper_bound":32321162.895440847},"point_estimate":25379575.59235664,"standard_error":3634033.308885464}}

benchmarks/aderyn/base/estimates.json

@@ -0,0 +1 @@
+{"sampling_mode":"Flat","iters":[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0],"times":[93792368.0,125797761.0,96254770.0,102374608.0,96493368.0,94031566.0,92396047.0,92177740.0,143306731.0,117915483.0,116549204.0,120464727.0,119302292.0,103040968.0,111625647.0,100710330.0,95988974.0,100853519.0,103700207.0,96537843.0,102143255.0,99264821.0,104100188.0,103455867.0,190820242.0,199394390.0,142701189.0,100542974.0,91910714.0,93222451.0,92382061.0,93195709.0,92210554.0,92074295.0,93916028.0,94650533.0,95918744.0,93382773.0,89506898.0,93195935.0,91561765.0,91837441.0,104168599.0,103359199.0,113947329.0,109202194.0,95463817.0,91603965.0,97421558.0,132146204.0,125159248.0,99479231.0,93373712.0,92970819.0,94648268.0,100649670.0,105911313.0,114752590.0,112190279.0,116569945.0,101618637.0,100205137.0,99485415.0,103899776.0,104103036.0,105496380.0,117068490.0,114958188.0,119318918.0,114960419.0,102964747.0,119196332.0,149610159.0,139797727.0,122250300.0,120796880.0,241071573.0,156090165.0,160000214.0,137333096.0,137873192.0,139612525.0,146369137.0,127092104.0,109784544.0,115614577.0,114103645.0,115323101.0,126611514.0,159452369.0,124081522.0,117945249.0,116389840.0,120472367.0,115915601.0,163522498.0,114205847.0,172101235.0,111150712.0,121448947.0]}

benchmarks/aderyn/base/sample.json

@@ -0,0 +1 @@
+[24074388.25,60254053.375,156733160.375,192912825.5]

benchmarks/aderyn/base/tukey.json

@@ -0,0 +1 @@
+{"mean":{"confidence_interval":{"confidence_level":0.95,"lower_bound":0.18459828421517133,"upper_bound":0.2908493398392847},"point_estimate":0.23709096764774906,"standard_error":0.0274031214685985},"median":{"confidence_interval":{"confidence_level":0.95,"lower_bound":0.1222985354981565,"upper_bound":0.25353205407930934},"point_estimate":0.1730272680196443,"standard_error":0.043756226418304595}}