bench(feat): Add benchmarking cobc -> clang executables.

Author: c272

clean.sh

@@ -59,6 +59,7 @@ rm -rf "$BUILD_DIR"
 echo -e "${CYAN}Cleaning generated benchmark artifacts...${NC}"
 BENCH_OUT_DIR="${SCRIPT_DIR}/bench_out"
 rm -rf $BENCH_OUT_DIR/*.o
+rm -rf $BENCH_OUT_DIR/*.c*
 rm -rf $BENCH_OUT_DIR/bench_bin
 
 # Done!

crates/bench/Cargo.lock

@@ -11,6 +11,7 @@ clap = { version = "4.5.4", features = ["derive"] }
 colored = "2.1.0"
 indicatif = "0.17.8"
 miette = { version = "7.2.0", features = ["fancy"] }
+regex = "1.10.4"
 sanitize-filename = "0.5.0"
 serde = { version = "1.0.199", features = ["derive"] }
 serde_json = "1.0.116"

crates/bench/Cargo.toml

@@ -38,6 +38,11 @@ pub struct Cli {
     #[arg(long, short = 'g', action)]
     pub run_comparative: bool,
 
+    /// Whether to run comparative tests against GnuCobol C -> clang.
+    /// Requires `run_comparative`.
+    #[arg(long, short = 'r', requires("run_comparative"), action)]
+    pub run_clang: bool,
+
     /// Whether to run only a build test and not execute benchmarks.
     #[arg(long, short = 'g', action)]
     pub build_only: bool,
@@ -112,6 +117,25 @@ impl TryInto<Cfg> for Cli {
             }
         }
 
+        // Check that `clang` is available if required.
+        if self.run_clang {
+            match Command::new("clang").output() {
+                Ok(_) => {}
+                Err(e) => {
+                    if let ErrorKind::NotFound = e.kind() {
+                        miette::bail!(
+                            "Clang comparisons enabled, but `clang` was not found! Check your PATH."
+                        );
+                    } else {
+                        miette::bail!(
+                            "An error occurred while verifying if `clang` was available: {}",
+                            e
+                        );
+                    }
+                }
+            }
+        }
+
         // Verify that the output directory exists.
         let output_dir = if let Some(path) = self.output_dir {
             path
@@ -223,6 +247,7 @@ impl TryInto<Cfg> for Cli {
             cobc_force_platform_linker: self.force_platform_linker,
             disable_hw_security: self.disable_hw_security,
             run_comparative: self.run_comparative,
+            run_clang: self.run_clang,
             build_only: self.build_only,
             output_dir,
             output_log,

crates/bench/src/cli.rs

@@ -69,6 +69,10 @@ pub(crate) struct BenchmarkExecution {
     /// Benchmark results for `cobc`, if present.
     /// Only generated when executed with `--run-comparative`.
     pub cobc_results: Option<BenchmarkResult>,
+
+    /// Benchmark results for `cobc` -> `clang`, if present.
+    /// Only generated when executed with `--run-clang`.
+    pub clang_results: Option<BenchmarkResult>,
 }
 
 /// Output for the result of a single benchmark compile/execute pair.

crates/bench/src/log.rs

@@ -7,6 +7,7 @@ use std::{
 
 use colored::Colorize;
 use miette::Result;
+use regex::Regex;
 
 use crate::{
     bench::Benchmark,
@@ -37,6 +38,9 @@ pub(crate) struct Cfg {
     /// Whether to run comparative tests against GnuCobol's `cobc`.
     pub run_comparative: bool,
 
+    /// Whether to run comparative tests against `cobc` -> `clang`.
+    pub run_clang: bool,
+
     /// Whether to only build and not execute benchmarks.
     pub build_only: bool,
 
@@ -72,13 +76,18 @@ pub(crate) fn run_single(cfg: &Cfg, benchmark: &Benchmark) -> Result<BenchmarkEx
         .run_comparative
         .then(|| run_cobc(cfg, benchmark))
         .transpose()?;
+    let clang_results = cfg
+        .run_clang
+        .then(|| run_clang(cfg, benchmark))
+        .transpose()?;
 
     Ok(BenchmarkExecution {
         benchmark: benchmark.clone(),
         started_at,
         ended_at: chrono::offset::Local::now().to_utc(),
         cobalt_results,
         cobc_results,
+        clang_results,
     })
 }
 
@@ -182,6 +191,122 @@ fn run_cobc(cfg: &Cfg, benchmark: &Benchmark) -> Result<BenchmarkResult> {
     })
 }
 
+/// Executes a single benchmark using GnuCobol's `cobc`'s C output followed by
+/// binary generation using `clang`.
+fn run_clang(cfg: &Cfg, benchmark: &Benchmark) -> Result<BenchmarkResult> {
+    // Calculate output file locations for benchmarking binary, C file.
+    // We also need a bootstrapping file for `main` since GnuCobol doesn't create
+    // one for us.
+    let mut bench_bin_path = cfg.output_dir.clone();
+    bench_bin_path.push(BENCH_BIN_NAME);
+    let mut bench_c_path = cfg.output_dir.clone();
+    bench_c_path.push(format!("{}.c", &benchmark.name));
+    let mut bootstrap_path = cfg.output_dir.clone();
+    bootstrap_path.push("bootstrap.c");
+
+    // Set up commands for transpiling then compiling from C.
+    let mut cobc = Command::new("cobc");
+    cobc.args(["-C", &format!("-O{}", cfg.cobc_opt_level), "-free"])
+        // Required to generate `cob_init()` in the transpiled C.
+        .arg("-fimplicit-init")
+        .args(["-o", bench_c_path.to_str().unwrap()])
+        .arg(&benchmark.source_file);
+    let mut clang = Command::new("clang");
+    clang
+        .args(["-lcob", &format!("-O{}", cfg.cobc_opt_level)])
+        .args(["-o", bench_bin_path.to_str().unwrap()])
+        .arg(bootstrap_path.to_str().unwrap());
+
+    // We require the program ID of the COBOL file to generate the bootstrapper.
+    // Attempt to grep for that from the source.
+    let bench_prog_func = fetch_program_func(benchmark)?;
+
+    // Generate the bootstrapping file.
+    let bootstrapper = format!(
+        "
+        #include \"{}.c\"
+        int main(void) {{
+            {}();
+        }}
+    ",
+        &benchmark.name, bench_prog_func
+    );
+    std::fs::write(bootstrap_path.clone(), bootstrapper)
+        .map_err(|e| miette::diagnostic!("Failed to write bootstrap file for `clang`: {e}"))?;
+
+    let before = Instant::now();
+    for _ in 0..100 {
+        // First, transpile to C.
+        let out = cobc
+            .output()
+            .map_err(|e| miette::diagnostic!("Failed to execute `cobc`: {e}"))?;
+        if !out.status.success() {
+            miette::bail!(
+                "Failed benchmark for '{}' with `cobc` compiler error: {}",
+                benchmark.source_file,
+                String::from_utf8_lossy(&out.stderr)
+            );
+        }
+
+        // Finally, perform compilation & linkage with `clang`.
+        let out = clang
+            .output()
+            .map_err(|e| miette::diagnostic!("Failed to execute `clang`: {e}"))?;
+        if !out.status.success() {
+            miette::bail!(
+                "Failed benchmark for '{}' with `clang` compiler error: {}",
+                benchmark.source_file,
+                String::from_utf8_lossy(&out.stderr)
+            );
+        }
+    }
+    let elapsed = before.elapsed();
+    println!(
+        "clang(compile): Total time {:.2?}, average/run of {:.6?}.",
+        elapsed,
+        elapsed / 100
+    );
+
+    // Run the target program.
+    let (execute_time_total, execute_time_avg) = if !cfg.build_only {
+        let (x, y) = run_bench_bin(cfg, benchmark)?;
+        (Some(x), Some(y))
+    } else {
+        (None, None)
+    };
+
+    Ok(BenchmarkResult {
+        compile_time_total: elapsed,
+        compile_time_avg: elapsed / 100,
+        execute_time_total,
+        execute_time_avg,
+    })
+}
+
+/// Attempts to fetch the output C function name of the given benchmark COBOL file.
+/// If not found, throws an error.
+fn fetch_program_func(benchmark: &Benchmark) -> Result<String> {
+    // First, read in source of benchmark.
+    let source = std::fs::read_to_string(&benchmark.source_file).map_err(|e| {
+        miette::diagnostic!(
+            "Failed to read COBOL source for benchmark '{}': {e}",
+            benchmark.name
+        )
+    })?;
+
+    // Search for a pattern matching "PROGRAM-ID ...".
+    let prog_id_pat = Regex::new(r"PROGRAM-ID\. [A-Z0-9a-z\-]+").unwrap();
+    let prog_id_str = prog_id_pat.find(&source).ok_or(miette::diagnostic!(
+        "Could not find program ID in sources for benchmark '{}'.",
+        &benchmark.name
+    ))?;
+
+    // Extract the program ID, format into final function name.
+    let mut prog_id = prog_id_str.as_str()["PROGRAM-ID ".len()..].to_string();
+    prog_id = prog_id.replace("-", "__");
+    Ok(prog_id)
+}
+
 /// Executes a single generated benchmarking binary.
 /// Returns the total execution time and average execution time per iteration.
 fn run_bench_bin(cfg: &Cfg, benchmark: &Benchmark) -> Result<(Duration, Duration)> {