refactor: split out library parts from regtest (#61)

tests/ui/hir/instantiation-missing-type-arguments.eight

@@ -1,4 +1,4 @@
-// RUN: not %eightc %s 2>&1 | %regtest %s.snap
+// RUN: not %eightc %s 2>&1 | %regtest test %s
 
 trait Add<A, B, R> {
   fn add(a: A, b: B) -> R;

tests/ui/syntax/too-long-integer.eight

@@ -1,4 +1,4 @@
-// RUN: not %eightc %s 2>&1 | %regtest %s.snap
+// RUN: not %eightc %s 2>&1 | %regtest test %s
 
 fn test() {
   let k = 1234773457276345671237572345;

tests/ui/syntax/unexpected-end-of-file.eight

@@ -1,3 +1,3 @@
-// RUN: not %eightc %s 2>&1 | %regtest %s.snap
+// RUN: not %eightc %s 2>&1 | %regtest test %s
 
 fn foo() -> *

tests/ui/syntax/unexpected-token.eight

@@ -1,3 +1,3 @@
-// RUN: not %eightc %s 2>&1 | %regtest %s.snap
+// RUN: not %eightc %s 2>&1 | %regtest test %s
 
 fn test() unexpected_token {

tests/ui/syntax/unterminated-token.eight

@@ -1,3 +1,3 @@
-// RUN: not %eightc %s 2>&1 | %regtest %s.snap
+// RUN: not %eightc %s 2>&1 | %regtest test %s
 
-|
+||

tests/ui/syntax/unterminated-token.eight.tmpsnap

@@ -0,0 +1,10 @@
+Error: syntax::unexpected_token
+
+  × found unexpected token during parsing
+   ╭─[unterminated-token.eight:3:1]
+ 2 │ 
+ 3 │ ||
+   · ─┬
+   ·  ╰── was not expecting to find '||' in this position
+   ╰────
+

tools/eight-regtest/Cargo.toml

@@ -3,9 +3,13 @@ name = "eight-regtest"
 version = "0.1.0"
 edition = "2021"
 
+[[bin]]
+name = "eight-regtest"
+path = "src/bin/regtest.rs"
+
 [dependencies]
 anyhow = "1.0.95"
 clap = { workspace = true }
+owo-colors = "4.1.0"
 similar = "2.7.0"
-termcolor = "1.4.1"
 thiserror = { workspace = true }

tools/eight-regtest/src/bin/regtest.rs

@@ -0,0 +1,111 @@
+use clap::{Parser, Subcommand};
+use eight_regtest::{
+    get_annotated_diff, get_regressed_snapshot_path, get_snapshot_state, get_unverified_snapshots,
+    get_updated_snapshot_path, get_verified_snapshot_path, SnapshotState,
+};
+use owo_colors::OwoColorize;
+use std::io::Read;
+
+#[derive(Parser)]
+struct Args {
+    #[command(subcommand)]
+    command: Command,
+}
+
+/// A subcommand selected for the regtest binary.
+#[derive(Subcommand)]
+enum Command {
+    /// Test stdin against the provided snapshot file.
+    Test(CommandTestArgs),
+    /// Interactively update each snapshot file in the provided directory, recursively.
+    Verify(CommandVerifyArgs),
+}
+
+#[derive(Parser)]
+struct CommandTestArgs {
+    snapshot: String,
+}
+
+#[derive(Parser)]
+struct CommandVerifyArgs {
+    directory: String,
+}
+
+fn main() -> anyhow::Result<()> {
+    let args = Args::parse();
+    match args.command {
+        Command::Test(args) => test(args),
+        Command::Verify(args) => verify(args),
+    }
+}
+
+/// Compare the file on stdin against the snapshot file.
+///
+/// If the snapshot file does not exist, it will be created, and marked as fresh. The program will
+/// exit with a non-zero exit code and print a diff.
+///
+/// It is now up to the user to verify the snapshot using `regtest verify`.
+fn test(args: CommandTestArgs) -> anyhow::Result<()> {
+    let snapshot_state = get_snapshot_state(&args.snapshot)?;
+    let snapshot = match &snapshot_state {
+        SnapshotState::Fresh | SnapshotState::Unverified(_) => "",
+        SnapshotState::Verified(snapshot) => snapshot,
+        SnapshotState::PreviouslyRegressed(regression, _) => regression,
+    };
+    let mut stdin = String::new();
+    std::io::stdin().lock().read_to_string(&mut stdin)?;
+    let (changed, diff) = get_annotated_diff(&stdin, snapshot);
+
+    // If there were no changes, and the snapshot file was previously verified, we can exit early.
+    if !changed && matches!(snapshot_state, SnapshotState::Verified(_)) {
+        std::process::exit(0);
+    }
+
+    // Otherwise, we let the user know that the snapshot file has changed.
+    match snapshot_state {
+        SnapshotState::Fresh => println!("{}\n", "A new snapshot has been created".cyan()),
+        SnapshotState::Unverified(_) => println!(
+            "{}\n",
+            "The snapshot has not been reviewed since last run".cyan()
+        ),
+        // The output has diverged, so we mark the file as regressed in addition to producing the
+        // new tmpsnap file.
+        SnapshotState::Verified(_) | SnapshotState::PreviouslyRegressed(_, _) => {
+            println!("{}\n", "The snapshot has regressed".cyan());
+        }
+    }
+
+    // This means the regression is fresh to this `regtest test` run, so we can write the regressed
+    // snapshot into the .regsnap file.
+    if matches!(snapshot_state, SnapshotState::Verified(_)) {
+        let verified_snapshot_path = get_verified_snapshot_path(&args.snapshot);
+        let regressed_snapshot_path = get_regressed_snapshot_path(&args.snapshot);
+        std::fs::rename(&verified_snapshot_path, &regressed_snapshot_path)?;
+    }
+
+    println!("{}", diff);
+    // Write back the updated snapshot into the .tmpsnap file
+    let updated_snapshot_path = get_updated_snapshot_path(&args.snapshot);
+    std::fs::write(&updated_snapshot_path, &stdin)?;
+    println!(
+        "{} {}\n",
+        "The snapshot has been written to".cyan(),
+        updated_snapshot_path.display()
+    );
+    std::process::exit(1);
+}
+
+/// Interactively verify each snapshot file in the provided directory.
+///
+/// This gives the user the choice between ACCEPT / IGNORE / REJECT each snapshot file. Based on the
+/// selection, the behavior is as follows:
+///
+/// 1. ACCEPT: The snapshot file is marked accepted
+/// 2. IGNORE: The snapshot file is left as-is with no modifications to its state
+/// 3. REJECT: The snapshot file is deleted. Consequently, future `regtest test` runs will end up
+///    creating the snapshot file again.
+fn verify(args: CommandVerifyArgs) -> anyhow::Result<()> {
+    let unverified_snapshots = get_unverified_snapshots(&args.directory)?;
+    dbg!(&unverified_snapshots);
+    Ok(())
+}

tools/eight-regtest/src/lib.rs

@@ -0,0 +1,178 @@
+//! Eight Regtest, a regression testing tool for the Eight compiler.
+//!
+//! Regtest takes two inputs, a truth file read from stdin, and a test file read from the first
+//! command line argument. It then compares the truth file with the snap file, and prints a diff
+//! if they differ.
+
+use owo_colors::OwoColorize;
+use similar::TextDiff;
+use std::io::Read;
+use std::path::{Path, PathBuf};
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+#[error("regtest error: {0}")]
+pub enum RegTestError {
+    #[error("io error: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("snapshot path is not a file: {0}")]
+    InvalidPath(PathBuf),
+    #[error("both fresh and updated snapshot files exist: {0} and {1}. delete one of them.")]
+    Conflict(PathBuf, PathBuf),
+}
+
+/// Enum describing the different states the snapshot file can be in.
+#[derive(Debug)]
+pub enum SnapshotState {
+    /// The snapshot file does not exist, and will be created this run.
+    ///
+    /// The creation of the file will happen regardless of whether `-u` is provided to the
+    /// command line. This is for convenience, as you'll likely want to generate the file if it
+    /// doesn't exist.
+    Fresh,
+    /// The file has seen some changes (i.e, has .tmpsnap extension)
+    Unverified(String),
+    /// The file has not seen any changes and was previously accepted. (i.e, has only .snap
+    /// extension)
+    Verified(String),
+    /// The file regressed due to some recent changes. It means that both .regsnap and .tmpsnap
+    /// exist.
+    ///
+    /// The tuple is (regsnap, tmpsnap) file contents respectively.
+    PreviouslyRegressed(String, String),
+}
+
+pub fn get_verified_snapshot_path<P: AsRef<Path>>(path: P) -> PathBuf {
+    let mut buf = path.as_ref().as_os_str().to_owned();
+    buf.push(".snap");
+    PathBuf::from(buf)
+}
+
+pub fn get_updated_snapshot_path<P: AsRef<Path>>(path: P) -> PathBuf {
+    let mut buf = path.as_ref().as_os_str().to_owned();
+    buf.push(".tmpsnap");
+    PathBuf::from(buf)
+}
+
+pub fn get_regressed_snapshot_path<P: AsRef<Path>>(path: P) -> PathBuf {
+    let mut buf = path.as_ref().as_os_str().to_owned();
+    buf.push(".regsnap");
+    PathBuf::from(buf)
+}
+
+/// Get the snapshot state for the given snapshot file.
+pub fn get_snapshot_state<P: AsRef<Path>>(path: P) -> Result<SnapshotState, RegTestError> {
+    let updated_snapshot_candidate_path = get_updated_snapshot_path(path.as_ref());
+    let updated_exists = std::fs::exists(&updated_snapshot_candidate_path)?;
+    let verified_snapshot_candidate_path = get_verified_snapshot_path(path.as_ref());
+    let verified_exists = std::fs::exists(&verified_snapshot_candidate_path)?;
+
+    match (updated_exists, verified_exists) {
+        // If both files exist, we have a conflict and should error out.
+        (true, true) => Err(RegTestError::Conflict(
+            updated_snapshot_candidate_path,
+            verified_snapshot_candidate_path,
+        )),
+        // An updated snapshot file exists, but the fresh snapshot file does not. This means that
+        // the snapshot should be verified by the user.
+        (true, false) => {
+            let metadata = std::fs::metadata(&updated_snapshot_candidate_path)?;
+            // If the inode at the path is not a file, then something is obstructing the inode path
+            // from being used properly, so this should be an error.
+            if !metadata.is_file() {
+                return Err(RegTestError::InvalidPath(updated_snapshot_candidate_path));
+            }
+            let mut snapshot = String::new();
+            std::fs::File::open(&updated_snapshot_candidate_path)?.read_to_string(&mut snapshot)?;
+            // If the file was a regression, we also read and return the regressed snapshot.
+            let regressed_snapshot_path = get_regressed_snapshot_path(path.as_ref());
+            if std::fs::exists(&regressed_snapshot_path)? {
+                let mut regressed_snapshot = String::new();
+                std::fs::File::open(&regressed_snapshot_path)?
+                    .read_to_string(&mut regressed_snapshot)?;
+                return Ok(SnapshotState::PreviouslyRegressed(
+                    regressed_snapshot,
+                    snapshot,
+                ));
+            }
+            Ok(SnapshotState::Unverified(snapshot))
+        }
+        // A verified snapshot file exists.
+        (false, true) => {
+            let metadata = std::fs::metadata(&verified_snapshot_candidate_path)?;
+            // If the inode at the path is not a file, then something is obstructing the inode path from
+            // being used properly, so this should be an error.
+            if !metadata.is_file() {
+                return Err(RegTestError::InvalidPath(verified_snapshot_candidate_path));
+            }
+            let mut snapshot = String::new();
+            std::fs::File::open(&verified_snapshot_candidate_path)?
+                .read_to_string(&mut snapshot)?;
+            Ok(SnapshotState::Verified(snapshot))
+        }
+        // Neither file exists, so we should create a fresh snapshot file.
+        (false, false) => Ok(SnapshotState::Fresh),
+    }
+}
+
+/// Get the annotated diff between the truth and snapshot.
+///
+/// Returns a tuple of (changed, diff) where changed is true if the snapshot file has changed.
+pub fn get_annotated_diff(truth: &str, snapshot: &str) -> (bool, String) {
+    let diff = TextDiff::from_lines(snapshot, truth);
+    let mut buf = String::new();
+    let mut changed = false;
+    for change in diff.iter_all_changes() {
+        // To avoid bleeding the color onto the next line, we write the newline after the change
+        // and reset have been applied. Not doing this bleeds the diff colors out into the `lit`
+        // output.
+        let c = change.to_string().replace('\n', "");
+        match change.tag() {
+            similar::ChangeTag::Equal => {
+                buf.push_str(&c);
+            }
+            similar::ChangeTag::Delete => {
+                changed = true;
+                buf.push_str(&format!("-{}", c.red()));
+            }
+            similar::ChangeTag::Insert => {
+                changed = true;
+                buf.push_str(&format!("+{}", c.green()));
+            }
+        };
+        buf.push('\n');
+    }
+
+    (changed, buf)
+}
+
+pub fn get_unverified_snapshots<P: AsRef<Path>>(path: P) -> Result<Vec<PathBuf>, RegTestError> {
+    let mut buf = Vec::new();
+    fn is_unverified_snapshot(path: &Path) -> bool {
+        path.is_file() && path.extension().is_some_and(|ext| ext == "tmpsnap")
+    }
+    /// Recursively visit the given path, and collect all unverified snapshots.
+    fn visit(path: &Path, buf: &mut Vec<PathBuf>) -> Result<(), RegTestError> {
+        let metadata = std::fs::metadata(path)?;
+        if metadata.is_dir() {
+            for entry in std::fs::read_dir(path)? {
+                let path = entry?.path();
+                if path.is_dir() {
+                    visit(&path, buf)?;
+                }
+                // If it's a file and it has the .tmpsnap extension, we count it as an unverified
+                // snapshot.
+                if is_unverified_snapshot(&path) {
+                    buf.push(path);
+                }
+            }
+        }
+        // Otherwise, if it's a file
+        if is_unverified_snapshot(path) {
+            buf.push(path.to_owned());
+        }
+        Ok(())
+    }
+    visit(path.as_ref(), &mut buf)?;
+    Ok(buf)
+}