Control Minio in unit test code

Adds helpers for running containers via Podman.

Moves controlling starting/stopping Minio for unit tests into the
relevant unit test modules.

Corrects some CI issues (i.e. clippy not being able to fail builds).

.github/workflows/ci.yml

@@ -31,21 +31,10 @@ jobs:
         run: nix develop --command cargo-deny check
 
       - name: Clippy
-        run: nix develop --command cargo clippy --all-targets
-
-      - name: Start MinIO
-        run: |
-          podman run \
-            -d \
-            -p 9000:9000 \
-            -e "MINIO_ACCESS_KEY=minioadmin" \
-            -e "MINIO_SECRET_KEY=minioadmin" \
-            minio/minio \
-            server \
-            /data
+        run: nix develop --command cargo clippy --all-targets -- -D warnings
 
       - name: Test
-        run: nix build -L --no-sandbox .#test
+        run: nix develop --command cargo test
 
   build:
     name: Build

Cargo.toml

@@ -4,6 +4,8 @@ members = [
   "common",
   "storage",
 
+  "testing-utils",
+
   "agent",
   "archiver",
   "ctl",

flake.nix

@@ -72,22 +72,7 @@
         };
 
         packages =
-          {
-            test = naersk'.buildPackage {
-              mode = "test";
-              src = ./.;
-
-              nativeBuildInputs = nativeBuildInputs;
-              buildInputs = buildInputs;
-
-              # Ensure detailed test output appears in nix build log
-              cargoTestOptions = x: x ++ ["1>&2"];
-
-              AWS_ACCESS_KEY_ID = "minioadmin";
-              AWS_SECRET_ACCESS_KEY = "minioadmin";
-            };
-          }
-          // import ./agent {inherit pkgs naersk' version gitRevision nativeBuildInputs buildInputs;}
+          import ./agent {inherit pkgs naersk' version gitRevision nativeBuildInputs buildInputs;}
           // import ./archiver {inherit pkgs naersk' version gitRevision nativeBuildInputs buildInputs;}
           // import ./ctl {inherit pkgs naersk' version gitRevision nativeBuildInputs buildInputs;}
           // import ./event-processor {inherit pkgs naersk' version gitRevision nativeBuildInputs buildInputs;};

storage/Cargo.toml

@@ -24,6 +24,9 @@ toml.workspace = true
 tracing = "0.1"
 
 [dev-dependencies]
+ctor = "0.2.5"
 hex = "0.4.3"
+lazy_static = "1.4.0"
 rand = "0.8.5"
+satori-testing-utils = { path = "../testing-utils" }
 tempfile = "3.4.0"

storage/src/providers/s3_object.rs

@@ -250,6 +250,36 @@ mod test {
     use super::*;
     use rand::Rng;
     use s3::BucketConfiguration;
+    use satori_testing_utils::MinioDriver;
+    use std::sync::{Arc, Mutex};
+
+    lazy_static::lazy_static! {
+        static ref MINIO: Arc<Mutex<Option<MinioDriver>>> = Arc::new(Mutex::new(None));
+    }
+
+    #[ctor::ctor]
+    fn init_minio() {
+        MINIO.lock().unwrap().replace(MinioDriver::default());
+
+        std::env::set_var("AWS_ACCESS_KEY_ID", "minioadmin");
+        std::env::set_var("AWS_SECRET_ACCESS_KEY", "minioadmin");
+    }
+
+    #[ctor::dtor]
+    fn shutdown_minio() {
+        MINIO.lock().unwrap().as_ref().unwrap().stop();
+    }
+
+    fn generate_random_bucket_name() -> String {
+        let id = rand::thread_rng()
+            .sample_iter(&rand::distributions::Alphanumeric)
+            .take(8)
+            .map(char::from)
+            .collect::<String>()
+            .to_lowercase();
+
+        format!("satori-storage-test-{id}")
+    }
 
     mod no_encryption {
         use super::*;
@@ -258,19 +288,15 @@ mod test {
             ( $test:ident ) => {
                 #[tokio::test]
                 async fn $test() {
-                    let id = rand::thread_rng()
-                        .sample_iter(&rand::distributions::Alphanumeric)
-                        .take(8)
-                        .map(char::from)
-                        .collect::<String>()
-                        .to_lowercase();
-                    let bucket_name = format!("satori-storage-test-{id}");
+                    let endpoint = MINIO.lock().unwrap().as_ref().unwrap().endpoint();
+
+                    let bucket = super::generate_random_bucket_name();
 
                     Bucket::create_with_path_style(
-                        &bucket_name,
+                        &bucket,
                         Region::Custom {
                             region: "".into(),
-                            endpoint: "http://localhost:9000".into(),
+                            endpoint: endpoint.clone(),
                         },
                         Credentials::default().unwrap(),
                         BucketConfiguration::default(),
@@ -279,9 +305,9 @@ mod test {
                     .unwrap();
 
                     let provider = crate::StorageConfig::S3(S3Config {
-                        bucket: bucket_name,
+                        bucket,
                         region: "".into(),
-                        endpoint: "http://localhost:9000".into(),
+                        endpoint,
                         encryption: EncryptionConfig::default(),
                     })
                     .create_provider();
@@ -301,19 +327,15 @@ mod test {
             ( $test:ident ) => {
                 #[tokio::test]
                 async fn $test() {
-                    let id = rand::thread_rng()
-                        .sample_iter(&rand::distributions::Alphanumeric)
-                        .take(8)
-                        .map(char::from)
-                        .collect::<String>()
-                        .to_lowercase();
-                    let bucket_name = format!("satori-storage-test-{id}");
+                    let endpoint = MINIO.lock().unwrap().as_ref().unwrap().endpoint();
+
+                    let bucket = super::generate_random_bucket_name();
 
                     Bucket::create_with_path_style(
-                        &bucket_name,
+                        &bucket,
                         Region::Custom {
                             region: "".into(),
-                            endpoint: "http://localhost:9000".into(),
+                            endpoint: endpoint.clone(),
                         },
                         Credentials::default().unwrap(),
                         BucketConfiguration::default(),
@@ -322,9 +344,9 @@ mod test {
                     .unwrap();
 
                     let provider = crate::StorageConfig::S3(S3Config {
-                        bucket: bucket_name,
+                        bucket,
                         region: "".into(),
-                        endpoint: "http://localhost:9000".into(),
+                        endpoint,
                         encryption: toml::from_str(
                             "
 [event]

testing-utils/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "satori-testing-utils"
+license.workspace = true
+version.workspace = true
+edition.workspace = true
+
+[dependencies]
+rand = "0.8.5"

testing-utils/src/lib.rs

@@ -0,0 +1,4 @@
+mod minio;
+mod podman;
+
+pub use self::{minio::MinioDriver, podman::PodmanDriver};

testing-utils/src/minio.rs

@@ -0,0 +1,33 @@
+use crate::PodmanDriver;
+
+pub struct MinioDriver {
+    pub podman: PodmanDriver,
+    pub endpoint: String,
+}
+
+impl Default for MinioDriver {
+    fn default() -> Self {
+        let port = rand::random::<u16>() % 1000 + 9000;
+
+        let podman = PodmanDriver::new(
+            "docker.io/minio/minio",
+            &[&format!("{port}:9000")],
+            &["MINIO_ACCESS_KEY=minioadmin", "MINIO_SECRET_KEY=minioadmin"],
+            &["server", "/data"],
+        );
+
+        let endpoint = format!("http://localhost:{}", port);
+
+        Self { podman, endpoint }
+    }
+}
+
+impl MinioDriver {
+    pub fn stop(&self) {
+        self.podman.stop();
+    }
+
+    pub fn endpoint(&self) -> String {
+        self.endpoint.clone()
+    }
+}

testing-utils/src/podman.rs

@@ -0,0 +1,60 @@
+use std::process::Command;
+
+pub struct PodmanDriver {
+    container_id: String,
+}
+
+impl PodmanDriver {
+    pub fn new(image: &str, ports: &[&str], env_vars: &[&str], args: &[&str]) -> Self {
+        // Build Podman args
+        let mut podman_args = vec!["run".to_string(), "--detach".to_string()];
+
+        for port in ports.iter() {
+            podman_args.push("-p".to_string());
+            podman_args.push(port.to_string());
+        }
+
+        for var in env_vars.iter() {
+            podman_args.push("-e".to_string());
+            podman_args.push(var.to_string());
+        }
+
+        podman_args.push(image.to_string());
+
+        for arg in args.iter() {
+            podman_args.push(arg.to_string());
+        }
+
+        // Start the container
+        let container_start = Command::new("podman").args(podman_args).output().unwrap();
+        println!("Container start: {:?}", container_start);
+
+        if container_start.status.success() && container_start.status.code().unwrap() == 0 {
+            let container_id = String::from_utf8(container_start.stdout)
+                .unwrap()
+                .trim()
+                .to_string();
+            println!("Container ID: {container_id}");
+
+            Self { container_id }
+        } else {
+            panic!("Failed to start container");
+        }
+    }
+
+    pub fn stop(&self) {
+        // Stop the container
+        let container_stop = Command::new("podman")
+            .args(vec!["stop", &self.container_id])
+            .output()
+            .unwrap();
+        println!("Container stop: {:?}", container_stop);
+
+        // Remove the container
+        let container_remove = Command::new("podman")
+            .args(vec!["rm", &self.container_id])
+            .output()
+            .unwrap();
+        println!("Container remove: {:?}", container_remove);
+    }
+}