boulder/analysis: Add compress man handler

compresses man/info pages

Author: joebonrichie

Cargo.lock

@@ -1,12 +1,6 @@
 [workspace]
-members = [
-    "boulder",
-    "moss",
-    "crates/*",
-]
-default-members = [
-    "moss"
-]
+members = ["boulder", "moss", "crates/*"]
+default-members = ["moss"]
 resolver = "2"
 
 [workspace.package]
@@ -23,25 +17,47 @@ criterion = { version = "0.5.1", features = ["html_reports"] }
 crossterm = "0.27.0"
 derive_more = "0.99.18"
 dialoguer = "0.11.0"
-diesel = { version = "2.2.1", features = ["sqlite", "returning_clauses_for_sqlite_3_35"] }
+diesel = { version = "2.2.1", features = [
+    "sqlite",
+    "returning_clauses_for_sqlite_3_35",
+] }
 diesel_migrations = "2.2.0"
 dirs = "5.0.1"
 elf = "0.7.4"
 indicatif = "0.17.8"
 itertools = "0.13.0"
 fs-err = { version = "2.11.0", features = ["tokio"] }
+filetime = "0.2.24"
 futures = "0.3.30"
 glob = "0.3.1"
 hex = "0.4.3"
 indextree = "4.6.1"
 libsqlite3-sys = { version = "0.28.0", features = ["bundled"] }
 log = "0.4.22"
 nom = "7.1.3"
-nix = { version = "0.27.1", features = ["user", "fs", "sched", "process", "mount", "hostname", "signal", "term"] }
+nix = { version = "0.27.1", features = [
+    "user",
+    "fs",
+    "sched",
+    "process",
+    "mount",
+    "hostname",
+    "signal",
+    "term",
+] }
 petgraph = "0.6.5"
 rayon = "1.10.0"
 regex = "1.10.5"
-reqwest = { version = "0.12.5", default-features = false, features = ["brotli", "charset", "deflate", "gzip", "http2", "rustls-tls", "stream", "zstd"] }
+reqwest = { version = "0.12.5", default-features = false, features = [
+    "brotli",
+    "charset",
+    "deflate",
+    "gzip",
+    "http2",
+    "rustls-tls",
+    "stream",
+    "zstd",
+] }
 serde = { version = "1.0.204", features = ["derive"] }
 serde_json = "1.0.120"
 serde_yaml = "0.9.34"

Cargo.toml

@@ -29,6 +29,7 @@ clap.workspace = true
 derive_more.workspace = true
 dirs.workspace = true
 elf.workspace = true
+filetime.workspace = true
 glob.workspace = true
 fs-err.workspace = true
 futures.workspace = true
@@ -45,3 +46,4 @@ thiserror.workspace = true
 tokio.workspace = true
 url.workspace = true
 mailparse.workspace = true
+zstd.workspace = true

boulder/Cargo.toml

@@ -39,6 +39,7 @@ impl<'a> Chain<'a> {
                 Box::new(handler::pkg_config),
                 Box::new(handler::python),
                 Box::new(handler::cmake),
+                Box::new(handler::compressman),
                 // Catch-all if not excluded
                 Box::new(handler::include_any),
             ],

boulder/src/package/analysis.rs

@@ -1,9 +1,16 @@
+use filetime::FileTime;
+use std::fs::File;
+use std::io::{BufReader, BufWriter, Write};
 use std::{
     path::PathBuf,
+    fs,
+    os::unix::fs::symlink,
+    path::{Component, PathBuf},
     process::{Command, Stdio},
 };
 
 use fs_err as fs;
+use itertools::Itertools;
 use moss::{dependency, Dependency, Provider};
 
 use crate::package::collect::PathInfo;
@@ -180,3 +187,88 @@ pub fn cmake(bucket: &mut BucketMut, info: &mut PathInfo) -> Result<Response, Bo
 
     Ok(Decision::NextHandler.into())
 }
+
+pub fn compressman(bucket: &mut BucketMut, info: &mut PathInfo) -> Result<Response, BoxError> {
+    if !bucket.recipe.parsed.options.compressman {
+        return Ok(Decision::NextHandler.into());
+    }
+
+    let is_man_file = info.path.components().contains(&Component::Normal("man".as_ref()))
+        && info.file_name().ends_with(|c| ('1'..'9').contains(&c));
+    let is_info_file =
+        info.path.components().contains(&Component::Normal("info".as_ref())) && info.file_name().ends_with(".info");
+
+    if !(is_man_file || is_info_file) {
+        return Ok(Decision::NextHandler.into());
+    }
+
+    let mut generated_path = PathBuf::new();
+
+    let metadata = fs::metadata(&info.path)?;
+    let atime = metadata.accessed()?;
+    let mtime = metadata.modified()?;
+
+    /* If we have a man/info symlink update the link to the compressed file */
+    if info.path.is_symlink() {
+        let new_original = format!("{}.zst", fs::canonicalize(&info.path)?.display());
+        let new_link = format!("{}.zst", &info.path.display());
+
+        /*
+         * Depending on the order the files get analysed the new compressed file may not yet exist,
+         * compress it _now_ so the correct metadata src info is returned to the binary writer.
+         */
+        if !std::path::Path::new(&new_original).exists() {
+            let compressed_file = compress_file_zstd(fs::canonicalize(&info.path)?)?;
+            let _ = bucket.paths.install().guest.join(compressed_file);
+        }
+
+        symlink(format!("{}.zst", fs::read_link(&info.path)?.display()), &new_link)?;
+
+        /* Restore the original {a,m}times for reproducibility */
+        filetime::set_symlink_file_times(
+            &new_link,
+            FileTime::from_system_time(atime),
+            FileTime::from_system_time(mtime),
+        )?;
+
+        generated_path.push(bucket.paths.install().guest.join(new_link));
+        return Ok(Decision::ReplaceFile {
+            newpath: generated_path,
+        }
+        .into());
+    }
+
+    let mut compressed_file = PathBuf::from(format!("{}.zst", info.path.display()));
+
+    /* We may have already compressed the file if we encountered a symlink to this file first */
+    if !&compressed_file.exists() {
+        compressed_file = compress_file_zstd(info.path.clone())?;
+    }
+
+    /* Restore the original {a,m}times for reproducibility */
+    filetime::set_file_handle_times(
+        &File::open(&compressed_file)?,
+        Some(FileTime::from_system_time(atime)),
+        Some(FileTime::from_system_time(mtime)),
+    )?;
+
+    generated_path.push(bucket.paths.install().guest.join(compressed_file));
+
+    pub fn compress_file_zstd(path: PathBuf) -> Result<PathBuf, BoxError> {
+        let output_path = PathBuf::from(format!("{}.zst", path.display()));
+        let input = File::create(&output_path)?;
+        let mut reader = BufReader::new(File::open(&path)?);
+        let mut writer = BufWriter::new(input);
+
+        zstd::stream::copy_encode(&mut reader, &mut writer, 16)?;
+
+        writer.flush()?;
+
+        Ok(output_path)
+    }
+
+    Ok(Decision::ReplaceFile {
+        newpath: generated_path,
+    }
+    .into())
+}

boulder/src/package/analysis/handler.rs

@@ -93,6 +93,8 @@ pub struct Options {
     pub strip: bool,
     #[serde(default, deserialize_with = "stringy_bool")]
     pub networking: bool,
+    #[serde(default, deserialize_with = "stringy_bool")]
+    pub compressman: bool,
 }
 
 #[derive(Debug, Clone, Deserialize)]