Merge pull request #5 from RemiKalbe/exit-on-non-utf8-files

Exit on non utf8 files

Author: RemiKalbe

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.3.0] - 2024-11-30
+
+### Added
+
+- Non UTF-8 files are ignored by default, can be disabled with the `--exit-on-non-utf8` flag. (fix [#3](https://github.com/RemiKalbe/cunw/issues/3))
+
 ## [0.2.3] - 2024-09-19
 
 ### Fixed
@@ -88,6 +94,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Minor improvements to error handling and logging
 
-[Unreleased]: https://github.com/RemiKalbe/cunw/compare/v0.2.0...HEAD
+[Unreleased]: https://github.com/RemiKalbe/cunw/compare/v0.3.0...HEAD
+[0.3.0]: https://github.com/RemiKalbe/cunw/releases/tag/v0.3.0
+[0.2.3]: https://github.com/RemiKalbe/cunw/releases/tag/v0.2.3
+[0.2.2]: https://github.com/RemiKalbe/cunw/releases/tag/v0.2.2
+[0.2.1]: https://github.com/RemiKalbe/cunw/releases/tag/v0.2.1
 [0.2.0]: https://github.com/RemiKalbe/cunw/releases/tag/v0.2.0
 [0.1.0]: https://github.com/RemiKalbe/cunw/releases/tag/v0.1.0

README.md

@@ -1,7 +1,7 @@
 # cunw
 
-![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/RemiKalbe/cunw/.github%2Fworkflows%2Ftests.yaml)
-![Crates.io Version](https://img.shields.io/crates/v/cunw)
+![GitHub Actions Workflow Status](https://github.com/RemiKalbe/cunw/actions)
+![Crates.io Version](https://crates.io/crates/cunw)
 
 cunw (codebase unwrap) is a command-line interface (CLI) tool that generates a structured representation of a codebase, making it easy to provide context to a large language model (LLM). It recursively traverses a directory, collects file content, and generates a single file that represents the structure and content of the codebase.
 

src/args.rs

@@ -13,6 +13,13 @@ pub struct Args {
     pub output: Option<PathBuf>,
     #[arg(short, long, help = "Exclude files or directories matching the specified pattern.", value_hint = ValueHint::Other, required = false, num_args = 0.., action = ArgAction::Append)]
     pub exclude: Option<Vec<Glob>>,
+    #[arg(
+        long,
+        help = "Exit on non-UTF-8 content.",
+        required = false,
+        default_value = "false"
+    )]
+    pub exit_on_non_utf8: bool,
     #[arg(
         long,
         help = "Do not consider the ignore files (.gitignore, .hgignore, .ignore, .git/info/exclude and core.excludesFile in .git/config).",
@@ -52,6 +59,7 @@ mod tests {
         assert_eq!(args.path.to_str().unwrap(), "/path/to/codebase");
         assert_eq!(args.output, Some(std::path::PathBuf::from("output.txt")));
         assert_eq!(args.exclude, None);
+        assert_eq!(args.exit_on_non_utf8, false);
         assert_eq!(args.do_not_consider_ignore_files, false);
         assert_eq!(args.dangerously_allow_dot_git_traversal, false);
         assert_eq!(args.max_depth, None);
@@ -67,6 +75,7 @@ mod tests {
             "custom_output.md",
             "-e",
             "*.txt",
+            "--exit-on-non-utf8",
             "--do-not-consider-ignore-files",
             "--dangerously-allow-dot-git-traversal",
             "-m",
@@ -80,6 +89,7 @@ mod tests {
             Some(std::path::PathBuf::from("custom_output.md"))
         );
         assert_eq!(args.exclude.unwrap()[0].glob(), "*.txt");
+        assert_eq!(args.exit_on_non_utf8, true);
         assert_eq!(args.do_not_consider_ignore_files, true);
         assert_eq!(args.dangerously_allow_dot_git_traversal, true);
         assert_eq!(args.max_depth, Some(3));

src/codebase/mod.rs

@@ -16,6 +16,7 @@ pub mod item;
 
 pub struct CodebaseBuilder {
     excluded_paths: Option<GlobSet>,
+    exit_on_non_utf8: Option<bool>,
     consider_gitignores: Option<bool>,
     max_depth: Option<usize>,
     follow_symlinks: Option<bool>,
@@ -26,6 +27,7 @@ impl CodebaseBuilder {
     pub fn new() -> Self {
         Self {
             excluded_paths: None,
+            exit_on_non_utf8: None,
             consider_gitignores: None,
             max_depth: None,
             follow_symlinks: None,
@@ -38,6 +40,11 @@ impl CodebaseBuilder {
         self
     }
 
+    pub fn exit_on_non_utf8(mut self, exit_on_non_utf8: bool) -> Self {
+        self.exit_on_non_utf8 = Some(exit_on_non_utf8);
+        self
+    }
+
     pub fn consider_gitignores(mut self, consider_gitignores: bool) -> Self {
         self.consider_gitignores = Some(consider_gitignores);
         self
@@ -205,8 +212,17 @@ impl CodebaseBuilder {
 
         // Wait for all files to be read
         let mut any_error = false;
+        let mut non_utf8_errors = Vec::new();
         while let Some(res) = files_handles.next().await {
             if let Err(err) = res.expect("Failed to await file content") {
+                if !self.exit_on_non_utf8.unwrap_or(false) {
+                    if let CunwErrorKind::Io(io_err) = &err.source {
+                        if io_err.kind() == std::io::ErrorKind::InvalidData {
+                            non_utf8_errors.push(err);
+                            continue;
+                        }
+                    }
+                }
                 Logger::warn(format!("Error while reading file: {:#?}", err).as_str());
                 any_error = true;
             }
@@ -216,6 +232,19 @@ impl CodebaseBuilder {
                 "Failed to read file(s) content(s)".to_string(),
             )));
         }
+        if !non_utf8_errors.is_empty() {
+            Logger::warn(
+                "Some files were ignored because they were not UTF-8 encoded and could not be read.",
+            );
+            for err in non_utf8_errors {
+                if let Some(file) = err.related_to_file {
+                    Logger::warn(format!("  - {}", file.display()).as_str());
+                }
+            }
+            Logger::warn(
+                "If you want to exit on non-UTF-8 files, use the --exit-on-non-utf8 flag.",
+            );
+        }
 
         Ok(Codebase { tree: root_tree })
     }