Initial POC

Author: dkorunic

.github/workflows/release.yml

@@ -0,0 +1,112 @@
+name: release
+on:
+  push:
+    tags:
+    - "[0-9]+.[0-9]+.[0-9]+"
+jobs:
+  create-release:
+    name: create-release
+    runs-on: ubuntu-latest
+    outputs:
+      upload_url: ${{ steps.release.outputs.upload_url }}
+      fl_version: ${{ env.FL_VERSION }}
+    steps:
+      - name: Get the release version from the tag
+        shell: bash
+        if: env.FL_VERSION == ''
+        run: |
+          echo "FL_VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
+          echo "version is: ${{ env.FL_VERSION }}"
+      - name: Create GitHub release
+        id: release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ env.FL_VERSION }}
+          release_name: ${{ env.FL_VERSION }}
+
+  build-release:
+    name: build-release
+    needs: ['create-release']
+    runs-on: ${{ matrix.os }}
+    env:
+      CARGO: cargo
+      TARGET_FLAGS: ""
+      TARGET_DIR: ./target
+      RUST_BACKTRACE: 1
+    strategy:
+      matrix:
+        build: [linux, linux-arm, macos]
+        include:
+        - build: linux
+          os: ubuntu-latest
+          rust: stable
+          target: x86_64-unknown-linux-musl
+        - build: linux-arm
+          os: ubuntu-latest
+          rust: stable
+          target: arm-unknown-linux-gnueabihf
+        - build: macos
+          os: macos-latest
+          rust: stable
+          target: x86_64-apple-darwin
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
+
+    - name: Install Rust
+      uses: dtolnay/rust-toolchain@v1
+      with:
+        toolchain: ${{ matrix.rust }}
+        target: ${{ matrix.target }}
+
+    - name: Use Cross
+      shell: bash
+      run: |
+        cargo install cross
+        echo "CARGO=cross" >> $GITHUB_ENV
+        echo "TARGET_FLAGS=--target ${{ matrix.target }}" >> $GITHUB_ENV
+        echo "TARGET_DIR=./target/${{ matrix.target }}" >> $GITHUB_ENV
+
+    - name: Show command used for Cargo
+      run: |
+        echo "cargo command is: ${{ env.CARGO }}"
+        echo "target flag is: ${{ env.TARGET_FLAGS }}"
+        echo "target dir is: ${{ env.TARGET_DIR }}"
+
+    - name: Build release binary
+      run: ${{ env.CARGO }} build --verbose --release ${{ env.TARGET_FLAGS }}
+
+    - name: Strip release binary (linux and macos)
+      if: matrix.build == 'linux' || matrix.build == 'macos'
+      run: strip "target/${{ matrix.target }}/release/findlargedir"
+
+    - name: Strip release binary (arm)
+      if: matrix.build == 'linux-arm'
+      run: |
+        docker run --rm -v \
+          "$PWD/target:/target:Z" \
+          rustembedded/cross:arm-unknown-linux-gnueabihf \
+          arm-linux-gnueabihf-strip \
+          /target/arm-unknown-linux-gnueabihf/release/findlargedir
+
+    - name: Build archive
+      shell: bash
+      run: |
+        outdir="$(ci/cargo-out-dir "${{ env.TARGET_DIR }}")"
+        staging="findlargedir-${{ needs.create-release.outputs.fl_version }}-${{ matrix.target }}"
+        cp {README.md,LICENSE} "$staging/"
+
+    - name: Upload release archive
+      uses: actions/[email protected]
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        upload_url: ${{ needs.create-release.outputs.upload_url }}
+        asset_path: ${{ env.ASSET }}
+        asset_name: ${{ env.ASSET }}
+        asset_content_type: application/octet-stream

.gitignore

@@ -0,0 +1,20 @@
+### Rust ###
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+# IntelliJ tools
+.idea/
+
+# End of https://www.toptal.com/developers/gitignore/api/rust

Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "findlargedir"
+version = "0.1.0"
+authors = ["Dinko Korunic <[email protected]>"]
+categories = ["command-line-utilities"]
+description = "find all blackhole directories with a huge amount of filesystem entries in a flat structure"
+repository = "https://github.com/dkorunic/findlargedir"
+readme = "README.md"
+license = "MIT"
+exclude = [".gitignore"]
+edition = "2021"
+
+[dependencies]
+jwalk = "0.6.0"
+rayon = "1.5.3"
+tikv-jemallocator = "0.5.0"
+num_cpus = "1.13.1"
+tempfile = "3.3.0"
+spinach = "2.1.0"
+anyhow = "1.0.65"
+human_format = "1.0.3"
+human_bytes = { version = "0.3.1", features = ["fast"] }
+clap = { version = "3.2.22", features = ["derive"] }
+ctrlc = { version = "3.2.3", features = ["termination"] }
+rm_rf = "0.6.2"
+ansi_term = "0.12.1"
+
+[profile.release]
+lto = true

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Dinko Korunic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,40 @@
+findlargedir
+===
+
+[![GitHub license](https://img.shields.io/github/license/dkorunic/findlargedir.svg)](https://github.com/dkorunic/findlargedir/blob/master/LICENSE.txt)
+[![GitHub release](https://img.shields.io/github/release/dkorunic/findlargedir.svg)](https://github.com/dkorunic/findlargedir/releases/latest)
+
+## About
+
+Findlargedir is a quick hack intended to help identifying "black hole" directories on an any filesystem having more than 100,000 entries in a single flat structure. Program will attempt to identify any number of such events and report on them.
+
+Program will **not follow symlinks** and **requires r/w permissions** to be able to calculate a directory inode size to number of entries ratio and estimate a number of entries in a directory without actually counting them. While this method is just an approximation of the actual number of entries in a directory, it is good enough to quickly scan for offending directories.
+
+## Caveats
+
+* requires r/w privileges for an each filesystem being tested, it will also create a temporary directory with a lot of temporary files which are cleaned up afterwards
+* accurate mode (`-a`) can cause an excessive I/O and an excessive memory use; only use when appropriate
+
+
+## Usage
+
+```shell
+USAGE:
+    findlargedir [OPTIONS] <PATH>...
+
+ARGS:
+    <PATH>...
+
+OPTIONS:
+    -a, --accurate <accurate>                          [default: false]
+    -A, --alert-threshold <ALERT_THRESHOLD>            [default: 10000]
+    -B, --blacklist-threshold <BLACKLIST_THRESHOLD>    [default: 100000]
+    -c, --calibration-count <CALIBRATION_COUNT>        [default: 10000]
+    -h, --help                                         Print help information
+    -o, --one-filesystem <one-filesystem>              [default: true]
+    -V, --version                                      Print version information
+```
+
+When using **accurate mode** (`-a` parameter) beware that large directory lookups will stall the process completely for extended periods of time. What this mode does is basically a secondary fully accurate pass on a possibly offending directory calculating exact number of entries.
+
+If you want to avoid descending into mounted filesystems (as in find -xdev option), use **onefilesystem mode** with `-o` parameter and this toggled by default.

src/calibrate.rs

@@ -0,0 +1,55 @@
+use anyhow::{Context, Error};
+use rm_rf::ensure_removed;
+use spinach::Spinach;
+use std::fs;
+use std::fs::File;
+use std::os::unix::fs::MetadataExt;
+use std::path::Path;
+use std::process;
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+pub const DEFAULT_TEST_COUNT: u64 = 10000;
+const ERROR_EXIT: i32 = 1;
+
+pub fn get_inode_ratio(
+    test_path: &Path,
+    shutdown: &Arc<AtomicBool>,
+    test_count: u64,
+) -> Result<u64, Error> {
+    println!(
+        "Running test directory calibration in: {}",
+        test_path.display(),
+    );
+
+    let s = Spinach::new("Starting calibration...");
+
+    for i in 0..test_count {
+        if shutdown.load(Ordering::Relaxed) {
+            s.stop();
+            println!("Requested program exit, stopping and deleting temporary files...",);
+            ensure_removed(test_path)
+                .expect("Unable to completely delete calibration directory, exiting");
+            process::exit(ERROR_EXIT);
+        }
+
+        File::create(test_path.join(i.to_string()))
+            .with_context(|| format!("Unable to create calibration test file {}", i))?;
+        if i % 1000 == 0 {
+            s.text(format!("Created {} files...", i));
+        }
+    }
+
+    s.text("Done, getting total size and deleting temp folder");
+
+    let tmp_dir_size = fs::metadata(test_path)
+        .with_context(|| format!("Unable to stat {} directory", test_path.display()))?
+        .size();
+
+    s.succeed("Finished with calibration.");
+
+    let size_inode_ratio = tmp_dir_size / test_count;
+    println!("Calculated size-to-inode ratio: {}", size_inode_ratio);
+
+    Ok(size_inode_ratio)
+}

src/interrupt.rs

@@ -0,0 +1,12 @@
+use anyhow::{Context, Error};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+
+pub fn setup_interrupt_handler(shutdown: Arc<AtomicBool>) -> Result<(), Error> {
+    ctrlc::set_handler(move || {
+        shutdown.store(true, Ordering::Relaxed);
+    })
+    .context("Unable to set Ctrl-C handler")?;
+
+    Ok(())
+}

src/main.rs

@@ -0,0 +1,83 @@
+#![warn(clippy::all, clippy::pedantic)]
+mod calibrate;
+mod interrupt;
+mod walk;
+
+use anyhow::{Context, Error, Result};
+use clap::Parser;
+use std::collections::HashSet;
+use std::fs;
+use std::sync::atomic::AtomicBool;
+use std::sync::Arc;
+use tempfile::TempDir;
+use tikv_jemallocator::Jemalloc;
+
+#[global_allocator]
+static GLOBAL: Jemalloc = Jemalloc;
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+    #[clap(short, long, default_value_t = calibrate::DEFAULT_TEST_COUNT)]
+    calibration_count: u64,
+
+    #[clap(short, long, default_value_t = false)]
+    accurate: bool,
+
+    #[clap(short, long, default_value_t = true)]
+    one_filesystem: bool,
+
+    #[clap(short = 'A', long, default_value_t = walk::ALERT_COUNT)]
+    alert_threshold: u64,
+
+    #[clap(short = 'B', long, default_value_t = walk::BLACKLIST_COUNT)]
+    blacklist_threshold: u64,
+
+    #[clap(required = true, allow_hyphen_values = true)]
+    path: Vec<String>,
+}
+
+fn main() -> Result<(), Error> {
+    let args = Args::parse();
+
+    let shutdown = Arc::new(AtomicBool::new(false));
+    let shutdown_calibrate = shutdown.clone();
+    interrupt::setup_interrupt_handler(shutdown)?;
+
+    let mut visited_paths = HashSet::new();
+
+    for path in args.path {
+        match visited_paths.get(&path) {
+            None => visited_paths.insert(path.clone()),
+            _ => continue,
+        };
+
+        let path_metadata =
+            fs::metadata(&path).with_context(|| format!("Unable to stat {} directory", &path))?;
+
+        let tmp_dir = Arc::new(
+            TempDir::new_in(&path).context("Unable to setup/create calibration test directory")?,
+        );
+
+        let size_inode_ratio =
+            calibrate::get_inode_ratio(tmp_dir.path(), &shutdown_calibrate, args.calibration_count)
+                .context("Unable to calibrate inode to size ratio")?;
+        drop(tmp_dir);
+
+        println!("Scanning filesystem path {} started", &path);
+
+        walk::parallel_search(
+            &path,
+            path_metadata,
+            size_inode_ratio,
+            args.accurate,
+            args.one_filesystem,
+            args.alert_threshold,
+            args.blacklist_threshold,
+        );
+
+        println!("Scanning filesystem path {} completed", &path);
+    }
+
+    Ok(())
+}