[scorpio]: Fix bug in overlayfs & init LFS code

scorpio/Cargo.toml

@@ -31,11 +31,19 @@ futures = "0.3.31"
 fuse3 = { version = "0.8.1" ,features = ["tokio-runtime","unprivileged"]}
 vmm-sys-util = { version = "0.11" }
 futures-util = { version = "0.3.30", features = ["sink"] }
+quote = "1.0.38"
+proc-macro2 = "1.0.93"
+syn = { version = "2.0.98", features = ["full", "extra-traits"] }
 
-[features]
+logging_macros = { path = "src/passthrough/logging_macros" }
+uuid = "1.14.0"
+
+
+[features] 
 async-io = []
 
 [workspace]
+members = ["src/passthrough/logging_macros"]
 
 [package.metadata.docs.rs]
 all-features = true

scorpio/README.md

@@ -56,7 +56,7 @@ The following interfaces are currently available:
 curl -X POST http://localhost:2725/api/fs/mount      -H "Content-Type: application/json"      -d '{"path": "third-part/mega/scorpio"}'
 curl -X GET http://localhost:2725/api/fs/mpoint
 curl -X POST http://localhost:2725/api/fs/umount      -H "Content-Type: application/json"      -d '{"path": "third-part/mega/scorpio"}'
-
+curl -X POST http://localhost:2725/api/fs/mount      -H "Content-Type: application/json"      -d '{"path": "third-part/mega/ts"}'
 ```
 ### How to Contribute?
 

scorpio/config.toml

@@ -3,4 +3,8 @@ workspace = "/home/luxian/megadir/mount"
 store_path = "/home/luxian/megadir/store"
 git_author = "MEGA"
 git_email = "[email protected]"
-works = []
+
+[[works]]
+path = "third-part/buck-hello"
+node = 8
+hash = "5f70f0f460d92d73eac0ea09ef2edb3840bfc68f"

scorpio/script/.gitignore

@@ -0,0 +1 @@
+output.txt

scorpio/script/log_analysis.py

@@ -0,0 +1,67 @@
+
+# Scorpio Log Analysis Tool, used for extracting abnormal requests (function not returning)
+import re
+from collections import defaultdict
+
+def extract_unique_id_lines(input_path, output_path):
+    """Extract lines with unique IDs and save them to a file"""
+
+    # Compile optimized regular expression (note the comments in re.VERBOSE mode)
+    uuid_regex = re.compile(
+        r'''^ID:                         # Fixed starting identifier
+        \{?                            # Optional left curly brace
+        (                              # Start capturing group
+          [0-9a-fA-F]{8}               # 8 hexadecimal digits
+          -                             # Separator
+          (?:[0-9a-fA-F]{4}-){3}       # Three middle groups (non-capturing group for performance)
+          [0-9a-fA-F]{12}              # Last 12 digits
+        )                              # End capturing group
+        \}?                            # Optional right curly brace
+        (?!\S)                         # Ensure ID is followed by a space or end of line
+        ''', 
+        re.IGNORECASE | re.VERBOSE
+    )
+    uuid_regex = re.compile(
+        r'^ID:\{?([0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12})\}?\b',
+        re.IGNORECASE
+    )
+
+    id_counter = defaultdict(int)
+    logs = defaultdict(int)
+    # First pass: Count occurrences of each ID
+    with open(input_path, 'r', encoding='utf-8') as f:
+        for line_num, line in enumerate(f, 1):
+            # Clean up line content (ignore comments and whitespace)
+            line = line.split('#')[0].strip()
+            if match := uuid_regex.search(line):
+                # Extract the standardized ID (convert to lowercase + remove braces)
+                raw_uuid = match.group(1).strip('{}')
+                standard_uuid = raw_uuid.lower()
+                logs[standard_uuid] = line
+                id_counter[standard_uuid] += 1
+
+    # Second pass: Record line numbers for unique IDs
+    unique_lines = []
+    with open(input_path, 'r', encoding='utf-8') as f:
+        for line_num, line in enumerate(f, 1):
+            line = line.split('#')[0].strip()
+            if match := uuid_regex.search(line):
+                raw_uuid = match.group(1).strip('{}')
+                standard_uuid = raw_uuid.lower()
+                if id_counter[standard_uuid] == 1:
+                    unique_lines.append(logs[standard_uuid])
+
+    # Write the results to a file (one line number per line)
+    with open(output_path, 'w', encoding='utf-8') as f:
+        f.write('\n'.join(map(str, unique_lines)))
+
+
+# For example, run this script with: python log_analysis.py input.log output.txt
+if __name__ == "__main__":
+    import sys
+    if len(sys.argv) != 2:
+        print(f"Usage: {sys.argv[0]} input.log")
+        sys.exit(1)
+
+    extract_unique_id_lines(sys.argv[1], "output.txt")
+    print(f"Unique ID line numbers have been saved")

scorpio/src/daemon/lfs.rs

@@ -0,0 +1,163 @@
+use axum::{
+    routing::{get, post, delete},
+    Router,
+    extract::{Query, Path},
+    Json,
+};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+pub fn create_app() -> Router {
+    Router::new()
+        .nest("/lfs", Router::new()
+            // Track LFS paths (equivalent to the track command)
+            .route("/attributes/track", post(track_lfs_path))
+            // Untrack paths (equivalent to the untrack command)
+            .route("/attributes/untrack", post(untrack_lfs_path))
+            // List locked files in the current branch (equivalent to lfs locks)
+            .route("/locks", get(list_locks))
+            // Lock a file (equivalent to lfs lock) 
+            .route("/locks/:path", post(create_lock))
+            // Unlock a file (equivalent to lfs unlock)
+            .route("/locks/:path", delete(remove_lock))
+            // Display LFS file information (equivalent to lfs ls - files)  
+            .route("/objects/metadata", get(list_lfs_files))
+        )
+}
+
+// Region 1: Attribute management endpoints ===============================================
+
+#[derive(Debug, Deserialize)]
+struct TrackPathsRequest {
+    patterns: Vec<String>,
+}
+
+async fn track_lfs_path(
+    Json(payload): Json<TrackPathsRequest>
+) -> Result<Json<HashMap<String, String>>, AppError> {
+    // Business logic:
+    // 1. Update the.gitattributes file
+    // 2. Return something like {"status": "tracked", "added_paths": [...]}
+    Ok(Json(HashMap::from([
+        ("status".to_string(), "success".to_string()),
+        ("added_paths".to_string(), payload.patterns.join(","))
+    ])))    
+}
+
+#[derive(Debug, Deserialize)]
+struct UntrackPathsRequest {
+    paths: Vec<String>,
+}
+
+async fn untrack_lfs_path(
+    Json(payload): Json<UntrackPathsRequest>
+) -> Result<Json<HashMap<String, String>>, AppError> {
+    // Business logic: Remove paths from.gitattributes
+    Ok(Json(HashMap::from([
+        ("status".to_string(), "success".to_string()),
+        ("removed_paths".to_string(), payload.paths.join(","))
+    ])))
+}
+
+// Region 2: File lock management endpoints ============================================
+
+#[derive(Debug, Deserialize)]
+struct ListLocksQuery { // Corresponds to the three option parameters of the CLI
+    id: Option<String>,
+    path: Option<String>,
+    limit: Option<u64>, 
+}
+
+#[derive(Debug, Serialize)]
+struct LockInfo {
+    id: String,
+    path: String,
+    owner: String,
+    locked_at: i64, // Timestamp
+}
+
+async fn list_locks(
+    Query(params): Query<ListLocksQuery>
+) -> Result<Json<Vec<LockInfo>>, AppError> {
+    // Business logic: Query the list of locks in the current branch
+    let mock_data = vec![LockInfo {
+        id: "123".to_string(),
+        path: params.path.unwrap_or_default(),
+        owner: "user1".to_string(),
+        locked_at: 1672531200
+    }];
+    Ok(Json(mock_data))
+}
+
+async fn create_lock(
+    Path(path): Path<String> // Get the file path from the URL path
+) -> Result<Json<LockInfo>, AppError> {
+    // Business logic: Create a new lock
+    Ok(Json(LockInfo {
+        id: "456".to_string(),
+        path,
+        owner: "current_user".to_string(),
+        locked_at: 1672531200
+    }))
+}
+
+#[derive(Debug, Deserialize)]
+struct UnlockParams { // CLI unlock parameters
+    force: bool,
+    id: Option<String>,
+}
+
+async fn remove_lock(
+    Path(path): Path<String>,
+    Query(params): Query<UnlockParams>
+) -> Result<Json<HashMap<String, String>>, AppError> {
+    // Business logic: Force or normal unlock
+    Ok(Json(HashMap::from([
+        ("status".to_string(), "unlocked".to_string()),
+        ("path".to_string(), path),
+        ("force_mode".to_string(), params.force.to_string())
+    ])))
+}
+
+// Region 3: LFS file information viewing endpoints =======================================
+
+#[derive(Debug, Deserialize)]
+struct MetadataQueryParams { // Corresponds to the CLI option parameters
+    long: Option<bool>,
+    size: Option<bool>, 
+    name_only: Option<bool>
+}
+
+#[derive(Debug, Serialize)]
+struct LFSFileMeta {
+    oid: String,
+    symbolic_type: String, // "*" or "-"
+    path: String,
+    size_human: Option<String> // Nullable field
+}
+
+async fn list_lfs_files(
+    Query(params): Query<MetadataQueryParams>
+) -> Result<Json<Vec<LFSFileMeta>>, AppError> {
+    // Business logic: Get the list of LFS files in the current branch
+    let mock_file = LFSFileMeta {
+        oid: "01ba4719...".to_string(),
+        symbolic_type: "*".to_string(),
+        path: "assets/image.png".to_string(),
+        size_human: params.size.then(|| "15.2 MB".to_string())
+    };
+    Ok(Json(vec![mock_file]))
+}
+
+// Error handling infrastructure
+#[derive(Debug)]
+struct AppError(anyhow::Error);
+
+impl IntoResponse for AppError {
+    fn into_response(self) -> axum::response::Response {
+        (
+            StatusCode::INTERNAL_SERVER_ERROR,
+            Json(HashMap::from([("error", self.0.to_string())]))
+        ).into_response()
+    }
+}

scorpio/src/dicfuse/mod.rs

@@ -41,7 +41,7 @@ impl Dicfuse{
             let name_os = OsString::from(&i.name);
             if name_os!=name{
                 continue;
-            }else if i.mode!=TreeItemMode::Blob{
+            }else if i.mode!=TreeItemMode::Blob && i.mode!=TreeItemMode::BlobExecutable{
                 return Ok(());
             }
 
@@ -78,7 +78,7 @@ impl Dicfuse{
         let client = Client::new();
         for i in tree.tree_items{
             //TODO & POS_BUG: how to deal with the link?
-            if i.mode==TreeItemMode::Commit || i.mode==TreeItemMode::Tree{
+            if i.mode!=TreeItemMode::Blob && i.mode!=TreeItemMode::BlobExecutable{
                 continue;
             }
             let url = format!("http://localhost:8000/api/v1/file/blob/{}",i.id);//TODO: configabel.

scorpio/src/dicfuse/tree_store.rs

@@ -22,6 +22,7 @@ pub struct StorageItem{
     is_dir: bool , // True for Directory . 
     children:Vec<u64>
 }
+#[allow(unused)]
 impl StorageItem {
     pub fn get_inode(&self) -> u64{
         self.inode

scorpio/src/fuse/inode_alloc.rs

@@ -18,6 +18,7 @@ pub struct InodeAlloc {
     alloc: Arc<Mutex<HashMap<u64,u64>> >,
 }
 
+#[allow(unused)]
 impl InodeAlloc{
     pub fn new()-> Self{
         InodeAlloc{

scorpio/src/fuse/mod.rs

@@ -29,7 +29,7 @@ pub struct MegaFuse{
 
 }
 
-
+#[allow(unused)]
 impl MegaFuse{
 
     /// Creates a new instance of `MegaFuse` asynchronously.

scorpio/src/lib.rs

@@ -1,7 +1,7 @@
 
-
 #[macro_use]
 extern crate log;
+
 pub mod passthrough;
 mod overlayfs;
 
@@ -12,4 +12,4 @@ pub mod manager;
 pub mod server;
 pub mod daemon;
 //const VFS_MAX_INO: u64 = 0xff_ffff_ffff_ffff;
-pub const READONLY_INODE :u64 = 0xffff_ffff;
+const READONLY_INODE :u64 = 0xffff_ffff;

scorpio/src/main.rs

@@ -11,11 +11,16 @@
 // use scorpio::deamon::deamon_main;
 // use tokio::runtime::Handle;
 
+
 use std::{ffi::OsStr, sync::Arc};
 
-use scorpio::{daemon::daemon_main, fuse::MegaFuse, manager::{fetch::CheckHash, ScorpioManager}, server::mount_filesystem};
+use scorpio::daemon::daemon_main;
+use scorpio::fuse::MegaFuse;
+use scorpio::manager::{fetch::CheckHash, ScorpioManager};
+use scorpio::server::mount_filesystem;
 use tokio::signal;
-use scorpio::passthrough::logfs::LoggingFileSystem;
+use scorpio::passthrough::newlogfs::LoggingFileSystem;
+
 #[tokio::main]
 async fn main() {
 

scorpio/src/manager/fetch.rs

@@ -14,9 +14,12 @@ use crate::manager::store::store_trees;
 use crate::util::GPath;
 
 use super::{ScorpioManager, WorkDir};
+
+#[allow(unused)]
 #[async_trait]
 pub trait CheckHash{
     async fn check(&mut self);
+
     async fn fetch<P: AsRef<Path>+ std::marker::Send  >(&mut self,inode:u64,monopath :P)-> WorkDir;
 }
 
@@ -381,7 +384,7 @@ mod tests {
         let client = Client::new();
 
         // Use the URL from environment variables or local test URL
-        let url = "http://localhost:8000/api/v1/file/blob/d12d12579799a658b29808fe695abd919a033ac9";
+        let url = "http://localhost:8000/api/v1/file/blob/841b6fe34540e866e1f458d77b1bd03d3cb0e782";
         // Send a GET request
         let response = client.get(url).send().await.unwrap();
 

scorpio/src/manager/push.rs

@@ -33,6 +33,7 @@ pub async fn pack(commit:Commit,trees:Vec<Tree>, blob:Vec<Blob>) -> Vec<u8>{
     pack_data
 
 }
+#[allow(unused)]
 pub async fn push(path:PathBuf,monopath:PathBuf){
     let mut lower  = path.clone();
     lower.push("lower");

scorpio/src/manager/store.rs

@@ -26,6 +26,7 @@ impl  TreeStore for sled::Db {
         Ok(decoded)
     }
 }
+#[allow(unused)]
 pub trait CommitStore{
     fn store_commit(&self,commit:Commit) -> Result<()>;
     fn get_commit(&self) -> Result<Commit>;