Merge pull request #39 from rygorous/master

Add directory filter callback to traverseDirectory to accelerate project scan
zeux · Oct 26, 2024 · 3dcd9a8 · 3dcd9a8
2 parents 41360d2 + 9ea605a
commit 3dcd9a8
Show file tree

Hide file tree

Showing 5 changed files with 40 additions and 13 deletions.
diff --git a/src/fileutil.cpp b/src/fileutil.cpp
@@ -63,6 +63,11 @@ bool traverseFileNeeded(const char* name)
 	return true;
 }
 
+bool passthroughDirectoryFilter(const char* name)
+{
+	return true;
+}
+
 void joinPaths(std::string& buf, const char* lhs, const char* rhs)
 {
 	buf = lhs;

diff --git a/src/fileutil.hpp b/src/fileutil.hpp
@@ -6,8 +6,9 @@
 
 #include <stdio.h>
 
-bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback);
+bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool (const char* name)>& directoryFilter);
 bool traverseFileNeeded(const char* name);
+bool passthroughDirectoryFilter(const char* name);
 
 void createDirectory(const char* path);
 void createPath(const char* path);
@@ -26,4 +27,4 @@ bool getFileAttributes(const char* path, uint64_t* mtime, uint64_t* size);
 
 FILE* openFile(const char* path, const char* mode);
 
-bool watchDirectory(const char* path, const std::function<void (const char* name)>& callback);
+bool watchDirectory(const char* path, const std::function<void (const char* name)>& callback);
diff --git a/src/fileutil_posix.cpp b/src/fileutil_posix.cpp
@@ -21,7 +21,7 @@
 #include <CoreServices/CoreServices.h>
 #endif
 
-static bool traverseDirectoryRec(const char* path, const char* relpath, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback)
+static bool traverseDirectoryRec(const char* path, const char* relpath, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool (const char* name)>& directoryFilter)
 {
 	int fd = open(path, O_DIRECTORY);
 	DIR* dir = fdopendir(fd);
@@ -65,7 +65,8 @@ static bool traverseDirectoryRec(const char* path, const char* relpath, const st
 
 			if (type == DT_DIR)
 			{
-				traverseDirectoryRec(buf.c_str(), relbuf.c_str(), callback);
+				if (directoryFilter(relbuf.c_str()))
+					traverseDirectoryRec(buf.c_str(), relbuf.c_str(), callback, directoryFilter);
 			}
 			else if (type == DT_REG)
 			{
@@ -83,9 +84,9 @@ static bool traverseDirectoryRec(const char* path, const char* relpath, const st
 	return true;
 }
 
-bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback)
+bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool(const char* name)>& directoryFilter)
 {
-	return traverseDirectoryRec(path, "", callback);
+	return traverseDirectoryRec(path, "", callback, directoryFilter);
 }
 
 bool renameFile(const char* oldpath, const char* newpath)

diff --git a/src/fileutil_win.cpp b/src/fileutil_win.cpp
@@ -41,7 +41,7 @@ static uint64_t combine(uint32_t hi, uint32_t lo)
 	return (static_cast<uint64_t>(hi) << 32) | lo;
 }
 
-static bool traverseDirectoryRec(const wchar_t* path, const char* relpath, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback)
+static bool traverseDirectoryRec(const wchar_t* path, const char* relpath, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool (const char* name)>& directoryFilter)
 {
 	std::wstring query = path + std::wstring(L"/*");
 
@@ -73,7 +73,8 @@ static bool traverseDirectoryRec(const wchar_t* path, const char* relpath, const
 				buf += '/';
 				buf += data.cFileName;
 
-				traverseDirectoryRec(buf.c_str(), relbuf.c_str(), callback);
+				if (directoryFilter(relbuf.c_str()))
+					traverseDirectoryRec(buf.c_str(), relbuf.c_str(), callback, directoryFilter);
 			}
 			else
 			{
@@ -91,9 +92,9 @@ static bool traverseDirectoryRec(const wchar_t* path, const char* relpath, const
 	return true;
 }
 
-bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback)
+bool traverseDirectory(const char* path, const std::function<void (const char* name, uint64_t mtime, uint64_t size)>& callback, const std::function<bool (const char* name)>& directoryFilter)
 {
-	return traverseDirectoryRec(fromUtf8(path).c_str(), "", callback);
+	return traverseDirectoryRec(fromUtf8(path).c_str(), "", callback, directoryFilter);
 }
 
 bool renameFile(const char* oldpath, const char* newpath)

diff --git a/src/project.cpp b/src/project.cpp
@@ -86,7 +86,7 @@ static std::vector<std::string> getProjectsByPrefix(const char* prefix)
 			{
 				result.push_back(pprefix + std::string(path, dot));
 			}
-		});
+		}, passthroughDirectoryFilter);
 	}
 
 	return result;
@@ -280,6 +280,23 @@ bool isFileAcceptable(ProjectGroup* group, const char* path)
 	return true;
 }
 
+// It can help enumeration times massively to be able to stop traversing
+// entire subtrees instead of enumerating them entirely only to reject
+// every single file inside them, so run exclusion rules on directory
+// names during traversal.
+bool isDirectoryAcceptable(ProjectGroup* group, const char* path)
+{
+	size_t length = strlen(path);
+
+	for (; group; group = group->parent)
+	{
+		if (group->exclude && group->exclude->search(path, length))
+			return false;
+	}
+
+	return true;
+}
+
 static void getProjectGroupFilesRec(Output* output, ProjectGroup* group, std::vector<FileInfo>& files)
 {
 	for (auto& path: group->files)
@@ -296,13 +313,15 @@ static void getProjectGroupFilesRec(Output* output, ProjectGroup* group, std::ve
 	{
 		std::string buf;
 
-		bool result = traverseDirectory(folder.c_str(), [&](const char* path, uint64_t mtime, uint64_t size) { 
+		bool result = traverseDirectory(folder.c_str(), [&](const char* path, uint64_t mtime, uint64_t size) {
 			if (isFileAcceptable(group, path))
 			{
 				joinPaths(buf, folder.c_str(), path);
 				files.push_back({ buf, mtime, size });
 			}
-		});
+			}, [&](const char* path) {
+				return isDirectoryAcceptable(group, path);
+			});
 
 		if (!result) output->error("Error reading folder %s\n", folder.c_str());
 	}