Ignore files based on name(glob), size and whether they're binary files

Author: senko

README.md

@@ -85,7 +85,7 @@ After you have Python and (optionally) PostgreSQL installed, follow these steps:
    - LLM Provider (OpenAI/Azure/Openrouter)
    - Your API key
    - database settings: SQLite/PostgreSQL (to change from SQLite to PostgreSQL, just set `DATABASE_TYPE=postgres`)
-   - optionally set IGNORE_FOLDERS for the folders which shouldn't be tracked by GPT Pilot in workspace, useful to ignore folders created by compilers (i.e. `IGNORE_FOLDERS=folder1,folder2,folder3`)
+   - optionally set IGNORE_PATHS for the folders which shouldn't be tracked by GPT Pilot in workspace, useful to ignore folders created by compilers (i.e. `IGNORE_PATHS=folder1,folder2,folder3`)
 9. `python db_init.py` (initialize the database)
 10. `python main.py` (start GPT Pilot)
 

docker-compose.yml

@@ -18,7 +18,7 @@ services:
             - DB_USER=pilot
             - DB_PASSWORD=pilot
             # Folders which shouldn't be tracked in workspace (useful to ignore folders created by compiler)
-            # IGNORE_FOLDERS=folder1,folder2
+            # IGNORE_PATHS=folder1,folder2
         volumes:
             - ~/gpt-pilot-workspace:/usr/src/app/workspace
         build:

pilot/.env.example

@@ -16,7 +16,7 @@ MODEL_NAME=gpt-4-1106-preview
 MAX_TOKENS=8192
 
 # Folders which shouldn't be tracked in workspace (useful to ignore folders created by compiler)
-# IGNORE_FOLDERS=folder1,folder2
+# IGNORE_PATHS=folder1,folder2
 
 # Database
 # DATABASE_TYPE=postgres

pilot/const/common.py

@@ -21,10 +21,7 @@
     'finished'
 ]
 
-additional_ignore_folders = os.environ.get('IGNORE_FOLDERS', '').split(',')
-
-# TODO: rename to IGNORE_PATHS as it also contains files
-IGNORE_FOLDERS = [
+DEFAULT_IGNORE_PATHS = [
     '.git',
     '.gpt-pilot',
     '.idea',
@@ -36,7 +33,16 @@
     'venv',
     'dist',
     'build',
-    'target'
-] + [folder for folder in additional_ignore_folders if folder]
-
+    'target',
+    "*.min.js",
+    "*.min.css",
+    "*.svg",
+    "*.csv",
+]
+IGNORE_PATHS = DEFAULT_IGNORE_PATHS + [
+    folder for folder
+    in os.environ.get('IGNORE_PATHS', '').split(',')
+    if folder
+]
+IGNORE_SIZE_THRESHOLD = 102400  # 100K+ files are ignored by default
 PROMPT_DATA_TO_IGNORE = {'directory_tree', 'name'}

pilot/helpers/Project.py

@@ -8,7 +8,7 @@
 
 from const.messages import CHECK_AND_CONTINUE, AFFIRMATIVE_ANSWERS, NEGATIVE_ANSWERS
 from utils.style import color_yellow_bold, color_cyan, color_white_bold, color_green
-from const.common import IGNORE_FOLDERS, STEPS
+from const.common import STEPS
 from database.database import delete_unconnected_steps_from, delete_all_app_development_data, update_app_status
 from const.ipc import MESSAGE_TYPE
 from prompts.prompts import ask_user
@@ -28,6 +28,7 @@
 from logger.logger import logger
 from utils.dot_gpt_pilot import DotGptPilot
 from utils.llm_connection import test_api_access
+from utils.ignore import IgnoreMatcher
 
 from utils.telemetry import telemetry
 
@@ -176,12 +177,7 @@ def get_directory_tree(self, with_descriptions=False):
         Returns:
             dict: The directory tree.
         """
-        # files = {}
-        # if with_descriptions and False:
-        #     files = File.select().where(File.app_id == self.args['app_id'])
-        #     files = {snapshot.name: snapshot for snapshot in files}
-        # return build_directory_tree_with_descriptions(self.root_path, ignore=IGNORE_FOLDERS, files=files, add_descriptions=False)
-        return build_directory_tree(self.root_path, ignore=IGNORE_FOLDERS)
+        return build_directory_tree(self.root_path)
 
     def get_test_directory_tree(self):
         """
@@ -191,7 +187,7 @@ def get_test_directory_tree(self):
             dict: The directory tree of tests.
         """
         # TODO remove hardcoded path
-        return build_directory_tree(self.root_path + '/tests', ignore=IGNORE_FOLDERS)
+        return build_directory_tree(self.root_path + '/tests')
 
     def get_all_coded_files(self):
         """
@@ -209,18 +205,7 @@ def get_all_coded_files(self):
                 )
             )
 
-        files = self.get_files([file.path + '/' + file.name for file in files])
-
-        # Don't send contents of binary files
-        for file in files:
-            if not isinstance(file["content"], str):
-                file["content"] = f"<<binary file, {len(file['content'])} bytes>>"
-
-        # TODO temoprary fix to eliminate files that are not in the project
-        files = [file for file in files if file['content'] != '']
-        # TODO END
-
-        return files
+        return self.get_files([file.path + '/' + file.name for file in files])
 
     def get_files(self, files):
         """
@@ -232,16 +217,20 @@ def get_files(self, files):
         Returns:
             list: A list of files with content.
         """
+        matcher = IgnoreMatcher(root_path=self.root_path)
         files_with_content = []
         for file_path in files:
             try:
                 # TODO path is sometimes relative and sometimes absolute - fix at one point
                 _, full_path = self.get_full_file_path(file_path, file_path)
                 file_data = get_file_contents(full_path, self.root_path)
             except ValueError:
+                full_path = None
                 file_data = {"path": file_path, "name": os.path.basename(file_path), "content": ''}
 
-            files_with_content.append(file_data)
+            if full_path and file_data["content"] != "" and not matcher.ignore(full_path):
+                files_with_content.append(file_data)
+
         return files_with_content
 
     def find_input_required_lines(self, file_content):
@@ -395,7 +384,7 @@ def normalize_path(path: str) -> Tuple[str, str]:
 
 
     def save_files_snapshot(self, development_step_id):
-        files = get_directory_contents(self.root_path, ignore=IGNORE_FOLDERS)
+        files = get_directory_contents(self.root_path)
         development_step, created = DevelopmentSteps.get_or_create(id=development_step_id)
 
         total_files = 0
@@ -431,7 +420,7 @@ def restore_files(self, development_step_id):
         development_step = DevelopmentSteps.get(DevelopmentSteps.id == development_step_id)
         file_snapshots = FileSnapshot.select().where(FileSnapshot.development_step == development_step)
 
-        clear_directory(self.root_path, IGNORE_FOLDERS + self.files)
+        clear_directory(self.root_path, ignore=self.files)
         for file_snapshot in file_snapshots:
             update_file(file_snapshot.file.full_path, file_snapshot.content, project=self)
             if file_snapshot.file.full_path not in self.files:

pilot/helpers/cli.py

@@ -10,6 +10,7 @@
 
 from logger.logger import logger
 from utils.style import color_yellow, color_green, color_red, color_yellow_bold
+from utils.ignore import IgnoreMatcher
 from database.database import get_saved_command_run, save_command_run
 from helpers.exceptions.TooDeepRecursionError import TooDeepRecursionError
 from helpers.exceptions.TokenLimitError import TokenLimitError
@@ -340,32 +341,41 @@ def check_if_command_successful(convo, command, cli_response, response, exit_cod
 
     return response
 
-def build_directory_tree(path, prefix='', is_root=True, ignore=None):
+def build_directory_tree(path, prefix='', root_path=None) -> str:
     """Build the directory tree structure in a simplified format.
 
-    Args:
-    - path: The starting directory path.
-    - prefix: Prefix for the current item, used for recursion.
-    - is_root: Flag to indicate if the current item is the root directory.
-    - ignore: a list of directories to ignore
-
-    Returns:
-    - A string representation of the directory tree.
+    :param path: The starting directory path.
+    :param prefix: Prefix for the current item, used for recursion.
+    :param root_path: The root directory path.
+    :return: A string representation of the directory tree.
     """
     output = ""
     indent = '  '
 
+    if root_path is None:
+        root_path = path
+
+    matcher = IgnoreMatcher(root_path=root_path)
+
     if os.path.isdir(path):
-        if is_root:
+        if root_path == path:
             output += '/'
         else:
             dir_name = os.path.basename(path)
             output += f'{prefix}/{dir_name}'
 
         # List items in the directory
         items = os.listdir(path)
-        dirs = [item for item in items if os.path.isdir(os.path.join(path, item)) and item not in ignore]
-        files = [item for item in items if os.path.isfile(os.path.join(path, item))]
+        dirs = []
+        files = []
+        for item in items:
+            item_path = os.path.join(path, item)
+            if matcher.ignore(item_path):
+                continue
+            if os.path.isdir(item_path):
+                dirs.append(item)
+            elif os.path.isfile(item_path):
+                files.append(item)
         dirs.sort()
         files.sort()
 
@@ -374,7 +384,7 @@ def build_directory_tree(path, prefix='', is_root=True, ignore=None):
             for index, dir_item in enumerate(dirs):
                 item_path = os.path.join(path, dir_item)
                 new_prefix = prefix + indent  # Updated prefix for recursion
-                output += build_directory_tree(item_path, new_prefix, is_root=False, ignore=ignore)
+                output += build_directory_tree(item_path, new_prefix, root_path)
 
             if files:
                 output += f"{prefix}  {', '.join(files)}\n"
@@ -387,36 +397,6 @@ def build_directory_tree(path, prefix='', is_root=True, ignore=None):
     return output
 
 
-def res_for_build_directory_tree(path, files=None):
-    return ' - ' + files[os.path.basename(path)].description + ' ' if files and os.path.basename(path) in files else ''
-
-
-def build_directory_tree_with_descriptions(path, prefix="", ignore=None, is_last=False, files=None):
-    """Build the directory tree structure in tree-like format.
-   Args:
-   - path: The starting directory path.
-   - prefix: Prefix for the current item, used for recursion.
-   - ignore: List of directory names to ignore.
-   - is_last: Flag to indicate if the current item is the last in its parent directory.
-   Returns:
-   - A string representation of the directory tree.
-   """
-    ignore |= []
-    if os.path.basename(path) in ignore:
-        return ""
-    output = ""
-    indent = '|   ' if not is_last else '    '
-    # It's a directory, add its name to the output and then recurse into it
-    output += prefix + f"|-- {os.path.basename(path)}{res_for_build_directory_tree(path, files)}/\n"
-    if os.path.isdir(path):
-        # List items in the directory
-        items = os.listdir(path)
-        for index, item in enumerate(items):
-            item_path = os.path.join(path, item)
-            output += build_directory_tree(item_path, prefix + indent, ignore, index == len(items) - 1, files)
-    return output
-
-
 def execute_command_and_check_cli_response(convo, command: dict):
     """
     Execute a command and check its CLI response.

pilot/helpers/files.py

@@ -3,7 +3,7 @@
 from typing import Optional, Union
 
 from utils.style import color_green
-
+from utils.ignore import IgnoreMatcher
 
 def update_file(path: str, new_content: Union[str, bytes], project=None):
     """
@@ -87,7 +87,8 @@ def get_file_contents(
 
 
 def get_directory_contents(
-    directory: str, ignore: Optional[list[str]] = None
+    directory: str,
+    ignore: Optional[list[str]] = None,
 ) -> list[dict[str, Union[str, bytes]]]:
     """
     Get the content of all files in the given directory.
@@ -101,19 +102,22 @@ def get_directory_contents(
     """
     return_array = []
 
-    if ignore is None:
-        ignore = []
+    matcher = IgnoreMatcher(ignore, root_path=directory)
 
     # TODO: Convert to use pathlib.Path.walk()
     for dpath, dirs, files in os.walk(directory):
         # In-place update of dirs so that os.walk() doesn't traverse them
-        dirs[:] = [d for d in dirs if d not in ignore]
+        dirs[:] = [
+            d for d in dirs
+            if not matcher.ignore(os.path.join(dpath, d))
+        ]
 
         for file in files:
-            if file in ignore:
+            full_path = os.path.join(dpath, file)
+            if matcher.ignore(full_path):
                 continue
 
-            return_array.append(get_file_contents(os.path.join(dpath, file), directory))
+            return_array.append(get_file_contents(full_path, directory))
 
     return return_array
 
@@ -125,20 +129,22 @@ def clear_directory(directory: str, ignore: Optional[list[str]] = None):
     :param dir_path: Full path to the directory to clear
     :param ignore: List of files or folders to ignore (optional)
     """
-    if ignore is None:
-        ignore = []
+    matcher = IgnoreMatcher(ignore, root_path=directory)
 
     # TODO: Convert to use pathlib.Path.walk()
     for dpath, dirs, files in os.walk(directory, topdown=True):
         # In-place update of dirs so that os.walk() doesn't traverse them
-        dirs[:] = [d for d in dirs if d not in ignore]
+        dirs[:] = [
+            d for d in dirs
+            if not matcher.ignore(os.path.join(dpath, d))
+        ]
 
         for file in files:
-            if file in ignore or os.path.join(directory, file) in ignore:
+            full_path = os.path.join(dpath, file)
+            if matcher.ignore(full_path):
                 continue
 
-            path = os.path.join(dpath, file)
-            os.remove(path)
+            os.remove(full_path)
 
         # Delete empty subdirectories not in ignore list
         for d in dirs:

pilot/helpers/test_Project.py

@@ -363,7 +363,7 @@ def setup_method(self):
             'user_review_goal': 'Test User Review Goal',
         }]
 
-        # with directories including common.IGNORE_FOLDERS
+        # with directories including common.IGNORE_PATHS
         src = os.path.join(project.root_path, 'src')
         foo = os.path.join(project.root_path, 'src/foo')
         files_no_folders = os.path.join(foo, 'files_no_folders')

pilot/test/helpers/test_files.py

@@ -147,14 +147,13 @@ def test_get_directory_contents_live():
     assert isinstance(this_file["content"], str)
     assert "test_get_directory_contents_live()" in this_file["content"]
 
-    # Check that the Python cache was loaded as a binary file
-    print("FILES", [(f["path"], f["name"]) for f in files])
-    pycache_file = [
+    # Check that the binary file was ignored
+    image_files = [
         f
         for f in files
         if f["path"] == "helpers" and f["name"] == "testlogo.png"
-    ][0]
-    assert isinstance(pycache_file["content"], bytes)
+    ]
+    assert image_files == []
 
     # Check that the ignore list works
     assert all(file["name"] != "__init__.py" for file in files)