Add pre-commit hooks to validate and cleanup

.devcontainer/Dockerfile

@@ -25,6 +25,3 @@ RUN cd /opt && \
 # [Optional] Uncomment this section to install additional OS packages.
 # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
 #     && apt-get -y install --no-install-recommends <your-package-list-here>
-
-
-

.devcontainer/devcontainer.json

@@ -56,4 +56,4 @@
 	},
 	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode"
-}
+}

.devcontainer/docker-compose.yml

@@ -14,7 +14,7 @@ services:
         NODE_VERSION: "lts/*"
 
     volumes:
-      - ..:/workspace:cached  
+      - ..:/workspace:cached
     init: true
 
     # Overrides default command so things don't shut down after the process ends.

.github/workflows/azure-container-webapp.yml

@@ -35,7 +35,7 @@ env:
 
 on:
   push:
-    branches: [ "main" ]
+    branches: [ main ]
   pull_request:
   workflow_dispatch:
 

.github/workflows/ci.yml

@@ -1,11 +1,21 @@
 name: CI
 on:
   push:
-    branches: main
+    branches: [ main ]
   pull_request:
   workflow_dispatch:
 
 jobs:
+  pre-commit-check:
+    runs-on: ubuntu-latest
+    name: "Run the pre-commit hooks"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - uses: pre-commit/[email protected]
+
   test:
     runs-on: ubuntu-latest
     name: test (Python ${{ matrix.python-version }})

.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+exclude: '.idea'
+repos:
+  -   repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.4.0
+      hooks:
+        -   id: trailing-whitespace
+        -   id: end-of-file-fixer
+        -   id: check-yaml
+        -   id: debug-statements
+        -   id: name-tests-test
+        -   id: requirements-txt-fixer
+  -   repo: https://github.com/asottile/add-trailing-comma
+      rev: v2.4.0
+      hooks:
+        -   id: add-trailing-comma
+            args: [--py36-plus]
+  -   repo: https://github.com/asottile/pyupgrade
+      rev: v3.3.1
+      hooks:
+        -   id: pyupgrade
+            args: [--py38-plus]
+  - repo: https://github.com/iamthefij/docker-pre-commit
+    rev: v3.0.1
+    hooks:
+      - id: docker-compose-check
+  - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
+    rev: v0.1.0
+    hooks:
+      - id: dockerfilelint
+  - repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.22.0
+    hooks:
+      - id: check-github-workflows
+      - id: check-dependabot

.vscode/launch.json

@@ -16,4 +16,4 @@
             "django": true
         }
     ]
-}
+}

.vscode/settings.json

@@ -39,4 +39,4 @@
     },
     "editor.bracketPairColorization.enabled": true,
     "html.format.templating": true
-}
+}

.vscode/tasks.json

@@ -90,4 +90,4 @@
             },
         },
     ]
-}
+}

CONTRIBUTING.md

@@ -0,0 +1,37 @@
+# Contributing
+
+## Getting Started
+
+Please read the [README](./README.md) for more information about the project.
+
+## Pre-Commit Hooks
+
+### Installing
+
+This project uses `pre-commit` to run a series of checks before each commit.
+If you don't already have `pre-commit` installed, you can install globally with:
+
+```bash
+pip install pre-commit
+```
+Alternatively, you can use homebrew on macOS:
+
+```bash
+brew install pre-commit
+```
+
+Then, to install the pre-commit hooks, run:
+
+```bash
+pre-commit install
+```
+
+### Executing
+
+The pre-commit hooks will run automatically before each commit.
+However, many of the hooks can be used to automatically fix issues as well.
+To run the pre-commit hooks manually and resolve easily fixed issues, run:
+
+```bash
+pre-commit run --all-files
+```

README.md

@@ -61,4 +61,3 @@ TBD
 ## Security
 
 See [SECURITY.md](https://github.com/ossf/omega-triage-portal/blob/main/SECURITY.md).
-

docker-entrypoint-initdb.d

@@ -5,4 +5,4 @@ psql -v ON_ERROR_STOP=1 --username "$DATABASE_USER" --dbname "$DATABASE_NAME" <<
 	CREATE USER triage_user;
 	CREATE DATABASE triage;
 	GRANT ALL PRIVILEGES ON DATABASE triage_user TO triage_user;
-EOSQL
+EOSQL

src/.env-template

@@ -21,4 +21,4 @@ CACHE_REDIS_PASSWORD=''
 
 #APPINSIGHTS_IKEY = ''
 
-OSSGADGET_PATH="/opt/OSSGadget"
+OSSGADGET_PATH="/opt/OSSGadget"

src/Dockerfile

@@ -23,7 +23,7 @@ RUN cd /opt && \
     mv OSSGadget_linux_${OSSGADGET_VERSION} OSSGadget
 
 
-RUN apk update && apk add gcc \
+RUN apk update && apk add --no-cache gcc \
                         libc-dev \
                         postgresql-dev \
                         musl-dev  \

src/core/__init__.py

@@ -17,7 +17,7 @@ def get_env_variable(var_name, optional=False):
         if optional:
             return False
         raise ImproperlyConfigured(
-            f"Error: You must set the {var_name} environment variable."
+            f"Error: You must set the {var_name} environment variable.",
         ) from ex
 
 

src/core/settings.py

@@ -15,7 +15,7 @@
     dotenv.load_dotenv(os.path.join(BASE_DIR, ".env-template"))
 except Exception as ex:
     raise ImproperlyConfigured(
-        "A .env-template file was not found. Environment variables are not set."
+        "A .env-template file was not found. Environment variables are not set.",
     ) from ex
 
 SECRET_KEY = os.getenv("SECRET_KEY")
@@ -92,7 +92,7 @@
         "HOST": os.getenv("DATABASE_HOST"),
         "PORT": os.getenv("DATABASE_PORT"),
         "OPTIONS": {"options": "-c statement_timeout=5000"},
-    }
+    },
 }
 
 
@@ -146,7 +146,7 @@
                     "TIMEOUT": DEFAULT_CACHE_TIMEOUT,
                     "PASSWORD": os.getenv("CACHE_REDIS_PASSWORD"),
                 },
-            }
+            },
         }
     else:
         CACHES = {
@@ -237,5 +237,5 @@
     "default": {
         "provider": "triage.util.content_managers.file_manager.FileManager",
         "args": {"root_path": "/home/vscode/omega-fs"},
-    }
+    },
 }

src/manage.py

@@ -14,7 +14,7 @@ def main():
         raise ImportError(
             "Couldn't import Django. Are you sure it's installed and "
             "available on your PYTHONPATH environment variable? Did you "
-            "forget to activate a virtual environment?"
+            "forget to activate a virtual environment?",
         ) from exc
     execute_from_command_line(sys.argv)
 

src/pyproject.toml

@@ -545,5 +545,3 @@ ignored-argument-names = "_.*|^ignored_|^unused_"
 # List of qualified module names which can have objects that can redefine
 # builtins.
 redefining-builtins-modules = ["six.moves", "past.builtins", "future.builtins", "builtins", "io"]
-
-

src/requirements.txt

@@ -4,13 +4,13 @@ azure-core==1.26.1
 azure-storage-blob==12.14.1
 bandit==1.7.5
 black==22.10.0
-cryptography==39.0.1
+celery==5.2.7
 certifi==2022.12.7
 cffi==1.15.1
-celery==5.2.7
 charset-normalizer<3,>=2
 click==8.1.3
 cryptography==39.0.1
+cryptography==39.0.1
 data==0.4
 Deprecated==1.2.13
 dill==0.3.6
@@ -53,8 +53,8 @@ pylint-django==2.5.3
 pylint-flask==0.6
 pylint-plugin-utils==0.7
 pyparsing==3.0.9
-python-magic==0.4.27
 python-dotenv==1.0.0
+python-magic==0.4.27
 pytz==2022.6
 PyYAML==6.0
 redis==4.5.4

src/triage/apps.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 """This module configures application-level settings for the Triage Portal."""
 
 import logging

src/triage/migrations/0001_initial.py

@@ -37,7 +37,10 @@ class Migration(migrations.Migration):
                 (
                     "package_url",
                     models.CharField(
-                        blank=True, db_index=True, max_length=1024, null=True
+                        blank=True,
+                        db_index=True,
+                        max_length=1024,
+                        null=True,
                     ),
                 ),
                 ("metadata", models.JSONField(null=True)),
@@ -83,7 +86,10 @@ class Migration(migrations.Migration):
                 (
                     "package_url",
                     models.CharField(
-                        blank=True, db_index=True, max_length=1024, null=True
+                        blank=True,
+                        db_index=True,
+                        max_length=1024,
+                        null=True,
                     ),
                 ),
                 ("metadata", models.JSONField(null=True)),
@@ -98,7 +104,8 @@ class Migration(migrations.Migration):
                 (
                     "project",
                     models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="triage.project"
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="triage.project",
                     ),
                 ),
                 (
@@ -144,7 +151,9 @@ class Migration(migrations.Migration):
                 (
                     "type",
                     models.CharField(
-                        choices=[("PY", "Python Function")], default="PY", max_length=2
+                        choices=[("PY", "Python Function")],
+                        default="PY",
+                        max_length=2,
                     ),
                 ),
             ],
@@ -246,7 +255,8 @@ class Migration(migrations.Migration):
                 (
                     "tool",
                     models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="triage.tool"
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="triage.tool",
                     ),
                 ),
                 (
@@ -413,7 +423,8 @@ class Migration(migrations.Migration):
                 (
                     "scan",
                     models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="triage.scan"
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="triage.scan",
                     ),
                 ),
                 (

src/triage/migrations/0003_auto_20211127_0020.py

@@ -65,7 +65,8 @@ class Migration(migrations.Migration):
                 (
                     "tool",
                     models.ForeignKey(
-                        on_delete=django.db.models.deletion.CASCADE, to="triage.tool"
+                        on_delete=django.db.models.deletion.CASCADE,
+                        to="triage.tool",
                     ),
                 ),
             ],

src/triage/migrations/0012_filecontent_remove_finding_file_path_file_and_more.py

@@ -28,7 +28,10 @@ class Migration(migrations.Migration):
                 (
                     "content_type",
                     models.CharField(
-                        blank=True, db_index=True, max_length=64, null=True
+                        blank=True,
+                        db_index=True,
+                        max_length=64,
+                        null=True,
                     ),
                 ),
                 ("data", models.BinaryField(blank=True, null=True)),