Pre-commit/CI and build improvements. (#1495)

* Small improvements to build script.
* Use ruff for linting and formatting.
* Use mypy daemon for (much) faster type-checking.
* Add typing to `cache.py` to fix it's requirements.
* Bump to latest mypy for increased performance.
* Bump husky to latest version.
* Only lint Python when Python files have changed.
* Linting for GitHub Actions.
  Requires actionlint: https://github.com/rhysd/actionlint
* Linting and formatting check in CI.

.github/workflows/ci.yaml

@@ -14,8 +14,8 @@ jobs:
   pytest:
     uses: ./.github/workflows/pytest.yaml
     secrets: inherit
-  mypy:
-    uses: ./.github/workflows/mypy.yaml
+  lint-backend:
+    uses: ./.github/workflows/lint-backend.yaml
     secrets: inherit
   e2e-tests:
     uses: ./.github/workflows/e2e-tests.yaml
@@ -26,7 +26,7 @@ jobs:
   ci:
     runs-on: ubuntu-latest
     name: Run CI
-    needs: [mypy, pytest, lint-ui, e2e-tests]
+    needs: [lint-backend, pytest, lint-ui, e2e-tests]
     steps:
       - name: Done
         run: echo "Done"

.github/workflows/mypy.yaml → .github/workflows/lint-backend.yaml

@@ -1,11 +1,11 @@
-name: Mypy
+name: LintBackend
 
 on: [workflow_call]
 
 permissions: read-all
 
 jobs:
-  mypy:
+  lint-backend:
     runs-on: ubuntu-latest
     env:
       BACKEND_DIR: ./backend
@@ -16,6 +16,17 @@ jobs:
         with:
           poetry-install-args: --with tests --with mypy --with custom-data --no-root
           poetry-working-directory: ${{ env.BACKEND_DIR }}
+      - name: Lint with ruff
+        uses: astral-sh/ruff-action@v1
+        with:
+          src: ${{ env.BACKEND_DIR }}
+          changed-files: "true"
+      - name: Check formatting with ruff
+        uses: astral-sh/ruff-action@v1
+        with:
+          src: ${{ env.BACKEND_DIR }}
+          changed-files: "true"
+          args: "format --check"
       - name: Run Mypy
         run: poetry run mypy chainlit/
         working-directory: ${{ env.BACKEND_DIR }}

.gitignore

@@ -60,3 +60,4 @@ dist-ssr
 .coverage
 
 backend/README.md
+backend/.dmypy.json

.husky/pre-commit

@@ -1,5 +1 @@
-#!/usr/bin/env sh
-. "$(dirname -- "$0")/_/husky.sh"
-
 pnpm lint-staged
-pnpm run lintPython

backend/build.py

@@ -1,62 +1,104 @@
 """Build script gets called on poetry/pip build."""
 
+import os
 import pathlib
 import shutil
+import signal
 import subprocess
+import sys
+from contextlib import contextmanager
+from typing import Optional
+
+
+class BuildError(Exception):
+    """Custom exception for build failures"""
+
+    pass
+
+
+def run_subprocess(cmd: list[str], cwd: os.PathLike) -> None:
+    """
+    Run a subprocess, allowing natural signal propagation.
+
+    Args:
+        cmd: Command and arguments as a list of strings
+        cwd: Working directory for the subprocess
+    """
+
+    print(f"-- Running: {' '.join(cmd)}")
+    subprocess.run(cmd, cwd=cwd, check=True)
 
 
 def pnpm_install(project_root, pnpm_path):
-    subprocess.run(
-        [pnpm_path, "install", "--frozen-lockfile"], cwd=project_root, check=True
-    )
+    run_subprocess([pnpm_path, "install", "--frozen-lockfile"], project_root)
 
 
 def pnpm_buildui(project_root, pnpm_path):
-    subprocess.run([pnpm_path, "buildUi"], cwd=project_root, check=True)
+    run_subprocess([pnpm_path, "buildUi"], project_root)
+
+
+def copy_directory(src, dst, description):
+    """Copy directory with proper error handling"""
+    print(f"Copying {src} to {dst}")
+    try:
+        dst.mkdir(parents=True, exist_ok=True)
+        shutil.copytree(src, dst, dirs_exist_ok=True)
+    except KeyboardInterrupt:
+        print("\nInterrupt received during copy operation...")
+        # Clean up partial copies
+        if dst.exists():
+            shutil.rmtree(dst)
+        raise
+    except Exception as e:
+        raise BuildError(f"Failed to copy {src} to {dst}: {str(e)}")
 
 
 def copy_frontend(project_root):
     """Copy the frontend dist directory to the backend for inclusion in the package."""
-
-    # Create backend frontend dist dir
     backend_frontend_dir = project_root / "backend" / "chainlit" / "frontend" / "dist"
-    backend_frontend_dir.mkdir(parents=True, exist_ok=True)
-
-    # Recursively copy frontend_dist to backend_frontend_dir
     frontend_dist = project_root / "frontend" / "dist"
-
-    print(f"Copying {frontend_dist} to {backend_frontend_dir}")
-    shutil.copytree(frontend_dist, backend_frontend_dir, dirs_exist_ok=True)
+    copy_directory(frontend_dist, backend_frontend_dir, "frontend assets")
 
 
 def copy_copilot(project_root):
     """Copy the copilot dist directory to the backend for inclusion in the package."""
-
-    # Create backend copilot dir
     backend_copilot_dir = project_root / "backend" / "chainlit" / "copilot" / "dist"
-    backend_copilot_dir.mkdir(parents=True, exist_ok=True)
-
-    # Recursively copy copilot_dist to backend_copilot_dir
     copilot_dist = project_root / "libs" / "copilot" / "dist"
-
-    print(f"Copying {copilot_dist} to {backend_copilot_dir}")
-    shutil.copytree(copilot_dist, backend_copilot_dir, dirs_exist_ok=True)
+    copy_directory(copilot_dist, backend_copilot_dir, "copilot assets")
 
 
 def build():
-    # Find directory containing this file.
-    backend_dir = pathlib.Path(__file__).resolve().parent
-    project_root = backend_dir.parent
-
-    pnpm = shutil.which("pnpm")
-    if not pnpm:
-        print("pnpm not found!")
-        exit(-1)
-
-    pnpm_install(project_root, pnpm)
-    pnpm_buildui(project_root, pnpm)
-    copy_frontend(project_root)
-    copy_copilot(project_root)
+    """Main build function with proper error handling"""
+
+    print(
+        "\n-- Building frontend, this might take a while!\n\n"
+        "   If you don't need to build the frontend and just want dependencies installed, use:\n"
+        "   `poetry install --no-root`\n"
+    )
+
+    try:
+        # Find directory containing this file
+        backend_dir = pathlib.Path(__file__).resolve().parent
+        project_root = backend_dir.parent
+
+        pnpm = shutil.which("pnpm")
+        if not pnpm:
+            raise BuildError("pnpm not found!")
+
+        pnpm_install(project_root, pnpm)
+        pnpm_buildui(project_root, pnpm)
+        copy_frontend(project_root)
+        copy_copilot(project_root)
+
+    except KeyboardInterrupt:
+        print("\nBuild interrupted by user")
+        sys.exit(1)
+    except BuildError as e:
+        print(f"\nBuild failed: {str(e)}")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\nUnexpected error: {str(e)}")
+        sys.exit(1)
 
 
 if __name__ == "__main__":

backend/chainlit/cache.py

@@ -1,6 +1,7 @@
 import importlib.util
 import os
 import threading
+from typing import Any
 
 from chainlit.config import config
 from chainlit.logger import logger
@@ -22,7 +23,7 @@ def init_lc_cache():
                 )
 
 
-_cache = {}
+_cache: dict[tuple, Any] = {}
 _cache_lock = threading.Lock()