feat: improve project security posture

.github/dependabot.yml

@@ -1,7 +1,11 @@
+# See https://docs.github.com/en/code-security/dependabot/working-with-dependabot/dependabot-options-reference
 version: 2
 updates:
   - package-ecosystem: "pip"
-    directory: "/" # Location of your requirements.txt or other package manifest
+    directory: "/"
     schedule:
-      interval: "weekly" # Check for updates daily
-    target-branch: "develop" # Target branch for updates
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/ci.yaml

@@ -14,6 +14,8 @@ on:
 
 jobs:
   run-tests:
+    permissions:
+      contents: read
     strategy:
       fail-fast: false
       matrix:
@@ -30,15 +32,20 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5.6
         with:
           python-version: ${{matrix.python-version}}
 
       - name: Install dependencies
-        run: python -m pip install pytest pytest-asyncio pytest-rerunfailures langchain-openai .
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          poetry install --all-groups
 
       - name: Run tests
-        run: pytest -v
+        run: poetry run pytest -v

.github/workflows/linting.yml

@@ -1,35 +1,37 @@
 name: Linting with Ruff
 
 on:
-    pull_request:
-        branches:
-          - main
-          - develop
-    workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+      - develop
+  workflow_dispatch:
 
 concurrency:
-    group: ${{ github.workflow }}-${{ github.ref }}
-    cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
 
 jobs:
-    ruff-formatting:
-        runs-on: ubuntu-latest
-        steps:
-        - uses: actions/checkout@v4
-        - name: Set up Python
-          uses: actions/setup-python@v5
-          with:
-            python-version: "3.9"
-            cache: 'pip'
-        - name: Get Ruff version and install
-          run: |
-              pip install poetry
-              RUFF_VERSION=$(poetry show --only=dev | grep '^ruff ' | awk '{print $3}')
-              echo "Installing ruff==$RUFF_VERSION"
-              pip install ruff==$RUFF_VERSION
-        - name: Lint with Ruff
-          run: |
-            ruff check uqlm/
-        - name: Check for unformatted files
-          run: |
-            ruff format --check uqlm/
+  ruff-formatting:
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4.2.2
+        with:
+          persist-credentials: false
+      - name: Set up Python
+        uses: actions/setup-python@v5.6
+        with:
+          python-version: "3.9"
+          cache: 'pip'
+      - name: Install Ruff
+        run: |
+            pip install poetry
+            poetry install --all-groups
+      - name: Lint with Ruff
+        run: |
+          poetry run ruff check
+      - name: Check for unformatted files
+        run: |
+          poetry run ruff format --check

.pre-commit-config.yaml

@@ -5,3 +5,25 @@ repos:
       - id: ruff
         args: [ --fix ]
       - id: ruff-format
+  - repo: https://github.com/PyCQA/bandit
+    rev: 1.8.6
+    hooks:
+      - id: bandit
+        args: [--config=bandit.yml]
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.27.2
+    hooks:
+      - id: gitleaks
+  - repo: https://github.com/fabasoad/pre-commit-snyk
+    rev: v1.0.2
+    hooks:
+      - id: snyk-test
+        args:
+          - --snyk-args=--file=poetry.lock
+          - --snyk-args=--package-manager=pip
+          - --snyk-args=--fail-on=upgradable
+  - repo: https://github.com/zizmorcore/zizmor-pre-commit
+    rev: v1.11.0
+    hooks:
+      # Run the linter.
+      - id: zizmor

bandit.yml

@@ -0,0 +1,6 @@
+exclude_dirs:
+  - assets
+  - examples
+  - tests
+skips:
+  - B101

examples/judges_demo.ipynb

@@ -66,7 +66,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "metadata": {
     "tags": []
    },

pyproject.toml

@@ -42,6 +42,9 @@ sentence-transformers = ">=3.4,<6.0"
 datasets = ">=3.3.2,<5.0.0"
 rich = "^13.8.0"
 ipywidgets = "^8.1.7"
+aiohttp = "^3.12.14"
+torch = "^2.7.1"
+
 
 [tool.poetry.group.dev]
 optional = true
@@ -51,6 +54,7 @@ python-dotenv = "^1.0.1"
 ruff = "0.9.7"
 pre-commit = "^4.1.0"
 ipykernel = "^6.29.5"
+bandit = "^1.8.6"
 
 [tool.poetry.group.test]
 optional = true
@@ -108,7 +112,7 @@ line-length = 400
 
 [tool.ruff.lint]
 #What rules to enable
-select = ["E", "F"]
+select = ["E", "F", "S"]
 # E = pycodestyle errors
 # F = pyflakes
 # I = isort (import sorting)
@@ -118,7 +122,8 @@ select = ["E", "F"]
 # S = bandit (security)
 
 #What rules to ignore
-ignore = []
+ignore = ["S101"]
+per-file-ignores = { "tests/test_similarity.py" = ["S603","S607"] }
 
 
 [tool.ruff.format]

security-insights.yml

@@ -0,0 +1,104 @@
+header:
+  schema-version: 2.0.0
+  last-updated: '2025-07-22'
+  last-reviewed: '2025-07-22'
+  url: https://github.com/cvs-health/uqlm
+
+project:
+  documentation:
+    code-of-conduct: https://github.com/cvs-health/uqlm/blob/main/CODE_OF_CONDUCT.md
+    quickstart-guide: https://cvs-health.github.io/uqlm/latest/getstarted.html
+  homepage: https://cvs-health.github.io/uqlm/
+  name: "uqlm: Uncertainty Quantification for Language Models"
+  administrators:
+    - name: Dylan Bouchard
+      affiliation: CVS Health
+      email: dylan.bouchard@cvshealth.com
+      primary: true
+    - name: Mohit Singh Chauhan
+      affiliation: CVS Health
+      email: mohitsingh.chauhan@cvshealth.com
+      primary: false
+    - name: David Skarbrevik
+      affiliation: CVS Health
+      email: david.skarbrevik@cvshealth.com
+    - name: Viren Bajaj
+      affiliation: CVS Health
+      email: bajajv@aetna.com
+    - name: Ho-Kyeong Ra
+      affiliation: CVS Health
+      email: doyajii1@gmail.com
+    - name: Zeya Ahmad
+      affiliation: CVS Health
+      email: zeya.ahmad@cvshealth.com
+  repositories:
+    - name: LangFair
+      url: https://github.com/cvs-health/uqlm
+      comment: cvs-health/uqlm is the primary repository for uqlm.
+  vulnerability-reporting:
+    bug-bounty-available: false
+    contact:
+      name: Dylan Bouchard
+      affiliation: CVS Health
+      email: dylan.bouchard@cvshealth.com
+      primary: true
+    reports-accepted: true
+    # security-policy: TODO: Add security policy URL
+
+repository:
+  url: https://github.com/cvs-health/uqlm
+  status: active
+  accepts-change-request: true
+  accepts-automated-change-request: true
+  core-team:
+    - name:        Dylan Bouchard
+      affiliation: CVS Health
+      email: dylan.bouchard@cvshealth.com
+      primary:     true
+  documentation:
+    contributing-guide: https://github.com/cvs-health/uqlm/blob/main/CONTRIBUTING.md
+  license:
+    url: https://github.com/cvs-health/uqlm/blob/main/LICENSE
+    expression: "Apache-2.0"
+  security:
+    assessments:
+      self:
+        comment: Self assessment with SCA, SAST and secret leak tools completed July 2025 to establish baseline.
+        date: '2025-07-22'
+    tools:
+      - name: Dependabot
+        type: SCA
+        version: "2"
+        rulesets:
+          - built-in
+        results:
+          adhoc:
+            name: Scheduled SCA Scan Results
+            predicate-uri: https://docs.github.com/en/graphql/reference/objects#repositoryvulnerabilityalert
+            location: https://github.com/cvs-health/uqlm/security/dependabot
+            comment: |
+              The results of the scheduled SCA scan are available in the Dependabot tab of the Security Insights page.
+        integration:
+          adhoc: true
+          ci: false
+          release: false
+      - name: bandit
+        comment: Used in pre-commit and CI to check for security issues in Python code.
+        type: SAST
+        version: "1.8.6"
+        rulesets:
+          - bandit.yml
+        integration:
+          adhoc: true
+          ci: true
+          release: true
+      - name: Snyk Open Source
+        comment: Used in pre-commit to check for vulnerabilities in dependencies.
+        type: SCA
+        version: "1.1298.0"
+        rulesets:
+          - default
+        integration:
+          adhoc: true
+          ci: false
+          release: false