Merge branch 'langgenius:main' into main

.devcontainer/Dockerfile

@@ -1,5 +1,5 @@
-FROM mcr.microsoft.com/devcontainers/python:3.10
+FROM mcr.microsoft.com/devcontainers/python:3.12
 
 # [Optional] Uncomment this section to install additional OS packages.
 # RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-#     && apt-get -y install --no-install-recommends <your-package-list-here>
+#     && apt-get -y install --no-install-recommends <your-package-list-here>

.devcontainer/devcontainer.json

@@ -1,7 +1,7 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the
 // README at: https://github.com/devcontainers/templates/tree/main/src/anaconda
 {
-	"name": "Python 3.10",
+	"name": "Python 3.12",
 	"build": { 
 		"context": "..",
 		"dockerfile": "Dockerfile"

.devcontainer/post_create_command.sh

@@ -7,5 +7,6 @@ echo 'alias start-api="cd /workspaces/dify/api && poetry run python -m flask run
 echo 'alias start-worker="cd /workspaces/dify/api && poetry run python -m celery -A app.celery worker -P gevent -c 1 --loglevel INFO -Q dataset,generation,mail,ops_trace,app_deletion"' >> ~/.bashrc
 echo 'alias start-web="cd /workspaces/dify/web && npm run dev"' >> ~/.bashrc
 echo 'alias start-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify up -d"' >> ~/.bashrc
+echo 'alias stop-containers="cd /workspaces/dify/docker && docker-compose -f docker-compose.middleware.yaml -p dify down"' >> ~/.bashrc
 
-source /home/vscode/.bashrc
+source /home/vscode/.bashrc

.devcontainer/post_start_command.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-poetry install -C api
+cd api && poetry install

.github/actions/setup-poetry/action.yml

@@ -0,0 +1,36 @@
+name: Setup Poetry and Python
+
+inputs:
+  python-version:
+    description: Python version to use and the Poetry installed with
+    required: true
+    default: '3.11'
+  poetry-version:
+    description: Poetry version to set up
+    required: true
+    default: '2.0.1'
+  poetry-lockfile:
+    description: Path to the Poetry lockfile to restore cache from
+    required: true
+    default: ''
+
+runs:
+  using: composite
+  steps:
+    - name: Set up Python ${{ inputs.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: pip
+
+    - name: Install Poetry
+      shell: bash
+      run: pip install poetry==${{ inputs.poetry-version }}
+
+    - name: Restore Poetry cache
+      if: ${{ inputs.poetry-lockfile != '' }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: poetry
+        cache-dependency-path: ${{ inputs.poetry-lockfile }}

.github/pull_request_template.md

@@ -1,34 +1,25 @@
-# Checklist:
+# Summary
 
-> [!IMPORTANT]  
-> Please review the checklist below before submitting your pull request.
-
-- [ ] Please open an issue before creating a PR or link to an existing issue
-- [ ] I have performed a self-review of my own code
-- [ ] I have commented my code, particularly in hard-to-understand areas
-- [ ] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods
-
-# Description
-
-Describe the big picture of your changes here to communicate to the maintainers why we should accept this pull request. If it fixes a bug or resolves a feature request, be sure to link to that issue. Close issue syntax: `Fixes #<issue number>`, see [documentation](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) for more details.
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
 
-Fixes 
+> [!Tip]
+> Close issue syntax: `Fixes #<issue number>` or `Resolves #<issue number>`, see [documentation](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword) for more details.
 
-## Type of Change
 
-- [ ] Bug fix (non-breaking change which fixes an issue)
-- [ ] New feature (non-breaking change which adds functionality)
-- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
-- [ ] This change requires a documentation update, included: [Dify Document](https://github.com/langgenius/dify-docs)
-- [ ] Improvement, including but not limited to code refactoring, performance optimization, and UI/UX improvement
-- [ ] Dependency upgrade
-
-# Testing Instructions
+# Screenshots
 
-Please describe the tests that you ran to verify your changes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+| Before | After |
+|--------|-------|
+| ...    | ...   |
 
-- [ ] Test A
-- [ ] Test B
+# Checklist
 
+> [!IMPORTANT]  
+> Please review the checklist below before submitting your pull request.
 
+- [ ] This change requires a documentation update, included: [Dify Document](https://github.com/langgenius/dify-docs)
+- [x] I understand that this PR may be closed in case there was no previous discussion or issues. (This doesn't apply to typos!)
+- [x] I've added a test for each change that was introduced, and I tried as much as possible to make a single atomic change.
+- [x] I've updated the documentation accordingly.
+- [x] I ran `dev/reformat`(backend) and `cd web && npx lint-staged`(frontend) to appease the lint gods
 

.github/workflows/api-tests.yml

@@ -7,6 +7,7 @@ on:
     paths:
       - api/**
       - docker/**
+      - .github/workflows/api-tests.yml
 
 concurrency:
   group: api-tests-${{ github.head_ref || github.run_id }}
@@ -19,24 +20,18 @@ jobs:
     strategy:
       matrix:
         python-version:
-          - "3.10"
           - "3.11"
           - "3.12"
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - name: Setup Poetry and Python ${{ matrix.python-version }}
+        uses: ./.github/actions/setup-poetry
         with:
           python-version: ${{ matrix.python-version }}
-          cache-dependency-path: |
-            api/pyproject.toml
-            api/poetry.lock
-
-      - name: Install Poetry
-        uses: abatilo/actions-poetry@v3
+          poetry-lockfile: api/poetry.lock
 
       - name: Check Poetry lockfile
         run: |
@@ -47,16 +42,23 @@ jobs:
         run: poetry install -C api --with dev
 
       - name: Check dependencies in pyproject.toml
-        run: poetry run -C api bash dev/pytest/pytest_artifacts.sh
+        run: poetry run -P api bash dev/pytest/pytest_artifacts.sh
 
       - name: Run Unit tests
-        run: poetry run -C api bash dev/pytest/pytest_unit_tests.sh
+        run: poetry run -P api bash dev/pytest/pytest_unit_tests.sh
 
       - name: Run ModelRuntime
-        run: poetry run -C api bash dev/pytest/pytest_model_runtime.sh
+        run: poetry run -P api bash dev/pytest/pytest_model_runtime.sh
+
+      - name: Run dify config tests
+        run: poetry run -P api python dev/pytest/pytest_config_tests.py
 
       - name: Run Tool
-        run: poetry run -C api bash dev/pytest/pytest_tools.sh
+        run: poetry run -P api bash dev/pytest/pytest_tools.sh
+
+      - name: Run mypy
+        run: |
+          poetry run -C api python -m mypy --install-types --non-interactive .
 
       - name: Set up dotenvs
         run: |
@@ -67,7 +69,7 @@ jobs:
         run: sh .github/workflows/expose_service_ports.sh
 
       - name: Set up Sandbox
-        uses: hoverkraft-tech/[email protected].0
+        uses: hoverkraft-tech/[email protected].2
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml
@@ -76,22 +78,4 @@ jobs:
             ssrf_proxy
 
       - name: Run Workflow
-        run: poetry run -C api bash dev/pytest/pytest_workflow.sh
-
-      - name: Set up Vector Stores (Weaviate, Qdrant, PGVector, Milvus, PgVecto-RS, Chroma, MyScale, ElasticSearch)
-        uses: hoverkraft-tech/[email protected]
-        with:
-          compose-file: |
-            docker/docker-compose.yaml
-          services: |
-            weaviate
-            qdrant
-            etcd
-            minio
-            milvus-standalone
-            pgvecto-rs
-            pgvector
-            chroma
-            elasticsearch
-      - name: Test Vector Stores
-        run: poetry run -C api bash dev/pytest/pytest_vdb.sh
+        run: poetry run -P api bash dev/pytest/pytest_workflow.sh

.github/workflows/build-push.yml

@@ -49,7 +49,7 @@ jobs:
           echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}
@@ -114,7 +114,7 @@ jobs:
           merge-multiple: true
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ env.DOCKERHUB_USER }}
           password: ${{ env.DOCKERHUB_TOKEN }}

.github/workflows/db-migration-test.yml

@@ -6,6 +6,7 @@ on:
       - main
     paths:
       - api/migrations/**
+      - .github/workflows/db-migration-test.yml
 
 concurrency:
   group: db-migration-test-${{ github.ref }}
@@ -14,25 +15,15 @@ concurrency:
 jobs:
   db-migration-test:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version:
-          - "3.10"
 
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+      - name: Setup Poetry and Python
+        uses: ./.github/actions/setup-poetry
         with:
-          python-version: ${{ matrix.python-version }}
-          cache-dependency-path: |
-            api/pyproject.toml
-            api/poetry.lock
-
-      - name: Install Poetry
-        uses: abatilo/actions-poetry@v3
+          poetry-lockfile: api/poetry.lock
 
       - name: Install dependencies
         run: poetry install -C api
@@ -43,7 +34,7 @@ jobs:
           cp middleware.env.example middleware.env
 
       - name: Set up Middlewares
-        uses: hoverkraft-tech/[email protected].0
+        uses: hoverkraft-tech/[email protected].2
         with:
           compose-file: |
             docker/docker-compose.middleware.yaml
@@ -57,6 +48,8 @@ jobs:
           cp .env.example .env
 
       - name: Run DB Migration
+        env:
+          DEBUG: true
         run: |
           cd api
           poetry run python -m flask upgrade-db

.github/workflows/expose_service_ports.sh

@@ -7,5 +7,8 @@ yq eval '.services["milvus-standalone"].ports += ["19530:19530"]' -i docker/dock
 yq eval '.services.pgvector.ports += ["5433:5432"]' -i docker/docker-compose.yaml
 yq eval '.services["pgvecto-rs"].ports += ["5431:5432"]' -i docker/docker-compose.yaml
 yq eval '.services["elasticsearch"].ports += ["9200:9200"]' -i docker/docker-compose.yaml
+yq eval '.services.couchbase-server.ports += ["8091-8096:8091-8096"]' -i docker/docker-compose.yaml
+yq eval '.services.couchbase-server.ports += ["11210:11210"]' -i docker/docker-compose.yaml
+yq eval '.services.tidb.ports += ["4000:4000"]' -i docker/docker-compose.yaml
 
-echo "Ports exposed for sandbox, weaviate, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch"
+echo "Ports exposed for sandbox, weaviate, tidb, qdrant, chroma, milvus, pgvector, pgvecto-rs, elasticsearch, couchbase"

.github/workflows/style.yml

@@ -22,33 +22,28 @@ jobs:
         id: changed-files
         uses: tj-actions/changed-files@v45
         with:
-          files: api/**
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        if: steps.changed-files.outputs.any_changed == 'true'
-        with:
-          python-version: '3.10'
+          files: |
+            api/**
+            .github/workflows/style.yml
 
-      - name: Install Poetry
+      - name: Setup Poetry and Python
         if: steps.changed-files.outputs.any_changed == 'true'
-        uses: abatilo/actions-poetry@v3
+        uses: ./.github/actions/setup-poetry
 
-      - name: Python dependencies
+      - name: Install dependencies
         if: steps.changed-files.outputs.any_changed == 'true'
         run: poetry install -C api --only lint
 
       - name: Ruff check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: poetry run -C api ruff check ./api
+        run: |
+          poetry run -C api ruff --version
+          poetry run -C api ruff check ./
+          poetry run -C api ruff format --check ./
 
       - name: Dotenv check
         if: steps.changed-files.outputs.any_changed == 'true'
-        run: poetry run -C api dotenv-linter ./api/.env.example ./web/.env.example
-
-      - name: Ruff formatter check
-        if: steps.changed-files.outputs.any_changed == 'true'
-        run: poetry run -C api ruff format --check ./api
+        run: poetry run -P api dotenv-linter ./api/.env.example ./web/.env.example
 
       - name: Lint hints
         if: failure()
@@ -87,6 +82,33 @@ jobs:
         if: steps.changed-files.outputs.any_changed == 'true'
         run: yarn run lint
 
+  docker-compose-template:
+    name: Docker Compose Template
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Check changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v45
+        with:
+          files: |
+            docker/generate_docker_compose
+            docker/.env.example
+            docker/docker-compose-template.yaml
+            docker/docker-compose.yaml
+
+      - name: Generate Docker Compose
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          cd docker
+          ./generate_docker_compose
+
+      - name: Check for changes
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: git diff --exit-code
 
   superlinter:
     name: SuperLinter