--- #12768
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: ci | |
on: | |
push: | |
tags: | |
# https://semver.org/#spec-item-2 | |
- 'v[0-9]+.[0-9]+.[0-9]+' | |
# https://semver.org/#spec-item-9 | |
- 'v[0-9]+.[0-9]+.[0-9]+-beta.rc[0-9]+' | |
branches-ignore: | |
- 'translations**' | |
pull_request: | |
branches-ignore: | |
- 'translations**' | |
env: | |
# This is the version of pipenv all the steps will use | |
# If changing this, change Dockerfile | |
DEFAULT_PIP_ENV_VERSION: "2023.12.1" | |
# This is the default version of Python to use in most steps which aren't specific | |
DEFAULT_PYTHON_VERSION: "3.10" | |
jobs: | |
pre-commit: | |
# We want to run on external PRs, but not on our own internal PRs as they'll be run | |
# by the push to the branch. Without this if check, checks are duplicated since | |
# internal PRs match both the push and pull_request events. | |
if: | |
github.event_name == 'push' || github.event.pull_request.head.repo.full_name != | |
github.repository | |
name: Linting Checks | |
runs-on: ubuntu-22.04 | |
steps: | |
- | |
name: Checkout repository | |
uses: actions/checkout@v4 | |
- | |
name: Install python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
- | |
name: Check files | |
uses: pre-commit/[email protected] | |
documentation: | |
name: "Build & Deploy Documentation" | |
runs-on: ubuntu-22.04 | |
needs: | |
- pre-commit | |
steps: | |
- | |
name: Checkout | |
uses: actions/checkout@v4 | |
- | |
name: Set up Python | |
id: setup-python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
cache: "pipenv" | |
cache-dependency-path: 'Pipfile.lock' | |
- | |
name: Install pipenv | |
run: | | |
pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} | |
- | |
name: Install dependencies | |
run: | | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev | |
- | |
name: List installed Python dependencies | |
run: | | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list | |
- | |
name: Make documentation | |
run: | | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs build --config-file ./mkdocs.yml | |
- | |
name: Deploy documentation | |
if: github.event_name == 'push' && github.ref == 'refs/heads/main' | |
run: | | |
echo "docs.paperless-ngx.com" > "${{ github.workspace }}/docs/CNAME" | |
git config --global user.name "${{ github.actor }}" | |
git config --global user.email "${{ github.actor }}@users.noreply.github.com" | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} run mkdocs gh-deploy --force --no-history | |
- | |
name: Upload artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: documentation | |
path: site/ | |
retention-days: 7 | |
tests-backend: | |
name: "Backend Tests (Python ${{ matrix.python-version }})" | |
runs-on: ubuntu-22.04 | |
needs: | |
- pre-commit | |
strategy: | |
matrix: | |
python-version: ['3.9', '3.10', '3.11'] | |
fail-fast: false | |
steps: | |
- | |
name: Checkout | |
uses: actions/checkout@v4 | |
- | |
name: Start containers | |
run: | | |
docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml pull --quiet | |
docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml up --detach | |
- | |
name: Set up Python | |
id: setup-python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: "${{ matrix.python-version }}" | |
cache: "pipenv" | |
cache-dependency-path: 'Pipfile.lock' | |
- | |
name: Install pipenv | |
run: | | |
pip install --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} | |
- | |
name: Install system dependencies | |
run: | | |
sudo apt-get update -qq | |
sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript libzbar0 poppler-utils | |
- | |
name: Configure ImageMagick | |
run: | | |
sudo cp docker/imagemagick-policy.xml /etc/ImageMagick-6/policy.xml | |
- | |
name: Install Python dependencies | |
run: | | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} run python --version | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev | |
- | |
name: List installed Python dependencies | |
run: | | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} run pip list | |
- | |
name: Tests | |
env: | |
PAPERLESS_CI_TEST: 1 | |
# Enable paperless_mail testing against real server | |
PAPERLESS_MAIL_TEST_HOST: ${{ secrets.TEST_MAIL_HOST }} | |
PAPERLESS_MAIL_TEST_USER: ${{ secrets.TEST_MAIL_USER }} | |
PAPERLESS_MAIL_TEST_PASSWD: ${{ secrets.TEST_MAIL_PASSWD }} | |
run: | | |
cd src/ | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} run pytest -ra | |
- | |
name: Upload coverage | |
if: ${{ matrix.python-version == env.DEFAULT_PYTHON_VERSION }} | |
uses: actions/upload-artifact@v4 | |
with: | |
name: backend-coverage-report | |
path: src/coverage.xml | |
retention-days: 7 | |
if-no-files-found: warn | |
- | |
name: Stop containers | |
if: always() | |
run: | | |
docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml logs | |
docker compose --file ${{ github.workspace }}/docker/compose/docker-compose.ci-test.yml down | |
install-frontend-depedendencies: | |
name: "Install Frontend Dependencies" | |
runs-on: ubuntu-22.04 | |
needs: | |
- pre-commit | |
steps: | |
- uses: actions/checkout@v4 | |
- | |
name: Use Node.js 20 | |
uses: actions/setup-node@v4 | |
with: | |
node-version: 20.x | |
cache: 'npm' | |
cache-dependency-path: 'src-ui/package-lock.json' | |
- name: Cache frontend dependencies | |
id: cache-frontend-deps | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.npm | |
~/.cache | |
key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/package-lock.json') }} | |
- | |
name: Install dependencies | |
if: steps.cache-frontend-deps.outputs.cache-hit != 'true' | |
run: cd src-ui && npm ci | |
- | |
name: Install Playwright | |
if: steps.cache-frontend-deps.outputs.cache-hit != 'true' | |
run: cd src-ui && npx playwright install --with-deps | |
tests-frontend: | |
name: "Frontend Tests (Node ${{ matrix.node-version }} - ${{ matrix.shard-index }}/${{ matrix.shard-count }})" | |
runs-on: ubuntu-22.04 | |
needs: | |
- install-frontend-depedendencies | |
strategy: | |
fail-fast: false | |
matrix: | |
node-version: [20.x] | |
shard-index: [1, 2, 3, 4] | |
shard-count: [4] | |
steps: | |
- uses: actions/checkout@v4 | |
- | |
name: Use Node.js 20 | |
uses: actions/setup-node@v4 | |
with: | |
node-version: 20.x | |
cache: 'npm' | |
cache-dependency-path: 'src-ui/package-lock.json' | |
- name: Cache frontend dependencies | |
id: cache-frontend-deps | |
uses: actions/cache@v4 | |
with: | |
path: | | |
~/.npm | |
~/.cache | |
key: ${{ runner.os }}-frontenddeps-${{ hashFiles('src-ui/package-lock.json') }} | |
- name: Re-link Angular cli | |
run: cd src-ui && npm link @angular/cli | |
- | |
name: Linting checks | |
run: cd src-ui && npm run lint | |
- | |
name: Run Jest unit tests | |
run: cd src-ui && npm run test -- --max-workers=2 --shard=${{ matrix.shard-index }}/${{ matrix.shard-count }} | |
- | |
name: Upload Jest coverage | |
if: always() | |
uses: actions/upload-artifact@v4 | |
with: | |
name: jest-coverage-report-${{ matrix.shard-index }} | |
path: | | |
src-ui/coverage/coverage-final.json | |
src-ui/coverage/lcov.info | |
src-ui/coverage/clover.xml | |
retention-days: 7 | |
if-no-files-found: warn | |
- | |
name: Run Playwright e2e tests | |
run: cd src-ui && npx playwright test --shard ${{ matrix.shard-index }}/${{ matrix.shard-count }} | |
- | |
name: Upload Playwright test results | |
if: always() | |
uses: actions/upload-artifact@v4 | |
with: | |
name: playwright-report-${{ matrix.shard-index }} | |
path: src-ui/playwright-report | |
retention-days: 7 | |
tests-coverage-upload: | |
name: "Upload Coverage" | |
runs-on: ubuntu-22.04 | |
needs: | |
- tests-backend | |
- tests-frontend | |
steps: | |
- | |
uses: actions/checkout@v4 | |
- | |
name: Download frontend jest coverage | |
uses: actions/download-artifact@v4 | |
with: | |
path: src-ui/coverage/ | |
pattern: jest-coverage-report-* | |
- | |
name: Download frontend playwright coverage | |
uses: actions/download-artifact@v4 | |
with: | |
path: src-ui/coverage/ | |
pattern: playwright-report-* | |
merge-multiple: true | |
- | |
name: Upload frontend coverage to Codecov | |
uses: codecov/codecov-action@v4 | |
with: | |
# not required for public repos, but intermittently fails otherwise | |
token: ${{ secrets.CODECOV_TOKEN }} | |
flags: frontend | |
directory: src-ui/coverage/ | |
# dont include backend coverage files here | |
files: '!coverage.xml' | |
- | |
name: Download backend coverage | |
uses: actions/download-artifact@v4 | |
with: | |
name: backend-coverage-report | |
path: src/ | |
- | |
name: Upload coverage to Codecov | |
uses: codecov/codecov-action@v4 | |
with: | |
# not required for public repos, but intermittently fails otherwise | |
token: ${{ secrets.CODECOV_TOKEN }} | |
# future expansion | |
flags: backend | |
directory: src/ | |
build-docker-image: | |
name: Build Docker image for ${{ github.ref_name }} | |
runs-on: ubuntu-22.04 | |
if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || startsWith(github.ref, 'refs/heads/fix-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || contains(github.ref, 'beta.rc') || startsWith(github.ref, 'refs/tags/v')) | |
concurrency: | |
group: ${{ github.workflow }}-build-docker-image-${{ github.ref_name }} | |
cancel-in-progress: true | |
needs: | |
- tests-backend | |
- tests-frontend | |
steps: | |
- | |
name: Check pushing to Docker Hub | |
id: push-other-places | |
# Only push to Dockerhub from the main repo AND the ref is either: | |
# main | |
# dev | |
# beta | |
# a tag | |
# Otherwise forks would require a Docker Hub account and secrets setup | |
run: | | |
if [[ ${{ github.repository_owner }} == "paperless-ngx" && ( ${{ github.ref_name }} == "dev" || ${{ github.ref_name }} == "beta" || ${{ startsWith(github.ref, 'refs/tags/v') }} == "true" ) ]] ; then | |
echo "Enabling DockerHub image push" | |
echo "enable=true" >> $GITHUB_OUTPUT | |
else | |
echo "Not pushing to DockerHub" | |
echo "enable=false" >> $GITHUB_OUTPUT | |
fi | |
- | |
name: Set ghcr repository name | |
id: set-ghcr-repository | |
run: | | |
ghcr_name=$(echo "${{ github.repository }}" | awk '{ print tolower($0) }') | |
echo "Name is ${ghcr_name}" | |
echo "ghcr-repository=${ghcr_name}" >> $GITHUB_OUTPUT | |
- | |
name: Gather Docker metadata | |
id: docker-meta | |
uses: docker/metadata-action@v5 | |
with: | |
images: | | |
ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }} | |
name=paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }} | |
name=quay.io/paperlessngx/paperless-ngx,enable=${{ steps.push-other-places.outputs.enable }} | |
tags: | | |
# Tag branches with branch name | |
type=ref,event=branch | |
# Process semver tags | |
# For a tag x.y.z or vX.Y.Z, output an x.y.z and x.y image tag | |
type=semver,pattern={{version}} | |
type=semver,pattern={{major}}.{{minor}} | |
- | |
name: Checkout | |
uses: actions/checkout@v4 | |
# If https://github.com/docker/buildx/issues/1044 is resolved, | |
# the append input with a native arm64 arch could be used to | |
# significantly speed up building | |
- | |
name: Set up Docker Buildx | |
uses: docker/setup-buildx-action@v3 | |
- | |
name: Set up QEMU | |
uses: docker/setup-qemu-action@v3 | |
with: | |
platforms: arm64 | |
- | |
name: Login to GitHub Container Registry | |
uses: docker/login-action@v3 | |
with: | |
registry: ghcr.io | |
username: ${{ github.actor }} | |
password: ${{ secrets.GITHUB_TOKEN }} | |
- | |
name: Login to Docker Hub | |
uses: docker/login-action@v3 | |
# Don't attempt to login is not pushing to Docker Hub | |
if: steps.push-other-places.outputs.enable == 'true' | |
with: | |
username: ${{ secrets.DOCKERHUB_USERNAME }} | |
password: ${{ secrets.DOCKERHUB_TOKEN }} | |
- | |
name: Login to Quay.io | |
uses: docker/login-action@v3 | |
# Don't attempt to login is not pushing to Quay.io | |
if: steps.push-other-places.outputs.enable == 'true' | |
with: | |
registry: quay.io | |
username: ${{ secrets.QUAY_USERNAME }} | |
password: ${{ secrets.QUAY_ROBOT_TOKEN }} | |
- | |
name: Build and push | |
uses: docker/build-push-action@v5 | |
with: | |
context: . | |
file: ./Dockerfile | |
platforms: linux/amd64,linux/arm64 | |
push: ${{ github.event_name != 'pull_request' }} | |
tags: ${{ steps.docker-meta.outputs.tags }} | |
labels: ${{ steps.docker-meta.outputs.labels }} | |
# Get cache layers from this branch, then dev | |
# This allows new branches to get at least some cache benefits, generally from dev | |
cache-from: | | |
type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }} | |
type=registry,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:dev | |
cache-to: | | |
type=registry,mode=max,ref=ghcr.io/${{ steps.set-ghcr-repository.outputs.ghcr-repository }}/builder/cache/app:${{ github.ref_name }} | |
- | |
name: Inspect image | |
run: | | |
docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }} | |
- | |
name: Export frontend artifact from docker | |
run: | | |
docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }} | |
docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/ | |
- | |
name: Upload frontend artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: frontend-compiled | |
path: src/documents/static/frontend/ | |
retention-days: 7 | |
build-release: | |
name: "Build Release" | |
needs: | |
- build-docker-image | |
- documentation | |
runs-on: ubuntu-22.04 | |
steps: | |
- | |
name: Checkout | |
uses: actions/checkout@v4 | |
- | |
name: Set up Python | |
id: setup-python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
cache: "pipenv" | |
cache-dependency-path: 'Pipfile.lock' | |
- | |
name: Install pipenv + tools | |
run: | | |
pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel | |
- | |
name: Install Python dependencies | |
run: | | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} sync --dev | |
- | |
name: Patch whitenoise | |
run: | | |
curl --fail --silent --show-error --location --output 484.patch https://github.com/evansd/whitenoise/pull/484.patch | |
patch -d $(pipenv --venv)/lib/python3.10/site-packages --verbose -p2 < 484.patch | |
rm 484.patch | |
- | |
name: Install system dependencies | |
run: | | |
sudo apt-get update -qq | |
sudo apt-get install -qq --no-install-recommends gettext liblept5 | |
- | |
name: Download frontend artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: frontend-compiled | |
path: src/documents/static/frontend/ | |
- | |
name: Download documentation artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: documentation | |
path: docs/_build/html/ | |
- | |
name: Generate requirements file | |
run: | | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} requirements > requirements.txt | |
- | |
name: Compile messages | |
run: | | |
cd src/ | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py compilemessages | |
- | |
name: Collect static files | |
run: | | |
cd src/ | |
pipenv --python ${{ steps.setup-python.outputs.python-version }} run python3 manage.py collectstatic --no-input | |
- | |
name: Move files | |
run: | | |
echo "Making dist folders" | |
for directory in dist \ | |
dist/paperless-ngx \ | |
dist/paperless-ngx/scripts; | |
do | |
mkdir --verbose --parents ${directory} | |
done | |
echo "Copying basic files" | |
for file_name in .dockerignore \ | |
.env \ | |
Dockerfile \ | |
Pipfile \ | |
Pipfile.lock \ | |
requirements.txt \ | |
LICENSE \ | |
README.md \ | |
paperless.conf.example \ | |
gunicorn.conf.py | |
do | |
cp --verbose ${file_name} dist/paperless-ngx/ | |
done | |
mv --verbose dist/paperless-ngx/paperless.conf.example dist/paperless-ngx/paperless.conf | |
echo "Copying Docker related files" | |
cp --recursive docker/ dist/paperless-ngx/docker | |
echo "Copying startup scripts" | |
cp --verbose scripts/*.service scripts/*.sh scripts/*.socket dist/paperless-ngx/scripts/ | |
echo "Copying source files" | |
cp --recursive src/ dist/paperless-ngx/src | |
echo "Copying documentation" | |
cp --recursive docs/_build/html/ dist/paperless-ngx/docs | |
mv --verbose static dist/paperless-ngx | |
- | |
name: Make release package | |
run: | | |
echo "Creating release archive" | |
cd dist | |
sudo chown -R 1000:1000 paperless-ngx/ | |
tar -cJf paperless-ngx.tar.xz paperless-ngx/ | |
- | |
name: Upload release artifact | |
uses: actions/upload-artifact@v4 | |
with: | |
name: release | |
path: dist/paperless-ngx.tar.xz | |
retention-days: 7 | |
publish-release: | |
name: "Publish Release" | |
runs-on: ubuntu-22.04 | |
outputs: | |
prerelease: ${{ steps.get_version.outputs.prerelease }} | |
changelog: ${{ steps.create-release.outputs.body }} | |
version: ${{ steps.get_version.outputs.version }} | |
needs: | |
- build-release | |
if: github.ref_type == 'tag' && (startsWith(github.ref_name, 'v') || contains(github.ref_name, '-beta.rc')) | |
steps: | |
- | |
name: Download release artifact | |
uses: actions/download-artifact@v4 | |
with: | |
name: release | |
path: ./ | |
- | |
name: Get version | |
id: get_version | |
run: | | |
echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT | |
if [[ ${{ contains(github.ref_name, '-beta.rc') }} == 'true' ]]; then | |
echo "prerelease=true" >> $GITHUB_OUTPUT | |
else | |
echo "prerelease=false" >> $GITHUB_OUTPUT | |
fi | |
- | |
name: Create Release and Changelog | |
id: create-release | |
uses: release-drafter/release-drafter@v6 | |
with: | |
name: Paperless-ngx ${{ steps.get_version.outputs.version }} | |
tag: ${{ steps.get_version.outputs.version }} | |
version: ${{ steps.get_version.outputs.version }} | |
prerelease: ${{ steps.get_version.outputs.prerelease }} | |
publish: true # ensures release is not marked as draft | |
env: | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
- | |
name: Upload release archive | |
id: upload-release-asset | |
uses: shogo82148/actions-upload-release-asset@v1 | |
with: | |
github_token: ${{ secrets.GITHUB_TOKEN }} | |
upload_url: ${{ steps.create-release.outputs.upload_url }} | |
asset_path: ./paperless-ngx.tar.xz | |
asset_name: paperless-ngx-${{ steps.get_version.outputs.version }}.tar.xz | |
asset_content_type: application/x-xz | |
append-changelog: | |
name: "Append Changelog" | |
runs-on: ubuntu-22.04 | |
needs: | |
- publish-release | |
if: needs.publish-release.outputs.prerelease == 'false' | |
steps: | |
- | |
name: Checkout | |
uses: actions/checkout@v4 | |
with: | |
ref: main | |
- | |
name: Set up Python | |
uses: actions/setup-python@v5 | |
with: | |
python-version: ${{ env.DEFAULT_PYTHON_VERSION }} | |
cache: "pipenv" | |
cache-dependency-path: 'Pipfile.lock' | |
- | |
name: Install pipenv + tools | |
run: | | |
pip install --upgrade --user pipenv==${{ env.DEFAULT_PIP_ENV_VERSION }} setuptools wheel | |
- | |
name: Append Changelog to docs | |
id: append-Changelog | |
working-directory: docs | |
run: | | |
git branch ${{ needs.publish-release.outputs.version }}-changelog | |
git checkout ${{ needs.publish-release.outputs.version }}-changelog | |
echo -e "# Changelog\n\n${{ needs.publish-release.outputs.changelog }}\n" > changelog-new.md | |
echo "Manually linking usernames" | |
sed -i -r 's|@(.+?) \(\[#|[@\1](https://github.com/\1) ([#|ig' changelog-new.md | |
CURRENT_CHANGELOG=`tail --lines +2 changelog.md` | |
echo -e "$CURRENT_CHANGELOG" >> changelog-new.md | |
mv changelog-new.md changelog.md | |
pipenv run pre-commit run --files changelog.md || true | |
git config --global user.name "github-actions" | |
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
git commit -am "Changelog ${{ needs.publish-release.outputs.version }} - GHA" | |
git push origin ${{ needs.publish-release.outputs.version }}-changelog | |
- | |
name: Create Pull Request | |
uses: actions/github-script@v7 | |
with: | |
script: | | |
const { repo, owner } = context.repo; | |
const result = await github.rest.pulls.create({ | |
title: 'Documentation: Add ${{ needs.publish-release.outputs.version }} changelog', | |
owner, | |
repo, | |
head: '${{ needs.publish-release.outputs.version }}-changelog', | |
base: 'main', | |
body: 'This PR is auto-generated by CI.' | |
}); | |
github.rest.issues.addLabels({ | |
owner, | |
repo, | |
issue_number: result.data.number, | |
labels: ['documentation', 'skip-changelog'] | |
}); |