Skip to content
Python

Python #1802

Sign in to view logs

Python

Python #1802

Workflow file for this run

.github/workflows/python.yml at 6795b1b
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
name: Python
on:
push:
pull_request:
schedule:
# Run daily at 00:00 so we get notified if CI is broken before a pull request is submitted.
# It also notifies us about new Arrow releases for which we need to release a corresponding version of PalletJack.
- cron: '0 0 * * *'
permissions:
contents: read
jobs:
test:
if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
runs-on: ubuntu-latest
strategy:
matrix:
mode: [None, IOUringReader1, ReadIntoMemoryIOUring, ReadIntoMemoryIOUring_ODirect]
fail-fast: false
steps:
- uses: actions/checkout@v5
- name: Set up Python 3.12
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Set environment variables
run: |
if [[ "${{ matrix.mode }}" == "IOUringReader1" ]]; then
echo "JJ_experimental_io_uring_mode=IOUringReader1" >> $GITHUB_ENV
elif [[ "${{ matrix.mode }}" == "ReadIntoMemoryIOUring" ]]; then
echo "JJ_experimental_io_uring_mode=ReadIntoMemoryIOUring" >> $GITHUB_ENV
elif [[ "${{ matrix.mode }}" == "ReadIntoMemoryIOUring_ODirect" ]]; then
echo "JJ_experimental_io_uring_mode=ReadIntoMemoryIOUring" >> $GITHUB_ENV
echo "JJ_experimental_O_DIRECT=ON" >> $GITHUB_ENV
fi
- name: Install dependencies
shell: bash
run: |
python3 -m pip install --upgrade pip
python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu
python3 -m pip install -r requirements.txt
python3 -m pip install flake8 pytest
- name: Lint with flake8
run: |
# stop the build if there are Python syntax errors or undefined names
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Test with pytest
run: |
sudo apt-get install liburing-dev
pip install --editable .
pytest
build-wheels:
if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
runs-on: ${{ matrix.runner }}
strategy:
matrix:
runner: [ubuntu-latest, windows-latest, macos-latest]
fail-fast: false
steps:
- uses: actions/checkout@v5
- name: Set up QEMU
if: runner.os == 'Linux'
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: Set up Python 3.12
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Install dependencies
shell: bash
run: |
python3 -m pip install --upgrade pip
python3 -m pip install -r requirements.txt
python3 -m pip install flake8 pytest cibuildwheel build
- name: Build sdist (Linux)
if: runner.os == 'linux'
run: |
python3 -m build --sdist
- name: Build wheels
run: python3 -m cibuildwheel --output-dir dist
env:
CIBW_BUILD_VERBOSITY: 1
- uses: actions/upload-artifact@v5
with:
name: dist-${{ matrix.runner }}
path: ./dist/*
test-binary:
needs: build-wheels
runs-on: ${{ matrix.runner }}
strategy:
matrix:
runner: [ubuntu-latest, windows-latest, macos-latest]
fail-fast: false
steps:
- uses: actions/checkout@v5
- name: Set up Python 3.12
uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Download artifacts
uses: actions/download-artifact@v6
with:
pattern: dist-*
path: dist
merge-multiple: true
- name: Setup torch
run: |
python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu
- name: Test with pytest
run: |
python3 -m pip install -r requirements.txt
python3 -m pip install pytest
# Keep in mind that if the local and remote versions are the same, the remote version will be installed
python3 -m pip install JollyJack --pre --find-links ./dist --break-system-packages --only-binary=:all:
# So now ensure that the local version is installed
python3 -m pip install JollyJack --pre --find-links ./dist --break-system-packages --only-binary=:all: --force-reinstall --no-index --no-deps
python3 test/test_jollyjack.py
test-binary-manylinux:
needs: build-wheels
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["cp310-cp310", "cp311-cp311", "cp312-cp312", "cp313-cp313", "cp314-cp314"]
fail-fast: false
steps:
- name: Checkout code
uses: actions/checkout@v5
- name: Download artifacts
uses: actions/download-artifact@v6
with:
pattern: dist-*
path: dist
merge-multiple: true
- name: Build and Run Docker Container for Tests
run: |
# Build the Docker image with Python version as build arg
docker build \
--build-arg PYTHON_VERSION=${{ matrix.python-version }} \
-t jollyjack-test-py${{ matrix.python-version }} \
--progress plain \
-f Dockerfile.test .
docker run jollyjack-test-py${{ matrix.python-version }}
# Virtual job that can be configured as a required check before a PR can be merged.
# As GitHub considers a check as successful if it is skipped, we need to check its status in
# another workflow (check-required.yml) and create a check there.
all-required-checks-done:
name: All required checks done
needs:
- test
- build-wheels
- test-binary
- test-binary-manylinux
runs-on: ubuntu-latest
steps:
- run: echo "All required checks done"
benchmarks:
needs: build-wheels
strategy:
matrix:
os: [ubuntu-latest, windows-latest]
mode: [CPU, FILE_SYSTEM]
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v5
- name: Set up Python 3.10
uses: actions/setup-python@v6
with:
python-version: "3.10"
- name: Download artifacts
uses: actions/download-artifact@v6
with:
pattern: dist-*
path: dist
merge-multiple: true
- name: Setup torch
run: |
python3 -m pip install torch --index-url https://download.pytorch.org/whl/cpu
- name: Run benchmarks
run: |
python3 -m pip install -r requirements.txt
# Keep in mind that if the local and remote versions are the same, the remote version will be installed
pip install JollyJack --pre --find-links ./dist --break-system-packages --only-binary=:all:
# So now ensure that the local version is installed
pip install JollyJack --pre --find-links ./dist --break-system-packages --only-binary=:all: --force-reinstall --no-index --no-deps
python3 ./benchmarks/benchmark_jollyjack.py
env:
JJ_benchmark_mode: ${{ matrix.mode }}
publish:
if: ${{ !github.event.repository.fork && startsWith(github.ref, 'refs/tags/v') }}
needs: [all-required-checks-done]
runs-on: ubuntu-latest
environment: release
permissions:
id-token: write
steps:
- name: Download artifacts
uses: actions/download-artifact@v6
with:
pattern: dist-*
path: dist
merge-multiple: true
- name: Display structure of downloaded files
run: ls -R dist
- name: Validate tag
shell: pwsh
run: |
$tag = "${{ github.ref }}".SubString(11)
$expectedFile = "dist/jollyjack-$tag.tar.gz"
# Check whether the tag and the package version match together
if (-not (Test-Path -Path $expectedFile)) {
echo "::error ::Expected file $expectedFile doesn't exist"
Get-ChildItem -Path dist
exit 1
}
- name: Publish to PyPi
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e
with:
packages-dir: dist