python-packaging.yml

# Workflow to build distribution, and push to PyPI or TestPyPI

name: TileDB-SOMA python sdist & wheels

on:
  # Trigger publication to TestPyPI via user workflow request:
  #     gh workflow run python-packaging.yml --ref branch
  # Specify the branch or other ref as required, allowing testing
  # of PRs/branches.
  workflow_dispatch:
  # Trigger publication to PyPi via new release event.
  release:
    types: [published]
  # Schedule wheel-build and smoke-test on a regular schedule.
  #
  # This will not publish to either PyPI or TestPyPI (see the conditionals
  # guarding those steps below).
  schedule:
    - cron: "42 9 * * *"

jobs:
  sdist:
    name: Build source distribution
    runs-on: ubuntu-24.04
    steps:
      - name: Checkout TileDB-SOMA
        uses: actions/checkout@v4
        with:
          fetch-depth: 0 # ensure we get all tags to inform package version determination
      - name: Install dependencies
        run: pip install --upgrade 'setuptools>=70.1' importlib-metadata  # cf. https://github.com/single-cell-data/TileDB-SOMA/pull/3076
      - run: pip list
      - name: Build sdist
        run: python setup.py sdist
        working-directory: ./apis/python
      - name: Upload sdist artifact to GitHub Actions storage
        uses: actions/upload-artifact@v4
        with:
          name: sdist
          path: apis/python/dist/

  # This step builds wheels and uploads them to GitHub Actions storage.
  # See also https://github.com/single-cell-data/TileDB-SOMA/issues/700.
  # See also https://github.com/single-cell-data/TileDB-SOMA/wiki/PyPI-packaging-WIP
  # for important transitional context.
  wheels:
    # Note: tries all supported Python versions as specified in apis/python/setup.py
    name: Build ${{ matrix.python-version }}-${{ matrix.wheel-name }} wheel
    needs: sdist
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: [ '39', '310', '311', '312' ]
        cibw_build: [ manylinux_x86_64, macosx_x86_64, macosx_arm64 ]
        include:
          - cibw_build: manylinux_x86_64
            os: ubuntu-24.04
            wheel-name: manylinux_2_28
            cc: gcc-13
            cxx: g++-13
          - cibw_build: macosx_x86_64
            os: macos-latest
            cibw_archs_macos: x86_64
            wheel-name: macos-x86_64
            cc: gcc-13
            cxx: g++-13
          - cibw_build: macosx_arm64
            os: macos-14  # Note: macos-14 is arm64 only
            cibw_archs_macos: arm64
            wheel-name: macos-arm64
            cc: gcc-13
            cxx: g++-13
    steps:
      - name: Download sdist artifact
        uses: actions/download-artifact@v4
        with:
          name: sdist
      - name: Rename sdist
        run: cp tiledbsoma-*.tar.gz tiledbsoma.tar.gz && ls -lh
      # This is crucial for ongoing debug (do not remove it) as this shows the
      # OS version as used by `pip install` to find wheel names. Importantly,
      # macos 12 and macos14 self-report as `macosx-10.9-universal2` via
      # `distutil.util.get_platform()`.
      - name: Show self-reported platform
        run: |
          echo "python --version"; python --version
          echo matrix.platform: ${{ matrix.platform }}
          echo matrix.arch: ${{ matrix.arch }}
          # This bit is crucial since it's used to match up to a component of the wheel-file name
          pip install setuptools
          python -c 'from distutils import util; print("distutil.util.get_platform:", util.get_platform())'
          python -c 'import platform; print("platform.platform()", platform.platform())'
      - name: Select XCode version
        if: startsWith(matrix.os, 'macos')
        uses: maxim-lobanov/setup-xcode@v1
        with:
          xcode-version: '15.4'
      - name: Build wheels
        uses: pypa/cibuildwheel@v2.18.1
        with:
          package-dir: tiledbsoma.tar.gz
          only: cp${{ matrix.python-version }}-${{ matrix.cibw_build }}
          cc: gcc-13
          cxx: g++-13
        env:
          CIBW_ENVIRONMENT_MACOS: CC=gcc-13 CXX=g++-13
          CIBW_BUILD: ${{ matrix.cibw_build }}
          CIBW_BUILD_VERBOSITY: 1
          CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28
          CIBW_BEFORE_BUILD_LINUX: yum -y remove gcc-toolset-12\*; yum -y install gcc-toolset-13; bash -x -c 'rm -rf tiledbsoma*/dist_links/dist/lib*'
          # ^ Delete lib folder that apis/python/setup.py:find_or_build() looks for in deciding to
          #   run CMake build or not. Otherwise it'll keep reusing the library file built in the
          #   first iteration of cibuildwheel's outer loop, resulting in wheels with the library
          #   built for the wrong python version.
          CIBW_BEFORE_BUILD: bash -x -c 'rm -rf tiledbsoma*/dist_links/dist/lib*'
          # ^ Delete lib folder that apis/python/setup.py:find_or_build() looks for in deciding to
          #   run CMake build or not. Otherwise it'll keep reusing the library file built in the
          #   first iteration of cibuildwheel's outer loop, resulting in wheels with the library
          #   built for the wrong python version.
          CIBW_ARCHS_MACOS: ${{ matrix.cibw_archs_macos }}
          CIBW_ENVIRONMENT_LINUX : CC=/opt/rh/gcc-toolset-13/root/usr/bin/gcc CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++
          CIBW_TEST_SKIP: "*_arm64"
          CMAKE_OSX_ARCHITECTURES: ${{ matrix.cibw_archs_macos }}
          MACOSX_DEPLOYMENT_TARGET: "11.0"

      - name: Upload wheel-${{ matrix.wheel-name }}-${{ matrix.python-version }} to GitHub Actions storage
        uses: actions/upload-artifact@v4
        with:
          name: wheel-${{ matrix.wheel-name }}-${{ matrix.python-version }}
          path: ./wheelhouse/*.whl

  # This step locally tries out the built wheels, without publishing to PyPI
  smoke-test:
    name: "Smoke test ${{ matrix.python.undotted-version }}-${{ matrix.wheel-name }} wheel"
    needs: wheels
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python:
          - undotted-version: '310'
            dotted-version: '3.10'
          - undotted-version: '311'
            dotted-version: '3.11'
          - undotted-version: '312'
            dotted-version: '3.12'
        wheel-name:
          - manylinux_2_28
          - macos-x86_64
          - macos-arm64
        include:
          - wheel-name: manylinux_2_28
            os: ubuntu-24.04
            arch: x86_64
            cc: gcc-13
            cxx: g++-13
          - wheel-name: macos-x86_64
            os: macos-13
            arch: x86_64
            # cc: clang
            # cxx: clang++
            cc: gcc-13
            cxx: g++-13
          - wheel-name: macos-arm64
            os: macos-14
            arch: arm64
            # cc: clang
            # cxx: clang++
            cc: gcc-13
            cxx: g++-13
      fail-fast: false
    steps:
      - name: Set up Python ${{ matrix.python.dotted-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python.dotted-version }}
      - name: Download artifacts
        uses: actions/download-artifact@v4
        with:
          name: wheel-${{ matrix.wheel-name }}-${{ matrix.python.undotted-version }}
      - name: Install wheel
        run: |
          set -x
          ls -lR
          WHL=$(find . -name 'tiledbsoma-*-cp${{ matrix.python.undotted-version }}-cp${{ matrix.python.undotted-version }}-*_${{ matrix.arch }}.whl')
          echo "WHL=$WHL"
          if [ -z "$WHL" ]; then echo "No wheel found"; exit 1; fi
          unzip -l $WHL
          pip install wheel
          pip install $WHL
          echo "WHL=$WHL" >> $GITHUB_ENV
      - name: Smoke test ${{ matrix.os }}
        run: python -c 'import tiledbsoma; print(tiledbsoma.pytiledbsoma.__file__); tiledbsoma.show_package_versions()'
        # TODO: more thorough local smoke test
      - name: Smoke test in docker
        if: ${{ matrix.os == 'ubuntu-24.04' }}
        run: |
          docker run -v $(pwd):/mnt python:${{ matrix.python.dotted-version }} bash -ec "
            apt-get -qq update && apt-get install -y python3-pip python3-wheel
            pip3 install /mnt/$WHL
            python3 -c 'import tiledbsoma; print(tiledbsoma.pytiledbsoma.__file__); tiledbsoma.show_package_versions()'
          "

  # Publish to TestPyPI upon user workflow request
  publish-to-test-pypi:
    name: Publish package to TestPyPI
    needs: smoke-test
    runs-on: ubuntu-24.04
    if: github.event_name == 'workflow_dispatch'
    steps:
      - name: Download artifacts
        uses: actions/download-artifact@v4
      - name: Create dist
        run: |
          set -x
          mkdir dist
          cp sdist/tiledbsoma-*.tar.gz wheel-*/*.whl dist
          ls -l dist
      - name: Publish packages to TestPyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        continue-on-error: true
        with:
          password: ${{ secrets.TEST_PYPI_TOKEN }}
          repository-url: https://test.pypi.org/legacy/
          packages_dir: dist
          verbose: true

  # Publish to PyPI upon release.
  publish-to-pypi:
    name: Publish package to PyPI
    needs: smoke-test
    runs-on: ubuntu-24.04
    if: github.event_name == 'release'
    steps:
      - name: Download artifacts
        uses: actions/download-artifact@v4
      - name: Create dist
        run: |
          set -x
          mkdir dist
          cp sdist/tiledbsoma-*.tar.gz wheel-*/*.whl dist
          ls -l dist
      - name: Publish packages to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_TOKEN }}
          packages_dir: dist
          verbose: true

  # File a bug report if anything fails, but don't file tickets for manual runs
  # -- only for scheduled ones.
  create_issue_on_fail:
    runs-on: ubuntu-24.04
    needs: [smoke-test, publish-to-test-pypi, publish-to-pypi]
    if: (failure() || cancelled()) && github.event_name != 'workflow_dispatch'
    steps:
      - name: Checkout TileDB-SOMA `main`
        uses: actions/checkout@v2
      - name: Create Issue if Build Fails
        uses: JasonEtco/create-an-issue@v2
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          filename: .github/workflows/daily-test-build-issue-template.md