automate building Release Candidate in github action (#1391)

Related to #1306 and #872 Devlist discussion: https://lists.apache.org/thread/oowhcfwv3fcjzdzm76tbn99k5q84mr75 This PR changes `.github/workflows/python-release.yml` to build the artifacts for both svn and pypi and update the "how to release doc" to match the new process.
apache · Feb 2, 2025 · fa78256 · fa78256
1 parent 29c9a96
commit fa78256
Show file tree

Hide file tree

Showing 5 changed files with 253 additions and 80 deletions.
diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml
@@ -17,29 +17,118 @@
 # under the License.
 #
 
-name: "Python Release"
+name: "Python Build Release Candidate"
 
 on:
+  push:
+    tags:
+      # Trigger this workflow when tag follows the versioning format: pyiceberg-<major>.<minor>.<patch>rc<release_candidate>
+      # Example valid tags: pyiceberg-0.8.0rc2, pyiceberg-1.0.0rc1
+      - 'pyiceberg-[0-9]+.[0-9]+.[0-9]+rc[0-9]+'
   workflow_dispatch:
     inputs:
       version:
-        description: 'Version'
+        description: 'Version (e.g., 0.8.0)'
         type: string
-        default: 'main'
-
+        required: true
+      rc:
+        description: 'Release Candidate (RC) (e.g., 1)'
+        type: number
+        required: true
 
 jobs:
-  build_wheels:
-    name: Build wheels on ${{ matrix.os }}
+  validate-inputs:
+    runs-on: ubuntu-latest
+    outputs:
+      VERSION: ${{ steps.validate-inputs.outputs.VERSION }} # e.g. 0.8.0
+      RC: ${{ steps.validate-inputs.outputs.RC }} # e.g. 1
+    steps:
+      - name: Validate and Extract Version and RC
+        id: validate-inputs
+        run: |
+          if [ "$GITHUB_EVENT_NAME" = "push" ]; then
+            echo "Workflow triggered by tag push."
+            TAG=${GITHUB_REF#refs/tags/} # Extract the tag name
+            VERSION_AND_RC=${TAG#pyiceberg-} # Remove the 'pyiceberg-' prefix
+            VERSION=${VERSION_AND_RC%rc*} # Extract VERSION by removing everything after 'rc'
+            RC=${VERSION_AND_RC#*rc} # Extract RC by keeping everything after 'rc'
+
+            if [[ -z "$VERSION" || -z "$RC" ]]; then
+              echo "Error: Unable to parse VERSION or RC from tag ($TAG). Ensure the tag format is correct."
+              exit 1
+            fi
+          else
+            echo "Workflow triggered manually via workflow_dispatch."
+            VERSION="${{ github.event.inputs.version }}"
+            RC="${{ github.event.inputs.rc }}"
+
+            # Validate version (e.g., 1.0.0)
+            if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+              echo "Error: version ($VERSION) must be in the format: <number>.<number>.<number>"
+              exit 1
+            fi
+
+            # Validate rc (e.g., 1)
+            if [[ ! "$RC" =~ ^[0-9]+$ ]]; then
+              echo "Error: rc ($RC) must be in the format: <number>"
+              exit 1
+            fi
+          fi
+
+          # Export variables for future steps
+          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
+          echo "RC=$RC" >> $GITHUB_OUTPUT
+
+      - name: Display Extracted Version and RC
+        run: |
+          echo "Using Version: ${{ steps.validate-inputs.outputs.VERSION }}"
+          echo "Using RC: ${{ steps.validate-inputs.outputs.RC }}"
+
+  validate-library-version:
+    runs-on: ubuntu-latest
+    needs:
+      - validate-inputs
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+
+      - name: Install Poetry
+        run: make install-poetry
+
+      - name: Validate current pyiceberg version
+        env:
+          VERSION: ${{ needs.validate-inputs.outputs.VERSION }}
+        run: |
+          # Extract the current version from Poetry
+          current_pyiceberg_version=$(poetry version --short)
+          echo "Detected Poetry version: $current_pyiceberg_version"
+
+          # Compare the input version with the Poetry version
+          if [[ "$VERSION" != "$current_pyiceberg_version" ]]; then
+            echo "Error: Input version ($VERSION) does not match the Poetry version ($current_pyiceberg_version)"
+            exit 1
+          fi
+
+  # SVN
+  svn-build-artifacts:
+    name: Build artifacts for SVN on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    needs:
+      - validate-inputs
+      - validate-library-version
     strategy:
       matrix:
         os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
 
     steps:
       - uses: actions/checkout@v4
         with:
-          fetch-depth: 0
+          fetch-depth: 1
 
       - uses: actions/setup-python@v5
         with:
@@ -50,11 +139,86 @@ jobs:
             3.12
 
       - name: Install poetry
-        run: pip install poetry
+        run: make install-poetry
 
-      - name: Set version
-        run: python -m poetry version "${{ inputs.version }}"
-        if: "${{ github.event.inputs.version != 'main' }}"
+      # Publish the source distribution with the version that's in
+      # the repository, otherwise the tests will fail
+      - name: Compile source distribution
+        run: python3 -m poetry build --format=sdist
+        if: startsWith(matrix.os, 'ubuntu')
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        with:
+          output-dir: wheelhouse
+          config-file: "pyproject.toml"
+        env:
+          # Ignore 32 bit architectures
+          CIBW_ARCHS: "auto64"
+          CIBW_PROJECT_REQUIRES_PYTHON: ">=3.9,<3.13"
+          CIBW_TEST_REQUIRES: "pytest==7.4.2 moto==5.0.1"
+          CIBW_TEST_EXTRAS: "s3fs,glue"
+          CIBW_TEST_COMMAND: "pytest {project}/tests/avro/test_decoder.py"
+          # There is an upstream issue with installing on MacOSX
+          # https://github.com/pypa/cibuildwheel/issues/1603
+          # Ignore tests for pypy since not all dependencies are compiled for it
+          # and would require a local rust build chain
+          CIBW_TEST_SKIP: "pp* *macosx*"
+
+      - name: Add source distribution
+        if: startsWith(matrix.os, 'ubuntu')
+        run: ls -lah dist/* && cp dist/* wheelhouse/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: "svn-release-candidate-${{ matrix.os }}"
+          path: ./wheelhouse/*
+
+  svn-merge-artifacts:
+    runs-on: ubuntu-latest
+    needs:
+      - validate-inputs
+      - svn-build-artifacts
+    steps:
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: "svn-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
+          pattern: svn-release-candidate*
+          delete-merged: true
+
+  # PyPi
+  pypi-build-artifacts:
+    name: Build artifacts for PyPi on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    needs:
+      - validate-inputs
+      - validate-library-version
+    strategy:
+      matrix:
+        os: [ ubuntu-22.04, windows-2022, macos-13, macos-14, macos-15 ]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: |
+            3.9
+            3.10
+            3.11
+            3.12
+
+      - name: Install poetry
+        run: make install-poetry
+
+      - name: Set version with RC
+        env:
+          VERSION: ${{ needs.validate-inputs.outputs.VERSION }}
+          RC: ${{ needs.validate-inputs.outputs.RC }}
+        run: python -m poetry version "${{ env.VERSION }}rc${{ env.RC }}" # e.g., 0.8.0rc1
 
       # Publish the source distribution with the version that's in
       # the repository, otherwise the tests will fail
@@ -86,15 +250,18 @@ jobs:
 
       - uses: actions/upload-artifact@v4
         with:
-          name: "release-${{ matrix.os }}"
+          name: "pypi-release-candidate-${{ matrix.os }}"
           path: ./wheelhouse/*
-  merge:
+
+  pypi-merge-artifacts:
     runs-on: ubuntu-latest
-    needs: build_wheels
+    needs:
+      - validate-inputs
+      - pypi-build-artifacts
     steps:
       - name: Merge Artifacts
         uses: actions/upload-artifact/merge@v4
         with:
-          name: "release-${{ github.event.inputs.version }}"
-          pattern: release-*
+          name: "pypi-release-candidate-${{ needs.validate-inputs.outputs.VERSION }}rc${{ needs.validate-inputs.outputs.RC }}"
+          pattern: pypi-release-candidate*
           delete-merged: true
diff --git a/mkdocs/docs/api.md b/mkdocs/docs/api.md
@@ -1496,7 +1496,7 @@ print(ray_dataset.take(2))
 
 ### Daft
 
-PyIceberg interfaces closely with Daft Dataframes (see also: [Daft integration with Iceberg](https://www.getdaft.io/projects/docs/en/latest/user_guide/integrations/iceberg.html)) which provides a full lazily optimized query engine interface on top of PyIceberg tables.
+PyIceberg interfaces closely with Daft Dataframes (see also: [Daft integration with Iceberg](https://www.getdaft.io/projects/docs/en/stable/integrations/iceberg/)) which provides a full lazily optimized query engine interface on top of PyIceberg tables.
 
 <!-- prettier-ignore-start -->
 

diff --git a/mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png b/mkdocs/docs/assets/images/ghactions-run-workflow-pypi-upload.png
diff --git a/mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png b/mkdocs/docs/assets/images/ghactions-run-workflow-svn-upload.png