CI: build Linux arm64 in a CentOS 7 container (#570)

- unify Linux native build CI
- remove vcpkg caching as it is not working anymore
- limit parallelization to the amount of logical cpu cores
- remove ability to cross-compile as it is currently broken with jemalloc/mimalloc
- bump the minimum required version of cmake to align with vcpkg

Author: jgiannuzzi

.github/workflows/ci.yml

@@ -18,8 +18,6 @@ permissions:
 
 jobs:
 
-  # Note that vcpkg dependencies takes the majority of the build time.
-  # We cache them using GitHub Actions cache, making the scripts below a bit more complex.
   check-format:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
     name: Check format
@@ -90,12 +88,6 @@ jobs:
         echo "root=$VCPKG_INSTALLATION_ROOT" >> $GITHUB_OUTPUT
       shell: bash
 
-    # Get cmake version, which is used by the CentOS 7 container.
-    - name: Get cmake version
-      id: cmake-info
-      run: echo "version=$(cmake --version | head -n1 | awk '{print $3}')" >> $GITHUB_OUTPUT
-      shell: bash
-
     # Ensure vcpkg builtin registry is up-to-date
     - name: Update vcpkg builtin registry
       working-directory: ${{ steps.vcpkg-info.outputs.root }}
@@ -116,22 +108,13 @@ jobs:
       run: |
         ./bootstrap-vcpkg.bat
 
-    # Setup a CentOS 7 container to build on Linux x64 for backwards compatibility.
+    # Setup a CentOS 7 container to build on Linux for backwards compatibility.
     - name: Start CentOS container and install toolchain
-      if: runner.os == 'Linux' && matrix.arch == 'x64'
+      if: runner.os == 'Linux'
       run: |
-        docker run -d --name centos --entrypoint tail -v $PWD:$PWD -v $VCPKG_INSTALLATION_ROOT:$VCPKG_INSTALLATION_ROOT quay.io/pypa/manylinux2014_x86_64 -f /dev/null
+        docker run -d --name centos --entrypoint tail -v $PWD:$PWD -v $VCPKG_INSTALLATION_ROOT:$VCPKG_INSTALLATION_ROOT quay.io/pypa/manylinux2014_$(uname -m) -f /dev/null
         docker exec centos sh -c "yum install -y devtoolset-10 rh-git227 httpd24-curl flex bison perl-Data-Dumper perl-IPC-Cmd && \
-                                  curl -fsSL -o /tmp/cmake.sh https://github.com/Kitware/CMake/releases/download/v${{ steps.cmake-info.outputs.version }}/cmake-${{ steps.cmake-info.outputs.version }}-linux-x86_64.sh && \
-                                  sh /tmp/cmake.sh --skip-license --prefix=/usr/local && \
-                                  rm /tmp/cmake.sh"
-
-    # Install arm64 cross-compilation toolchain if required
-    - name: Install arm64 cross-compilation toolchain
-      if: runner.os == 'Linux' && matrix.arch == 'arm64'
-      run: |
-        sudo apt-get update
-        sudo apt install g++-aarch64-linux-gnu
+                                  uv tool install ninja"
 
     # Install vcpkg dependencies
     - name: Install vcpkg build dependencies (macOS)
@@ -147,25 +130,19 @@ jobs:
       if: runner.os == 'Windows'
       uses: microsoft/setup-msbuild@v2
 
-    # Expose GitHub Runtime environment variables for vcpkg caching.
-    - name: Expose GitHub Runtime
-      uses: crazy-max/ghaction-github-runtime@v3
-
     # Compile ParquetSharp and C++ dependencies (and upload the native library as an artifact).
-    - name: Compile native ParquetSharp library (Unix)
-      if: runner.os == 'Linux' || runner.os == 'macOS'
+    - name: Compile native ParquetSharp library (Linux)
+      if: runner.os == 'Linux'
       run: |
-        if [ "${{ runner.os }}" == "Linux" ] && [ "${{ matrix.arch }}" == "x64" ]; then
-          exec="docker exec -w $PWD -e GITHUB_ACTIONS -e ACTIONS_CACHE_URL -e ACTIONS_RUNTIME_TOKEN -e VCPKG_BINARY_SOURCES -e VCPKG_INSTALLATION_ROOT centos scl enable devtoolset-10 rh-git227 httpd24 --"
-        fi
-        $exec ./build_unix.sh ${{ matrix.arch }}
-      env:
-        VCPKG_BINARY_SOURCES: clear;x-gha,readwrite
+        docker exec -w $PWD -e GITHUB_ACTIONS -e VCPKG_INSTALLATION_ROOT centos \
+          scl enable devtoolset-10 rh-git227 httpd24 -- \
+          sh -c 'PATH=$(uv tool dir --bin):$PATH ./build_unix.sh'
+    - name: Compile native ParquetSharp library (macOS)
+      if: runner.os == 'macOS'
+      run: ./build_unix.sh
     - name: Compile native ParquetSharp library (Windows)
       if: runner.os == 'Windows'
       run: ./build_windows.ps1
-      env:
-        VCPKG_BINARY_SOURCES: clear;x-gha,readwrite
     - name: Upload vcpkg arrow logs
       if: success() || failure()
       uses: actions/upload-artifact@v4
@@ -184,7 +161,7 @@ jobs:
         path: bin
 
     - name: Stop CentOS container
-      if: runner.os == 'Linux' && matrix.arch == 'x64'
+      if: runner.os == 'Linux'
       run: docker rm -f centos
 
   # Download all native shared libraries and create the nuget package.

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.8)
+cmake_minimum_required(VERSION 3.10)
 project(ParquetSharp)
 
 # Exclude MinSizeRel and RelWithDebugInfo, to simplify integration with C#

build_unix.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -e
 
-case ${1:-$(uname -m)} in
+case $(uname -m) in
   x86_64|x64)
     vcpkg_arch=x64
     linux_arch=x86_64
@@ -21,15 +21,11 @@ esac
 case $(uname) in
   Linux)
     os=linux
-    [ -f /etc/redhat-release ] && platform=redhat-linux || platform=linux-gnu
-    options="-D CMAKE_SYSTEM_PROCESSOR=$linux_arch \
-             -D CMAKE_C_COMPILER=$(which $linux_arch-$platform-gcc) \
-             -D CMAKE_CXX_COMPILER=$(which $linux_arch-$platform-g++) \
-             -D CMAKE_STRIP=$(which $linux_arch-$platform-strip 2>/dev/null || which strip)"
+    nproc=$(nproc)
     ;;
   Darwin)
     os=osx
-    options="-D CMAKE_OSX_ARCHITECTURES=$osx_arch"
+    nproc=$(sysctl -n hw.logicalcpu)
     if ! which brew >/dev/null || [ ! -x $(brew --prefix)/opt/bison/bin/bison ]
     then
       echo 'The version of bison provided with macOS is too old.'
@@ -86,5 +82,5 @@ do
   echo ">> Building ParquetSharpNative $build_type for $triplet"
   build_dir=build/$triplet-$(tr A-Z a-z <<<$build_type)
   cmake -B $build_dir -S . -D VCPKG_TARGET_TRIPLET=$triplet -D CMAKE_TOOLCHAIN_FILE=$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake -D CMAKE_BUILD_TYPE=$build_type $options
-  cmake --build $build_dir -j
+  cmake --build $build_dir -j ${nproc}
 done