Utilizing docker scripts in github workflows

TimSchneider42 · May 25, 2024 · 06fafd4 · 06fafd4
1 parent c3e66e7
commit 06fafd4
Show file tree

Hide file tree

Showing 9 changed files with 28 additions and 137 deletions.
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
@@ -20,43 +20,14 @@ jobs:
       - name: Install apt dependencies
         run: |
           sudo apt-get update
-          sudo apt-get -y install build-essential cmake git libpoco-dev catch2 python3-dev
+          sudo apt-get -y install build-essential cmake git libpoco-dev python3-dev
 
-      - name: Install Eigen3
+      - name: Install other dependencies
         run: |
-          git clone https://gitlab.com/libeigen/eigen.git
-          cd eigen
-          git checkout 3.4.0
-          mkdir build && cd build
-          cmake ..
-          sudo make install
-
-      - name: Install libfranka
-        run: |
-          git clone --recursive https://github.com/frankaemika/libfranka.git
-          cd libfranka
-          git checkout ${{ inputs.libfranka-version }}
-          
-          # Cherry-pick missing imports commit
-          git config user.email "[email protected]"
-          git config user.name "John Doe"
-          if ! git merge-base --is-ancestor f1f46fb HEAD; then git cherry-pick -m 1 f1f46fb; fi
-          
-          git submodule update
-          mkdir build && cd build
-          cmake -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF ..
-          make -j2
-          sudo make install
-
-      - name: Install pybind11
-        run: |
-          git clone https://github.com/pybind/pybind11.git
-          cd pybind11
-          git checkout v2.9.1
-          mkdir build && cd build
-          cmake -DPYBIND11_TEST=OFF ..
-          make -j2
-          sudo make install
+          set -a
+          . ./dependency_versions
+          set +a
+          ./docker/common/install-dependencies
 
       - name: Configure & make
         run: |

diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -23,110 +23,21 @@ jobs:
         with:
           submodules: recursive
 
-      - name: Log Python versions
+      - name: Setting up pip
         run: |
-          echo "Found the following Python versions:"
-          for PYBIN in /opt/python/*/bin; do
-            if [[ "${PYBIN}" =~ ${PYBIN_SUPPORTED_VERSIONS} ]]; then
-              echo "${PYBIN}" | cut -d / -f 4
-            fi
-          done
+          ./docker/common/manylinux-setup-pip
 
-      - name: Install build essentials
+      - name: Install dependencies
         run: |
-          yum -y install poco-devel*
-          
-          for PYBIN in /opt/python/*/bin; do
-            if [[ "${PYBIN}" =~ ${PYBIN_SUPPORTED_VERSIONS} ]]; then
-              "${PYBIN}/pip" install cmake setuptools wheel --user
-            fi
-          done
-          
-          ln -f -s $HOME/.local/bin/cmake /usr/bin/cmake
-
-      - name: Install Eigen3
-        run: |
-          git clone https://gitlab.com/libeigen/eigen.git
-          cd eigen
-          git checkout 3.4.0
-          mkdir build && cd build
-          cmake ..
-          make install -j2
-
-      - name: Install pybind11
-        run: |
-          git clone https://github.com/pybind/pybind11.git
-          cd pybind11
-          git checkout v2.11.1
-          mkdir build && cd build
-          cmake -DPYBIND11_TEST=OFF ..
-          make -j2
-          make install
-
-      - name: Install Catch2
-        run: |
-          git clone https://github.com/catchorg/Catch2.git
-          cd Catch2
-          git checkout v2.13.8
-          mkdir build && cd build
-          cmake -DCATCH_BUILD_TESTING=OFF -DCATCH_ENABLE_WERROR=OFF -DCATCH_INSTALL_DOCS=OFF -DCATCH_INSTALL_HELPERS=OFF ..
-          make install
-
-      - name: Install libfranka
-        run: |
-          git clone --recursive https://github.com/frankaemika/libfranka.git
-          cd libfranka
-          git checkout ${{ inputs.libfranka-version }}
-          
-          # Cherry-pick missing imports commit
-          git config user.email "[email protected]"
-          git config user.name "John Doe"
-          if ! git merge-base --is-ancestor f1f46fb HEAD; then git cherry-pick -m 1 f1f46fb; fi
-          
-          git submodule update
-          mkdir build && cd build
-          cmake -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF ..
-          make -j2
-          make install
+          set -a 
+          . ./dependency_versions
+          set +a
+          ./docker/common/install-dependencies
 
       - name: Python package
         run: |
           yum -y install zip
-          mkdir -p wheels
-          mkdir -p dist
-          mkdir -p dist_full
-          
-          # Compile wheels
-          for PYBIN in /opt/python/*/bin; do
-            if [[ "${PYBIN}" =~ ${PYBIN_SUPPORTED_VERSIONS} ]]; then
-              "${PYBIN}/pip" wheel . -w wheels/
-            fi
-          done
-          
-          # Bundle external shared libraries into the wheels
-          for whl in wheels/*.whl; do
-            if [[ "$whl" =~ wheels/franky_panda-.*\.whl ]]; then
-              auditwheel repair "$whl" -w dist_full/
-            else
-              cp "$whl" dist_full/
-            fi
-          done
-          
-          # Install packages and test
-          for PYBIN in /opt/python/*/bin/; do
-            if [[ "${PYBIN}" =~ ${PYBIN_SUPPORTED_VERSIONS} ]]; then
-              "${PYBIN}/pip" install franky-panda --no-index -f dist_full
-              # (cd "$HOME"; "${PYBIN}/nosetests" -w /io/tests)
-            fi
-          done
-          
-          cp dist_full/franky_panda-*.whl dist/
-          
-          ls dist
-          
-          mkdir -p output
-          libfranka_version=${{ inputs.libfranka-version }}
-          zip -r "output/libfranka_${libfranka_version//./-}_wheels.zip" dist/
+          ./docker/common/manylinux-build-wheels
 
       - name: Upload wheels
         uses: actions/upload-artifact@v3

diff --git a/docker/build/Dockerfile b/docker/build/Dockerfile
@@ -11,6 +11,8 @@ RUN set -a && . /tmp/dependency_versions && set +a \
     && /tmp/common/install-dependencies
 
 ARG PYBIN_SUPPORTED_VERSIONS="cp3(7m|8|9|10|11|12)"
+# Preserve argument for runtime
+ENV PYBIN_SUPPORTED_VERSIONS=${PYBIN_SUPPORTED_VERSIONS}
 RUN /tmp/common/manylinux-setup-pip
 
 # Set working directory
@@ -19,7 +21,7 @@ RUN mkdir -p /build
 VOLUME /code
 VOLUME /build
 WORKDIR /build
-ADD docker/build/build-wheels /usr/bin/
+ADD docker/common/manylinux-build-wheels /usr/bin/build-wheels
 ADD docker/build/run-tests /usr/bin/
 
 # Cleaning up

diff --git a/docker/common/install-catch2 b/docker/common/install-catch2
@@ -9,7 +9,7 @@ git checkout "v${CATCH2_VERSION}"
 mkdir build
 cd build
 cmake -DCATCH_BUILD_TESTING=OFF -DCATCH_ENABLE_WERROR=OFF -DCATCH_INSTALL_DOCS=OFF -DCATCH_INSTALL_HELPERS=OFF ..
-make install
+sudo make install || make install
 cd /
 rm -rf "${TMP_DIR}"
 echo "Catch2 installed successfully"
diff --git a/docker/common/install-eigen b/docker/common/install-eigen
@@ -9,7 +9,7 @@ git checkout "${EIGEN_VERSION}"
 mkdir build
 cd build
 cmake ..
-make install
+sudo make install || make install
 cd /
 rm -rf "${TMP_DIR}"
 echo "Eigen installed successfully"
diff --git a/docker/common/install-libfranka b/docker/common/install-libfranka
@@ -19,7 +19,7 @@ mkdir build
 cd build
 cmake -DBUILD_TESTS=OFF -DBUILD_EXAMPLES=OFF ..
 make -j "$(nproc)"
-make install
+sudo make install || make install
 
 cd /
 rm -rf "${TMP_DIR}"

diff --git a/docker/common/install-pybind11 b/docker/common/install-pybind11
@@ -10,7 +10,7 @@ mkdir build
 cd build
 cmake -DPYBIND11_TEST=OFF ..
 make -j "$(nproc)"
-make install
+sudo make install || make install
 cd /
 rm -rf "${TMP_DIR}"
 echo "pybind11 installed successfully"
diff --git a/docker/build/build-wheels → docker/common/manylinux-build-wheels b/docker/build/build-wheels → docker/common/manylinux-build-wheels
diff --git a/docker/common/manylinux-setup-pip b/docker/common/manylinux-setup-pip
@@ -1,6 +1,13 @@
 #!/bin/bash
 set -e
 
+echo "Found the following Python versions:"
+for PYBIN in /opt/python/*/bin; do
+  if [[ "${PYBIN}" =~ ${PYBIN_SUPPORTED_VERSIONS} ]]; then
+     echo "${PYBIN}" | cut -d / -f 4
+  fi
+done
+
 echo "Setting up pip..."
 for PYBIN in /opt/python/*/bin; do
   if [[ "${PYBIN}" =~ ${PYBIN_SUPPORTED_VERSIONS} ]]; then