feat: Simplify unify containers, but with artifacts [WIP] #183
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # TODO: | |
| # - detect whether there was a release created (over in release-please.yml) | |
| # based on commit message, and use that to trigger an upload to quay.io, | |
| # after building the images again here. | |
| # | |
| # NOTES: | |
| # - export env vars to GITHUB_ENV so that upload/download artifacts have access | |
| # - artifacts cannot have ":" in filenames | |
| # | |
| name: Build images | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - '.circleci/**' | |
| - 'docs/**' | |
| - 'test/**' | |
| env: | |
| BIOCONDA_UTILS_VERSION: ${{ github.event.release && github.event.release.tag_name || github.head_ref || github.ref_name }} | |
| jobs: | |
| # JOBS FOR BUILDING IMAGES | |
| # ---------------------------------------------------------------------- | |
| # Build images for all archs, upload as artifacts. | |
| demo: | |
| name: demo | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: export env vars marked as GITHUB_ENV | |
| run: | | |
| source images/versions.sh | |
| for var in $(grep "# GITHUB_ENV$" images/versions.sh | cut -f1 -d "="); do | |
| echo "$var=${!var}" >> $GITHUB_ENV | |
| done | |
| - name: check env vars | |
| run: | | |
| set -x | |
| echo $ARCHS | |
| echo $BASE_BUSYBOX_IMAGE_NAME | |
| echo $BIOCONDA_IMAGE_TAG | |
| mkdir -p image-artifacts | |
| echo hi > image-artifacts/$BASE_BUSYBOX_IMAGE_NAME | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: demo | |
| path: | | |
| image-artifacts/${{ env.BASE_BUSYBOX_IMAGE_NAME }} | |
| next: | |
| name: next | |
| needs: demo | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: demo | |
| path: image-artifacts | |
| - name: export env vars marked as GITHUB_ENV | |
| run: | | |
| source images/versions.sh | |
| for var in $(grep "# GITHUB_ENV$" images/versions.sh | cut -f1 -d "="); do | |
| echo "$var=${!var}" >> $GITHUB_ENV | |
| done | |
| - name: show | |
| run: | | |
| cat image-artifacts/$BASE_BUSYBOX_IMAGE_NAME | |
| # Inspect quay.io to see which, if any, of the images we're trying to build | |
| # are already on quay.io. Export the results so they can be seen by other rules. | |
| detect-existing: | |
| name: detect-existing | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| DO_BUILD: ${{ steps.detect-existing.outputs.DO_BUILD }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect existing | |
| id: detect-existing | |
| run: | | |
| source images/versions.sh | |
| source images/env_var_inventory.sh | |
| # If all versions exist on quay.io, then don't do the build. | |
| if tag_exists $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG \ | |
| && tag_exists $BASE_DEBIAN_IMAGE_NAME $BASE_TAG \ | |
| && tag_exists $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG \ | |
| && tag_exists $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG; then | |
| echo "DO_BUILD=false" >> $GITHUB_OUTPUT | |
| else | |
| echo "DO_BUILD=true" >> $GITHUB_OUTPUT | |
| fi | |
| build-base-debian: | |
| name: Build base-debian | |
| runs-on: ubuntu-24.04 | |
| needs: [ detect-existing, next ] | |
| if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }} | |
| container: | |
| # travier/podman-action contains newer podman/buildah versions. | |
| image: quay.io/travier/podman-action | |
| options: --privileged | |
| steps: | |
| - name: install git in podman-action container | |
| run: dnf install -qy git | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup | |
| run: | | |
| source images/github_actions_setup.sh | |
| - name: base-debian | |
| id: base-debian | |
| run: | | |
| source images/versions.sh | |
| cd images && bash build.sh base-glibc-debian-bash | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: base-debian | |
| path: | | |
| image-artifacts/${{ env.BASE_DEBIAN_IMAGE_NAME }}*.tar | |
| build-base-busybox: | |
| name: Build base-busybox | |
| runs-on: ubuntu-24.04 | |
| needs: [ detect-existing, next ] | |
| if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }} | |
| container: | |
| # travier/podman-action contains newer podman/buildah versions. | |
| image: quay.io/travier/podman-action | |
| options: --privileged | |
| steps: | |
| - name: install git in podman-action container | |
| run: dnf install -qy git | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup | |
| run: | | |
| source images/github_actions_setup.sh | |
| - name: base-busybox | |
| id: base-busybox | |
| run: | | |
| source images/versions.sh | |
| cd images && bash build.sh base-glibc-busybox-bash | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: base-busybox | |
| path: | | |
| image-artifacts/${{ env.BASE_BUSYBOX_IMAGE_NAME }}*.tar | |
| build-build-env: | |
| name: Build build-env | |
| runs-on: ubuntu-22.04 | |
| needs: [ build-base-busybox, detect-existing ] | |
| if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }} | |
| container: | |
| image: quay.io/travier/podman-action | |
| options: --privileged | |
| steps: | |
| - name: install git in podman-action container | |
| run: dnf install -qy git | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup | |
| run: | | |
| source images/github_actions_setup.sh | |
| - name: Download image artifacts from base-busybox | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: base-busybox | |
| path: image-artifacts | |
| - name: Load image artifacts into podman | |
| run: | | |
| for image in image-artifacts/*.tar; do | |
| podman load -i $image | |
| done | |
| podman images | |
| - name: build-env | |
| id: build-env | |
| run: | | |
| source images/versions.sh | |
| cd images && bash build.sh build-env | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-env | |
| path: | | |
| image-artifacts/${{ env.BUILD_ENV_IMAGE_NAME }}*.tar | |
| build-create-env: | |
| name: Build create-env | |
| runs-on: ubuntu-24.04 | |
| needs: [build-build-env, build-base-busybox, detect-existing ] | |
| if: ${{ needs.detect-existing.outputs.DO_BUILD == 'true' }} | |
| container: | |
| image: quay.io/travier/podman-action | |
| options: --privileged | |
| steps: | |
| - name: install git in podman-action container | |
| run: dnf install -qy git | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Setup | |
| run: | | |
| source images/github_actions_setup.sh | |
| - name: Download artifacts from base-busybox | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: base-busybox | |
| path: image-artifacts | |
| - name: Download artifacts from build-env | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: build-env | |
| path: image-artifacts | |
| - name: Load image artifacts into podman | |
| run: | | |
| for image in image-artifacts/*.tar; do | |
| podman load -i $image | |
| done | |
| podman images | |
| - name: Build create-env | |
| run: | | |
| source images/versions.sh | |
| cd images && bash build.sh create-env | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: create-env | |
| path: | | |
| image-artifacts/${{ env.CREATE_ENV_IMAGE_NAME }}*.tar | |
| # END OF BUILDING IMAGES | |
| # ---------------------------------------------------------------------- | |
| # START TESTING | |
| # These testing jobs will run the respective Dockerfile.test in each image | |
| # directory. | |
| test: | |
| name: test bioconda-utils with images | |
| runs-on: ubuntu-24.04 | |
| needs: [build-base-busybox, build-build-env, build-create-env, build-base-debian, detect-existing] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: export env vars marked as GITHUB_ENV | |
| run: | | |
| source images/versions.sh | |
| for var in $(grep "# GITHUB_ENV$" images/versions.sh | cut -f1 -d "="); do | |
| echo "$var=${!var}" >> $GITHUB_ENV | |
| done | |
| - name: Install bioconda-utils | |
| run: | | |
| export BIOCONDA_DISABLE_BUILD_PREP=1 | |
| BRANCH=simplify-unify-containers | |
| wget https://raw.githubusercontent.com/bioconda/bioconda-common/${BRANCH}/{common,install-and-set-up-conda,configure-conda}.sh | |
| source images/versions.sh | |
| # Ensure install-and-set-up-conda uses same version as in the container | |
| # (which uses images/versions.sh) | |
| export BIOCONDA_UTILS_TAG=$BIOCONDA_UTILS_VERSION | |
| bash install-and-set-up-conda.sh | |
| eval "$(conda shell.bash hook)" | |
| conda create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt | |
| conda activate bioconda | |
| python setup.py install | |
| - name: Download artifacts from base-busybox | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: base-busybox | |
| path: image-artifacts | |
| - name: Download artifacts from build-env | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: build-env | |
| path: image-artifacts | |
| - name: Download artifacts from create-env | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: create-env | |
| path: image-artifacts | |
| - name: Load image artifacts into docker for testing | |
| run: | | |
| for image in image-artifacts/*.tar; do | |
| docker load -i $image | |
| done | |
| docker images | |
| - name: test | |
| run: | | |
| eval "$(conda shell.bash hook)" | |
| conda activate bioconda | |
| source images/versions.sh | |
| # Tell mulled-build which image to use | |
| # | |
| # DEST_BASE_IMAGE, DEFAULT_BASE_IMAGE, and DEFAULT_EXTENDED_BASE_IMAGE | |
| # are hard-coded by mulled-build, e.g. | |
| # https://github.com/galaxyproject/galaxy/blob/957f6f5475f8f96c6af110be10791b5acab3a0df/lib/galaxy/tool_util/deps/mulled/mulled_build.py#L62-L71 | |
| # We keep DEST_BASE_IMAGE unset so it defaults to DEFAULT_BASE_IMAGE or | |
| # DEFAULT_EXTENDED_BASE_IMAGE. | |
| export DEFAULT_BASE_IMAGE="${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" | |
| export DEFAULT_EXTENDED_BASE_IMAGE="${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG}" | |
| export BUILD_ENV_IMAGE="${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" | |
| export CREATE_ENV_IMAGE="${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" | |
| source images/env_var_inventory.sh | |
| [ command -v podman > /dev/null ] && podman images | |
| [ command -v docker > /dev/null ] && docker images | |
| py.test --durations=0 test/ -v --log-level=DEBUG -k "docker" --tb=native | |
| # END TESTING | |
| # ------------------------------------------------------------------------ | |
| # START PUSHING IMAGES | |
| # For these push steps, a repository must first exist on quay.io/bioconda | |
| # AND that repository must also be configured to allow write access for the | |
| # appropriate service account. This must be done by a user with admin | |
| # access to quay.io/bioconda. | |
| push: | |
| name: push images | |
| if: (github.ref == 'refs/heads/master') && (needs.detect-existing.outputs.DO_BUILD == 'true') | |
| runs-on: ubuntu-24.04 | |
| needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env, test, detect-existing] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: push base-debian | |
| if: ${{ ! needs.base-debian.outputs.TAG_EXISTS_base-debian }} | |
| run: | | |
| echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| source images/versions.sh | |
| push_to_quay ${BASE_DEBIAN_IMAGE_NAME} ${BASE_TAG} | |
| push_to_quay ${BASE_BUSYBOX_IMAGE_NAME} ${BASE_TAG} | |
| push_to_quay ${CREATE_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} | |
| push_to_quay ${BUILD_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} |