feat: Simplify unify containers, but with artifacts [WIP] #150
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Build all container images. | |
| # | |
| name: Build images | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| on: | |
| pull_request: | |
| paths-ignore: | |
| - '.circleci/**' | |
| - 'docs/**' | |
| - 'test/**' | |
| env: | |
| BIOCONDA_UTILS_VERSION: ${{ github.event.release && github.event.release.tag_name || github.head_ref || github.ref_name }} | |
| jobs: | |
| # JOBS FOR BUILDING IMAGES | |
| # ---------------------------------------------------------------------- | |
| # Build images for all archs, upload as artifacts. | |
| demo: | |
| name: demo | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: export env vars | |
| run: | | |
| source images/versions.sh | |
| set -x | |
| for var in $(grep "# GITHUB_ENV$" images/versions.sh | cut -f1 -d "="); do | |
| echo "$var=${!var}" >> $GITHUB_ENV | |
| done | |
| cat $GITHUB_ENV | |
| - name: check env vars | |
| run: | | |
| set -x | |
| echo $ARCHS | |
| echo $BASE_BUSYBOX_IMAGE_NAME | |
| echo $BIOCONDA_IMAGE_TAG | |
| mkdir -p image-artifacts | |
| echo hi > image-artifacts/$BASE_BUSYBOX_IMAGE_NAME | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: demo | |
| path: | | |
| image-artifacts/${{ env.BASE_BUSYBOX_IMAGE_NAME }} | |
| next: | |
| name: next | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: demo | |
| - name: show | |
| run: | | |
| cat image-artifacts/$BASE_BUSYBOX_IMAGE_NAME | |
| exit 1 | |
| build-base-busybox: | |
| name: Build base-busybox | |
| runs-on: ubuntu-24.04 | |
| container: | |
| # travier/podman-action contains newer podman/buildah versions. | |
| image: quay.io/travier/podman-action | |
| options: --privileged | |
| outputs: | |
| TAG_EXISTS_base-busybox: ${{ steps.base-busybox.outputs.TAG_EXISTS_base-busybox }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set up QEMU | |
| run: | | |
| podman run --rm --privileged \ | |
| docker.io/tonistiigi/binfmt --install arm64 | |
| - name: Install Tools | |
| run: | | |
| set -eu | |
| # jq nor git are not installed in travier/podman-action | |
| dnf install -qy \ | |
| jq | |
| rpm -q \ | |
| buildah podman \ | |
| coreutils findutils sed \ | |
| curl jq \ | |
| | ( | |
| while read -r line ; do | |
| printf %s\\n "${line}" | |
| case "${line}" in (*' not installed'*) | |
| err=1 ;; | |
| esac | |
| done | |
| exit "${err-0}" | |
| ) | |
| - name: export env vars | |
| run: | | |
| source images/versions.sh | |
| for var in $(grep "# GITHUB_ENV$" $HERE | cut -f1 -d "="); do | |
| echo "$var=\"${!var}\"" >> $GITHUB_ENV | |
| done | |
| - name: base-busybox | |
| id: base-busybox | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG) ]; then | |
| echo "TAG_EXISTS_base-busybox=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh base-glibc-busybox-bash | |
| fi | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: base-busybox | |
| path: | | |
| image-artifacts/${{ env.BASE_BUSYBOX_IMAGE_NAME }}*.tar | |
| build-build-env: | |
| name: Build build-env | |
| needs: build-base-busybox | |
| outputs: | |
| TAG_EXISTS_build-env: ${{ steps.build-env.outputs.TAG_EXISTS_build-env }} | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: base-busybox | |
| - name: Install qemu dependency | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y qemu-user-static | |
| - name: Load images | |
| run: | | |
| for image in image-artifacts/*.tar; do | |
| podman load -i $image | |
| done | |
| - name: build-env | |
| id: build-env | |
| run: | | |
| source images/versions.sh | |
| if [ $(tag_exists $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then | |
| echo "TAG_EXISTS_build-env=true" >> $GITHUB_OUTPUT | |
| else | |
| cd images && bash build.sh build-env | |
| fi | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-env | |
| path: | | |
| image-artifacts/tmp-build-env-*.tar | |
| # build-create-env: | |
| # name: Build create-env | |
| # needs: [build-build-env, build-base-busybox] | |
| # outputs: | |
| # TAG_EXISTS_create-env: ${{ steps.create-env.outputs.TAG_EXISTS_create-env }} | |
| # runs-on: ubuntu-24.04 | |
| # container: | |
| # # travier/podman-action contains newer podman/buildah versions. | |
| # image: quay.io/travier/podman-action | |
| # options: --privileged | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # fetch-depth: 0 | |
| # - name: Set up QEMU | |
| # run: | | |
| # podman run --rm --privileged \ | |
| # docker.io/tonistiigi/binfmt --install arm64 | |
| # - name: Install Tools | |
| # run: | | |
| # set -eu | |
| # # jq is not installed in travier/podman-action | |
| # dnf install -qy \ | |
| # jq | |
| # rpm -q \ | |
| # buildah podman \ | |
| # coreutils findutils sed \ | |
| # curl jq \ | |
| # | ( | |
| # while read -r line ; do | |
| # printf %s\\n "${line}" | |
| # case "${line}" in (*' not installed'*) | |
| # err=1 ;; | |
| # esac | |
| # done | |
| # exit "${err-0}" | |
| # ) | |
| # - name: Build create-env | |
| # id: create-env | |
| # run: | | |
| # source images/versions.sh | |
| # echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| # if [ $(tag_exists $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then | |
| # echo "TAG_EXISTS_create-env=true" >> $GITHUB_OUTPUT | |
| # else | |
| # cd images && bash build.sh create-env | |
| # fi | |
| # - name: push to ghcr | |
| # if: '${{ ! steps.create-env.outputs.TAG_EXISTS_create-env }}' | |
| # run: | | |
| # echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin | |
| # source images/versions.sh | |
| # push_to_ghcr $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG | |
| # # END OF BUILDING IMAGES | |
| # # ---------------------------------------------------------------------- | |
| # # START TESTING | |
| # # These testing jobs will run the respective Dockerfile.test in each image | |
| # # directory. | |
| # test: | |
| # name: test bioconda-utils with images | |
| # runs-on: ubuntu-24.04 | |
| # needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env] | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # fetch-depth: 0 | |
| # - name: Install bioconda-utils | |
| # run: | | |
| # export BIOCONDA_DISABLE_BUILD_PREP=1 | |
| # BRANCH=simplify-unify-containers | |
| # wget https://raw.githubusercontent.com/bioconda/bioconda-common/${BRANCH}/{common,install-and-set-up-conda,configure-conda}.sh | |
| # source images/versions.sh | |
| # # Ensure install-and-set-up-conda uses same version as in the container | |
| # # (which uses images/versions.sh | |
| # export BIOCONDA_UTILS_TAG=$BIOCONDA_UTILS_VERSION | |
| # bash install-and-set-up-conda.sh | |
| # eval "$(conda shell.bash hook)" | |
| # conda create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt | |
| # conda activate bioconda | |
| # python setup.py install | |
| # - name: test | |
| # run: | | |
| # eval "$(conda shell.bash hook)" | |
| # conda activate bioconda | |
| # source images/versions.sh | |
| # # Figure out which registry to use for each image. If we built an image | |
| # # during this workflow run, it was pushed to ghcr.io as a staging area. | |
| # # If it was not built during this run, then we use the exissting one on | |
| # # quay.io. | |
| # [ ${{ needs.build-build-env.outputs.TAG_EXISTS_build-env }} ] && BUILD_ENV_REGISTRY='quay.io/bioconda' || BUILD_ENV_REGISTRY="ghcr.io/bioconda" | |
| # [ ${{ needs.build-create-env.outputs.TAG_EXISTS_create-env }} ] && CREATE_ENV_REGISTRY='quay.io/bioconda' || CREATE_ENV_REGISTRY="ghcr.io/bioconda" | |
| # [ ${{ needs.build-base-busybox.outputs.TAG_EXISTS_base_busybox }} ] && DEST_BASE_REGISTRY='quay.io/bioconda' || DEST_BASE_REGISTRY="ghcr.io/bioconda" | |
| # [ ${{ needs.build-base-debian.outputs.TAG_EXISTS_base_debian }} ] && DEST_EXTENDED_BASE_REGISTRY='quay.io/bioconda' || DEST_EXTENDED_BASE_REGISTRY="ghcr.io/bioconda" | |
| # # Tell mulled-build which image to use | |
| # # | |
| # # DEST_BASE_IMAGE, DEFAULT_BASE_IMAGE, and DEFAULT_EXTENDED_BASE_IMAGE | |
| # # are hard-coded by mulled-build, e.g. | |
| # # https://github.com/galaxyproject/galaxy/blob/957f6f5475f8f96c6af110be10791b5acab3a0df/lib/galaxy/tool_util/deps/mulled/mulled_build.py#L62-L71 | |
| # # We keep DEST_BASE_IMAGE unset so it defaults to DEFAULT_BASE_IMAGE or | |
| # # DEFAULT_EXTENDED_BASE_IMAGE. | |
| # export DEFAULT_BASE_IMAGE="${DEST_BASE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}" | |
| # export DEFAULT_EXTENDED_BASE_IMAGE="${DEST_EXTENDED_BASE_REGISTRY}/${BASE_DEBIAN_IMAGE_NAME}:${BASE_TAG}" | |
| # export BUILD_ENV_IMAGE="${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" | |
| # export CREATE_ENV_IMAGE="${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" | |
| # source images/env_var_inventory.sh | |
| # [ command -v podman > /dev/null ] && podman images | |
| # [ command -v docker > /dev/null ] && docker images | |
| # py.test --durations=0 test/ -v --log-level=DEBUG -k "docker" --tb=native | |
| # # END TESTING | |
| # # ------------------------------------------------------------------------ | |
| # # START PUSHING IMAGES | |
| # # For these push steps, a repository must first exist on quay.io/bioconda | |
| # # AND that repository must also be configured to allow write access for the | |
| # # appropriate service account. This must be done by a user with admin | |
| # # access to quay.io/bioconda. | |
| # # | |
| # # This uses the TAG_EXISTS_* outputs from previous jobs to determine if | |
| # # a push to quay.io should happen. | |
| # push: | |
| # name: push images | |
| # if: github.ref == 'refs/heads/master' | |
| # runs-on: ubuntu-24.04 | |
| # needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env, test] | |
| # steps: | |
| # - uses: actions/checkout@v4 | |
| # with: | |
| # fetch-depth: 0 | |
| # - name: push base-debian | |
| # if: ${{ ! needs.base-debian.outputs.TAG_EXISTS_base-debian }} | |
| # run: | | |
| # echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| # source images/versions.sh | |
| # move_from_ghcr_to_quay ${BASE_DEBIAN_IMAGE_NAME} ${BASE_TAG} | |
| # - name: push base-busybox | |
| # if: ${{ ! needs.base-busybox.outputs.TAG_EXISTS_base-busybox }} | |
| # run: | | |
| # echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| # source images/versions.sh | |
| # move_from_ghcr_to_quay ${BASE_BUSYBOX_IMAGE_NAME} ${BASE_TAG} | |
| # - name: push create-env | |
| # if: ${{ ! needs.create-env.outputs.TAG_EXISTS_create-env }} | |
| # run: | | |
| # echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| # source images/versions.sh | |
| # move_from_ghcr_to_quay ${CREATE_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} | |
| # - name: push build-env | |
| # if: ${{ ! needs.build-env.outputs.TAG_EXISTS_build-env }} | |
| # run: | | |
| # echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin | |
| # source images/versions.sh | |
| # move_from_ghcr_to_quay ${BUILD_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG} |