From 8578f2f86021eeae01f5d3443733d5650d81560f Mon Sep 17 00:00:00 2001 From: Damian Mee Date: Fri, 16 Aug 2019 01:42:00 +0700 Subject: [PATCH 1/5] Simplify build process, and introduce versioning --- .gitignore | 2 +- .travis.yml | 71 ++++++++------- build-docker.sh | 92 ++++++++++++++++++++ containers/register/{.gitkeep => .gitignore} | 0 generate_tarballs.sh | 20 +++-- publish.sh | 52 ----------- test.sh | 28 ++---- update.sh | 68 --------------- 8 files changed, 152 insertions(+), 181 deletions(-) create mode 100755 build-docker.sh rename containers/register/{.gitkeep => .gitignore} (100%) delete mode 100755 publish.sh delete mode 100755 update.sh diff --git a/.gitignore b/.gitignore index f5b7784..683a5f7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,5 @@ /containers/latest/qemu-*-static /containers/register/Dockerfile /containers/register/register.sh -/containers/x86_64_qemu-*/ +/containers/qemu-*/ /releases diff --git a/.travis.yml b/.travis.yml index aa8b935..712129b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,42 +3,49 @@ services: docker language: bash addons: apt: - config: - retries: true update: true packages: - - jq - rpm2cpio - cpio env: - global: - - VERSION=4.0.0-5 - # See qemu-user-static's RPM spec file on Fedora to check the new version. - # https://src.fedoraproject.org/rpms/qemu/blob/master/f/qemu.spec - - DOCKER_SERVER=docker.io - # - DOCKER_SERVER=quay.io - # Git repository - - REPO=multiarch/qemu-user-static - # Container repository - - DOCKER_REPO=$DOCKER_SERVER/multiarch/qemu-user-static - # - DOCKER_REPO=$DOCKER_SERVER/your_username/qemu-user-static - - PACKAGE_URI="https://kojipkgs.fedoraproject.org/packages/qemu/4.0.0/5.fc31/x86_64/qemu-user-static-4.0.0-5.fc31.x86_64.rpm" - - PACKAGE_FILENAME=$(basename "$PACKAGE_URI") + global: + # See qemu-user-static's RPM spec file on Fedora to check the new version. + # https://src.fedoraproject.org/rpms/qemu/blob/master/f/qemu.spec + # Container repository + - PACKAGE_URI="https://kojipkgs.fedoraproject.org/packages/qemu/4.0.0/5.fc31/x86_64/qemu-user-static-4.0.0-5.fc31.x86_64.rpm" + - PACKAGE_FILENAME=$(basename "$PACKAGE_URI") + +stages: + - build + - test + - name: deploy + if: tag IS present + + before_script: - - wget --content-disposition $PACKAGE_URI - - rpm2cpio $PACKAGE_FILENAME | cpio -dimv -script: + - wget --content-disposition ${PACKAGE_URI} + - rpm2cpio ${PACKAGE_FILENAME} | cpio -dimv - ./generate_tarballs.sh - - | - if [[ $TRAVIS_BRANCH == 'master' && $TRAVIS_PULL_REQUEST == 'false' ]]; then - ./publish.sh -v "$VERSION" -t "$GITHUB_TOKEN" -r "$REPO" - fi - - ./update.sh -v "$VERSION" -r "$REPO" -d "$DOCKER_REPO" - - docker images - - ./test.sh -d "$DOCKER_REPO" -after_success: - - | - if [[ $TRAVIS_BRANCH == 'master' && $TRAVIS_PULL_REQUEST == 'false' ]]; then - docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" "$DOCKER_SERVER" && \ - docker push "$DOCKER_REPO" - fi + +jobs: + include: + - stage: build + name: "Build Docker image for amd64" + script: ./build-docker.sh + + - stage: test + name: "Test generated images" + script: ./test.sh + + - name: "Upload qemu binaries to Github Releases" + stage: deploy + deploy: + skip_cleanup: true + provider: releases + api_key: + secure: Q2ueVCN6jsXOn9iaf8ucAMWsLH+KkYrleSKRbMkJPu+qVacVcYFynm6yMwM63koVA8ADN80+yl7JUvqWx+MZLfCwR+/MRyGC2J0AlCDTBuvK55ETT4Kggtwb/2j6XrjQLlV+9Qkhb0JVpyO1475CjnoT9aOpWsHEHBymddQC13IQMn27XV1LJhunLw7+7Pqb73Yrml8PvnRLFqfqaaLZMoEUjbCjayjmgJY1HqUA2dtQBUnlASDLo3H6epJaDD8qpZTU4850I6TFwavSpRXq786+uqLWfXki0H9EJFkPMgEluul09BRXNS4RABC2darrIluQfK0EOS+acW4AHkTiM5s5YiT402WrD5VVjRCV0GoLrUjnnAFj+jm5fCXykSVJ04/tOMrTMnIgusmUYDRLJ9JUhFb6LS5pzg1ib9pfBmkUfRfEqymFfmsV32sZUyI3+C3NNjOuBiePtQ0gx3vzy1jWwqf5y4IlvMjYyQE11J0PZBjrcZWSXaFlQ34Wl7jZgr+WXJ3a717LX+TbpVjTAuJdPv4vTDJ755avOAM3IgGLFONg7x/5+yroe5dtnHsS5g2Gy8ZYF+Kb0I14x+8Hgi/jRxdF3cKvOoO8MvnK6zeDB4EDolWyTXHwfmpkJOtW+LbvJTJffg5bhoeevpJWaebJqzkusY3/nevG8FYAv2Y= + file_glob: true + file: releases/*.tgz + on: + repo: ${TRAVIS_REPO_SLUG} + tags: true diff --git a/build-docker.sh b/build-docker.sh new file mode 100755 index 0000000..89d8dca --- /dev/null +++ b/build-docker.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +set -xe + +# +## This script builds 3 types of Docker images (assuming version `VERSION`): +# +# 1. Image containing all qemu binaries. Built in `./containers/latest/`. Tagged with `:latest`, and `:VERSION`. +# 2. Register image. Built in `./containers/register/`. Tagged with `:register`, and `:register-VERSION`. +# 3. A series of images, each bundled with a single guest-architecture qemu binary. Built in `./containers/qemu-ARCH`. +# Tagged with `:ARCH`, and `:ARCH-VERSION`. + + +# Convert Travis-provided repo SLUG to lowercase - Docker's requirement for tags +SLUG="$(echo "${TRAVIS_REPO_SLUG}" | tr '[:upper:]' '[:lower:]')" + +# +BUILD_DIR="containers" + +# list of all supported architectures for guest machines +guest_architectures="aarch64 aarch64_be alpha armeb arm cris hppa i386 m68k microblazeel microblaze mips64el mips64 mipsel mipsn32el mipsn32 mips nios2 or1k ppc64abi32 ppc64le ppc64 ppc riscv32 riscv64 s390x sh4eb sh4 sparc32plus sparc64 sparc tilegx trace-stap xtensaeb xtensa" + +# NOTE: All qemu binaries have been downloaded, and unpacked to `./usr/bin/` in `before_script` already. + +## +# Prepare & build image containing all qemu binaries (`:latest`) +## +cp -p ./usr/bin/qemu-*-static "${BUILD_DIR}/latest/" +docker build -t "${SLUG}:latest" "${BUILD_DIR}/latest" + + +## +# Prepare & build register image (`:register`) +## +cp -p ${BUILD_DIR}/latest/{Dockerfile,register.sh} "${BUILD_DIR}/register/" + +# Register image does not need `qemu-*-static` binaries copied into it. This line removes the COPY directive from Dockerfile. +sed -i '/^COPY qemu/ s/^/#/' "${BUILD_DIR}/register/Dockerfile" +docker build -t "${SLUG}:register" "${BUILD_DIR}/register" + + +## +# Build images for individual target architectures (`:ARCH`) +## +for guest_arch in ${guest_architectures}; do + work_dir="${BUILD_DIR}/qemu-${guest_arch}" + + mkdir -p "${work_dir}" + + # copy a single binary to the image + cp -p "./usr/bin/qemu-${guest_arch}-static" "${work_dir}" + + # create a minimal `Dockerfile` that only copies that specific binary into the image + cat > "${work_dir}/Dockerfile" -<&2 - exit 1 -fi +# Convert Travis-provided repo SLUG to lowercase - Docker's requirement for tags +SLUG="$(echo "${TRAVIS_REPO_SLUG}" | tr '[:upper:]' '[:lower:]')" # Test cases - # ------------------------------------------------ # multiarch/qemu-user-static image # It should register binfmt_misc entry with 'flags: F' # by given "-p yes" option. -sudo docker run --rm --privileged ${DOCKER_REPO} --reset -p yes +sudo docker run --rm --privileged "${SLUG}" --reset -p yes cat /proc/sys/fs/binfmt_misc/qemu-aarch64 grep -q '^flags: F$' /proc/sys/fs/binfmt_misc/qemu-aarch64 @@ -53,7 +40,7 @@ docker run --rm -t arm64v8/fedora uname -m # It should register binfmt_misc entry with 'flags: ' # by given no "-p yes" option. -sudo docker run --rm --privileged ${DOCKER_REPO}:register --reset +sudo docker run --rm --privileged "${SLUG}:register" --reset cat /proc/sys/fs/binfmt_misc/qemu-aarch64 grep -q '^flags: $' /proc/sys/fs/binfmt_misc/qemu-aarch64 @@ -62,13 +49,12 @@ grep -q '^flags: $' /proc/sys/fs/binfmt_misc/qemu-aarch64 # multiarch/qemu-user-static:$from_arch-$to_arch image # /usr/bin/qemu-aarch64-static should be included. -docker run --rm -t ${DOCKER_REPO}:aarch64 /usr/bin/qemu-aarch64-static --version -docker run --rm -t ${DOCKER_REPO}:x86_64-aarch64 /usr/bin/qemu-aarch64-static --version +docker run --rm -t "${SLUG}:aarch64" /usr/bin/qemu-aarch64-static --version # ------------------------------------------------ # Integration test docker build --rm -t "test/integration/ubuntu" -<&1 - echo "check https://github.com/${REPO}/releases for available versions" 2>&1 - exit 1 -fi - -shift $((OPTIND-1)) - -[ "$1" = "--" ] && shift - -from_arch="x86_64" -to_archs="aarch64 aarch64_be alpha armeb arm cris hppa i386 m68k microblazeel microblaze mips64el mips64 mipsel mipsn32el mipsn32 mips nios2 or1k ppc64abi32 ppc64le ppc64 ppc riscv32 riscv64 s390x sh4eb sh4 sparc32plus sparc64 sparc tilegx trace-stap x86_64 xtensaeb xtensa" -# For casual test -# to_archs="aarch64" - -# Build container images creating the directory. -# containers/ -# latest/ - An image including /usr/bin/qemu-$arch-status and /register script. -# ${from_arch}_qemu-${to_arch}/ - Images including /usr/bin/qemu-$arch-status -# register/ - An image including /register script. -out_dir="containers" - -# Generate register files. -cp -p "${out_dir}/latest/register.sh" "${out_dir}/register/" -cp -p "${out_dir}/latest/Dockerfile" "${out_dir}/register/" -# Comment out the line to copy qemu-*-static not to provide those. -sed -i '/^COPY qemu/ s/^/#/' "${out_dir}/register/Dockerfile" - -for to_arch in $to_archs; do - if [ "$from_arch" != "$to_arch" ]; then - work_dir="${out_dir}/${from_arch}_qemu-${to_arch}" - mkdir -p "${work_dir}" - curl -sSL -o "${work_dir}/${from_arch}_qemu-${to_arch}-static.tar.gz" \ - "https://github.com/${REPO}/releases/download/v${VERSION}/${from_arch}_qemu-${to_arch}-static.tar.gz" - tar xzvf "${work_dir}/${from_arch}_qemu-${to_arch}-static.tar.gz" -C "${work_dir}" - rm -f "${work_dir}/${from_arch}_qemu-${to_arch}-static.tar.gz" - - cp -p "${work_dir}/qemu-${to_arch}-static" "${out_dir}/latest/" - - cat > ${work_dir}/Dockerfile -< Date: Fri, 16 Aug 2019 14:23:35 +0700 Subject: [PATCH 2/5] apply requested changes part 1 --- .gitignore | 2 +- .travis.yml | 58 +++++++++++++------------- build-docker.sh | 92 ---------------------------------------- build_containers.sh | 99 ++++++++++++++++++++++++++++++++++++++++++++ generate_tarballs.sh | 8 ++-- test.sh | 2 +- 6 files changed, 134 insertions(+), 127 deletions(-) delete mode 100755 build-docker.sh create mode 100755 build_containers.sh diff --git a/.gitignore b/.gitignore index 683a5f7..6ee71bc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,5 @@ /containers/latest/qemu-*-static /containers/register/Dockerfile /containers/register/register.sh -/containers/qemu-*/ +/containers/qemu-single/ /releases diff --git a/.travis.yml b/.travis.yml index 712129b..c73d325 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,18 +8,18 @@ addons: - rpm2cpio - cpio env: - global: - # See qemu-user-static's RPM spec file on Fedora to check the new version. - # https://src.fedoraproject.org/rpms/qemu/blob/master/f/qemu.spec - # Container repository - - PACKAGE_URI="https://kojipkgs.fedoraproject.org/packages/qemu/4.0.0/5.fc31/x86_64/qemu-user-static-4.0.0-5.fc31.x86_64.rpm" - - PACKAGE_FILENAME=$(basename "$PACKAGE_URI") + global: + # See qemu-user-static's RPM spec file on Fedora to check the new version. + # https://src.fedoraproject.org/rpms/qemu/blob/master/f/qemu.spec + # Container repository + - PACKAGE_URI="https://kojipkgs.fedoraproject.org/packages/qemu/4.0.0/5.fc31/x86_64/qemu-user-static-4.0.0-5.fc31.x86_64.rpm" + - PACKAGE_FILENAME=$(basename "$PACKAGE_URI") stages: - - build - - test - - name: deploy - if: tag IS present + - build + - test + - name: deploy + if: tag IS present before_script: @@ -28,24 +28,24 @@ before_script: - ./generate_tarballs.sh jobs: - include: - - stage: build - name: "Build Docker image for amd64" - script: ./build-docker.sh + include: + - stage: build + name: "Build container images for x86_64" + script: ./build_containers.sh - - stage: test - name: "Test generated images" - script: ./test.sh + - stage: test + name: "Test generated images" + script: ./test.sh - - name: "Upload qemu binaries to Github Releases" - stage: deploy - deploy: - skip_cleanup: true - provider: releases - api_key: - secure: Q2ueVCN6jsXOn9iaf8ucAMWsLH+KkYrleSKRbMkJPu+qVacVcYFynm6yMwM63koVA8ADN80+yl7JUvqWx+MZLfCwR+/MRyGC2J0AlCDTBuvK55ETT4Kggtwb/2j6XrjQLlV+9Qkhb0JVpyO1475CjnoT9aOpWsHEHBymddQC13IQMn27XV1LJhunLw7+7Pqb73Yrml8PvnRLFqfqaaLZMoEUjbCjayjmgJY1HqUA2dtQBUnlASDLo3H6epJaDD8qpZTU4850I6TFwavSpRXq786+uqLWfXki0H9EJFkPMgEluul09BRXNS4RABC2darrIluQfK0EOS+acW4AHkTiM5s5YiT402WrD5VVjRCV0GoLrUjnnAFj+jm5fCXykSVJ04/tOMrTMnIgusmUYDRLJ9JUhFb6LS5pzg1ib9pfBmkUfRfEqymFfmsV32sZUyI3+C3NNjOuBiePtQ0gx3vzy1jWwqf5y4IlvMjYyQE11J0PZBjrcZWSXaFlQ34Wl7jZgr+WXJ3a717LX+TbpVjTAuJdPv4vTDJ755avOAM3IgGLFONg7x/5+yroe5dtnHsS5g2Gy8ZYF+Kb0I14x+8Hgi/jRxdF3cKvOoO8MvnK6zeDB4EDolWyTXHwfmpkJOtW+LbvJTJffg5bhoeevpJWaebJqzkusY3/nevG8FYAv2Y= - file_glob: true - file: releases/*.tgz - on: - repo: ${TRAVIS_REPO_SLUG} - tags: true + - name: "Upload qemu binaries to Github Releases" + stage: deploy + deploy: + skip_cleanup: true + provider: releases + api_key: + secure: Q2ueVCN6jsXOn9iaf8ucAMWsLH+KkYrleSKRbMkJPu+qVacVcYFynm6yMwM63koVA8ADN80+yl7JUvqWx+MZLfCwR+/MRyGC2J0AlCDTBuvK55ETT4Kggtwb/2j6XrjQLlV+9Qkhb0JVpyO1475CjnoT9aOpWsHEHBymddQC13IQMn27XV1LJhunLw7+7Pqb73Yrml8PvnRLFqfqaaLZMoEUjbCjayjmgJY1HqUA2dtQBUnlASDLo3H6epJaDD8qpZTU4850I6TFwavSpRXq786+uqLWfXki0H9EJFkPMgEluul09BRXNS4RABC2darrIluQfK0EOS+acW4AHkTiM5s5YiT402WrD5VVjRCV0GoLrUjnnAFj+jm5fCXykSVJ04/tOMrTMnIgusmUYDRLJ9JUhFb6LS5pzg1ib9pfBmkUfRfEqymFfmsV32sZUyI3+C3NNjOuBiePtQ0gx3vzy1jWwqf5y4IlvMjYyQE11J0PZBjrcZWSXaFlQ34Wl7jZgr+WXJ3a717LX+TbpVjTAuJdPv4vTDJ755avOAM3IgGLFONg7x/5+yroe5dtnHsS5g2Gy8ZYF+Kb0I14x+8Hgi/jRxdF3cKvOoO8MvnK6zeDB4EDolWyTXHwfmpkJOtW+LbvJTJffg5bhoeevpJWaebJqzkusY3/nevG8FYAv2Y= + file_glob: true + file: releases/*.tgz + on: + repo: ${TRAVIS_REPO_SLUG} + tags: true diff --git a/build-docker.sh b/build-docker.sh deleted file mode 100755 index 89d8dca..0000000 --- a/build-docker.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -set -xe - -# -## This script builds 3 types of Docker images (assuming version `VERSION`): -# -# 1. Image containing all qemu binaries. Built in `./containers/latest/`. Tagged with `:latest`, and `:VERSION`. -# 2. Register image. Built in `./containers/register/`. Tagged with `:register`, and `:register-VERSION`. -# 3. A series of images, each bundled with a single guest-architecture qemu binary. Built in `./containers/qemu-ARCH`. -# Tagged with `:ARCH`, and `:ARCH-VERSION`. - - -# Convert Travis-provided repo SLUG to lowercase - Docker's requirement for tags -SLUG="$(echo "${TRAVIS_REPO_SLUG}" | tr '[:upper:]' '[:lower:]')" - -# -BUILD_DIR="containers" - -# list of all supported architectures for guest machines -guest_architectures="aarch64 aarch64_be alpha armeb arm cris hppa i386 m68k microblazeel microblaze mips64el mips64 mipsel mipsn32el mipsn32 mips nios2 or1k ppc64abi32 ppc64le ppc64 ppc riscv32 riscv64 s390x sh4eb sh4 sparc32plus sparc64 sparc tilegx trace-stap xtensaeb xtensa" - -# NOTE: All qemu binaries have been downloaded, and unpacked to `./usr/bin/` in `before_script` already. - -## -# Prepare & build image containing all qemu binaries (`:latest`) -## -cp -p ./usr/bin/qemu-*-static "${BUILD_DIR}/latest/" -docker build -t "${SLUG}:latest" "${BUILD_DIR}/latest" - - -## -# Prepare & build register image (`:register`) -## -cp -p ${BUILD_DIR}/latest/{Dockerfile,register.sh} "${BUILD_DIR}/register/" - -# Register image does not need `qemu-*-static` binaries copied into it. This line removes the COPY directive from Dockerfile. -sed -i '/^COPY qemu/ s/^/#/' "${BUILD_DIR}/register/Dockerfile" -docker build -t "${SLUG}:register" "${BUILD_DIR}/register" - - -## -# Build images for individual target architectures (`:ARCH`) -## -for guest_arch in ${guest_architectures}; do - work_dir="${BUILD_DIR}/qemu-${guest_arch}" - - mkdir -p "${work_dir}" - - # copy a single binary to the image - cp -p "./usr/bin/qemu-${guest_arch}-static" "${work_dir}" - - # create a minimal `Dockerfile` that only copies that specific binary into the image - cat > "${work_dir}/Dockerfile" -< "${work_dir}/Dockerfile" -< Date: Fri, 16 Aug 2019 14:31:22 +0700 Subject: [PATCH 3/5] change docker creds env vars for travis --- build_containers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_containers.sh b/build_containers.sh index 2c18084..1735388 100755 --- a/build_containers.sh +++ b/build_containers.sh @@ -69,7 +69,7 @@ if [[ -n "${TRAVIS_TAG}" ]]; then fi # Login to Docker Hub - echo "${DOCKER_PASS}" | docker login -u="${DOCKER_USER}" --password-stdin + echo "${DOCKER_PASSWORD}" | docker login -u="${DOCKER_USERNAME}" --password-stdin MINOR_TAG="$(echo "${TRAVIS_TAG}" | cut -d. -f1).$(echo "${TRAVIS_TAG}" | cut -d. -f2)" From a9f12753fe8fa47c74d195eac70388c67195b5a8 Mon Sep 17 00:00:00 2001 From: Damian Mee Date: Fri, 16 Aug 2019 15:07:31 +0700 Subject: [PATCH 4/5] shebangs reverted --- build_containers.sh | 2 +- generate_tarballs.sh | 2 +- test.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/build_containers.sh b/build_containers.sh index 1735388..82f2de7 100755 --- a/build_containers.sh +++ b/build_containers.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash -e set -xeo pipefail # diff --git a/generate_tarballs.sh b/generate_tarballs.sh index ce889b5..4ac2e81 100755 --- a/generate_tarballs.sh +++ b/generate_tarballs.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash -e # ## This script compresses all qemu binaries, and prepares them for upload to Github Release later in the deploy step. diff --git a/test.sh b/test.sh index 47b754d..646e6d6 100755 --- a/test.sh +++ b/test.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash -e set -xeo pipefail # Convert Travis-provided repo SLUG to lowercase - Docker's requirement for tags From ddc8e0952eb26e0da1dcc80004e39d1880a01723 Mon Sep 17 00:00:00 2001 From: Damian Mee Date: Fri, 16 Aug 2019 15:21:22 +0700 Subject: [PATCH 5/5] add back travis property --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index c73d325..7c81c38 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,6 +3,8 @@ services: docker language: bash addons: apt: + config: + retries: true update: true packages: - rpm2cpio