Simplify build process, and introduce versioning

Author: meeDamian

.gitignore

@@ -4,5 +4,5 @@
 /containers/latest/qemu-*-static
 /containers/register/Dockerfile
 /containers/register/register.sh
-/containers/x86_64_qemu-*/
+/containers/qemu-*/
 /releases

.travis.yml

@@ -3,42 +3,49 @@ services: docker
 language: bash
 addons:
     apt:
-        config:
-            retries: true
         update: true
         packages:
-            - jq
             - rpm2cpio
             - cpio
 env:
-    global:
-        - VERSION=4.0.0-5
-        # See qemu-user-static's RPM spec file on Fedora to check the new version.
-        # https://src.fedoraproject.org/rpms/qemu/blob/master/f/qemu.spec
-        - DOCKER_SERVER=docker.io
-        # - DOCKER_SERVER=quay.io
-        # Git repository
-        - REPO=multiarch/qemu-user-static
-        # Container repository
-        - DOCKER_REPO=$DOCKER_SERVER/multiarch/qemu-user-static
-        # - DOCKER_REPO=$DOCKER_SERVER/your_username/qemu-user-static
-        - PACKAGE_URI="https://kojipkgs.fedoraproject.org/packages/qemu/4.0.0/5.fc31/x86_64/qemu-user-static-4.0.0-5.fc31.x86_64.rpm"
-        - PACKAGE_FILENAME=$(basename "$PACKAGE_URI")
+  global:
+    # See qemu-user-static's RPM spec file on Fedora to check the new version.
+    # https://src.fedoraproject.org/rpms/qemu/blob/master/f/qemu.spec
+    # Container repository
+    - PACKAGE_URI="https://kojipkgs.fedoraproject.org/packages/qemu/4.0.0/5.fc31/x86_64/qemu-user-static-4.0.0-5.fc31.x86_64.rpm"
+    - PACKAGE_FILENAME=$(basename "$PACKAGE_URI")
+
+stages:
+  - build
+  - test
+  - name: deploy
+    if: tag IS present
+
+
 before_script:
-    - wget --content-disposition $PACKAGE_URI
-    - rpm2cpio $PACKAGE_FILENAME | cpio -dimv
-script:
+    - wget --content-disposition ${PACKAGE_URI}
+    - rpm2cpio ${PACKAGE_FILENAME} | cpio -dimv
     - ./generate_tarballs.sh
-    - |
-      if [[ $TRAVIS_BRANCH == 'master' && $TRAVIS_PULL_REQUEST == 'false' ]]; then
-          ./publish.sh -v "$VERSION" -t "$GITHUB_TOKEN" -r "$REPO"
-      fi
-    - ./update.sh -v "$VERSION" -r "$REPO" -d "$DOCKER_REPO"
-    - docker images
-    - ./test.sh -d "$DOCKER_REPO"
-after_success:
-    - |
-      if [[ $TRAVIS_BRANCH == 'master' && $TRAVIS_PULL_REQUEST == 'false' ]]; then
-          docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD" "$DOCKER_SERVER" && \
-              docker push "$DOCKER_REPO"
-      fi
+
+jobs:
+  include:
+    - stage: build
+      name: "Build Docker image for amd64"
+      script: ./build-docker.sh
+
+    - stage: test
+      name: "Test generated images"
+      script: ./test.sh
+
+    - name: "Upload qemu binaries to Github Releases"
+      stage: deploy
+      deploy:
+        skip_cleanup: true
+        provider: releases
+        api_key:
+          secure: Q2ueVCN6jsXOn9iaf8ucAMWsLH+KkYrleSKRbMkJPu+qVacVcYFynm6yMwM63koVA8ADN80+yl7JUvqWx+MZLfCwR+/MRyGC2J0AlCDTBuvK55ETT4Kggtwb/2j6XrjQLlV+9Qkhb0JVpyO1475CjnoT9aOpWsHEHBymddQC13IQMn27XV1LJhunLw7+7Pqb73Yrml8PvnRLFqfqaaLZMoEUjbCjayjmgJY1HqUA2dtQBUnlASDLo3H6epJaDD8qpZTU4850I6TFwavSpRXq786+uqLWfXki0H9EJFkPMgEluul09BRXNS4RABC2darrIluQfK0EOS+acW4AHkTiM5s5YiT402WrD5VVjRCV0GoLrUjnnAFj+jm5fCXykSVJ04/tOMrTMnIgusmUYDRLJ9JUhFb6LS5pzg1ib9pfBmkUfRfEqymFfmsV32sZUyI3+C3NNjOuBiePtQ0gx3vzy1jWwqf5y4IlvMjYyQE11J0PZBjrcZWSXaFlQ34Wl7jZgr+WXJ3a717LX+TbpVjTAuJdPv4vTDJ755avOAM3IgGLFONg7x/5+yroe5dtnHsS5g2Gy8ZYF+Kb0I14x+8Hgi/jRxdF3cKvOoO8MvnK6zeDB4EDolWyTXHwfmpkJOtW+LbvJTJffg5bhoeevpJWaebJqzkusY3/nevG8FYAv2Y=
+        file_glob: true
+        file: releases/*.tgz
+        on:
+          repo: ${TRAVIS_REPO_SLUG}
+          tags: true

build-docker.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -xe
+
+#
+## This script builds 3 types of Docker images (assuming version `VERSION`):
+#
+#   1. Image containing all qemu binaries. Built in `./containers/latest/`. Tagged with `:latest`, and `:VERSION`.
+#   2. Register image.  Built in `./containers/register/`. Tagged with `:register`, and `:register-VERSION`.
+#   3. A series of images, each bundled with a single guest-architecture qemu binary. Built in `./containers/qemu-ARCH`.
+#      Tagged with `:ARCH`, and `:ARCH-VERSION`.
+
+
+# Convert Travis-provided repo SLUG to lowercase - Docker's requirement for tags
+SLUG="$(echo "${TRAVIS_REPO_SLUG}" | tr '[:upper:]' '[:lower:]')"
+
+#
+BUILD_DIR="containers"
+
+# list of all supported architectures for guest machines
+guest_architectures="aarch64 aarch64_be alpha armeb arm cris hppa i386 m68k microblazeel microblaze mips64el mips64 mipsel mipsn32el mipsn32 mips nios2 or1k ppc64abi32 ppc64le ppc64 ppc riscv32 riscv64 s390x sh4eb sh4 sparc32plus sparc64 sparc tilegx trace-stap xtensaeb xtensa"
+
+# NOTE: All qemu binaries have been downloaded, and unpacked to `./usr/bin/` in `before_script` already.
+
+##
+# Prepare & build image containing all qemu binaries (`:latest`)
+##
+cp -p ./usr/bin/qemu-*-static "${BUILD_DIR}/latest/"
+docker build  -t "${SLUG}:latest"  "${BUILD_DIR}/latest"
+
+
+##
+# Prepare & build register image (`:register`)
+##
+cp -p ${BUILD_DIR}/latest/{Dockerfile,register.sh} "${BUILD_DIR}/register/"
+
+# Register image does not need `qemu-*-static` binaries copied into it.  This line removes the COPY directive from Dockerfile.
+sed -i '/^COPY qemu/ s/^/#/' "${BUILD_DIR}/register/Dockerfile"
+docker build  -t "${SLUG}:register"  "${BUILD_DIR}/register"
+
+
+##
+# Build images for individual target architectures (`:ARCH`)
+##
+for guest_arch in ${guest_architectures}; do
+  work_dir="${BUILD_DIR}/qemu-${guest_arch}"
+
+  mkdir -p "${work_dir}"
+
+  # copy a single binary to the image
+  cp -p "./usr/bin/qemu-${guest_arch}-static" "${work_dir}"
+
+  # create a minimal `Dockerfile` that only copies that specific binary into the image
+  cat > "${work_dir}/Dockerfile" -<<EOF
+FROM scratch
+COPY qemu-${guest_arch}-static /usr/bin/
+EOF
+
+  docker build  -t "${SLUG}:${guest_arch}"  "${work_dir}"
+
+  # cleanup
+  rm -rf "${work_dir}"
+done
+
+
+# If git tag is provided, tag all images with VERSION, and push them to Docker Hub
+if [[ -n "${TRAVIS_TAG}" ]]; then
+  if [[ -z "${DOCKER_USER}" ]] || [[ -z "${DOCKER_PASS}" ]]; then
+    echo "For deployment to Docker Hub to work both DOCKER_USER and DOCKER_PASS must be provided in Travis build settings."
+    exit 1
+  fi
+
+  # Login to Docker Hub
+  echo "${DOCKER_PASS}" | docker login -u="${DOCKER_USER}" --password-stdin
+
+  # Tag `:latest` with a specific qemu version, and push both
+  docker tag  "${SLUG}:latest" "${SLUG}:${TRAVIS_TAG}"
+  docker push "${SLUG}:latest"
+  docker push "${SLUG}:${TRAVIS_TAG}"
+
+  # Tag `:register` with specific qemu version, and push both
+  docker tag  "${SLUG}:register" "${SLUG}:register-${TRAVIS_TAG}"
+  docker push "${SLUG}:register"
+  docker push "${SLUG}:register-${TRAVIS_TAG}"
+
+  # For each architecture, create a versioned tag, and push all
+  for guest_arch in ${guest_architectures}; do
+    docker tag  "${SLUG}:${guest_arch}"  "${SLUG}:${guest_arch}-${TRAVIS_TAG}"
+
+    docker push "${SLUG}:${guest_arch}"
+    docker push "${SLUG}:${guest_arch}-${TRAVIS_TAG}"
+  done
+fi

containers/register/.gitkeep renamed to containers/register/.gitignore

@@ -1,11 +1,17 @@
-#!/bin/bash -e
+#!/usr/bin/env bash
+
+#
+## This script compresses all qemu binaries, and prepares them for upload to Github Release later in the deploy step.
+
+if [[ -z "${TRAVIS_TAG}" ]]; then
+  echo "TRAVIS_TAG not specified.  Skipping tarball generation."
+  exit 0
+fi
 
 rm -rf releases
 mkdir -p releases
-# find . -regex './qemu-.*' -not -regex './qemu-system-.*' -exec cp {} releases \;
-cp ./usr/bin/qemu-*-static releases/
-cd releases/
-for file in *; do
-    tar -czf $file.tar.gz $file;
-    cp $file.tar.gz x86_64_$file.tar.gz
+
+for file in ./usr/bin/qemu-*-static; do
+  name="$(basename "${file}").tgz"
+  tar -cvzf "releases/${TRAVIS_TAG}-${name}" "${file}"
 done

generate_tarballs.sh

@@ -1,29 +1,16 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -xeo pipefail
 
-# A POSIX variable
-OPTIND=1 # Reset in case getopts has been used previously in the shell.
-
-while getopts "d:" opt; do
-    case "$opt" in
-        d)  DOCKER_REPO=$OPTARG
-        ;;
-    esac
-done
-
-if [ "${DOCKER_REPO}" = "" ]; then
-    echo "DOCKER_REPO is required." 1>&2
-    exit 1
-fi
+# Convert Travis-provided repo SLUG to lowercase - Docker's requirement for tags
+SLUG="$(echo "${TRAVIS_REPO_SLUG}" | tr '[:upper:]' '[:lower:]')"
 
 # Test cases
-
 # ------------------------------------------------
 # multiarch/qemu-user-static image
 
 # It should register binfmt_misc entry with 'flags: F'
 # by given "-p yes" option.
-sudo docker run --rm --privileged ${DOCKER_REPO} --reset -p yes
+sudo docker run --rm --privileged "${SLUG}" --reset -p yes
 cat /proc/sys/fs/binfmt_misc/qemu-aarch64
 grep -q '^flags: F$' /proc/sys/fs/binfmt_misc/qemu-aarch64
 
@@ -53,7 +40,7 @@ docker run --rm -t arm64v8/fedora uname -m
 
 # It should register binfmt_misc entry with 'flags: '
 # by given no "-p yes" option.
-sudo docker run --rm --privileged ${DOCKER_REPO}:register --reset
+sudo docker run --rm --privileged "${SLUG}:register" --reset
 cat /proc/sys/fs/binfmt_misc/qemu-aarch64
 grep -q '^flags: $' /proc/sys/fs/binfmt_misc/qemu-aarch64
 
@@ -62,13 +49,12 @@ grep -q '^flags: $' /proc/sys/fs/binfmt_misc/qemu-aarch64
 # multiarch/qemu-user-static:$from_arch-$to_arch image
 
 # /usr/bin/qemu-aarch64-static should be included.
-docker run --rm -t ${DOCKER_REPO}:aarch64 /usr/bin/qemu-aarch64-static --version
-docker run --rm -t ${DOCKER_REPO}:x86_64-aarch64 /usr/bin/qemu-aarch64-static --version
+docker run --rm -t "${SLUG}:aarch64" /usr/bin/qemu-aarch64-static --version
 
 # ------------------------------------------------
 # Integration test
 docker build --rm -t "test/integration/ubuntu" -<<EOF
-FROM ${DOCKER_REPO}:x86_64-aarch64 as qemu
+FROM ${SLUG}:aarch64 as qemu
 FROM arm64v8/ubuntu
 COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin
 EOF