Merge pull request moby#1829 from crazy-max/ghactions-generator

Refactor generators

.github/workflows/validate.yml

@@ -1,44 +1,44 @@
-name: validate
-
-on:
-  push:
-    branches:
-      - 'master'
-    tags:
-      - 'v*'
-      - 'dockerfile/*'
-  pull_request:
-    branches:
-      - 'master'
-
-env:
-  REPO_SLUG_ORIGIN: "moby/buildkit:latest"
-
-jobs:
-  validate:
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - script: ./hack/lint
-          - script: ./hack/validate-vendor
-          - script: ./hack/validate-generated-files
-          - script: ./hack/validate-shfmt
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
-      -
-        name: Run
-        run: |
-          ${{ matrix.script }}
-      -
-        name: Dump context
-        if: always()
-        uses: crazy-max/ghaction-dump-context@v1
+name: validate
+
+on:
+  push:
+    branches:
+      - 'master'
+    tags:
+      - 'v*'
+      - 'dockerfile/*'
+  pull_request:
+    branches:
+      - 'master'
+
+env:
+  REPO_SLUG_ORIGIN: "moby/buildkit:latest"
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - script: ./hack/lint
+          - script: ./hack/validate-vendor
+          - script: ./hack/validate-generated-files
+          - script: ./hack/validate-shfmt
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          driver-opts: image=${{ env.REPO_SLUG_ORIGIN }}
+      -
+        name: Run
+        run: |
+          ${{ matrix.script }}
+      -
+        name: Dump context
+        if: always()
+        uses: crazy-max/ghaction-dump-context@v1

.travis.yml

@@ -4,6 +4,7 @@ language: minimal
 env:
   global:
     - DOCKER_CHANNEL="stable"
+    - DOCKER_CLI_EXPERIMENTAL="enabled"
     - PLATFORMS="linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le"
     - PREFER_BUILDCTL="1"
 
@@ -46,7 +47,7 @@ jobs:
        - TESTPKGS=./frontend ./hack/test
       name: "Unit Tests & Lint & Vendor & Proto"
     - script:
-       - TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=6 --timeout=20m' ./hack/test
+       - TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=5 --timeout=30m' ./hack/test
       name: "Dockerfile integration tests"
     - script: TESTPKGS=./frontend/dockerfile ./hack/test dockerfile
       name: "External Dockerfile tests"

hack/shfmt

@@ -1,26 +1,10 @@
 #!/usr/bin/env bash
 
 . $(dirname $0)/util
-set -eu -o pipefail -x
+set -e
 
-: ${CONTINUOUS_INTEGRATION=}
-
-progressFlag=""
-if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
-
-case $buildmode in
-  "buildkit")
-    buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
-      --opt target=update \
-      --opt filename=./hack/dockerfiles/shfmt.Dockerfile \
-      --output type=local,dest=.
-    ;;
-  "docker-buildkit")
-    export DOCKER_BUILDKIT=1
-    docker build $progressFlag -f ./hack/dockerfiles/shfmt.Dockerfile --target update -o . .
-    ;;
-  *)
-    echo "Unsupported build mode: $buildmode" >&2
-    exit 1
-    ;;
-esac
+buildxCmd build \
+  --target "update" \
+  --output "type=local,dest=." \
+  --file "./hack/dockerfiles/shfmt.Dockerfile" \
+  .

hack/update-generated-files

@@ -1,40 +1,17 @@
 #!/usr/bin/env bash
 
 . $(dirname $0)/util
-set -eu -o pipefail -x
-
-: ${CONTINUOUS_INTEGRATION=}
-
-progressFlag=""
-if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
+set -eu
 
 gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' go.mod)
-case $buildmode in
-  "buildkit")
-    output=$(mktemp -d -t buildctl-output.XXXXXXXXXX)
-    buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
-      --opt build-arg:GOGO_VERSION=$gogo_version \
-      --opt target=update \
-      --opt filename=./hack/dockerfiles/generated-files.Dockerfile \
-      --output type=local,dest=$output
-    cp -R "$output/generated-files/" .
-    rm -rf $output
-    ;;
-  "docker-buildkit")
-    iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-    export DOCKER_BUILDKIT=1
-    docker build --build-arg GOGO_VERSION=$gogo_version --iidfile $iidfile -f ./hack/dockerfiles/generated-files.Dockerfile --target update --force-rm .
-    iid=$(cat $iidfile)
-    cid=$(docker create $iid noop)
-
-    docker export $cid | tar -xf - --strip-components=1 generated-files
+output=$(mktemp -d -t buildctl-output.XXXXXXXXXX)
 
-    docker rm $cid
+buildxCmd build \
+  --target "update" \
+  --build-arg "GOGO_VERSION=$gogo_version" \
+  --output "type=local,dest=$output" \
+  --file "./hack/dockerfiles/generated-files.Dockerfile" \
+  .
 
-    rm -f $iidfile
-    ;;
-  *)
-    echo "Unsupported build mode: $buildmode" >&2
-    exit 1
-    ;;
-esac
+cp -R "$output/generated-files/" .
+rm -rf $output

hack/update-vendor

@@ -1,41 +1,16 @@
 #!/usr/bin/env bash
 
 . $(dirname $0)/util
-set -eu -o pipefail -x
+set -eu
 
-: ${CONTINUOUS_INTEGRATION=}
+output=$(mktemp -d -t buildx-output.XXXXXXXXXX)
 
-progressFlag=""
-if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi
+buildxCmd build \
+  --target "update" \
+  --output "type=local,dest=$output" \
+  --file "./hack/dockerfiles/vendor.Dockerfile" \
+  .
 
-case $buildmode in
-  "buildkit")
-    output=$(mktemp -d -t buildctl-output.XXXXXXXXXX)
-    buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \
-      --opt target=update \
-      --opt filename=./hack/dockerfiles/vendor.Dockerfile \
-      --output type=local,dest=$output
-    rm -rf ./vendor
-    cp -R "$output"/out/* .
-    rm -rf $output
-    ;;
-  "docker-buildkit")
-    iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
-    export DOCKER_BUILDKIT=1
-    docker build --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --target update --force-rm .
-    iid=$(cat $iidfile)
-    cid=$(docker create $iid noop)
-    rm -rf ./vendor
-
-    docker cp $cid:/out/go.mod .
-    docker cp $cid:/out/go.sum .
-    docker cp $cid:/out/vendor .
-
-    docker rm $cid
-    rm -f $iidfile
-    ;;
-  *)
-    echo "Unsupported build mode: $buildmode" >&2
-    exit 1
-    ;;
-esac
+rm -rf ./vendor
+cp -R "$output"/out/* .
+rm -rf $output

hack/util

@@ -1,6 +1,8 @@
 #!/usr/bin/env sh
 export BUILDX_NO_DEFAULT_LOAD=true
 
+: ${PREFER_BUILDCTL=}
+: ${PREFER_LEGACY=}
 : ${CONTINUOUS_INTEGRATION=}
 : ${CI=}
 : ${GITHUB_ACTIONS=}
@@ -14,7 +16,27 @@ if [ "$CONTINUOUS_INTEGRATION" = "true" ] || [ "$CI" = "true" ]; then
   progressFlag="--progress=plain"
 fi
 
-buildmode="buildkit"
+newerEqualThan() { # $1=minimum wanted version $2=actual-version
+  [ "$1" = "$(printf "$1\n$2" | sort -V | head -n 1)" ]
+}
+
+buildmode="legacy"
+if [ "$PREFER_BUILDCTL" = "1" ]; then
+  buildmode="buildkit"
+else
+  serverVersion=$(docker info --format '{{.ServerVersion}}')
+  experimental=$(docker info --format '{{.ExperimentalBuild}}')
+  if [ "$PREFER_LEGACY" != "1" ] && (newerEqualThan "18.09" $serverVersion ||
+    (newerEqualThan "18.06" $serverVersion && [ "true" = "$experimental" ]) ||
+    [ "$DOCKER_BUILDKIT" = "1" ]); then
+    buildmode="docker-buildkit"
+  fi
+fi
+if [ "$buildmode" = "legacy" ]; then
+  echo >&2 "ERROR: \"legacy\" buildmode is no longer supported. Please enable DOCKER_BUILDKIT or install standalone BuildKit"
+  exit 1
+fi
+
 buildxCmd() {
   if docker buildx version >/dev/null 2>&1; then
     set -x