Rebuild changes from buildkite-plugins#310 as single commit

This change rebuilds the changes from the unmerged PR buildkite-plugins#310 so that we
can reapply it more cleanly when pulling in changes from the upstream
repo in the future.

README.md

@@ -2,6 +2,11 @@
 
 A [Buildkite plugin](https://buildkite.com/docs/agent/v3/plugins) that lets you build, run and push build steps using [Docker Compose](https://docs.docker.com/compose/).
 
+This fork of Buildkite's default plugin. It allows for the skipping of build
+steps if an image with the specified tag already exists. This can dramatically
+speed up certain steps such as dependency installs or asset builds if your
+images are tagged with a proper cache key.
+
 * Containers are built, run and linked on demand using Docker Compose
 * Containers are namespaced to each build job, and cleaned up after use
 * Supports pre-building of images, allowing for fast parallel builds across distributed agents
@@ -374,6 +379,22 @@ steps:
 In the example above, the `myservice_intermediate:buildkite-build-${BUILDKITE_BUILD_NUMBER}` is one group named "intermediate", and `myservice:${BUILDKITE_BRANCH}` and `myservice:latest`
 are another (with a default name). The first successfully downloaded image in each group will be used as a cache.
 
+## Reusing previously built images
+
+If an build image has already been built previously, you can skip it if it
+already exists on the repository.
+
+```yaml
+steps:
+ - label: ":docker: Build an image"
+ plugins:
+ - docker-compose#v3.8.0:
+ build: app
+ image-repository: index.docker.io/myorg/myrepo
+ cache-from: app:index.docker.io/myorg/myrepo/myapp:123
+ use-prior-image: true
+```
+
 ## Configuration
 
 ### Main Commands
@@ -715,6 +736,11 @@ It will add the `--ssh` option to the build command with the passed value (if `t
 
 All elements in this array will be passed literally to the `build` command as parameters of the [`--secrets` option](https://docs.docker.com/engine/reference/commandline/buildx_build/#secret). Note that you must have BuildKit enabled for this option to have any effect and special `RUN` stanzas in your Dockerfile to actually make use of them.
 
+### `use-prior-image` (optional, build only)
+
+When true, the build step will be skipped if the `cache-from` image exists on
+the remote repository. The run will use the existing image in later steps.
+
 ## Developing
 
 To run the tests:

commands/build.sh

@@ -76,6 +76,21 @@ if [[ "$(plugin_read_config NO_CACHE "false")" == "false" ]] ; then
  # The variable with this name will hold an array of group names:
  cache_image_name="$(service_name_cache_from_var "$service_name")"
 
+ # If we're using prior images and we find an exact match, terminate and skip
+ # all further processing. This won't work for a multi-image build step, only
+ # a single service can be built this way
+ if [[ "$(plugin_read_config USE_PRIOR_IMAGE "false")" == "true" ]] && \
+ docker manifest inspect "$service_image" > /dev/null; then
+ echo ":docker: Found an image! Marking and skipping $service_image"
+ set_prebuilt_image "$service_name" "$service_image"
+
+ for service_alias in $(plugin_read_list BUILD_ALIAS) ; do
+ set_prebuilt_image "$service_alias" "$service_image"
+ done
+
+ exit 0
+ fi
+
  if [[ -n ${!cache_image_name+x} ]]; then
  if [[ "$(named_array_values "${cache_image_name}")" =~ ${cache_from_group_name} ]]; then
  continue # skipping since there's already a pulled cache image for this service+group

plugin.yml

@@ -120,6 +120,8 @@ configuration:
  minimum: 1
  workdir:
  type: string
+ use-prior-image:
+ type: boolean
  anyOf:
  - required:
  - run
@@ -158,3 +160,4 @@ configuration:
  volumes: [ run ]
  wait: [ run ]
  workdir: [ run ]
+ use-prior-image: [ build ]

tests/build.bats

@@ -515,7 +515,7 @@ load '../lib/shared'
  assert_output --partial "- my.repository:port/myservice_cache:branch-name"
  refute_output --partial "- my.repository:port/myservice_cache:latest"
  assert_output --partial "built helloworld"
- 
+
  unstub docker
  unstub docker-compose
 }
@@ -920,3 +920,57 @@ load '../lib/shared'
  assert_output --partial "built myservice"
  unstub docker-compose
 }
+
+@test "Build with use-prior-image and a previously-built image" {
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml"
+ export BUILDKITE_JOB_ID=1111
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas
+ export BUILDKITE_PIPELINE_SLUG=test
+ export BUILDKITE_BUILD_NUMBER=1
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USE_PRIOR_IMAGE=true
+
+ stub docker \
+ "manifest inspect my.repository/myservice_cache:latest : exit 0"
+
+ stub buildkite-agent \
+ "meta-data set docker-compose-plugin-built-image-tag-helloworld-tests/composefiles/docker-compose.v3.2.yml my.repository/myservice_cache:latest : echo set image metadata for myservice"
+
+ run $PWD/hooks/command
+
+ assert_success
+ assert_output --partial "Found an image! Marking and skipping my.repository/myservice_cache:latest"
+ refute_output --partial "pulled cache image"
+ unstub docker
+ unstub buildkite-agent
+}
+
+@test "Build with use-prior-image and no previously-built image found" {
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CONFIG="tests/composefiles/docker-compose.v3.2.yml"
+ export BUILDKITE_JOB_ID=1111
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_BUILD_0=helloworld
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_CACHE_FROM_0=helloworld:my.repository/myservice_cache:latest
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_IMAGE_REPOSITORY=my.repository/llamas
+ export BUILDKITE_PIPELINE_SLUG=test
+ export BUILDKITE_BUILD_NUMBER=1
+ export BUILDKITE_PLUGIN_DOCKER_COMPOSE_USE_PRIOR_IMAGE=true
+
+ stub docker \
+ "manifest inspect my.repository/myservice_cache:latest : exit 1" \
+ "pull my.repository/myservice_cache:latest : exit 1"
+ stub docker-compose \
+ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml build --pull helloworld : echo built helloworld" \
+ "-f tests/composefiles/docker-compose.v3.2.yml -p buildkite1111 -f docker-compose.buildkite-1-override.yml push helloworld : echo pushed helloworld"
+ stub buildkite-agent \
+ "meta-data set docker-compose-plugin-built-image-tag-helloworld-tests/composefiles/docker-compose.v3.2.yml my.repository/llamas:test-helloworld-build-1 : echo set image metadata for myservice"
+
+ run $PWD/hooks/command
+
+ assert_success
+ refute_output --partial "Found matching image cache_from__helloworld, marking and skipping build"
+ assert_output --partial "pushed helloworld"
+ unstub docker
+ unstub docker-compose
+ unstub buildkite-agent
+}