diff --git a/.gitignore b/.gitignore index 879421e26659..8c9a9545f55c 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ cluster/local/certs **.pem **.crt **.csr +_out diff --git a/.travis.yml b/.travis.yml index 87e60ebc0ea0..ffbb860ed553 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,43 +11,28 @@ go: - 1.8 - 1.9 -before_install: -- sudo add-apt-repository ppa:jacob/virtualisation -y -- sudo apt-get update -q +env: +- BUILDER_NAME="kubevirt/${TRAVIS_JOB_ID}builder" install: - - git reset --hard # we are caching the vendor folder, make sure we see the new vendor.json file - - go get github.com/mattn/goveralls - - go get -u github.com/Masterminds/glide - - go get golang.org/x/tools/cmd/goimports - - go get -u mvdan.cc/sh/cmd/shfmt - - go get -u github.com/golang/mock/gomock - - go get -u github.com/rmohr/mock/mockgen - - go get -u github.com/rmohr/go-swagger-utils/swagger-doc - - go get -u github.com/onsi/ginkgo/ginkgo - - go get -u k8s.io/code-generator/cmd/deepcopy-gen - - go get -u k8s.io/code-generator/cmd/defaulter-gen - - sudo apt-get install libvirt-dev make - - make sync + - git reset --hard + - ./hack/dockerized make sync script: -- make fmt fmt-bash +- ./hack/dockerized make fmt fmt-bash - if git diff --name-only | grep '.*.go'; then echo "It seems like you did not run `make fmt fmt-bash`. Please run it and commit the changes"; false; fi -- make generate -- make fmt +- ./hack/dockerized make generate fmt - if git diff --name-only | grep 'generated.*.go'; then echo "Content of generated files changed. Please regenerate and commit them."; false; fi - if git diff --name-only | grep 'swagger.json' ; then echo "Content of generated files changed. Please regenerate and commit them." ; false ; fi - if diff <(git grep -c '') <(git grep -cI '') | egrep -v 'docs/.*\.png|swagger-ui' | grep '^<'; then echo "Binary files are present in git repostory."; false; fi -- make check -- make build -- if [[ $TRAVIS_REPO_SLUG == "kubevirt/kubevirt" ]]; then $HOME/gopath/bin/goveralls - -service=travis-ci -package=./pkg/... -ignore=$(find -regextype posix-egrep -regex ".*generated_mock.*\.go|.*swagger_generated\.go" -printf - "%P\n" | paste -d, -s) ; else make test; fi -- make apidocs +- ./hack/dockerized make check +- ./hack/dockerized make build +- if [[ $TRAVIS_REPO_SLUG == "kubevirt/kubevirt" ]]; then ./hack/dockerized make goveralls TRAVIS_JOB_ID=${TRAVIS_JOB_ID} TRAVIS_PULL_REQUEST=${TRAVIS_PULL_REQUEST} TRAVIS_BRANCH=${TRAVIS_BRANCH}; else ./hack/dockerized make test; fi +- ./hack/dockerized make apidocs - make manifests DOCKER_TAG=$TRAVIS_TAG # falls back to latest if not on a tag cache: @@ -74,9 +59,8 @@ deploy: secure: dy3AlSki7p9sRjItOiWoPv4s28+hZ9wMcuvtZv0bWHbEZ7y/EvzXcwWmn+/Tjjk4Np59LImUjJw1E5Yv2Q16ViuNwKbX7TA7fu+wAcTz4DF4wFLIrJBcCRetiud4175aJ1ylOvmtHOuWHfo6Wx5HRxRT0fLcMFdSoagpe4Dh7BJYfI4Sgu/jghMMJ4wcDsMLZqi6RRM6jywFXQXMmBY87E57/ZLlXZ5Nud5RoWXJxfFfc/vTn/frCBu7r+JW8ayGeGVPEkEMtWX1huWnIvVjNu1twuZnL3zWmLNmpFXfk3j/gLbdIwLn3jgtariNz/Rvigqaj18F0Fa/QNL6VcruqgXd6ryWxmGm9sD6MYC3kk0HoqwSI7WBf88tlfz27FR+cHFZtjTBXnZHBtM5skuzUpT4kUKhLkwcfgA5PmoW/7NcmnN0eKvOQuQYq/8pREwdmYAvYsGPWJZjNYHBCv9KRT1lDRN5jRLMi5wn7NWFMR1p+mbdHyd7i43zM8gnumqvGF/chZ9PEH38hnm835RZAwnzdJ3EQ4YFAaOE2AaUBe9Yw1FP6f22MOC8+N9CY+a1AlGQBmovPjaf+x4f+Jx6y9jjxahJpoXxEKgeFvfSVUoKbkMPMOHxMAb0UL9Rqq3BhlLnPQIjcwdCZU/+wTcV/sFHdcnIqdl9/wb/L+mMjOs= file_glob: true file: - - cmd/virtctl/virtctl-* + - _out/cmd/virtctl/virtctl-* - manifests/release/kubevirt.yaml - - manifests/release/spice-proxy.yaml prerelease: true on: tags: true diff --git a/Makefile b/Makefile index b0670e071dca..3e2efca127c8 100644 --- a/Makefile +++ b/Makefile @@ -18,14 +18,13 @@ generate: sync tools/openapispec/openapispec --dump-api-spec-path api/openapi-spec/swagger.json apidocs: generate - docker run -u `stat -c "%u" hack/gen-swagger-doc/` --rm \ - -v ${PWD}:/home/gradle/kubevirt:rw,z \ - -w /home/gradle/kubevirt \ - gradle \ - bash hack/gen-swagger-doc/gen-swagger-docs.sh v1 html + ./hack/gen-swagger-doc/gen-swagger-docs.sh v1 html build: checksync fmt vet compile +goveralls: + ./hack/goveralls.sh + compile: ./hack/build-go.sh install ${WHAT} @@ -42,11 +41,11 @@ test: build ./hack/build-go.sh test ${WHAT} functest: - ./hack/build-go.sh functest ${WHAT} + ./hack/functests.sh clean: ./hack/build-go.sh clean ${WHAT} - rm ./bin -rf + rm _out/ -rf rm tools/openapispec/openapispec -rf distclean: clean @@ -56,8 +55,8 @@ distclean: clean glide cc checksync: - test -f .glide.yaml.hash || ${HASH} glide.yaml > .glide.yaml.hash - if [ "`${HASH} glide.yaml`" != "`cat .glide.yaml.hash`" ]; then \ + @test -f .glide.yaml.hash || ${HASH} glide.yaml > .glide.yaml.hash + @if [ "`${HASH} glide.yaml`" != "`cat .glide.yaml.hash`" ]; then \ glide cc; \ glide update --strip-vendor; \ ${HASH} glide.yaml > .glide.yaml.hash; \ @@ -70,7 +69,8 @@ sync: glide install --strip-vendor ${HASH} glide.lock > .glide.lock.hash -docker: build +docker: + hack/dockerized make build ./hack/build-docker.sh build ${WHAT} publish: docker diff --git a/docs/getting-started.md b/docs/getting-started.md index a3fd1a28d5cb..524bf99aee38 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -6,9 +6,25 @@ A quick start guide to get KubeVirt up and running inside Vagrant. **Note:** Fedora 24 is known to have a bug which affects our vagrant setup. - ## Building +### Dockerized building and code generating + +All go-related make targets can be invoked in an included container. To build +the project within the builder-container, run + +```bash +./hack/dockerized make fmt fmt-bash build +``` + +To generate code, run + +```bash +./hack/dockerized make generate +``` + +If you want to build KubeVirt without docker, read on. + ### Go (1.8 or higher) [Go](https://golang.org) needs to be setup to be able to compile the sources. diff --git a/hack/build-docker.sh b/hack/build-docker.sh index 2b334ba5d5d6..67b32f3e1702 100755 --- a/hack/build-docker.sh +++ b/hack/build-docker.sh @@ -20,6 +20,11 @@ set -e source hack/config.sh +KUBEVIRT_DIR="$( + cd "$(dirname "$0")/../" + pwd +)" +OUT_DIR=$KUBEVIRT_DIR/_out/cmd/ if [ -z "$1" ]; then target="build" @@ -38,21 +43,22 @@ fi for arg in $args; do if [ "${target}" = "build" ]; then + BIN_NAME=$(basename $arg) + rsync -avz --exclude "**/*.md" --exclude "**/*.go" --exclude "**/.*" $arg/ ${OUT_DIR}/${BIN_NAME}/ # TODO the version of docker we're using in our vagrant dev environment # does not support using ARGS in FROM field. # https://docs.docker.com/engine/reference/builder/#understand-how-arg-and-from-interact # Because of this we have to manipulate the Dockerfile for kubevirt containers # that depend on other kubevirt containers. - cat $arg/Dockerfile | sed s/registry-disk-v1alpha/registry-disk-v1alpha\:$docker_tag/g >$arg/.GeneratedDockerfile + cat $arg/Dockerfile | sed s/registry-disk-v1alpha/registry-disk-v1alpha\:$docker_tag/g >${OUT_DIR}/${BIN_NAME}/Dockerfile ( - cd $arg - docker $target -t ${docker_prefix}/$(basename $arg):${docker_tag} -f .GeneratedDockerfile . + cd ${OUT_DIR}/${BIN_NAME}/ + docker $target -t ${docker_prefix}/${BIN_NAME}:${docker_tag} . ) - rm $arg/.GeneratedDockerfile elif [ "${target}" = "push" ]; then ( - cd $arg - docker $target ${docker_prefix}/$(basename $arg):${docker_tag} + cd ${OUT_DIR}/${BIN_NAME}/ + docker $target ${docker_prefix}/${BIN_NAME}:${docker_tag} ) fi done diff --git a/hack/build-go.sh b/hack/build-go.sh index c67f970edab5..5d49322c591e 100755 --- a/hack/build-go.sh +++ b/hack/build-go.sh @@ -20,6 +20,11 @@ set -e source hack/config.sh +KUBEVIRT_DIR="$( + cd "$(dirname "$0")/../" + pwd +)" +OUT_DIR=$KUBEVIRT_DIR/_out/cmd if [ -z "$1" ]; then target="install" @@ -30,6 +35,7 @@ fi if [ $# -eq 0 ]; then args=$binaries + build_tests="true" else args=$@ fi @@ -42,12 +48,6 @@ if [ $# -eq 0 ]; then cd pkg go ${target} -v ./... ) - elif [ "${target}" = "functest" ]; then - ( - cd tests - go test -kubeconfig=${kubeconfig} -timeout 30m ${FUNC_TEST_ARGS} - ) - exit else ( cd pkg @@ -61,6 +61,11 @@ if [ $# -eq 0 ]; then fi # handle binaries + +if [ "${target}" = "install" ]; then + rm -rf ${OUT_DIR} +fi + for arg in $args; do if [ "${target}" = "test" ]; then ( @@ -69,16 +74,14 @@ for arg in $args; do ) elif [ "${target}" = "install" ]; then eval "$(go env)" - ARCHBIN=$(basename $arg)-$(git describe --always)-$GOHOSTOS-$GOHOSTARCH - ALIASLNK=$(basename $arg) - rm $arg/$ALIASLNK $arg/$(basename $arg)-*-$GOHOSTOS-$GOHOSTARCH || : + BIN_NAME=$(basename $arg) + ARCHBIN=${BIN_NAME}-$(git describe --always)-${GOHOSTOS}-${GOHOSTARCH} + mkdir -p ${OUT_DIR}/${BIN_NAME} ( cd $arg - GOBIN=$PWD go build -o $ARCHBIN + go build -o ${OUT_DIR}/${BIN_NAME}/${ARCHBIN} + (cd ${OUT_DIR}/${BIN_NAME} && ln -sf ${ARCHBIN} ${BIN_NAME}) ) - mkdir -p bin - ln -sf $ARCHBIN $arg/$ALIASLNK - ln -sf ../$arg/$ARCHBIN bin/$ALIASLNK else ( cd $arg @@ -86,3 +89,9 @@ for arg in $args; do ) fi done + +if [[ "${target}" == "install" && "${build_tests}" == "true" ]]; then + mkdir -p ${KUBEVIRT_DIR}/_out/tests/ + ginkgo build ${KUBEVIRT_DIR}/tests + mv ${KUBEVIRT_DIR}/tests/tests.test ${KUBEVIRT_DIR}/_out/tests/ +fi diff --git a/hack/docker-builder/Dockerfile b/hack/docker-builder/Dockerfile new file mode 100644 index 000000000000..63bb89cd5652 --- /dev/null +++ b/hack/docker-builder/Dockerfile @@ -0,0 +1,33 @@ +FROM fedora:27 + +ENV LIBVIRT_VERSION 3.7.0 + +RUN dnf -y install libvirt-devel-${LIBVIRT_VERSION} make git mercurial sudo gcc findutils gradle rsync-daemon rsync && \ + dnf -y clean all + +ENV GIMME_GO_VERSION=1.9.2 + +RUN mkdir -p /gimme && curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | HOME=/gimme bash >> /etc/profile.d/gimme.sh + +ENV GOPATH="/go" GOBIN="/usr/bin" + +ADD rsyncd.conf /etc/rsyncd.conf + +RUN \ + mkdir -p /go && \ + source /etc/profile.d/gimme.sh && \ + go get github.com/mattn/goveralls && \ + go get -u github.com/Masterminds/glide && \ + go get golang.org/x/tools/cmd/goimports && \ + go get -u mvdan.cc/sh/cmd/shfmt && \ + go get -u github.com/golang/mock/gomock && \ + go get -u github.com/rmohr/mock/mockgen && \ + go get -u github.com/rmohr/go-swagger-utils/swagger-doc && \ + go get -u github.com/onsi/ginkgo/ginkgo && \ + go get -u k8s.io/code-generator/cmd/deepcopy-gen && \ + go get -u k8s.io/code-generator/cmd/defaulter-gen + + +ADD entrypoint.sh /entrypoint.sh + +ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/hack/docker-builder/entrypoint.sh b/hack/docker-builder/entrypoint.sh new file mode 100755 index 000000000000..247ba188cdb4 --- /dev/null +++ b/hack/docker-builder/entrypoint.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source /etc/profile.d/gimme.sh +export GOPATH="/root/go" +"$@" diff --git a/hack/docker-builder/rsyncd.conf b/hack/docker-builder/rsyncd.conf new file mode 100644 index 000000000000..240768952d29 --- /dev/null +++ b/hack/docker-builder/rsyncd.conf @@ -0,0 +1,14 @@ +gid = 0 +uid = 0 +log file = /dev/stdout +reverse lookup = no +[build] + hosts allow = * + read only = false + path = /root/go/src/kubevirt.io/kubevirt/ + comment = input sources +[out] + hosts allow = * + read only = false + path = /root/go/src/kubevirt.io/kubevirt/_out + comment = build output diff --git a/hack/dockerized b/hack/dockerized new file mode 100755 index 000000000000..98544e6b630e --- /dev/null +++ b/hack/dockerized @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +DIR="$( + cd "$(dirname "$0")" + pwd +)" +DOCKER_DIR=${DIR}/docker-builder +KUBEVIRT_DIR="$( + cd "$DIR/../" + pwd +)" + +BUILDER=${BUILDER_NAME:-kubevirtbuilder} +RSYNCD_PORT=${RSYNCD_PORT:-10873} + +# Build the build container +(cd ${DOCKER_DIR} && docker build . -q -t ${BUILDER}) + +# Create the persistent docker volume +if [ -z "$(docker volume list | grep ${BUILDER})" ]; then + docker volume create --name ${BUILDER} +fi + +# Make sure that the output directory exists +docker run -v "${BUILDER}:/root:rw,z" --rm -it ${BUILDER} mkdir -p /root/go/src/kubevirt.io/kubevirt/_out + +# Start an rsyncd instance and make sure it gets stopped after the script exits +RSYNC_CID=$(docker run -d -v "${BUILDER}:/root:rw,z" -p 127.0.0.1:${RSYNCD_PORT}:873 ${BUILDER} /usr/bin/rsync --no-detach --daemon --verbose) +function finish() { + docker stop ${RSYNC_CID} + docker rm ${RSYNC_CID} +} +trap finish EXIT +# TODO really check if rsyncd comes up. For now just give it some time to come up +sleep 1 + +_rsync() { + rsync -avl "$@" +} + +# Copy kubevirt into the persistent docker volume +_rsync --delete --exclude '.glide*' --exclude "cluster/vagrant/.kubeconfig" --exclude "cluster/vagrant/.kubectl" --exclude "vendor" --exclude "_out" --exclude ".vagrant" ${KUBEVIRT_DIR}/ "rsync://root@127.0.0.1:${RSYNCD_PORT}/build" + +# Run the command +docker run --rm -v "${BUILDER}:/root:rw,z" -w "/root/go/src/kubevirt.io/kubevirt" -it ${BUILDER} "$@" + +# Copy the whole kubevirt data out to get generated sources and formatting changes +_rsync --exclude '.glide*' --exclude "vendor" --exclude "_out" --exclude ".vagrant" --exclude ".git" "rsync://root@127.0.0.1:${RSYNCD_PORT}/build" ${KUBEVIRT_DIR}/ +# Copy the build output out of the container, make sure that _out exactly matches the build result +_rsync --delete "rsync://root@127.0.0.1:${RSYNCD_PORT}/out" $KUBEVIRT_DIR/_out diff --git a/hack/functests.sh b/hack/functests.sh new file mode 100755 index 000000000000..2637d5282c61 --- /dev/null +++ b/hack/functests.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# This file is part of the KubeVirt project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Copyright 2017 Red Hat, Inc. +# + +set -e + +source hack/config.sh +KUBEVIRT_DIR="$( + cd "$(dirname "$0")/../" + pwd +)" + +${KUBEVIRT_DIR}/_out/tests/tests.test -kubeconfig=${kubeconfig} -test.timeout 30m ${FUNC_TEST_ARGS} diff --git a/hack/gen-swagger-doc/deploy.sh b/hack/gen-swagger-doc/deploy.sh index cf420b85b5b2..23b345c6353f 100755 --- a/hack/gen-swagger-doc/deploy.sh +++ b/hack/gen-swagger-doc/deploy.sh @@ -10,7 +10,7 @@ git clone \ "https://${API_REFERENCE_PUSH_TOKEN}@${GITHUB_FQDN}/${API_REF_REPO}.git" \ "${API_REF_DIR}" >/dev/null 2>&1 rm -rf "${API_REF_DIR}/content/"* -cp -f hack/gen-swagger-doc/html5/content/*.html "${API_REF_DIR}/content/" +cp -f _out/apidocs/html/*.html "${API_REF_DIR}/content/" cd "${API_REF_DIR}" diff --git a/hack/gen-swagger-doc/gen-swagger-docs.sh b/hack/gen-swagger-doc/gen-swagger-docs.sh index 25709427d7fc..1d6cb762addd 100755 --- a/hack/gen-swagger-doc/gen-swagger-docs.sh +++ b/hack/gen-swagger-doc/gen-swagger-docs.sh @@ -8,6 +8,11 @@ set -o errexit set -o nounset set -o pipefail +KUBEVIRT_DIR="$( + cd "$(dirname "$0")/../../" + pwd +)" + VERSION="${1:-v1}" OUTPUT_FORMAT="${2:-html}" if [ "$OUTPUT_FORMAT" = "html" ]; then @@ -79,4 +84,7 @@ elif [ "$OUTPUT_FORMAT" = "markdown" ]; then cd - fi +mkdir -p ${KUBEVIRT_DIR}/_out/apidocs/html +mv ${WORKDIR}/html5/content/* ${KUBEVIRT_DIR}/_out/apidocs/html +mv ${WORKDIR}/*.adoc ${KUBEVIRT_DIR}/_out/apidocs/ echo "SUCCESS" diff --git a/hack/goveralls.sh b/hack/goveralls.sh new file mode 100755 index 000000000000..28af570c9f08 --- /dev/null +++ b/hack/goveralls.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +goveralls -service=travis-ci -package=./pkg/... -ignore=$(find -regextype posix-egrep -regex ".*generated_mock.*\.go|.*swagger_generated\.go" -printf "%P\n" | paste -d, -s) -v diff --git a/pkg/virt-dhcp/monitor_test.go b/pkg/virt-dhcp/monitor_test.go index 3c5f74461737..23eca1055f58 100644 --- a/pkg/virt-dhcp/monitor_test.go +++ b/pkg/virt-dhcp/monitor_test.go @@ -62,7 +62,7 @@ var _ = Describe("Virt-DHCP Monitor test", func() { dir := os.Getenv("PWD") dir = strings.TrimSuffix(dir, "pkg/virt-dhcp") processName := "fake-dnsmasq-process" - processPath := dir + "/cmd/fake-dnsmasq-process/" + processName + processPath := dir + "/_out/cmd/fake-dnsmasq-process/" + processName processArgs := []string{"-k", "-d", "--strict-order", "--bind-dynamic"} StartProcess := func() { diff --git a/pkg/virt-launcher/monitor_test.go b/pkg/virt-launcher/monitor_test.go index 82af9486dbc5..be1404888092 100644 --- a/pkg/virt-launcher/monitor_test.go +++ b/pkg/virt-launcher/monitor_test.go @@ -45,7 +45,7 @@ var _ = Describe("VirtLauncher", func() { dir = strings.TrimSuffix(dir, "pkg/virt-launcher") processName := "fake-qemu-process" - processPath := dir + "/cmd/fake-qemu-process/" + processName + processPath := dir + "/_out/cmd/fake-qemu-process/" + processName processStarted := false