From 2870e0e91e453ed4dd818ea6a3f11c7ac2b0b1e7 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 17 Feb 2023 09:40:21 +0100 Subject: [PATCH] test: use gotestsum to generate reports Signed-off-by: CrazyMax --- .github/workflows/.test.yml | 46 +++++++++-------------------------- .github/workflows/test-os.yml | 26 +++++++++++++++++--- .gitignore | 1 - Dockerfile | 9 +++++++ hack/test | 19 ++++++--------- 5 files changed, 51 insertions(+), 50 deletions(-) diff --git a/.github/workflows/.test.yml b/.github/workflows/.test.yml index 2e6eea1f06ff..ad40ed8b812c 100644 --- a/.github/workflows/.test.yml +++ b/.github/workflows/.test.yml @@ -31,6 +31,7 @@ env: SETUP_BUILDX_VERSION: "latest" SETUP_BUILDKIT_IMAGE: "moby/buildkit:latest" TESTFLAGS: "-v --parallel=6 --timeout=30m" + GOTESTSUM_FORMAT: "standard-verbose" jobs: prepare: @@ -143,6 +144,7 @@ jobs: name: Test continue-on-error: ${{ matrix.tags == 'nydus' }} run: | + export TEST_REPORT_SUFFIX=-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.kind }}-${{ matrix.worker }}-${{ matrix.tags }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]') if [ -n "${{ matrix.tags }}" ]; then TESTFLAGS="${TESTFLAGS} --tags=${{ matrix.tags }}" export BUILDKITD_TAGS="${{ matrix.tags }}" @@ -151,48 +153,22 @@ jobs: export TESTFLAGS="${TESTFLAGS} --run=//worker=${{ matrix.worker }}$" fi ./hack/test ${{ matrix.kind }} - mv ./coverage/coverage.txt ./coverage/coverage-${{ github.job }}-$(echo "${{ matrix.pkg }}-${{ matrix.skip-integration-tests }}-${{ matrix.kind }}-${{ matrix.worker }}-${{ matrix.tags }}" | tr -dc '[:alnum:]-\n\r' | tr '[:upper:]' '[:lower:]').txt env: TEST_COVERAGE: 1 TESTPKGS: ${{ matrix.pkg }} SKIP_INTEGRATION_TESTS: ${{ matrix.skip-integration-tests }} CACHE_FROM: type=gha,scope=${{ inputs.cache_scope }} - - - name: Upload coverage file - continue-on-error: ${{ matrix.tags == 'nydus' }} - uses: actions/upload-artifact@v3 - with: - name: coverage - path: ./coverage - - upload-coverage: - runs-on: ubuntu-20.04 - needs: - - run - steps: - - - name: Checkout - uses: actions/checkout@v3 - - - name: Download coverage files - uses: actions/download-artifact@v3 - with: - name: coverage - path: ./coverage - - - name: List coverage files - id: files - uses: actions/github-script@v6 - with: - result-encoding: string - script: | - return require('fs').readdirSync('./coverage', {withFileTypes: true}) - .filter(item => !item.isDirectory()) - .map(item => `./coverage/${item.name}`) - .join(','); - name: Send to Codecov + if: always() uses: codecov/codecov-action@v3 with: - files: ${{ steps.files.outputs.result }} + directory: ./bin/testreports flags: ${{ matrix.codecov_flags }} + - + name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: ./bin/testreports diff --git a/.github/workflows/test-os.yml b/.github/workflows/test-os.yml index f8bdce9da706..3b40bef9f080 100644 --- a/.github/workflows/test-os.yml +++ b/.github/workflows/test-os.yml @@ -21,6 +21,8 @@ on: env: GO_VERSION: "1.20" TESTFLAGS: "-v --parallel=6 --timeout=30m" + GOTESTSUM_VERSION: "v1.9.0" # same as one in Dockerfile + GOTESTSUM_FORMAT: "standard-verbose" jobs: test: @@ -42,6 +44,10 @@ jobs: with: go-version: "${{ env.GO_VERSION }}" cache: true + - + name: Install gotestsum + run: | + go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }} - name: Go mod run: | @@ -52,13 +58,27 @@ jobs: TMPDIR: ${{ runner.temp }} SKIP_INTEGRATION_TESTS: 1 run: | - mkdir -p ./coverage - go test -coverprofile=./coverage/coverage.txt -covermode=atomic ${TESTFLAGS} ./... + mkdir -p ./bin/testreports + gotestsum \ + --jsonfile="./bin/testreports/go-test-report.json" \ + --junitfile="./bin/testreports/junit-report.xml" \ + --packages="./..." \ + -- \ + "-coverprofile" "./bin/testreports/coverage.txt" \ + "-covermode" "atomic" ${TESTFLAGS} shell: bash - name: Send to Codecov + if: always() uses: codecov/codecov-action@v3 with: - files: ./coverage/coverage.txt + directory: ./bin/testreports env_vars: RUNNER_OS flags: unit + - + name: Upload test reports + if: always() + uses: actions/upload-artifact@v3 + with: + name: test-reports + path: ./bin/testreports diff --git a/.gitignore b/.gitignore index bd9babd17443..eac5a4e2b7da 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,5 @@ # if you want to ignore files created by your editor/tools, # please consider a global .gitignore https://help.github.com/articles/ignoring-files bin -coverage .certs .tmp diff --git a/Dockerfile b/Dockerfile index b9d0f62d0c3f..f1806bce26cb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,7 @@ ARG NYDUS_VERSION=v2.1.0 ARG MINIO_VERSION=RELEASE.2022-05-03T20-36-08Z ARG MINIO_MC_VERSION=RELEASE.2022-05-04T06-07-55Z ARG AZURITE_VERSION=3.18.0 +ARG GOTESTSUM_VERSION=v1.9.0 ARG GO_VERSION=1.20 ARG ALPINE_VERSION=3.17 @@ -207,6 +208,12 @@ SHELL ["/bin/bash", "-c"] RUN wget https://github.com/dragonflyoss/image-service/releases/download/$NYDUS_VERSION/nydus-static-$NYDUS_VERSION-$TARGETOS-$TARGETARCH.tgz RUN mkdir -p /out/nydus-static && tar xzvf nydus-static-$NYDUS_VERSION-$TARGETOS-$TARGETARCH.tgz -C /out +FROM gobuild-base AS gotestsum +ARG GOTESTSUM_VERSION +RUN --mount=target=/root/.cache,type=cache \ + GOBIN=/out/ go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" && \ + /out/gotestsum --version + FROM buildkit-export AS buildkit-linux COPY --link --from=binaries / /usr/bin/ ENTRYPOINT ["buildkitd"] @@ -249,6 +256,8 @@ ENTRYPOINT ["/docker-entrypoint.sh"] ENV BUILDKIT_INTEGRATION_CONTAINERD_EXTRA="containerd-1.6=/opt/containerd-alt-16/bin" ENV BUILDKIT_INTEGRATION_SNAPSHOTTER=stargz ENV CGO_ENABLED=0 +ENV GOTESTSUM_FORMAT=standard-verbose +COPY --link --from=gotestsum /out/gotestsum /usr/bin/ COPY --link --from=minio /opt/bin/minio /usr/bin/ COPY --link --from=minio-mc /usr/bin/mc /usr/bin/ COPY --link --from=nydus /out/nydus-static/* /usr/bin/ diff --git a/hack/test b/hack/test index a7d5bf046105..5bf50fe2b1d2 100755 --- a/hack/test +++ b/hack/test @@ -11,7 +11,7 @@ set -eu -o pipefail : ${TEST_DOCKERFILE=} : ${TEST_DOCKERD=} : ${TEST_DOCKERD_BINARY=$(which dockerd)} -: ${TEST_COVERAGE=} +: ${TEST_REPORT_SUFFIX=} : ${TEST_KEEP_CACHE=} : ${DOCKERFILE_RELEASES=} : ${BUILDKIT_WORKER_RANDOM=} @@ -54,14 +54,11 @@ done iid="buildkit-tests" iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) -coverageVol="" -coverageFlags="" -if [ "$TEST_COVERAGE" = "1" ]; then - covdir="$(pwd)/coverage" - mkdir -p "$covdir" - coverageVol="-v $covdir:/coverage" - coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic" -fi +testReportsDir="$(pwd)/bin/testreports" +mkdir -p "$testReportsDir" +testReportsVol="-v $testReportsDir:/testreports" +gotestsumArgs="--format=standard-verbose --jsonfile=/testreports/go-test-report$TEST_REPORT_SUFFIX.json --junitfile=/testreports/junit-report$TEST_REPORT_SUFFIX.xml" +gotestArgs="-coverprofile=/testreports/coverage-report$TEST_REPORT_SUFFIX.txt -covermode=atomic" buildxCmd build $cacheFromFlags \ --build-arg BUILDKITD_TAGS \ @@ -78,7 +75,7 @@ if ! docker container inspect "$cacheVolume" >/dev/null 2>/dev/null; then fi if [ "$TEST_INTEGRATION" == 1 ]; then - cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_TEST_ENABLE_FEATURES -e BUILDKIT_TEST_DISABLE_FEATURES ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...}) + cid=$(docker create --rm -v /tmp $testReportsVol --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e SKIP_INTEGRATION_TESTS -e BUILDKIT_TEST_ENABLE_FEATURES -e GOTESTSUM_FORMAT ${BUILDKIT_INTEGRATION_SNAPSHOTTER:+"-eBUILDKIT_INTEGRATION_SNAPSHOTTER"} -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS --privileged $iid gotestsum $gotestsumArgs --packages="${TESTPKGS:-./...}" -- $gotestArgs ${TESTFLAGS:--v}) if [ "$TEST_DOCKERD" = "1" ]; then docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd fi @@ -118,7 +115,7 @@ if [ "$TEST_DOCKERFILE" == 1 ]; then if [ -s $tarout ]; then if [ "$release" = "mainline" ] || [ "$release" = "labs" ] || [ -n "$DOCKERFILE_RELEASES_CUSTOM" ] || [ "$GITHUB_ACTIONS" = "true" ]; then - cid=$(docker create -v /tmp $coverageVol --rm --privileged --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_TEST_ENABLE_FEATURES -e BUILDKIT_TEST_DISABLE_FEATURES -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid go test $coverageFlags --count=1 -tags "$buildtags" ${TESTFLAGS:--v} ./frontend/dockerfile) + cid=$(docker create -v /tmp $testReportsVol --rm --privileged --volumes-from=$cacheVolume -e GITHUB_REF -e ACTIONS_RUNTIME_TOKEN -e ACTIONS_CACHE_URL -e TEST_DOCKERD -e BUILDKIT_TEST_ENABLE_FEATURES -e GOTESTSUM_FORMAT -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry -e BUILDKIT_WORKER_RANDOM -e BUILDKIT_INTEGRATION_DOCKERD_FLAGS -e FRONTEND_GATEWAY_ONLY=local:/$release.tar -e EXTERNAL_DF_FRONTEND=/dockerfile-frontend $iid gotestsum $gotestsumArgs --packages=./frontend/dockerfile -- $gotestArgs --count=1 -tags "$buildtags" ${TESTFLAGS:--v}) docker cp $tarout $cid:/$release.tar if [ "$TEST_DOCKERD" = "1" ]; then docker cp "$TEST_DOCKERD_BINARY" $cid:/usr/bin/dockerd