Recyclable pipeline #3023
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| pull_request: | |
| branches: | |
| - "main" | |
| - "rollout-riders" | |
| jobs: | |
| codegen: | |
| name: Codegen | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Restore go build cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ hashFiles('**/go.mod') }} | |
| - name: Setup Golang | |
| uses: actions/[email protected] | |
| with: | |
| go-version: '1.23' | |
| - name: Add bins to PATH | |
| run: | | |
| echo /home/runner/go/bin >> $GITHUB_PATH | |
| echo /usr/local/bin >> $GITHUB_PATH | |
| - name: Get dependencies | |
| run: go mod download | |
| - name: Make codegen | |
| run: | | |
| echo 'GOPATH=/home/runner/go' >> $GITHUB_ENV | |
| make -B codegen | |
| - name: Ensure nothing changed | |
| run: git diff --exit-code | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| env: | |
| GOPATH: /home/runner/go | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.23' | |
| - name: golangci-lint | |
| uses: golangci/golangci-lint-action@v6 | |
| with: | |
| version: v1.61 | |
| args: --timeout=10m | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 25 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Golang | |
| uses: actions/[email protected] | |
| with: | |
| go-version: '1.23' | |
| id: go | |
| - name: Install MockGen | |
| run: go install github.com/golang/mock/[email protected] | |
| - name: Restore Go build cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} | |
| - name: Get dependencies | |
| run: go mod download | |
| - name: Test | |
| run: make test | |
| - name: Archive unit test code coverage | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: unit-tests-coverage | |
| path: coverage.out | |
| retention-days: 7 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Verify docker image | |
| run: make image | |
| e2e-tests: | |
| name: E2E Tests | |
| runs-on: ubuntu-latest | |
| # We already have a timeout in "go test" call: we don't also need this | |
| #timeout-minutes: 35 # If we start getting error in CI as "The operation was canceled", we need to increase this value. | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| strategy: ["pause-and-drain", "progressive", "no-strategy"] | |
| case: ["functional-pipeline", "functional-monovertex", "functional-nc", "concurrent", "rollback"] | |
| include: | |
| - strategy: "pause-and-drain" | |
| case: "ppnd" | |
| - strategy: "progressive" | |
| case: "progressive-monovertex" | |
| - strategy: "progressive" | |
| case: "progressive-pipeline" | |
| - strategy: "progressive" | |
| case: "rider" | |
| - strategy: "progressive" | |
| case: "progressive-analysis-monovertex" | |
| - strategy: "progressive" | |
| case: "progressive-analysis-pipeline" | |
| - strategy: "progressive" | |
| case: "force-drain" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Restore go build cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ hashFiles('**/go.mod') }} | |
| - name: Setup Golang | |
| uses: actions/[email protected] | |
| with: | |
| go-version: '1.23' | |
| - name: Install Ginkgo | |
| run: go install github.com/onsi/ginkgo/v2/ginkgo | |
| - name: Add bins to PATH | |
| run: | | |
| echo /home/runner/go/bin >> $GITHUB_PATH | |
| echo /usr/local/bin >> $GITHUB_PATH | |
| - name: Install k3d | |
| run: | | |
| curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash | |
| echo 'k3d installed' | |
| k3d --version | |
| - name: Create a cluster | |
| run: | | |
| export PATH=$PATH:/usr/local/bin | |
| k3d cluster create e2e --runtime-ulimit "nofile=65535:65535" | |
| k3d kubeconfig get e2e > ~/.kube/config | |
| echo '127.0.0.1 localhost' | sudo tee -a /etc/hosts | |
| echo 'Waiting for the cluster to be ready...' | |
| until kubectl cluster-info; do sleep 1; done | |
| - name: Install Numaplane | |
| env: | |
| GOPATH: /home/runner/go | |
| run: KUBECONFIG=~/.kube/config VERSION=${{ github.sha }} DOCKER_PUSH=true STRATEGY=${{matrix.strategy}} TEST_TYPE=e2e make start | |
| - name: Wait for Numaplane to be ready | |
| run: | | |
| export POD_NAME=$(kubectl get pods -n numaplane-system -l app.kubernetes.io/name=controller-manager -o jsonpath="{.items[0].metadata.name}") | |
| echo 'Waiting for the numaplane-controller-manager to be ready...' | |
| kubectl get po -n numaplane-system | |
| kubectl wait --for=condition=Ready --timeout=5m pod/$POD_NAME -n numaplane-system | |
| - name: Run e2e tests | |
| env: | |
| GOPATH: /home/runner/go | |
| run: | | |
| KUBECONFIG=~/.kube/config VERSION=${{ github.sha }} DOCKER_PUSH=true STRATEGY=${{matrix.strategy}} ENABLE_POD_LOGS=true make test-${{matrix.case}}-e2e | |
| ## This is a temporary solution to kill the process twice, until we have fix for a stuck process after initial kill. | |
| - name: Stop numaplane process | |
| if: always() | |
| run: | | |
| export POD_NAME=$(kubectl get pods -n numaplane-system -l app.kubernetes.io/name=controller-manager -o jsonpath="{.items[0].metadata.name}") | |
| kubectl exec -n numaplane-system $POD_NAME -- /bin/bash -c 'kill -SIGTERM `cat /numaplane.pid`' | |
| sleep 10s | |
| kubectl exec -n numaplane-system $POD_NAME -- /bin/bash -c 'kill -SIGTERM `cat /numaplane.pid`' | |
| - name: Copy report from Numaplane pod | |
| run: | | |
| mkdir -p tests/e2e/output/resources/coverage/ | |
| POD_NAME=$(kubectl get pods -n numaplane-system -l app.kubernetes.io/name=controller-manager -o jsonpath="{.items[0].metadata.name}") | |
| kubectl cp $POD_NAME:/coverage -n numaplane-system tests/e2e/output/resources/coverage/ | |
| - name: Archive code coverage | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: coverage-${{matrix.strategy}}-${{matrix.case}} | |
| path: | | |
| tests/e2e/output/resources/coverage/ | |
| retention-days: 7 | |
| - name: Archive resource changes | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: resource-changes-${{matrix.strategy}}-${{matrix.case}} | |
| path: | | |
| tests/e2e/output/resources/ | |
| retention-days: 7 | |
| - name: Archive pod logs | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: pod-logs-${{matrix.strategy}}-${{matrix.case}} | |
| path: | | |
| tests/e2e/output/logs/ | |
| retention-days: 7 | |
| coverage: | |
| name: Generate coverage report | |
| runs-on: ubuntu-latest | |
| needs: | |
| - e2e-tests | |
| - unit-tests | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v4 | |
| - name: Setup Golang | |
| uses: actions/[email protected] | |
| with: | |
| go-version: '1.23' | |
| - name: Install gocovmerge | |
| run: go install github.com/bookerzzz/gocovmerge@latest | |
| - name: Copy all e2e coverage files in a single directory | |
| run: | | |
| mkdir -p merged-coverage | |
| find . -type f -path "./coverage-*" -exec cp {} merged-coverage \; | |
| echo "################# Merged e2e coverage files: #################" | |
| ls -l merged-coverage | |
| echo "##################################" | |
| go tool covdata percent -i=merged-coverage/ -o=coverage-e2e.out | |
| cp unit-tests-coverage/coverage.out . | |
| gocovmerge coverage.out coverage-e2e.out > merged_coverage.out | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/[email protected] | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| disable_search: true | |
| verbose: true | |
| files: merged_coverage.out | |
| - name: Archive merged code coverage | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: merged-coverage | |
| path: merged-coverage | |
| retention-days: 7 |