From 0b106e8f169c936443f3018767556387230fc50b Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 10 Jul 2024 13:53:43 +0200 Subject: [PATCH 1/9] Summary of srtool and new CLI arguments --- .github/workflows/build-wasm.yml | 111 +++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index ee19e6c085..80481fb0a6 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -18,14 +18,13 @@ jobs: # Benchmark (with cache hit) # ubuntu-latest 50 min # ubuntu-latest-4-cores 34 min - runs-on: ubuntu-latest-4-cores + runs-on: ubuntu-latest-4-cores strategy: matrix: - target: [build-runtime] - package: [altair-runtime, centrifuge-runtime, development-runtime] - name: ${{ matrix.target }}-${{ matrix.package }} + chain: [ altair, centrifuge, development ] + name: build-runtime-${{ matrix.chain }} steps: - # PREP + # PREP - name: Check out code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #4.1.4 @@ -42,22 +41,12 @@ jobs: ./target/srtool cache-directories: ./runtime - # BUILD - - name: Run srtool # https://docs.substrate.io/reference/command-line-tools/srtool/ - run: > - docker run --rm --user root --platform=linux/amd64 - -e PACKAGE=${{ matrix.package }} -v /home/runner/.cargo:/cargo-home - -v ${{ github.workspace }}:/build paritytech/srtool:1.75.0-0.14.0 - /srtool/build --app - - # Alternative way of running SRTool that allows for "script-like" execution, - # extremely useful to debug: - + # Use this to debug what's going on inside the srtool container # - name: Run Docker SRTool # uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 # with: # image: paritytech/srtool:${{ matrix.rust_version }} - # options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e BUILD_OPTS=${{ env.BUILD_OPTS || ''}} -e PACKAGE=${{ matrix.package }} + # options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e BUILD_OPTS=${{ env.BUILD_OPTS || ''}} -e PACKAGE=${{ matrix.chain }}-runtime # run: | # echo "---- Env VARS ----" # echo "BUILD_OPT=${BUILD_OPT}" @@ -66,13 +55,60 @@ jobs: # ls -la /cargo-home/ # du -sh /cargo-home/* # echo "---- RUNNING BUILD ----" - # /srtool/build --save + # /srtool/build --app --json -cM + + + # Build and setup env vars: + - name: Run srtool # https://docs.substrate.io/reference/command-line-tools/srtool/ + id: srtool_build + shell: bash + run: | + ##"RUNNING SRTOOL" + echo ::group::Docker run srtool build ${{ inputs.chain }} + CMD="docker run --rm --user root --platform=linux/amd64 + -e PROFILE=release -e PACKAGE=${{ matrix.chain }}-runtime -e BUILD_OPTS="--features=on-chain-release-build" + -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build + paritytech/srtool:1.77.0-0.15.0 /srtool/build --app --json -cM" + echo ::debug::build::docker_run $CMD + echo ::endgroup + # here we keep streaming the progress and fetch the last line for the json result + stdbuf -oL $CMD | { + while IFS= read -r line + do + echo ║ $line + JSON="$line" + done + echo "json=$JSON" >> $GITHUB_OUTPUT - - name: Upload srtool report - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 #v4.3.0 - with: - name: srtool-wasm-report.txt - path: ./${{ matrix.package }}-srtool-wasm-report.json + PROP=`echo $JSON | jq -r .runtimes.compact.prop` + echo "proposal_hash=$PROP" >> $GITHUB_OUTPUT + + WASM=`echo $JSON | jq -r .runtimes.compact.wasm` + echo "wasm=$WASM" >> $GITHUB_OUTPUT + + Z_WASM=`echo $JSON | jq -r .runtimes.compressed.wasm` + echo "wasm_compressed=$Z_WASM" >> $GITHUB_OUTPUT + + IPFS=`echo $JSON | jq -r .runtimes.compact.ipfs` + echo "ipfs=$IPFS" >> $GITHUB_OUTPUT + } + + - name: Summary + run: | + ## Summary and JSON output + echo "::group::Summary" + echo "- version: ${{ steps.srtool_build.outputs.version }}" + echo "- info: ${{ steps.srtool_build.outputs.info }}" + echo "- prop: ${{ steps.srtool_build.outputs.proposal_hash }}" + echo "- runtime location: ${{ steps.srtool_build.outputs.wasm }}" + echo "::endgroup" + + echo "::group::JSON output" + PRETTY_JSON=$(echo '${{ steps.srtool_build.outputs.json }}' | jq .) + echo '${{ steps.srtool_build.outputs.json }}' | jq . > "${{ matrix.chain }}-srtool-digest.json" + echo "JSON Output:" + cat "${{ matrix.chain }}-srtool-digest.json" + echo "::endgroup" - id: 'auth' if: github.event_name != 'pull_request' @@ -89,8 +125,29 @@ jobs: if: ${{ steps.auth.outcome == 'success' }} shell: bash run: | - runtime_name=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//" ) - filename=$(echo "${{ matrix.package }}" | sed -e 's/-/_/g' ) + ## Publish WASM and JSON summary + echo '${{ steps.srtool_build.outputs.json }}' | jq . > "${{ matrix.chain }}-srtool-digest.json" gsutil cp \ - ./runtime/${runtime_name}/target/srtool/release/wbuild/${{ matrix.package }}/${filename}.compact.compressed.wasm \ - gs://centrifuge-wasm-repo/${runtime_name}/${runtime_name}-$(git rev-parse --short HEAD).wasm + "${{ steps.srtool_build.outputs.wasm }}" \ + "gs://centrifuge-wasm-repo/${{ matrix.chain }}/${{ matrix.chain }}-$(git rev-parse --short HEAD).wasm" + gsutil cp \ + "${{ matrix.chain }}-srtool-digest.json" \ + "gs://centrifuge-wasm-repo/${{ matrix.chain }}/${{ matrix.chain }}-$(git rev-parse --short HEAD)-srtool-digest.json" + + - name: Upload WASM to release tag ${{ github.ref_name }} + uses: ncipollo/release-action@a2e71bdd4e7dab70ca26a852f29600c98b33153e # @v1 + if: github.event_name == 'release' + # https://github.com/ncipollo/release-action + with: + tag: ${{ github.ref_name }} + commit: ${{ github.sha }} + artifacts: "${{ steps.srtool_build.outputs.wasm }}, ${{ matrix.chain }}-srtool-digest.json" + replacesArtifacts: true + allowUpdates: true + artifactErrorsFailBuild: true + omitBodyDuringUpdate: true + omitDraftDuringUpdate: true + omitNameDuringUpdate: true + omitPrereleaseDuringUpdate: true + # updateOnlyUnreleased: true # When allowUpdates is enabled, this will fail the action if the release it is updating is not a draft or a prerelease. + \ No newline at end of file From c2c3505a6e2a682f6dd7f316e96e96f077c597bb Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 10 Jul 2024 13:54:09 +0200 Subject: [PATCH 2/9] Do not update release w/ docker tag and avoid extra containers in ghcr --- .github/workflows/build-docker.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index b7a1574731..885320f8ca 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -86,6 +86,7 @@ jobs: - name: Build and push centrifugeio/centrifuge-chain uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 #v5.3.0 with: + provenance: false # Solves untagged containers getting into the GH registry https://github.com/docker/build-push-action/issues/894 context: . file: ./docker/centrifuge-chain/Dockerfile build-args: | @@ -111,14 +112,14 @@ jobs: short-description: ${{ github.event.repository.description }} enable-url-completion: true - - name: Update GitHub release - if: github.event_name == 'release' && github.event.action == 'released' && matrix.target == 'release' - uses: softprops/action-gh-release@v1 - with: - append_body: true - body: | - **Docker tags (${{ env.NOW }}):** - ${{ steps.meta.outputs.tags }} + # - name: Update GitHub release + # if: github.event_name == 'release' && github.event.action == 'released' && matrix.target == 'release' + # uses: softprops/action-gh-release@v1 + # with: + # append_body: true + # body: | + # **Docker tags (${{ env.NOW }}):** + # ${{ steps.meta.outputs.tags }} - if: failure() name: Check available space after build failed From c798c91cd6d43a6774ef48cc654e35ed9595497f Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 10 Jul 2024 13:54:21 +0200 Subject: [PATCH 3/9] delete untagged container in the GH registry --- .github/workflows/delete_untagged.yml | 45 +++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/workflows/delete_untagged.yml diff --git a/.github/workflows/delete_untagged.yml b/.github/workflows/delete_untagged.yml new file mode 100644 index 0000000000..2499e3bb29 --- /dev/null +++ b/.github/workflows/delete_untagged.yml @@ -0,0 +1,45 @@ +name: Delete untagged +on: + workflow_dispatch: +jobs: + delete_untagged: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + # id-token: write + steps: + - name: Delete untagged ghcr + uses: Chizkiyahu/delete-untagged-ghcr-action@v3 + with: + # Personal access token (PAT) used to fetch the repository. The PAT is configured + # with the local git config, which enables your scripts to run authenticated git + # commands. The post-job step removes the PAT. + # needs delete:packages permissions + # required: true + # [Learn more about creating and using encrypted secrets](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets) + token: ${{ secrets.GITHUB_TOKEN }} + # Repository name or name with owner + # Delete only from repository name + # Default: ${{ github.repository }} + repository: 'centrifuge-chain' + # 'The repository owner name' + # Default: ${{ github.repository_owner }} + # repository_owner: '' + # 'The package names' + # Delete only from comma separated package names + # required: false + package_name: 'centrifuge-chain' + # Delete only package versions without tag + # required: false + # Default: true + # choices: true, false + untagged_only: true + # Except untagged multiplatform packages from deletion + # only for untagged_only=true + # needs docker installed + except_untagged_multiplatform: false + # the owner type + # required: true + # choices: org, user + owner_type: 'org' \ No newline at end of file From 08ba39ade1165249e584281df8724e0d8959c25d Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 10 Jul 2024 13:54:33 +0200 Subject: [PATCH 4/9] avoid long running jobs on sanity-checks --- .github/workflows/sanity-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index bc6f41776e..999f4603fc 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -9,6 +9,7 @@ permissions: contents: read jobs: test-n-lint: + timeout-minutes: 90 name: ${{ matrix.target }} runs-on: ubuntu-latest-8-cores strategy: @@ -40,6 +41,7 @@ jobs: RUSTC_WRAPPER: "sccache" benchmark-check: + # timeout-minutes: 90 # <- Not needed as ubunt-latest is free of charge name: bench-check-${{ matrix.runtime }} runs-on: ubuntu-latest #-4-cores strategy: From 1e74672f1f3d3cc887cd0d12cbf34da7aefeb1ad Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Tue, 23 Jul 2024 10:43:31 +0200 Subject: [PATCH 5/9] Fix wasm summary --- .github/workflows/build-wasm.yml | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 80481fb0a6..0f9053a1fb 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -23,6 +23,8 @@ jobs: matrix: chain: [ altair, centrifuge, development ] name: build-runtime-${{ matrix.chain }} + env: + SRTOOL_IMAGE: "paritytech/srtool:1.77.0-0.15.0" steps: # PREP - name: Check out code @@ -68,7 +70,7 @@ jobs: CMD="docker run --rm --user root --platform=linux/amd64 -e PROFILE=release -e PACKAGE=${{ matrix.chain }}-runtime -e BUILD_OPTS="--features=on-chain-release-build" -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build - paritytech/srtool:1.77.0-0.15.0 /srtool/build --app --json -cM" + ${{ env.SRTOOL_IMAGE }} /srtool/build --app --json -cM" echo ::debug::build::docker_run $CMD echo ::endgroup # here we keep streaming the progress and fetch the last line for the json result @@ -96,11 +98,25 @@ jobs: - name: Summary run: | ## Summary and JSON output - echo "::group::Summary" - echo "- version: ${{ steps.srtool_build.outputs.version }}" - echo "- info: ${{ steps.srtool_build.outputs.info }}" - echo "- prop: ${{ steps.srtool_build.outputs.proposal_hash }}" + + echo ::group::Srtool version + CMD="docker run -i --rm -v ${{ github.workspace }}:/build ${{ env.SRTOOL_IMAGE }} version -cM" + JSON=`$CMD` + echo $JSON | jq . + echo "version=$JSON" >> $GITHUB_OUTPUT + echo ::endgroup:: + + echo ::group::Srtool info + CMD="docker run -i --rm -v ${{ github.workspace }}:/build ${{ env.SRTOOL_IMAGE }} info -cM" + JSON=`$CMD` + echo $JSON | jq . + echo "info=$JSON" >> $GITHUB_OUTPUT + echo ::endgroup:: + + echo "::group::Technical info" + echo "- proposal hash: ${{ steps.srtool_build.outputs.proposal_hash }}" echo "- runtime location: ${{ steps.srtool_build.outputs.wasm }}" + echo "- IPFS: ${{ steps.srtool_build.outputs.ipfs }}" echo "::endgroup" echo "::group::JSON output" From 2d12f50d2ca61fd4679918d18ad715788606579f Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Tue, 23 Jul 2024 15:58:40 +0200 Subject: [PATCH 6/9] fix sanity-checks job not triggering when PR is ready --- .github/workflows/sanity-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index 832ecf3642..d102ab1f31 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -1,5 +1,6 @@ on: pull_request: + types: [opened, synchronize, reopened, ready_for_review] name: Sanity checks concurrency: group: 'tests-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' From f92b80a2c87833860a26a912d055415df74948bd Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Tue, 23 Jul 2024 15:58:56 +0200 Subject: [PATCH 7/9] Simplier (and cheaper) build-wasm --- .github/workflows/build-wasm.yml | 61 ++++---------------------------- 1 file changed, 6 insertions(+), 55 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index d7891fa00b..6ac6189d7b 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -18,8 +18,8 @@ jobs: # Benchmark (with cache hit) # ubuntu-latest 50 min # ubuntu-latest-4-cores 34 min - runs-on: ubuntu-latest-4-cores - runs-on: ubuntu-latest-4-cores + # As this is not a critical PR feedback item, we can run it on free ubuntu-latest + runs-on: ubuntu-latest strategy: matrix: chain: [ altair, centrifuge, development ] @@ -27,7 +27,6 @@ jobs: env: SRTOOL_IMAGE: "paritytech/srtool:1.77.0-0.15.0" steps: - # PREP # PREP - name: Check out code uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b #4.1.4 @@ -45,7 +44,6 @@ jobs: ./target/srtool cache-directories: ./runtime - # Use this to debug what's going on inside the srtool container # Use this to debug what's going on inside the srtool container # - name: Run Docker SRTool # uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 @@ -85,44 +83,17 @@ jobs: JSON="$line" done echo "json=$JSON" >> $GITHUB_OUTPUT - - PROP=`echo $JSON | jq -r .runtimes.compact.prop` - echo "proposal_hash=$PROP" >> $GITHUB_OUTPUT WASM=`echo $JSON | jq -r .runtimes.compact.wasm` echo "wasm=$WASM" >> $GITHUB_OUTPUT Z_WASM=`echo $JSON | jq -r .runtimes.compressed.wasm` echo "wasm_compressed=$Z_WASM" >> $GITHUB_OUTPUT - - IPFS=`echo $JSON | jq -r .runtimes.compact.ipfs` - echo "ipfs=$IPFS" >> $GITHUB_OUTPUT } - name: Summary run: | - ## Summary and JSON output - - echo ::group::Srtool version - CMD="docker run -i --rm -v ${{ github.workspace }}:/build ${{ env.SRTOOL_IMAGE }} version -cM" - JSON=`$CMD` - echo $JSON | jq . - echo "version=$JSON" >> $GITHUB_OUTPUT - echo ::endgroup:: - - echo ::group::Srtool info - CMD="docker run -i --rm -v ${{ github.workspace }}:/build ${{ env.SRTOOL_IMAGE }} info -cM" - JSON=`$CMD` - echo $JSON | jq . - echo "info=$JSON" >> $GITHUB_OUTPUT - echo ::endgroup:: - - echo "::group::Technical info" - echo "- proposal hash: ${{ steps.srtool_build.outputs.proposal_hash }}" - echo "- runtime location: ${{ steps.srtool_build.outputs.wasm }}" - echo "- IPFS: ${{ steps.srtool_build.outputs.ipfs }}" - echo "::endgroup" - + ## JSON output echo "::group::JSON output" PRETTY_JSON=$(echo '${{ steps.srtool_build.outputs.json }}' | jq .) echo '${{ steps.srtool_build.outputs.json }}' | jq . > "${{ matrix.chain }}-srtool-digest.json" @@ -148,12 +119,12 @@ jobs: ## Publish WASM and JSON summary echo '${{ steps.srtool_build.outputs.json }}' | jq . > "${{ matrix.chain }}-srtool-digest.json" gsutil cp \ - "${{ steps.srtool_build.outputs.wasm }}" \ + "${{ steps.srtool_build.wasm_compressed }}" \ "gs://centrifuge-wasm-repo/${{ matrix.chain }}/${{ matrix.chain }}-$(git rev-parse --short HEAD).wasm" ## Publish WASM and JSON summary echo '${{ steps.srtool_build.outputs.json }}' | jq . > "${{ matrix.chain }}-srtool-digest.json" gsutil cp \ - "${{ steps.srtool_build.outputs.wasm }}" \ + "${{ steps.srtool_build.wasm_compressed }}" \ "gs://centrifuge-wasm-repo/${{ matrix.chain }}/${{ matrix.chain }}-$(git rev-parse --short HEAD).wasm" gsutil cp \ "${{ matrix.chain }}-srtool-digest.json" \ @@ -166,27 +137,7 @@ jobs: with: tag: ${{ github.ref_name }} commit: ${{ github.sha }} - artifacts: "${{ steps.srtool_build.outputs.wasm }}, ${{ matrix.chain }}-srtool-digest.json" - replacesArtifacts: true - allowUpdates: true - artifactErrorsFailBuild: true - omitBodyDuringUpdate: true - omitDraftDuringUpdate: true - omitNameDuringUpdate: true - omitPrereleaseDuringUpdate: true - # updateOnlyUnreleased: true # When allowUpdates is enabled, this will fail the action if the release it is updating is not a draft or a prerelease. - - "${{ matrix.chain }}-srtool-digest.json" \ - "gs://centrifuge-wasm-repo/${{ matrix.chain }}/${{ matrix.chain }}-$(git rev-parse --short HEAD)-srtool-digest.json" - - - name: Upload WASM to release tag ${{ github.ref_name }} - uses: ncipollo/release-action@a2e71bdd4e7dab70ca26a852f29600c98b33153e # @v1 - if: github.event_name == 'release' - # https://github.com/ncipollo/release-action - with: - tag: ${{ github.ref_name }} - commit: ${{ github.sha }} - artifacts: "${{ steps.srtool_build.outputs.wasm }}, ${{ matrix.chain }}-srtool-digest.json" + artifacts: "${{ steps.srtool_build.wasm_compressed }}, ${{ matrix.chain }}-srtool-digest.json" replacesArtifacts: true allowUpdates: true artifactErrorsFailBuild: true From 2cf5644ba9850960b233125e1bf625329cef3173 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Tue, 23 Jul 2024 16:11:10 +0200 Subject: [PATCH 8/9] Limit the runs of the codecov action --- .github/workflows/codecov.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 29c544a396..15441652fe 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -2,6 +2,10 @@ on: push: branches: [main] pull_request: + types: [opened, synchronize, reopened, ready_for_review] + pull_request_review: + types: [submitted] + workflow_dispatch: name: Code coverage concurrency: group: 'codecov-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' @@ -12,7 +16,12 @@ permissions: jobs: codecov: runs-on: ubuntu-latest-8-cores - if: github.event.pull_request.draft == false || github.ref == 'refs/heads/main' + if: > + github.event.pull_request.draft == false || + github.ref == 'refs/heads/main' || + (github.event_name == 'pull_request_review' && github.event.review.state == 'approved') || + github.event.action == 'ready_for_review' || + github.event_name == 'workflow_dispatch' steps: - uses: actions/checkout@master From de8ba4763eaa39bf8d3eea5c775843e8ca1167b0 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Tue, 23 Jul 2024 16:12:20 +0200 Subject: [PATCH 9/9] reduce codecov machine size --- .github/workflows/codecov.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 15441652fe..ccb532ca56 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -15,7 +15,7 @@ permissions: contents: read jobs: codecov: - runs-on: ubuntu-latest-8-cores + runs-on: ubuntu-latest-4-cores if: > github.event.pull_request.draft == false || github.ref == 'refs/heads/main' ||