From 943ca6ace9e28bf024cb026bc201f86198926c95 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 14 Sep 2023 23:31:13 -0400 Subject: [PATCH 01/67] Renmove unnecessary/old workflow files --- .github/workflows/build-matrix.json | 48 ---------------- .github/workflows/build-nix.yml | 25 --------- .github/workflows/build.yml | 86 ----------------------------- .github/workflows/docker-nix.yml | 31 ----------- .github/workflows/docker.yml | 73 ------------------------ .github/workflows/lints.yml | 39 ------------- .github/workflows/tests.yml | 39 ------------- 7 files changed, 341 deletions(-) delete mode 100644 .github/workflows/build-matrix.json delete mode 100644 .github/workflows/build-nix.yml delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/docker-nix.yml delete mode 100644 .github/workflows/docker.yml delete mode 100644 .github/workflows/lints.yml delete mode 100644 .github/workflows/tests.yml diff --git a/.github/workflows/build-matrix.json b/.github/workflows/build-matrix.json deleted file mode 100644 index e8633688d3..0000000000 --- a/.github/workflows/build-matrix.json +++ /dev/null @@ -1,48 +0,0 @@ -[ - { - "os": "ubuntu-latest", - "target": "build-node", - "run_on_event": "any", - "rust_toolchain": "nightly-2022-11-14" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime", - "package": "altair-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime", - "package": "centrifuge-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime-fast", - "package": "altair-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime-fast", - "package": "centrifuge-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - }, - - { - "os": "ubuntu-latest", - "target": "build-runtime-testnet", - "package": "altair-runtime", - "run_on_event": "push", - "rust_toolchain": "1.64.0" - } -] diff --git a/.github/workflows/build-nix.yml b/.github/workflows/build-nix.yml deleted file mode 100644 index cbb02cf1e7..0000000000 --- a/.github/workflows/build-nix.yml +++ /dev/null @@ -1,25 +0,0 @@ -#### NOTE: This job is disabled until nix supports the latest cargo workspace features. -#### See: https://github.com/NixOS/nixpkgs/pull/217232 -# -#name: nix-build -#on: -# pull_request: -# push: -# branches: [main, 'release-v**'] -#jobs: -# build: -# runs-on: ubuntu-latest -# steps: -# - uses: actions/checkout@v3 -# with: -# # Nix Flakes doesn't work on shallow clones -# fetch-depth: 0 -# - uses: cachix/install-nix-action@v17 -# with: -# install_url: https://releases.nixos.org/nix/nix-2.10.3/install -# - uses: cachix/cachix-action@v10 -# with: -# name: centrifuge-chain -# authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" -# - name: Build centrifuge-chain -# run: nix build --no-allow-dirty -L \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index a01782f0a5..0000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,86 +0,0 @@ -on: - push: - branches: [main, 'release-v**'] - pull_request: -name: Build -jobs: - matrix_prep: - name: matrix-prep - runs-on: 'ubuntu-latest' - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - # Required as the JSON input file needs to be read - - uses: actions/checkout@v3 - - id: set-matrix - uses: JoshuaTheMiller/conditional-build-matrix@9ff11e2cad4aa166e1d1d834097121a969da8276 - with: - inputFile: '.github/workflows/build-matrix.json' - # we run build-node on every workflow but build-runtime only on pushes to branches - filter: '[?run_on_event==`any` || run_on_event==`${{ github.event_name }}`]' - build: - needs: matrix_prep - strategy: - matrix: ${{fromJson(needs.matrix_prep.outputs.matrix)}} - name: ${{ matrix.target }} - runs-on: ${{ matrix.os }} - env: - RUST_TOOLCHAIN: "nightly-2022-11-14" - steps: - - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - name: Check out code - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true - - if: startsWith(matrix.target, 'build-runtime') - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - with: # cache the runtime target instead of root target for build-runtime - working-directory: './runtime' - target-dir: './target/srtool' - - if: matrix.target == 'build-node' - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - - name: ${{ matrix.target }} - run: ./ci/script.sh - env: - TARGET: ${{ matrix.target }} - RUST_TOOLCHAIN: ${{ matrix.rust_toolchain }} - PACKAGE: ${{matrix.package}} - - if: startsWith(matrix.target, 'build-runtime') - name: Setup - gcloud / gsutil - uses: google-github-actions/setup-gcloud@v0 - with: - service_account_key: ${{ secrets.GCS_SA_KEY }} - project_id: ${{ secrets.GCS_PROJECT }} - export_default_credentials: true - - if: ${{ matrix.target == 'build-runtime' && matrix.package == 'altair-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/altair/target/srtool/release/wbuild/altair-runtime/altair_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/parachain/altair_runtime-$(git rev-parse --short HEAD).compact.compressed.wasm - - if: ${{ matrix.target == 'build-runtime-fast' && matrix.package == 'altair-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/altair/target/srtool/release/wbuild/altair-runtime/altair_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/test-parachain/altair_runtime-$(git rev-parse --short HEAD).compact.compressed.wasm - - if: ${{ matrix.target == 'build-runtime-testnet' && matrix.package == 'altair-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/altair/target/srtool/release/wbuild/altair-runtime/altair_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/parachain/algol-$(git rev-parse --short HEAD).compact.compressed.wasm - - if: ${{ matrix.target == 'build-runtime' && matrix.package == 'centrifuge-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/centrifuge/target/srtool/release/wbuild/centrifuge-runtime/centrifuge_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/parachain/centrifuge_runtime-$(git rev-parse --short HEAD).compact.compressed.wasm - - if: ${{ matrix.target == 'build-runtime-fast' && matrix.package == 'centrifuge-runtime' }} - name: Publish to GCS - run: | - gsutil cp ./runtime/centrifuge/target/srtool/release/wbuild/centrifuge-runtime/centrifuge_runtime.compact.compressed.wasm gs://centrifuge-artifact-releases/test-parachain/centrifuge_runtime-$(git rev-parse --short HEAD).compact.compressed.wasm \ No newline at end of file diff --git a/.github/workflows/docker-nix.yml b/.github/workflows/docker-nix.yml deleted file mode 100644 index 31e285cedb..0000000000 --- a/.github/workflows/docker-nix.yml +++ /dev/null @@ -1,31 +0,0 @@ -# name: docker -# on: -# push: -# branches: [main, 'release-v**'] -# jobs: -# docker: -# strategy: -# matrix: -# os: [ ubuntu-latest ] -# target: [".#dockerImage", ".#dockerImageFastRuntime"] -# runs-on: ${{ matrix.os }} -# steps: -# - uses: actions/checkout@v3 -# with: -# fetch-depth: 0 -# - uses: cachix/install-nix-action@v17 -# with: -# install_url: https://releases.nixos.org/nix/nix-2.10.3/install -# - name: Build Docker image -# run: | -# nix build --no-allow-dirty -L ${{ matrix.target }} -# docker load < result -# - name: Login to Docker Hub -# uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 -# with: -# username: ${{ secrets.DOCKER_HUB_USERNAME }} -# password: ${{ secrets.DOCKER_HUB_TOKEN }} -# - name: List images -# run: docker images -# - name: Push image to Docker Hub -# run: docker push --all-tags --quiet centrifugeio/centrifuge-chain diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 8c496ec10f..0000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: docker -on: - push: - branches: [main, 'release-v**'] - pull_request: - paths: - - ".github/workflows/docker.yml" - workflow_dispatch: - inputs: - docker_tag: - description: 'Docker Image tag' - required: true - default: '' -env: - RUST_TOOLCHAIN: "nightly-2022-11-14" -jobs: - docker: - strategy: - matrix: - os: [ ubuntu-latest-4-cores ] - target: [ release, test ] - runs-on: ${{ matrix.os }} - env: - WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Free space - # https://github.com/actions/runner-images/issues/2840#issuecomment-1284059930 - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - - - name: check available docker space - run: | - docker volume ls - df -h - - - name: Set docker tag - run: | - if [ "${WORKFLOW_TAG}" != "" ]; then - echo "DOCKER_TAG=${WORKFLOW_TAG}" >> $GITHUB_ENV - else - echo "DOCKER_TAG=$(echo $GITHUB_REF | cut -d'/' -f 3)" >> $GITHUB_ENV - fi - - if: matrix.target == 'release' - name: Build docker release - run: docker build --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -t centrifugeio/centrifuge-chain:${{ env.DOCKER_TAG }}-latest . - - if: matrix.target == 'test' - name: Build docker test - run: docker build --build-arg RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} --build-arg OPTS="--features=fast-runtime" -t centrifugeio/centrifuge-chain:test-${{ env.DOCKER_TAG }}-latest . - - if: failure() - name: Check available space after build failed - run: | - docker volume ls - df -h - - name: Login to Docker Hub - uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - if: matrix.target == 'release' - name: Tag image release - run: docker tag centrifugeio/centrifuge-chain:${{ env.DOCKER_TAG }}-latest "centrifugeio/centrifuge-chain:${{ env.DOCKER_TAG }}-$(date -u +%Y%m%d%H%M%S)-$(git rev-parse --short HEAD)" - - if: matrix.target == 'test' - name: Tag image test - run: docker tag centrifugeio/centrifuge-chain:test-${{ env.DOCKER_TAG }}-latest "centrifugeio/centrifuge-chain:test-${{ env.DOCKER_TAG }}-$(date -u +%Y%m%d%H%M%S)-$(git rev-parse --short HEAD)" - - name: List images - run: docker images - - name: Push image to Docker Hub - run: docker push --all-tags --quiet centrifugeio/centrifuge-chain - if: contains(github.ref, 'refs/tags/release-v') || github.ref == 'refs/heads/main' diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml deleted file mode 100644 index abc9c8fc43..0000000000 --- a/.github/workflows/lints.yml +++ /dev/null @@ -1,39 +0,0 @@ -on: - push: - branches: [main, 'release-v**'] - pull_request: -name: Lints -jobs: - lints: - name: ${{ matrix.target }} - strategy: - matrix: - os: [ubuntu-latest] - target: [fmt, clippy] - runs-on: ${{ matrix.os }} - env: - RUST_TOOLCHAIN: "nightly-2022-11-14" - steps: - - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - name: Check out code - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true - - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - - name: Run lints - run: ./ci/script.sh - env: - TARGET: ${{ matrix.target }} - diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml deleted file mode 100644 index d00c4333d6..0000000000 --- a/.github/workflows/tests.yml +++ /dev/null @@ -1,39 +0,0 @@ -on: - push: - branches: [main, 'release-v**'] - pull_request: -name: Tests -jobs: - tests: - name: ${{ matrix.target }} - strategy: - matrix: - os: [ubuntu-latest] - target: [tests, integration] - runs-on: ${{ matrix.os }} - env: - RUST_TOOLCHAIN: "nightly-2022-11-14" - steps: - - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - name: Check out code - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true - - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - - name: Run tests - run: ./ci/script.sh - env: - TARGET: ${{ matrix.target }} - From 9ea2c6fbe9b3ca544f3c4edb811b4633546ae7e3 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 14 Sep 2023 23:32:18 -0400 Subject: [PATCH 02/67] Add PR common checks and wasm build(s) --- .github/workflows/build-wasm.yml | 96 +++++++++++++++++++++++++++++ .github/workflows/sanity-checks.yml | 69 +++++++++++++++++++++ 2 files changed, 165 insertions(+) create mode 100644 .github/workflows/build-wasm.yml create mode 100644 .github/workflows/sanity-checks.yml diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml new file mode 100644 index 0000000000..0003ae19a7 --- /dev/null +++ b/.github/workflows/build-wasm.yml @@ -0,0 +1,96 @@ +on: + push: + branches: [main, 'release-v**'] + pull_request: +name: Build&Publish WASM +concurrency: + group: 'build-wasm-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true +jobs: + build-runtime-wasms: + environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} + # Benchmark (with cache hit) + # ubuntu-latest 50 min + # ubuntu-latest-4-cores 34 min + runs-on: ubuntu-latest-8-cores + strategy: + matrix: + # To test until I get one right + # target: [build-runtime, build-runtime-fast, build-runtime-testnet] + # package: [centrifuge-runtime, altair-runtime] + target: [build-runtime, build-runtime-testnet, build-runtime-fast] + package: [altair-runtime, centrifuge-runtime] + rust_version: ["1.66.0"] + exclude: + - target: "build-runtime-testnet" + package: "centrifuge-runtime" + name: ${{ matrix.target }}-${{ matrix.package }} + # Cannot make sccache work with the srtool container + # env: + # SCCACHE_GHA_ENABLED: "true" + # RUSTC_WRAPPER: "sccache" + steps: + - name: Check out code + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 + + - name: Prep build on Ubuntu + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ matrix.rust_version }} + + # sccache does not work inside the srtool container + - uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # 2.6.2 + with: + prefix-key: build #-${{ matrix.package }} + shared-key: wasm + # save-if: ${{ github.ref == 'refs/heads/master' }} + + - name: Setup build options + id: buildopt + run: | + if ${{ matrix.target == 'build-runtime-testnet'}} ; then + echo "BUILD_OPTS="--features=testnet-runtime"" >> GITHUB_ENV + elif ${{ matrix.target == 'build-runtime-fast'}} ; then + echo "BUILD_OPTS="--features=fast-runtime"" >> GITHUB_ENV + fi + + - name: Run Docker SRTool + uses: addnab/docker-run-action@v3 + env: + RUSTC_VERSION: ${{ matrix.rust_version }} + BUILD_OPTS: ${{ env.BUILD_OPTS }} + with: + image: paritytech/srtool:${{ matrix.rust_version }} + options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e PACKAGE=${{ matrix.package }} + run: | + echo "BUILD_OPT=${BUILD_OPT}" + echo "PACKAGE=${PACKAGE}" + echo "---- List Cache Folder ----" + ls -la /cargo-home/ + du -sh /cargo-home/* + echo "---- Rust Versions ----" + rustc --version + rustup --version + cargo --version + echo "---- RUNNING BUILD ----" + /srtool/build + + - id: 'auth' + uses: 'google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033' # v1.1.1 + with: + workload_identity_provider: ${{ secrets.GWIP_ARTIFACTS }} + service_account: ${{ secrets.GSA_ARTIFACTS }} + + - name: Setup - gcloud / gsutil + uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 + + + - name: Publish artifacts to GCS + # if: ${{ github.ref == 'refs/heads/main' }} + shell: bash + run: | + target=$(echo "${{ matrix.target }}" | sed -e "s/^build-//" ) + runtime_name=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//" ) + gsutil cp \ + ./runtime/$runtime_name/target/srtool/release/wbuild/${{ matrix.package }}/${{ matrix.package }}.compact.compressed.wasm \ + gs://centrifuge-wasm-repo/$RUNTIME/$target/$runtime_name-$target-$(git rev-parse --short HEAD).wasm diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml new file mode 100644 index 0000000000..6f924f7bcd --- /dev/null +++ b/.github/workflows/sanity-checks.yml @@ -0,0 +1,69 @@ +on: + # push: + # branches: [main, 'release-v**'] + pull_request: + paths: + - 'src/' + - '.github/workflows/sanity-checks.yml' +name: Sanity checks +concurrency: + group: 'tests-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true +permissions: + id-token: write + contents: read +jobs: + test-n-lint: + name: ${{ matrix.target }} + runs-on: ubuntu-latest-4-cores + strategy: + matrix: + target: [test-general, test-integration, + lint-fmt, lint-clippy, cargo-build] # ,lint-taplo] + env: + RUST_TOOLCHAIN: "nightly-2022-11-14" + RUSTC_WRAPPER: "sccache" + steps: + - name: Check out code + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 + + - name: Prep build + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + cache: enabled + # Cache needs Google credentials: + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} + + - name: Runing cargo ${{ matrix.target }} + run: ./ci/run-check.sh + env: + TARGET: ${{ matrix.target }} + + benchmark-check: + name: bench-${{ matrix.runtime }} + runs-on: ubuntu-latest #r-4-cores + strategy: + matrix: + runtime: [development, altair, centrifuge] + env: + RUST_TOOLCHAIN: "nightly-2022-11-14" + steps: + - name: Check out code + uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 + + - name: Prep build + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + setup_cache: true + cache: enabled + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} + + - name: Runing cargo ${{ matrix.target }} + run: ./ci/run-check.sh + env: + TARGET: benchmark-check + RUNTIME: ${{ matrix.runtime }} From 298c3078d131722230600f1c23f859deca01700c Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 14 Sep 2023 23:32:27 -0400 Subject: [PATCH 03/67] Modify docs buld --- .github/workflows/docs.yml | 39 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4c8539c6da..bcf1217ca2 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,37 +7,29 @@ jobs: docs: permissions: contents: write - runs-on: ubuntu-latest + id-token: write + runs-on: ubuntu-latest-8-cores env: - RUST_TOOLCHAIN: "nightly-2022-11-14" + RUST_TOOLCHAIN: "1.66.0" RUSTDOCFLAGS: "-D warnings" + RUSTC_WRAPPER: "sccache" steps: - name: Checkout repository uses: actions/checkout@v3 - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 + uses: ./.github/actions/prep-ubuntu with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true + RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + cache: enabled + # Cache needs Google credentials: + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} - - name: Build Documentation + - name: Build documentation id: build_docs - uses: actions-rs/cargo@v1 - with: - command: doc - args: --all --no-deps + run: cargo doc --all --no-deps + - name: Build Documentation failed if: always() && steps.build_docs.outcome == 'failure' @@ -46,10 +38,11 @@ jobs: - name: Create Index file if: github.ref == 'refs/heads/main' - run: ./ci/create_index_for_rust_docs.sh + run: | + echo "" \ > ./target/doc/index.html - name: Deploy Docs - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@373f7f263a76c20808c831209c920827a82a2847 # 3.9.3 if: github.ref == 'refs/heads/main' with: github_token: ${{ secrets.GITHUB_TOKEN }} From 06dc4ef7490647486989a00d0dec65306241a855 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 14 Sep 2023 23:32:39 -0400 Subject: [PATCH 04/67] modify benchmark --- .github/workflows/benchmark.yml | 35 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index b65bdbe0ed..c1057f5ee8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -6,11 +6,13 @@ jobs: benchmark: permissions: pull-requests: write + id-token: write name: benchmark runs-on: ubuntu-latest-4-cores if: ${{ startsWith(github.event.comment.body, '/benchmark ') && github.event.issue.pull_request && contains(github.event.comment.author_association, 'CONTRIBUTOR') }} env: - RUST_TOOLCHAIN: "nightly-2022-11-14" + RUST_TOOLCHAIN: "1.66.0" + RUSTC_WRAPPER: "sccache" steps: - uses: actions/github-script@v6 name: Parse Runtime From Comment @@ -34,15 +36,6 @@ jobs: body: `:clock1: Running Benchmarks for \`${{steps.parse-runtime.outputs.result}}\` :clock1:` }) return comment.data.id - - name: Prep build on Ubuntu - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - name: Get PR Hash id: get_pr_hash uses: actions/github-script@v6 @@ -60,21 +53,27 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ steps.get_pr_hash.outputs.result }} - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 + + - name: Prep build on Ubuntu + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ matrix.rust_version }} + + - name: SCcache setup + uses: ./.github/actions/sccache-gcloud with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} + - name: Run Benchmark ${{steps.parse-runtime.outputs.result}} - run: ./ci/script.sh - env: - TARGET: benchmark - RUNTIME: ${{steps.parse-runtime.outputs.result}} + run: ./scripts/runtime_benchmarks.sh ${{steps.parse-runtime.outputs.result}} + - name: Upload artifact uses: actions/upload-artifact@v3 with: name: ${{steps.parse-runtime.outputs.result}}-weights path: runtime/${{steps.parse-runtime.outputs.result}}/src/weights/ + - uses: actions/github-script@v6 name: Notify Benchmark finished with: From 77948d529169ab0c43ac0bbd24d0c065d761f606 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 14 Sep 2023 23:33:05 -0400 Subject: [PATCH 05/67] Modify CI script (simpler) --- ci/run-check.sh | 37 +++++++++++++++++++++++++++ ci/script.sh | 66 ------------------------------------------------- 2 files changed, 37 insertions(+), 66 deletions(-) create mode 100755 ci/run-check.sh delete mode 100755 ci/script.sh diff --git a/ci/run-check.sh b/ci/run-check.sh new file mode 100755 index 0000000000..5434dbb3a0 --- /dev/null +++ b/ci/run-check.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +set -eux + +# Enable warnings about unused extern crates +# export RUSTFLAGS=" -W unused-extern-crates" +rustc --version +rustup --version +cargo --version + +case $TARGET in + cargo-build) + cargo build --release "$@" + ;; + + test-general) + RUST_MIN_STACK=8388608 cargo test --workspace --release --features runtime-benchmarks,try-runtime --exclude runtime-integration-tests + ;; + + test-integration) + RUST_MIN_STACK=8388608 cargo test --release --package runtime-integration-tests --features fast-runtime + ;; + + lint-fmt) + cargo fmt -- --check + ;; + + lint-taplo) + taplo fmt --check + ;; + + lint-clippy) + cargo clippy --workspace -- -D warnings -A clippy::unnecessary-cast -A clippy::bool-to-int-with-if + ;; + benchmark-check) + ./scripts/check_benchmarks.sh $RUNTIME +esac diff --git a/ci/script.sh b/ci/script.sh deleted file mode 100755 index 7b576d984e..0000000000 --- a/ci/script.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash - -set -eux - -RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-nightly-2023-02-07}" -SRTOOL_VERSION="${SRTOOL_VERSION:-1.66.1-0.9.25}" -PACKAGE="${PACKAGE:-centrifuge-runtime}" # Need to replicate job for all runtimes -RUNTIME="${RUNTIME:-centrifuge}" - -# Enable warnings about unused extern crates -export RUSTFLAGS=" -W unused-extern-crates" - -./scripts/init.sh install-toolchain - -rustc --version -rustup --version -cargo --version - -case $TARGET in - build-node) - cargo build --release "$@" - ;; - - build-runtime) - export RUSTC_VERSION=$RUST_TOOLCHAIN - docker run --rm -e PACKAGE=$PACKAGE -v $PWD:/build -v /tmp/cargo:/cargo-home paritytech/srtool:$SRTOOL_VERSION build - ;; - - build-runtime-fast) - export RUSTC_VERSION=$RUST_TOOLCHAIN - docker run --rm -e PACKAGE=$PACKAGE -e BUILD_OPTS="--features=fast-runtime" -v $PWD:/build -v /tmp/cargo:/cargo-home paritytech/srtool:$SRTOOL_VERSION build - ;; - - build-runtime-testnet) - export RUSTC_VERSION=$RUST_TOOLCHAIN - docker run --rm -e PACKAGE=$PACKAGE -e BUILD_OPTS="--features=testnet-runtime" -v $PWD:/build -v /tmp/cargo:/cargo-home paritytech/srtool:$SRTOOL_VERSION build - ;; - - tests) - RUST_MIN_STACK=8388608 cargo test --workspace --release --features runtime-benchmarks,try-runtime --exclude runtime-integration-tests - ;; - - integration) - RUST_MIN_STACK=8388608 cargo test --release --package runtime-integration-tests --features fast-runtime - ;; - - fmt) - cargo fmt -- --check - ;; - - taplo) - taplo fmt --check - ;; - - clippy) - cargo clippy --workspace -- -D warnings -A clippy::unnecessary-cast -A clippy::bool-to-int-with-if - ;; - - benchmark) - ./scripts/runtime_benchmarks.sh $RUNTIME - ;; - - benchmark-check) - ./scripts/check_benchmarks.sh $RUNTIME - -esac From 949dbeb64cb15fb93ad6598f63c71ea1bf9cded6 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 14 Sep 2023 23:33:16 -0400 Subject: [PATCH 06/67] Add docker build --- .dockerignore | 4 ++ .github/workflows/build-docker.yml | 90 ++++++++++++++++++++++++ Dockerfile | 109 +++++++++++++++++------------ 3 files changed, 160 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/build-docker.yml diff --git a/.dockerignore b/.dockerignore index f4ceea7856..93c3c5d6db 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1 +1,5 @@ **/target/ +ci/ +.github/ +scripts/ +docker-compose* \ No newline at end of file diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml new file mode 100644 index 0000000000..d1d20df52d --- /dev/null +++ b/.github/workflows/build-docker.yml @@ -0,0 +1,90 @@ +name: Docker Build +on: + push: + branches: [main, 'release-v**'] + pull_request: + paths: + - ".github/workflows/build-docker.yml" + - "Dockerfile" + workflow_dispatch: + +env: + RUST_TOOLCHAIN: "1.66" +jobs: + docker: + strategy: + matrix: + os: [ ubuntu-latest-4-cores ] + target: [ release, test ] + runs-on: ${{ matrix.os }} + env: + WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} + steps: + + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Free space + # https://github.com/actions/runner-images/issues/2840#issuecomment-1284059930 + run: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + + - name: check available docker space + run: | + docker volume ls + df -h + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: DockerHub Login + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + + - run: echo "NOW=$(date -u +%Y%m%d)" >> $GITHUB_ENV + + - name: Setup docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: centrifugeio/centrifuge-chain + flavor: | + prefix=${{ matrix.target == 'test' && 'test-' || '' }} + suffix=${{ env.NOW }} + tags: | + type=schedule + type=ref,event=branch + type=ref,event=tag + type=semver,pattern={{version}} + type=semver,pattern={{major}} + type=sha,format=short,prefix='git-' + + - name: Build and push centrifugeio/centrifuge-chain + uses: docker/build-push-action@v5 + with: + context: . + push: ${{ github.ref == '/refs/heads/main' && true || false }} + tags: ${{ steps.meta.outputs.tags }} + # Cache options: + # https://docs.docker.com/build/ci/github-actions/cache/ + cache-from: type=registry,ref=centrifugeio/centrifuge-chain + # https://docs.docker.com/build/cache/backends/inline/ + cache-to: inline + + - name: Update DockerHub descriptions + if: contains(github.ref, 'refs/tags/release-v') + uses: peter-evans/dockerhub-description@v3 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} + repository: centrifuge/centrifuge-chain + short-description: ${{ github.event.repository.description }} + enable-url-completion: true + + - if: failure() + name: Check available space after build failed + run: | + docker volume ls + df -h \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 0dcf4af01c..d78a10ec4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,58 +1,81 @@ -# Based from https://github.com/paritytech/substrate/blob/master/.maintain/Dockerfile +# Based on +# https://github.com/paritytech/polkadot-sdk/blob/master/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile +FROM docker.io/library/ubuntu:22.04 as builder -FROM phusion/baseimage:focal-1.0.0 as builder -LABEL maintainer="Centrifuge Team" -LABEL description="This is the build stage for the Centrifuge Chain client. Here the binary is created." + # Defaults + ENV RUST_BACKTRACE 1 + ENV DEBIAN_FRONTEND=noninteractive + ENV RUST_TOOLCHAIN=$RUST_TOOLCHAIN + ARG FEATURES="" + ARG RUST_TOOLCHAIN="1.66" + + RUN apt-get update && \ + # apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ + apt-get install -y \ + cmake \ + pkg-config \ + libssl-dev \ + git \ + clang \ + libclang-dev \ + protobuf-compiler \ + curl + + RUN curl https://sh.rustup.rs -sSf | sh -s -- -y + ENV PATH="${PATH}:/root/.cargo/bin" + + # BUILD + COPY . centrifuge-chain + WORKDIR /centrifuge-chain + RUN echo $(ls -l /centrifuge-chain/) -ARG RUST_TOOLCHAIN=nightly -ENV DEBIAN_FRONTEND=noninteractive -ENV RUST_TOOLCHAIN=$RUST_TOOLCHAIN + RUN rustup default $RUST_TOOLCHAIN && \ + rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN && \ + cargo build "--release" --features=${FEATURES} -ARG PROFILE=release -ARG OPTS="" -WORKDIR /centrifuge-chain - -COPY . /centrifuge-chain - -RUN apt-get update && \ - apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ - apt-get install -y cmake pkg-config libssl-dev git clang libclang-dev protobuf-compiler +# ===== SECOND STAGE ====== -RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && \ - export PATH="$PATH:$HOME/.cargo/bin" && \ - rustup default $RUST_TOOLCHAIN && \ - rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN && \ - cargo build "--$PROFILE" $OPTS +FROM docker.io/library/ubuntu:22.04 -# ===== SECOND STAGE ====== + LABEL io.centrifuge.image.authors="guillermo@k-f.co" \ + io.centrifuge.image.vendor="Centrifuge" \ + io.centrifuge.image.title="centrifugeio/centrifuge-chain" \ + io.centrifuge.image.description="Centrifuge, the layer 1 of RWA. This is the official Centrifuge image with an injected binary." \ + io.centrifuge.image.source="https://github.com/centrifuge/centrifuge-chain/blob/main/Dockerfile" \ + # io.centrifuge.image.revision="${VCS_REF}" \ + io.centrifuge.image.created="${BUILD_DATE}" -FROM phusion/baseimage:focal-1.0.0 -LABEL maintainer="Centrifuge Team" -LABEL description="This is the 2nd stage: a very small image that contains the centrifuge-chain binary and will be used by users." -ARG PROFILE=release + COPY --from=builder /centrifuge-chain/target/release/centrifuge-chain /usr/local/bin -RUN mv /usr/share/ca* /tmp && \ - rm -rf /usr/share/* && \ - mv /tmp/ca-certificates /usr/share/ && \ - mkdir -p /root/.local/share/centrifuge-chain && \ - ln -s /root/.local/share/centrifuge-chain /data - # && \ - # useradd -m -u 1000 -U -s /bin/sh -d /centrifuge-chain centrifuge-chain # commented out since users do not seem to work with PVCs we currently use: https://stackoverflow.com/questions/46873796/allowing-access-to-a-persistentvolumeclaim-to-non-root-user/46907452 + RUN useradd -m -u 1000 -U -s /bin/sh -d /centrifuge centrifuge && \ + mkdir -p /data /centrifuge/.local/share && \ + chown -R centrifuge:centrifuge /data && \ + ln -s /data /centrifuge/.local/share/centrifuge -COPY --from=builder /centrifuge-chain/target/$PROFILE/centrifuge-chain /usr/local/bin + # checks + RUN ldd /usr/local/bin/centrifuge-chain && \ + /usr/local/bin/centrifuge-chain --version -# checks -RUN ldd /usr/local/bin/centrifuge-chain && \ - /usr/local/bin/centrifuge-chain --version + # Save sh and bash + RUN cp /usr/bin/sh /usr/bin/bash /usr/local/bin/ /root/ -# Shrinking -RUN rm -rf /usr/lib/python* && \ - rm -rf /usr/bin /usr/sbin /usr/share/man + # Unclutter + RUN mv /usr/share/ca* /tmp && \ + rm -rf /usr/share/* && \ + mv /tmp/ca-certificates /usr/share/ && \ + mkdir -p /root/.local/share/centrifuge-chain && \ + ln -s /root/.local/share/centrifuge-chain /data \ + # minimize the attack surface + rm -rf /usr/bin /usr/sbin && \ + rm -rf /usr/lib/python* && \ + # check if executable works in this container + ldd /usr/local/bin/centrifuge-chain && \ + /usr/local/bin/centrifuge-chain --version -# Add chain resources to image -COPY res /resources/ + # Add chain resources to image + COPY res /resources/ -# USER centrifuge-chain # see above +USER centrifuge EXPOSE 30333 9933 9944 VOLUME ["/data"] From 934145dba49b6e8b722e51adcbc1fe0e597a2cb0 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 14 Sep 2023 23:33:53 -0400 Subject: [PATCH 07/67] Add prep action for common Ubuntu steps. +Add myself to codeowners --- .github/CODEOWNERS | 18 +++++----- .github/actions/prep-ubuntu/action.yml | 49 ++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) create mode 100644 .github/actions/prep-ubuntu/action.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b525bfd297..b6d6611823 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,19 +1,19 @@ # Centrifudge Code Owners ## Changes to .github -.github/* @mikiquantum @mustermeiszer @branan @NunoAlexandre - +.github/* @mustermeiszer @branan @NunoAlexandre +.github/workflows @gpmayorga ## Changes to ci -ci/* @mikiquantum @mustermeiszer @branan @NunoAlexandre +ci/* @mustermeiszer @branan @NunoAlexandre ## Changes to the service of our chain. -node/* @mikiquantum @mustermeiszer @branan @NunoAlexandre +node/* @mustermeiszer @branan @NunoAlexandre ## Changes to chain-specs -node/res/* @mikiquantum @mustermeiszer @branan @NunoAlexandre @wischli +node/res/* @mustermeiszer @branan @NunoAlexandre @wischli ## Changes to toml files -*.toml @mikiquantum @mustermeiszer @branan @NunoAlexandre @lemunozm @wischli @cdamian @thea-leake +*.toml @mustermeiszer @branan @NunoAlexandre @lemunozm @wischli @cdamian @thea-leake ## Changes to specific pallets pallets/liquidity-pools/* @NunoAlexandre @cdamian @wischli @mustermeiszer @@ -41,9 +41,9 @@ libs/traits/src/changes.rs @lemunozm libs/traits/src/data.rs @lemunozm ## Changes to runtime -runtime/common/* @branan @mikiquantum @mustermeiszer @NunoAlexandre @offerijns @lemunozm -runtime/altair/* @branan @mikiquantum @mustermeiszer @NunoAlexandre @offerijns @wischli -runtime/centrifuge/* @branan @mikiquantum @mustermeiszer @NunoAlexandre @offerijns @wischli +runtime/common/* @branan @mustermeiszer @NunoAlexandre @offerijns @lemunozm +runtime/altair/* @branan @mustermeiszer @NunoAlexandre @offerijns @wischli +runtime/centrifuge/* @branan @mustermeiszer @NunoAlexandre @offerijns @wischli ## Changes to integration tests runtime/integration-tests/* @mustermeiszer @NunoAlexandre @wischli @cdamian diff --git a/.github/actions/prep-ubuntu/action.yml b/.github/actions/prep-ubuntu/action.yml new file mode 100644 index 0000000000..3069e3e0ed --- /dev/null +++ b/.github/actions/prep-ubuntu/action.yml @@ -0,0 +1,49 @@ +name: Prepare Ubuntu for Rust builds +description: cleanup and Rust Tools setup +inputs: + RUST_TOOLCHAIN: + description: toolchain version + default: "1.66" + cache: + description: cache type (enabled if set) + default: "disabled" + GWIP: + description: "Google Workload identity provider" + default: '' + GSA: + description: "Google Service Account" + default: '' +runs: + using: composite + steps: + - name: Prep build on Ubuntu + id: ubuntu_prep + shell: sh + run: | + echo "Pre cleanup" + df -h + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + echo "Post cleanup" + df -h + sudo apt-get install protobuf-compiler + + - name: Install latest nightly + uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 + with: + toolchain: ${{ inputs.RUST_TOOLCHAIN }} + default: true + + - name: SCcache setup + if: ${{ inputs.cache == 'enabled' }} + uses: ./.github/actions/sccache-gcloud + with: + GWIP: ${{ inputs.GWIP }} + GSA: ${{ inputs.GSA }} + + - name: setup Rust sccache wrapper + if: ${{ inputs.cache == 'enabled' }} + shell: sh + run: echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV + + From 3641a117c0925fca038359e92bb2ee59af5dfb01 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 14 Sep 2023 23:57:32 -0400 Subject: [PATCH 08/67] separate sccache gcloud action --- .github/actions/prep-ubuntu/action.yml | 6 +--- .github/actions/sccache-gcloud/action.yml | 37 +++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 .github/actions/sccache-gcloud/action.yml diff --git a/.github/actions/prep-ubuntu/action.yml b/.github/actions/prep-ubuntu/action.yml index 3069e3e0ed..c37827a9aa 100644 --- a/.github/actions/prep-ubuntu/action.yml +++ b/.github/actions/prep-ubuntu/action.yml @@ -40,10 +40,6 @@ runs: with: GWIP: ${{ inputs.GWIP }} GSA: ${{ inputs.GSA }} - - - name: setup Rust sccache wrapper - if: ${{ inputs.cache == 'enabled' }} - shell: sh - run: echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV + diff --git a/.github/actions/sccache-gcloud/action.yml b/.github/actions/sccache-gcloud/action.yml new file mode 100644 index 0000000000..598c00089f --- /dev/null +++ b/.github/actions/sccache-gcloud/action.yml @@ -0,0 +1,37 @@ +name: Install SCcache w/ GCloud bucket +description: Configure Scache to use a Gcloud bucket through workload identity + +inputs: + GWIP: + description: Google Workflow Identity provider + required: true + GSA: + description: Google Service Account + required: true +runs: + using: composite + steps: + + - name: Auth gcloud + id: gauth + uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # @v1 + with: + workload_identity_provider: ${{ inputs.GWIP }} + service_account: ${{ inputs.GSA }} + # export_environment_variables: true + + - name: Run sccache-cache + uses: mozilla-actions/sccache-action@8417cffc2ec64127ad83077aceaa8631f7cdc83e #v0.0.3 + + - name: set GH variables + shell: bash + run: | + # The gauth step should already take care of this: + # echo "SCCACHE_GCS_KEY_PATH=${{ steps.gauth.credentials_file_path }}" >> $GITHUB_ENV + echo "SCCACHE_GCS_BUCKET=centrifuge-chain-sccache-backend" >> $GITHUB_ENV + echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> $GITHUB_ENV + + - name: setup Rust sccache wrapper + if: ${{ inputs.cache == 'enabled' }} + shell: sh + run: echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV \ No newline at end of file From 98ca8f3ecbfcfeeb17611e79bbada8ad15c5bb90 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 00:28:13 -0400 Subject: [PATCH 09/67] fix dockertag pattern --- .github/workflows/build-docker.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index d1d20df52d..ad04e83dc4 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -14,9 +14,8 @@ jobs: docker: strategy: matrix: - os: [ ubuntu-latest-4-cores ] target: [ release, test ] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest-4-cores env: WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} steps: @@ -28,6 +27,7 @@ jobs: # https://github.com/actions/runner-images/issues/2840#issuecomment-1284059930 run: | sudo rm -rf /usr/share/dotnet + # sudo rm -rf "/usr/local/share/boost" sudo rm -rf "$AGENT_TOOLSDIRECTORY" - name: check available docker space @@ -51,15 +51,16 @@ jobs: with: images: centrifugeio/centrifuge-chain flavor: | - prefix=${{ matrix.target == 'test' && 'test-' || '' }} - suffix=${{ env.NOW }} + suffix=-${{ env.NOW }} + # latest=auto is already default, but the logic needs a value at the end and '' isn't valid + ${{ matrix.target == 'test' && 'prefix=test-' || 'latest=auto' }} tags: | - type=schedule type=ref,event=branch type=ref,event=tag + type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}} - type=sha,format=short,prefix='git-' + type=edge - name: Build and push centrifugeio/centrifuge-chain uses: docker/build-push-action@v5 From c2bc2cf7ed30cdb881d755cbfa912f299be3a7e0 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 00:35:14 -0400 Subject: [PATCH 10/67] delete old benchmark check --- .github/workflows/benchmark-check.yml | 40 --------------------------- 1 file changed, 40 deletions(-) delete mode 100644 .github/workflows/benchmark-check.yml diff --git a/.github/workflows/benchmark-check.yml b/.github/workflows/benchmark-check.yml deleted file mode 100644 index f661bcf5f7..0000000000 --- a/.github/workflows/benchmark-check.yml +++ /dev/null @@ -1,40 +0,0 @@ -on: - push: - branches: [main, 'release-v**'] - pull_request: -name: Check Benchmarks -jobs: - tests: - name: ${{ matrix.target }} - strategy: - matrix: - os: [ubuntu-latest] - target: [benchmark-check] - runtime: [development, altair, centrifuge] - runs-on: ${{ matrix.os }} - env: - RUST_TOOLCHAIN: "nightly-2022-05-09" - steps: - - name: Prep build on Ubuntu - if: ${{ matrix.os }} == 'ubuntu-latest' - run: | - echo "Pre cleanup" - df -h - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - echo "Post cleanup" - df -h - sudo apt-get install protobuf-compiler - - name: Check out code - uses: actions/checkout@v3 - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ env.RUST_TOOLCHAIN }} - default: true - - uses: Swatinem/rust-cache@cb2cf0cc7c5198d3364b9630e2c3d457f160790c - - name: Run fast benchmarks - run: ./ci/script.sh - env: - TARGET: ${{ matrix.target }} - RUNTIME: ${{ matrix.runtime }} From 79eff04205432b8e49708a1c4cfdc224426e76d0 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 01:41:37 -0400 Subject: [PATCH 11/67] Trick the wasm publish for this branch --- .github/workflows/build-wasm.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 0003ae19a7..458458501a 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -8,7 +8,8 @@ concurrency: cancel-in-progress: true jobs: build-runtime-wasms: - environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} + # environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} + environment: production # Benchmark (with cache hit) # ubuntu-latest 50 min # ubuntu-latest-4-cores 34 min @@ -76,17 +77,19 @@ jobs: /srtool/build - id: 'auth' + # if: ${{ github.ref == 'refs/heads/main' }} uses: 'google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033' # v1.1.1 with: workload_identity_provider: ${{ secrets.GWIP_ARTIFACTS }} service_account: ${{ secrets.GSA_ARTIFACTS }} - name: Setup - gcloud / gsutil + if: ${{ steps.auth.outcome == 'success }} uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 - name: Publish artifacts to GCS - # if: ${{ github.ref == 'refs/heads/main' }} + if: ${{ steps.auth.outcome == 'success }} shell: bash run: | target=$(echo "${{ matrix.target }}" | sed -e "s/^build-//" ) From bb5713e5070e5c7063c3a9c36c4f252cf81711fc Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 01:29:21 -0400 Subject: [PATCH 12/67] new benchmark pipeline --- .github/workflows/benchmark-auto-pr.yml | 51 +++++++++++++++++++++++++ .github/workflows/benchmark.yml | 12 +++--- 2 files changed, 56 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/benchmark-auto-pr.yml diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml new file mode 100644 index 0000000000..8ce6e5f93b --- /dev/null +++ b/.github/workflows/benchmark-auto-pr.yml @@ -0,0 +1,51 @@ +name: Benchmark +on: + push: + branches: [main] + pull_request: + paths: + - '.github/workflows/benchmark-auto-pr.yml' +jobs: + benchmark: + permissions: + pull-requests: write + id-token: write + runs-on: ubuntu-latest-8-cores + strategy: + matrix: + runtimes: [centrifuge, altair] + env: + RUST_TOOLCHAIN: "1.66.0" + RUSTC_WRAPPER: "sccache" + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: 'main' + + - name: Prep build on Ubuntu + uses: ./.github/actions/prep-ubuntu + with: + RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + cache: enabled + # Cache needs Google credentials: + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} + + - name: Run Benchmark ${{ matrix.runtimes }} + run: ./scripts/runtime_benchmarks.sh ${{ matrix.runtimes }} + + - name: Create PR with new benchmarks + uses: peter-evans/create-pull-request@v5 + with: + add-paths: runtime/${{ matrix.runtimes }}/src/weights/ + commit-message: "New benchmark weights generated from main by Github Actions Bot" + base: main + branch: benchmarks/ + delete-branch: true + + # - name: Upload artifact + # uses: actions/upload-artifact@v3 + # with: + # name: ${{steps.parse-runtime.outputs.result}}-weights + # path: runtime/${{steps.parse-runtime.outputs.result}}/src/weights/ diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c1057f5ee8..c10e9acc6d 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -1,7 +1,7 @@ name: Benchmark on: issue_comment: - types: [created] + types: [created] jobs: benchmark: permissions: @@ -58,12 +58,10 @@ jobs: uses: ./.github/actions/prep-ubuntu with: RUST_TOOLCHAIN: ${{ matrix.rust_version }} - - - name: SCcache setup - uses: ./.github/actions/sccache-gcloud - with: - GWIP: ${{ secrets.GWIP_SCCACHE }} - GSA: ${{ secrets.GSA_SCCACHE }} + cache: enabled + # Cache needs Google credentials: + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} - name: Run Benchmark ${{steps.parse-runtime.outputs.result}} run: ./scripts/runtime_benchmarks.sh ${{steps.parse-runtime.outputs.result}} From c6be0525dd4b39faa11924d56ca6f048a3af2318 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 01:29:35 -0400 Subject: [PATCH 13/67] additional cache options --- .github/workflows/build-wasm.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 458458501a..e04b89de13 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -44,7 +44,11 @@ jobs: with: prefix-key: build #-${{ matrix.package }} shared-key: wasm - # save-if: ${{ github.ref == 'refs/heads/master' }} + workspaces: | + . + ./target/srtool + cache-directories: ./runtime + # save-if: ${{ github.ref == 'refs/heads/master' }} - name: Setup build options id: buildopt From 309afdb61843d7cd09e8ba82a498aab933defd2a Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 01:45:36 -0400 Subject: [PATCH 14/67] exclude runtime integration tests from dockerfile --- .github/workflows/benchmark.yml | 4 ++-- Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c10e9acc6d..c8e85aa6c9 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -60,8 +60,8 @@ jobs: RUST_TOOLCHAIN: ${{ matrix.rust_version }} cache: enabled # Cache needs Google credentials: - GWIP: ${{ secrets.GWIP_SCCACHE }} - GSA: ${{ secrets.GSA_SCCACHE }} + GWIP: ${{ secrets.GWIP_SCCACHE }} + GSA: ${{ secrets.GSA_SCCACHE }} - name: Run Benchmark ${{steps.parse-runtime.outputs.result}} run: ./scripts/runtime_benchmarks.sh ${{steps.parse-runtime.outputs.result}} diff --git a/Dockerfile b/Dockerfile index d78a10ec4f..7550def7eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ FROM docker.io/library/ubuntu:22.04 as builder RUN rustup default $RUST_TOOLCHAIN && \ rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN && \ - cargo build "--release" --features=${FEATURES} + cargo build "--release" --exclude runtime-integration-tests --features=${FEATURES} # ===== SECOND STAGE ====== From c2b8fc7a3febba282ae5a72f3f9ab5d77dc8abdf Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 02:24:20 -0400 Subject: [PATCH 15/67] fix buildwasm --- .github/workflows/build-wasm.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index e04b89de13..9e8dd1170d 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -6,6 +6,9 @@ name: Build&Publish WASM concurrency: group: 'build-wasm-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true +permissions: + id-token: write + contents: read jobs: build-runtime-wasms: # environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} @@ -33,12 +36,12 @@ jobs: steps: - name: Check out code uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 - + - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu with: RUST_TOOLCHAIN: ${{ matrix.rust_version }} - + # sccache does not work inside the srtool container - uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # 2.6.2 with: @@ -48,7 +51,7 @@ jobs: . ./target/srtool cache-directories: ./runtime - # save-if: ${{ github.ref == 'refs/heads/master' }} + # save-if: ${{ github.ref == 'refs/heads/master' }} - name: Setup build options id: buildopt @@ -76,7 +79,7 @@ jobs: echo "---- Rust Versions ----" rustc --version rustup --version - cargo --version + cargo --version echo "---- RUNNING BUILD ----" /srtool/build @@ -85,15 +88,15 @@ jobs: uses: 'google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033' # v1.1.1 with: workload_identity_provider: ${{ secrets.GWIP_ARTIFACTS }} - service_account: ${{ secrets.GSA_ARTIFACTS }} + service_account: ${{ secrets.GSA_ARTIFACTS }} - name: Setup - gcloud / gsutil - if: ${{ steps.auth.outcome == 'success }} + if: ${{ steps.auth.outcome == 'success' }} uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 - + - name: Publish artifacts to GCS - if: ${{ steps.auth.outcome == 'success }} + if: ${{ steps.auth.outcome == 'success' }} shell: bash run: | target=$(echo "${{ matrix.target }}" | sed -e "s/^build-//" ) From 53a21a0da055358c54a61ff2e21d95cdfcf7bc17 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 02:35:11 -0400 Subject: [PATCH 16/67] fix docker build & fix benchmarks --- .github/workflows/benchmark-auto-pr.yml | 7 +++++-- Dockerfile | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index 8ce6e5f93b..e7a23b3426 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -20,8 +20,6 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 - with: - ref: 'main' - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu @@ -32,6 +30,11 @@ jobs: GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: 'main' + - name: Run Benchmark ${{ matrix.runtimes }} run: ./scripts/runtime_benchmarks.sh ${{ matrix.runtimes }} diff --git a/Dockerfile b/Dockerfile index 7550def7eb..d78a10ec4f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ FROM docker.io/library/ubuntu:22.04 as builder RUN rustup default $RUST_TOOLCHAIN && \ rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN && \ - cargo build "--release" --exclude runtime-integration-tests --features=${FEATURES} + cargo build "--release" --features=${FEATURES} # ===== SECOND STAGE ====== From f79e94bcc9d0b0c192ba1dfd20001a8efe21bdac Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 02:38:37 -0400 Subject: [PATCH 17/67] remove old benchmark job --- .github/actions/sccache-gcloud/action.yml | 4 +- .github/workflows/benchmark.yml | 88 ----------------------- 2 files changed, 2 insertions(+), 90 deletions(-) delete mode 100644 .github/workflows/benchmark.yml diff --git a/.github/actions/sccache-gcloud/action.yml b/.github/actions/sccache-gcloud/action.yml index 598c00089f..946db27561 100644 --- a/.github/actions/sccache-gcloud/action.yml +++ b/.github/actions/sccache-gcloud/action.yml @@ -26,10 +26,10 @@ runs: - name: set GH variables shell: bash run: | - # The gauth step should already take care of this: - # echo "SCCACHE_GCS_KEY_PATH=${{ steps.gauth.credentials_file_path }}" >> $GITHUB_ENV echo "SCCACHE_GCS_BUCKET=centrifuge-chain-sccache-backend" >> $GITHUB_ENV echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> $GITHUB_ENV + # The gauth step should already take care of this: + # echo "SCCACHE_GCS_KEY_PATH=${{ steps.gauth.credentials_file_path }}" >> $GITHUB_ENV - name: setup Rust sccache wrapper if: ${{ inputs.cache == 'enabled' }} diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml deleted file mode 100644 index c8e85aa6c9..0000000000 --- a/.github/workflows/benchmark.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Benchmark -on: - issue_comment: - types: [created] -jobs: - benchmark: - permissions: - pull-requests: write - id-token: write - name: benchmark - runs-on: ubuntu-latest-4-cores - if: ${{ startsWith(github.event.comment.body, '/benchmark ') && github.event.issue.pull_request && contains(github.event.comment.author_association, 'CONTRIBUTOR') }} - env: - RUST_TOOLCHAIN: "1.66.0" - RUSTC_WRAPPER: "sccache" - steps: - - uses: actions/github-script@v6 - name: Parse Runtime From Comment - id: parse-runtime - with: - result-encoding: string - script: | - console.log("Comment body", context.payload.comment.body) - const [, , runtime] = context.payload.comment.body.split(/\W+/) - if (!['centrifuge', 'altair'].includes(runtime)) throw new Error('Unsupported Runtime: ${runtime}') - return `${runtime}` - - uses: actions/github-script@v6 - name: Acknowledge Run Comment - id: ack-run-comment - with: - script: | - const comment = await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `:clock1: Running Benchmarks for \`${{steps.parse-runtime.outputs.result}}\` :clock1:` - }) - return comment.data.id - - name: Get PR Hash - id: get_pr_hash - uses: actions/github-script@v6 - with: - result-encoding: string - script: | - const number = context.payload.issue.number; - const pr = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: number - }); - return pr.data.head.sha - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: ${{ steps.get_pr_hash.outputs.result }} - - - name: Prep build on Ubuntu - uses: ./.github/actions/prep-ubuntu - with: - RUST_TOOLCHAIN: ${{ matrix.rust_version }} - cache: enabled - # Cache needs Google credentials: - GWIP: ${{ secrets.GWIP_SCCACHE }} - GSA: ${{ secrets.GSA_SCCACHE }} - - - name: Run Benchmark ${{steps.parse-runtime.outputs.result}} - run: ./scripts/runtime_benchmarks.sh ${{steps.parse-runtime.outputs.result}} - - - name: Upload artifact - uses: actions/upload-artifact@v3 - with: - name: ${{steps.parse-runtime.outputs.result}}-weights - path: runtime/${{steps.parse-runtime.outputs.result}}/src/weights/ - - - uses: actions/github-script@v6 - name: Notify Benchmark finished - with: - script: | - const id = `${{steps.ack-run-comment.outputs.result}}` - await github.rest.issues.updateComment({ - comment_id: id, - owner: context.repo.owner, - repo: context.repo.repo, - body: ` - :white_check_mark: Uploaded benchmarks for: \`${{steps.parse-runtime.outputs.result}}\` :white_check_mark: - Find the artifact here: https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId} - ` - }) From 97fa94ebed8e3742969d7263c239893e8bc08522 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 02:52:04 -0400 Subject: [PATCH 18/67] debug benchmark cache --- .github/actions/sccache-gcloud/action.yml | 6 +++++- .github/workflows/benchmark-auto-pr.yml | 11 ++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/actions/sccache-gcloud/action.yml b/.github/actions/sccache-gcloud/action.yml index 946db27561..9afcd3adca 100644 --- a/.github/actions/sccache-gcloud/action.yml +++ b/.github/actions/sccache-gcloud/action.yml @@ -34,4 +34,8 @@ runs: - name: setup Rust sccache wrapper if: ${{ inputs.cache == 'enabled' }} shell: sh - run: echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV \ No newline at end of file + run: echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV + + - name: Run sccache stat for check + shell: bash + run: ${SCCACHE_PATH} --show-stats \ No newline at end of file diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index e7a23b3426..9d4854ae88 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -1,4 +1,4 @@ -name: Benchmark +name: Benchmark PR on: push: branches: [main] @@ -9,6 +9,7 @@ jobs: benchmark: permissions: pull-requests: write + contents: read id-token: write runs-on: ubuntu-latest-8-cores strategy: @@ -30,10 +31,10 @@ jobs: GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} - - name: Checkout code - uses: actions/checkout@v3 - with: - ref: 'main' + # - name: Checkout code + # uses: actions/checkout@v3 + # with: + # ref: 'main' - name: Run Benchmark ${{ matrix.runtimes }} run: ./scripts/runtime_benchmarks.sh ${{ matrix.runtimes }} From 6a4c523a63692476efa427429974032ee0bb71bf Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 03:23:00 -0400 Subject: [PATCH 19/67] fix wasm location upload --- .github/workflows/build-wasm.yml | 8 ++++---- .github/workflows/sanity-checks.yml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 9e8dd1170d..046b0520d7 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -81,7 +81,7 @@ jobs: rustup --version cargo --version echo "---- RUNNING BUILD ----" - /srtool/build + /srtool/build --json - id: 'auth' # if: ${{ github.ref == 'refs/heads/main' }} @@ -94,13 +94,13 @@ jobs: if: ${{ steps.auth.outcome == 'success' }} uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1 - - name: Publish artifacts to GCS if: ${{ steps.auth.outcome == 'success' }} shell: bash run: | target=$(echo "${{ matrix.target }}" | sed -e "s/^build-//" ) runtime_name=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//" ) + filename=$(echo "${{ matrix.package }}" | sed -i 's/-/_/g' ) gsutil cp \ - ./runtime/$runtime_name/target/srtool/release/wbuild/${{ matrix.package }}/${{ matrix.package }}.compact.compressed.wasm \ - gs://centrifuge-wasm-repo/$RUNTIME/$target/$runtime_name-$target-$(git rev-parse --short HEAD).wasm + ./runtime/${runtime_name}/target/srtool/release/wbuild/${{ matrix.package }}/${filename}.compact.compressed.wasm \ + gs://centrifuge-wasm-repo/$RUNTIME/${target}/${runtime_name}-${target}-$(git rev-parse --short HEAD).wasm diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index 6f924f7bcd..c4eb7227e2 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -42,7 +42,7 @@ jobs: TARGET: ${{ matrix.target }} benchmark-check: - name: bench-${{ matrix.runtime }} + name: bench-check-${{ matrix.runtime }} runs-on: ubuntu-latest #r-4-cores strategy: matrix: From 12292b06714befd9f437f7bbdecb5bad9231f390 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 03:53:26 -0400 Subject: [PATCH 20/67] better benchmark PR --- .github/workflows/benchmark-auto-pr.yml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index 9d4854ae88..d6fd640c62 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -39,17 +39,28 @@ jobs: - name: Run Benchmark ${{ matrix.runtimes }} run: ./scripts/runtime_benchmarks.sh ${{ matrix.runtimes }} + - run: echo "NOW=$(date -u +%Y-%m-%d)" >> $GITHUB_ENV + - name: Create PR with new benchmarks uses: peter-evans/create-pull-request@v5 with: add-paths: runtime/${{ matrix.runtimes }}/src/weights/ - commit-message: "New benchmark weights generated from main by Github Actions Bot" base: main - branch: benchmarks/ + branch: benchmarks/${{ matrix.runtimes }}-${{ env.NOW }} delete-branch: true - + reviewers: ${{ github.actor }} + draft: true + labels: gha, benchmarks + commit-message: "New benchmark weights generated from main by Github Actions Bot" + title: New benchmark weights for ${{ matrix.runtimes }} + body: | + Automated PR generated by Benchmark PR Github Actions job + Runtime: ${{ matrix.runtimes }} + Commit that triggered: + ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} + # - name: Upload artifact # uses: actions/upload-artifact@v3 # with: - # name: ${{steps.parse-runtime.outputs.result}}-weights - # path: runtime/${{steps.parse-runtime.outputs.result}}/src/weights/ + # name: ${{ matrix.runtimes }}-weights + # path: runtime/${{ matrix.runtimes }}/src/weights/ From 9486c1eb20ef9cf7a3698fe41252755bb2840c1e Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 03:59:15 -0400 Subject: [PATCH 21/67] small fixes --- .github/workflows/build-docker.yml | 9 +++++---- .github/workflows/build-wasm.yml | 8 +++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index ad04e83dc4..a1d80ff901 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -7,11 +7,13 @@ on: - ".github/workflows/build-docker.yml" - "Dockerfile" workflow_dispatch: - +concurrency: + group: build-docker-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} + cancel-in-progress: true env: RUST_TOOLCHAIN: "1.66" jobs: - docker: + docker: strategy: matrix: target: [ release, test ] @@ -19,7 +21,6 @@ jobs: env: WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} steps: - - uses: actions/checkout@v3 with: fetch-depth: 0 @@ -43,7 +44,7 @@ jobs: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - - run: echo "NOW=$(date -u +%Y%m%d)" >> $GITHUB_ENV + - run: echo "NOW=$(date -u +%y-%m-%d)" >> $GITHUB_ENV - name: Setup docker metadata id: meta diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 046b0520d7..4a7aea9cc5 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -61,6 +61,7 @@ jobs: elif ${{ matrix.target == 'build-runtime-fast'}} ; then echo "BUILD_OPTS="--features=fast-runtime"" >> GITHUB_ENV fi + echo "BUILD_OPTS=${{ env.BUILD_OPT }}" - name: Run Docker SRTool uses: addnab/docker-run-action@v3 @@ -71,15 +72,12 @@ jobs: image: paritytech/srtool:${{ matrix.rust_version }} options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e PACKAGE=${{ matrix.package }} run: | + echo "---- Env VARS ----" echo "BUILD_OPT=${BUILD_OPT}" echo "PACKAGE=${PACKAGE}" echo "---- List Cache Folder ----" ls -la /cargo-home/ du -sh /cargo-home/* - echo "---- Rust Versions ----" - rustc --version - rustup --version - cargo --version echo "---- RUNNING BUILD ----" /srtool/build --json @@ -100,7 +98,7 @@ jobs: run: | target=$(echo "${{ matrix.target }}" | sed -e "s/^build-//" ) runtime_name=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//" ) - filename=$(echo "${{ matrix.package }}" | sed -i 's/-/_/g' ) + filename=$(echo "${{ matrix.package }}" | sed -e` 's/-/_/g' ) gsutil cp \ ./runtime/${runtime_name}/target/srtool/release/wbuild/${{ matrix.package }}/${filename}.compact.compressed.wasm \ gs://centrifuge-wasm-repo/$RUNTIME/${target}/${runtime_name}-${target}-$(git rev-parse --short HEAD).wasm From ed7ec02c425b237c35370972ed146bc5e0816c05 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 14:09:42 -0400 Subject: [PATCH 22/67] fix typo on build-wasm --- .github/workflows/build-wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 4a7aea9cc5..ae5041b3a2 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -98,7 +98,7 @@ jobs: run: | target=$(echo "${{ matrix.target }}" | sed -e "s/^build-//" ) runtime_name=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//" ) - filename=$(echo "${{ matrix.package }}" | sed -e` 's/-/_/g' ) + filename=$(echo "${{ matrix.package }}" | sed -e 's/-/_/g' ) gsutil cp \ ./runtime/${runtime_name}/target/srtool/release/wbuild/${{ matrix.package }}/${filename}.compact.compressed.wasm \ gs://centrifuge-wasm-repo/$RUNTIME/${target}/${runtime_name}-${target}-$(git rev-parse --short HEAD).wasm From 5200f383affa29735a638e1c9cbc279e10472c3d Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 14:40:35 -0400 Subject: [PATCH 23/67] better Docker build and management --- .github/workflows/build-docker.yml | 3 +++ .../centrifuge-chain/.dockerignore | 0 .../centrifuge-chain/Dockerfile | 24 +++++++------------ .../docker-compose-local-chain.yml | 0 .../docker-compose-local-relay.yml | 0 5 files changed, 12 insertions(+), 15 deletions(-) rename .dockerignore => docker/centrifuge-chain/.dockerignore (100%) rename Dockerfile => docker/centrifuge-chain/Dockerfile (85%) rename docker-compose-local-chain.yml => docker/docker-compose-local-chain.yml (100%) rename docker-compose-local-relay.yml => docker/docker-compose-local-relay.yml (100%) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index a1d80ff901..3f20bd29fa 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -65,8 +65,11 @@ jobs: - name: Build and push centrifugeio/centrifuge-chain uses: docker/build-push-action@v5 + env: + FEATURES: ${{ matrix.target == 'test' && '"fast-runtime"' || '' }} with: context: . + file: ./docker/centrifuge-chain/Dockerfile push: ${{ github.ref == '/refs/heads/main' && true || false }} tags: ${{ steps.meta.outputs.tags }} # Cache options: diff --git a/.dockerignore b/docker/centrifuge-chain/.dockerignore similarity index 100% rename from .dockerignore rename to docker/centrifuge-chain/.dockerignore diff --git a/Dockerfile b/docker/centrifuge-chain/Dockerfile similarity index 85% rename from Dockerfile rename to docker/centrifuge-chain/Dockerfile index d78a10ec4f..507abaed0b 100644 --- a/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -1,11 +1,12 @@ # Based on # https://github.com/paritytech/polkadot-sdk/blob/master/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile + +# ToDo: create a CI/builder image with preloaded tools FROM docker.io/library/ubuntu:22.04 as builder # Defaults ENV RUST_BACKTRACE 1 ENV DEBIAN_FRONTEND=noninteractive - ENV RUST_TOOLCHAIN=$RUST_TOOLCHAIN ARG FEATURES="" ARG RUST_TOOLCHAIN="1.66" @@ -34,7 +35,7 @@ FROM docker.io/library/ubuntu:22.04 as builder cargo build "--release" --features=${FEATURES} # ===== SECOND STAGE ====== - +# ToDo: create a secure image as a base for the binary FROM docker.io/library/ubuntu:22.04 LABEL io.centrifuge.image.authors="guillermo@k-f.co" \ @@ -45,35 +46,28 @@ FROM docker.io/library/ubuntu:22.04 # io.centrifuge.image.revision="${VCS_REF}" \ io.centrifuge.image.created="${BUILD_DATE}" - COPY --from=builder /centrifuge-chain/target/release/centrifuge-chain /usr/local/bin + # Add chain resources to image + COPY res /resources/ RUN useradd -m -u 1000 -U -s /bin/sh -d /centrifuge centrifuge && \ mkdir -p /data /centrifuge/.local/share && \ chown -R centrifuge:centrifuge /data && \ ln -s /data /centrifuge/.local/share/centrifuge - + + COPY --from=builder /centrifuge-chain/target/release/centrifuge-chain /usr/local/bin # checks RUN ldd /usr/local/bin/centrifuge-chain && \ /usr/local/bin/centrifuge-chain --version - # Save sh and bash - RUN cp /usr/bin/sh /usr/bin/bash /usr/local/bin/ /root/ - # Unclutter RUN mv /usr/share/ca* /tmp && \ rm -rf /usr/share/* && \ mv /tmp/ca-certificates /usr/share/ && \ mkdir -p /root/.local/share/centrifuge-chain && \ - ln -s /root/.local/share/centrifuge-chain /data \ + ln -s /root/.local/share/centrifuge-chain /data # minimize the attack surface - rm -rf /usr/bin /usr/sbin && \ - rm -rf /usr/lib/python* && \ - # check if executable works in this container - ldd /usr/local/bin/centrifuge-chain && \ - /usr/local/bin/centrifuge-chain --version + # rm -rf /usr/bin /usr/sbin /usr/lib/python* && \ - # Add chain resources to image - COPY res /resources/ USER centrifuge EXPOSE 30333 9933 9944 diff --git a/docker-compose-local-chain.yml b/docker/docker-compose-local-chain.yml similarity index 100% rename from docker-compose-local-chain.yml rename to docker/docker-compose-local-chain.yml diff --git a/docker-compose-local-relay.yml b/docker/docker-compose-local-relay.yml similarity index 100% rename from docker-compose-local-relay.yml rename to docker/docker-compose-local-relay.yml From d4674406caba097a6fe4e30d1bda2673e6c02695 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 15:07:16 -0400 Subject: [PATCH 24/67] small fixes --- .github/workflows/benchmark-auto-pr.yml | 3 +++ .github/workflows/build-wasm.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index d6fd640c62..eeba4f87fb 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -5,6 +5,9 @@ on: pull_request: paths: - '.github/workflows/benchmark-auto-pr.yml' +concurrency: + group: build-benchmark-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} + cancel-in-progress: true jobs: benchmark: permissions: diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index ae5041b3a2..0088c007c1 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -101,4 +101,4 @@ jobs: filename=$(echo "${{ matrix.package }}" | sed -e 's/-/_/g' ) gsutil cp \ ./runtime/${runtime_name}/target/srtool/release/wbuild/${{ matrix.package }}/${filename}.compact.compressed.wasm \ - gs://centrifuge-wasm-repo/$RUNTIME/${target}/${runtime_name}-${target}-$(git rev-parse --short HEAD).wasm + gs://centrifuge-wasm-repo/${runtime_name}/${target}/${runtime_name}-${target}-$(git rev-parse --short HEAD).wasm From 0bfbf7a43cc6aa30e77aac35f9f25cdf0baa4b9b Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 15:17:55 -0400 Subject: [PATCH 25/67] cleanup and concurrency review --- .github/workflows/benchmark-auto-pr.yml | 2 +- .github/workflows/build-docker.yml | 16 ++++++++-------- .github/workflows/build-wasm.yml | 2 +- .github/workflows/docs.yml | 3 +++ 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index eeba4f87fb..fbc5777e87 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -6,7 +6,7 @@ on: paths: - '.github/workflows/benchmark-auto-pr.yml' concurrency: - group: build-benchmark-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} + group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} cancel-in-progress: true jobs: benchmark: diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 3f20bd29fa..6ca8c3383d 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -8,12 +8,12 @@ on: - "Dockerfile" workflow_dispatch: concurrency: - group: build-docker-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}${{ github.ref == '/refs/heads/main' && github.sha || 'false' }} + cancel-in-progress: true env: RUST_TOOLCHAIN: "1.66" jobs: - docker: + docker: strategy: matrix: target: [ release, test ] @@ -37,12 +37,12 @@ jobs: df -h - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v3 - name: DockerHub Login uses: docker/login-action@v3 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} + password: ${{ secrets.DOCKER_HUB_TOKEN }} - run: echo "NOW=$(date -u +%y-%m-%d)" >> $GITHUB_ENV @@ -61,7 +61,7 @@ jobs: type=ref,event=pr type=semver,pattern={{version}} type=semver,pattern={{major}} - type=edge + type=edge - name: Build and push centrifugeio/centrifuge-chain uses: docker/build-push-action@v5 @@ -72,7 +72,7 @@ jobs: file: ./docker/centrifuge-chain/Dockerfile push: ${{ github.ref == '/refs/heads/main' && true || false }} tags: ${{ steps.meta.outputs.tags }} - # Cache options: + # Cache options: # https://docs.docker.com/build/ci/github-actions/cache/ cache-from: type=registry,ref=centrifugeio/centrifuge-chain # https://docs.docker.com/build/cache/backends/inline/ @@ -86,7 +86,7 @@ jobs: password: ${{ secrets.DOCKER_HUB_TOKEN }} repository: centrifuge/centrifuge-chain short-description: ${{ github.event.repository.description }} - enable-url-completion: true + enable-url-completion: true - if: failure() name: Check available space after build failed diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 0088c007c1..eaeba5801b 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -4,7 +4,7 @@ on: pull_request: name: Build&Publish WASM concurrency: - group: 'build-wasm-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + group: '${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' cancel-in-progress: true permissions: id-token: write diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index bcf1217ca2..d6d9233fd8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,6 +3,9 @@ on: push: branches: [main] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} + cancel-in-progress: true jobs: docs: permissions: From 9fddac4087ed45304c53352401daa6083a890320 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 15 Sep 2023 17:38:31 -0400 Subject: [PATCH 26/67] simplify docker and fix bench-check cache --- .github/workflows/sanity-checks.yml | 11 ++++------- docker/centrifuge-chain/Dockerfile | 25 +++++++++++++++---------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index c4eb7227e2..d63e6c4bbc 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -20,9 +20,6 @@ jobs: matrix: target: [test-general, test-integration, lint-fmt, lint-clippy, cargo-build] # ,lint-taplo] - env: - RUST_TOOLCHAIN: "nightly-2022-11-14" - RUSTC_WRAPPER: "sccache" steps: - name: Check out code uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 @@ -30,7 +27,7 @@ jobs: - name: Prep build uses: ./.github/actions/prep-ubuntu with: - RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + RUST_TOOLCHAIN: "nightly-2022-11-14" cache: enabled # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} @@ -40,6 +37,7 @@ jobs: run: ./ci/run-check.sh env: TARGET: ${{ matrix.target }} + RUSTC_WRAPPER: "sccache" benchmark-check: name: bench-check-${{ matrix.runtime }} @@ -47,8 +45,6 @@ jobs: strategy: matrix: runtime: [development, altair, centrifuge] - env: - RUST_TOOLCHAIN: "nightly-2022-11-14" steps: - name: Check out code uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 @@ -56,7 +52,7 @@ jobs: - name: Prep build uses: ./.github/actions/prep-ubuntu with: - RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + RUST_TOOLCHAIN: "nightly-2022-11-14" setup_cache: true cache: enabled GWIP: ${{ secrets.GWIP_SCCACHE }} @@ -67,3 +63,4 @@ jobs: env: TARGET: benchmark-check RUNTIME: ${{ matrix.runtime }} + RUSTC_WRAPPER: "sccache" diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index 507abaed0b..8c604f6a08 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -29,10 +29,14 @@ FROM docker.io/library/ubuntu:22.04 as builder COPY . centrifuge-chain WORKDIR /centrifuge-chain RUN echo $(ls -l /centrifuge-chain/) - + RUN \ + rustup-init -y --profile minimal --default-toolchain stable; \ + cargo install sccache; + RUN rustup default $RUST_TOOLCHAIN && \ - rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN && \ - cargo build "--release" --features=${FEATURES} + rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN + + # RUN cargo build "--release" --features=${FEATURES} # ===== SECOND STAGE ====== # ToDo: create a secure image as a base for the binary @@ -59,14 +63,15 @@ FROM docker.io/library/ubuntu:22.04 RUN ldd /usr/local/bin/centrifuge-chain && \ /usr/local/bin/centrifuge-chain --version - # Unclutter - RUN mv /usr/share/ca* /tmp && \ - rm -rf /usr/share/* && \ - mv /tmp/ca-certificates /usr/share/ && \ - mkdir -p /root/.local/share/centrifuge-chain && \ - ln -s /root/.local/share/centrifuge-chain /data - # minimize the attack surface + # Unclutter + # RUN mv /usr/share/ca* /tmp && \ + # rm -rf /usr/share/* && \ + # mv /tmp/ca-certificates /usr/share/ + # minimize the attack surface # rm -rf /usr/bin /usr/sbin /usr/lib/python* && \ + RUN mkdir -p /root/.local/share/centrifuge-chain && \ + ln -s /root/.local/share/centrifuge-chain /data + USER centrifuge From 9ca1e9aea96d7213780056920b7d50e04343ac08 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 04:18:54 -0400 Subject: [PATCH 27/67] inject sccache in the docker image --- .dockerignore | 7 ++++ .github/workflows/build-docker.yml | 14 ++++++- ci/install-sccache-tarball.sh | 45 ++++++++++++++++++++++ docker/centrifuge-chain/.dockerignore | 5 --- docker/centrifuge-chain/Dockerfile | 55 +++++++++++++++++---------- 5 files changed, 100 insertions(+), 26 deletions(-) create mode 100644 .dockerignore create mode 100644 ci/install-sccache-tarball.sh delete mode 100644 docker/centrifuge-chain/.dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..50b1114787 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +**/target/ +.github/ +scripts/ +docker-compos +.gitignore +docker +README.md \ No newline at end of file diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 6ca8c3383d..3d69fa745b 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -63,13 +63,25 @@ jobs: type=semver,pattern={{major}} type=edge + - name: Configure sccache + uses: actions/github-script@v6 + with: + script: | + core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); + core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); + - name: Build and push centrifugeio/centrifuge-chain uses: docker/build-push-action@v5 env: - FEATURES: ${{ matrix.target == 'test' && '"fast-runtime"' || '' }} + BUILDKIT_PROGRESS: plain + DOCKER_BUILDKIT: 1 with: context: . file: ./docker/centrifuge-chain/Dockerfile + build-args: | + FEATURES=${{ matrix.target == 'test' && '"fast-runtime"' || '' }} + SCCACHE_GHA_ENABLED="true" + RUSTC_WRAPPER=sccache push: ${{ github.ref == '/refs/heads/main' && true || false }} tags: ${{ steps.meta.outputs.tags }} # Cache options: diff --git a/ci/install-sccache-tarball.sh b/ci/install-sccache-tarball.sh new file mode 100644 index 0000000000..e9f47a6f1b --- /dev/null +++ b/ci/install-sccache-tarball.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Used by centrifuge Linux Docker image docker/centrifuge-chain/Dockerfile +set -eux +# Define URLs and file names +URL="https://github.com/mozilla/sccache/releases/download/v0.5.4/" +TARBALL_URL="${URL}/sccache-v0.5.4-aarch64-unknown-linux-musl.tar.gz" +CHECKSUM_URL="${URL}/sccache-v0.5.4-aarch64-unknown-linux-musl.tar.gz.sha256" +TARBALL_FILENAME="sccache.tar.gz" +CHECKSUM_FILENAME="sccache.sha256" + +# Define the target directory where you want to extract the binary +TARGET_DIR="/usr/local/cargo/bin" + +# Download the tarball and checksum +echo "Downloading tarball..." +curl -L "$TARBALL_URL" -o "$TARBALL_FILENAME" + +echo "Downloading checksum..." +curl -L "$CHECKSUM_URL" -o "$CHECKSUM_FILENAME" + +# Verify the checksum +echo "Verifying checksum..." +EXPECTED_SHA256=$(cat "$CHECKSUM_FILENAME" | awk '{print $1}') +ACTUAL_SHA256=$(sha256sum "$TARBALL_FILENAME" | awk '{print $1}') + +if [ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]; then + echo "Checksum verification failed. Aborting." + rm "$TARBALL_FILENAME" "$CHECKSUM_FILENAME" + exit 1 +fi + +# Extract the tarball +echo "Extracting tarball..." +mkdir sccache +tar -vxzf sccache.tar.gz -C ./sccache/ --strip-components 1 + +# Copy the sccache binary to the target directory +echo "Copying sccache binary to $TARGET_DIR" +cp "sccache/sccache" "$TARGET_DIR/" + +# Clean up downloaded files and extracted folder +rm "$TARBALL_FILENAME" "$CHECKSUM_FILENAME" +rm -rf "sccache" + +echo "Installation completed successfully." diff --git a/docker/centrifuge-chain/.dockerignore b/docker/centrifuge-chain/.dockerignore deleted file mode 100644 index 93c3c5d6db..0000000000 --- a/docker/centrifuge-chain/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -**/target/ -ci/ -.github/ -scripts/ -docker-compose* \ No newline at end of file diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index 8c604f6a08..3bdcb82ab6 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -2,13 +2,11 @@ # https://github.com/paritytech/polkadot-sdk/blob/master/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile # ToDo: create a CI/builder image with preloaded tools -FROM docker.io/library/ubuntu:22.04 as builder +FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder # Defaults ENV RUST_BACKTRACE 1 ENV DEBIAN_FRONTEND=noninteractive - ARG FEATURES="" - ARG RUST_TOOLCHAIN="1.66" RUN apt-get update && \ # apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ @@ -21,26 +19,40 @@ FROM docker.io/library/ubuntu:22.04 as builder libclang-dev \ protobuf-compiler \ curl - - RUN curl https://sh.rustup.rs -sSf | sh -s -- -y - ENV PATH="${PATH}:/root/.cargo/bin" - + + ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH +# install rustup, use minimum components + RUN curl -L "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" \ + -o rustup-init; \ + chmod +x rustup-init; \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain stable; \ + rm rustup-init; +# install sccache + COPY ./ci/install-sccache-tarball.sh ./ + RUN chmod +x install-sccache-tarball.sh && \ + ./install-sccache-tarball.sh + # BUILD - COPY . centrifuge-chain - WORKDIR /centrifuge-chain - RUN echo $(ls -l /centrifuge-chain/) - RUN \ - rustup-init -y --profile minimal --default-toolchain stable; \ - cargo install sccache; - + ARG FEATURES="" + ARG RUST_TOOLCHAIN="1.66" + ARG RUSTC_WRAPPER=sccache + ARG SCCACHE_GHA_ENABLED="false" + RUN rustup default $RUST_TOOLCHAIN && \ rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN - - # RUN cargo build "--release" --features=${FEATURES} + + RUN echo $(sccache --show-stats) + COPY . centrifuge-chain + WORKDIR /centrifuge-chain + RUN --mount=type=cache,target=/root/.cache/sccache \ + cargo build "--release" --features=${FEATURES} + RUN sccache --show-stats # ===== SECOND STAGE ====== # ToDo: create a secure image as a base for the binary -FROM docker.io/library/ubuntu:22.04 +FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 LABEL io.centrifuge.image.authors="guillermo@k-f.co" \ io.centrifuge.image.vendor="Centrifuge" \ @@ -67,13 +79,16 @@ FROM docker.io/library/ubuntu:22.04 # RUN mv /usr/share/ca* /tmp && \ # rm -rf /usr/share/* && \ # mv /tmp/ca-certificates /usr/share/ + # minimize the attack surface - # rm -rf /usr/bin /usr/sbin /usr/lib/python* && \ + # RUN rm -rf /usr/bin /usr/sbin /usr/lib/python* + + RUN apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* ; RUN mkdir -p /root/.local/share/centrifuge-chain && \ ln -s /root/.local/share/centrifuge-chain /data - - + +ENV RUST_BACKTRACE 1 USER centrifuge EXPOSE 30333 9933 9944 VOLUME ["/data"] From 61cbe489070c7eaea68d371b56733c9c73401f7b Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 04:23:08 -0400 Subject: [PATCH 28/67] fix benchmark write access for PRs --- .github/workflows/benchmark-auto-pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index fbc5777e87..4a867848c8 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -12,7 +12,7 @@ jobs: benchmark: permissions: pull-requests: write - contents: read + contents: write id-token: write runs-on: ubuntu-latest-8-cores strategy: From d72014a852971205eb0aa78f230e545df7d4851e Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 04:48:40 -0400 Subject: [PATCH 29/67] test gha cache for docker --- .github/workflows/build-docker.yml | 16 +++++++++------- docker/centrifuge-chain/Dockerfile | 23 ++++++++++++----------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 3d69fa745b..cd0547f59e 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -72,23 +72,25 @@ jobs: - name: Build and push centrifugeio/centrifuge-chain uses: docker/build-push-action@v5 - env: - BUILDKIT_PROGRESS: plain - DOCKER_BUILDKIT: 1 + # env: + # BUILDKIT_PROGRESS: plain + # DOCKER_BUILDKIT: 1 with: context: . file: ./docker/centrifuge-chain/Dockerfile build-args: | FEATURES=${{ matrix.target == 'test' && '"fast-runtime"' || '' }} - SCCACHE_GHA_ENABLED="true" - RUSTC_WRAPPER=sccache + # SCCACHE_GHA_ENABLED="true" + # RUSTC_WRAPPER=sccache push: ${{ github.ref == '/refs/heads/main' && true || false }} tags: ${{ steps.meta.outputs.tags }} # Cache options: # https://docs.docker.com/build/ci/github-actions/cache/ - cache-from: type=registry,ref=centrifugeio/centrifuge-chain + cache-from: type=gha + # cache-from: type=registry,ref=centrifugeio/centrifuge-chain:${{ github.ref }} # https://docs.docker.com/build/cache/backends/inline/ - cache-to: inline + cache-to: type=gha, mode=max + # cache-to: type=registry,ref=centrifugeio/centrifuge-chain:${{ github.ref }}, mode=max - name: Update DockerHub descriptions if: contains(github.ref, 'refs/tags/release-v') diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index 3bdcb82ab6..8d54b82e7e 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -4,7 +4,7 @@ # ToDo: create a CI/builder image with preloaded tools FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder - # Defaults + # Defaults ENV RUST_BACKTRACE 1 ENV DEBIAN_FRONTEND=noninteractive @@ -23,32 +23,33 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ PATH=/usr/local/cargo/bin:$PATH -# install rustup, use minimum components + # install rustup, use minimum components RUN curl -L "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" \ -o rustup-init; \ chmod +x rustup-init; \ ./rustup-init -y --no-modify-path --profile minimal --default-toolchain stable; \ - rm rustup-init; -# install sccache + rm rustup-init; + + # install sccache COPY ./ci/install-sccache-tarball.sh ./ RUN chmod +x install-sccache-tarball.sh && \ ./install-sccache-tarball.sh + RUN echo $(sccache --show-stats) + RUN sccache --show-stats - # BUILD + # BUILD ARG FEATURES="" ARG RUST_TOOLCHAIN="1.66" - ARG RUSTC_WRAPPER=sccache + ARG RUSTC_WRAPPER='' ARG SCCACHE_GHA_ENABLED="false" RUN rustup default $RUST_TOOLCHAIN && \ rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN - - RUN echo $(sccache --show-stats) + COPY . centrifuge-chain WORKDIR /centrifuge-chain - RUN --mount=type=cache,target=/root/.cache/sccache \ - cargo build "--release" --features=${FEATURES} - RUN sccache --show-stats + RUN cargo build "--release" --features=${FEATURES} + # ===== SECOND STAGE ====== # ToDo: create a secure image as a base for the binary From c9264d8b3bbbdb4f0c33d77ea6602b6f730134f7 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 05:02:13 -0400 Subject: [PATCH 30/67] remove sccache from dockerfile --- .github/workflows/build-docker.yml | 2 +- docker/centrifuge-chain/Dockerfile | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index cd0547f59e..661c7b9649 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -79,7 +79,7 @@ jobs: context: . file: ./docker/centrifuge-chain/Dockerfile build-args: | - FEATURES=${{ matrix.target == 'test' && '"fast-runtime"' || '' }} + FEATURES=${{ matrix.target == 'test' && 'fast-runtime' || '' }} # SCCACHE_GHA_ENABLED="true" # RUSTC_WRAPPER=sccache push: ${{ github.ref == '/refs/heads/main' && true || false }} diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index 8d54b82e7e..f4ea94d4af 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -30,13 +30,6 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder ./rustup-init -y --no-modify-path --profile minimal --default-toolchain stable; \ rm rustup-init; - # install sccache - COPY ./ci/install-sccache-tarball.sh ./ - RUN chmod +x install-sccache-tarball.sh && \ - ./install-sccache-tarball.sh - RUN echo $(sccache --show-stats) - RUN sccache --show-stats - # BUILD ARG FEATURES="" ARG RUST_TOOLCHAIN="1.66" From e23b574483cc46366f669d10a8fedec4df2a3b54 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 12:07:36 -0400 Subject: [PATCH 31/67] adjust machine size and fix quotation in Dockfile --- .github/workflows/build-docker.yml | 4 ++-- .github/workflows/build-wasm.yml | 2 +- .github/workflows/docs.yml | 2 +- docker/centrifuge-chain/Dockerfile | 5 +++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 661c7b9649..434ca1045b 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: target: [ release, test ] - runs-on: ubuntu-latest-4-cores + runs-on: ubuntu-latest-8-cores env: WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} steps: @@ -79,7 +79,7 @@ jobs: context: . file: ./docker/centrifuge-chain/Dockerfile build-args: | - FEATURES=${{ matrix.target == 'test' && 'fast-runtime' || '' }} + FEATURES=${{ matrix.target == 'test' && join(['fast', 'runtime'], '-') || '' }} # SCCACHE_GHA_ENABLED="true" # RUSTC_WRAPPER=sccache push: ${{ github.ref == '/refs/heads/main' && true || false }} diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index eaeba5801b..f039396e6c 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -16,7 +16,7 @@ jobs: # Benchmark (with cache hit) # ubuntu-latest 50 min # ubuntu-latest-4-cores 34 min - runs-on: ubuntu-latest-8-cores + runs-on: ubuntu-latest-4-cores strategy: matrix: # To test until I get one right diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d6d9233fd8..70ed8f5e07 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -11,7 +11,7 @@ jobs: permissions: contents: write id-token: write - runs-on: ubuntu-latest-8-cores + runs-on: ubuntu-latest-4-cores env: RUST_TOOLCHAIN: "1.66.0" RUSTDOCFLAGS: "-D warnings" diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index f4ea94d4af..f07e65235c 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -41,7 +41,8 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder COPY . centrifuge-chain WORKDIR /centrifuge-chain - RUN cargo build "--release" --features=${FEATURES} + RUN FEATURES=$(echo ${FEATURES} | tr -d '"') \ + cargo build "--release" --features=${FEATURES} # ===== SECOND STAGE ====== @@ -52,7 +53,7 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 io.centrifuge.image.vendor="Centrifuge" \ io.centrifuge.image.title="centrifugeio/centrifuge-chain" \ io.centrifuge.image.description="Centrifuge, the layer 1 of RWA. This is the official Centrifuge image with an injected binary." \ - io.centrifuge.image.source="https://github.com/centrifuge/centrifuge-chain/blob/main/Dockerfile" \ + io.centrifuge.image.source="https://github.com/centrifuge/centrifuge-chain/blob/main/docker/centrifuge-chain/Dockerfile" \ # io.centrifuge.image.revision="${VCS_REF}" \ io.centrifuge.image.created="${BUILD_DATE}" From 8087d9342eb749bdb1eb8d4cf2df297b3e66ff5d Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 12:37:56 -0400 Subject: [PATCH 32/67] try fix docker var quoting --- .github/workflows/build-docker.yml | 2 +- docker/centrifuge-chain/Dockerfile | 22 +++++++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 434ca1045b..d504ab5c8d 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -79,7 +79,7 @@ jobs: context: . file: ./docker/centrifuge-chain/Dockerfile build-args: | - FEATURES=${{ matrix.target == 'test' && join(['fast', 'runtime'], '-') || '' }} + FEATURES=${{ matrix.target == 'test' && 'fast-runtime' || '' }} # SCCACHE_GHA_ENABLED="true" # RUSTC_WRAPPER=sccache push: ${{ github.ref == '/refs/heads/main' && true || false }} diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index f07e65235c..09aeea6e04 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -7,7 +7,7 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder # Defaults ENV RUST_BACKTRACE 1 ENV DEBIAN_FRONTEND=noninteractive - + RUN apt-get update && \ # apt-get dist-upgrade -y -o Dpkg::Options::="--force-confold" && \ apt-get install -y \ @@ -22,7 +22,7 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH + PATH=/usr/local/cargo/bin:$PATH # install rustup, use minimum components RUN curl -L "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" \ -o rustup-init; \ @@ -31,16 +31,16 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder rm rustup-init; # BUILD - ARG FEATURES="" ARG RUST_TOOLCHAIN="1.66" ARG RUSTC_WRAPPER='' - ARG SCCACHE_GHA_ENABLED="false" + ARG SCCACHE_GHA_ENABLED="false" RUN rustup default $RUST_TOOLCHAIN && \ rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN - + COPY . centrifuge-chain - WORKDIR /centrifuge-chain + WORKDIR /centrifuge-chain + ARG FEATURES="" RUN FEATURES=$(echo ${FEATURES} | tr -d '"') \ cargo build "--release" --features=${FEATURES} @@ -64,7 +64,7 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 mkdir -p /data /centrifuge/.local/share && \ chown -R centrifuge:centrifuge /data && \ ln -s /data /centrifuge/.local/share/centrifuge - + COPY --from=builder /centrifuge-chain/target/release/centrifuge-chain /usr/local/bin # checks RUN ldd /usr/local/bin/centrifuge-chain && \ @@ -77,12 +77,12 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 # minimize the attack surface # RUN rm -rf /usr/bin /usr/sbin /usr/lib/python* - + RUN apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* ; RUN mkdir -p /root/.local/share/centrifuge-chain && \ - ln -s /root/.local/share/centrifuge-chain /data - - + ln -s /root/.local/share/centrifuge-chain /data + + ENV RUST_BACKTRACE 1 USER centrifuge EXPOSE 30333 9933 9944 From 3b1d5b379236ee4460a31adc7136d0c66a2faea1 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 16:18:00 -0400 Subject: [PATCH 33/67] pin all github action versions --- .github/workflows/benchmark-auto-pr.yml | 50 +++++++++++++------------ .github/workflows/build-docker.yml | 12 +++--- .github/workflows/build-wasm.yml | 4 +- .github/workflows/docs.yml | 2 +- 4 files changed, 35 insertions(+), 33 deletions(-) diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index 4a867848c8..55e5ce1a19 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -23,7 +23,7 @@ jobs: RUSTC_WRAPPER: "sccache" steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu @@ -35,7 +35,7 @@ jobs: GSA: ${{ secrets.GSA_SCCACHE }} # - name: Checkout code - # uses: actions/checkout@v3 + # uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 # with: # ref: 'main' @@ -43,27 +43,29 @@ jobs: run: ./scripts/runtime_benchmarks.sh ${{ matrix.runtimes }} - run: echo "NOW=$(date -u +%Y-%m-%d)" >> $GITHUB_ENV - - - name: Create PR with new benchmarks - uses: peter-evans/create-pull-request@v5 + + - name: Upload artifact + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3 with: - add-paths: runtime/${{ matrix.runtimes }}/src/weights/ - base: main - branch: benchmarks/${{ matrix.runtimes }}-${{ env.NOW }} - delete-branch: true - reviewers: ${{ github.actor }} - draft: true - labels: gha, benchmarks - commit-message: "New benchmark weights generated from main by Github Actions Bot" - title: New benchmark weights for ${{ matrix.runtimes }} - body: | - Automated PR generated by Benchmark PR Github Actions job - Runtime: ${{ matrix.runtimes }} - Commit that triggered: - ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} - - # - name: Upload artifact - # uses: actions/upload-artifact@v3 + name: ${{ matrix.runtimes }}-weights + path: runtime/${{ matrix.runtimes }}/src/weights/ + +# Disabled a the org level +# fails with 'GitHub Actions is not permitted to create or approve pull requests.' + # - name: Create PR with new benchmarks + # uses: peter-evans/create-pull-request@v5 # with: - # name: ${{ matrix.runtimes }}-weights - # path: runtime/${{ matrix.runtimes }}/src/weights/ + # add-paths: runtime/${{ matrix.runtimes }}/src/weights/ + # base: main + # branch: benchmarks/${{ matrix.runtimes }}-${{ env.NOW }} + # delete-branch: true + # reviewers: ${{ github.actor }} + # draft: true + # labels: gha, benchmarks + # commit-message: "New benchmark weights generated from main by Github Actions Bot" + # title: New benchmark weights for ${{ matrix.runtimes }} + # body: | + # Automated PR generated by Benchmark PR Github Actions job + # Runtime: ${{ matrix.runtimes }} + # Commit that triggered: + # ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index d504ab5c8d..bd9631a971 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -21,7 +21,7 @@ jobs: env: WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 with: fetch-depth: 0 - name: Free space @@ -37,7 +37,7 @@ jobs: df -h - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 #v3 - name: DockerHub Login uses: docker/login-action@v3 with: @@ -48,7 +48,7 @@ jobs: - name: Setup docker metadata id: meta - uses: docker/metadata-action@v5 + uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 #v5 with: images: centrifugeio/centrifuge-chain flavor: | @@ -64,14 +64,14 @@ jobs: type=edge - name: Configure sccache - uses: actions/github-script@v6 + uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 #v6 with: script: | core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || ''); core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || ''); - name: Build and push centrifugeio/centrifuge-chain - uses: docker/build-push-action@v5 + uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 #v5 # env: # BUILDKIT_PROGRESS: plain # DOCKER_BUILDKIT: 1 @@ -94,7 +94,7 @@ jobs: - name: Update DockerHub descriptions if: contains(github.ref, 'refs/tags/release-v') - uses: peter-evans/dockerhub-description@v3 + uses: peter-evans/dockerhub-description@dc67fad7001ef9e8e3c124cb7a64e16d0a63d864 #v3.4.2 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index f039396e6c..3d001b93ee 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -64,7 +64,7 @@ jobs: echo "BUILD_OPTS=${{ env.BUILD_OPT }}" - name: Run Docker SRTool - uses: addnab/docker-run-action@v3 + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 env: RUSTC_VERSION: ${{ matrix.rust_version }} BUILD_OPTS: ${{ env.BUILD_OPTS }} @@ -79,7 +79,7 @@ jobs: ls -la /cargo-home/ du -sh /cargo-home/* echo "---- RUNNING BUILD ----" - /srtool/build --json + srtool build - id: 'auth' # if: ${{ github.ref == 'refs/heads/main' }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 70ed8f5e07..d83a8a980c 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -18,7 +18,7 @@ jobs: RUSTC_WRAPPER: "sccache" steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu From 06ec8e17d40339710e873764b83042014e98bc38 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 16:40:33 -0400 Subject: [PATCH 34/67] fix srtool build --- .github/workflows/build-wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 3d001b93ee..3d24f991c0 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -79,7 +79,7 @@ jobs: ls -la /cargo-home/ du -sh /cargo-home/* echo "---- RUNNING BUILD ----" - srtool build + srtool/build --save - id: 'auth' # if: ${{ github.ref == 'refs/heads/main' }} From 0271fdbb8b620f2d88850d3f7e8775b3a5eaca4b Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 16:53:25 -0400 Subject: [PATCH 35/67] Upload srtool report from wasm build --- .github/workflows/build-wasm.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 3d24f991c0..26e2b6e269 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -79,7 +79,15 @@ jobs: ls -la /cargo-home/ du -sh /cargo-home/* echo "---- RUNNING BUILD ----" - srtool/build --save + /srtool/build --save + + - id: set_runtime + run: echo "RUNTIME=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//")" >> $GITHUB_ENV + - name: upload srtool report + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3 + with: + name: srtool-wasm-report.txt + path: ./runtime/${{ env.RUNTIME }}/target/srtool/srtool-wasm-report-*.txt - id: 'auth' # if: ${{ github.ref == 'refs/heads/main' }} From ec97693cd4267a64adda934eebfd99cdf218ae10 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 17:33:14 -0400 Subject: [PATCH 36/67] add code coverage to wasm build --- .github/workflows/build-wasm.yml | 8 +++++--- .github/workflows/sanity-checks.yml | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 26e2b6e269..3cd4d4bc31 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -34,6 +34,7 @@ jobs: # SCCACHE_GHA_ENABLED: "true" # RUSTC_WRAPPER: "sccache" steps: + # PREP - name: Check out code uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 @@ -52,7 +53,7 @@ jobs: ./target/srtool cache-directories: ./runtime # save-if: ${{ github.ref == 'refs/heads/master' }} - + # BUILD - name: Setup build options id: buildopt run: | @@ -68,6 +69,7 @@ jobs: env: RUSTC_VERSION: ${{ matrix.rust_version }} BUILD_OPTS: ${{ env.BUILD_OPTS }} + TARPAULIN_VERSION: 0.19.1 # travis-ci codecov generator with: image: paritytech/srtool:${{ matrix.rust_version }} options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e PACKAGE=${{ matrix.package }} @@ -80,10 +82,10 @@ jobs: du -sh /cargo-home/* echo "---- RUNNING BUILD ----" /srtool/build --save - + - id: set_runtime run: echo "RUNTIME=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//")" >> $GITHUB_ENV - - name: upload srtool report + - name: Upload srtool report uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3 with: name: srtool-wasm-report.txt diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index d63e6c4bbc..f0295556a1 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -38,6 +38,27 @@ jobs: env: TARGET: ${{ matrix.target }} RUSTC_WRAPPER: "sccache" + + - name: generate codecov reports + run: | + if ${{ contains(matrix.target, 'test' ) }}; then + echo "---- GENERATE CODE COVERAGE ----" + echo "Generate code coverage for ${{ matrix.package }}" + cargo binstall cargo-tarpaulin + # make Cargo.toml + cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml + fi + + # UPLOAD REPORTS + - name: Upload codecov report + uses: codecov/codecov-action@v3 + with: + # token: ${{ secrets.CODECOV_TOKEN }} + # files: ./coverage1.xml,./coverage2.xml # optional + # flags: unittests # optional + # name: codecov-umbrella # optional + # fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false) benchmark-check: name: bench-check-${{ matrix.runtime }} From 8ed4eda79c9ca08f71f66298be824979f4de7a75 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 19:00:33 -0400 Subject: [PATCH 37/67] fix typo --- .github/workflows/sanity-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index f0295556a1..b521b6aff1 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -44,7 +44,7 @@ jobs: if ${{ contains(matrix.target, 'test' ) }}; then echo "---- GENERATE CODE COVERAGE ----" echo "Generate code coverage for ${{ matrix.package }}" - cargo binstall cargo-tarpaulin + cargo install cargo-tarpaulin # make Cargo.toml cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml fi From b0b406c1fd39e48b687ee7899caa825c33fc9911 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Sat, 16 Sep 2023 19:08:18 -0400 Subject: [PATCH 38/67] add nix build (fully commented) --- .github/workflows/xperimental-nix.yml | 57 +++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 .github/workflows/xperimental-nix.yml diff --git a/.github/workflows/xperimental-nix.yml b/.github/workflows/xperimental-nix.yml new file mode 100644 index 0000000000..10e805d22d --- /dev/null +++ b/.github/workflows/xperimental-nix.yml @@ -0,0 +1,57 @@ +#### NOTE: This job is disabled until nix supports the latest cargo workspace features. +#### See: https://github.com/NixOS/nixpkgs/pull/217232 +# +#name: nix-build +#on: +# pull_request: +# push: +# branches: [main, 'release-v**'] +#jobs: +# build: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# with: +# # Nix Flakes doesn't work on shallow clones +# fetch-depth: 0 +# - uses: cachix/install-nix-action@v17 +# with: +# install_url: https://releases.nixos.org/nix/nix-2.10.3/install +# - uses: cachix/cachix-action@v10 +# with: +# name: centrifuge-chain +# authToken: "${{ secrets.CACHIX_AUTH_TOKEN }}" +# - name: Build centrifuge-chain +# run: nix build --no-allow-dirty -L + +# name: docker +# on: +# push: +# branches: [main, 'release-v**'] +# jobs: +# docker: +# strategy: +# matrix: +# os: [ ubuntu-latest ] +# target: [".#dockerImage", ".#dockerImageFastRuntime"] +# runs-on: ${{ matrix.os }} +# steps: +# - uses: actions/checkout@v3 +# with: +# fetch-depth: 0 +# - uses: cachix/install-nix-action@v17 +# with: +# install_url: https://releases.nixos.org/nix/nix-2.10.3/install +# - name: Build Docker image +# run: | +# nix build --no-allow-dirty -L ${{ matrix.target }} +# docker load < result +# - name: Login to Docker Hub +# uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 +# with: +# username: ${{ secrets.DOCKER_HUB_USERNAME }} +# password: ${{ secrets.DOCKER_HUB_TOKEN }} +# - name: List images +# run: docker images +# - name: Push image to Docker Hub +# run: docker push --all-tags --quiet centrifugeio/centrifuge-chain From 64d1822c73e048be4975518962ef81835c212e26 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Mon, 18 Sep 2023 05:15:02 -0400 Subject: [PATCH 39/67] Update build-wasm.yml --- .github/workflows/build-wasm.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 3cd4d4bc31..f39f2ce328 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -77,9 +77,12 @@ jobs: echo "---- Env VARS ----" echo "BUILD_OPT=${BUILD_OPT}" echo "PACKAGE=${PACKAGE}" - echo "---- List Cache Folder ----" + echo "---- List Folders ----" + echo "-List cargo folders-" ls -la /cargo-home/ du -sh /cargo-home/* + echo "-List build folder-" + ls /build/ -la echo "---- RUNNING BUILD ----" /srtool/build --save From 78689140e31f6d3394a7c174350adbc2bd67a0d1 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Mon, 18 Sep 2023 05:35:35 -0400 Subject: [PATCH 40/67] change build wasm docker run --- .github/workflows/build-wasm.yml | 47 ++++++++++--------- ...-nix.yml => xperimental-nix.yml.commented} | 0 .gitignore | 2 + 3 files changed, 28 insertions(+), 21 deletions(-) rename .github/workflows/{xperimental-nix.yml => xperimental-nix.yml.commented} (100%) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index f39f2ce328..6348cc88a5 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -63,29 +63,34 @@ jobs: echo "BUILD_OPTS="--features=fast-runtime"" >> GITHUB_ENV fi echo "BUILD_OPTS=${{ env.BUILD_OPT }}" - - - name: Run Docker SRTool - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 - env: - RUSTC_VERSION: ${{ matrix.rust_version }} - BUILD_OPTS: ${{ env.BUILD_OPTS }} - TARPAULIN_VERSION: 0.19.1 # travis-ci codecov generator - with: - image: paritytech/srtool:${{ matrix.rust_version }} - options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e PACKAGE=${{ matrix.package }} - run: | - echo "---- Env VARS ----" - echo "BUILD_OPT=${BUILD_OPT}" - echo "PACKAGE=${PACKAGE}" - echo "---- List Folders ----" - echo "-List cargo folders-" - ls -la /cargo-home/ - du -sh /cargo-home/* - echo "-List build folder-" - ls /build/ -la - echo "---- RUNNING BUILD ----" + + - name: Run srtool + run: | + docker run -it --rm \ + -e PACKAGE=centrifuge-runtime \ + -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} + --platform=linux/amd64 \ + -v $PWD:/build paritytech/srtool:1.66.0 \ /srtool/build --save + # - name: Run Docker SRTool + # uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 + # env: + # RUSTC_VERSION: ${{ matrix.rust_version }} + # TARPAULIN_VERSION: 0.19.1 # travis-ci codecov generator + # with: + # image: paritytech/srtool:${{ matrix.rust_version }} + # options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e BUILD_OPTS=${{ env.BUILD_OPTS || ''}} -e PACKAGE=${{ matrix.package }} + # run: | + # echo "---- Env VARS ----" + # echo "BUILD_OPT=${BUILD_OPT}" + # echo "PACKAGE=${PACKAGE}" + # echo "---- List Cache Folder ----" + # ls -la /cargo-home/ + # du -sh /cargo-home/* + # echo "---- RUNNING BUILD ----" + # /srtool/build --save + - id: set_runtime run: echo "RUNTIME=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//")" >> $GITHUB_ENV - name: Upload srtool report diff --git a/.github/workflows/xperimental-nix.yml b/.github/workflows/xperimental-nix.yml.commented similarity index 100% rename from .github/workflows/xperimental-nix.yml rename to .github/workflows/xperimental-nix.yml.commented diff --git a/.gitignore b/.gitignore index 45adb64f1e..295aecc95f 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ Temporary Items # Node Modules **/node_modules +# Vistual studio +.vscode \ No newline at end of file From fae328833a87d705f535a2ecebe04c7073cee9d8 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Mon, 18 Sep 2023 05:35:49 -0400 Subject: [PATCH 41/67] Move the code coverage up to see erros --- .github/workflows/sanity-checks.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index b521b6aff1..d4cc05536c 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -32,22 +32,21 @@ jobs: # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} - - - name: Runing cargo ${{ matrix.target }} - run: ./ci/run-check.sh - env: - TARGET: ${{ matrix.target }} - RUSTC_WRAPPER: "sccache" - - name: generate codecov reports run: | if ${{ contains(matrix.target, 'test' ) }}; then echo "---- GENERATE CODE COVERAGE ----" - echo "Generate code coverage for ${{ matrix.package }}" + echo "# Install Tarpaulin" cargo install cargo-tarpaulin # make Cargo.toml + echo "Generate code coverage for ${{ matrix.target }}" cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml - fi + fi + - name: Runing cargo ${{ matrix.target }} + run: ./ci/run-check.sh + env: + TARGET: ${{ matrix.target }} + RUSTC_WRAPPER: "sccache" # UPLOAD REPORTS - name: Upload codecov report From c47131e08af4ee344957e312c3c7afb0f8d1a3c9 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Mon, 18 Sep 2023 06:17:27 -0400 Subject: [PATCH 42/67] check if wasm builds --- .github/workflows/build-wasm.yml | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 6348cc88a5..8c01b66660 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -54,7 +54,7 @@ jobs: cache-directories: ./runtime # save-if: ${{ github.ref == 'refs/heads/master' }} # BUILD - - name: Setup build options + - id: buildopt run: | if ${{ matrix.target == 'build-runtime-testnet'}} ; then @@ -63,15 +63,9 @@ jobs: echo "BUILD_OPTS="--features=fast-runtime"" >> GITHUB_ENV fi echo "BUILD_OPTS=${{ env.BUILD_OPT }}" - + - name: Run srtool - run: | - docker run -it --rm \ - -e PACKAGE=centrifuge-runtime \ - -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} - --platform=linux/amd64 \ - -v $PWD:/build paritytech/srtool:1.66.0 \ - /srtool/build --save + run: docker run -it --rm -e PACKAGE=centrifuge-runtime -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} --platform=linux/amd64 -v ${{ github.workspace }}:/build paritytech/srtool:1.66.0 /srtool/build # - name: Run Docker SRTool # uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 @@ -91,13 +85,11 @@ jobs: # echo "---- RUNNING BUILD ----" # /srtool/build --save - - id: set_runtime - run: echo "RUNTIME=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//")" >> $GITHUB_ENV - name: Upload srtool report uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 #v3.1.3 with: name: srtool-wasm-report.txt - path: ./runtime/${{ env.RUNTIME }}/target/srtool/srtool-wasm-report-*.txt + path: ./${{ matrix.package }}-srtool-wasm-report.json - id: 'auth' # if: ${{ github.ref == 'refs/heads/main' }} From 30a8468fc9f733f1ddcb4dc151bbcef53002074d Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Mon, 18 Sep 2023 06:32:33 -0400 Subject: [PATCH 43/67] try and force the report generation --- .github/workflows/build-wasm.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 8c01b66660..2cde31189d 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -65,7 +65,11 @@ jobs: echo "BUILD_OPTS=${{ env.BUILD_OPT }}" - name: Run srtool - run: docker run -it --rm -e PACKAGE=centrifuge-runtime -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} --platform=linux/amd64 -v ${{ github.workspace }}:/build paritytech/srtool:1.66.0 /srtool/build + run: > + docker run --rm + -e PACKAGE=centrifuge-runtime -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} + --platform=linux/amd64 -v ${{ github.workspace }}:/build paritytech/srtool:1.66.0 + /srtool/build # - name: Run Docker SRTool # uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 From 31660ddcb2942cf024650f5e06a411b89fca0b5c Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Tue, 7 Nov 2023 17:30:21 +0100 Subject: [PATCH 44/67] review the PR with @wischli --- .github/workflows/benchmark-auto-pr.yml | 5 ++++- .github/workflows/build-docker.yml | 4 ++-- .github/workflows/build-wasm.yml | 15 ++++++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index 55e5ce1a19..0ea8c53a9a 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -1,7 +1,10 @@ name: Benchmark PR +# ToDo: add an arbitrary label to keep this cache separate from the builds on: + workflow_call: + # Manually triggered by an authorized repo contributor. push: - branches: [main] + branches: main # To build the "benchmark-only" cache pull_request: paths: - '.github/workflows/benchmark-auto-pr.yml' diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index bd9631a971..19ec6994b5 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -59,8 +59,8 @@ jobs: type=ref,event=branch type=ref,event=tag type=ref,event=pr - type=semver,pattern={{version}} - type=semver,pattern={{major}} + type=semver,pattern={{raw}} + # type=semver,pattern={{major}} type=edge - name: Configure sccache diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 2cde31189d..878e0b90fc 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -1,6 +1,8 @@ on: push: - branches: [main, 'release-v**'] + tags: + - '**' + # branches: [main, 'release-v**'] pull_request: name: Build&Publish WASM concurrency: @@ -22,12 +24,9 @@ jobs: # To test until I get one right # target: [build-runtime, build-runtime-fast, build-runtime-testnet] # package: [centrifuge-runtime, altair-runtime] - target: [build-runtime, build-runtime-testnet, build-runtime-fast] + target: [build-runtime, build-runtime-fast] package: [altair-runtime, centrifuge-runtime] rust_version: ["1.66.0"] - exclude: - - target: "build-runtime-testnet" - package: "centrifuge-runtime" name: ${{ matrix.target }}-${{ matrix.package }} # Cannot make sccache work with the srtool container # env: @@ -57,9 +56,7 @@ jobs: - id: buildopt run: | - if ${{ matrix.target == 'build-runtime-testnet'}} ; then - echo "BUILD_OPTS="--features=testnet-runtime"" >> GITHUB_ENV - elif ${{ matrix.target == 'build-runtime-fast'}} ; then + if ${{ matrix.target == 'build-runtime-fast'}} ; then echo "BUILD_OPTS="--features=fast-runtime"" >> GITHUB_ENV fi echo "BUILD_OPTS=${{ env.BUILD_OPT }}" @@ -115,4 +112,4 @@ jobs: filename=$(echo "${{ matrix.package }}" | sed -e 's/-/_/g' ) gsutil cp \ ./runtime/${runtime_name}/target/srtool/release/wbuild/${{ matrix.package }}/${filename}.compact.compressed.wasm \ - gs://centrifuge-wasm-repo/${runtime_name}/${target}/${runtime_name}-${target}-$(git rev-parse --short HEAD).wasm + gs://centrifuge-wasm-repo/${runtime_name}/${runtime_name}-${target}-$(git rev-parse --short HEAD).wasm From eed117c80bbababdb1c924bb6658b2468e054f38 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 13:46:33 +0100 Subject: [PATCH 45/67] Try cargo install --locked --- .github/workflows/sanity-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index d4cc05536c..16df9c6ba0 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -37,7 +37,7 @@ jobs: if ${{ contains(matrix.target, 'test' ) }}; then echo "---- GENERATE CODE COVERAGE ----" echo "# Install Tarpaulin" - cargo install cargo-tarpaulin + cargo install --locked cargo-tarpaulin # make Cargo.toml echo "Generate code coverage for ${{ matrix.target }}" cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml From d03c379a592cf20d227c2af51e34fc0a851e7cf8 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 13:47:51 +0100 Subject: [PATCH 46/67] Use toolchain from toml file --- .github/actions/prep-ubuntu/action.yml | 14 +++++++++----- .github/workflows/benchmark-auto-pr.yml | 4 ++-- .github/workflows/build-wasm.yml | 7 ++++--- docker/centrifuge-chain/Dockerfile | 4 +--- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/actions/prep-ubuntu/action.yml b/.github/actions/prep-ubuntu/action.yml index c37827a9aa..145a98a384 100644 --- a/.github/actions/prep-ubuntu/action.yml +++ b/.github/actions/prep-ubuntu/action.yml @@ -28,11 +28,15 @@ runs: df -h sudo apt-get install protobuf-compiler - - name: Install latest nightly - uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - with: - toolchain: ${{ inputs.RUST_TOOLCHAIN }} - default: true + # - name: Install latest nightly + # uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 + # with: + # toolchain: ${{ inputs.RUST_TOOLCHAIN }} + # default: true + - name: Install toolchain from rust-toolchain.toml + run: | + TOOLCHAIN_VERSION=$(grep 'channel =' rust-toolchain.toml | awk -F'"' '{print $2}') + rustup toolchain install "$TOOLCHAIN_VERSION" - name: SCcache setup if: ${{ inputs.cache == 'enabled' }} diff --git a/.github/workflows/benchmark-auto-pr.yml b/.github/workflows/benchmark-auto-pr.yml index 0ea8c53a9a..ba97a0f0ef 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ b/.github/workflows/benchmark-auto-pr.yml @@ -22,7 +22,7 @@ jobs: matrix: runtimes: [centrifuge, altair] env: - RUST_TOOLCHAIN: "1.66.0" + # RUST_TOOLCHAIN: "1.66.0" RUSTC_WRAPPER: "sccache" steps: - name: Checkout code @@ -31,7 +31,7 @@ jobs: - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu with: - RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} + # RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} cache: enabled # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 878e0b90fc..e8501b8d25 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -26,7 +26,7 @@ jobs: # package: [centrifuge-runtime, altair-runtime] target: [build-runtime, build-runtime-fast] package: [altair-runtime, centrifuge-runtime] - rust_version: ["1.66.0"] + # rust_version: ["1.66.0"] name: ${{ matrix.target }}-${{ matrix.package }} # Cannot make sccache work with the srtool container # env: @@ -39,8 +39,9 @@ jobs: - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu - with: - RUST_TOOLCHAIN: ${{ matrix.rust_version }} + # Use default toolchain + # with: + # RUST_TOOLCHAIN: ${{ matrix.rust_version }} # sccache does not work inside the srtool container - uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # 2.6.2 diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index 09aeea6e04..91f84f6567 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -31,12 +31,10 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder rm rustup-init; # BUILD - ARG RUST_TOOLCHAIN="1.66" ARG RUSTC_WRAPPER='' ARG SCCACHE_GHA_ENABLED="false" - RUN rustup default $RUST_TOOLCHAIN && \ - rustup target add wasm32-unknown-unknown --toolchain $RUST_TOOLCHAIN + RUN rustup target add wasm32-unknown-unknown COPY . centrifuge-chain WORKDIR /centrifuge-chain From d51ee77fb46293c3cf619bfa76a8466ef8725f78 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 13:52:16 +0100 Subject: [PATCH 47/67] small fixes --- .github/actions/prep-ubuntu/action.yml | 1 + .github/workflows/sanity-checks.yml | 4 ++-- ci/run-check.sh | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/actions/prep-ubuntu/action.yml b/.github/actions/prep-ubuntu/action.yml index 145a98a384..3048ee8ab1 100644 --- a/.github/actions/prep-ubuntu/action.yml +++ b/.github/actions/prep-ubuntu/action.yml @@ -34,6 +34,7 @@ runs: # toolchain: ${{ inputs.RUST_TOOLCHAIN }} # default: true - name: Install toolchain from rust-toolchain.toml + shell: sh run: | TOOLCHAIN_VERSION=$(grep 'channel =' rust-toolchain.toml | awk -F'"' '{print $2}') rustup toolchain install "$TOOLCHAIN_VERSION" diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index 16df9c6ba0..d47d6b6fa1 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -27,7 +27,7 @@ jobs: - name: Prep build uses: ./.github/actions/prep-ubuntu with: - RUST_TOOLCHAIN: "nightly-2022-11-14" + # RUST_TOOLCHAIN: "nightly-2023-02-07" cache: enabled # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} @@ -72,7 +72,7 @@ jobs: - name: Prep build uses: ./.github/actions/prep-ubuntu with: - RUST_TOOLCHAIN: "nightly-2022-11-14" + # RUST_TOOLCHAIN: "nightly-2022-11-14" setup_cache: true cache: enabled GWIP: ${{ secrets.GWIP_SCCACHE }} diff --git a/ci/run-check.sh b/ci/run-check.sh index 5434dbb3a0..f3fe6b094b 100755 --- a/ci/run-check.sh +++ b/ci/run-check.sh @@ -14,11 +14,11 @@ case $TARGET in ;; test-general) - RUST_MIN_STACK=8388608 cargo test --workspace --release --features runtime-benchmarks,try-runtime --exclude runtime-integration-tests + cargo test --workspace --release --features runtime-benchmarks,try-runtime --exclude runtime-integration-tests ;; test-integration) - RUST_MIN_STACK=8388608 cargo test --release --package runtime-integration-tests --features fast-runtime + cargo test --release --package runtime-integration-tests --features fast-runtime ;; lint-fmt) From 31d1884fd570d923415be31aa4a4543144f25a1a Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 13:58:14 +0100 Subject: [PATCH 48/67] comment code coverage reports --- .github/workflows/sanity-checks.yml | 41 +++++++++++++++-------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index d47d6b6fa1..8881caee48 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -32,32 +32,33 @@ jobs: # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} - - name: generate codecov reports - run: | - if ${{ contains(matrix.target, 'test' ) }}; then - echo "---- GENERATE CODE COVERAGE ----" - echo "# Install Tarpaulin" - cargo install --locked cargo-tarpaulin - # make Cargo.toml - echo "Generate code coverage for ${{ matrix.target }}" - cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml - fi + # Tarpaulin requires cargo 1.70.0: + # - name: generate codecov reports + # run: | + # if ${{ contains(matrix.target, 'test' ) }}; then + # echo "---- GENERATE CODE COVERAGE ----" + # echo "# Install Tarpaulin" + # cargo install --locked cargo-tarpaulin + # # make Cargo.toml + # echo "Generate code coverage for ${{ matrix.target }}" + # cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml + # fi - name: Runing cargo ${{ matrix.target }} run: ./ci/run-check.sh env: TARGET: ${{ matrix.target }} RUSTC_WRAPPER: "sccache" - # UPLOAD REPORTS - - name: Upload codecov report - uses: codecov/codecov-action@v3 - with: - # token: ${{ secrets.CODECOV_TOKEN }} - # files: ./coverage1.xml,./coverage2.xml # optional - # flags: unittests # optional - # name: codecov-umbrella # optional - # fail_ci_if_error: true # optional (default = false) - verbose: true # optional (default = false) + # UPLOAD REPORTS (requires cargo 1.70.0) + # - name: Upload codecov report + # uses: codecov/codecov-action@v3 + # with: + # # token: ${{ secrets.CODECOV_TOKEN }} + # # files: ./coverage1.xml,./coverage2.xml # optional + # # flags: unittests # optional + # # name: codecov-umbrella # optional + # # fail_ci_if_error: true # optional (default = false) + # verbose: true # optional (default = false) benchmark-check: name: bench-check-${{ matrix.runtime }} From 7d486b7b9c8cf22b24ded9c1039573857a69807a Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 14:01:26 +0100 Subject: [PATCH 49/67] trigger wasm build only on tag push --- .github/workflows/build-wasm.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index e8501b8d25..0144f84bf0 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -1,9 +1,7 @@ on: push: tags: - - '**' - # branches: [main, 'release-v**'] - pull_request: + - '*' name: Build&Publish WASM concurrency: group: '${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' From 44012ad762b15a66356095016ca8d5795ee6122d Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 14:19:30 +0100 Subject: [PATCH 50/67] Rename benchmarks job --- .../workflows/run-benchmarks\\.yml" | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .github/workflows/benchmark-auto-pr.yml => ".github/workflows/run-benchmarks\\.yml" (98%) diff --git a/.github/workflows/benchmark-auto-pr.yml "b/.github/workflows/run-benchmarks\\.yml" similarity index 98% rename from .github/workflows/benchmark-auto-pr.yml rename to ".github/workflows/run-benchmarks\\.yml" index ba97a0f0ef..35499dbeba 100644 --- a/.github/workflows/benchmark-auto-pr.yml +++ "b/.github/workflows/run-benchmarks\\.yml" @@ -1,7 +1,7 @@ -name: Benchmark PR +name: Run benchmarks # ToDo: add an arbitrary label to keep this cache separate from the builds on: - workflow_call: + workflow_dispatch: # Manually triggered by an authorized repo contributor. push: branches: main # To build the "benchmark-only" cache From 2ce5707194bb423668a52f4c63aeebeb599891d6 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 14:19:53 +0100 Subject: [PATCH 51/67] try and fix the docker tag logic --- .github/workflows/build-docker.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 19ec6994b5..727a8ddd40 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -53,8 +53,7 @@ jobs: images: centrifugeio/centrifuge-chain flavor: | suffix=-${{ env.NOW }} - # latest=auto is already default, but the logic needs a value at the end and '' isn't valid - ${{ matrix.target == 'test' && 'prefix=test-' || 'latest=auto' }} + prefix=${{ matrix.target == 'test' && 'test-' || '' }} tags: | type=ref,event=branch type=ref,event=tag From 780adbfb94dd1b39c493f77efa62dff1011c4c63 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 16:03:26 +0100 Subject: [PATCH 52/67] small fixes --- .github/workflows/build-docker.yml | 1 - .../workflows/run-benchmarks.yml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) rename ".github/workflows/run-benchmarks\\.yml" => .github/workflows/run-benchmarks.yml (98%) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 727a8ddd40..904149d66b 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -57,7 +57,6 @@ jobs: tags: | type=ref,event=branch type=ref,event=tag - type=ref,event=pr type=semver,pattern={{raw}} # type=semver,pattern={{major}} type=edge diff --git "a/.github/workflows/run-benchmarks\\.yml" b/.github/workflows/run-benchmarks.yml similarity index 98% rename from ".github/workflows/run-benchmarks\\.yml" rename to .github/workflows/run-benchmarks.yml index 35499dbeba..1d1e437a5c 100644 --- "a/.github/workflows/run-benchmarks\\.yml" +++ b/.github/workflows/run-benchmarks.yml @@ -7,7 +7,7 @@ on: branches: main # To build the "benchmark-only" cache pull_request: paths: - - '.github/workflows/benchmark-auto-pr.yml' + - '.github/workflows/run-benchmarks.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }} cancel-in-progress: true From f0f25af7ddef50ff7441fe004d892a3948ff8edb Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 17:28:14 +0100 Subject: [PATCH 53/67] more minor fixes --- .github/actions/sccache-gcloud/action.yml | 6 ++++-- .github/workflows/build-docker.yml | 6 +++--- .github/workflows/build-wasm.yml | 11 +++++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/actions/sccache-gcloud/action.yml b/.github/actions/sccache-gcloud/action.yml index 9afcd3adca..72cf06ec22 100644 --- a/.github/actions/sccache-gcloud/action.yml +++ b/.github/actions/sccache-gcloud/action.yml @@ -23,7 +23,7 @@ runs: - name: Run sccache-cache uses: mozilla-actions/sccache-action@8417cffc2ec64127ad83077aceaa8631f7cdc83e #v0.0.3 - - name: set GH variables + - name: set GCS bucket sccache variables shell: bash run: | echo "SCCACHE_GCS_BUCKET=centrifuge-chain-sccache-backend" >> $GITHUB_ENV @@ -34,7 +34,9 @@ runs: - name: setup Rust sccache wrapper if: ${{ inputs.cache == 'enabled' }} shell: sh - run: echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV + run: | + echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV + echo "CARGO_INCREMENTAL=false" >> $GITHUB_ENV # https://github.com/mozilla/sccache#rust - name: Run sccache stat for check shell: bash diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 904149d66b..acb05c0c8f 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -5,7 +5,7 @@ on: pull_request: paths: - ".github/workflows/build-docker.yml" - - "Dockerfile" + - "docker/centrifuge-chain/Dockerfile" workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}${{ github.ref == '/refs/heads/main' && github.sha || 'false' }} @@ -53,12 +53,12 @@ jobs: images: centrifugeio/centrifuge-chain flavor: | suffix=-${{ env.NOW }} - prefix=${{ matrix.target == 'test' && 'test-' || '' }} + prefix=${{ matrix.target == 'test' && 'test-' || 'latest=auto' }} tags: | type=ref,event=branch type=ref,event=tag type=semver,pattern={{raw}} - # type=semver,pattern={{major}} + type=semver,pattern={{major}},prefix=${{ matrix.target == 'test' && 'test-' || '' }}pr- type=edge - name: Configure sccache diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 0144f84bf0..d55adfcaf9 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -2,6 +2,9 @@ on: push: tags: - '*' + pull_request: + paths: + - ".github/workflows/build-wasm.yml" name: Build&Publish WASM concurrency: group: '${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' @@ -24,7 +27,7 @@ jobs: # package: [centrifuge-runtime, altair-runtime] target: [build-runtime, build-runtime-fast] package: [altair-runtime, centrifuge-runtime] - # rust_version: ["1.66.0"] + # rust_version: ["1.69.0"] name: ${{ matrix.target }}-${{ matrix.package }} # Cannot make sccache work with the srtool container # env: @@ -62,9 +65,9 @@ jobs: - name: Run srtool run: > - docker run --rm - -e PACKAGE=centrifuge-runtime -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} - --platform=linux/amd64 -v ${{ github.workspace }}:/build paritytech/srtool:1.66.0 + docker run --rm --user root --platform=linux/amd64 + -e PACKAGE=${{ matrix.package }} -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} + -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build paritytech/srtool:latest /srtool/build # - name: Run Docker SRTool From 0d414a8f11b319c0d130b0feb6bb2ba61a2dbd0c Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 17:31:10 +0100 Subject: [PATCH 54/67] fix cache warning --- .github/workflows/sanity-checks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index 8881caee48..c731405830 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -74,7 +74,6 @@ jobs: uses: ./.github/actions/prep-ubuntu with: # RUST_TOOLCHAIN: "nightly-2022-11-14" - setup_cache: true cache: enabled GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} From 2ce04c8c1831c09d60f0186bdca44e7a83b110a1 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 17:37:02 +0100 Subject: [PATCH 55/67] srtool has no latest tag on the repo, using 1.69 --- .github/workflows/build-wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index d55adfcaf9..e929fcfb9c 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -67,7 +67,7 @@ jobs: run: > docker run --rm --user root --platform=linux/amd64 -e PACKAGE=${{ matrix.package }} -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} - -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build paritytech/srtool:latest + -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build paritytech/srtool:1.69.0 /srtool/build # - name: Run Docker SRTool From 3eda7256b8225de01c116d8ebd103a965b17177f Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Wed, 8 Nov 2023 17:41:05 +0100 Subject: [PATCH 56/67] remove production environment restrictions --- .github/workflows/build-wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index e929fcfb9c..0058994a20 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -15,7 +15,7 @@ permissions: jobs: build-runtime-wasms: # environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} - environment: production + # environment: production # Benchmark (with cache hit) # ubuntu-latest 50 min # ubuntu-latest-4-cores 34 min From aeaaeddb0f1742f6886c10f2c429b4d33cf1c931 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 9 Nov 2023 10:17:00 +0100 Subject: [PATCH 57/67] Fix GITHUB_ENV and bump srtool version --- .github/actions/sccache-gcloud/action.yml | 2 +- .github/workflows/build-wasm.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/sccache-gcloud/action.yml b/.github/actions/sccache-gcloud/action.yml index 72cf06ec22..c0b02aad93 100644 --- a/.github/actions/sccache-gcloud/action.yml +++ b/.github/actions/sccache-gcloud/action.yml @@ -35,7 +35,7 @@ runs: if: ${{ inputs.cache == 'enabled' }} shell: sh run: | - echo "RUSTC_WRAPPER="sccache"" >> $GITHUB_ENV + echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV echo "CARGO_INCREMENTAL=false" >> $GITHUB_ENV # https://github.com/mozilla/sccache#rust - name: Run sccache stat for check diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 0058994a20..72ee9756a6 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -59,15 +59,15 @@ jobs: id: buildopt run: | if ${{ matrix.target == 'build-runtime-fast'}} ; then - echo "BUILD_OPTS="--features=fast-runtime"" >> GITHUB_ENV + echo "BUILD_OPTS=--features=fast-runtime" >> GITHUB_ENV fi - echo "BUILD_OPTS=${{ env.BUILD_OPT }}" + echo "BUILD_OPTS=${{ env.BUILD_OPTS }}" - name: Run srtool run: > docker run --rm --user root --platform=linux/amd64 - -e PACKAGE=${{ matrix.package }} -e BUILD_OPTS=${{ env.BUILD_OPTS || '' }} - -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build paritytech/srtool:1.69.0 + -e PACKAGE=${{ matrix.package }} -v /home/runner/.cargo:/cargo-home + -v ${{ github.workspace }}:/build paritytech/srtool:1.70.0 /srtool/build # - name: Run Docker SRTool From 8879d86032f72557968d12e9c69560469c815099 Mon Sep 17 00:00:00 2001 From: William Freudenberger Date: Mon, 13 Nov 2023 11:38:57 +0100 Subject: [PATCH 58/67] chore: update srtool version to latest --- .github/workflows/build-wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 72ee9756a6..07a7ce9c39 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -67,7 +67,7 @@ jobs: run: > docker run --rm --user root --platform=linux/amd64 -e PACKAGE=${{ matrix.package }} -v /home/runner/.cargo:/cargo-home - -v ${{ github.workspace }}:/build paritytech/srtool:1.70.0 + -v ${{ github.workspace }}:/build paritytech/srtool:1.73.0-0.12.0 /srtool/build # - name: Run Docker SRTool From 61fcacb7619f8e7f4a6a3a2b9bdc84a457ab7db0 Mon Sep 17 00:00:00 2001 From: William Freudenberger Date: Mon, 13 Nov 2023 11:40:07 +0100 Subject: [PATCH 59/67] fix: remove unused rust toolchain from setup --- .github/actions/prep-ubuntu/action.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/actions/prep-ubuntu/action.yml b/.github/actions/prep-ubuntu/action.yml index 3048ee8ab1..d03f7566f7 100644 --- a/.github/actions/prep-ubuntu/action.yml +++ b/.github/actions/prep-ubuntu/action.yml @@ -1,9 +1,6 @@ name: Prepare Ubuntu for Rust builds description: cleanup and Rust Tools setup inputs: - RUST_TOOLCHAIN: - description: toolchain version - default: "1.66" cache: description: cache type (enabled if set) default: "disabled" From 9dfdcfbacc8a7e3b1769cd043d82db7862cb4250 Mon Sep 17 00:00:00 2001 From: William Freudenberger Date: Mon, 13 Nov 2023 11:44:21 +0100 Subject: [PATCH 60/67] fix: toolchain version docker build --- .github/workflows/build-docker.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index acb05c0c8f..7e00931317 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -10,8 +10,6 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}${{ github.ref == '/refs/heads/main' && github.sha || 'false' }} cancel-in-progress: true -env: - RUST_TOOLCHAIN: "1.66" jobs: docker: strategy: @@ -36,6 +34,12 @@ jobs: docker volume ls df -h + - name: Install toolchain from rust-toolchain.toml + shell: sh + run: | + TOOLCHAIN_VERSION=$(grep 'channel =' rust-toolchain.toml | awk -F'"' '{print $2}') + rustup toolchain install "$TOOLCHAIN_VERSION" + - name: Set up Docker Buildx uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 #v3 - name: DockerHub Login From 43e5d7aed284e3ae4fe09ece01c602217f137c1b Mon Sep 17 00:00:00 2001 From: William Freudenberger Date: Mon, 13 Nov 2023 16:23:18 +0100 Subject: [PATCH 61/67] fix: remove unused thread_local --- pallets/pool-system/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/pallets/pool-system/src/lib.rs b/pallets/pool-system/src/lib.rs index 36388430c0..e36312e1d9 100644 --- a/pallets/pool-system/src/lib.rs +++ b/pallets/pool-system/src/lib.rs @@ -12,7 +12,6 @@ // GNU General Public License for more details. #![cfg_attr(not(feature = "std"), no_std)] #![allow(clippy::or_fun_call)] -#![feature(thread_local)] use cfg_traits::{Permissions, PoolInspect, PoolMutate, PoolNAV, PoolReserve, Seconds, TimeAsSecs}; use cfg_types::{ From 5177624ff089cfe8016762298647394a01d881d8 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Tue, 14 Nov 2023 14:04:02 +0100 Subject: [PATCH 62/67] Update build-wasm.yml enable production google credentials --- .github/workflows/build-wasm.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 07a7ce9c39..d60f9028e2 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -15,7 +15,7 @@ permissions: jobs: build-runtime-wasms: # environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} - # environment: production + environment: production # Benchmark (with cache hit) # ubuntu-latest 50 min # ubuntu-latest-4-cores 34 min From 8398d7a10c091443995d65cec1cc13ba49147b59 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Thu, 16 Nov 2023 18:51:02 +0100 Subject: [PATCH 63/67] Final review of CI changes --- .github/workflows/build-docker.yml | 18 +++----- .github/workflows/build-wasm.yml | 26 +++-------- .github/workflows/run-benchmarks.yml | 37 +--------------- .github/workflows/sanity-checks.yml | 33 +++----------- .../xperimental-codecov.yml.commented | 44 +++++++++++++++++++ 5 files changed, 64 insertions(+), 94 deletions(-) create mode 100644 .github/workflows/xperimental-codecov.yml.commented diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 7e00931317..9bf12a38ae 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -1,12 +1,14 @@ name: Docker Build on: + # Keep in mind the Docker tagging on the "metadata" step if you add new triggers push: - branches: [main, 'release-v**'] + branches: [main] + release: + types: [published] pull_request: paths: - ".github/workflows/build-docker.yml" - "docker/centrifuge-chain/Dockerfile" - workflow_dispatch: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}${{ github.ref == '/refs/heads/main' && github.sha || 'false' }} cancel-in-progress: true @@ -16,8 +18,6 @@ jobs: matrix: target: [ release, test ] runs-on: ubuntu-latest-8-cores - env: - WORKFLOW_TAG: ${{ github.event.inputs.docker_tag }} steps: - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 with: @@ -60,9 +60,8 @@ jobs: prefix=${{ matrix.target == 'test' && 'test-' || 'latest=auto' }} tags: | type=ref,event=branch - type=ref,event=tag - type=semver,pattern={{raw}} - type=semver,pattern={{major}},prefix=${{ matrix.target == 'test' && 'test-' || '' }}pr- + type=semver,pattern={{raw}},suffix= + type=semver,pattern={{major}},prefix=${{ matrix.target == 'test' && 'test-' || '' }} type=edge - name: Configure sccache @@ -74,16 +73,11 @@ jobs: - name: Build and push centrifugeio/centrifuge-chain uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 #v5 - # env: - # BUILDKIT_PROGRESS: plain - # DOCKER_BUILDKIT: 1 with: context: . file: ./docker/centrifuge-chain/Dockerfile build-args: | FEATURES=${{ matrix.target == 'test' && 'fast-runtime' || '' }} - # SCCACHE_GHA_ENABLED="true" - # RUSTC_WRAPPER=sccache push: ${{ github.ref == '/refs/heads/main' && true || false }} tags: ${{ steps.meta.outputs.tags }} # Cache options: diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index d60f9028e2..655b074db6 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -14,25 +14,16 @@ permissions: contents: read jobs: build-runtime-wasms: - # environment: ${{ github.ref == 'refs/heads/main' && 'production' || '' }} - environment: production + environment: production # It needs the production GCP credentials to upload WASM # Benchmark (with cache hit) # ubuntu-latest 50 min # ubuntu-latest-4-cores 34 min runs-on: ubuntu-latest-4-cores strategy: matrix: - # To test until I get one right - # target: [build-runtime, build-runtime-fast, build-runtime-testnet] - # package: [centrifuge-runtime, altair-runtime] target: [build-runtime, build-runtime-fast] package: [altair-runtime, centrifuge-runtime] - # rust_version: ["1.69.0"] name: ${{ matrix.target }}-${{ matrix.package }} - # Cannot make sccache work with the srtool container - # env: - # SCCACHE_GHA_ENABLED: "true" - # RUSTC_WRAPPER: "sccache" steps: # PREP - name: Check out code @@ -40,9 +31,6 @@ jobs: - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu - # Use default toolchain - # with: - # RUST_TOOLCHAIN: ${{ matrix.rust_version }} # sccache does not work inside the srtool container - uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # 2.6.2 @@ -55,8 +43,7 @@ jobs: cache-directories: ./runtime # save-if: ${{ github.ref == 'refs/heads/master' }} # BUILD - - - id: buildopt + - id: buildopt run: | if ${{ matrix.target == 'build-runtime-fast'}} ; then echo "BUILD_OPTS=--features=fast-runtime" >> GITHUB_ENV @@ -68,13 +55,14 @@ jobs: docker run --rm --user root --platform=linux/amd64 -e PACKAGE=${{ matrix.package }} -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build paritytech/srtool:1.73.0-0.12.0 - /srtool/build + /srtool/build --app + # -a, --app + # Enable the "app" mode which is a mix of json output and outputting progress during the build. This flag is recommended for CI. the json output will be provided as a single line at the end in compact mode + # Alternative way of running SRTool that allows for "script-like" execution, + # extremely useful to debug: # - name: Run Docker SRTool # uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 - # env: - # RUSTC_VERSION: ${{ matrix.rust_version }} - # TARPAULIN_VERSION: 0.19.1 # travis-ci codecov generator # with: # image: paritytech/srtool:${{ matrix.rust_version }} # options: --user root -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build -e BUILD_OPTS=${{ env.BUILD_OPTS || ''}} -e PACKAGE=${{ matrix.package }} diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 1d1e437a5c..94b83fe61e 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -1,10 +1,7 @@ name: Run benchmarks # ToDo: add an arbitrary label to keep this cache separate from the builds on: - workflow_dispatch: - # Manually triggered by an authorized repo contributor. - push: - branches: main # To build the "benchmark-only" cache + workflow_dispatch: # Manually triggered by an authorized repo contributor. pull_request: paths: - '.github/workflows/run-benchmarks.yml' @@ -14,16 +11,12 @@ concurrency: jobs: benchmark: permissions: - pull-requests: write - contents: write + contents: read id-token: write runs-on: ubuntu-latest-8-cores strategy: matrix: runtimes: [centrifuge, altair] - env: - # RUST_TOOLCHAIN: "1.66.0" - RUSTC_WRAPPER: "sccache" steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 @@ -31,17 +24,11 @@ jobs: - name: Prep build on Ubuntu uses: ./.github/actions/prep-ubuntu with: - # RUST_TOOLCHAIN: ${{ env.RUST_TOOLCHAIN }} cache: enabled # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} - # - name: Checkout code - # uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 - # with: - # ref: 'main' - - name: Run Benchmark ${{ matrix.runtimes }} run: ./scripts/runtime_benchmarks.sh ${{ matrix.runtimes }} @@ -52,23 +39,3 @@ jobs: with: name: ${{ matrix.runtimes }}-weights path: runtime/${{ matrix.runtimes }}/src/weights/ - -# Disabled a the org level -# fails with 'GitHub Actions is not permitted to create or approve pull requests.' - # - name: Create PR with new benchmarks - # uses: peter-evans/create-pull-request@v5 - # with: - # add-paths: runtime/${{ matrix.runtimes }}/src/weights/ - # base: main - # branch: benchmarks/${{ matrix.runtimes }}-${{ env.NOW }} - # delete-branch: true - # reviewers: ${{ github.actor }} - # draft: true - # labels: gha, benchmarks - # commit-message: "New benchmark weights generated from main by Github Actions Bot" - # title: New benchmark weights for ${{ matrix.runtimes }} - # body: | - # Automated PR generated by Benchmark PR Github Actions job - # Runtime: ${{ matrix.runtimes }} - # Commit that triggered: - # ${{ github.server_url }}/${{ github.repository }}/commit/${{ github.sha }} \ No newline at end of file diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index c731405830..838ebf3458 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -2,9 +2,9 @@ on: # push: # branches: [main, 'release-v**'] pull_request: - paths: - - 'src/' - - '.github/workflows/sanity-checks.yml' + # paths: + # - 'src/' + # - '.github/workflows/sanity-checks.yml' name: Sanity checks concurrency: group: 'tests-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' @@ -27,38 +27,16 @@ jobs: - name: Prep build uses: ./.github/actions/prep-ubuntu with: - # RUST_TOOLCHAIN: "nightly-2023-02-07" cache: enabled # Cache needs Google credentials: GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} - # Tarpaulin requires cargo 1.70.0: - # - name: generate codecov reports - # run: | - # if ${{ contains(matrix.target, 'test' ) }}; then - # echo "---- GENERATE CODE COVERAGE ----" - # echo "# Install Tarpaulin" - # cargo install --locked cargo-tarpaulin - # # make Cargo.toml - # echo "Generate code coverage for ${{ matrix.target }}" - # cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml - # fi + - name: Runing cargo ${{ matrix.target }} run: ./ci/run-check.sh env: TARGET: ${{ matrix.target }} - RUSTC_WRAPPER: "sccache" - - # UPLOAD REPORTS (requires cargo 1.70.0) - # - name: Upload codecov report - # uses: codecov/codecov-action@v3 - # with: - # # token: ${{ secrets.CODECOV_TOKEN }} - # # files: ./coverage1.xml,./coverage2.xml # optional - # # flags: unittests # optional - # # name: codecov-umbrella # optional - # # fail_ci_if_error: true # optional (default = false) - # verbose: true # optional (default = false) + RUSTC_WRAPPER: "sccache" benchmark-check: name: bench-check-${{ matrix.runtime }} @@ -73,7 +51,6 @@ jobs: - name: Prep build uses: ./.github/actions/prep-ubuntu with: - # RUST_TOOLCHAIN: "nightly-2022-11-14" cache: enabled GWIP: ${{ secrets.GWIP_SCCACHE }} GSA: ${{ secrets.GSA_SCCACHE }} diff --git a/.github/workflows/xperimental-codecov.yml.commented b/.github/workflows/xperimental-codecov.yml.commented new file mode 100644 index 0000000000..6600aff7c0 --- /dev/null +++ b/.github/workflows/xperimental-codecov.yml.commented @@ -0,0 +1,44 @@ +## THIS IS A WORK IN PROGRESS +on: + # push: + # branches: [main, 'release-v**'] + pull_request: +name: Sanity checks +concurrency: + group: 'tests-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' + cancel-in-progress: true +permissions: + id-token: write + contents: read +jobs: + codecov: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + # Tarpaulin requires cargo 1.70.0: + - name: generate codecov reports + run: | + if ${{ contains(matrix.target, 'test' ) }}; then + echo "---- GENERATE CODE COVERAGE ----" + echo "# Install Tarpaulin" + cargo install --locked cargo-tarpaulin + # make Cargo.toml + echo "Generate code coverage for ${{ matrix.target }}" + cargo +nightly tarpaulin --verbose --no-fail-fast --workspace --timeout 300 --out Xml + fi + - name: Runing cargo ${{ matrix.target }} + run: ./ci/run-check.sh + env: + TARGET: ${{ matrix.target }} + RUSTC_WRAPPER: "sccache" + + # UPLOAD REPORTS (requires cargo 1.70.0) + - name: Upload codecov report + uses: codecov/codecov-action@v3 + with: + # token: ${{ secrets.CODECOV_TOKEN }} + # files: ./coverage1.xml,./coverage2.xml # optional + # flags: unittests # optional + # name: codecov-umbrella # optional + # fail_ci_if_error: true # optional (default = false) + verbose: true # optional (default = false) From 13e2085775be894a119de06b4215f82b156af15f Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 17 Nov 2023 11:23:26 +0100 Subject: [PATCH 64/67] review of sanity-checks --- .github/workflows/sanity-checks.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index 838ebf3458..01fb9bf5f1 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -1,10 +1,7 @@ on: # push: - # branches: [main, 'release-v**'] + # branches: [main] pull_request: - # paths: - # - 'src/' - # - '.github/workflows/sanity-checks.yml' name: Sanity checks concurrency: group: 'tests-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}' @@ -15,7 +12,7 @@ permissions: jobs: test-n-lint: name: ${{ matrix.target }} - runs-on: ubuntu-latest-4-cores + runs-on: ubuntu-latest-8-cores strategy: matrix: target: [test-general, test-integration, @@ -40,10 +37,10 @@ jobs: benchmark-check: name: bench-check-${{ matrix.runtime }} - runs-on: ubuntu-latest #r-4-cores + runs-on: ubuntu-latest #-4-cores strategy: matrix: - runtime: [development, altair, centrifuge] + runtime: [altair, centrifuge] steps: - name: Check out code uses: actions/checkout@8e5e7e5ab8b370d6c329ec480221332ada57f0ab #3.5.2 From 3e9b8651e91d815276f34306cad483368950496a Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 17 Nov 2023 12:22:10 +0100 Subject: [PATCH 65/67] Final review of CI PR with @wischli --- .dockerignore | 2 +- .github/actions/prep-ubuntu/action.yml | 11 ++--------- .github/actions/sccache-gcloud/action.yml | 5 ++--- .github/workflows/build-docker.yml | 8 ++++---- .github/workflows/build-wasm.yml | 24 +++++++---------------- .github/workflows/run-benchmarks.yml | 2 ++ .github/workflows/sanity-checks.yml | 2 -- docker/centrifuge-chain/Dockerfile | 16 +++++---------- scripts/install_toolchain.sh | 5 +++-- 9 files changed, 26 insertions(+), 49 deletions(-) diff --git a/.dockerignore b/.dockerignore index 50b1114787..6ec026e904 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,6 @@ **/target/ .github/ -scripts/ +!scripts/install_toolchain.sh docker-compos .gitignore docker diff --git a/.github/actions/prep-ubuntu/action.yml b/.github/actions/prep-ubuntu/action.yml index d03f7566f7..61420caac3 100644 --- a/.github/actions/prep-ubuntu/action.yml +++ b/.github/actions/prep-ubuntu/action.yml @@ -24,17 +24,10 @@ runs: echo "Post cleanup" df -h sudo apt-get install protobuf-compiler - - # - name: Install latest nightly - # uses: actions-rs/toolchain@88dc2356392166efad76775c878094f4e83ff746 - # with: - # toolchain: ${{ inputs.RUST_TOOLCHAIN }} - # default: true + - name: Install toolchain from rust-toolchain.toml shell: sh - run: | - TOOLCHAIN_VERSION=$(grep 'channel =' rust-toolchain.toml | awk -F'"' '{print $2}') - rustup toolchain install "$TOOLCHAIN_VERSION" + run: scripts/install_toolchain.sh - name: SCcache setup if: ${{ inputs.cache == 'enabled' }} diff --git a/.github/actions/sccache-gcloud/action.yml b/.github/actions/sccache-gcloud/action.yml index c0b02aad93..cd9b0accfb 100644 --- a/.github/actions/sccache-gcloud/action.yml +++ b/.github/actions/sccache-gcloud/action.yml @@ -14,11 +14,10 @@ runs: - name: Auth gcloud id: gauth - uses: google-github-actions/auth@ef5d53e30bbcd8d0836f4288f5e50ff3e086997d # @v1 + uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # @v1.1.1 with: workload_identity_provider: ${{ inputs.GWIP }} service_account: ${{ inputs.GSA }} - # export_environment_variables: true - name: Run sccache-cache uses: mozilla-actions/sccache-action@8417cffc2ec64127ad83077aceaa8631f7cdc83e #v0.0.3 @@ -33,7 +32,7 @@ runs: - name: setup Rust sccache wrapper if: ${{ inputs.cache == 'enabled' }} - shell: sh + shell: bash run: | echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV echo "CARGO_INCREMENTAL=false" >> $GITHUB_ENV # https://github.com/mozilla/sccache#rust diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 9bf12a38ae..f3b9181bbd 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -4,7 +4,7 @@ on: push: branches: [main] release: - types: [published] + types: [unpublished] # GITHUB_REF == tag pushed with the release pull_request: paths: - ".github/workflows/build-docker.yml" @@ -59,9 +59,9 @@ jobs: suffix=-${{ env.NOW }} prefix=${{ matrix.target == 'test' && 'test-' || 'latest=auto' }} tags: | - type=ref,event=branch - type=semver,pattern={{raw}},suffix= - type=semver,pattern={{major}},prefix=${{ matrix.target == 'test' && 'test-' || '' }} + type=raw,event=branch,value={{branch}}-{{sha}} + type=semver,pattern={{raw}},suffix=,prefix=${{ matrix.target == 'test' && 'test-' || 'latest=auto' }} + type=semver,pattern={{major}},prefix=${{ matrix.target == 'test' && 'test-' || '' }},suffix= type=edge - name: Configure sccache diff --git a/.github/workflows/build-wasm.yml b/.github/workflows/build-wasm.yml index 655b074db6..1e0bee53a1 100644 --- a/.github/workflows/build-wasm.yml +++ b/.github/workflows/build-wasm.yml @@ -21,8 +21,8 @@ jobs: runs-on: ubuntu-latest-4-cores strategy: matrix: - target: [build-runtime, build-runtime-fast] - package: [altair-runtime, centrifuge-runtime] + target: [build-runtime] + package: [altair-runtime, centrifuge-runtime, development-runtime] name: ${{ matrix.target }}-${{ matrix.package }} steps: # PREP @@ -35,32 +35,24 @@ jobs: # sccache does not work inside the srtool container - uses: Swatinem/rust-cache@e207df5d269b42b69c8bc5101da26f7d31feddb4 # 2.6.2 with: - prefix-key: build #-${{ matrix.package }} + prefix-key: build shared-key: wasm workspaces: | . ./target/srtool cache-directories: ./runtime - # save-if: ${{ github.ref == 'refs/heads/master' }} - # BUILD - - id: buildopt - run: | - if ${{ matrix.target == 'build-runtime-fast'}} ; then - echo "BUILD_OPTS=--features=fast-runtime" >> GITHUB_ENV - fi - echo "BUILD_OPTS=${{ env.BUILD_OPTS }}" - - name: Run srtool + # BUILD + - name: Run srtool # https://docs.substrate.io/reference/command-line-tools/srtool/ run: > docker run --rm --user root --platform=linux/amd64 -e PACKAGE=${{ matrix.package }} -v /home/runner/.cargo:/cargo-home -v ${{ github.workspace }}:/build paritytech/srtool:1.73.0-0.12.0 /srtool/build --app - # -a, --app - # Enable the "app" mode which is a mix of json output and outputting progress during the build. This flag is recommended for CI. the json output will be provided as a single line at the end in compact mode # Alternative way of running SRTool that allows for "script-like" execution, # extremely useful to debug: + # - name: Run Docker SRTool # uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 # with: @@ -83,7 +75,6 @@ jobs: path: ./${{ matrix.package }}-srtool-wasm-report.json - id: 'auth' - # if: ${{ github.ref == 'refs/heads/main' }} uses: 'google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033' # v1.1.1 with: workload_identity_provider: ${{ secrets.GWIP_ARTIFACTS }} @@ -97,9 +88,8 @@ jobs: if: ${{ steps.auth.outcome == 'success' }} shell: bash run: | - target=$(echo "${{ matrix.target }}" | sed -e "s/^build-//" ) runtime_name=$(echo "${{ matrix.package }}" | sed -e "s/-runtime$//" ) filename=$(echo "${{ matrix.package }}" | sed -e 's/-/_/g' ) gsutil cp \ ./runtime/${runtime_name}/target/srtool/release/wbuild/${{ matrix.package }}/${filename}.compact.compressed.wasm \ - gs://centrifuge-wasm-repo/${runtime_name}/${runtime_name}-${target}-$(git rev-parse --short HEAD).wasm + gs://centrifuge-wasm-repo/${runtime_name}/${runtime_name}-$(git rev-parse --short HEAD).wasm diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 94b83fe61e..7118dd9617 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -31,6 +31,8 @@ jobs: - name: Run Benchmark ${{ matrix.runtimes }} run: ./scripts/runtime_benchmarks.sh ${{ matrix.runtimes }} + env: + RUSTC_WRAPPER: "sccache" - run: echo "NOW=$(date -u +%Y-%m-%d)" >> $GITHUB_ENV diff --git a/.github/workflows/sanity-checks.yml b/.github/workflows/sanity-checks.yml index 01fb9bf5f1..60f4abe409 100644 --- a/.github/workflows/sanity-checks.yml +++ b/.github/workflows/sanity-checks.yml @@ -1,6 +1,4 @@ on: - # push: - # branches: [main] pull_request: name: Sanity checks concurrency: diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index 91f84f6567..d35f95e89f 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -23,12 +23,15 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ PATH=/usr/local/cargo/bin:$PATH + COPY ./scripts scripts + COPY rust-toolchain.toml ./ # install rustup, use minimum components RUN curl -L "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" \ -o rustup-init; \ chmod +x rustup-init; \ - ./rustup-init -y --no-modify-path --profile minimal --default-toolchain stable; \ - rm rustup-init; + mv rustup-init /usr/local/bin/rustup + + RUN scripts/install_toolchain.sh # BUILD ARG RUSTC_WRAPPER='' @@ -68,19 +71,10 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 RUN ldd /usr/local/bin/centrifuge-chain && \ /usr/local/bin/centrifuge-chain --version - # Unclutter - # RUN mv /usr/share/ca* /tmp && \ - # rm -rf /usr/share/* && \ - # mv /tmp/ca-certificates /usr/share/ - - # minimize the attack surface - # RUN rm -rf /usr/bin /usr/sbin /usr/lib/python* - RUN apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* ; RUN mkdir -p /root/.local/share/centrifuge-chain && \ ln -s /root/.local/share/centrifuge-chain /data - ENV RUST_BACKTRACE 1 USER centrifuge EXPOSE 30333 9933 9944 diff --git a/scripts/install_toolchain.sh b/scripts/install_toolchain.sh index e31f30452a..5d924e8559 100755 --- a/scripts/install_toolchain.sh +++ b/scripts/install_toolchain.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash -set -eux +# set -eux + +RUST_TOOLCHAIN=$(grep 'channel =' rust-toolchain.toml | awk -F'"' '{print $2}') -RUST_TOOLCHAIN="${RUST_TOOLCHAIN:-nightly}" echo "Using rust toolchain: ${RUST_TOOLCHAIN}" echo "*** Initializing WASM build environment" From bfc67a59d0f62597f30538274087fff4ebe0c923 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 17 Nov 2023 13:32:49 +0100 Subject: [PATCH 66/67] Use Rust docker image for building the binary --- docker/centrifuge-chain/Dockerfile | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index d35f95e89f..c4067a2a30 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -2,8 +2,8 @@ # https://github.com/paritytech/polkadot-sdk/blob/master/docker/dockerfiles/polkadot/polkadot_injected_release.Dockerfile # ToDo: create a CI/builder image with preloaded tools -FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder - +# FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder +FROM --platform=linux/amd64 docker.io/library/rust:bookworm as builder # Defaults ENV RUST_BACKTRACE 1 ENV DEBIAN_FRONTEND=noninteractive @@ -20,17 +20,10 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder protobuf-compiler \ curl - ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH COPY ./scripts scripts COPY rust-toolchain.toml ./ - # install rustup, use minimum components - RUN curl -L "https://static.rust-lang.org/rustup/dist/x86_64-unknown-linux-gnu/rustup-init" \ - -o rustup-init; \ - chmod +x rustup-init; \ - mv rustup-init /usr/local/bin/rustup - + # RUN rustup update && rustup default + ENV PATH=~/.cargo/bin:$PATH RUN scripts/install_toolchain.sh # BUILD @@ -48,7 +41,7 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 as builder # ===== SECOND STAGE ====== # ToDo: create a secure image as a base for the binary -FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 +FROM --platform=linux/amd64 docker.io/library/ubuntu:latest LABEL io.centrifuge.image.authors="guillermo@k-f.co" \ io.centrifuge.image.vendor="Centrifuge" \ @@ -75,6 +68,7 @@ FROM --platform=linux/amd64 docker.io/library/ubuntu:22.04 RUN mkdir -p /root/.local/share/centrifuge-chain && \ ln -s /root/.local/share/centrifuge-chain /data + ENV RUST_BACKTRACE 1 USER centrifuge EXPOSE 30333 9933 9944 From 2fac7af4d51685c5bfe32ca952f5396be12ef151 Mon Sep 17 00:00:00 2001 From: Guillermo Perez Date: Fri, 17 Nov 2023 16:13:52 +0100 Subject: [PATCH 67/67] Fix review comments --- .github/CODEOWNERS | 4 ++-- .github/workflows/run-benchmarks.yml | 3 +-- docker/centrifuge-chain/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index eb1880020d..8429e0298c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,8 +2,8 @@ ## Changes to .github .github/* @mustermeiszer @NunoAlexandre @lemunozm -.github/workflows @gpmayorga -.github/actions @gpmayorga +.github/workflows @gpmayorga @wischli +.github/actions @gpmayorga @wischli ## Changes to ci ci/* @mustermeiszer @NunoAlexandre @lemunozm diff --git a/.github/workflows/run-benchmarks.yml b/.github/workflows/run-benchmarks.yml index 7118dd9617..a95cbd0dea 100644 --- a/.github/workflows/run-benchmarks.yml +++ b/.github/workflows/run-benchmarks.yml @@ -1,5 +1,4 @@ name: Run benchmarks -# ToDo: add an arbitrary label to keep this cache separate from the builds on: workflow_dispatch: # Manually triggered by an authorized repo contributor. pull_request: @@ -16,7 +15,7 @@ jobs: runs-on: ubuntu-latest-8-cores strategy: matrix: - runtimes: [centrifuge, altair] + runtimes: [centrifuge, altair, development] steps: - name: Checkout code uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4 diff --git a/docker/centrifuge-chain/Dockerfile b/docker/centrifuge-chain/Dockerfile index c4067a2a30..f401485280 100644 --- a/docker/centrifuge-chain/Dockerfile +++ b/docker/centrifuge-chain/Dockerfile @@ -41,7 +41,7 @@ FROM --platform=linux/amd64 docker.io/library/rust:bookworm as builder # ===== SECOND STAGE ====== # ToDo: create a secure image as a base for the binary -FROM --platform=linux/amd64 docker.io/library/ubuntu:latest +FROM --platform=linux/amd64 docker.io/library/ubuntu:jammy LABEL io.centrifuge.image.authors="guillermo@k-f.co" \ io.centrifuge.image.vendor="Centrifuge" \