GHA pipeline rewrite for ease and speed (#1551)

* Removed all the old pipelines and separated them by "Trigger" (when they have to run)
* Introduced sccache for all PR checks builds and benchmarking
* Improved Dockerfile to make use of layers. Layers are then cached in the native GHA cache
* Srtool (Wasm build) will keep using the GHA cache (2GB each cache package) 
* Better Docker tagging and metadata to publish Docker images.
* Abstracted the common GHA pipline code into usable actions
* Reviewed the whole pipleine with a Rust developer to clean up unnecessary steps
* Publish wasm to a new bucket with better naming.

---------

Co-authored-by: William Freudenberger <[email protected]>

.dockerignore

@@ -1 +1,7 @@
 **/target/
+.github/
+!scripts/install_toolchain.sh
+docker-compos
+.gitignore
+docker
+README.md

.github/CODEOWNERS

@@ -2,6 +2,8 @@
 
 ## Changes to .github
 .github/* @mustermeiszer @NunoAlexandre @lemunozm
+.github/workflows @gpmayorga @wischli
+.github/actions @gpmayorga @wischli
 
 ## Changes to ci
 ci/* @mustermeiszer @NunoAlexandre @lemunozm

.github/actions/prep-ubuntu/action.yml

@@ -0,0 +1,40 @@
+name: Prepare Ubuntu for Rust builds
+description: cleanup and Rust Tools setup
+inputs:
+  cache:
+    description: cache type (enabled if set)
+    default: "disabled"
+  GWIP:
+    description: "Google Workload identity provider"
+    default: ''
+  GSA:
+    description: "Google Service Account"
+    default: ''
+runs:
+  using: composite
+  steps:
+    - name: Prep build on Ubuntu
+      id: ubuntu_prep
+      shell: sh
+      run: |
+        echo "Pre cleanup"
+        df -h
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        echo "Post cleanup"
+        df -h
+        sudo apt-get install protobuf-compiler
+     
+    - name: Install toolchain from rust-toolchain.toml
+      shell: sh
+      run: scripts/install_toolchain.sh
+
+    - name: SCcache setup
+      if: ${{ inputs.cache == 'enabled' }}
+      uses: ./.github/actions/sccache-gcloud
+      with:
+        GWIP: ${{ inputs.GWIP }}
+        GSA: ${{ inputs.GSA }}
+
+
+

.github/actions/sccache-gcloud/action.yml

@@ -0,0 +1,42 @@
+name: Install SCcache w/ GCloud bucket
+description: Configure Scache to use a Gcloud bucket through workload identity
+
+inputs:
+  GWIP:
+    description: Google Workflow Identity provider
+    required: true
+  GSA:
+    description: Google Service Account
+    required: true
+runs:
+  using: composite
+  steps:
+
+    - name: Auth gcloud
+      id: gauth
+      uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # @v1.1.1
+      with:
+        workload_identity_provider: ${{ inputs.GWIP }}
+        service_account: ${{ inputs.GSA }}
+
+    - name: Run sccache-cache
+      uses: mozilla-actions/sccache-action@8417cffc2ec64127ad83077aceaa8631f7cdc83e #v0.0.3
+
+    - name: set GCS bucket sccache variables
+      shell: bash
+      run: |
+        echo "SCCACHE_GCS_BUCKET=centrifuge-chain-sccache-backend" >> $GITHUB_ENV
+        echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> $GITHUB_ENV
+        # The gauth step should already take care of this:
+        # echo "SCCACHE_GCS_KEY_PATH=${{ steps.gauth.credentials_file_path }}" >> $GITHUB_ENV        
+
+    - name: setup Rust sccache wrapper
+      if: ${{ inputs.cache == 'enabled' }}
+      shell: bash
+      run: |
+        echo "RUSTC_WRAPPER=sccache" >> $GITHUB_ENV
+        echo "CARGO_INCREMENTAL=false" >> $GITHUB_ENV # https://github.com/mozilla/sccache#rust
+    
+    - name: Run sccache stat for check
+      shell: bash
+      run: ${SCCACHE_PATH} --show-stats      

.github/workflows/build-docker.yml

@@ -0,0 +1,105 @@
+name: Docker Build
+on:
+  # Keep in mind the Docker tagging on the "metadata" step if you add new triggers
+  push:
+    branches: [main]
+  release:
+    types: [unpublished] # GITHUB_REF == tag pushed with the release
+  pull_request:
+    paths:
+      - ".github/workflows/build-docker.yml"
+      - "docker/centrifuge-chain/Dockerfile"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.head.label || github.head_ref || github.ref }}${{ github.ref == '/refs/heads/main' && github.sha || 'false' }}
+  cancel-in-progress: true
+jobs:
+  docker:
+    strategy:
+      matrix:
+        target: [ release, test ]
+    runs-on: ubuntu-latest-8-cores
+    steps:
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac #v4
+        with:
+          fetch-depth: 0
+      - name: Free space
+      # https://github.com/actions/runner-images/issues/2840#issuecomment-1284059930
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          # sudo rm -rf "/usr/local/share/boost"
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+
+      - name: check available docker space
+        run: |
+          docker volume ls
+          df -h
+
+      - name: Install toolchain from rust-toolchain.toml
+        shell: sh
+        run: |
+          TOOLCHAIN_VERSION=$(grep 'channel =' rust-toolchain.toml | awk -F'"' '{print $2}')
+          rustup toolchain install "$TOOLCHAIN_VERSION"
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 #v3
+      - name: DockerHub Login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+
+      - run: echo "NOW=$(date -u +%y-%m-%d)" >> $GITHUB_ENV
+
+      - name: Setup docker metadata
+        id: meta
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 #v5
+        with:
+          images: centrifugeio/centrifuge-chain
+          flavor: |
+            suffix=-${{ env.NOW }}
+            prefix=${{ matrix.target == 'test' && 'test-' || 'latest=auto' }}
+          tags: |
+            type=raw,event=branch,value={{branch}}-{{sha}}
+            type=semver,pattern={{raw}},suffix=,prefix=${{ matrix.target == 'test' && 'test-' || 'latest=auto' }}
+            type=semver,pattern={{major}},prefix=${{ matrix.target == 'test' && 'test-' || '' }},suffix=
+            type=edge
+
+      - name: Configure sccache
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 #v6
+        with:
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+
+      - name: Build and push centrifugeio/centrifuge-chain
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 #v5
+        with:
+          context: .
+          file: ./docker/centrifuge-chain/Dockerfile
+          build-args: |
+            FEATURES=${{ matrix.target == 'test' && 'fast-runtime' || '' }}
+          push: ${{ github.ref == '/refs/heads/main' && true || false }}
+          tags: ${{ steps.meta.outputs.tags }}
+          # Cache options:
+          # https://docs.docker.com/build/ci/github-actions/cache/
+          cache-from: type=gha
+          # cache-from: type=registry,ref=centrifugeio/centrifuge-chain:${{ github.ref }}
+          # https://docs.docker.com/build/cache/backends/inline/
+          cache-to: type=gha, mode=max
+          # cache-to: type=registry,ref=centrifugeio/centrifuge-chain:${{ github.ref }}, mode=max
+
+      - name: Update DockerHub descriptions
+        if: contains(github.ref, 'refs/tags/release-v')
+        uses: peter-evans/dockerhub-description@dc67fad7001ef9e8e3c124cb7a64e16d0a63d864 #v3.4.2
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_TOKEN }}
+          repository: centrifuge/centrifuge-chain
+          short-description: ${{ github.event.repository.description }}
+          enable-url-completion: true
+
+      - if: failure()
+        name: Check available space after build failed
+        run: |
+          docker volume ls
+          df -h