Merge pull request #4751 from mnaamani/nara-update-from-master

Nara update from master

.github/workflows/joystream-node-docker-publish.yml

@@ -0,0 +1,43 @@
+# Publish production runtime build joystream/node:latest
+name: joystream-node-docker-publish
+
+on:
+ - workflow_dispatch
+
+env:
+ REPOSITORY: joystream/node
+
+jobs:
+ main:
+ name: Publish joystream/node:latest docker image
+ runs-on: ubuntu-latest
+ if: github.ref == 'refs/heads/master'
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v3
+
+ - uses: actions/setup-node@v3
+ with:
+ node-version: '14.x'
+
+ - id: compute_shasum
+ name: Compute runtime code shasum
+ run: |
+ export TAG_SHASUM=`scripts/runtime-code-shasum.sh`
+
+ - name: Login to DockerHub
+ uses: docker/login-action@v2
+ with:
+ username: ${{ secrets.DOCKERHUB_USERNAME }}
+ password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+ - name: Publish manifest
+ run: |
+ IMAGE=${{ env.REPOSITORY }}:${{ env.TAG_SHASUM }}
+ LATEST_TAG=${{ env.REPOSITORY }}:latest
+ docker pull $IMAGE-amd64
+ docker pull $IMAGE-arm64
+ docker manifest create $LATEST_TAG $IMAGE-amd64 $IMAGE-arm64
+ docker manifest annotate $LATEST_TAG $IMAGE-amd64 --arch amd64
+ docker manifest annotate $LATEST_TAG $IMAGE-arm64 --arch arm64
+ docker manifest push $LATEST_TAG

.github/workflows/joystream-node-docker.yml

@@ -162,12 +162,3 @@ jobs:
  docker manifest annotate $IMAGE $IMAGE-arm64 --arch arm64
  docker manifest push $IMAGE
 
- - name: Create manifest with latest tag for master
- if: github.ref == 'refs/heads/master'
- run: |
- IMAGE=${{ env.REPOSITORY }}:${{ env.TAG_SHASUM }}
- LATEST_TAG=${{ env.REPOSITORY }}:latest
- docker manifest create $LATEST_TAG $IMAGE-amd64 $IMAGE-arm64
- docker manifest annotate $LATEST_TAG $IMAGE-amd64 --arch amd64
- docker manifest annotate $LATEST_TAG $IMAGE-arm64 --arch arm64
- docker manifest push $LATEST_TAG

.github/workflows/run-network-tests.yml

@@ -45,6 +45,7 @@ jobs:
  runs-on: ubuntu-latest
  outputs:
  use_artifact: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz
+ runtime: ${{ steps.compute_shasum.outputs.shasum }}
  steps:
  - uses: actions/checkout@v3
  - uses: actions/setup-node@v3
@@ -65,7 +66,7 @@ jobs:
  - name: Cache docker images
  uses: actions/cache@v3
  env:
- cache-name: joystream-node-docker
+ cache-name: joystream-node-docker-1
  with:
  path: ~/docker-images
  key: ${{ env.cache-name }}-${{ steps.compute_shasum.outputs.shasum }}
@@ -83,8 +84,7 @@ jobs:
  run: |
  if ! [ -f joystream-node-docker-image.tar.gz ]; then
  docker pull joystream/node:${{ steps.compute_shasum.outputs.shasum }}
- docker image tag joystream/node:${{ steps.compute_shasum.outputs.shasum }} joystream/node:latest
- docker save --output joystream-node-docker-image.tar joystream/node:latest
+ docker save --output joystream-node-docker-image.tar joystream/node:${{ steps.compute_shasum.outputs.shasum }}
  gzip joystream-node-docker-image.tar
  cp joystream-node-docker-image.tar.gz ~/docker-images/
  fi
@@ -117,7 +117,7 @@ jobs:
  CARGO_FEATURES=testing-runtime
  CODE_SHASUM=${{ steps.compute_shasum.outputs.shasum }}
  push: false
- tags: joystream/node
+ tags: joystream/node:${{ steps.compute_shasum.outputs.shasum }}
  cache-from: type=local,src=/tmp/.buildx-cache
  cache-to: type=local,dest=/tmp/.buildx-cache
  builder: ${{ steps.builder1.outputs.name }}
@@ -126,7 +126,7 @@ jobs:
 
  - name: Save joystream/node image to cache
  run: |
- docker save --output joystream-node-docker-image.tar joystream/node
+ docker save --output joystream-node-docker-image.tar joystream/node:${{ steps.compute_shasum.outputs.shasum }}
  gzip joystream-node-docker-image.tar
  cp joystream-node-docker-image.tar.gz ~/docker-images/
  if: steps.check_files.outputs.files_exists == 'false'
@@ -171,125 +171,74 @@ jobs:
  run: yarn workspace network-tests build
  - name: Execute network tests
  run: |
- export RUNTIME=latest
+ export RUNTIME=${{ needs.build_images.outputs.runtime }}
  export NO_STORAGE=${{ matrix.no_storage }}
  tests/network-tests/run-tests.sh ${{ matrix.scenario }}
 
- build_master_testing:
- name: Build joystream node images for mainnet-testing
+ runtime_upgrade:
+ name: Runtime Upgrade from production runtime
+ needs: build_images
  runs-on: ubuntu-latest
  if: github.ref != 'refs/heads/master'
- outputs:
- use_artifact: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz
  steps:
+ # Checkout master branch
  - name: check out master repo
  uses: actions/checkout@v3
  with:
  repository: Joystream/joystream
  ref: master
 
- - id: compute_shasum
+ - name: pull base runtime image
+ id: pull_base_image
  env:
  RUNTIME_PROFILE: 'TESTING'
  run: |
- export RUNTIME_CODE_SHASUM=`scripts/runtime-code-shasum.sh`
- echo "::set-output name=shasum::${RUNTIME_CODE_SHASUM}"
-
- - name: Setup cache directory
- run: mkdir ~/docker-images
-
- - name: Cache docker images
- uses: actions/cache@v3
- env:
- cache-name: joystream-node-docker-image-master
- with:
- path: ~/docker-images
- key: ${{ env.cache-name }}-${{ steps.compute_shasum.outputs.shasum }}
-
- - name: Check if we have cached image
- continue-on-error: true
- run: |
- if [ -f ~/docker-images/joystream-node-docker-image-master.tar.gz ]; then
- docker load --input ~/docker-images/joystream-node-docker-image-master.tar.gz
- cp ~/docker-images/joystream-node-docker-image-master.tar.gz .
- fi
-
- - name: Check if we have pre-built image on Dockerhub
- continue-on-error: true
- run: |
- if ! [ -f joystream-node-docker-image-master.tar.gz ]; then
- docker pull joystream/node:${{ steps.compute_shasum.outputs.shasum }}
- docker image tag joystream/node:${{ steps.compute_shasum.outputs.shasum }} joystream/node:mainnetDev
- docker save --output joystream-node-docker-image-master.tar joystream/node:mainnetDev
- gzip joystream-node-docker-image.tar
- cp joystream-node-docker-image-master.tar.gz ~/docker-images/
- fi
-
- - name: Check we now have an image
- id: check_master_image
- uses: andstor/file-existence-action@v1
- with:
- files: 'joystream-node-docker-image-master.tar.gz'
-
- - name: Set up Docker Buildx master
- uses: docker/setup-buildx-action@dc7b9719a96d48369863986a06765841d7ea23f6 # v1.7
- id: builder2
- if: steps.check_master_image.outputs.files_exists == 'false'
+ export RUNTIME=`scripts/runtime-code-shasum.sh`
+ echo "::set-output name=shasum::${RUNTIME}"
+ docker pull joystream/node:$RUNTIME
+ docker images
 
- - name: Build master image
- uses: docker/build-push-action@v3
+ # Get new runtime built for this workflow (target runtime)
+ # tagged in local repo as joystream/node:latest
+ - name: Get artifacts
+ uses: actions/download-artifact@v3
  with:
- context: .
- file: joystream-node.Dockerfile
- platforms: linux/amd64
- build-args: |
- CARGO_FEATURES=testing-runtime
- CODE_SHASUM=${{ steps.compute_shasum_master.outputs.shasum }}
- push: false
- tags: joystream/node:mainnetDev
- builder: ${{ steps.builder2.outputs.name }}
- load: true
- if: steps.check_master_image.outputs.files_exists == 'false'
-
- - name: Save joystream/node image to cache
+ name: ${{ needs.build_images.outputs.use_artifact }}
+ - name: Install artifacts
  run: |
- docker save --output joystream-node-docker-image-master.tar joystream/node:mainnetDev
- gzip joystream-node-docker-image-master.tar
- cp joystream-node-docker-image-master.tar.gz ~/docker-images/
- if: steps.check_master_image.outputs.files_exists == 'false'
+ docker load --input joystream-node-docker-image.tar.gz
+ docker images
 
- - name: Save joystream/node image (master) to Artifacts
- uses: actions/upload-artifact@v3
+ # Checkout workflow's branch/tag/commit
+ - name: checkout workflow branch
+ uses: actions/checkout@v3
+ - name: Check for runtime code changes
+ uses: technote-space/get-diff-action@v3
  with:
- name: ${{ steps.compute_shasum.outputs.shasum }}-joystream-node-docker-image.tar.gz
- path: joystream-node-docker-image-master.tar.gz
-
- runtime_upgrade:
- name: Runtime Upgrade From production runtime
- needs: [build_master_testing, build_images]
- if: github.ref != 'refs/heads/master'
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
- - uses: actions/setup-node@v3
+ PREFIX_FILTER: |
+ runtime
+ runtime-modules
+ SUFFIX_FILTER: |
+ .rs
+ FILES: |
+ Cargo.lock
+ Cargo.toml
+ - name: setup node
+ uses: actions/setup-node@v3
  with:
  node-version: '14.x'
- - name: Install artifacts
- uses: actions/download-artifact@v3
- - name: load docker images
- run: |
- docker load --input ${{needs.build_images.outputs.use_artifact}}/*
- docker load --input ${{needs.build_master_testing.outputs.use_artifact}}/*
- docker images
  - name: Install packages and dependencies
+ if: env.GIT_DIFF
  run: |
  yarn build:packages
  - name: Ensure tests are runnable
+ if: env.GIT_DIFF
  run: yarn workspace network-tests build
  - name: Execute network tests
+ if: env.GIT_DIFF
  env:
- RUNTIME: mainnetDev
- TARGET_RUNTIME: latest
+ TARGET_RUNTIME: ${{ needs.build_images.outputs.runtime }}
+ RUNTIME: ${{ steps.pull_base_image.outputs.shasum }}
  run: |
  export HOME=${PWD}
  mkdir -p ${HOME}/.local/share/joystream-cli

devops/ansible/roles/common/tasks/install-tools.yml

@@ -1,16 +1,10 @@
 ---
 # Install developer tools
 
-- name: Install aptitude
- become: true
- apt:
- name: aptitude
- state: latest
- update_cache: true
-
 - name: Install required system packages
  become: true
  apt:
+ update_cache: yes
  pkg:
  - coreutils
  - apt-transport-https

joystream-node.Dockerfile

@@ -8,7 +8,9 @@ RUN rustup default nightly-2022-05-11
 RUN rustup target add wasm32-unknown-unknown --toolchain nightly-2022-05-11
 RUN rustup component add --toolchain nightly-2022-05-11 clippy
 RUN rustup install nightly
-RUN cargo +nightly install cargo-chef
+# Temporary build fix by using older version of cargo-chef
+# https://github.com/Joystream/joystream/issues/4745
+RUN cargo +nightly install [email protected]
 
 FROM rust AS planner
 LABEL description="Cargo chef prepare"

tests/network-tests/run-runtime-upgrade-tests.sh

@@ -150,16 +150,16 @@ function cleanup() {
 
 # entrypoint
 function main {
- # Start a query-node
- if [ "${NO_QN}" != true ]; then
- ../../query-node/start.sh
- fi
-
  CONTAINER_ID=""
  export JOYSTREAM_NODE_TAG=${RUNTIME}
  if [ $TARGET_RUNTIME == $RUNTIME ]; then
  echo >&2 "Same tag for runtime and target runtime aborting..."
- exit 1
+ exit 0
+ fi
+
+ # Start a query-node
+ if [ "${NO_QN}" != true ]; then
+ ../../query-node/start.sh
  fi
 
  # 0. Generate config files