Merge pull request #155 from gluwa/testnet-release-3.12.0

[CSUB-800] Testnet release 3.12.0

.github/authorized_keys

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDOYiBGthrSNqUJdN9h9PHzXQL8cP8gj5pP9LZDx7BVgt/Knm9NwAe9hD/7fs9zmyECmZ5ubHDqG0x7Hb7DAjl+oPkCOxqRj5Npfvl1VRwwgXl3ymfI3JJpF7Cna4n0XdylBsTiwOL1/zoVXEJgTYEDEsP4gv65i8M/uWlsrfFwHLDEr3EQKnA0H4Ekz1CU2n9MFprX1hzA5IItozQUsYxKTPr1mxNTi1AFMhDEztMelvPO1OuC8MBZURR9S/+SlZ8ydCUcwl0gdCcgpUfouiuN9Yr9UbiV/yrAxEY3oKX8OegFmWEioUIZoFSiXl3sNP39ntOR4i+GV54g4omN6JbN3ios9LXBPMuvCy1NgFYPmDmmSEuo7n2IsP3pcXjZXpl4Ymwvn4RJviOZq8vghF5p0YMtCis1LxHL90epibxGoV6nc5isrOzqfgNUA9IS1zx7ZMoaF1PT8QcT8NzgNocrrhH4xGeTLD7WOOxykVASBnIXea8p7dl29/G0ThmS7Bc= [email protected]

.github/bench.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -e
+
+# Example: Compile and benchmark the bridge pallet.
+# ./scripts/bench.sh -p bridge -cb
+
+PALLET=bridge
+COMMAND=build
+BENCH=1
+BUILD=1
+REPEAT=30
+STEPS=50
+
+while getopts "fcbp:r:s:" opt;do
+    case $opt in
+        (f) COMMAND=check BUILD=0;;
+        (c) BUILD=0;;
+        (p) PALLET=$OPTARG;;
+        (b) BENCH=0;;
+        (r) REPEAT=$OPTARG;;
+        (s) STEPS=$OPTARG;;
+        (*)
+            echo "ERROR: Invalid flag detected"
+            exit 3
+    esac
+done
+
+
+OUTPUT="./pallets/${PALLET//_/-}/src/weights.rs"
+mkdir -p "pallets/$PALLET/src"
+
+if [[ $BUILD -eq 0 ]]
+then
+    cargo $COMMAND --release --features runtime-benchmarks || exit 1;
+fi
+
+if [[ $BENCH -eq 0 ]]
+then
+    ./target/release/creditcoin3-node benchmark pallet --chain dev --steps="$STEPS" --repeat="$REPEAT" --pallet "pallet_$PALLET" --extrinsic='*' --wasm-execution=compiled --heap-pages=10000 --output "$OUTPUT"
+    sed -i "s/pallet_$PALLET/crate/" "$OUTPUT"
+fi

.github/check-extrinsics.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+# This script:
+# - Checks for changes in transaction version in runtime/src/version.rs
+# - Downloads latest release binary from gluwa/creditcoin (RELEASE_BIN)
+# - Compiles and build a binary from the current branch (HEAD_BIN)
+# - Runs the two nodes
+
+set -ex
+
+HEAD_BIN=./target/release/creditcoin3-node
+HEAD_WS=ws://localhost:9944
+RELEASE_WS=ws://localhost:9955
+
+runtimes=(
+  "creditcoin3-runtime"
+)
+
+# First we fetch the latest released binary
+latest_release_tag() {
+  # WARNING: $GITHUB_TOKEN must be defined in the calling environment because this is a private repository
+  curl --silent --header "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/$1/releases/latest" | jq -r '.tag_name'
+}
+
+latest_release_url() {
+  # WARNING: $GITHUB_TOKEN must be defined in the calling environment because this is a private repository
+  curl --silent --header "Authorization: Bearer $GITHUB_TOKEN" "https://api.github.com/repos/$1/releases/latest" | jq -r '.url'
+}
+
+latest_tag=$(latest_release_tag 'gluwa/creditcoin3')
+latest_url=$(latest_release_url 'gluwa/creditcoin3')
+RELEASE_BIN="./creditcoin3-node"
+echo "[+] Fetching binary for Creditcoin version $latest_tag"
+# WARNING: $GITHUB_TOKEN must be defined in the calling environment because this is a private repository
+asset_url=$(curl --silent --header "Authorization: Bearer $GITHUB_TOKEN" \
+    "${latest_url}" | jq -r ".assets[] | select(.name==\"creditcoin-${latest_tag}-x86_64-unknown-linux-gnu.zip\") | .url"
+)
+curl --header "Authorization: Bearer $GITHUB_TOKEN" --header "Accept: application/octet-stream" -L \
+    "${asset_url}" --output creditcoin.zip
+unzip creditcoin.zip
+chmod +x "$RELEASE_BIN"
+git fetch --depth="${GIT_DEPTH:-100}" origin 'refs/tags/*:refs/tags/*'
+
+
+for RUNTIME in "${runtimes[@]}"; do
+  echo "[+] Checking runtime: ${RUNTIME}"
+
+  release_transaction_version=$(git show "tags/$latest_tag:runtime/src/version.rs" | grep 'transaction_version')
+
+  current_transaction_version=$(
+    grep 'transaction_version' "./runtime/src/version.rs"
+  )
+
+  echo "[+] Release: ${release_transaction_version}"
+  echo "[+] Ours: ${current_transaction_version}"
+
+
+  # Start running the nodes in the background
+  $HEAD_BIN --chain=local --tmp >head-node.log 2>&1 &
+  $RELEASE_BIN --chain=local --rpc-port 9955 --tmp --port 30555 >release-node.log 2>&1 &
+  jobs
+
+  #Wait for HEAD BINARY
+  ./.github/wait-for-creditcoin.sh 'http://127.0.0.1:9944'
+  #Wait for RELEASE BINARY
+  ./.github/wait-for-creditcoin.sh 'http://127.0.0.1:9955'
+
+  changed_extrinsics=$(
+    polkadot-js-metadata-cmp "$RELEASE_WS" "$HEAD_WS" \
+      | sed 's/^ \+//g' | grep -e 'idx: [0-9]\+ -> [0-9]\+' || true
+  )
+
+  # compare to mainnet and testnet explicitly b/c latest release could be any of them
+  # for now this comparison is only used to provide more info in CI
+  # polkadot-js-metadata-cmp wss://rpc.cc3-mainnet.creditcoin.network/ws "$HEAD_WS" > metadata-cmp-with-mainnet.txt
+  # polkadot-js-metadata-cmp wss://rpc.cc3-testnet.creditcoin.network/ws "$HEAD_WS" > metadata-cmp-with-testnet.txt
+
+  if [ -n "$changed_extrinsics" ]; then
+    echo "[!] Extrinsics indexing/ordering has changed in the ${RUNTIME} runtime! If this change is intentional, please bump transaction_version in lib.rs. Changed extrinsics:"
+    echo "$changed_extrinsics"
+
+    if [ "$release_transaction_version" == "$current_transaction_version" ]; then
+        exit 1
+    else
+        echo "[+] Transaction version for ${RUNTIME} has been bumped since last release. Exiting."
+    fi
+  fi
+
+  echo "[+] No change in extrinsics ordering for the ${RUNTIME} runtime"
+  jobs -p | xargs kill -9
+done

.github/check-for-used-forks.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# Check if there are git [forks] specified in Cargo.toml files!
+# Generally we don't want anything coming from git repositories b/c
+# it will not be updated by Dependabot. Substrate is a notable exception!
+#
+# Any other dependency coming from a git repository could be a fork
+# which is probably old and may lead to issues when upgrading.
+
+USED_FORKS=$(find ./ -name Cargo.toml -print0 | xargs --null grep git |
+# whitelist begin
+    grep -v "repository =" |
+    # frontier doesn't publish to crates.io and depends on polkadot-sdk
+    grep -v github.com/gluwa/frontier |
+    grep -v github.com/gluwa/polkadot-sdk
+# whitelist end
+)
+echo "INFO: Used forks in Cargo.toml files"
+echo "$USED_FORKS"
+
+if [ -n "$USED_FORKS" ]; then
+    echo "FAIL: Cargo.toml files seem to specify dependencies from git forks"
+    echo
+    echo "TODO: For each individual dependency"
+    cat <<_EOF_
+1) Open a PR with upstream
+2) Open a Jira ticket with a reference to the upstream PR +
+   action item to replace the git repository in Cargo.toml start with crates.io +
+   action item to clean-up the whitelist above!
+3) Update the whitelist at the top of ./scripts/check-for-used-forks.sh to pass CI
+_EOF_
+    echo "TODO: ===== end ====="
+
+    exit 1
+fi
+
+echo "PASS: Cargo.toml files don't specify dependencies from git forks"
+exit 0

.github/check-hardware.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+cat /proc/cpuinfo
+free -m
+cat /proc/meminfo
+
+./target/release/creditcoin3-node benchmark machine --chain dev

.github/check-parity-bw-extrinsics-benchmarks-weights.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -e
+
+PALLET_WHITELIST='xxx'
+FILES_WITH_EXTRINSICS=$(grep -E "pallet::call|pallet::hooks" pallets/* -R | grep -Ev "$PALLET_WHITELIST" | cut -f1 -d: | sort | uniq)
+PALLETS=$(echo "$FILES_WITH_EXTRINSICS" | cut -f2 -d/ | sort | uniq | xargs)
+
+if [ "$1" == "--show-pallets" ]; then
+    echo "$PALLETS" | tr "-" "_"
+    exit 0
+fi
+
+# prepare the value for use with grep -E
+FILES_WITH_EXTRINSICS=$(echo "$FILES_WITH_EXTRINSICS" | xargs)
+WHITELIST=""
+
+# NOTE: $FILES_WITH_EXTRINSICS isn't quoted below because we want the shell
+# to split the words, i.e. tell grep to search only in specific files
+# shellcheck disable=SC2086
+# WARNING: /dev/null coaxes grep into thinking that it is dealing with multiple files
+# and allows the following commands to work even when we have a single file defining extrinsics
+EXTRINSICS=$(grep "pub fn" /dev/null $FILES_WITH_EXTRINSICS | cut -f2 -d":" | cut -f1 -d"(" | sed 's/pub fn //' | tr -d ' \t' | sort)
+
+echo "----- Detected extrinsics are -----"
+echo "$EXTRINSICS"
+
+for EXTRINSIC in $EXTRINSICS; do
+    if [[ $WHITELIST =~ $EXTRINSIC ]]; then
+        echo "***** Skipping $EXTRINSIC - white listed"
+        continue
+    fi
+
+    echo "----- Searching weights for $EXTRINSIC -----"
+    grep "fn $EXTRINSIC" pallets/*/src/weights.rs
+
+    # makes sure the weight for an extrinsic function has the
+    # same name as the extrinsic - in case copy&paste errors, etc
+    echo "----- Double check weights for $EXTRINSIC -----"
+    grep -B1 "pub fn $EXTRINSIC" pallets/*/src/lib.rs | grep "WeightInfo::$EXTRINSIC"
+
+    echo "----- Searching benchmarks for $EXTRINSIC -----"
+    grep "$EXTRINSIC {" pallets/*/src/benchmarking.rs
+done
+
+echo "----- DONE - ALL PASS -----"

.github/check-spec-increase.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+set -eu
+
+
+GITHUB_BASE_REF="${1:-origin/testnet}"
+GITHUB_HEAD_REF="${2:-HEAD}"
+
+BASE_REF=$(git rev-parse "${GITHUB_BASE_REF}")
+
+echo "base ref = $BASE_REF"
+
+HEAD_REF=$(git rev-parse "${GITHUB_HEAD_REF}")
+
+echo "head ref = $HEAD_REF"
+
+get_spec_version () {
+    local rev=$1
+
+    git checkout "$rev" >/dev/null 2>&1
+
+    local version
+    version=$(grep -Eow -m 1 "spec_version: [0-9]+" ./runtime/src/version.rs | grep -Eow -m 1 "[0-9]+")
+
+    if [[ -z $version ]]; then
+        >&2 echo "Could not find spec version in version.rs at $rev"
+        exit 1
+    fi
+
+    git checkout - >/dev/null 2>&1
+
+    echo "$version"
+}
+
+base_version=$(get_spec_version "$BASE_REF")
+
+head_version=$(get_spec_version "$HEAD_REF")
+
+echo "base version = $base_version ; head version = $head_version"
+
+if [[ "$base_version" == "$head_version" ]]; then
+    echo 'No spec version change!'
+    echo "::set-output name=needs_bench::0"
+else
+    echo 'Spec version changed!'
+    echo "::set-output name=needs_bench::1"
+fi

.github/check-usage-of-log-warn.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+# shellcheck disable=SC2038
+OUTPUT=$(find . -wholename "*/migrations/*.rs" | xargs grep "log::warn" || true)
+if [ -n "$OUTPUT" ]; then
+    echo "FAIL"
+    echo "$OUTPUT"
+    exit 1
+fi
+
+echo "PASS"
+exit 0

.github/check-version-vs-git-tag.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -xeuo pipefail
+
+VERSION_FROM_CARGO_TOML=$(grep "^version =" Cargo.toml  | cut -f2 -d'=' | tr -d "' \"")
+VERSION_FROM_GIT_TAG=$(git describe --tag)
+
+# when releasing version strings in Cargo.toml and git tags must be in sync
+echo "INFO: Cargo.toml version is $VERSION_FROM_CARGO_TOML"
+echo "INFO: git tag version is $VERSION_FROM_GIT_TAG"
+
+if [[ ! "$VERSION_FROM_GIT_TAG" =~ "$VERSION_FROM_CARGO_TOML"* ]]; then
+    echo "FAIL: Versions in Cargo.toml and git tag are not in sync"
+    exit 2
+fi
+
+echo "PASS: Versions in Cargo.toml and git tag are in sync"
+exit 0

.github/check-version.sh

@@ -26,7 +26,7 @@ VERSION_FROM_CARGO_TOML=$(grep "^version =" Cargo.toml  | cut -f2 -d'=' | tr -d
 
 SPEC_VERSION=$(grep spec_version: runtime/src/version.rs | cut -f2 -d: | tr -d " ,")
 IMPL_VERSION=$(grep impl_version: runtime/src/version.rs | cut -f2 -d: | tr -d " ,")
-VERSION_FROM_VERSION_RS="1.$SPEC_VERSION.$IMPL_VERSION"
+VERSION_FROM_VERSION_RS="3.$SPEC_VERSION.$IMPL_VERSION"
 
 # Since PR #969 version strings in Cargo.toml and version.rs should be in sync
 echo "INFO: Cargo.toml version is $VERSION_FROM_CARGO_TOML"
@@ -57,8 +57,8 @@ if git --no-pager diff --name-only "${FROM}"..."${TO}" | grep -e '^runtime'; the
     check_version "${FROM}" "${TO}"
 else
     greenprint "INFO: runtime/src/ didn't change. Will inspect Cargo.lock"
-    if git --no-pager diff "${FROM}"..."${TO}" Cargo.lock | grep '+source = "git+https://github.com/paritytech'; then
-        echo "INFO: Cargo.lock references to 'frontier' or 'polkadot-sdk' have been modified"
+    if git --no-pager diff "${FROM}"..."${TO}" Cargo.lock | grep '+source = "git+https://github.com/'; then
+        echo "INFO: Cargo.lock references to dependencies via GitHub have been modified"
         check_version "${FROM}" "${TO}"
     else
         greenprint "INFO: Cargo.lock references to Substrate did not change"

.github/dependabot.yml

@@ -29,21 +29,32 @@ updates:
     schedule:
       interval: "monthly"
 
+  - package-ecosystem: "cargo"
+    directory: "/pallets/bridge"
+    schedule:
+      interval: "monthly"
+
   - package-ecosystem: "cargo"
     directory: "/runtime"
     schedule:
       interval: "monthly"
 
+  - package-ecosystem: "cargo"
+    directory: "/runtime/generate-bags"
+    schedule:
+      interval: "monthly"
+
   - package-ecosystem: "npm"
-    directory: "/examples/contract-erc20"
+    directory: "/testing"
     schedule:
       interval: "monthly"
     ignore:
       - dependency-name: "*"
         update-types: ["version-update:semver-patch"]
 
+  # Maintain dependencies for Node.js
   - package-ecosystem: "npm"
-    directory: "/examples/contract-hello"
+    directory: "/cli"
     schedule:
       interval: "monthly"
     ignore:

.github/provision-github-runner.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+mkdir actions-runner
+pushd actions-runner || exit 1
+
+curl -L https://github.com/actions/runner/releases/download/v2.311.0/actions-runner-linux-x64-2.311.0.tar.gz > runner.tar.gz
+
+tar xzf ./runner.tar.gz
+sudo ./bin/installdependencies.sh
+sudo apt install -y jq
+
+OWNER_REPO_SLUG="gluwa/creditcoin3"
+REPOSITORY_URL="https://github.com/$OWNER_REPO_SLUG"
+EPHEMERAL=${LC_RUNNER_EPHEMERAL:-true}
+
+# we need a temporary registration token first
+REGISTRATION_TOKEN=$(curl --silent -X POST \
+    -H "Accept: application/vnd.github+json" \
+    -H "Authorization: Bearer $LC_GITHUB_REPO_ADMIN_TOKEN" \
+    -H "X-GitHub-Api-Version: 2022-11-28" \
+    "https://api.github.com/repos/$OWNER_REPO_SLUG/actions/runners/registration-token" | jq -r '.token')
+
+# Important: ephemeral runners are removed after a single job is executed on them
+# which is inline with the VM lifecycle
+./config.sh --unattended --ephemeral "$EPHEMERAL" --url "$REPOSITORY_URL" --token "$REGISTRATION_TOKEN" --labels "$LC_RUNNER_VM_NAME"
+nohup ./run.sh >/dev/null 2>&1 </dev/null &