Merge branch 'development' into feat_upgrade_polkadot_1.2.0

.github/workflows/010_build_and_test.yaml

@@ -39,6 +39,7 @@ jobs:
       - name: Build
         run: |
           cd substrate-node
+          cargo clean
           cargo build --release
 
       - name: Unit tests

.github/workflows/020_lint_and_test_go_client.yaml

@@ -6,29 +6,28 @@ on:
       - clients/tfchain-client-go/**
   workflow_dispatch:
 
-defaults:
-  run:
-    working-directory: clients/tfchain-client-go
-
 jobs:
   lint:
     name: lint
     runs-on: ubuntu-latest
     timeout-minutes: 5
-    steps:
+    steps:    
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+        with:
+          submodules: "true"
+          sparse-checkout: clients/tfchain-client-go
+
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: "1.20"
+          cache: false
+          # cache-dependency-path: clients/tfchain-client-go/go.sum
         id: go
 
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v3.5.0
-        with:
-          submodules: "true"
-
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v3.7.0
         with:
           args: --timeout 3m --verbose
           working-directory: clients/tfchain-client-go
@@ -44,4 +43,5 @@ jobs:
       - name: gofmt
         uses: Jerome1337/gofmt-action@v1.0.5
         with:
-          gofmt-flags: "-l -d"
+          gofmt-path: './clients/tfchain-client-go'
+          gofmt-flags: "-l -d"

.github/workflows/060_generate_benchmark_weights.yml

@@ -0,0 +1,80 @@
+name: Generate benchmark weights
+
+on:
+  workflow_dispatch:
+
+jobs:
+  generate-benchmark-weights:
+    runs-on: [self-hosted, tfchainrunner01]
+    container:
+      image: threefolddev/tfchain:4
+      env:
+        DEBIAN_FRONTEND: noninteractive
+        PATH: /root/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin
+        RUSTUP_HOME: /root/.rustup
+        CARGO_HOME: /root/.cargo
+
+    steps:
+      - name: Fail if branch is main
+        if: github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/development'
+        run: |
+          echo "This workflow should not be triggered with workflow_dispatch on main branch"
+          exit 1
+
+      - name: Checkout the repo
+        uses: actions/checkout@v3
+
+      - name: Chown user
+        run: |
+          chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - name: Cache build
+        uses: actions/cache@v3
+        timeout-minutes: 6
+        continue-on-error: true
+        with:
+          path: |
+            /root/.cargo/bin/
+            /root/.cargo/registry/index/
+            /root/.cargo/registry/cache/
+            /root/.cargo/git/db/
+            substrate-node/target/
+          key: ${{ runner.os }}-tfchain-build-cache-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: ${{ runner.os }}-tfchain-build-cache
+
+      - name: Build
+        run: |
+          cd substrate-node
+          cargo build --profile=production --features runtime-benchmarks
+
+      - name: Run benchmarking
+        shell: bash
+        run: |
+          cd substrate-node
+          for weights_rs_file in ./pallets/*/src/weights.rs
+          do
+            rm $weights_rs_file
+            pal_name=$(awk -F'pallets/|/src' '{print $2}' <<< $weights_rs_file)
+            ./target/production/tfchain benchmark pallet \
+            --chain=dev \
+            --wasm-execution=compiled \
+            --pallet="$pal_name" \
+            --extrinsic="*" \
+            --steps=50 \
+            --repeat=20 \
+            --heap-pages=409 \
+            --output ./pallets/"$pal_name"/src/weights.rs \
+            --template ./.maintain/frame-weight-template.hbs
+          done
+          
+      - name: Git config
+        run: |
+          git config --global --add safe.directory /__w/tfchain/tfchain
+          git status
+
+      - name: Commit & Push changes
+        uses: actions-js/push@master
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          message: 'chore: update benchmark `weights.rs` file for all pallets'
+          branch: ${{ github.ref_name }}

.github/workflows/060_publish_tfchain_image.yml

@@ -0,0 +1,42 @@
+name: Publish tfchain image
+
+on:
+  push:
+    tags:
+      - v*
+  workflow_dispatch:
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ghcr.io/threefoldtech/tfchain
+          tags: |
+            type=semver,pattern={{version}}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          file: ./substrate-node/Dockerfile
+          context: ./substrate-node/.
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/070_lint_and_test_go_bridge.yaml

@@ -0,0 +1,49 @@
+name: Lint Go bridge
+
+on:
+  push:
+    paths:
+      - bridge/tfchain_bridge/**
+  workflow_dispatch:
+
+jobs:
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:    
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+        with:
+          submodules: "true"
+          sparse-checkout: |
+            clients/tfchain-client-go
+            bridge/tfchain_bridge
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: "1.20"
+          cache: false
+          # cache-dependency-path: bridge/tfchain_bridge/go.sum
+        id: go
+
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3.7.0
+        with:
+          args: --timeout 3m --verbose
+          working-directory: bridge/tfchain_bridge
+
+      - name: staticcheck
+        uses: dominikh/staticcheck-action@v1.3.0
+        with:
+          version: "2022.1.3"
+          working-directory: bridge/tfchain_bridge
+        env:
+          GO111MODULE: on
+
+      - name: gofmt
+        uses: Jerome1337/gofmt-action@v1.0.5
+        with:
+          gofmt-path: './bridge/tfchain_bridge'
+          gofmt-flags: "-l -d"

.github/workflows/080_check_benchmark_weights.yml

@@ -0,0 +1,77 @@
+name: Check benchmark weights
+
+on:
+  push:
+    branches-ignore:
+      - development
+    paths:
+      - '**.rs'
+      - 'substrate-node/pallets/**'
+
+jobs:
+  check-benchmark-weights:
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v3
+
+      - name: Get all rust files that have changed in pallets
+        id: pallets-changed-rust-files
+        uses: tj-actions/changed-files@v42
+        with:
+          files: |
+            substrate-node/pallets/**/src/*.rs
+
+      - name: Get all pallets with changes in src dir
+        id: pallets-changed-src-dir
+        uses: tj-actions/changed-files@v42
+        with:
+          dir_names: "true"
+          files: |
+            substrate-node/pallets/**/src/*.rs
+
+      - name: List all changed files
+        env:
+          ALL_CHANGED_FILES: ${{ steps.pallets-changed-rust-files.outputs.all_changed_files }}
+        run: |
+          for file in ${ALL_CHANGED_FILES}; do
+            echo "$file file was changed"
+          done
+
+      - name: List all changed dir
+        env:
+          ALL_CHANGED_DIR: ${{ steps.pallets-changed-src-dir.outputs.all_changed_files }}
+        run: |
+          for dir in ${ALL_CHANGED_DIR}; do
+            echo "$dir has changes"
+          done
+
+      - name: Run benchmarking
+        env:
+          ALL_CHANGED_PALLETS_SRC_DIR: ${{ steps.pallets-changed-src-dir.outputs.all_changed_files }}
+          ALL_CHANGED_PALLETS_FILES: ${{ steps.pallets-changed-rust-files.outputs.all_changed_files }}
+        run: |
+          count=0
+          for pallet_src_dir in ${ALL_CHANGED_PALLETS_SRC_DIR}; do
+            echo "pallet src dir: $pallet_src_dir"
+            weights_file="$pallet_src_dir"/weights.rs
+            echo "weights file: $weights_file"
+            updated_weights=false
+            for changed_file in ${ALL_CHANGED_PALLETS_FILES}; do
+              if [ "$changed_file" = "$weights_file" ]; then
+                updated_weights=true
+                break
+              fi
+            done
+            if [ "$updated_weights" = false ] ; then
+              let "count=count+1"
+            fi
+          done
+          if [ "$count" -gt 0 ]; then
+            echo "Found changes on src rust file(s) for $count pallet(s) and respective weights.rs file(s) was not updated."
+            echo "Make sure to generate these files again if pallet logic has changed by running generate_benchmark_weights workflow on branch."
+            exit 1
+          else
+            echo "Found changes on src rust file(s) and respective weights.rs file(s) was updated."
+          fi

Makefile

@@ -0,0 +1,46 @@
+.PHONY: version-bump
+
+# * Usage Examples:*
+# type=patch make version-bump
+# type=minor make version-bump
+# type=major make version-bump
+# ** skip increment spec_version in substrate-node/runtime/src/lib.rs **
+# type=patch retain_spec_version=1 make version-bump
+version-bump:
+	set -e; \
+	if [ "$(type)" = "patch" ] || [ "$(type)" = "minor" ] || [ "$(type)" = "major" ]; then \
+		default_branch=$$(git symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@'); \
+		git checkout $$default_branch; \
+		git pull origin $$default_branch; \
+		new_version=$$(npx semver -i $(type) $$(jq -r .version clients/tfchain-client-js/package.json)); \
+		branch_name="$$default_branch-bump-version-to-$$new_version"; \
+		git checkout -b $$branch_name; \
+		current_spec_version=$$(sed -n -e 's/^.*spec_version: \([0-9]\+\),$$/\1/p' substrate-node/runtime/src/lib.rs); \
+		if [ -z "$${retain_spec_version}" ]; then \
+			current_spec_version=$$(sed -n -e 's/^.*spec_version: \([0-9]\+\),$$/\1/p' substrate-node/runtime/src/lib.rs); \
+			echo "Current spec_version: $$current_spec_version"; \
+			new_spec_version=$$((current_spec_version + 1)); \
+			echo "New spec_version: $$new_spec_version"; \
+			sed -i "s/spec_version: $$current_spec_version,/spec_version: $$new_spec_version,/" substrate-node/runtime/src/lib.rs; \
+		fi; \
+		jq ".version = \"$$new_version\"" activation-service/package.json > temp.json && mv temp.json activation-service/package.json; \
+		jq ".version = \"$$new_version\"" clients/tfchain-client-js/package.json > temp.json && mv temp.json clients/tfchain-client-js/package.json; \
+		jq ".version = \"$$new_version\"" scripts/package.json > temp.json && mv temp.json scripts/package.json; \
+		jq ".version = \"$$new_version\"" tools/fork-off-substrate/package.json > temp.json && mv temp.json tools/fork-off-substrate/package.json; \
+		sed -i "s/^version = .*/version = \"$$new_version\"/" substrate-node/Cargo.toml; \
+		sed -i "s/^version: .*/version: $$new_version/" substrate-node/charts/substrate-node/Chart.yaml; \
+		sed -i "s/^appVersion: .*/appVersion: '$$new_version'/" substrate-node/charts/substrate-node/Chart.yaml; \
+		sed -i "s/^version: .*/version: $$new_version/" bridge/tfchain_bridge/chart/tfchainbridge/Chart.yaml; \
+		sed -i "s/^appVersion: .*/appVersion: '$$new_version'/" bridge/tfchain_bridge/chart/tfchainbridge/Chart.yaml; \
+		sed -i "s/^version: .*/version: $$new_version/" activation-service/helm/tfchainactivationservice/Chart.yaml; \
+		sed -i "s/^appVersion: .*/appVersion: '$$new_version'/" activation-service/helm/tfchainactivationservice/Chart.yaml; \
+		cd substrate-node && cargo metadata -q 1> /dev/null && cd ..; \
+		git add substrate-node/Cargo.toml substrate-node/Cargo.lock substrate-node/charts/substrate-node/Chart.yaml bridge/tfchain_bridge/chart/tfchainbridge/Chart.yaml activation-service/helm/tfchainactivationservice/Chart.yaml activation-service/package.json clients/tfchain-client-js/package.json scripts/package.json tools/fork-off-substrate/package.json substrate-node/runtime/src/lib.rs; \
+		if [ -z "$${new_spec_version}" ]; then \
+			git commit -m "Bump version to $$new_version"; \
+		else \
+			git commit -m "Bump version to $$new_version (spec v$$new_spec_version)"; \
+		fi \
+	else \
+		echo "Invalid version type. Please use patch, minor, or major."; \
+	fi

activation-service/Dockerfile

@@ -4,11 +4,10 @@ FROM node:16
 WORKDIR /usr/src/app
 
 # Install app dependencies
-# A wildcard is used to ensure both package.json AND package-lock.json are copied
-# where available (npm@5+)
-COPY package*.json ./
+COPY package.json yarn.lock ./
+
+RUN yarn install && yarn cache clean
 
-RUN yarn install
 # If you are building your code for production
 # RUN npm ci --only=production
 

activation-service/helm/tfchainactivationservice/Chart.yaml

@@ -2,5 +2,5 @@ apiVersion: v2
 name: tfchainactivationservice
 description: TFchain account activation funding service
 type: application
-version: 2.6.0-rc1
-appVersion: '2.6.0-rc1'
+version: 2.9.0
+appVersion: '2.9.0'

activation-service/package.json

@@ -1,6 +1,6 @@
 {
   "name": "substrate-funding-service",
-  "version": "2.6.0-rc1",
+  "version": "2.9.0",
   "description": "Substrate funding service",
   "main": "index.js",
   "scripts": {
@@ -36,4 +36,4 @@
     "pino-pretty": "^5.0.2",
     "standard": "^16.0.3"
   }
-}
+}