feat: 3-node localnet (#4086)

* feat: 3-node localnet

* fix: ignore setup

* fix: try with sudo and debug logs

* chore: revert change

* fix: create directory

* fix: sudo and -p

* fix: sudo everywhere :(

* fix: don't sudo a var

* chore: no sudo

* chore: quick bouncer

* chore: sudo manage

* chore: debug logs

* fix: try mkdirs before

* fix: set selected nodes to 1

* chore: revert ci changes

* fix: add bashful bootnode

* fix: logs

* fix: add suggestion

* fix: suffix NODE_COUNT variable

* fix: replace dev null

* chore: use custom dev-3 chainspec for 3-node localnet

* fix rebase issues

* fix: restore order of tests

* chore: put gas limit test before concurrent

---------

Co-authored-by: kylezs <zsembery.kyle@gmail.com>

.github/workflows/_40_post_check.yml

@@ -67,9 +67,14 @@ jobs:
 
       - name: Start a localnet
         env:
-          BINARIES_LOCATION: .
+          BINARY_ROOT_PATH: .
         run: |
-          chmod +x ${{ env.BINARIES_LOCATION }}/chainflip-*
+          set -x
+          mkdir -p /tmp/chainflip/bashful
+          mkdir -p /tmp/chainflip/doc
+          mkdir -p /tmp/chainflip/dopey
+          chmod +x ${{ env.BINARY_ROOT_PATH }}/chainflip-*
+          touch ./localnet/.setup_complete
           ./localnet/manage.sh
 
       - name: Run EinfachHeuteLeiderNicht.exe
@@ -88,12 +93,12 @@ jobs:
       - name: Print chainflip-engine logs
         if: failure()
         run: |
-          cat /tmp/chainflip/chainflip-engine.log
+          cat /tmp/chainflip/*/chainflip-engine.log
 
       - name: Print chainflip-node logs
         if: failure()
         run: |
-          cat /tmp/chainflip/chainflip-node.log
+          cat /tmp/chainflip/*/chainflip-node.log
 
       - name: Upload Localnet Logs 💾
         if: always()
@@ -102,7 +107,7 @@ jobs:
         with:
           name: localnet-logs
           path: |
-            /tmp/chainflip/chainflip-*.log
+            /tmp/chainflip/*/chainflip-*.log
 
       - name: Clean Up docker containers 🧹
         if: always()

README.md

@@ -157,7 +157,7 @@ These commands can be used to control which logs the engine outputs at runtime.
 - `curl -X POST -H 'Content-Type: application/json' -d '"debug,warp=off,hyper=off,jsonrpc=off,web3=off,reqwest=off"' 127.0.0.1:36079/tracing` (Equivalent to the above, but without using the --sjon short-hand)
 
 The `RUST_LOG` environment variable controls the initial filtering directives if specified at engine startup.
-
+[manage.sh](localnet%2Fmanage.sh)
 The syntax for specifying filtering directives is given here: <https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/struct.EnvFilter.html>
 
 ## Testnet

bouncer/shared/send_dot.ts

@@ -6,7 +6,7 @@ import { polkadotSigningMutex, sleep, amountToFineAmount } from './utils';
 
 export async function sendDot(address: string, amount: string) {
   const aliceUri = process.env.POLKADOT_ALICE_URI || '//Alice';
-  const polkadotEndpoint = process.env.POLKADOT_ENDPOINT || 'ws://127.0.0.1:9945';
+  const polkadotEndpoint = process.env.POLKADOT_ENDPOINT || 'ws://127.0.0.1:9947';
 
   const planckAmount = amountToFineAmount(amount, assetDecimals.DOT);
   await cryptoWaitReady();

bouncer/shared/utils.ts

@@ -132,7 +132,7 @@ export const getChainflipApi = getCachedSubstrateApi(
   process.env.CF_NODE_ENDPOINT ?? 'ws://127.0.0.1:9944',
 );
 export const getPolkadotApi = getCachedSubstrateApi(
-  process.env.POLKADOT_ENDPOINT ?? 'ws://127.0.0.1:9945',
+  process.env.POLKADOT_ENDPOINT ?? 'ws://127.0.0.1:9947',
 );
 
 export const polkadotSigningMutex = new Mutex();

localnet/docker-compose.yml

@@ -1,7 +1,7 @@
 version: "3.9"
 services:
   init:
-    image: ghcr.io/chainflip-io/chainflip-eth-contracts/localnet-initial-state:v0.8.0-1-node
+    image: ghcr.io/chainflip-io/chainflip-eth-contracts/localnet-initial-state:v0.8.0-${NODE_COUNT}
     container_name: init
     platform: linux/amd64
     volumes:
@@ -16,7 +16,7 @@ services:
       - "/tmp/chainflip/data/redis-data:/data"
 
   geth:
-    image: ghcr.io/chainflip-io/chainflip-eth-contracts/geth:v0.8.0-1-node
+    image: ghcr.io/chainflip-io/chainflip-eth-contracts/geth:v0.8.0-${NODE_COUNT}
     container_name: geth
     platform: linux/amd64
     ports:
@@ -103,15 +103,15 @@ services:
       - --max-runtime-instances=256
       - --wasmtime-instantiation-strategy=recreate-instance-copy-on-write
     ports:
-      - 9945:9944
+      - 9947:9944
     healthcheck:
       test:
         [
           "CMD-SHELL",
           "curl",
           "-H 'Content-Type: application/json;'",
           "-d '{\"id\":1, \"jsonrpc\":\"2.0\", \"method\": \"chain_getBlockHash\", \"params\" : [0]}'",
-          "http://localhost:9944"
+          "http://localhost:9947"
         ]
       interval: 10s
       timeout: 5s

localnet/helper.sh

@@ -20,11 +20,12 @@ function check_endpoint_health() {
 
 function print_success() {
   logs=$(cat <<EOM
+---------------------------------------------------------------------------------------
 🚀 Network is live
 🪵 To get logs run: ./localnet/manage.sh
 👆 Then select logs (4)
 💚 Head to https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9944#/explorer to access PolkadotJS of Chainflip Network
-🧡 Head to https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9945#/explorer to access PolkadotJS of the Private Polkadot Network
+🧡 Head to https://polkadot.js.org/apps/?rpc=ws%3A%2F%2F127.0.0.1%3A9947#/explorer to access PolkadotJS of the Private Polkadot Network
 💜 Head to http://localhost:3002 to access the local Bitcoin explorer (credentials: flip / flip)
 👮‍ To run the bouncer: ./localnet/manage.sh -> (6)
 EOM

localnet/init/config/Settings.toml

@@ -1,18 +1,18 @@
 # Default configurations for the CFE
 [node_p2p]
-node_key_file = "./localnet/init/keys/node_key_file"
+#node_key_file = "./localnet/init/keys/node_key_file"
 ip_address = "127.0.0.1"
-port = 8078
+#port = 8078
 allow_local_ip = true
 
-[state_chain]
+#[state_chain]
 # 32 byte hex secret key - associated with the node's public id (public key)
-signing_key_file = "./localnet/init/keys/signing_key_file"
-ws_endpoint = "ws://localhost:9944"
+#signing_key_file = "./localnet/init/keys/signing_key_file"
+#ws_endpoint = "ws://localhost:9944"
 
-[eth]
+#[eth]
 # Ethereum private key file path. Default is the docker secrets path. This file should contain a hex-encoded private key.
-private_key_file = "./localnet/init/keys/eth_private_key_file"
+#private_key_file = "./localnet/init/keys/eth_private_key_file"
 
 [eth.rpc]
 ws_endpoint = "ws://localhost:8546"
@@ -24,8 +24,8 @@ http_endpoint = "http://localhost:8545"
 #http_endpoint = "http://localhost:8555"
 
 [dot.rpc]
-ws_endpoint = "ws://localhost:9945"
-http_endpoint = "http://localhost:9945"
+ws_endpoint = "ws://localhost:9947"
+http_endpoint = "http://localhost:9947"
 
 # optional
 #[dot.backup_rpc]
@@ -46,15 +46,15 @@ basic_auth_password = "flip"
 # optional
 [health_check]
 hostname = "0.0.0.0"
-port = 5555
+#port = 5555
 
 # optional
-[prometheus]
-hostname = "0.0.0.0"
-port = 5566
+#[prometheus]
+#hostname = "0.0.0.0"
+#port = 5566
 
-[signing]
-db_file = "/tmp/chainflip/bashful.db"
+#[signing]
+#db_file = "/tmp/chainflip/bashful.db"
 
 [logging]
 command_server_port = 4321

localnet/init/env/eth.env

@@ -6,7 +6,5 @@ export ADDRESS_CHECKER_ADDRESS=e7f1725E7734CE288F8367e1Bb143E90bb3F0512
 export ETH_USDC_ADDRESS=9fE46736679d2D9a65F0992F2272dE9f3c7fa6e0
 export ETH_CF_RECEIVER_ADDRESS=A51c1fc2f0D1a1b8494Ed1FE312d7C3a78Ed91C0
 export ETH_CF_RECEIVER_GRIEFER=0DCd1Bf9A1b36cE34237eEaFef220932846BCD82
-export ETH_INIT_AGG_KEY=0361ef4fcc2f464a81f4f1a60387056e937d67e6f7f717936c143b4b0102f710d6
-export DOT_INIT_AGG_KEY=ba55be45b8424b3b0781ab3d9e631fe4159e30c6996c864e6dd44b5fcff79c21
 export ETHEREUM_CHAIN_ID=10997
 export ETH_DEPLOYMENT_BLOCK=1

localnet/init/keys/doc/eth_private_key_file

@@ -0,0 +1 @@
+ef895dc87b545ba42ece3162b8441f712c892d0681ae34682eeb3e5443afefaa

localnet/init/keys/doc/node_key_file

@@ -0,0 +1 @@
+bac76708516faf25aa2b0cf28d93db721b2ba6beaa2f8eabc9c195d4b115df01

localnet/init/keys/doc/signing_key_file

@@ -0,0 +1 @@
+92c070c2d06e91d2d38941c81518a0f49a0e59f864eac2dc4b0757ebe8634898

localnet/init/keys/dopey/eth_private_key_file

@@ -0,0 +1 @@
+ef895dc87b545ba42ece3162b8441f712c892d0681ae34682eeb3e5443afefaa

localnet/init/keys/dopey/node_key_file

@@ -0,0 +1 @@
+a132ff372e0cef12f78bf6ec04d964ab6aa9a62b9e0eca9b5280ccc25be1c157

localnet/init/keys/dopey/signing_key_file

@@ -0,0 +1 @@
+a5cc9f6f5454218310d169b4f79f412793921ff566673a14d085d0d51729e5bb

localnet/init/scripts/start-engine.sh

@@ -1,5 +1,19 @@
 #!/bin/bash
 set -e
-binary_location=$1
+BINARY_ROOT_PATH=$1
+NODE_NAME=$2
+PORT=$3
+HEALTH_PORT=$4
+RPC_PORT=$5
+LOG_PORT=$6
 source ./localnet/init/env/cfe.env
-$binary_location/chainflip-engine --config-root=./localnet/init/ > /tmp/chainflip/chainflip-engine.log 2>&1 &
+$BINARY_ROOT_PATH/chainflip-engine \
+  --config-root=./localnet/init/ \
+  --eth.private_key_file=./localnet/init/keys/$NODE_NAME/eth_private_key_file \
+  --state_chain.signing_key_file=./localnet/init/keys/$NODE_NAME/signing_key_file \
+  --state_chain.ws_endpoint=ws://localhost:$RPC_PORT \
+  --p2p.node_key_file=./localnet/init/keys/$NODE_NAME/node_key_file \
+  --p2p.port=$PORT \
+  --logging.command_server_port=$LOG_PORT \
+  --signing.db_file=/tmp/chainflip/$NODE_NAME/$NODE_NAME.db \
+  --health_check.port=$HEALTH_PORT > /tmp/chainflip/$NODE_NAME/chainflip-engine.log 2>&1 &

localnet/init/scripts/start-node.sh

@@ -1,20 +1,35 @@
 #!/bin/bash
 set -e
-binary_location=$1
+BINARY_ROOT_PATH=$1
+NODE_NAME=$2
+PORT=$3
+RPC_PORT=$4
+NODE_COUNT=$5
+
+CHAIN="dev"
+if [ $NODE_COUNT == "3-node" ]; then
+    CHAIN="dev-3"
+fi
+
 source ./localnet/init/env/eth.env
 source ./localnet/init/env/arb.env
 source ./localnet/init/env/node.env
-$binary_location/chainflip-node key insert --chain=dev --base-path=/tmp/chainflip/chaindata --suri=0x$(cat ./localnet/init/keys/signing_key_file) --key-type=aura --scheme=sr25519
-$binary_location/chainflip-node key insert --chain=dev --base-path=/tmp/chainflip/chaindata --suri=0x$(cat ./localnet/init/keys/signing_key_file) --key-type=gran --scheme=ed25519
-$binary_location/chainflip-node --chain=dev \
-  --base-path=/tmp/chainflip/chaindata \
-  --node-key-file=./localnet/init/keys/node_key_file \
+export ETH_INIT_AGG_KEY=$(jq -r '.eth_agg_key' ./localnet/init/keyshare/$NODE_COUNT/agg_keys.json)
+export DOT_INIT_AGG_KEY=$(jq -r '.dot_agg_key' ./localnet/init/keyshare/$NODE_COUNT/agg_keys.json)
+$BINARY_ROOT_PATH/chainflip-node key insert --chain=$CHAIN --base-path=/tmp/chainflip/$NODE_NAME/chaindata --suri=0x$(cat ./localnet/init/keys/$NODE_NAME/signing_key_file) --key-type=aura --scheme=sr25519
+$BINARY_ROOT_PATH/chainflip-node key insert --chain=$CHAIN --base-path=/tmp/chainflip/$NODE_NAME/chaindata --suri=0x$(cat ./localnet/init/keys/$NODE_NAME/signing_key_file) --key-type=gran --scheme=ed25519
+$BINARY_ROOT_PATH/chainflip-node --chain=$CHAIN \
+  --base-path=/tmp/chainflip/$NODE_NAME/chaindata \
+  --node-key-file=./localnet/init/keys/$NODE_NAME/node_key_file \
   --validator \
   --force-authoring \
   --rpc-cors=all \
   --unsafe-rpc-external \
   --rpc-methods=unsafe \
-  --name=bashful \
+  --name=$NODE_NAME \
+  --port=$PORT \
+  --rpc-port=$RPC_PORT \
   --blocks-pruning=archive \
   --state-pruning=archive \
-  --trie-cache-size=0 > /tmp/chainflip/chainflip-node.log 2>&1 &
+  --bootnodes=/ip4/127.0.0.1/tcp/30333/p2p/12D3KooWFD3YMDyQocSQBAx6VCcUruYihi6xYufduzf321FnvvoY \
+  --trie-cache-size=0 > /tmp/chainflip/$NODE_NAME/chainflip-node.log 2>&1 &