From 57c51d7e1cd765535002ac9bce3edc0fb4dccdd7 Mon Sep 17 00:00:00 2001 From: Piotr Heilman Date: Thu, 9 May 2024 12:01:58 +0200 Subject: [PATCH] Added e2e testing with docker compose. --- Cargo.lock | 27 ++ Cargo.toml | 2 +- e2e_tests/docker-compose/compose.yml | 127 ++++++++ e2e_tests/docker-compose/compose.yml.bak | 137 +++++++++ e2e_tests/docker-compose/haproxy/haproxy.cfg | 10 + .../signup_sequencer/config.toml | 28 ++ e2e_tests/scenarios/Cargo.toml | 20 ++ e2e_tests/scenarios/tests/common/api.rs | 141 +++++++++ .../scenarios/tests/common/docker_compose.rs | 271 ++++++++++++++++++ e2e_tests/scenarios/tests/common/mod.rs | 83 ++++++ e2e_tests/scenarios/tests/insert_1k.rs | 83 ++++++ .../scenarios/tests/insert_delete_insert.rs | 155 ++++++++++ .../scenarios/tests/insert_restart_insert.rs | 151 ++++++++++ 13 files changed, 1234 insertions(+), 1 deletion(-) create mode 100644 e2e_tests/docker-compose/compose.yml create mode 100644 e2e_tests/docker-compose/compose.yml.bak create mode 100644 e2e_tests/docker-compose/haproxy/haproxy.cfg create mode 100644 e2e_tests/docker-compose/signup_sequencer/config.toml create mode 100644 e2e_tests/scenarios/Cargo.toml create mode 100644 e2e_tests/scenarios/tests/common/api.rs create mode 100644 e2e_tests/scenarios/tests/common/docker_compose.rs create mode 100644 e2e_tests/scenarios/tests/common/mod.rs create mode 100644 e2e_tests/scenarios/tests/insert_1k.rs create mode 100644 e2e_tests/scenarios/tests/insert_delete_insert.rs create mode 100644 e2e_tests/scenarios/tests/insert_restart_insert.rs diff --git a/Cargo.lock b/Cargo.lock index adb17e80..c7bd915b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1977,6 +1977,24 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56ce8c6da7551ec6c462cbaf3bfbc75131ebbfa1c944aeaa9dab51ca1c5f0c3b" +[[package]] +name = "e2e-tests" +version = "0.1.0" +dependencies = [ + "anyhow", + "ethers", + "hex", + "hyper", + "rand", + "retry", + "serde_json", + "tokio", + "tracing", + "tracing-futures", + "tracing-subscriber 0.3.18", + "tracing-test", +] + [[package]] name = "ecdsa" version = "0.16.9" @@ -4873,6 +4891,15 @@ dependencies = [ "winreg", ] +[[package]] +name = "retry" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9166d72162de3575f950507683fac47e30f6f2c3836b71b7fbc61aa517c9c5f4" +dependencies = [ + "rand", +] + [[package]] name = "rfc6979" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 22c6946a..09ffe2ca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ license-file = "LICENSE.md" build = "build.rs" [workspace] -members = ["crates/*"] +members = ["crates/*", "e2e_tests/scenarios"] [features] default = [] diff --git a/e2e_tests/docker-compose/compose.yml b/e2e_tests/docker-compose/compose.yml new file mode 100644 index 00000000..2e466da5 --- /dev/null +++ b/e2e_tests/docker-compose/compose.yml @@ -0,0 +1,127 @@ +services: + chain: + image: ghcr.io/foundry-rs/foundry + hostname: chain + platform: linux/amd64 + ports: + - ${CHAIN_PORT:-8545}:8545 + command: [ "anvil --host 0.0.0.0 --chain-id 31337 --block-time 30 --base-fee 0 --gas-limit 0 --gas-price 0 --fork-url https://eth-sepolia.g.alchemy.com/v2/${ALCHEMY_API_KEY}@5091094" ] + tx-sitter-db: + image: postgres:latest + hostname: tx-sitter-db + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=tx-sitter + ports: + - ${TX_SITTER_DB_PORT:-5460}:5432 + volumes: + - tx_sitter_db_data:/var/lib/postgresql/data + sequencer-db: + image: postgres:latest + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=sequencer + ports: + - ${SEQUENCER_DB_PORT:-5461}:5432 + volumes: + - sequencer_db_data:/var/lib/postgresql/data + tx-sitter: + image: ghcr.io/worldcoin/tx-sitter-monolith:latest + hostname: tx-sitter + depends_on: + - tx-sitter-db + - chain + restart: always + ports: + - ${TX_SITTER_PORT:-3000}:3000 + environment: + - RUST_LOG=info + - TX_SITTER__SERVICE__ESCALATION_INTERVAL=3s + - TX_SITTER__DATABASE__KIND=connection_string + - TX_SITTER__DATABASE__CONNECTION_STRING=postgres://postgres:postgres@tx-sitter-db:5432/tx-sitter?sslmode=disable + - TX_SITTER__KEYS__KIND=local + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__CHAIN_ID=31337 + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__NAME=Anvil + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__HTTP_RPC=http://chain:8545 + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__WS_RPC=ws://chain:8545 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__ID=1b908a34-5dc1-4d2d-a146-5eb46e975830 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__NAME=Relayer + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__CHAIN_ID=31337 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__KEY_ID=d10607662a85424f02a33fb1e6d095bd0ac7154396ff09762e41f82ff2233aaa + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__API_KEY=G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH + - TX_SITTER__SERVER__HOST=0.0.0.0:3000 + - TX_SITTER__SERVER__DISABLE_AUTH=true + semaphore-insertion: + image: semaphore-mtb + hostname: semaphore-insertion + restart: always + ports: + - ${SEMAPHORE_INSERTION_PORT:-3001}:3001 + command: [ "start", "--keys-file", "/mtb/keys", "--prover-address", "0.0.0.0:3001", "--mode", "insertion" ] + volumes: + - ./keys/insertion_b10t30.ps:/mtb/keys + environment: + BATCH_TIMEOUT_SECONDS: 1 + semaphore-deletion: + image: semaphore-mtb + hostname: semaphore-deletion + restart: always + ports: + - ${SEMAPHORE_DELETION_PORT:-3002}:3001 + command: [ "start", "--keys-file", "/mtb/keys", "--prover-address", "0.0.0.0:3001", "--mode", "deletion" ] + volumes: + - ./keys/deletion_b10t30.ps:/mtb/keys + environment: + BATCH_DELETION_TIMEOUT_SECONDS: 1 + signup-sequencer-balancer: + image: haproxy:3.0.0 + hostname: signup-sequencer-balancer + restart: always + ports: + - ${SIGNUP_SEQUENCER_BALANCER_PORT:-8080}:8080 + volumes: + - ./haproxy:/usr/local/etc/haproxy + depends_on: + - signup-sequencer-0 + signup-sequencer-0: &signup-sequencer-def + image: signup-sequencer + hostname: signup-sequencer-0 + profiles: [ e2e-ha ] + build: + context: ./../../ + depends_on: + - sequencer-db + - chain + - semaphore-insertion + - semaphore-deletion + - tx-sitter + restart: always + ports: + - ${SIGNUP_SEQUENCER_0_PORT:-9080}:8080 + volumes: + - ./signup_sequencer/config.toml:/config.toml + command: [ "/config.toml" ] + environment: + - RUST_LOG=debug +# signup-sequencer-1: +# <<: *signup-sequencer-def +# hostname: signup-sequencer-1 +# ports: +# - ${SIGNUP_SEQUENCER_0_PORT:-9081}:8080 +# signup-sequencer-2: +# <<: *signup-sequencer-def +# hostname: signup-sequencer-2 +# ports: +# - ${SIGNUP_SEQUENCER_0_PORT:-9082}:8080 +# signup-sequencer-3: +# <<: *signup-sequencer-def +# hostname: signup-sequencer-3 +# ports: +# - ${SIGNUP_SEQUENCER_0_PORT:-9083}:8080 +volumes: + tx_sitter_db_data: + driver: local + sequencer_db_data: + driver: local diff --git a/e2e_tests/docker-compose/compose.yml.bak b/e2e_tests/docker-compose/compose.yml.bak new file mode 100644 index 00000000..37aac900 --- /dev/null +++ b/e2e_tests/docker-compose/compose.yml.bak @@ -0,0 +1,137 @@ +version: "3" +services: + chain: + container_name: chain + image: ghcr.io/foundry-rs/foundry + hostname: chain + platform: linux/amd64 + ports: + - "8545:8545" + command: [ "anvil --host 0.0.0.0 --chain-id 31337 --block-time 10 --base-fee 0 --gas-limit 0 --gas-price 0 --fork-url https://eth-sepolia.g.alchemy.com/v2/Hkj3vTy6ee49NbI4Imhe6r5mRM1vkR10@5091094" ] + tx-sitter-db: + container_name: tx-sitter-db + image: postgres:latest + hostname: tx-sitter-db + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=tx-sitter + ports: + - "5460:5432" + volumes: + - tx_sitter_db_data:/var/lib/postgresql/data + sequencer-db: + container_name: sequencer-db + image: postgres:latest + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + - POSTGRES_DB=sequencer + ports: + - "5461:5432" + volumes: + - sequencer_db_data:/var/lib/postgresql/data + - /home/piotr/Downloads/dump.sql:/tmp2/dump.sql + tx-sitter: + container_name: tx-sitter + image: tx-sitter-monolith + hostname: tx-sitter + depends_on: + - tx-sitter-db + - chain + restart: always + ports: + - "3000:3000" + environment: + - RUST_LOG=info + - TX_SITTER__SERVICE__ESCALATION_INTERVAL=3s + - TX_SITTER__DATABASE__KIND=connection_string + - TX_SITTER__DATABASE__CONNECTION_STRING=postgres://postgres:postgres@tx-sitter-db:5432/tx-sitter?sslmode=disable + - TX_SITTER__KEYS__KIND=local + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__CHAIN_ID=31337 + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__NAME=Anvil + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__HTTP_RPC=http://chain:8545 + - TX_SITTER__SERVICE__PREDEFINED__NETWORK__WS_RPC=ws://chain:8545 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__ID=1b908a34-5dc1-4d2d-a146-5eb46e975830 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__NAME=Relayer + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__CHAIN_ID=31337 + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__KEY_ID=d10607662a85424f02a33fb1e6d095bd0ac7154396ff09762e41f82ff2233aaa + - TX_SITTER__SERVICE__PREDEFINED__RELAYER__API_KEY=G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH + - TX_SITTER__SERVER__HOST=0.0.0.0:3000 + - TX_SITTER__SERVER__DISABLE_AUTH=true + semaphore-insertion: + container_name: semaphore-insertion + image: semaphore-mtb + hostname: semaphore-insertion + restart: always + ports: + - "3001:3001" + command: [ "start", "--keys-file", "/mtb/keys", "--prover-address", "0.0.0.0:3001", "--mode", "insertion" ] + volumes: + - ./keys/insertion_b10t30.ps:/mtb/keys + environment: + BATCH_TIMEOUT_SECONDS: 1 + semaphore-deletion: + container_name: semaphore-deletion + image: semaphore-mtb + hostname: semaphore-deletion + restart: always + ports: + - "3002:3001" + command: [ "start", "--keys-file", "/mtb/keys", "--prover-address", "0.0.0.0:3001", "--mode", "deletion" ] + volumes: + - ./keys/deletion_b10t30.ps:/mtb/keys + environment: + BATCH_DELETION_TIMEOUT_SECONDS: 1 + signup-sequencer-0: &signup-sequencer-def + container_name: signup-sequencer-0 + image: signup-sequencer + profiles: [ e2e-ha ] + build: + context: ./../../ + depends_on: + - sequencer-db + - chain + - semaphore-insertion + - semaphore-deletion + - tx-sitter + restart: always + ports: + - "9080:8080" + environment: + - RUST_LOG=debug + - SEQ__TREE__TREE_DEPTH=30 + - SEQ__TREE__DENSE_TREE_PREFIX_DEPTH=10 + - SEQ__TREE__TREE_GC_THRESHOLD=10000000 + - SEQ__TREE__CACHE_FILE=./cache_file + - SEQ__SERVER__ADDRESS=0.0.0.0:8080 + - SEQ__NETWORK__IDENTITY_MANAGER_ADDRESS=0x48483748eb0446A16cAE79141D0688e3F624Cb73 + - SEQ__RELAYER__KIND=tx_sitter + - SEQ__RELAYER__TX_SITTER_URL=http://tx-sitter:3000/1/api/G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH + - SEQ__RELAYER__TX_SITTER_ADDRESS=0x1d7ffed610cc4cdC097ecDc835Ae5FEE93C9e3Da + - SEQ__RELAYER__TX_SITTER_GAS_LIMIT=2000000 + - SEQ__PROVIDERS__PRIMARY_NETWORK_PROVIDER=http://chain:8545 + - 'SEQ__APP__PROVERS_URLS=[{"url": "http://semaphore-insertion:3001", "prover_type": "insertion", "batch_size": 10,"timeout_s": 30}, {"url": "http://semaphore-deletion:3001", "prover_type": "deletion", "batch_size": 10,"timeout_s": 30}]' + - SEQ__DATABASE__DATABASE=postgres://postgres:postgres@sequencer-db:5432/sequencer?sslmode=disable + - SEQ__APP__BATCH_INSERTION_TIMEOUT=30s + - SEQ__APP__BATCH_DELETION_TIMEOUT=1s +# signup-sequencer-1: +# <<: *signup-sequencer-def +# container_name: signup-sequencer-1 +# ports: +# - "9081:8080" +# signup-sequencer-2: +# <<: *signup-sequencer-def +# container_name: signup-sequencer-2 +# ports: +# - "9082:8080" +# signup-sequencer-3: +# <<: *signup-sequencer-def +# container_name: signup-sequencer-3 +# ports: +# - "9083:8080" +volumes: + tx_sitter_db_data: + driver: local + sequencer_db_data: + driver: local \ No newline at end of file diff --git a/e2e_tests/docker-compose/haproxy/haproxy.cfg b/e2e_tests/docker-compose/haproxy/haproxy.cfg new file mode 100644 index 00000000..16533b6e --- /dev/null +++ b/e2e_tests/docker-compose/haproxy/haproxy.cfg @@ -0,0 +1,10 @@ +frontend http-in + bind *:8080 + default_backend http_back + +backend http_back + balance roundrobin + server signup-sequencer-0 signup-sequencer-0:8080 check +# server signup-sequencer-1 signup-sequencer-1:8080 check +# server signup-sequencer-2 signup-sequencer-2:8080 check +# server signup-sequencer-3 signup-sequencer-3:8080 check diff --git a/e2e_tests/docker-compose/signup_sequencer/config.toml b/e2e_tests/docker-compose/signup_sequencer/config.toml new file mode 100644 index 00000000..7c5f302a --- /dev/null +++ b/e2e_tests/docker-compose/signup_sequencer/config.toml @@ -0,0 +1,28 @@ +[tree] +tree_depth = 30 +dense_tree_prefix_depth = 10 +tree_gc_threshold = 10000000 +cache_file = "./cache_file" + +[server] +address = "0.0.0.0:8080" + +[network] +identity_manager_address = "0x48483748eb0446A16cAE79141D0688e3F624Cb73" + +[relayer] +kind = "tx_sitter" +tx_sitter_url = "http://tx-sitter:3000/1/api/G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH" +tx_sitter_address = "0x1d7ffed610cc4cdC097ecDc835Ae5FEE93C9e3Da" +tx_sitter_gas_limit = 2000000 + +[providers] +primary_network_provider = "http://chain:8545" + +[app] +provers_urls = '[{"url": "http://semaphore-insertion:3001", "prover_type": "insertion", "batch_size": 10,"timeout_s": 30}, {"url": "http://semaphore-deletion:3001", "prover_type": "deletion", "batch_size": 10,"timeout_s": 30}]' +batch_insertion_timeout = "30s" +batch_deletion_timeout = "1s" + +[database] +database = "postgres://postgres:postgres@sequencer-db:5432/sequencer?sslmode=disable" diff --git a/e2e_tests/scenarios/Cargo.toml b/e2e_tests/scenarios/Cargo.toml new file mode 100644 index 00000000..ab3198b8 --- /dev/null +++ b/e2e_tests/scenarios/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "e2e-tests" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = "1.0" +ethers = { version = "2.0.10" } +hex = "0.4.3" +hyper = { version = "^0.14.17", features = ["tcp", "http1", "http2", "client"] } +rand = "0.8.5" +retry = "2.0.0" +serde_json = "1.0" +tokio = { version = "1.0", features = ["full"] } +tracing = "0.1" +tracing-futures = "0.2" +tracing-subscriber = "0.3.11" +tracing-test = "0.2" diff --git a/e2e_tests/scenarios/tests/common/api.rs b/e2e_tests/scenarios/tests/common/api.rs new file mode 100644 index 00000000..14510873 --- /dev/null +++ b/e2e_tests/scenarios/tests/common/api.rs @@ -0,0 +1,141 @@ +use std::time::Duration; + +use anyhow::Error; +use hyper::client::HttpConnector; +use hyper::{Body, Client, Request}; +use serde_json::{json, Value}; +use tracing::debug; + +use crate::common::prelude::StatusCode; + +pub struct RawResponse { + pub status_code: StatusCode, + pub body: String, +} + +pub async fn insert_identity( + client: &Client, + uri: &String, + commitment: &String, +) -> anyhow::Result<()> { + debug!("Calling /insertIdentity"); + let body = Body::from( + json!({ + "identityCommitment": format!("0x{}", commitment), + }) + .to_string(), + ); + + let req = Request::builder() + .method("POST") + .uri(uri.to_owned() + "/insertIdentity") + .header("Content-Type", "application/json") + .body(body) + .expect("Failed to create insert identity hyper::Body"); + + let mut response = client + .request(req) + .await + .expect("Failed to execute request."); + let bytes = hyper::body::to_bytes(response.body_mut()) + .await + .expect("Failed to convert response body to bytes"); + if !response.status().is_success() { + return Err(Error::msg(format!( + "Failed to insert identity: response = {}", + response.status() + ))); + } + + assert!(bytes.is_empty()); + + Ok(()) +} + +pub async fn delete_identity( + client: &Client, + uri: &String, + commitment: &String, +) -> anyhow::Result<()> { + debug!("Calling /deleteIdentity"); + let body = Body::from( + json!({ + "identityCommitment": format!("0x{}", commitment), + }) + .to_string(), + ); + + let req = Request::builder() + .method("POST") + .uri(uri.to_owned() + "/deleteIdentity") + .header("Content-Type", "application/json") + .body(body) + .expect("Failed to create delete identity hyper::Body"); + + let mut response = client + .request(req) + .await + .expect("Failed to execute request."); + let bytes = hyper::body::to_bytes(response.body_mut()) + .await + .expect("Failed to convert response body to bytes"); + if !response.status().is_success() { + return Err(Error::msg(format!( + "Failed to delete identity: response = {}", + response.status() + ))); + } + + assert!(bytes.is_empty()); + + Ok(()) +} + +pub async fn inclusion_proof_raw( + client: &Client, + uri: &String, + commitment: &String, +) -> anyhow::Result { + debug!("Calling /inclusionProof"); + let body = Body::from( + json!({ + "identityCommitment": format!("0x{}", commitment), + }) + .to_string(), + ); + + let req = Request::builder() + .method("POST") + .uri(uri.to_owned() + "/inclusionProof") + .header("Content-Type", "application/json") + .body(body) + .expect("Failed to create inclusion proof hyper::Body"); + + let mut response = client + .request(req) + .await + .expect("Failed to execute request."); + let bytes = hyper::body::to_bytes(response.body_mut()) + .await + .expect("Failed to convert response body to bytes"); + let result = String::from_utf8(bytes.into_iter().collect()) + .expect("Could not parse response bytes to utf-8"); + + Ok(RawResponse { + status_code: response.status(), + body: result, + }) +} + +pub async fn inclusion_proof( + client: &Client, + uri: &String, + commitment: &String, +) -> anyhow::Result { + let result = inclusion_proof_raw(client, uri, commitment).await?; + + let result_json = + serde_json::from_str::(&result.body).expect("Failed to parse response as json"); + + Ok(result_json) +} diff --git a/e2e_tests/scenarios/tests/common/docker_compose.rs b/e2e_tests/scenarios/tests/common/docker_compose.rs new file mode 100644 index 00000000..b11ae9d3 --- /dev/null +++ b/e2e_tests/scenarios/tests/common/docker_compose.rs @@ -0,0 +1,271 @@ +use std::collections::HashMap; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::{Duration, Instant}; + +use anyhow::{Context, Error}; +use hyper::{Body, Client}; +use rand::distributions::Alphanumeric; +use rand::{thread_rng, Rng}; +use tracing::{debug, info}; +use tracing_subscriber::fmt::format; + +use crate::common::prelude::{Request, StatusCode}; + +const LOCAL_ADDR: &str = "localhost"; + +#[derive(Debug)] +pub struct DockerComposeGuard<'a> { + // Current working dir containing compose.yml + cwd: &'a str, + project_name: String, + chain_port: u32, + tx_sitter_db_port: u32, + sequencer_db_port: u32, + tx_sitter_port: u32, + semaphore_insertion_port: u32, + semaphore_deletion_port: u32, + signup_sequencer_0_port: u32, + signup_sequencer_balancer_port: u32, +} + +impl<'a> DockerComposeGuard<'a> { + pub fn get_local_addr(&self) -> String { + format!("{}:{}", LOCAL_ADDR, self.signup_sequencer_balancer_port) + } + + pub async fn restart_sequencer(&self) -> anyhow::Result<()> { + let (stdout, stderr) = run_cmd_to_output( + self.cwd, + self.envs_with_ports(), + self.generate_command("restart signup-sequencer-0"), + ) + .context("Restarting sequencer.")?; + + debug!( + "Docker compose rstart output:\n stdout:\n{}\nstderr:\n{}\n", + stdout, stderr + ); + + await_running(self).await + } + + fn envs_with_ports(&self) -> HashMap { + let mut res = HashMap::new(); + + res.insert(String::from("CHAIN_PORT"), self.chain_port.to_string()); + res.insert( + String::from("TX_SITTER_DB_PORT"), + self.tx_sitter_db_port.to_string(), + ); + res.insert( + String::from("SEQUENCER_DB_PORT"), + self.sequencer_db_port.to_string(), + ); + res.insert( + String::from("TX_SITTER_PORT"), + self.tx_sitter_port.to_string(), + ); + res.insert( + String::from("SEMAPHORE_INSERTION_PORT"), + self.semaphore_insertion_port.to_string(), + ); + res.insert( + String::from("SEMAPHORE_DELETION_PORT"), + self.semaphore_deletion_port.to_string(), + ); + res.insert( + String::from("SIGNUP_SEQUENCER_0_PORT"), + self.signup_sequencer_0_port.to_string(), + ); + res.insert( + String::from("SIGNUP_SEQUENCER_BALANCER_PORT"), + self.signup_sequencer_balancer_port.to_string(), + ); + + res + } + + fn generate_command(&self, args: &str) -> String { + format!( + "docker compose -p {} --profile e2e-ha {}", + self.project_name, args + ) + } + + fn update_balancer_port(&mut self, signup_sequencer_balancer_port: u32) { + self.signup_sequencer_balancer_port = signup_sequencer_balancer_port + } +} + +impl<'a> Drop for DockerComposeGuard<'a> { + fn drop(&mut self) { + // May run when compose is not up but better to be sure its down. + // Parameter '-v' is removing all volumes and networks. + if let Err(err) = run_cmd( + self.cwd, + self.envs_with_ports(), + self.generate_command("down -v"), + ) { + eprintln!("Failed to put down docker compose: {}", err); + } + } +} + +/// Starts a docker compose infrastructure. It will be stopped and removed when +/// the guard is dropped. +/// +/// Note that we're using sync code here so we'll block the executor - but this +/// is fine, because the spawned container will still run in the background. +pub async fn setup(cwd: &str) -> anyhow::Result { + let mut res = DockerComposeGuard { + cwd, + project_name: generate_project_name(), + chain_port: 0, + tx_sitter_db_port: 0, + sequencer_db_port: 0, + tx_sitter_port: 0, + semaphore_insertion_port: 0, + semaphore_deletion_port: 0, + signup_sequencer_0_port: 0, + signup_sequencer_balancer_port: 0, + }; + + debug!("Configuration: {:#?}", res); + + let (stdout, stderr) = run_cmd_to_output( + res.cwd, + res.envs_with_ports(), + res.generate_command("up -d"), + ) + .context("Starting e2e test docker compose infrastructure.")?; + + debug!( + "Docker compose starting output:\n stdout:\n{}\nstderr:\n{}\n", + stdout, stderr + ); + + tokio::time::sleep(Duration::from_secs(1)).await; + + let (stdout, stderr) = run_cmd_to_output( + res.cwd, + res.envs_with_ports(), + res.generate_command("port signup-sequencer-balancer 8080"), + ) + .context("Looking for balancer selected port.")?; + + debug!( + "Docker compose starting output:\n stdout:\n{}\nstderr:\n{}\n", + stdout, stderr + ); + + let balancer_port = parse_exposed_port(stdout); + res.update_balancer_port(balancer_port); + + _ = await_running(&res).await?; + + return Ok(res); +} + +fn generate_project_name() -> String { + let charset: &[u8] = b"abcdefghijklmnopqrstuvwxyz"; + let mut rng = rand::thread_rng(); + (0..8) + .map(|_| { + let idx = rng.gen_range(0..charset.len()); + charset[idx] as char + }) + .collect() +} + +async fn await_running(docker_compose_guard: &DockerComposeGuard<'_>) -> anyhow::Result<()> { + let timeout = Duration::from_secs_f32(30.0); + let check_interval = Duration::from_secs_f32(2.0); + let min_success_counts = 5; + let mut success_counter = 0; + + let timer = Instant::now(); + loop { + let healthy = check_health(docker_compose_guard.get_local_addr()).await; + if healthy.is_ok() && healthy.unwrap() { + success_counter = success_counter + 1; + } + + if success_counter >= min_success_counts { + return Ok(()); + } + + if timer.elapsed() > timeout { + return Err(Error::msg("Timed out waiting for healthcheck.")); + } + + tokio::time::sleep(check_interval).await; + } +} + +async fn check_health(local_addr: String) -> anyhow::Result { + let uri = format!("http://{}", local_addr); + let client = Client::new(); + + let healthcheck = Request::builder() + .method("GET") + .uri(format!("{uri}/health")) + .header("Content-Type", "application/json") + .body(Body::empty()) + .unwrap(); + + let response = client.request(healthcheck).await?; + + return Ok(response.status() == StatusCode::OK); +} + +fn run_cmd_to_output( + cwd: &str, + envs: HashMap, + cmd_str: String, +) -> anyhow::Result<(String, String)> { + let args: Vec<_> = cmd_str.split(' ').collect(); + let mut command = Command::new(args[0]); + + for arg in &args[1..] { + command.arg(arg); + } + + command + .current_dir(cwd) + .envs(envs) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + let output = command + .output() + .expect(&format!("Failed to run command: {}", cmd_str)); + + let stdout_utf = String::from_utf8(output.stdout)?; + let stderr_utf = String::from_utf8(output.stderr)?; + + Ok((stdout_utf.trim().to_string(), stderr_utf.trim().to_string())) +} + +fn run_cmd(cwd: &str, envs: HashMap, cmd_str: String) -> anyhow::Result<()> { + run_cmd_to_output(cwd, envs, cmd_str)?; + + Ok(()) +} + +fn parse_exposed_port(s: String) -> u32 { + let parts: Vec<_> = s + .split_whitespace() + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .collect(); + + parts + .last() + .unwrap() + .split(":") + .last() + .unwrap() + .parse::() + .unwrap() +} diff --git a/e2e_tests/scenarios/tests/common/mod.rs b/e2e_tests/scenarios/tests/common/mod.rs new file mode 100644 index 00000000..7f034fdd --- /dev/null +++ b/e2e_tests/scenarios/tests/common/mod.rs @@ -0,0 +1,83 @@ +// We include this module in multiple in multiple integration +// test crates - so some code may not be used in some cases +#![allow(dead_code, clippy::too_many_arguments, unused_imports)] + +use ethers::types::U256; +use tracing::error; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::fmt::time::Uptime; + +mod api; +pub mod docker_compose; + +#[allow(unused)] +pub mod prelude { + pub use std::time::Duration; + + pub use anyhow::{Context, Error}; + pub use hyper::client::HttpConnector; + pub use hyper::{Body, Client, Request, StatusCode}; + pub use retry::delay::Fixed; + pub use retry::retry; + pub use serde_json::json; + pub use tokio::spawn; + pub use tokio::task::JoinHandle; + pub use tracing::{error, info, instrument}; + pub use tracing_subscriber::fmt::format; + + pub use super::{generate_test_commitments, init_tracing_subscriber}; + pub use crate::common::api::{ + delete_identity, inclusion_proof, inclusion_proof_raw, insert_identity, + }; +} + +/// Initializes the tracing subscriber. +/// +/// Set the `QUIET_MODE` environment variable to reduce the complexity of the +/// log output. +pub fn init_tracing_subscriber() { + let quiet_mode = std::env::var("QUIET_MODE").is_ok(); + let rust_log = std::env::var("RUST_LOG").unwrap_or("info".to_string()); + let result = if quiet_mode { + tracing_subscriber::fmt() + .with_env_filter(rust_log) + .compact() + .with_timer(Uptime::default()) + .try_init() + } else { + tracing_subscriber::fmt() + .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE) + .with_line_number(true) + .with_env_filter(rust_log) + .with_timer(Uptime::default()) + // .pretty() + .try_init() + }; + if let Err(error) = result { + error!(error, "Failed to initialize tracing_subscriber"); + } +} + +/// Generates identities for the purposes of testing. The identities are encoded +/// in hexadecimal as a string but without the "0x" prefix as required by the +/// testing utilities. +/// +/// # Note +/// This utilises a significantly smaller portion of the 256-bit identities than +/// would be used in reality. This is both to make them easier to generate and +/// to ensure that we do not run afoul of the element numeric limit for the +/// snark scalar field. +pub fn generate_test_commitments(count: usize) -> Vec { + let mut commitments = vec![]; + + for _ in 0..count { + // Generate the identities using the just the last 64 bits (of 256) has so we're + // guaranteed to be less than SNARK_SCALAR_FIELD. + let bytes: [u8; 32] = U256::from(rand::random::()).into(); + let identity_string: String = hex::encode(bytes); + + commitments.push(identity_string); + } + + commitments +} diff --git a/e2e_tests/scenarios/tests/insert_1k.rs b/e2e_tests/scenarios/tests/insert_1k.rs new file mode 100644 index 00000000..5d68badc --- /dev/null +++ b/e2e_tests/scenarios/tests/insert_1k.rs @@ -0,0 +1,83 @@ +mod common; + +use common::prelude::*; +use serde_json::Value; +use tokio::time::sleep; + +use crate::common::docker_compose; + +#[tokio::test] +async fn insert_1k() -> anyhow::Result<()> { + // Initialize logging for the test. + init_tracing_subscriber(); + info!("Starting e2e test"); + + let docker_compose = docker_compose::setup("./../docker-compose").await?; + + let uri = format!("http://{}", docker_compose.get_local_addr()); + let client = Client::new(); + + let identities = generate_test_commitments(1000); + + for commitment in identities.iter() { + _ = insert_identity_with_retries(&client, &uri, commitment, 10, 3.0).await?; + } + + for commitment in identities.iter() { + _ = mined_inclusion_proof_with_retries(&client, &uri, commitment, 60, 10.0).await?; + } + + Ok(()) +} + +async fn insert_identity_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result<()> { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = insert_identity(&client, &uri, &commitment).await; + + if last_res.is_ok() { + break; + } + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + if let Err(err) = last_res { + return Err(err); + }; + + last_res +} + +async fn mined_inclusion_proof_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = inclusion_proof(&client, &uri, &commitment).await; + + if let Ok(ref inclusion_proof_json) = last_res { + if inclusion_proof_json["status"] == "mined" { + break; + } + }; + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + let inclusion_proof_json = last_res?; + + assert_eq!(inclusion_proof_json["status"], "mined"); + + Ok(inclusion_proof_json) +} diff --git a/e2e_tests/scenarios/tests/insert_delete_insert.rs b/e2e_tests/scenarios/tests/insert_delete_insert.rs new file mode 100644 index 00000000..c56dae40 --- /dev/null +++ b/e2e_tests/scenarios/tests/insert_delete_insert.rs @@ -0,0 +1,155 @@ +mod common; + +use common::prelude::*; +use serde_json::Value; +use tokio::time::sleep; +use tracing::debug; +use tracing::field::debug; + +use crate::common::docker_compose; + +#[tokio::test] +async fn insert_delete_insert() -> anyhow::Result<()> { + // Initialize logging for the test. + init_tracing_subscriber(); + info!("Starting e2e test"); + + let docker_compose = docker_compose::setup("./../docker-compose").await?; + + let uri = format!("http://{}", docker_compose.get_local_addr()); + let client = Client::new(); + + let identities = generate_test_commitments(10); + + for commitment in identities.iter() { + _ = insert_identity_with_retries(&client, &uri, commitment, 10, 3.0).await?; + } + + for commitment in identities.iter() { + _ = mined_inclusion_proof_with_retries(&client, &uri, commitment, 60, 10.0).await?; + } + + let first_commitment = identities.first().unwrap(); + + _ = delete_identity_with_retries(&client, &uri, &first_commitment, 10, 3.0).await?; + _ = bad_request_inclusion_proof_with_retries(&client, &uri, &first_commitment, 60, 10.0) + .await?; + + let new_identities = generate_test_commitments(10); + + for commitment in new_identities.iter() { + _ = insert_identity_with_retries(&client, &uri, commitment, 10, 3.0).await?; + } + + for commitment in new_identities.iter() { + _ = mined_inclusion_proof_with_retries(&client, &uri, commitment, 60, 10.0).await?; + } + + Ok(()) +} + +async fn insert_identity_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result<()> { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = insert_identity(&client, &uri, &commitment).await; + + if last_res.is_ok() { + break; + } + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + if let Err(err) = last_res { + return Err(err); + }; + + last_res +} + +async fn mined_inclusion_proof_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = inclusion_proof(&client, &uri, &commitment).await; + + if let Ok(ref inclusion_proof_json) = last_res { + if inclusion_proof_json["status"] == "mined" { + break; + } + }; + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + let inclusion_proof_json = last_res?; + + assert_eq!(inclusion_proof_json["status"], "mined"); + + Ok(inclusion_proof_json) +} + +async fn bad_request_inclusion_proof_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result<()> { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = inclusion_proof_raw(&client, &uri, &commitment).await; + + if let Ok(ref response) = last_res { + if response.status_code == StatusCode::BAD_REQUEST { + break; + } + } else { + error!("error: {}", last_res.as_ref().err().unwrap()); + } + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + let response = last_res?; + + assert_eq!(response.status_code, StatusCode::BAD_REQUEST); + + Ok(()) +} + +async fn delete_identity_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result<()> { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = delete_identity(&client, &uri, &commitment).await; + + if last_res.is_ok() { + break; + } + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + if let Err(err) = last_res { + return Err(err); + }; + + last_res +} diff --git a/e2e_tests/scenarios/tests/insert_restart_insert.rs b/e2e_tests/scenarios/tests/insert_restart_insert.rs new file mode 100644 index 00000000..b6988d3a --- /dev/null +++ b/e2e_tests/scenarios/tests/insert_restart_insert.rs @@ -0,0 +1,151 @@ +mod common; + +use common::prelude::*; +use serde_json::Value; +use tokio::time::sleep; +use tracing::debug; +use tracing::field::debug; + +use crate::common::docker_compose; + +#[tokio::test] +async fn insert_restart_insert() -> anyhow::Result<()> { + // Initialize logging for the test. + init_tracing_subscriber(); + info!("Starting e2e test"); + + let docker_compose = docker_compose::setup("./../docker-compose").await?; + + let uri = format!("http://{}", docker_compose.get_local_addr()); + let client = Client::new(); + + let identities = generate_test_commitments(10); + + for commitment in identities.iter() { + _ = insert_identity_with_retries(&client, &uri, commitment, 10, 3.0).await?; + } + + for commitment in identities.iter() { + _ = mined_inclusion_proof_with_retries(&client, &uri, commitment, 60, 10.0).await?; + } + + _ = docker_compose.restart_sequencer().await?; + + let identities = generate_test_commitments(10); + + for commitment in identities.iter() { + _ = insert_identity_with_retries(&client, &uri, commitment, 10, 3.0).await?; + } + + for commitment in identities.iter() { + _ = mined_inclusion_proof_with_retries(&client, &uri, commitment, 60, 10.0).await?; + } + + Ok(()) +} + +async fn insert_identity_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result<()> { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = insert_identity(&client, &uri, &commitment).await; + + if last_res.is_ok() { + break; + } + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + if let Err(err) = last_res { + return Err(err); + }; + + last_res +} + +async fn mined_inclusion_proof_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = inclusion_proof(&client, &uri, &commitment).await; + + if let Ok(ref inclusion_proof_json) = last_res { + if inclusion_proof_json["status"] == "mined" { + break; + } + }; + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + let inclusion_proof_json = last_res?; + + assert_eq!(inclusion_proof_json["status"], "mined"); + + Ok(inclusion_proof_json) +} + +async fn bad_request_inclusion_proof_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result<()> { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = inclusion_proof_raw(&client, &uri, &commitment).await; + + if let Ok(ref response) = last_res { + if response.status_code == StatusCode::BAD_REQUEST { + break; + } + } else { + error!("error: {}", last_res.as_ref().err().unwrap()); + } + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + let response = last_res?; + + assert_eq!(response.status_code, StatusCode::BAD_REQUEST); + + Ok(()) +} + +async fn delete_identity_with_retries( + client: &Client, + uri: &String, + commitment: &String, + retries_count: usize, + retries_interval: f32, +) -> anyhow::Result<()> { + let mut last_res = Err(Error::msg("No calls at all")); + for _i in 0..retries_count { + last_res = delete_identity(&client, &uri, &commitment).await; + + if last_res.is_ok() { + break; + } + + _ = sleep(Duration::from_secs_f32(retries_interval)).await; + } + + if let Err(err) = last_res { + return Err(err); + }; + + last_res +}