E2E testing with docker compose

.github/workflows/test.yml

@@ -9,7 +9,7 @@ on:
     branches:
       - main
   pull_request:
-    types: [opened, synchronize, reopened]
+    types: [ opened, synchronize, reopened ]
     branches:
       - main
 
@@ -98,12 +98,12 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: nextest
-          args: run --workspace --no-run
+          args: run --workspace --exclude e2e-tests --no-run
       - name: Run tests
         uses: actions-rs/cargo@v1
         with:
           command: nextest
-          args: run --workspace
+          args: run --workspace --exclude e2e-tests
 
   cargo-vet:
     name: Vet Dependencies

.gitignore

@@ -8,3 +8,6 @@ commitments.json
 !.editorconfig
 !.dockerignore
 !.cargo
+
+e2e_tests/docker-compose/keys/*
+!e2e_tests/docker-compose/keys/.keep

Cargo.toml

@@ -16,7 +16,7 @@ license-file = "LICENSE.md"
 build = "build.rs"
 
 [workspace]
-members = ["crates/*"]
+members = ["crates/*", "e2e_tests/scenarios"]
 
 [features]
 default = []

Readme.md

@@ -24,15 +24,22 @@ Sequencer has 6 API routes.
        The database is polled every few seconds and added to insertion task.
     2. Processing: The processing of identities, where current batching tree is taken and processed so we
        end up with pre root (the root of tree before proofs are generated), post root, start index and
-       identity commitments (with their proofs). All of those get sent to a [prover](#semaphore-mtb) for proof generation.
-       The identities transaction is then mined, with aforementioned fields and pending identities are sent to task to be mined on-chain.
+       identity commitments (with their proofs). All of those get sent to a [prover](#semaphore-mtb) for proof
+       generation.
+       The identities transaction is then mined, with aforementioned fields and pending identities are sent to task to
+       be mined on-chain.
     3. Mining: The transaction ID from processing task gets mined and Sequencer database gets updated accordingly.
        Now with blockchain and database being in sync, the mined tree gets updated as well.
-2. `/inclusionProof` - Takes the identity commitment hash, and checks for any errors that might have occurred in the insert identity steps.
-   Then leaf index is fetched from the database, corresponding to the identity hash provided, and then we check if the identity is
+2. `/inclusionProof` - Takes the identity commitment hash, and checks for any errors that might have occurred in the
+   insert identity steps.
+   Then leaf index is fetched from the database, corresponding to the identity hash provided, and then we check if the
+   identity is
    indeed in the tree. The inclusion proof is then returned to the API caller.
-3. `/deleteIdentity` - Takes an identity commitment hash, ensures that it exists and hasn't been deleted yet. This identity is then scheduled for deletion.
-4. `/recoverIdentity` - Takes two identity commitment hashes. The first must exist and will be scheduled for deletion and the other will be inserted as a replacement after the first identity has been deleted and a set amount of time (depends on configuration parameters) has passed.
+3. `/deleteIdentity` - Takes an identity commitment hash, ensures that it exists and hasn't been deleted yet. This
+   identity is then scheduled for deletion.
+4. `/recoverIdentity` - Takes two identity commitment hashes. The first must exist and will be scheduled for deletion
+   and the other will be inserted as a replacement after the first identity has been deleted and a set amount of time (
+   depends on configuration parameters) has passed.
 5. `/verifySemaphoreProof` - This call takes root, signal hash, nullifier hash, external nullifier hash and a proof.
    The proving key is fetched based on the depth index, and verification key as well.
    The list of prime fields is created based on request input mentioned before, and then we proceed to verify the proof.
@@ -63,7 +70,8 @@ docker pull postgres
 
 ### Local Node
 
-You'll need to run a local node like geth or [ganache](https://archive.trufflesuite.com/ganache/). Start up a new chain and take note of the dev addresses. You can follow instructions [here](https://book.getfoundry.sh/anvil/).
+You'll need to run a local node like geth or [ganache](https://archive.trufflesuite.com/ganache/). Start up a new chain
+and take note of the dev addresses. You can follow instructions [here](https://book.getfoundry.sh/anvil/).
 
 ### Worldcoin id contracts
 
@@ -75,10 +83,12 @@ Clone [contracts-deployer](https://github.com/worldcoin/contract-deployer.git) a
 
 Semaphore-mtb is a service for batch processing of Merkle tree updates.
 
-Clone [semaphore-mtb](https://github.com/worldcoin/semaphore-mtb) and execute `go build .` (you will need a golang compiler)
+Clone [semaphore-mtb](https://github.com/worldcoin/semaphore-mtb) and execute `go build .` (you will need a golang
+compiler)
 
 Go build will create an executable named gnark-mbu. If you went through the contracts-deployer,
-you will have a generated a keys file that is used by semaphore-mtb. If your deployment contains more than one prover, then you must run this command for each one and configure them to listen on different ports.
+you will have a generated a keys file that is used by semaphore-mtb. If your deployment contains more than one prover,
+then you must run this command for each one and configure them to listen on different ports.
 
 ```shell
 ./gnark-mbu start --keys-file path/to/contracts-deployer/<DEPLOYMENT_NAME>/.cache/keys/<KEY_NAME> --mode <insertion/deletion>
@@ -102,7 +112,7 @@ Now you need to create a `config.toml` file for signup-sequencer:
 
 ```toml
 [app]
-provers_urls ='[]'
+provers_urls = '[]'
 
 [tree]
 
@@ -138,10 +148,27 @@ sudo ln -sf `pwd`/signup_sequencer_data /data
 ```
 
 And then run the daemon:
+
 ```shell
 cargo run config.toml
 ```
 
+### Docker compose
+
+Docker compose from E2E tests can also be used for local development. To run it first export alchemy API key
+for anvil fork to work:
+
+```shell
+export ALCHEMY_API_KEY=<api-key>
+```
+
+Then you can run docker compose (without signup sequencer):
+
+```shell
+cd e2e_tests/docker-compose
+docker compose up
+```
+
 ## Tests
 
 Lint, build, test
@@ -152,6 +179,21 @@ First ensure you have the docker daemon up and running, then:
 cargo fmt && cargo clippy --all-targets && cargo build --all-targets && cargo test --all-targets
 ```
 
+## E2E Tests
+
+Before running please make sure to build signup-sequencer image.
+
+```shell
+docker build -t signup-sequencer .
+```
+
+Then run tests. You need alchemy API key to run docker compose which is used by E2E tests.
+
+```shell
+export ALCHEMY_API_KEY=<api-key>
+cd e2e_tests/scenarios && cargo test
+```
+
 ## Contributing
 
 We welcome your pull requests! But also consider the following:

e2e_tests/docker-compose/compose.yml

@@ -0,0 +1,162 @@
+services:
+  chain:
+    image: ghcr.io/foundry-rs/foundry
+    hostname: chain
+    platform: linux/amd64
+    ports:
+      - ${CHAIN_PORT:-8545}:8545
+    command: [ "anvil --host 0.0.0.0 --chain-id 31337 --block-time 30 --base-fee 0 --gas-limit 0 --gas-price 0 --fork-url https://eth-sepolia.g.alchemy.com/v2/${ALCHEMY_API_KEY}@5091094" ]
+  tx-sitter-db:
+    image: postgres:latest
+    hostname: tx-sitter-db
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=tx-sitter
+    ports:
+      - ${TX_SITTER_DB_PORT:-5460}:5432
+    volumes:
+      - tx_sitter_db_data:/var/lib/postgresql/data
+  sequencer-db:
+    image: postgres:latest
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=sequencer
+    ports:
+      - ${SEQUENCER_DB_PORT:-5461}:5432
+    volumes:
+      - sequencer_db_data:/var/lib/postgresql/data
+  tx-sitter:
+    image: ghcr.io/worldcoin/tx-sitter-monolith:latest
+    hostname: tx-sitter
+    depends_on:
+      - tx-sitter-db
+      - chain
+    restart: always
+    ports:
+      - ${TX_SITTER_PORT:-3000}:3000
+    environment:
+      - RUST_LOG=info
+      - TX_SITTER__SERVICE__ESCALATION_INTERVAL=3s
+      - TX_SITTER__DATABASE__KIND=connection_string
+      - TX_SITTER__DATABASE__CONNECTION_STRING=postgres://postgres:postgres@tx-sitter-db:5432/tx-sitter?sslmode=disable
+      - TX_SITTER__KEYS__KIND=local
+      - TX_SITTER__SERVICE__PREDEFINED__NETWORK__CHAIN_ID=31337
+      - TX_SITTER__SERVICE__PREDEFINED__NETWORK__NAME=Anvil
+      - TX_SITTER__SERVICE__PREDEFINED__NETWORK__HTTP_RPC=http://chain:8545
+      - TX_SITTER__SERVICE__PREDEFINED__NETWORK__WS_RPC=ws://chain:8545
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__ID=1b908a34-5dc1-4d2d-a146-5eb46e975830
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__NAME=Relayer
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__CHAIN_ID=31337
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__KEY_ID=d10607662a85424f02a33fb1e6d095bd0ac7154396ff09762e41f82ff2233aaa
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__API_KEY=G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH
+      - TX_SITTER__SERVER__HOST=0.0.0.0:3000
+      - TX_SITTER__SERVER__DISABLE_AUTH=true
+  semaphore-keys-init-insertion:
+    image: curlimages/curl:latest
+    user: "0:0"
+    volumes:
+      - ./keys:/keys
+    entrypoint: /bin/sh
+    restart: on-failure
+    command:
+      - "-c"
+      - >
+        if [ ! -f "/keys/insertion_b10t30.ps" ]; then
+          curl "https://semaphore-mtb-trusted-setup-ceremony.s3.amazonaws.com/insertion_b10/insertion_b10t30.ps" -o /keys/insertion_b10t30.ps &&
+          chown -f $(stat -c "%u:%g" /keys/.keep) /keys/insertion_b10t30.ps;
+        fi
+  semaphore-keys-init-deletion:
+    image: curlimages/curl:latest
+    user: "0:0"
+    volumes:
+      - ./keys:/keys
+    entrypoint: /bin/sh
+    restart: on-failure
+    command:
+      - "-c"
+      - >
+        if [ ! -f "/keys/deletion_b10t30.ps" ]; then
+          curl "https://semaphore-mtb-trusted-setup-ceremony.s3.amazonaws.com/deletion_b10/deletion_b10t30.ps" -o /keys/deletion_b10t30.ps &&
+          chown -f $(stat -c "%u:%g" /keys/.keep) /keys/deletion_b10t30.ps;
+        fi
+  semaphore-insertion:
+    image: ghcr.io/worldcoin/semaphore-mtb:latest
+    hostname: semaphore-insertion
+    restart: always
+    ports:
+      - ${SEMAPHORE_INSERTION_PORT:-3001}:3001
+    command: [ "start", "--keys-file", "/mtb/keys/insertion_b10t30.ps", "--prover-address", "0.0.0.0:3001", "--mode", "insertion" ]
+    volumes:
+      - ./keys:/mtb/keys
+    environment:
+      BATCH_TIMEOUT_SECONDS: 1
+    depends_on:
+      semaphore-keys-init-insertion:
+        condition: service_completed_successfully
+  semaphore-deletion:
+    image: ghcr.io/worldcoin/semaphore-mtb:latest
+    hostname: semaphore-deletion
+    restart: always
+    ports:
+      - ${SEMAPHORE_DELETION_PORT:-3002}:3001
+    command: [ "start", "--keys-file", "/mtb/keys/deletion_b10t30.ps", "--prover-address", "0.0.0.0:3001", "--mode", "deletion" ]
+    volumes:
+      - ./keys:/mtb/keys
+    environment:
+      BATCH_DELETION_TIMEOUT_SECONDS: 1
+    depends_on:
+      semaphore-keys-init-deletion:
+        condition: service_completed_successfully
+  signup-sequencer-balancer:
+    image: haproxy:3.0.0
+    hostname: signup-sequencer-balancer
+    restart: always
+    profiles: [ e2e-ha ]
+    ports:
+      - ${SIGNUP_SEQUENCER_BALANCER_PORT:-8080}:8080
+    volumes:
+      - ./haproxy:/usr/local/etc/haproxy
+    depends_on:
+      - signup-sequencer-0
+  signup-sequencer-0: &signup-sequencer-def
+    image: signup-sequencer
+    hostname: signup-sequencer-0
+    profiles: [ e2e-ha ]
+    build:
+      context: ./../../
+    depends_on:
+      - sequencer-db
+      - chain
+      - semaphore-insertion
+      - semaphore-deletion
+      - tx-sitter
+    restart: always
+    ports:
+      - ${SIGNUP_SEQUENCER_0_PORT:-9080}:8080
+    volumes:
+      - ./signup_sequencer/config.toml:/config.toml
+    command: [ "/config.toml" ]
+    environment:
+      - RUST_LOG=debug
+#  signup-sequencer-1:
+#    <<: *signup-sequencer-def
+#    hostname: signup-sequencer-1
+#    ports:
+#      - ${SIGNUP_SEQUENCER_0_PORT:-9081}:8080
+#  signup-sequencer-2:
+#    <<: *signup-sequencer-def
+#    hostname: signup-sequencer-2
+#    ports:
+#      - ${SIGNUP_SEQUENCER_0_PORT:-9082}:8080
+#  signup-sequencer-3:
+#    <<: *signup-sequencer-def
+#    hostname: signup-sequencer-3
+#    ports:
+#      - ${SIGNUP_SEQUENCER_0_PORT:-9083}:8080
+volumes:
+  tx_sitter_db_data:
+    driver: local
+  sequencer_db_data:
+    driver: local

e2e_tests/docker-compose/haproxy/haproxy.cfg

@@ -0,0 +1,10 @@
+frontend http-in
+    bind *:8080
+    default_backend http_back
+
+backend http_back
+   balance roundrobin
+   server signup-sequencer-0 signup-sequencer-0:8080 check
+#    server signup-sequencer-1 signup-sequencer-1:8080 check
+#    server signup-sequencer-2 signup-sequencer-2:8080 check
+#    server signup-sequencer-3 signup-sequencer-3:8080 check

e2e_tests/docker-compose/signup_sequencer/config.toml

@@ -0,0 +1,28 @@
+[tree]
+tree_depth = 30
+dense_tree_prefix_depth = 10
+tree_gc_threshold = 10000000
+cache_file = "./cache_file"
+
+[server]
+address = "0.0.0.0:8080"
+
+[network]
+identity_manager_address = "0x48483748eb0446A16cAE79141D0688e3F624Cb73"
+
+[relayer]
+kind = "tx_sitter"
+tx_sitter_url = "http://tx-sitter:3000/1/api/G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH"
+tx_sitter_address = "0x1d7ffed610cc4cdC097ecDc835Ae5FEE93C9e3Da"
+tx_sitter_gas_limit = 2000000
+
+[providers]
+primary_network_provider = "http://chain:8545"
+
+[app]
+provers_urls = '[{"url": "http://semaphore-insertion:3001", "prover_type": "insertion", "batch_size": 10,"timeout_s": 30}, {"url": "http://semaphore-deletion:3001", "prover_type": "deletion", "batch_size": 10,"timeout_s": 30}]'
+batch_insertion_timeout = "30s"
+batch_deletion_timeout = "1s"
+
+[database]
+database = "postgres://postgres:postgres@sequencer-db:5432/sequencer?sslmode=disable"