High Availability - sync over DB, multiple process instances support

Dockerfile

@@ -1,4 +1,5 @@
 FROM debian:12 as build-env
+ARG BUILD_DEV
 
 WORKDIR /src
 
@@ -20,11 +21,16 @@ ENV PATH="/root/.cargo/bin:${PATH}"
 ENV RUSTUP_HOME="/root/.rustup"
 ENV CARGO_HOME="/root/.cargo"
 
+ENV CARGO_ARGS=${BUILD_DEV:+}
+ENV CARGO_ARGS=${CARGO_ARGS:---release}
+
+RUN echo "CARGO_ARGS: ${CARGO_ARGS}"
+
 # Install the toolchain
 RUN rustup component add cargo
 
 # Build the sequencer
-RUN cargo build --release
+RUN cargo build ${CARGO_ARGS}
 
 # cc variant because we need libgcc and others
 FROM gcr.io/distroless/cc-debian12:nonroot
@@ -36,7 +42,10 @@ LABEL prometheus.io/scrape="true"
 LABEL prometheus.io/port="9998"
 LABEL prometheus.io/path="/metrics"
 
+ENV BIN_PATH=${BUILD_DEV:+/src/target/debug/signup-sequencer}
+ENV BIN_PATH=${BIN_PATH:-/src/target/release/signup-sequencer}
+
 # Copy the sequencer binary
-COPY --from=build-env --chown=0:10001 --chmod=454 /src/target/release/signup-sequencer /bin/signup-sequencer
+COPY --from=build-env --chown=0:10001 --chmod=454 ${BIN_PATH} /bin/signup-sequencer
 
 ENTRYPOINT [ "/bin/signup-sequencer" ]

e2e_tests/compose.yml

@@ -0,0 +1,206 @@
+version: "3"
+services:
+  chain:
+    container_name: chain
+    image: ghcr.io/foundry-rs/foundry
+    platform: linux/amd64
+    ports:
+      - "8545:8545"
+    command: [ "anvil --host 0.0.0.0 --chain-id 31337 --block-time 30 --base-fee 0 --gas-limit 0 --gas-price 0 --fork-url https://eth-sepolia.g.alchemy.com/v2/Hkj3vTy6ee49NbI4Imhe6r5mRM1vkR10@5091094" ]
+  tx-sitter-db:
+    container_name: tx-sitter-db
+    image: postgres:latest
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=tx-sitter
+    ports:
+      - "5460:5432"
+    volumes:
+      - tx_sitter_db_data:/var/lib/postgresql/data
+  sequencer-db:
+    container_name: sequencer-db
+    image: postgres:latest
+    environment:
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=postgres
+      - POSTGRES_DB=sequencer
+    ports:
+      - "5461:5432"
+    volumes:
+      - sequencer_db_data:/var/lib/postgresql/data
+  tx-sitter:
+    container_name: tx-sitter
+    image: tx-sitter-monolith
+    depends_on:
+      - tx-sitter-db
+      - chain
+    restart: always
+    ports:
+      - "3000:3000"
+    environment:
+      - TX_SITTER__SERVICE__ESCALATION_INTERVAL=3s
+      - TX_SITTER__DATABASE__KIND=connection_string
+      - TX_SITTER__DATABASE__CONNECTION_STRING=postgres://postgres:postgres@tx-sitter-db:5432/tx-sitter?sslmode=disable
+      - TX_SITTER__KEYS__KIND=local
+      - TX_SITTER__SERVICE__PREDEFINED__NETWORK__CHAIN_ID=31337
+      - TX_SITTER__SERVICE__PREDEFINED__NETWORK__NAME=Anvil
+      - TX_SITTER__SERVICE__PREDEFINED__NETWORK__HTTP_RPC=http://chain:8545
+      - TX_SITTER__SERVICE__PREDEFINED__NETWORK__WS_RPC=ws://chain:8545
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__ID=1b908a34-5dc1-4d2d-a146-5eb46e975830
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__NAME=Relayer
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__CHAIN_ID=31337
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__KEY_ID=d10607662a85424f02a33fb1e6d095bd0ac7154396ff09762e41f82ff2233aaa
+      - TX_SITTER__SERVICE__PREDEFINED__RELAYER__API_KEY=G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH
+      - TX_SITTER__SERVER__HOST=0.0.0.0:3000
+      - TX_SITTER__SERVER__DISABLE_AUTH=true
+      - RUST_LOG=info
+  semaphore-insertion:
+    container_name: semaphore-insertion
+    image: semaphore-mtb
+    hostname: semaphore-insertion
+    restart: always
+    command: [ "start", "--keys-file", "/mtb/keys", "--prover-address", "0.0.0.0:3001", "--mode", "insertion" ]
+    volumes:
+      - ./keys/insertion_b10t30.ps:/mtb/keys
+    environment:
+      BATCH_TIMEOUT_SECONDS: 1
+  semaphore-deletion:
+    container_name: semaphore-deletion
+    image: semaphore-mtb
+    hostname: semaphore-deletion
+    restart: always
+    command: [ "start", "--keys-file", "/mtb/keys", "--prover-address", "0.0.0.0:3001", "--mode", "deletion" ]
+    volumes:
+      - ./keys/deletion_b10t30.ps:/mtb/keys
+    environment:
+      BATCH_DELETION_TIMEOUT_SECONDS: 1
+  signup-sequencer-0:
+    container_name: signup-sequencer-0
+    image: signup-sequencer
+    build:
+      context: ./../
+      args:
+        BUILD_DEV: 1
+    depends_on:
+      - sequencer-db
+      - chain
+      - semaphore-insertion
+      - semaphore-deletion
+    restart: always
+    ports:
+      - "9080:8080"
+    environment:
+      - SEQ__TREE__TREE_DEPTH=30
+      - SEQ__TREE__DENSE_TREE_PREFIX_DEPTH=22
+      - SEQ__TREE__TREE_GC_THRESHOLD=10000000
+      - SEQ__TREE__CACHE_FILE=./cache_file
+      - SEQ__SERVER__ADDRESS=0.0.0.0:8080
+      - SEQ__NETWORK__IDENTITY_MANAGER_ADDRESS=0x48483748eb0446A16cAE79141D0688e3F624Cb73
+      - SEQ__RELAYER__KIND=tx_sitter
+      - SEQ__RELAYER__TX_SITTER_URL=http://tx-sitter:3000/1/api/G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH
+      - SEQ__RELAYER__TX_SITTER_ADDRESS=0x1d7ffed610cc4cdC097ecDc835Ae5FEE93C9e3Da
+      - SEQ__RELAYER__TX_SITTER_GAS_LIMIT=2000000
+      - SEQ__PROVIDERS__PRIMARY_NETWORK_PROVIDER=http://chain:8545
+      - 'SEQ__APP__PROVERS_URLS=[{"url": "http://semaphore-insertion:3001", "prover_type": "insertion", "batch_size": 10,"timeout_s": 30}, {"url": "http://semaphore-deletion:3001", "prover_type": "deletion", "batch_size": 10,"timeout_s": 30}]'
+      - SEQ__DATABASE__DATABASE=postgres://postgres:postgres@sequencer-db:5432/sequencer?sslmode=disable
+      - SEQ__APP__BATCH_INSERTION_TIMEOUT=30s
+      - SEQ__APP__BATCH_DELETION_TIMEOUT_SECONDS=1s
+  signup-sequencer-1:
+    container_name: signup-sequencer-1
+    image: signup-sequencer
+    build:
+      context: ./../
+      args:
+        BUILD_DEV: 1
+    depends_on:
+      - sequencer-db
+      - chain
+      - semaphore-insertion
+      - semaphore-deletion
+    restart: always
+    ports:
+      - "9081:8080"
+    environment:
+      - SEQ__TREE__TREE_DEPTH=30
+      - SEQ__TREE__DENSE_TREE_PREFIX_DEPTH=22
+      - SEQ__TREE__TREE_GC_THRESHOLD=10000000
+      - SEQ__TREE__CACHE_FILE=./cache_file
+      - SEQ__SERVER__ADDRESS=0.0.0.0:8080
+      - SEQ__NETWORK__IDENTITY_MANAGER_ADDRESS=0x48483748eb0446A16cAE79141D0688e3F624Cb73
+      - SEQ__RELAYER__KIND=tx_sitter
+      - SEQ__RELAYER__TX_SITTER_URL=http://tx-sitter:3000/1/api/G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH
+      - SEQ__RELAYER__TX_SITTER_ADDRESS=0x1d7ffed610cc4cdC097ecDc835Ae5FEE93C9e3Da
+      - SEQ__RELAYER__TX_SITTER_GAS_LIMIT=2000000
+      - SEQ__PROVIDERS__PRIMARY_NETWORK_PROVIDER=http://chain:8545
+      - 'SEQ__APP__PROVERS_URLS=[{"url": "http://semaphore-insertion:3001", "prover_type": "insertion", "batch_size": 10,"timeout_s": 30}, {"url": "http://semaphore-deletion:3001", "prover_type": "deletion", "batch_size": 10,"timeout_s": 30}]'
+      - SEQ__DATABASE__DATABASE=postgres://postgres:postgres@sequencer-db:5432/sequencer?sslmode=disable
+      - SEQ__APP__BATCH_INSERTION_TIMEOUT=30s
+      - SEQ__APP__BATCH_DELETION_TIMEOUT_SECONDS=1s
+  signup-sequencer-2:
+    container_name: signup-sequencer-2
+    image: signup-sequencer
+    build:
+      context: ./../
+      args:
+        BUILD_DEV: 1
+    depends_on:
+      - sequencer-db
+      - chain
+      - semaphore-insertion
+      - semaphore-deletion
+    restart: always
+    ports:
+      - "9082:8080"
+    environment:
+      - SEQ__TREE__TREE_DEPTH=30
+      - SEQ__TREE__DENSE_TREE_PREFIX_DEPTH=22
+      - SEQ__TREE__TREE_GC_THRESHOLD=10000000
+      - SEQ__TREE__CACHE_FILE=./cache_file
+      - SEQ__SERVER__ADDRESS=0.0.0.0:8080
+      - SEQ__NETWORK__IDENTITY_MANAGER_ADDRESS=0x48483748eb0446A16cAE79141D0688e3F624Cb73
+      - SEQ__RELAYER__KIND=tx_sitter
+      - SEQ__RELAYER__TX_SITTER_URL=http://tx-sitter:3000/1/api/G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH
+      - SEQ__RELAYER__TX_SITTER_ADDRESS=0x1d7ffed610cc4cdC097ecDc835Ae5FEE93C9e3Da
+      - SEQ__RELAYER__TX_SITTER_GAS_LIMIT=2000000
+      - SEQ__PROVIDERS__PRIMARY_NETWORK_PROVIDER=http://chain:8545
+      - 'SEQ__APP__PROVERS_URLS=[{"url": "http://semaphore-insertion:3001", "prover_type": "insertion", "batch_size": 10,"timeout_s": 30}, {"url": "http://semaphore-deletion:3001", "prover_type": "deletion", "batch_size": 10,"timeout_s": 30}]'
+      - SEQ__DATABASE__DATABASE=postgres://postgres:postgres@sequencer-db:5432/sequencer?sslmode=disable
+      - SEQ__APP__BATCH_INSERTION_TIMEOUT=30s
+      - SEQ__APP__BATCH_DELETION_TIMEOUT_SECONDS=1s
+  signup-sequencer-3:
+    container_name: signup-sequencer-3
+    image: signup-sequencer
+    build:
+      context: ./../
+      args:
+        BUILD_DEV: 1
+    depends_on:
+      - sequencer-db
+      - chain
+      - semaphore-insertion
+      - semaphore-deletion
+    restart: always
+    ports:
+      - "9083:8080"
+    environment:
+      - SEQ__TREE__TREE_DEPTH=30
+      - SEQ__TREE__DENSE_TREE_PREFIX_DEPTH=22
+      - SEQ__TREE__TREE_GC_THRESHOLD=10000000
+      - SEQ__TREE__CACHE_FILE=./cache_file
+      - SEQ__SERVER__ADDRESS=0.0.0.0:8080
+      - SEQ__NETWORK__IDENTITY_MANAGER_ADDRESS=0x48483748eb0446A16cAE79141D0688e3F624Cb73
+      - SEQ__RELAYER__KIND=tx_sitter
+      - SEQ__RELAYER__TX_SITTER_URL=http://tx-sitter:3000/1/api/G5CKNF3BTS2hRl60bpdYMNPqXvXsP-QZd2lrtmgctsnllwU9D3Z4D8gOt04M0QNH
+      - SEQ__RELAYER__TX_SITTER_ADDRESS=0x1d7ffed610cc4cdC097ecDc835Ae5FEE93C9e3Da
+      - SEQ__RELAYER__TX_SITTER_GAS_LIMIT=2000000
+      - SEQ__PROVIDERS__PRIMARY_NETWORK_PROVIDER=http://chain:8545
+      - 'SEQ__APP__PROVERS_URLS=[{"url": "http://semaphore-insertion:3001", "prover_type": "insertion", "batch_size": 10,"timeout_s": 30}, {"url": "http://semaphore-deletion:3001", "prover_type": "deletion", "batch_size": 10,"timeout_s": 30}]'
+      - SEQ__DATABASE__DATABASE=postgres://postgres:postgres@sequencer-db:5432/sequencer?sslmode=disable
+      - SEQ__APP__BATCH_INSERTION_TIMEOUT=30s
+      - SEQ__APP__BATCH_DELETION_TIMEOUT_SECONDS=1s
+volumes:
+  tx_sitter_db_data:
+    driver: local
+  sequencer_db_data:
+    driver: local

e2e_tests/create_identities.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+NUMBER=${1:-1}
+SLEEP=${2:-0}
+
+for run in $(seq $NUMBER); do
+  echo "running";
+  curl -X POST -H "Content-Type: application/json" -d "{\"identityCommitment\":\"0x$(openssl rand -hex 16)\"}" localhost:9080/insertIdentity -vv;
+  sleep $SLEEP
+done

e2e_tests/keys/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

schemas/database/013_batches_and_transactions.sql

@@ -0,0 +1,25 @@
+-- Create ENUM for prover type
+CREATE TYPE batch_type_enum AS ENUM ('Insertion', 'Deletion');
+
+CREATE TABLE batches
+(
+    next_root    BYTEA           NOT NULL UNIQUE PRIMARY KEY,
+    prev_root    BYTEA UNIQUE,
+    created_at   TIMESTAMPTZ     NOT NULL,
+    batch_type   batch_type_enum NOT NULL,
+    commitments  BYTEA[]         NOT NULL,
+    leaf_indexes BIGINT[]        NOT NULL CHECK (array_length(leaf_indexes, 1) = array_length(commitments, 1)),
+
+    FOREIGN KEY (prev_root) REFERENCES batches (next_root)
+);
+
+CREATE UNIQUE INDEX i_single_null_prev_root ON batches ((batches.prev_root IS NULL)) WHERE batches.prev_root IS NULL;
+
+CREATE TABLE transactions
+(
+    transaction_id  VARCHAR(256) NOT NULL UNIQUE PRIMARY KEY,
+    batch_next_root BYTEA        NOT NULL UNIQUE,
+    created_at      TIMESTAMPTZ  NOT NULL,
+
+    FOREIGN KEY (batch_next_root) REFERENCES batches (next_root)
+);

src/app.rs

@@ -259,8 +259,7 @@ impl App {
             processed_builder.update(&processed_item);
         }
 
-        let (processed, batching_builder) = processed_builder.seal_and_continue();
-        let (batching, mut latest_builder) = batching_builder.seal_and_continue();
+        let (processed, mut latest_builder) = processed_builder.seal_and_continue();
         let pending_items = self
             .database
             .get_commitments_by_status(ProcessedStatus::Pending)
@@ -269,7 +268,7 @@ impl App {
             latest_builder.update(&update);
         }
         let latest = latest_builder.seal();
-        Ok(Some(TreeState::new(mined, processed, batching, latest)))
+        Ok(Some(TreeState::new(mined, processed, latest)))
     }
 
     pub fn tree_state(&self) -> anyhow::Result<&TreeState> {
@@ -335,8 +334,7 @@ impl App {
         })
         .await?;
 
-        let (processed, batching_builder) = processed_builder.seal_and_continue();
-        let (batching, mut latest_builder) = batching_builder.seal_and_continue();
+        let (processed, mut latest_builder) = processed_builder.seal_and_continue();
 
         let pending_items = self
             .database
@@ -355,7 +353,7 @@ impl App {
 
         let latest = latest_builder.seal();
 
-        Ok(TreeState::new(mined, processed, batching, latest))
+        Ok(TreeState::new(mined, processed, latest))
     }
 
     /// Queues an insert into the merkle tree.
@@ -542,13 +540,15 @@ impl App {
                 // A pending identity can be present in the batching tree and therefore status
                 // should be set to Batched
                 IdentityHistoryEntryStatus::Pending => {
-                    if let Some(leaf_index) = entry.leaf_index {
-                        if self.tree_state()?.get_batching_tree().get_leaf(leaf_index)
-                            == entry.commitment
-                        {
-                            status = IdentityHistoryEntryStatus::Batched;
-                        }
-                    }
+                    // todo(piotrh): what to do with it?
+                    // if let Some(leaf_index) = entry.leaf_index {
+                    //     if self.tree_state()?.get_batching_tree().get_leaf(leaf_index)
+                    //         == entry.commitment
+                    //     {
+                    //         status = IdentityHistoryEntryStatus::Batched;
+                    //     }
+                    // }
+                    ()
                 }
                 IdentityHistoryEntryStatus::Buffered if entry.held_back => {
                     status = IdentityHistoryEntryStatus::Queued;
@@ -732,7 +732,6 @@ impl App {
     ) -> Result<(), ServerError> {
         let tree_state = self.tree_state()?;
         let latest_root = tree_state.get_latest_tree().get_root();
-        let batching_root = tree_state.get_batching_tree().get_root();
         let processed_root = tree_state.get_processed_tree().get_root();
         let mined_root = tree_state.get_mined_tree().get_root();
 
@@ -742,7 +741,7 @@ impl App {
 
         match root_state.status {
             // Pending status implies the batching or latest tree
-            ProcessedStatus::Pending if latest_root == root || batching_root == root => {
+            ProcessedStatus::Pending if latest_root == root => {
                 tracing::warn!("Root is pending - skipping");
                 return Ok(());
             }