From e89ec2ca908224e6d2b1800dff596ea0005ca920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Tr=C4=85d?= Date: Wed, 20 Sep 2023 20:19:37 +0200 Subject: [PATCH] Clean & optimize CI (#606) --- .cargo/config.toml | 6 - .github/workflows/test.yml | 97 ++--------- Cargo.lock | 165 +------------------ Cargo.toml | 55 ++++--- Readme.md | 85 +++++----- crates/micro-oz/Cargo.toml | 10 +- crates/oz-api/Cargo.toml | 10 +- src/task_monitor/tasks/process_identities.rs | 3 + tests/delete_identities.rs | 1 - tests/delete_padded_identity.rs | 1 - tests/dynamic_batch_sizes.rs | 1 - tests/insert_identity_and_proofs.rs | 1 - tests/recover_identities.rs | 1 - tests/validate_proof_with_age.rs | 12 +- 14 files changed, 115 insertions(+), 333 deletions(-) delete mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 467590bd..00000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,6 +0,0 @@ -[alias] -criterion = "bench --bench=criterion --features=bench,proptest" - -[build] -# Enable tokio-console support -# rustflags = ["--cfg", "tokio_unstable"] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0c44769c..317259ad 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,12 +63,12 @@ jobs: - uses: actions-rs/clippy-check@v1 with: token: ${{ secrets.GITHUB_TOKEN }} - args: --locked --features "bench, mimalloc" --all-targets + args: --locked --features "mimalloc" --all-targets - name: Check docs uses: actions-rs/cargo@v1 with: command: doc - args: --locked --features "bench, mimalloc" --no-deps --document-private-items + args: --locked --features "mimalloc" --no-deps --document-private-items test: name: Test @@ -99,92 +99,19 @@ jobs: uses: foundry-rs/foundry-toolchain@v1 with: version: nightly - # Cargo doc test is not included in `--all-targets` - # See - # Cargo doc test also doesn't support `--no-run`, so we run it but - # have it just print `--help`. + - name: Install latest nextest release + uses: taiki-e/install-action@nextest - name: Build tests - run: | - cargo test --locked --features "bench, mimalloc" --all-targets --no-run - cargo test --locked --features "bench, mimalloc" --doc -- --help - - name: Run tests - run: | - cargo test --locked --features "bench, mimalloc" --all-targets -- --nocapture - cargo test --locked --features "bench, mimalloc" --doc -- --nocapture - - codecov: - # See - name: Coverage - runs-on: ubuntu-latest - env: - RUSTFLAGS: -Cinstrument-coverage - RUSTDOCFLAGS: -C instrument-coverage -Z unstable-options --persist-doctests target/debug/doctestbins - LLVM_PROFILE_FILE: profile-%m.profraw - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Install rust - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ env.NIGHTLY_VERSION }} - override: true - components: llvm-tools-preview - - name: Install protobuf-compiler - run: sudo apt-get install -y protobuf-compiler - - name: Cache - uses: actions/cache@v3 - continue-on-error: false - with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - key: ${{ env.RUST_VERSION }}-${{ env.NIGHTLY_VERSION }}-cargo-cov-${{ hashFiles('**/Cargo.lock') }}-cov - restore-keys: ${{ env.RUST_VERSION }}-${{ env.NIGHTLY_VERSION }}-cargo-cov- - - name: Install cargo-binutils - run: cargo install cargo-binutils - - name: Install Foundry - uses: foundry-rs/foundry-toolchain@v1 + uses: actions-rs/cargo@v1 with: - version: nightly - - name: Build tests with coverage - run: | - cargo test --locked --features "bench, mimalloc" --all-targets --no-fail-fast --no-run - cargo test --locked --features "bench, mimalloc" --doc --no-fail-fast -- --help - - name: Run tests with coverage - run: | - cargo test --locked --features "bench, mimalloc" --all-targets --no-fail-fast -- --nocapture - cargo test --locked --features "bench, mimalloc" --doc --no-fail-fast - - name: Merge execution traces - run: cargo profdata -- merge -sparse $(find . -iname "profile-*.profraw") -o profile.profdata - - name: Export to lcov format for codecov - # See - run: - cargo cov -- export - --format=lcov > profile.lcov - --instr-profile=profile.profdata - $( - for file in - $( - cargo test --locked --features "bench, mimalloc" --all-targets - --no-fail-fast --no-run --message-format=json - | jq -r "select(.profile.test == true) | .filenames[]" - | grep -v dSYM - - ) - target/debug/doctestbins/*/rust_out; - do - [[ -x $file ]] && printf "%s %s " -object $file ; - done - ) - - name: Submit to codecov.io - uses: codecov/codecov-action@v3.1.1 + command: nextest + args: run --workspace --features "mimalloc" --no-run + - name: Run tests + uses: actions-rs/cargo@v1 with: - token: ${{ secrets.CODECOV_TOKEN }} # Optional for public repos - flags: test - fail_ci_if_error: true - verbose: true + command: nextest + args: run --workspace --features "mimalloc" + cargo-vet: name: Vet Dependencies runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index c0996924..b1ad1c3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -85,12 +85,6 @@ dependencies = [ "libc", ] -[[package]] -name = "anes" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" - [[package]] name = "ansi_term" version = "0.12.1" @@ -182,7 +176,7 @@ dependencies = [ "byteorder", "cfg-if", "color-eyre 0.5.11", - "criterion 0.3.6", + "criterion", "ethers-core 2.0.3", "fnv", "hex", @@ -1162,33 +1156,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "ciborium" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c137568cc60b904a7724001b35ce2630fd00d5d84805fbb608ab89509d788f" -dependencies = [ - "ciborium-io", - "ciborium-ll", - "serde", -] - -[[package]] -name = "ciborium-io" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346de753af073cc87b52b2083a506b38ac176a44cfb05497b622e27be899b369" - -[[package]] -name = "ciborium-ll" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213030a2b5a4e0c0892b6652260cf6ccac84827b83a85a534e178e3906c4cf1b" -dependencies = [ - "ciborium-io", - "half", -] - [[package]] name = "cipher" version = "0.3.0" @@ -1698,7 +1665,7 @@ dependencies = [ "atty", "cast", "clap 2.34.0", - "criterion-plot 0.4.5", + "criterion-plot", "csv", "itertools", "lazy_static", @@ -1715,34 +1682,6 @@ dependencies = [ "walkdir", ] -[[package]] -name = "criterion" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c76e09c1aae2bc52b3d2f29e13c6572553b30c4aa1b8a49fd70de6412654cb" -dependencies = [ - "anes", - "atty", - "cast", - "ciborium", - "clap 3.2.23", - "criterion-plot 0.5.0", - "futures", - "itertools", - "lazy_static", - "num-traits", - "oorandom", - "plotters", - "rayon", - "regex", - "serde", - "serde_derive", - "serde_json", - "tinytemplate", - "tokio", - "walkdir", -] - [[package]] name = "criterion-plot" version = "0.4.5" @@ -1753,16 +1692,6 @@ dependencies = [ "itertools", ] -[[package]] -name = "criterion-plot" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" -dependencies = [ - "cast", - "itertools", -] - [[package]] name = "crossbeam-channel" version = "0.5.6" @@ -4652,26 +4581,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "proptest" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0d9cc07f18492d879586c92b485def06bc850da3118075cd45d50e9c95b0e5" -dependencies = [ - "bit-set", - "bitflags", - "byteorder", - "lazy_static", - "num-traits", - "quick-error 2.0.1", - "rand", - "rand_chacha", - "rand_xorshift", - "regex-syntax 0.6.28", - "rusty-fork", - "tempfile", -] - [[package]] name = "prost" version = "0.11.9" @@ -4721,18 +4630,6 @@ dependencies = [ "syn 1.0.107", ] -[[package]] -name = "quick-error" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" - -[[package]] -name = "quick-error" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" - [[package]] name = "quote" version = "1.0.31" @@ -4784,15 +4681,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_xorshift" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" -dependencies = [ - "rand_core", -] - [[package]] name = "rayon" version = "1.7.0" @@ -5214,18 +5102,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97477e48b4cf8603ad5f7aaf897467cf42ab4218a38ef76fb14c2d6773a6d6a8" -[[package]] -name = "rusty-fork" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" -dependencies = [ - "fnv", - "quick-error 1.2.3", - "tempfile", - "wait-timeout", -] - [[package]] name = "ryu" version = "1.0.11" @@ -5548,31 +5424,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serial_test" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "538c30747ae860d6fb88330addbbd3e0ddbe46d662d032855596d8a8ca260611" -dependencies = [ - "dashmap", - "futures", - "lazy_static", - "log", - "parking_lot 0.12.1", - "serial_test_derive", -] - -[[package]] -name = "serial_test_derive" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "079a83df15f85d89a68d64ae1238f142f172b1fa915d0d76b26a7cba1b659a69" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.107", -] - [[package]] name = "sha-1" version = "0.10.0" @@ -5693,7 +5544,6 @@ dependencies = [ "clap 4.3.14", "cli-batteries", "cognitoauth", - "criterion 0.4.0", "ethers", "eyre", "futures", @@ -5707,14 +5557,12 @@ dependencies = [ "oz-api", "postgres-docker-utils", "prometheus", - "proptest", "regex", "reqwest", "ruint", "semaphore", "serde", "serde_json", - "serial_test", "sqlx", "take_mut", "tempfile", @@ -6830,15 +6678,6 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" -[[package]] -name = "wait-timeout" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" -dependencies = [ - "libc", -] - [[package]] name = "walkdir" version = "2.3.2" diff --git a/Cargo.toml b/Cargo.toml index 16b8e6fd..f350c9cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,8 @@ name = "signup-sequencer" version = "0.1.0" authors = [ "Remco Bloemen ", - "Lucas Ege "] + "Lucas Ege ", +] edition = "2021" build = "build.rs" homepage = "https://github.com/worldcoin/signup-sequencer" @@ -19,7 +20,6 @@ members = ["crates/*"] [features] default = [] -bench = ["criterion", "proptest"] mimalloc = ["cli-batteries/mimalloc"] [dependencies] @@ -31,9 +31,14 @@ axum-server = "0.4.4" bytes = "1.4.0" chrono = { version = "0.4.19", features = ["serde"] } clap = { version = "4.0", features = ["derive"] } -cli-batteries = { git = "https://github.com/recmo/cli-batteries", rev = "fc1186d1aba6a25120570fe04ad3362b08c8adfd", features = ["signals", "prometheus", "metered-allocator", "otlp", "datadog"] } +cli-batteries = { git = "https://github.com/recmo/cli-batteries", rev = "fc1186d1aba6a25120570fe04ad3362b08c8adfd", features = [ + "signals", + "prometheus", + "metered-allocator", + "otlp", + "datadog", +] } cognitoauth = { git = "https://github.com/lucdew/cognito-srp-auth.git" } -criterion = { version = "0.4", optional = true, features = ["async_tokio"] } # For `bench` ethers = { version = "1.0.0", features = ["ws", "ipc", "openssl", "abigen"] } eyre = "0.6" futures = "0.3" @@ -43,17 +48,32 @@ hyper = { version = "^0.14.17", features = ["server", "tcp", "http1", "http2"] } once_cell = "1.8" oz-api = { path = "crates/oz-api" } prometheus = "0.13.3" # We need upstream PR#465 to fix #272. -proptest = { version = "1.0", optional = true } # For `bench` reqwest = { version = "0.11.18", features = ["json"] } ruint = { version = "1.3", features = ["primitive-types", "sqlx"] } -semaphore = { git = "https://github.com/worldcoin/semaphore-rs", branch = "main", features = ["depth_30"] } +semaphore = { git = "https://github.com/worldcoin/semaphore-rs", branch = "main", features = [ + "depth_30", +] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -sqlx = { version = "0.6", features = ["runtime-tokio-native-tls", "any", "postgres", "chrono"] } +sqlx = { version = "0.6", features = [ + "runtime-tokio-native-tls", + "any", + "postgres", + "chrono", +] } take_mut = "0.2.2" tempfile = "3.3.0" thiserror = "1.0" -tokio = { version = "1.17", features = ["signal", "macros", "rt", "sync", "time", "rt-multi-thread", "tracing", "test-util"] } +tokio = { version = "1.17", features = [ + "signal", + "macros", + "rt", + "sync", + "time", + "rt-multi-thread", + "tracing", + "test-util", +] } tracing = "0.1" tracing-futures = "0.2" url = { version = "2.2", features = ["serde"] } @@ -62,16 +82,18 @@ url = { version = "2.2", features = ["serde"] } # `cargo update --package primitive-types@0.12.1 --precise 0.11.1` [dev-dependencies] -cli-batteries = { git = "https://github.com/recmo/cli-batteries", rev = "fc1186d1aba6a25120570fe04ad3362b08c8adfd", features = ["mock-shutdown"] } +cli-batteries = { git = "https://github.com/recmo/cli-batteries", rev = "fc1186d1aba6a25120570fe04ad3362b08c8adfd", features = [ + "mock-shutdown", +] } hex = "0.4.3" hex-literal = "0.3" maplit = "1.0.2" micro-oz = { path = "crates/micro-oz" } postgres-docker-utils = { path = "crates/postgres-docker-utils" } -proptest = { version = "1.0" } regex = { version = "1.7.1", features = ["std"] } -semaphore = { git = "https://github.com/worldcoin/semaphore-rs", branch = "main", features = ["depth_20"] } -serial_test = { version = "1.0.0" } +semaphore = { git = "https://github.com/worldcoin/semaphore-rs", branch = "main", features = [ + "depth_20", +] } test-case = "3.0" tracing-subscriber = "0.3.11" tracing-test = "0.2" @@ -82,15 +104,8 @@ cli-batteries = { git = "https://github.com/recmo/cli-batteries", rev = "fc1186d [profile.release] panic = "abort" overflow-checks = true -codegen-units = 1 -lto = true +lto = "thin" strip = true [profile.dev] panic = "abort" - -# Compilation profile for any non-workspace member. -# Dependencies are optimized, even in a dev build. This improves dev performance -# while having neglible impact on incremental build times. -[profile.dev.package."*"] -opt-level = 3 diff --git a/Readme.md b/Readme.md index 35a47287..d952a7a7 100644 --- a/Readme.md +++ b/Readme.md @@ -1,7 +1,6 @@ ![lines of code](https://img.shields.io/tokei/lines/github/worldcoin/signup-sequencer) [![dependency status](https://deps.rs/repo/github/worldcoin/signup-sequencer/status.svg)](https://deps.rs/repo/github/worldcoin/signup-sequencer) -[![codecov](https://codecov.io/gh/worldcoin/signup-sequencer/branch/main/graph/badge.svg?token=WBPZ9U4TTO)](https://codecov.io/gh/worldcoin/signup-sequencer) [![CI](https://github.com/worldcoin/signup-sequencer/actions/workflows/test.yml/badge.svg)](https://github.com/worldcoin/signup-sequencer/actions/workflows/test.yml) [![Audit](https://github.com/worldcoin/signup-sequencer/actions/workflows/audit.yml/badge.svg)](https://github.com/worldcoin/signup-sequencer/actions/workflows/audit.yml) @@ -19,31 +18,31 @@ Sign-up Sequencer does sequencing of data (identities) that are commited in a ba Sequencer has 6 API routes. -1. `/insertIdentity` - Accepts identity commitment hash as input which gets added in queue for processing. - Identities go trough three tasks. - 1. Insertion: In the initial stage, the identities are placed into the Sequencer's database. - The database is polled every few seconds and added to insertion task. - 2. Processing: The processing of identities, where current batching tree is taken and processed so we we - end up with pre root (the root of tree before proofs are generated), post root, start index and - identity commitments (with their proofs). All of those get sent to a [prover](#semaphore-mtb) for proof generation. - The identities transaction is then mined, with aforementioned fields and pending identities are sent to task to be mined on-chain. - 3. Mining: The transaction ID from processing task gets mined and Sequencer database gets updated accordingly. - Now with blockchain and database being in sync, the mined tree gets updated as well. -2. `/inclusionProof` - Takes the identity commitment hash, and checks for any errors that might have occurred in the insert identity steps. - Then leaf index is fetched from the database, corresponding to the identity hash provided, and then the we check if the identity is - indeed in the tree. The inclusion proof is then returned to the API caller. -3. `/verifySemaphoreProof` - This call takes root, signal hash, nullifier hash, external nullifier hash and a proof. - The proving key is fetched based on the depth index, and verification key as well. - The list of prime fields is created based on request input mentioned before, and then we proceed to verify the proof. +1. `/insertIdentity` - Accepts identity commitment hash as input which gets added in queue for processing. + Identities go trough three tasks. + 1. Insertion: In the initial stage, the identities are placed into the Sequencer's database. + The database is polled every few seconds and added to insertion task. + 2. Processing: The processing of identities, where current batching tree is taken and processed so we we + end up with pre root (the root of tree before proofs are generated), post root, start index and + identity commitments (with their proofs). All of those get sent to a [prover](#semaphore-mtb) for proof generation. + The identities transaction is then mined, with aforementioned fields and pending identities are sent to task to be mined on-chain. + 3. Mining: The transaction ID from processing task gets mined and Sequencer database gets updated accordingly. + Now with blockchain and database being in sync, the mined tree gets updated as well. +2. `/inclusionProof` - Takes the identity commitment hash, and checks for any errors that might have occurred in the insert identity steps. + Then leaf index is fetched from the database, corresponding to the identity hash provided, and then the we check if the identity is + indeed in the tree. The inclusion proof is then returned to the API caller. +3. `/verifySemaphoreProof` - This call takes root, signal hash, nullifier hash, external nullifier hash and a proof. + The proving key is fetched based on the depth index, and verification key as well. + The list of prime fields is created based on request input mentioned before, and then we proceed to verify the proof. Sequencer uses groth16 zk-SNARK implementation. - The API call returns the proof as response. -4. `/addBatchSize` - Adds a prover with specific batch size to a list of provers. -5. `/removeBatchSize` - Removes the prover based on batch size. -6. `/listBatchSizes` - Lists all provers that are added to the Sequencer. - + The API call returns the proof as response. +4. `/addBatchSize` - Adds a prover with specific batch size to a list of provers. +5. `/removeBatchSize` - Removes the prover based on batch size. +6. `/listBatchSizes` - Lists all provers that are added to the Sequencer. -## Getting Started + +## Getting Started ### (Local development) Install Protobuf compiler @@ -61,22 +60,22 @@ docker pull postgres ``` ### Worldcoin id contracts -Worldcoin id contracts are ethereum smart contracts that are used by the sequencer +Worldcoin id contracts are ethereum smart contracts that are used by the sequencer -Clone [worldcoin id contracts](https://github.com/worldcoin/world-id-contracts) and execute `make build && npm install` +Clone [worldcoin id contracts](https://github.com/worldcoin/world-id-contracts) and execute `make build && npm install` -You will need a local ethereum chain, like geth or ganache. -Start up your local ethereum network and generate an account. -Go to your worldcoin id contracts repo folder and do `make deploy`. Follow the steps provided by the script. -Make sure you use a private key from your local ethereum network account. -!! Make a note of all the addresses generated by the script !! +You will need a local ethereum chain, like geth or ganache. +Start up your local ethereum network and generate an account. +Go to your worldcoin id contracts repo folder and do `make deploy`. Follow the steps provided by the script. +Make sure you use a private key from your local ethereum network account. +!! Make a note of all the addresses generated by the script !! ### Semaphore-mtb Semaphore-mtb is a service for batch processing of Merkle tree updates. -Clone [semaphore-mtb](https://github.com/worldcoin/semaphore-mtb) and execute `go build .` (you will need a golang compiler) +Clone [semaphore-mtb](https://github.com/worldcoin/semaphore-mtb) and execute `go build .` (you will need a golang compiler) -Go build will create a executable named gnark-mbu. If you went trough the worldcoin id contracts deploy script, +Go build will create a executable named gnark-mbu. If you went trough the worldcoin id contracts deploy script, you will have a generated keys file that is used by semaphore-mtb. ```shell ./gnark-mbu start --keys-file path/to/world-id-contracts/mtb/keys @@ -88,9 +87,9 @@ you will have a generated keys file that is used by semaphore-mtb. docker run --rm -ti -p 5432:5432 -e POSTGRES_PASSWORD=password postgres ``` -Now you are ready to start up sequencer service! +Now you are ready to start up sequencer service! ```shell -TREE_DEPTH=*your tree depth (eg. 16)* cargo run -- --batch-size *batch size for semaphore-mtb (eg. 3)* --batch-timeout-seconds 10 --database postgres://postgres:password@0.0.0.0:5432 --identity-manager-address *address from worldcoin id contracts identity manager* +TREE_DEPTH=*your tree depth (eg. 16)* cargo run -- --batch-size *batch size for semaphore-mtb (eg. 3)* --batch-timeout-seconds 10 --database postgres://postgres:password@0.0.0.0:5432 --identity-manager-address *address from worldcoin id contracts identity manager* --signing-key *private key you used to deploy smart contracts* ``` @@ -104,15 +103,15 @@ cargo fmt && cargo clippy --all-targets && cargo build --all-targets && cargo te ## Contributing -We welcome your pull requests! But also consider the following: +We welcome your pull requests! But also consider the following: -1. Fork this repo from `main` branch. -2. If you added code that should be tested, please add tests. -3. If you changed the API routes, please update this readme in your PR. -4. Ensure that CI tests suite passes (lints as well). -5. If you added dependencies, make sure you add exemptions for `cargo vet` +1. Fork this repo from `main` branch. +2. If you added code that should be tested, please add tests. +3. If you changed the API routes, please update this readme in your PR. +4. Ensure that CI tests suite passes (lints as well). +5. If you added dependencies, make sure you add exemptions for `cargo vet` -When you submit code changes, your submissions are understood to be under the same MIT License that covers the project. -Feel free to contact the maintainers if that's a concern. +When you submit code changes, your submissions are understood to be under the same MIT License that covers the project. +Feel free to contact the maintainers if that's a concern. -Report bugs using github issues. +Report bugs using github issues. diff --git a/crates/micro-oz/Cargo.toml b/crates/micro-oz/Cargo.toml index b675d27e..5e0a1463 100644 --- a/crates/micro-oz/Cargo.toml +++ b/crates/micro-oz/Cargo.toml @@ -5,17 +5,17 @@ edition = "2021" publish = false [dependencies] -oz-api = { path = "../oz-api" } anyhow = "1.0.72" +async-trait = "0.1.71" axum = "0.6.19" +chrono = "0.4.26" clap = { version = "4.3.14", features = ["env", "derive"] } +ethers = { version = "1.0.0", features = ["openssl"] } +hyper = "0.14.27" +oz-api = { path = "../oz-api" } serde = "1.0.171" serde_json = "1.0.103" tokio = { version = "1.29.1", features = ["full"] } tracing = "0.1.37" tracing-subscriber = "0.3.17" -ethers = { version = "1.0.0", features = ["openssl"] } -async-trait = "0.1.71" -chrono = "0.4.26" -hyper = "0.14.27" diff --git a/crates/oz-api/Cargo.toml b/crates/oz-api/Cargo.toml index d10163dd..12e04db0 100644 --- a/crates/oz-api/Cargo.toml +++ b/crates/oz-api/Cargo.toml @@ -5,15 +5,15 @@ edition = "2021" publish = false [dependencies] +anyhow = "1.0" chrono = { version = "0.4.23", features = ["serde"] } +cognitoauth = { git = "https://github.com/lucdew/cognito-srp-auth.git" } ethers = { version = "1.0.0", features = [ "ws", "ipc", "openssl", "abigen" ] } +hyper = { version = "^0.14.17", features = ["server", "tcp", "http1", "http2"] } reqwest = "0.11.14" serde = { version = "1.0.154", features = ["derive"] } serde_json = "1.0.94" -tracing = "0.1.37" -cognitoauth = { git = "https://github.com/lucdew/cognito-srp-auth.git" } -anyhow = "1.0" -hyper = { version = "^0.14.17", features = ["server", "tcp", "http1", "http2"] } -tokio = { version = "1.17", features = ["full"] } thiserror = "1.0.39" +tokio = { version = "1.17", features = ["full"] } +tracing = "0.1.37" url = "2.3.1" diff --git a/src/task_monitor/tasks/process_identities.rs b/src/task_monitor/tasks/process_identities.rs index cb5d3f39..f9e57460 100644 --- a/src/task_monitor/tasks/process_identities.rs +++ b/src/task_monitor/tasks/process_identities.rs @@ -140,6 +140,9 @@ async fn process_identities( // We have _at most_ one complete batch here. let updates = batching_tree.peek_next_updates(batch_size); + if updates.is_empty() { + continue; + } // If there are not enough identities to insert at this // stage we can wait. The timer will ensure that the API diff --git a/tests/delete_identities.rs b/tests/delete_identities.rs index 3cf97ed6..3b6a7492 100644 --- a/tests/delete_identities.rs +++ b/tests/delete_identities.rs @@ -8,7 +8,6 @@ const SUPPORTED_DEPTH: usize = 18; const IDLE_TIME: u64 = 7; #[tokio::test] -#[serial_test::serial] async fn delete_identities() -> anyhow::Result<()> { // Initialize logging for the test. init_tracing_subscriber(); diff --git a/tests/delete_padded_identity.rs b/tests/delete_padded_identity.rs index 298b8832..ac9a2e97 100644 --- a/tests/delete_padded_identity.rs +++ b/tests/delete_padded_identity.rs @@ -8,7 +8,6 @@ const SUPPORTED_DEPTH: u8 = 18; const IDLE_TIME: u64 = 7; #[tokio::test] -#[serial_test::serial] async fn delete_padded_identity() -> anyhow::Result<()> { // Initialize logging for the test. init_tracing_subscriber(); diff --git a/tests/dynamic_batch_sizes.rs b/tests/dynamic_batch_sizes.rs index 8947b23f..921ee2cc 100644 --- a/tests/dynamic_batch_sizes.rs +++ b/tests/dynamic_batch_sizes.rs @@ -11,7 +11,6 @@ const SUPPORTED_DEPTH: usize = 20; const IDLE_TIME: u64 = 7; #[tokio::test] -#[serial_test::serial] async fn dynamic_batch_sizes() -> anyhow::Result<()> { // Initialize logging for the test. init_tracing_subscriber(); diff --git a/tests/insert_identity_and_proofs.rs b/tests/insert_identity_and_proofs.rs index 71cfca83..004e2ea8 100644 --- a/tests/insert_identity_and_proofs.rs +++ b/tests/insert_identity_and_proofs.rs @@ -6,7 +6,6 @@ const SUPPORTED_DEPTH: usize = 20; const IDLE_TIME: u64 = 7; #[tokio::test] -#[serial_test::serial] async fn insert_identity_and_proofs() -> anyhow::Result<()> { // Initialize logging for the test. init_tracing_subscriber(); diff --git a/tests/recover_identities.rs b/tests/recover_identities.rs index f3e1c5da..7995a1af 100644 --- a/tests/recover_identities.rs +++ b/tests/recover_identities.rs @@ -8,7 +8,6 @@ const SUPPORTED_DEPTH: usize = 18; const IDLE_TIME: u64 = 7; #[tokio::test] -#[serial_test::serial] async fn recover_identities() -> anyhow::Result<()> { // Initialize logging for the test. init_tracing_subscriber(); diff --git a/tests/validate_proof_with_age.rs b/tests/validate_proof_with_age.rs index 2d51c9dc..60232209 100644 --- a/tests/validate_proof_with_age.rs +++ b/tests/validate_proof_with_age.rs @@ -1,5 +1,7 @@ mod common; +use std::time::Instant; + use common::prelude::*; use crate::common::test_verify_proof_with_age; @@ -82,6 +84,8 @@ async fn validate_proof_with_age() -> anyhow::Result<()> { let signal_hash = hash_to_field(b"signal_hash"); let external_nullifier_hash = hash_to_field(b"external_hash"); + let time_of_identity_insertion = Instant::now(); + // Insert only the 1st identity let (merkle_proof, root) = test_insert_identity(&uri, &client, &mut ref_tree, &test_leaves, 0).await; @@ -138,6 +142,12 @@ async fn validate_proof_with_age() -> anyhow::Result<()> { ) .await; + let max_age_of_proof = (Instant::now() - time_of_identity_insertion).as_secs(); + assert!( + max_age_of_proof > sleep_duration_seconds * 2, + "Proof age should be at least 2 batch times" + ); + // Test proof which is new enough test_verify_proof_with_age( &uri, @@ -147,7 +157,7 @@ async fn validate_proof_with_age() -> anyhow::Result<()> { nullifier_hash, external_nullifier_hash, proof, - (2 * sleep_duration_seconds) as i64 + 5, // 5 seconds margin + max_age_of_proof as i64, None, ) .await;