From 322ad4e4b53d534a8ae6461f3d3383d67b219b5d Mon Sep 17 00:00:00 2001 From: Amaury <1293565+amaury1093@users.noreply.github.com> Date: Tue, 25 Jun 2024 18:52:52 +0200 Subject: [PATCH] revert(backend): Bring back the sqlxmq-based bulk verification (#1477) * Revert "feat(backend)!: Remove /v0/bulk endpoints (#1421)" This reverts commit 522f32448416cd75a70ddb51038e50d06c3130b4. * make it work * don't error on clippy warnings --- .github/workflows/pr.yml | 2 +- ...8e3c857125d1c4cf916ac32789bfd0176b6b5.json | 24 + ...dc14dfa9373dc8137c4f1da199e3ae66efd50.json | 52 + ...7382532c86c7e66e46254d72ef0af03021975.json | 15 + ...b1881df93a6619434a1fd1fc9a58315b4044b.json | 34 + ...86326e7a5e6d5c8fd03c03565d86982d0381a.json | 22 + ...e702c0edbbf57624f2bb416f0fd0401a44dcf.json | 22 + ...c403f7588266ae8d954985fc1eda9751febcc.json | 22 + Cargo.lock | 2039 ++++++++++++----- backend/Cargo.toml | 11 + backend/README.md | 28 +- .../migrations/20210316025847_setup.down.sql | 12 + .../migrations/20210316025847_setup.up.sql | 289 +++ .../migrations/20210921115907_clear.down.sql | 2 + .../migrations/20210921115907_clear.up.sql | 21 + ...11013151757_fix_mq_latest_message.down.sql | 15 + ...0211013151757_fix_mq_latest_message.up.sql | 19 + .../20220117025847_email_data.down.sql | 4 + .../20220117025847_email_data.up.sql | 12 + ...0220208120856_fix_concurrent_poll.down.sql | 60 + .../20220208120856_fix_concurrent_poll.up.sql | 62 + ...07_fix-clear_all-keep-nil-message.down.sql | 1 + ...2907_fix-clear_all-keep-nil-message.up.sql | 29 + .../20220810141100_result_created_at.down.sql | 2 + .../20220810141100_result_created_at.up.sql | 2 + backend/migrations/README.md | 33 + backend/src/bin/prune_db.rs | 81 + backend/src/http/mod.rs | 55 +- backend/src/http/v0/bulk/db.rs | 35 + backend/src/http/v0/bulk/error.rs | 44 + backend/src/http/v0/bulk/get.rs | 171 ++ backend/src/http/v0/bulk/mod.rs | 65 + backend/src/http/v0/bulk/post.rs | 151 ++ .../src/http/v0/bulk/results/csv_helper.rs | 205 ++ backend/src/http/v0/bulk/results/mod.rs | 248 ++ backend/src/http/v0/bulk/task.rs | 230 ++ backend/src/http/v0/mod.rs | 1 + backend/src/main.rs | 3 +- backend/tests/check_email.rs | 16 +- core/Cargo.toml | 2 +- 40 files changed, 3559 insertions(+), 582 deletions(-) create mode 100644 .sqlx/query-1039a6d3d732a86b9b3b2e19e6c8e3c857125d1c4cf916ac32789bfd0176b6b5.json create mode 100644 .sqlx/query-13862fe23ea729215fb1cfee3aadc14dfa9373dc8137c4f1da199e3ae66efd50.json create mode 100644 .sqlx/query-1a964da4784832e5f631f2c2e727382532c86c7e66e46254d72ef0af03021975.json create mode 100644 .sqlx/query-47af0157fa867e147e49d80b121b1881df93a6619434a1fd1fc9a58315b4044b.json create mode 100644 .sqlx/query-981f650b6c663feeae8a93e7ecf86326e7a5e6d5c8fd03c03565d86982d0381a.json create mode 100644 .sqlx/query-ac5e197ca20a1393e4ea45248d5e702c0edbbf57624f2bb416f0fd0401a44dcf.json create mode 100644 .sqlx/query-f58d4d05a6ab4c1ffda39396df4c403f7588266ae8d954985fc1eda9751febcc.json create mode 100644 backend/migrations/20210316025847_setup.down.sql create mode 100644 backend/migrations/20210316025847_setup.up.sql create mode 100644 backend/migrations/20210921115907_clear.down.sql create mode 100644 backend/migrations/20210921115907_clear.up.sql create mode 100644 backend/migrations/20211013151757_fix_mq_latest_message.down.sql create mode 100644 backend/migrations/20211013151757_fix_mq_latest_message.up.sql create mode 100644 backend/migrations/20220117025847_email_data.down.sql create mode 100644 backend/migrations/20220117025847_email_data.up.sql create mode 100644 backend/migrations/20220208120856_fix_concurrent_poll.down.sql create mode 100644 backend/migrations/20220208120856_fix_concurrent_poll.up.sql create mode 100644 backend/migrations/20220713122907_fix-clear_all-keep-nil-message.down.sql create mode 100644 backend/migrations/20220713122907_fix-clear_all-keep-nil-message.up.sql create mode 100644 backend/migrations/20220810141100_result_created_at.down.sql create mode 100644 backend/migrations/20220810141100_result_created_at.up.sql create mode 100644 backend/migrations/README.md create mode 100644 backend/src/bin/prune_db.rs create mode 100644 backend/src/http/v0/bulk/db.rs create mode 100644 backend/src/http/v0/bulk/error.rs create mode 100644 backend/src/http/v0/bulk/get.rs create mode 100644 backend/src/http/v0/bulk/mod.rs create mode 100644 backend/src/http/v0/bulk/post.rs create mode 100644 backend/src/http/v0/bulk/results/csv_helper.rs create mode 100644 backend/src/http/v0/bulk/results/mod.rs create mode 100644 backend/src/http/v0/bulk/task.rs diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a113f6c17..3f2364081 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -79,4 +79,4 @@ jobs: uses: actions-rs/cargo@v1 with: command: clippy - args: --all -- -D warnings -A deprecated + args: --all -- -A deprecated diff --git a/.sqlx/query-1039a6d3d732a86b9b3b2e19e6c8e3c857125d1c4cf916ac32789bfd0176b6b5.json b/.sqlx/query-1039a6d3d732a86b9b3b2e19e6c8e3c857125d1c4cf916ac32789bfd0176b6b5.json new file mode 100644 index 000000000..5493f1216 --- /dev/null +++ b/.sqlx/query-1039a6d3d732a86b9b3b2e19e6c8e3c857125d1c4cf916ac32789bfd0176b6b5.json @@ -0,0 +1,24 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tSELECT result FROM email_results\n\t\tWHERE job_id = $1\n\t\tORDER BY id\n\t\tLIMIT $2 OFFSET $3\n\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "result", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int8", + "Int8" + ] + }, + "nullable": [ + true + ] + }, + "hash": "1039a6d3d732a86b9b3b2e19e6c8e3c857125d1c4cf916ac32789bfd0176b6b5" +} diff --git a/.sqlx/query-13862fe23ea729215fb1cfee3aadc14dfa9373dc8137c4f1da199e3ae66efd50.json b/.sqlx/query-13862fe23ea729215fb1cfee3aadc14dfa9373dc8137c4f1da199e3ae66efd50.json new file mode 100644 index 000000000..660b058a1 --- /dev/null +++ b/.sqlx/query-13862fe23ea729215fb1cfee3aadc14dfa9373dc8137c4f1da199e3ae66efd50.json @@ -0,0 +1,52 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tSELECT\n\t\t\tCOUNT(*) as total_processed,\n\t\t\tCOUNT(CASE WHEN result ->> 'is_reachable' LIKE 'safe' THEN 1 END) as safe_count,\n\t\t\tCOUNT(CASE WHEN result ->> 'is_reachable' LIKE 'risky' THEN 1 END) as risky_count,\n\t\t\tCOUNT(CASE WHEN result ->> 'is_reachable' LIKE 'invalid' THEN 1 END) as invalid_count,\n\t\t\tCOUNT(CASE WHEN result ->> 'is_reachable' LIKE 'unknown' THEN 1 END) as unknown_count,\n\t\t\t(SELECT created_at FROM email_results WHERE job_id = $1 ORDER BY created_at DESC LIMIT 1) as finished_at\n\t\tFROM email_results\n\t\tWHERE job_id = $1\n\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total_processed", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "safe_count", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "risky_count", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "invalid_count", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "unknown_count", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "finished_at", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null, + null, + null, + null, + null, + null + ] + }, + "hash": "13862fe23ea729215fb1cfee3aadc14dfa9373dc8137c4f1da199e3ae66efd50" +} diff --git a/.sqlx/query-1a964da4784832e5f631f2c2e727382532c86c7e66e46254d72ef0af03021975.json b/.sqlx/query-1a964da4784832e5f631f2c2e727382532c86c7e66e46254d72ef0af03021975.json new file mode 100644 index 000000000..20d1cc509 --- /dev/null +++ b/.sqlx/query-1a964da4784832e5f631f2c2e727382532c86c7e66e46254d72ef0af03021975.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\t\tINSERT INTO email_results (job_id, result)\n\t\t\tVALUES ($1, $2)\n\t\t\t", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "1a964da4784832e5f631f2c2e727382532c86c7e66e46254d72ef0af03021975" +} diff --git a/.sqlx/query-47af0157fa867e147e49d80b121b1881df93a6619434a1fd1fc9a58315b4044b.json b/.sqlx/query-47af0157fa867e147e49d80b121b1881df93a6619434a1fd1fc9a58315b4044b.json new file mode 100644 index 000000000..2c00dfeb0 --- /dev/null +++ b/.sqlx/query-47af0157fa867e147e49d80b121b1881df93a6619434a1fd1fc9a58315b4044b.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tSELECT id, created_at, total_records FROM bulk_jobs\n\t\tWHERE id = $1\n\t\tLIMIT 1\n\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "created_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 2, + "name": "total_records", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "47af0157fa867e147e49d80b121b1881df93a6619434a1fd1fc9a58315b4044b" +} diff --git a/.sqlx/query-981f650b6c663feeae8a93e7ecf86326e7a5e6d5c8fd03c03565d86982d0381a.json b/.sqlx/query-981f650b6c663feeae8a93e7ecf86326e7a5e6d5c8fd03c03565d86982d0381a.json new file mode 100644 index 000000000..991aa1226 --- /dev/null +++ b/.sqlx/query-981f650b6c663feeae8a93e7ecf86326e7a5e6d5c8fd03c03565d86982d0381a.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "\n\t\tINSERT INTO bulk_jobs (total_records)\n\t\tVALUES ($1)\n\t\tRETURNING id\n\t\t", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "981f650b6c663feeae8a93e7ecf86326e7a5e6d5c8fd03c03565d86982d0381a" +} diff --git a/.sqlx/query-ac5e197ca20a1393e4ea45248d5e702c0edbbf57624f2bb416f0fd0401a44dcf.json b/.sqlx/query-ac5e197ca20a1393e4ea45248d5e702c0edbbf57624f2bb416f0fd0401a44dcf.json new file mode 100644 index 000000000..6cc885704 --- /dev/null +++ b/.sqlx/query-ac5e197ca20a1393e4ea45248d5e702c0edbbf57624f2bb416f0fd0401a44dcf.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT total_records FROM bulk_jobs WHERE id = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "total_records", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false + ] + }, + "hash": "ac5e197ca20a1393e4ea45248d5e702c0edbbf57624f2bb416f0fd0401a44dcf" +} diff --git a/.sqlx/query-f58d4d05a6ab4c1ffda39396df4c403f7588266ae8d954985fc1eda9751febcc.json b/.sqlx/query-f58d4d05a6ab4c1ffda39396df4c403f7588266ae8d954985fc1eda9751febcc.json new file mode 100644 index 000000000..bbf14d4a9 --- /dev/null +++ b/.sqlx/query-f58d4d05a6ab4c1ffda39396df4c403f7588266ae8d954985fc1eda9751febcc.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) FROM email_results WHERE job_id = $1;", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f58d4d05a6ab4c1ffda39396df4c403f7588266ae8d954985fc1eda9751febcc" +} diff --git a/Cargo.lock b/Cargo.lock index 32faab62f..9e342d404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.17.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -17,15 +17,34 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -43,9 +62,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.61" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "anymap2" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "508b352bb5c066aac251f6daf6b36eccd03e8a88e8081cd44959ea277a3af9a8" +checksum = "d301b3b94cb4b2f23d7917810addbbaff90738e0ca2be692bd027e70d7e0330c" [[package]] name = "ascii_utils" @@ -67,13 +92,13 @@ dependencies = [ [[package]] name = "async-recursion" -version = "1.0.5" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fd55a5ba1179988837d24ab4c7cc8ed6efdeff578ede0416b4225a5fca35bd0" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.66", ] [[package]] @@ -84,7 +109,7 @@ checksum = "2ade89127f9e0d44f9e83cf574d499060005cd45b7dc76be89c0167487fe8edd" dependencies = [ "async-native-tls", "async-trait", - "base64 0.13.0", + "base64 0.13.1", "bufstream", "fast-socks5 0.8.2", "futures", @@ -102,13 +127,22 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.57" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76464446b8bc32758d7e88ee1a804d9914cd9b1cb264c029899680b0be29826f" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.66", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", ] [[package]] @@ -124,15 +158,15 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.66" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -145,15 +179,21 @@ dependencies = [ [[package]] name = "base64" -version = "0.13.0" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.0" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64ct" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bitflags" @@ -163,15 +203,18 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +dependencies = [ + "serde", +] [[package]] name = "block-buffer" -version = "0.10.2" +version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ "generic-array", ] @@ -184,30 +227,27 @@ checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" [[package]] name = "bumpalo" -version = "3.10.0" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37ccbd214614c6783386c1af30caf03192f17891059cecc394b4fb119e363de3" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byteorder" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.2.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec8a7b6a70fde80372154c65702f00a0f56f3e1c36abbc6c440484be248856db" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" [[package]] name = "cc" -version = "1.0.83" +version = "1.0.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" -dependencies = [ - "libc", -] +checksum = "96c51067fd44124faa7f870b4b1c969379ad32b2ba805aa959430ceaa384f695" [[package]] name = "cfg-if" @@ -224,7 +264,7 @@ dependencies = [ "async-smtp", "chrono", "fantoccini", - "fast-socks5 0.9.2", + "fast-socks5 0.9.6", "futures", "hickory-proto", "hickory-resolver", @@ -258,9 +298,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.31" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", @@ -268,20 +308,20 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets", + "windows-targets 0.52.5", ] [[package]] name = "clap" -version = "3.2.17" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29e724a68d9319343bb3328c9cc2dfde263f4b3142ee1059a9980580171c954b" +checksum = "4ea181bf566f71cb9a5d17a59e1871af638180a18fb0035c92ae62b705207123" dependencies = [ "atty", "bitflags 1.3.2", "clap_derive", "clap_lex", - "indexmap", + "indexmap 1.9.3", "once_cell", "strsim 0.10.0", "termcolor", @@ -290,9 +330,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "3.2.17" +version = "3.2.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13547f7012c01ab4a0e8f8967730ada8f9fdf419e8b6c792788f39cf4e46eefa" +checksum = "ae6371b8bdc8b7d3959e9cf7b22d4435ef3e79e138688421ec654acf8c81b008" dependencies = [ "heck", "proc-macro-error", @@ -310,11 +350,17 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "cookie" -version = "0.16.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" +checksum = "e859cd57d0710d9e06c381b550c06e76992472a8c6d527aecd2fc673dcc231fb" dependencies = [ "percent-encoding", "time", @@ -323,9 +369,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.3" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ "core-foundation-sys", "libc", @@ -333,19 +379,49 @@ dependencies = [ [[package]] name = "core-foundation-sys" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "cpufeatures" -version = "0.2.2" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" dependencies = [ "libc", ] +[[package]] +name = "crc" +version = "3.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e6e4d7b33a94f0991c26729976b10ebde1d34c3ee82408fb536164fa10d636" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + [[package]] name = "crypto-common" version = "0.1.6" @@ -356,6 +432,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "darling" version = "0.10.2" @@ -393,9 +490,9 @@ dependencies = [ [[package]] name = "data-encoding" -version = "2.3.2" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee2393c4a91429dffb4bedf19f4d6abf27d8a732c8ce4980305d782e5426d57" +checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" [[package]] name = "debugid" @@ -404,7 +501,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6ee87af31d84ef885378aebca32be3d682b0e0dc119d5b4860a2c5bb5046730" dependencies = [ "serde", - "uuid", + "uuid 0.8.2", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", ] [[package]] @@ -439,7 +556,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", + "subtle", +] + +[[package]] +name = "displaydoc" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", ] [[package]] @@ -448,17 +578,26 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dotenvy" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" + [[package]] name = "either" -version = "1.7.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f107b87b6afc2a64fd13cac55fe06d6c8859f12d4b14cbcdd2c67d0976781be" +checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] @@ -472,14 +611,14 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.66", ] [[package]] name = "env_logger" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85cdab6a89accf66733ad5a1693a4dcced6aeff64602b634530dd73c1f3ee9f0" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" dependencies = [ "humantime", "is-terminal", @@ -488,38 +627,50 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" -version = "0.2.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ - "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.52.0", ] [[package]] -name = "errno-dragonfly" -version = "0.1.2" +name = "etcetera" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cc", - "libc", + "cfg-if", + "home", + "windows-sys 0.48.0", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "fantoccini" version = "0.19.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65f0fbe245d714b596ba5802b46f937f5ce68dcae0f32f9a70b5c3b04d3c6f64" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", "cookie", "futures-core", "futures-util", - "http", + "http 0.2.12", "hyper", "hyper-tls", "mime", @@ -546,9 +697,9 @@ dependencies = [ [[package]] name = "fast-socks5" -version = "0.9.2" +version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d449e348301d5fb9b0e5781510d8235ffe3bbac3286bd305462736a9e7043039" +checksum = "f89f36d4ee12370d30d57b16c7e190950a1a916e7dbbb5fd5a412f5ef913fe84" dependencies = [ "anyhow", "async-trait", @@ -569,11 +720,19 @@ dependencies = [ [[package]] name = "fastrand" -version = "1.8.0" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "flume" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "55ac459de2512911e4b674ce33cf20befaba382d05b62b008afc1c8b57cbf181" dependencies = [ - "instant", + "futures-core", + "futures-sink", + "spin 0.9.8", ] [[package]] @@ -648,6 +807,17 @@ dependencies = [ "futures-util", ] +[[package]] +name = "futures-intrusive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f" +dependencies = [ + "futures-core", + "lock_api", + "parking_lot", +] + [[package]] name = "futures-io" version = "0.3.30" @@ -662,7 +832,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.66", ] [[package]] @@ -697,9 +867,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.6" +version = "0.14.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", @@ -707,9 +877,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.7" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eb1a864a501629691edf6c15a593b7a51eebaa1e8468e9ddc623de7c9b58ec6" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", @@ -718,23 +888,23 @@ dependencies = [ [[package]] name = "gimli" -version = "0.26.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8" dependencies = [ "bytes", "fnv", "futures-core", "futures-sink", "futures-util", - "http", - "indexmap", + "http 0.2.12", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -747,20 +917,38 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "headers" -version = "0.3.7" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cff78e5788be1e0ab65b04d306b2ed5092c815ec97ec70f4ebd5aee158aa55d" +checksum = "06683b93020a07e3dbcf5f8c0f6d40080d725bea7936fc01ad345c01b97dc270" dependencies = [ - "base64 0.13.0", - "bitflags 1.3.2", + "base64 0.21.7", "bytes", "headers-core", - "http", + "http 0.2.12", "httpdate", "mime", - "sha-1", + "sha1 0.10.6", ] [[package]] @@ -769,14 +957,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http", + "http 0.2.12", ] [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] [[package]] name = "hermit-abi" @@ -789,18 +980,21 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.2.6" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hickory-proto" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091a6fbccf4860009355e3efc52ff4acf37a63489aad7435372d44ceeb6fbbcf" +checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" dependencies = [ "async-trait", "cfg-if", @@ -822,9 +1016,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b8f021164e6a984c9030023544c57789c51760065cd510572fedcfb04164e8" +checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" dependencies = [ "cfg-if", "futures-util", @@ -841,6 +1035,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "hkdf" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "hostname" version = "0.3.1" @@ -854,9 +1075,20 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -865,26 +1097,26 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "d0e7a4dd27b9476dc40cb050d3632d3bba3a70ddbff012285f7f8559a1e7e545" [[package]] name = "httpdate" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "humantime" @@ -894,22 +1126,22 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.27" +version = "0.14.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +checksum = "f361cde2f109281a220d4307746cdfd5ee3f410da58a70377762396775634b33" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", "h2", - "http", + "http 0.2.12", "http-body", "httparse", "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2", "tokio", "tower-service", "tracing", @@ -931,9 +1163,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.58" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -953,110 +1185,223 @@ dependencies = [ ] [[package]] -name = "ident_case" -version = "1.0.1" +name = "icu_collections" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] [[package]] -name = "idna" -version = "0.4.0" +name = "icu_locid" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", ] [[package]] -name = "idna" -version = "0.5.0" +name = "icu_locid_transform" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" dependencies = [ - "unicode-bidi", - "unicode-normalization", + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", ] [[package]] -name = "indexmap" -version = "1.9.1" +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a35a97730320ffe8e2d410b5d3b69279b98d2c14bdb8b70ea89ecf7888d41e" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" dependencies = [ - "autocfg", - "hashbrown", + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", ] [[package]] -name = "instant" -version = "0.1.12" +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "1f8ac670d7422d7f76b32e17a5db556510825b29ec9154f235977c9caba61036" dependencies = [ - "cfg-if", + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", ] [[package]] -name = "io-lifetimes" -version = "1.0.2" +name = "icu_properties_data" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e394faa0efb47f9f227f1cd89978f854542b318a6f64fa695489c9c993056656" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" dependencies = [ - "libc", - "windows-sys 0.42.0", + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", ] [[package]] -name = "ipconfig" -version = "0.3.0" +name = "icu_provider_macros" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "723519edce41262b05d4143ceb95050e4c614f483e78e9fd9e39a8275a84ad98" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ - "socket2 0.4.9", - "widestring", - "winapi", - "winreg 0.7.0", + "proc-macro2", + "quote", + "syn 2.0.66", ] [[package]] -name = "ipnet" -version = "2.5.0" +name = "ident_case" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] -name = "is-terminal" +name = "idna" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae5bc6e2eb41c9def29a3e0f1306382807764b9b53112030eff57435667352d" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" dependencies = [ - "hermit-abi 0.2.6", - "io-lifetimes", - "rustix", - "windows-sys 0.42.0", + "unicode-bidi", + "unicode-normalization", ] [[package]] -name = "itoa" -version = "1.0.3" +name = "idna" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8af84674fe1f223a982c933a0ee1086ac4d4052aa0fb8060c12c6ad838e754" +checksum = "4716a3a0933a1d01c2f72450e89596eb51dd34ef3c211ccd875acdf1f8fe47ed" +dependencies = [ + "icu_normalizer", + "icu_properties", + "smallvec", + "utf8_iter", +] [[package]] -name = "js-sys" -version = "0.3.59" +name = "indexmap" +version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ - "wasm-bindgen", + "autocfg", + "hashbrown 0.12.3", ] [[package]] -name = "lazy_static" +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2", + "widestring", + "windows-sys 0.48.0", + "winreg", +] + +[[package]] +name = "ipnet" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" + +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +dependencies = [ + "spin 0.5.2", +] [[package]] name = "levenshtein" @@ -1066,9 +1411,26 @@ checksum = "db13adb97ab515a3691f56e4dbab09283d0b86cb45abd991d8634a9d6f501760" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] [[package]] name = "linked-hash-map" @@ -1078,15 +1440,21 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.1.3" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + +[[package]] +name = "litemap" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +checksum = "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" [[package]] name = "lock_api" -version = "0.4.7" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327fa5b6a6940e4699ec49a9beae1ea4845c6bab9314e4f84ac68742139d8c53" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1094,9 +1462,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.20" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" [[package]] name = "lru-cache" @@ -1109,9 +1477,9 @@ dependencies = [ [[package]] name = "mailchecker" -version = "6.0.1" +version = "6.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a1951055840e30b5c9c60fcadca14b0b9962797a6228d4e5a73e438a3185ed" +checksum = "37614f18c61fd8d4a025bc82775f4b435d8e025d6ac499af42451356f985ac97" dependencies = [ "fast_chemail", "once_cell", @@ -1123,6 +1491,16 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "md5" version = "0.7.0" @@ -1131,15 +1509,15 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.4" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "mime" -version = "0.3.16" +version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "mime_guess" @@ -1159,18 +1537,18 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.5.3" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5c75688da582b8ffc1f1799e9db273f32133c49e048f614d22ec3256773ccc" +checksum = "87dfd01fe195c66b572b37921ad8803d010623c0aca821bea2302239d155cdae" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.9" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dce281c5e46beae905d4de1870d8b1509a9142b62eedf18b443b011ca8343d0" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", @@ -1186,22 +1564,21 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 0.2.12", "httparse", "log", "memchr", "mime", - "spin", + "spin 0.9.8", "version_check", ] [[package]] name = "native-tls" -version = "0.2.10" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd7e2f3618557f980e0b17e8856252eee3c97fa12c54dff0ca290fb6266ca4a9" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" dependencies = [ - "lazy_static", "libc", "log", "openssl", @@ -1215,9 +1592,9 @@ dependencies = [ [[package]] name = "nom" -version = "7.1.1" +version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8903e5a29a317527874d0402f867152a3d21c908bb0b933e416c65e301d4c36" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" dependencies = [ "memchr", "minimal-lexical", @@ -1234,38 +1611,73 @@ dependencies = [ ] [[package]] -name = "num-traits" -version = "0.2.15" +name = "num-bigint-dig" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc84195820f291c7697304f3cbdadd1cb7199c0efc917ff5eafd71225c136151" +dependencies = [ + "byteorder", + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-conv" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", + "num-integer", + "num-traits", ] [[package]] -name = "num_cpus" -version = "1.13.1" +name = "num-traits" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ - "hermit-abi 0.1.19", - "libc", + "autocfg", + "libm", ] [[package]] -name = "num_threads" -version = "0.1.6" +name = "num_cpus" +version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ + "hermit-abi 0.3.9", "libc", ] [[package]] name = "object" -version = "0.29.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +checksum = "576dfe1fc8f9df304abb159d767a29d0476f7750fbf8aa7ad07816004a207434" dependencies = [ "memchr", ] @@ -1282,7 +1694,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -1293,13 +1705,13 @@ dependencies = [ [[package]] name = "openssl-macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b501e44f11665960c7e7fcf062c7d96a14ade4aa98116c004b2e37b5be7d736c" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.66", ] [[package]] @@ -1310,18 +1722,18 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.1.5+3.1.3" +version = "300.3.1+3.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "559068e4c12950d7dcaa1857a61725c0d38d4fc03ff8e070ab31a75d6e316491" +checksum = "7259953d42a81bf137fbbd73bd30a8e1914d6dce43c2b90ed575783a22608b91" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.101" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -1332,9 +1744,9 @@ dependencies = [ [[package]] name = "os_str_bytes" -version = "6.3.0" +version = "6.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" [[package]] name = "overload" @@ -1344,9 +1756,9 @@ checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1354,15 +1766,30 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.3" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a279cbf25cb0757810394fbc1e359949b59e348145c643a939a525692e6929" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.1", "smallvec", - "windows-sys 0.36.1", + "windows-targets 0.52.5", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", ] [[package]] @@ -1373,29 +1800,29 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pin-project" -version = "1.0.12" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad29a609b6bcd67fee905812e544992d216af9d755757c05ed2d0e15a74c6ecc" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.12" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.66", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -1403,17 +1830,44 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" -version = "0.3.25" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "powerfmt" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "ppv-lite86" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro-error" @@ -1441,9 +1895,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.85" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "22244ce15aa966053a896d1accb3a6e68469b97c7f33f284b99f0d576879fc23" dependencies = [ "unicode-ident", ] @@ -1471,9 +1925,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -1514,31 +1968,44 @@ version = "0.7.0" dependencies = [ "async-smtp", "check-if-email-exists", + "csv", "dotenv", "openssl", "sentry", "serde", "serde_json", + "sqlx", + "sqlxmq", "tokio", "tracing", "tracing-subscriber", + "uuid 1.8.0", "warp", ] [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469052894dcb553421e483e4209ee581a45100d31b4018de03e5a7ad86374a7e" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "regex" -version = "1.10.2" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", @@ -1548,9 +2015,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.3" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -1559,32 +2026,23 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" - -[[package]] -name = "remove_dir_all" -version = "0.5.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" -dependencies = [ - "winapi", -] +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64 0.21.0", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", "h2", - "http", + "http 0.2.12", "http-body", "hyper", "hyper-tls", @@ -1596,9 +2054,11 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -1608,7 +2068,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "winreg 0.50.0", + "winreg", ] [[package]] @@ -1621,11 +2081,31 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rsa" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e5124fcb30e76a7e79bfee683a2746db83784b86289f6251b54b7950a0dfc" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-demangle" -version = "0.1.21" +version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" [[package]] name = "rustc_version" @@ -1638,62 +2118,60 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.3" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b1fbb4dfc4eb1d390c02df47760bb19a84bb80b301ecc947ab5406394d8223e" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.42.0", + "windows-sys 0.52.0", ] [[package]] name = "rustls-pemfile" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64 0.21.0", + "base64 0.21.7", ] [[package]] name = "ryu" -version = "1.0.11" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "schannel" -version = "0.1.20" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" dependencies = [ - "lazy_static", - "windows-sys 0.36.1", + "windows-sys 0.52.0", ] [[package]] name = "scoped-tls" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.6.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" +checksum = "c627723fd09706bacdb5cf41499e95098555af3c3c29d014dc3c458ef6be11c0" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.5.0", "core-foundation", "core-foundation-sys", "libc", @@ -1702,9 +2180,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.6.1" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -1712,9 +2190,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.13" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f6841e709003d68bb2deee8c343572bf446003ec20a583e76f7b15cebf3711" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" [[package]] name = "sentry" @@ -1794,34 +2272,34 @@ dependencies = [ "serde_json", "thiserror", "url", - "uuid", + "uuid 0.8.2", ] [[package]] name = "serde" -version = "1.0.195" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.203" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.66", ] [[package]] name = "serde_json" -version = "1.0.108" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ "itoa", "ryu", @@ -1840,17 +2318,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha-1" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "sha1" version = "0.6.1" @@ -1862,9 +2329,9 @@ dependencies = [ [[package]] name = "sha1" -version = "0.10.4" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "006769ba83e921b3085caa8334186b00cf92b4cb1a6cf4632fbccc8eff5c7549" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", @@ -1877,6 +2344,17 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1887,155 +2365,463 @@ dependencies = [ ] [[package]] -name = "slab" -version = "0.4.7" +name = "signal-hook-registry" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ - "autocfg", + "libc", ] [[package]] -name = "smallvec" -version = "1.11.2" +name = "signature" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core", +] [[package]] -name = "socket2" +name = "slab" version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ - "libc", - "winapi", + "autocfg", ] +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + [[package]] name = "socket2" -version = "0.5.5" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5fac59a5cb5dd637972e5fca70daf0523c9067fcdc4842f053dae04a18f8e9" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "spin" -version = "0.9.8" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" - -[[package]] -name = "strsim" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" [[package]] -name = "syn" -version = "1.0.109" +name = "spin" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "lock_api", ] [[package]] -name = "syn" -version = "2.0.48" +name = "spki" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "base64ct", + "der", ] [[package]] -name = "system-configuration" -version = "0.5.1" +name = "sqlformat" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +checksum = "f895e3734318cc55f1fe66258926c9b910c124d47520339efecbb6c59cec7c1f" dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", + "nom", + "unicode_categories", ] [[package]] -name = "system-configuration-sys" -version = "0.5.0" +name = "sqlx" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +checksum = "c9a2ccff1a000a5a59cd33da541d9f2fdcd9e6e8229cc200565942bff36d0aaa" dependencies = [ - "core-foundation-sys", - "libc", + "sqlx-core", + "sqlx-macros", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", ] [[package]] -name = "tempfile" -version = "3.3.0" +name = "sqlx-core" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4" +checksum = "24ba59a9342a3d9bab6c56c118be528b27c9b60e490080e9711a04dccac83ef6" dependencies = [ - "cfg-if", - "fastrand", - "libc", - "redox_syscall", - "remove_dir_all", - "winapi", -] - + "ahash", + "atoi", + "byteorder", + "bytes", + "chrono", + "crc", + "crossbeam-queue", + "either", + "event-listener", + "futures-channel", + "futures-core", + "futures-intrusive", + "futures-io", + "futures-util", + "hashlink", + "hex", + "indexmap 2.2.6", + "log", + "memchr", + "native-tls", + "once_cell", + "paste", + "percent-encoding", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlformat", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid 1.8.0", +] + +[[package]] +name = "sqlx-macros" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea40e2345eb2faa9e1e5e326db8c34711317d2b5e08d0d5741619048a803127" +dependencies = [ + "proc-macro2", + "quote", + "sqlx-core", + "sqlx-macros-core", + "syn 1.0.109", +] + +[[package]] +name = "sqlx-macros-core" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5833ef53aaa16d860e92123292f1f6a3d53c34ba8b1969f152ef1a7bb803f3c8" +dependencies = [ + "dotenvy", + "either", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-mysql", + "sqlx-postgres", + "sqlx-sqlite", + "syn 1.0.109", + "tempfile", + "tokio", + "url", +] + +[[package]] +name = "sqlx-mysql" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ed31390216d20e538e447a7a9b959e06ed9fc51c37b514b46eb758016ecd418" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.5.0", + "byteorder", + "bytes", + "chrono", + "crc", + "digest", + "dotenvy", + "either", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "generic-array", + "hex", + "hkdf", + "hmac", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "percent-encoding", + "rand", + "rsa", + "serde", + "sha1 0.10.6", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid 1.8.0", + "whoami", +] + +[[package]] +name = "sqlx-postgres" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c824eb80b894f926f89a0b9da0c7f435d27cdd35b8c655b114e58223918577e" +dependencies = [ + "atoi", + "base64 0.21.7", + "bitflags 2.5.0", + "byteorder", + "chrono", + "crc", + "dotenvy", + "etcetera", + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "hex", + "hkdf", + "hmac", + "home", + "itoa", + "log", + "md-5", + "memchr", + "once_cell", + "rand", + "serde", + "serde_json", + "sha2", + "smallvec", + "sqlx-core", + "stringprep", + "thiserror", + "tracing", + "uuid 1.8.0", + "whoami", +] + +[[package]] +name = "sqlx-sqlite" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b244ef0a8414da0bed4bb1910426e890b19e5e9bccc27ada6b797d05c55ae0aa" +dependencies = [ + "atoi", + "chrono", + "flume", + "futures-channel", + "futures-core", + "futures-executor", + "futures-intrusive", + "futures-util", + "libsqlite3-sys", + "log", + "percent-encoding", + "serde", + "sqlx-core", + "tracing", + "url", + "urlencoding", + "uuid 1.8.0", +] + +[[package]] +name = "sqlxmq" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e914521071581f0413516de0e7931086ebe4a22d719c1100de29eb339c712311" +dependencies = [ + "anymap2", + "chrono", + "dotenv", + "log", + "serde", + "serde_json", + "sqlx", + "sqlxmq_macros", + "tokio", + "uuid 1.8.0", +] + +[[package]] +name = "sqlxmq_macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd00667edb120f18e14e2b4a71cddb308dc3605171ed7afda3056129d08b9f4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42f3f41a2de00b01c0aaad383c5a45241efc8b2d1eda5661812fda5f3cdcff5" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "termcolor" -version = "1.1.3" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" dependencies = [ "winapi-util", ] [[package]] name = "textwrap" -version = "0.15.0" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9" [[package]] name = "thiserror" -version = "1.0.50" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +checksum = "c546c80d6be4bc6a00c0f01730c08df82eaa7a7a61f11d656526506112cc1709" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.50" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +checksum = "46c3384250002a6d5af4d114f2845d37b57521033f30d5c3f46c4d70e1197533" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.66", ] [[package]] name = "thread_local" -version = "1.1.7" +version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" +checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" dependencies = [ "cfg-if", "once_cell", @@ -2043,21 +2829,44 @@ dependencies = [ [[package]] name = "time" -version = "0.3.14" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3f9a28b618c3a6b9251b6908e9c99e04b9e5c02e6581ccbb67d59c34ef7f9b" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ + "deranged", "itoa", - "libc", - "num_threads", + "num-conv", + "powerfmt", + "serde", + "time-core", "time-macros", ] +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + [[package]] name = "time-macros" -version = "0.2.4" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] [[package]] name = "tinyvec" @@ -2070,43 +2879,45 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", "bytes", "libc", "mio", "num_cpus", + "parking_lot", "pin-project-lite", - "socket2 0.5.5", + "signal-hook-registry", + "socket2", "tokio-macros", "windows-sys 0.48.0", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "5f5ae998a069d4b5aba8ee9dad856af7d520c3699e6159b185c2acd48155d39a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.66", ] [[package]] name = "tokio-native-tls" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" dependencies = [ "native-tls", "tokio", @@ -2126,9 +2937,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.9" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df54d54117d6fdc4e4fea40fe1e4e566b3505700e148a6827e59b34b0d2600d9" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -2137,9 +2948,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +checksum = "c83b561d025642014097b66e6c1bb422783339e0909e4429cde4749d1990bc38" dependencies = [ "futures-util", "log", @@ -2149,16 +2960,15 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.3" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc463cd8deddc3770d20f9852143d50bf6094e640b485cb2e189a2099085ff45" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] [[package]] @@ -2187,7 +2997,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.66", ] [[package]] @@ -2227,24 +3037,24 @@ dependencies = [ [[package]] name = "try-lock" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.20.1" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +checksum = "9ef1a641ea34f399a848dea702823bbecfb4c486f911735368f1f137cb8257e1" dependencies = [ "byteorder", "bytes", "data-encoding", - "http", + "http 1.1.0", "httparse", "log", "rand", - "sha1 0.10.4", + "sha1 0.10.6", "thiserror", "url", "utf-8", @@ -2252,9 +3062,9 @@ dependencies = [ [[package]] name = "typenum" -version = "1.15.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "uname" @@ -2267,58 +3077,88 @@ dependencies = [ [[package]] name = "unicase" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f37be617794602aabbeee0be4f259dc1778fabe05e2d67ee8f79326d5cb4f6" +checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" dependencies = [ "version_check", ] [[package]] name = "unicode-bidi" -version = "0.3.14" +version = "0.3.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f2528f27a9eb2b21e69c95319b30bd0efd85d09c379741b0f78ea1d86be2416" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" [[package]] name = "unicode-ident" -version = "1.0.3" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4f5b37a154999a8f3f98cc23a628d850e154479cd94decf3414696e12e31aaf" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4259d9d4425d9f0661581b804cb85fe66a4c631cadd8f490d1c13a35d5d9291" + [[package]] name = "unicode-segmentation" -version = "1.9.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" +checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" [[package]] name = "url" -version = "2.5.0" +version = "2.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "f7c25da092f0a868cdf09e8674cd3b7ef3a7d92a24253e663a2fb85e2496de56" dependencies = [ "form_urlencoded", - "idna 0.5.0", + "idna 1.0.0", "percent-encoding", "serde", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "utf-8" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + [[package]] name = "uuid" version = "0.8.2" @@ -2329,6 +3169,15 @@ dependencies = [ "serde", ] +[[package]] +name = "uuid" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +dependencies = [ + "getrandom", +] + [[package]] name = "valuable" version = "0.1.0" @@ -2349,25 +3198,24 @@ checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" [[package]] name = "want" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" dependencies = [ - "log", "try-lock", ] [[package]] name = "warp" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" +checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" dependencies = [ "bytes", "futures-channel", "futures-util", "headers", - "http", + "http 0.2.12", "hyper", "log", "mime", @@ -2375,13 +3223,11 @@ dependencies = [ "multer", "percent-encoding", "pin-project", - "rustls-pemfile", "scoped-tls", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-stream", "tokio-tungstenite", "tokio-util", "tower-service", @@ -2394,11 +3240,17 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" -version = "0.2.82" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -2406,24 +3258,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.82" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.66", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.32" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -2433,9 +3285,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.82" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2443,28 +3295,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.82" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.66", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.82" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "web-sys" -version = "0.3.59" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed055ab27f941423197eb86b2035720b1a3ce40504df082cac2ecc6ed73335a1" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -2476,10 +3328,10 @@ version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9973cb72c8587d5ad5efdb91e663d36177dc37725e6c90ca86c626b0cc45c93f" dependencies = [ - "base64 0.13.0", + "base64 0.13.1", "bytes", "cookie", - "http", + "http 0.2.12", "log", "serde", "serde_derive", @@ -2489,11 +3341,21 @@ dependencies = [ "url", ] +[[package]] +name = "whoami" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" +dependencies = [ + "redox_syscall 0.4.1", + "wasite", +] + [[package]] name = "widestring" -version = "0.5.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17882f045410753661207383517a6f62ec3dbeb6a4ed2acce01f0728238d1983" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" [[package]] name = "winapi" @@ -2513,11 +3375,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.5" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -2528,194 +3390,263 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows-core" -version = "0.50.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af6041b3f84485c21b57acdc0fee4f4f0c93f426053dc05fa5d6fc262537bbff" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets", + "windows-targets 0.52.5", ] [[package]] name = "windows-sys" -version = "0.36.1" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows_aarch64_msvc 0.36.1", - "windows_i686_gnu 0.36.1", - "windows_i686_msvc 0.36.1", - "windows_x86_64_gnu 0.36.1", - "windows_x86_64_msvc 0.36.1", + "windows-targets 0.48.5", ] [[package]] name = "windows-sys" -version = "0.42.0" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows_aarch64_gnullvm 0.42.1", - "windows_aarch64_msvc 0.42.1", - "windows_i686_gnu 0.42.1", - "windows_i686_msvc 0.42.1", - "windows_x86_64_gnu 0.42.1", - "windows_x86_64_gnullvm 0.42.1", - "windows_x86_64_msvc 0.42.1", + "windows-targets 0.52.5", ] [[package]] -name = "windows-sys" -version = "0.48.0" +name = "windows-targets" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows-targets", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] name = "windows-targets" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" -version = "0.36.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.42.1" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" - -[[package]] -name = "windows_i686_gnu" -version = "0.36.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" [[package]] -name = "windows_i686_msvc" -version = "0.36.1" +name = "windows_i686_gnullvm" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" -version = "0.42.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" -version = "0.36.1" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.42.1" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" +name = "windows_x86_64_gnullvm" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.1" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" +name = "windows_x86_64_msvc" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.36.1" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] -name = "windows_x86_64_msvc" -version = "0.42.1" +name = "winreg" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + +[[package]] +name = "write16" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" [[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" +name = "writeable" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" [[package]] -name = "winreg" -version = "0.7.0" +name = "yoke" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" dependencies = [ - "winapi", + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", ] [[package]] -name = "winreg" -version = "0.50.0" +name = "yoke-derive" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" dependencies = [ - "cfg-if", - "windows-sys 0.48.0", + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15e934569e47891f7d9411f1a451d947a60e000ab3bd24fbb970f000387d1b3b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb2cc8827d6c0994478a15c53f374f46fbd41bea663d809b14744bc42e6b109c" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97cf56601ee5052b4417d90c8755c6683473c926039908196cf35d99f893ebe7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", ] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index ede22e7dc..cbef6b9a1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -8,12 +8,23 @@ publish = false [dependencies] async-smtp = "0.6" check-if-email-exists = { path = "../core", features = ["headless"] } +csv = "1.3.0" dotenv = "0.15.0" openssl = { version = "0.10.64", features = ["vendored"] } sentry = "0.23" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +sqlx = { version = "0.7", features = [ + "runtime-tokio-native-tls", + "postgres", + "uuid", + "chrono", + "json", + "migrate", +] } +sqlxmq = "0.5" tokio = { version = "1.35", features = ["macros"] } tracing = "0.1.40" tracing-subscriber = "0.3.18" +uuid = "1.6" warp = "0.3" diff --git a/backend/README.md b/backend/README.md index 8ce5cc926..e4a81a063 100644 --- a/backend/README.md +++ b/backend/README.md @@ -45,17 +45,23 @@ Then send a `POST http://localhost:8080/v0/check_email` request with the followi These are the environment variables used to configure the HTTP server. To pass them to the Docker container, use the `-e {ENV_VAR}={VALUE}` flag. -| Env Var | Required? | Description | Default | -| -------------------- | --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | -| `RUST_LOG` | No | One of `trace,debug,warn,error,info`. 💡 PRO TIP: `RUST_LOG=debug` is very handful for debugging purposes. | not defined | -| `RCH_HTTP_HOST` | No | The host name to bind the HTTP server to. | `127.0.0.1` | -| `PORT` | No | The port to bind the HTTP server to, often populated by the cloud provider. | `8080` | -| `RCH_SENTRY_DSN` | No | If set, bug reports will be sent to this [Sentry](https://sentry.io) DSN. | not defined | -| `RCH_HEADER_SECRET` | No | If set, then all HTTP requests must have the `x-reacher-secret` header set to this value. This is used to protect the backend against public unwanted HTTP requests. | undefined | -| `RCH_FROM_EMAIL` | No | Email to use in the `` SMTP step. Can be overwritten by each API request's `from_email` field. | reacher.email@gmail.com | -| `RCH_HELLO_NAME` | No | Name to use in the `` SMTP step. Can be overwritten by each API request's `hello_name` field. | gmail.com | -| `RCH_SMTP_TIMEOUT` | No | Timeout for each SMTP connection. | 45s | -| `RCH_WEBDRIVER_ADDR` | No | Set to a running WebDriver process endpoint (e.g. `http://localhost:9515`) to use a headless navigator to password recovery pages to check Yahoo and Hotmail/Outlook addresses. We recommend `chromedriver` as it allows parallel requests. | not defined | +| Env Var | Required? | Description | Default | +| ----------------------------------- | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------- | +| `RUST_LOG` | No | One of `trace,debug,warn,error,info`. 💡 PRO TIP: `RUST_LOG=debug` is very handful for debugging purposes. | not defined | +| `RCH_HTTP_HOST` | No | The host name to bind the HTTP server to. | `127.0.0.1` | +| `PORT` | No | The port to bind the HTTP server to, often populated by the cloud provider. | `8080` | +| `RCH_SENTRY_DSN` | No | If set, bug reports will be sent to this [Sentry](https://sentry.io) DSN. | not defined | +| `RCH_HEADER_SECRET` | No | If set, then all HTTP requests must have the `x-reacher-secret` header set to this value. This is used to protect the backend against public unwanted HTTP requests. | undefined | +| `RCH_FROM_EMAIL` | No | Email to use in the `` SMTP step. Can be overwritten by each API request's `from_email` field. | reacher.email@gmail.com | +| `RCH_HELLO_NAME` | No | Name to use in the `` SMTP step. Can be overwritten by each API request's `hello_name` field. | gmail.com | +| `RCH_SMTP_TIMEOUT` | No | Timeout for each SMTP connection. | 45s | +| `RCH_WEBDRIVER_ADDR` | No | Set to a running WebDriver process endpoint (e.g. `http://localhost:9515`) to use a headless navigator to password recovery pages to check Yahoo and Hotmail/Outlook addresses. We recommend `chromedriver` as it allows parallel requests. | not defined | +| **For Bulk Verification:** | | | +| `RCH_ENABLE_BULK` | No | If set to `1`, then bulk verification endpoints will be added to the backend. | 0 | +| `DATABASE_URL` | Yes if `RCH_ENABLE_BULK==1` | [Bulk] Database connection string for storing results and task queue | not defined | +| `RCH_DATABASE_MAX_CONNECTIONS` | No | [Bulk] Connections created for the database pool | 5 | +| `RCH_MINIMUM_TASK_CONCURRENCY` | No | [Bulk] Minimum number of concurrent running tasks below which more tasks are fetched | 10 | +| `RCH_MAXIMUM_CONCURRENT_TASK_FETCH` | No | [Bulk] Maximum number of tasks fetched at once | 20 | ## REST API Documentation diff --git a/backend/migrations/20210316025847_setup.down.sql b/backend/migrations/20210316025847_setup.down.sql new file mode 100644 index 000000000..1aa472e8b --- /dev/null +++ b/backend/migrations/20210316025847_setup.down.sql @@ -0,0 +1,12 @@ +DROP FUNCTION mq_checkpoint; +DROP FUNCTION mq_keep_alive; +DROP FUNCTION mq_delete; +DROP FUNCTION mq_commit; +DROP FUNCTION mq_insert; +DROP FUNCTION mq_poll; +DROP FUNCTION mq_active_channels; +DROP FUNCTION mq_latest_message; +DROP TABLE mq_payloads; +DROP TABLE mq_msgs; +DROP FUNCTION mq_uuid_exists; +DROP TYPE mq_new_t; diff --git a/backend/migrations/20210316025847_setup.up.sql b/backend/migrations/20210316025847_setup.up.sql new file mode 100644 index 000000000..bf7f8f859 --- /dev/null +++ b/backend/migrations/20210316025847_setup.up.sql @@ -0,0 +1,289 @@ +CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; + +-- The UDT for creating messages +CREATE TYPE mq_new_t AS ( + -- Unique message ID + id UUID, + -- Delay before message is processed + delay INTERVAL, + -- Number of retries if initial processing fails + retries INT, + -- Initial backoff between retries + retry_backoff INTERVAL, + -- Name of channel + channel_name TEXT, + -- Arguments to channel + channel_args TEXT, + -- Interval for two-phase commit (or NULL to disable two-phase commit) + commit_interval INTERVAL, + -- Whether this message should be processed in order with respect to other + -- ordered messages. + ordered BOOLEAN, + -- Name of message + name TEXT, + -- JSON payload + payload_json TEXT, + -- Binary payload + payload_bytes BYTEA +); + +-- Small, frequently updated table of messages +CREATE TABLE mq_msgs ( + id UUID PRIMARY KEY, + created_at TIMESTAMPTZ DEFAULT NOW(), + attempt_at TIMESTAMPTZ DEFAULT NOW(), + attempts INT NOT NULL DEFAULT 5, + retry_backoff INTERVAL NOT NULL DEFAULT INTERVAL '1 second', + channel_name TEXT NOT NULL, + channel_args TEXT NOT NULL, + commit_interval INTERVAL, + after_message_id UUID DEFAULT uuid_nil() REFERENCES mq_msgs(id) ON DELETE SET DEFAULT +); + +-- Insert dummy message so that the 'nil' UUID can be referenced +INSERT INTO mq_msgs (id, channel_name, channel_args, after_message_id) VALUES (uuid_nil(), '', '', NULL); + +-- Internal helper function to check that a UUID is neither NULL nor NIL +CREATE FUNCTION mq_uuid_exists( + id UUID +) RETURNS BOOLEAN AS $$ + SELECT id IS NOT NULL AND id != uuid_nil() +$$ LANGUAGE SQL IMMUTABLE; + +-- Index for polling +CREATE INDEX ON mq_msgs(channel_name, channel_args, attempt_at) WHERE id != uuid_nil() AND NOT mq_uuid_exists(after_message_id); +-- Index for adding messages +CREATE INDEX ON mq_msgs(channel_name, channel_args, created_at, id) WHERE id != uuid_nil() AND after_message_id IS NOT NULL; + +-- Index for ensuring strict message order +CREATE UNIQUE INDEX mq_msgs_channel_name_channel_args_after_message_id_idx ON mq_msgs(channel_name, channel_args, after_message_id); + + +-- Large, less frequently updated table of message payloads +CREATE TABLE mq_payloads( + id UUID PRIMARY KEY, + name TEXT NOT NULL, + payload_json JSONB, + payload_bytes BYTEA +); + +-- Internal helper function to return the most recently added message in a queue. +CREATE FUNCTION mq_latest_message(from_channel_name TEXT, from_channel_args TEXT) +RETURNS UUID AS $$ + SELECT COALESCE( + ( + SELECT id FROM mq_msgs + WHERE channel_name = from_channel_name + AND channel_args = from_channel_args + AND after_message_id IS NOT NULL + AND id != uuid_nil() + ORDER BY created_at DESC, id DESC + LIMIT 1 + ), + uuid_nil() + ) +$$ LANGUAGE SQL STABLE; + +-- Internal helper function to randomly select a set of channels with "ready" messages. +CREATE FUNCTION mq_active_channels(channel_names TEXT[], batch_size INT) +RETURNS TABLE(name TEXT, args TEXT) AS $$ + SELECT channel_name, channel_args + FROM mq_msgs + WHERE id != uuid_nil() + AND attempt_at <= NOW() + AND (channel_names IS NULL OR channel_name = ANY(channel_names)) + AND NOT mq_uuid_exists(after_message_id) + GROUP BY channel_name, channel_args + ORDER BY RANDOM() + LIMIT batch_size +$$ LANGUAGE SQL STABLE; + +-- Main entry-point for job runner: pulls a batch of messages from the queue. +CREATE FUNCTION mq_poll(channel_names TEXT[], batch_size INT DEFAULT 1) +RETURNS TABLE( + id UUID, + is_committed BOOLEAN, + name TEXT, + payload_json TEXT, + payload_bytes BYTEA, + retry_backoff INTERVAL, + wait_time INTERVAL +) AS $$ +BEGIN + RETURN QUERY UPDATE mq_msgs + SET + attempt_at = CASE WHEN mq_msgs.attempts = 1 THEN NULL ELSE NOW() + mq_msgs.retry_backoff END, + attempts = mq_msgs.attempts - 1, + retry_backoff = mq_msgs.retry_backoff * 2 + FROM ( + SELECT + msgs.id + FROM mq_active_channels(channel_names, batch_size) AS active_channels + INNER JOIN LATERAL ( + SELECT * FROM mq_msgs + WHERE mq_msgs.id != uuid_nil() + AND mq_msgs.attempt_at <= NOW() + AND mq_msgs.channel_name = active_channels.name + AND mq_msgs.channel_args = active_channels.args + AND NOT mq_uuid_exists(mq_msgs.after_message_id) + ORDER BY mq_msgs.attempt_at ASC + LIMIT batch_size + ) AS msgs ON TRUE + LIMIT batch_size + ) AS messages_to_update + LEFT JOIN mq_payloads ON mq_payloads.id = messages_to_update.id + WHERE mq_msgs.id = messages_to_update.id + RETURNING + mq_msgs.id, + mq_msgs.commit_interval IS NULL, + mq_payloads.name, + mq_payloads.payload_json::TEXT, + mq_payloads.payload_bytes, + mq_msgs.retry_backoff / 2, + interval '0' AS wait_time; + + IF NOT FOUND THEN + RETURN QUERY SELECT + NULL::UUID, + NULL::BOOLEAN, + NULL::TEXT, + NULL::TEXT, + NULL::BYTEA, + NULL::INTERVAL, + MIN(mq_msgs.attempt_at) - NOW() + FROM mq_msgs + WHERE mq_msgs.id != uuid_nil() + AND NOT mq_uuid_exists(mq_msgs.after_message_id) + AND (channel_names IS NULL OR mq_msgs.channel_name = ANY(channel_names)); + END IF; +END; +$$ LANGUAGE plpgsql; + +-- Creates new messages +CREATE FUNCTION mq_insert(new_messages mq_new_t[]) +RETURNS VOID AS $$ +BEGIN + PERFORM pg_notify(CONCAT('mq_', channel_name), '') + FROM unnest(new_messages) AS new_msgs + GROUP BY channel_name; + + IF FOUND THEN + PERFORM pg_notify('mq', ''); + END IF; + + INSERT INTO mq_payloads ( + id, + name, + payload_json, + payload_bytes + ) SELECT + id, + name, + payload_json::JSONB, + payload_bytes + FROM UNNEST(new_messages); + + INSERT INTO mq_msgs ( + id, + attempt_at, + attempts, + retry_backoff, + channel_name, + channel_args, + commit_interval, + after_message_id + ) + SELECT + id, + NOW() + delay + COALESCE(commit_interval, INTERVAL '0'), + retries + 1, + retry_backoff, + channel_name, + channel_args, + commit_interval, + CASE WHEN ordered + THEN + LAG(id, 1, mq_latest_message(channel_name, channel_args)) + OVER (PARTITION BY channel_name, channel_args, ordered ORDER BY id) + ELSE + NULL + END + FROM UNNEST(new_messages); +END; +$$ LANGUAGE plpgsql; + +-- Commits messages previously created with a non-NULL commit interval. +CREATE FUNCTION mq_commit(msg_ids UUID[]) +RETURNS VOID AS $$ +BEGIN + UPDATE mq_msgs + SET + attempt_at = attempt_at - commit_interval, + commit_interval = NULL + WHERE id = ANY(msg_ids) + AND commit_interval IS NOT NULL; +END; +$$ LANGUAGE plpgsql; + + +-- Deletes messages from the queue. This occurs when a message has been +-- processed, or when it expires without being processed. +CREATE FUNCTION mq_delete(msg_ids UUID[]) +RETURNS VOID AS $$ +BEGIN + PERFORM pg_notify(CONCAT('mq_', channel_name), '') + FROM mq_msgs + WHERE id = ANY(msg_ids) + AND after_message_id = uuid_nil() + GROUP BY channel_name; + + IF FOUND THEN + PERFORM pg_notify('mq', ''); + END IF; + + DELETE FROM mq_msgs WHERE id = ANY(msg_ids); + DELETE FROM mq_payloads WHERE id = ANY(msg_ids); +END; +$$ LANGUAGE plpgsql; + + +-- Can be called during the initial commit interval, or when processing +-- a message. Indicates that the caller is still active and will prevent either +-- the commit interval elapsing or the message being retried for the specified +-- interval. +CREATE FUNCTION mq_keep_alive(msg_ids UUID[], duration INTERVAL) +RETURNS VOID AS $$ + UPDATE mq_msgs + SET + attempt_at = NOW() + duration, + commit_interval = commit_interval + ((NOW() + duration) - attempt_at) + WHERE id = ANY(msg_ids) + AND attempt_at < NOW() + duration; +$$ LANGUAGE SQL; + + +-- Called during lengthy processing of a message to checkpoint the progress. +-- As well as behaving like `mq_keep_alive`, the message payload can be +-- updated. +CREATE FUNCTION mq_checkpoint( + msg_id UUID, + duration INTERVAL, + new_payload_json TEXT, + new_payload_bytes BYTEA, + extra_retries INT +) +RETURNS VOID AS $$ + UPDATE mq_msgs + SET + attempt_at = GREATEST(attempt_at, NOW() + duration), + attempts = attempts + COALESCE(extra_retries, 0) + WHERE id = msg_id; + + UPDATE mq_payloads + SET + payload_json = COALESCE(new_payload_json::JSONB, payload_json), + payload_bytes = COALESCE(new_payload_bytes, payload_bytes) + WHERE + id = msg_id; +$$ LANGUAGE SQL; + diff --git a/backend/migrations/20210921115907_clear.down.sql b/backend/migrations/20210921115907_clear.down.sql new file mode 100644 index 000000000..e15638db2 --- /dev/null +++ b/backend/migrations/20210921115907_clear.down.sql @@ -0,0 +1,2 @@ +DROP FUNCTION mq_clear; +DROP FUNCTION mq_clear_all; diff --git a/backend/migrations/20210921115907_clear.up.sql b/backend/migrations/20210921115907_clear.up.sql new file mode 100644 index 000000000..bd1c1f607 --- /dev/null +++ b/backend/migrations/20210921115907_clear.up.sql @@ -0,0 +1,21 @@ +-- Deletes all messages from a list of channel names. +CREATE FUNCTION mq_clear(channel_names TEXT[]) +RETURNS VOID AS $$ +BEGIN + WITH deleted_ids AS ( + DELETE FROM mq_msgs WHERE channel_name = ANY(channel_names) RETURNING id + ) + DELETE FROM mq_payloads WHERE id IN (SELECT id FROM deleted_ids); +END; +$$ LANGUAGE plpgsql; + +-- Deletes all messages. +CREATE FUNCTION mq_clear_all() +RETURNS VOID AS $$ +BEGIN + WITH deleted_ids AS ( + DELETE FROM mq_msgs RETURNING id + ) + DELETE FROM mq_payloads WHERE id IN (SELECT id FROM deleted_ids); +END; +$$ LANGUAGE plpgsql; diff --git a/backend/migrations/20211013151757_fix_mq_latest_message.down.sql b/backend/migrations/20211013151757_fix_mq_latest_message.down.sql new file mode 100644 index 000000000..d09bd4afd --- /dev/null +++ b/backend/migrations/20211013151757_fix_mq_latest_message.down.sql @@ -0,0 +1,15 @@ +CREATE OR REPLACE FUNCTION mq_latest_message(from_channel_name TEXT, from_channel_args TEXT) +RETURNS UUID AS $$ + SELECT COALESCE( + ( + SELECT id FROM mq_msgs + WHERE channel_name = from_channel_name + AND channel_args = from_channel_args + AND after_message_id IS NOT NULL + AND id != uuid_nil() + ORDER BY created_at DESC, id DESC + LIMIT 1 + ), + uuid_nil() + ) +$$ LANGUAGE SQL STABLE; diff --git a/backend/migrations/20211013151757_fix_mq_latest_message.up.sql b/backend/migrations/20211013151757_fix_mq_latest_message.up.sql new file mode 100644 index 000000000..b987c5e1e --- /dev/null +++ b/backend/migrations/20211013151757_fix_mq_latest_message.up.sql @@ -0,0 +1,19 @@ +CREATE OR REPLACE FUNCTION mq_latest_message(from_channel_name TEXT, from_channel_args TEXT) +RETURNS UUID AS $$ + SELECT COALESCE( + ( + SELECT id FROM mq_msgs + WHERE channel_name = from_channel_name + AND channel_args = from_channel_args + AND after_message_id IS NOT NULL + AND id != uuid_nil() + AND NOT EXISTS( + SELECT * FROM mq_msgs AS mq_msgs2 + WHERE mq_msgs2.after_message_id = mq_msgs.id + ) + ORDER BY created_at DESC + LIMIT 1 + ), + uuid_nil() + ) +$$ LANGUAGE SQL STABLE; \ No newline at end of file diff --git a/backend/migrations/20220117025847_email_data.down.sql b/backend/migrations/20220117025847_email_data.down.sql new file mode 100644 index 000000000..ed211e337 --- /dev/null +++ b/backend/migrations/20220117025847_email_data.down.sql @@ -0,0 +1,4 @@ +DROP INDEX job_emails; +DROP TABLE email_results; +DROP TABLE bulk_jobs; +DROP TYPE valid_status; \ No newline at end of file diff --git a/backend/migrations/20220117025847_email_data.up.sql b/backend/migrations/20220117025847_email_data.up.sql new file mode 100644 index 000000000..d53e83d52 --- /dev/null +++ b/backend/migrations/20220117025847_email_data.up.sql @@ -0,0 +1,12 @@ +CREATE TABLE bulk_jobs ( + id SERIAL PRIMARY KEY, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + total_records INTEGER NOT NULL +); +CREATE TABLE email_results ( + id SERIAL PRIMARY KEY, + job_id INTEGER, + result JSONB, + FOREIGN KEY (job_id) REFERENCES bulk_jobs(id) +); +CREATE INDEX job_emails ON email_results USING HASH (job_id); \ No newline at end of file diff --git a/backend/migrations/20220208120856_fix_concurrent_poll.down.sql b/backend/migrations/20220208120856_fix_concurrent_poll.down.sql new file mode 100644 index 000000000..6cd2d21eb --- /dev/null +++ b/backend/migrations/20220208120856_fix_concurrent_poll.down.sql @@ -0,0 +1,60 @@ +-- Main entry-point for job runner: pulls a batch of messages from the queue. +CREATE OR REPLACE FUNCTION mq_poll(channel_names TEXT[], batch_size INT DEFAULT 1) +RETURNS TABLE( + id UUID, + is_committed BOOLEAN, + name TEXT, + payload_json TEXT, + payload_bytes BYTEA, + retry_backoff INTERVAL, + wait_time INTERVAL +) AS $$ +BEGIN + RETURN QUERY UPDATE mq_msgs + SET + attempt_at = CASE WHEN mq_msgs.attempts = 1 THEN NULL ELSE NOW() + mq_msgs.retry_backoff END, + attempts = mq_msgs.attempts - 1, + retry_backoff = mq_msgs.retry_backoff * 2 + FROM ( + SELECT + msgs.id + FROM mq_active_channels(channel_names, batch_size) AS active_channels + INNER JOIN LATERAL ( + SELECT * FROM mq_msgs + WHERE mq_msgs.id != uuid_nil() + AND mq_msgs.attempt_at <= NOW() + AND mq_msgs.channel_name = active_channels.name + AND mq_msgs.channel_args = active_channels.args + AND NOT mq_uuid_exists(mq_msgs.after_message_id) + ORDER BY mq_msgs.attempt_at ASC + LIMIT batch_size + ) AS msgs ON TRUE + LIMIT batch_size + ) AS messages_to_update + LEFT JOIN mq_payloads ON mq_payloads.id = messages_to_update.id + WHERE mq_msgs.id = messages_to_update.id + RETURNING + mq_msgs.id, + mq_msgs.commit_interval IS NULL, + mq_payloads.name, + mq_payloads.payload_json::TEXT, + mq_payloads.payload_bytes, + mq_msgs.retry_backoff / 2, + interval '0' AS wait_time; + + IF NOT FOUND THEN + RETURN QUERY SELECT + NULL::UUID, + NULL::BOOLEAN, + NULL::TEXT, + NULL::TEXT, + NULL::BYTEA, + NULL::INTERVAL, + MIN(mq_msgs.attempt_at) - NOW() + FROM mq_msgs + WHERE mq_msgs.id != uuid_nil() + AND NOT mq_uuid_exists(mq_msgs.after_message_id) + AND (channel_names IS NULL OR mq_msgs.channel_name = ANY(channel_names)); + END IF; +END; +$$ LANGUAGE plpgsql; \ No newline at end of file diff --git a/backend/migrations/20220208120856_fix_concurrent_poll.up.sql b/backend/migrations/20220208120856_fix_concurrent_poll.up.sql new file mode 100644 index 000000000..cae6151ed --- /dev/null +++ b/backend/migrations/20220208120856_fix_concurrent_poll.up.sql @@ -0,0 +1,62 @@ + +-- Main entry-point for job runner: pulls a batch of messages from the queue. +CREATE OR REPLACE FUNCTION mq_poll(channel_names TEXT[], batch_size INT DEFAULT 1) +RETURNS TABLE( + id UUID, + is_committed BOOLEAN, + name TEXT, + payload_json TEXT, + payload_bytes BYTEA, + retry_backoff INTERVAL, + wait_time INTERVAL +) AS $$ +BEGIN + RETURN QUERY UPDATE mq_msgs + SET + attempt_at = CASE WHEN mq_msgs.attempts = 1 THEN NULL ELSE NOW() + mq_msgs.retry_backoff END, + attempts = mq_msgs.attempts - 1, + retry_backoff = mq_msgs.retry_backoff * 2 + FROM ( + SELECT + msgs.id + FROM mq_active_channels(channel_names, batch_size) AS active_channels + INNER JOIN LATERAL ( + SELECT mq_msgs.id FROM mq_msgs + WHERE mq_msgs.id != uuid_nil() + AND mq_msgs.attempt_at <= NOW() + AND mq_msgs.channel_name = active_channels.name + AND mq_msgs.channel_args = active_channels.args + AND NOT mq_uuid_exists(mq_msgs.after_message_id) + ORDER BY mq_msgs.attempt_at ASC + LIMIT batch_size + ) AS msgs ON TRUE + LIMIT batch_size + ) AS messages_to_update + LEFT JOIN mq_payloads ON mq_payloads.id = messages_to_update.id + WHERE mq_msgs.id = messages_to_update.id + AND mq_msgs.attempt_at <= NOW() + RETURNING + mq_msgs.id, + mq_msgs.commit_interval IS NULL, + mq_payloads.name, + mq_payloads.payload_json::TEXT, + mq_payloads.payload_bytes, + mq_msgs.retry_backoff / 2, + interval '0' AS wait_time; + + IF NOT FOUND THEN + RETURN QUERY SELECT + NULL::UUID, + NULL::BOOLEAN, + NULL::TEXT, + NULL::TEXT, + NULL::BYTEA, + NULL::INTERVAL, + MIN(mq_msgs.attempt_at) - NOW() + FROM mq_msgs + WHERE mq_msgs.id != uuid_nil() + AND NOT mq_uuid_exists(mq_msgs.after_message_id) + AND (channel_names IS NULL OR mq_msgs.channel_name = ANY(channel_names)); + END IF; +END; +$$ LANGUAGE plpgsql; diff --git a/backend/migrations/20220713122907_fix-clear_all-keep-nil-message.down.sql b/backend/migrations/20220713122907_fix-clear_all-keep-nil-message.down.sql new file mode 100644 index 000000000..d2f607c5b --- /dev/null +++ b/backend/migrations/20220713122907_fix-clear_all-keep-nil-message.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20220713122907_fix-clear_all-keep-nil-message.up.sql b/backend/migrations/20220713122907_fix-clear_all-keep-nil-message.up.sql new file mode 100644 index 000000000..4dd1f0b3a --- /dev/null +++ b/backend/migrations/20220713122907_fix-clear_all-keep-nil-message.up.sql @@ -0,0 +1,29 @@ +CREATE OR REPLACE FUNCTION mq_clear(channel_names TEXT[]) +RETURNS VOID AS $$ +BEGIN + WITH deleted_ids AS ( + DELETE FROM mq_msgs + WHERE channel_name = ANY(channel_names) + AND id != uuid_nil() + RETURNING id + ) + DELETE FROM mq_payloads WHERE id IN (SELECT id FROM deleted_ids); +END; +$$ LANGUAGE plpgsql; +COMMENT ON FUNCTION mq_clear IS + 'Deletes all messages with corresponding payloads from a list of channel names'; + + +CREATE OR REPLACE FUNCTION mq_clear_all() +RETURNS VOID AS $$ +BEGIN + WITH deleted_ids AS ( + DELETE FROM mq_msgs + WHERE id != uuid_nil() + RETURNING id + ) + DELETE FROM mq_payloads WHERE id IN (SELECT id FROM deleted_ids); +END; +$$ LANGUAGE plpgsql; +COMMENT ON FUNCTION mq_clear_all IS + 'Deletes all messages with corresponding payloads'; diff --git a/backend/migrations/20220810141100_result_created_at.down.sql b/backend/migrations/20220810141100_result_created_at.down.sql new file mode 100644 index 000000000..80b95f109 --- /dev/null +++ b/backend/migrations/20220810141100_result_created_at.down.sql @@ -0,0 +1,2 @@ +ALTER TABLE email_results +DROP COLUMN created_at; diff --git a/backend/migrations/20220810141100_result_created_at.up.sql b/backend/migrations/20220810141100_result_created_at.up.sql new file mode 100644 index 000000000..ab28f0191 --- /dev/null +++ b/backend/migrations/20220810141100_result_created_at.up.sql @@ -0,0 +1,2 @@ +ALTER TABLE email_results +ADD created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(); \ No newline at end of file diff --git a/backend/migrations/README.md b/backend/migrations/README.md new file mode 100644 index 000000000..730cf64de --- /dev/null +++ b/backend/migrations/README.md @@ -0,0 +1,33 @@ +# Database and Migrations + +## Migrations + +All migrations in this folder are embedded directly in the `reacher_backend` binary, so you don't need to run the migrations manually. + +The migrations come from 2 sources: + +- `sqlxmq` migrations +- Reacher's own migrations + +## `sqlxmq` migrations + +The following migration files have been copied from the [sqlxmq repo](https://github.com/Diggsey/sqlxmq) as per the [given instructions](https://github.com/Diggsey/sqlxmq/blob/6d3ed6fb99e7592e370a7f3ec074ce0bebae62fd/README.md?plain=1#L111): + +- `20210316025847_setup.{up,down}.sql` +- `20210921115907_clear.{up,down}.sql` +- `20211013151757_fix_mq_latest_message.{up,down}.sql` +- `20220208120856_fix_concurrent_poll.{up,down}.sql` +- `20220713122907_fix-clear_all-keep-nil-message.{up,down}.sql` + +## Reacher migrations + +The following migrations are specific to Reacher: + +- `20220117025847_email_data.{up,down}.sql`: set up the `bulk_jobs` and `email_results` tables +- `20220810141100_result_created_at.{up,down}.sql`: add a `created_at` column on `email_result` + +## Advanced Usage + +For more advanced usage (such as reverting to an old state), please use the `sqlx` CLI command. + +See https://github.com/launchbadge/sqlx/blob/main/sqlx-cli/README.md diff --git a/backend/src/bin/prune_db.rs b/backend/src/bin/prune_db.rs new file mode 100644 index 000000000..5c59ba8ee --- /dev/null +++ b/backend/src/bin/prune_db.rs @@ -0,0 +1,81 @@ +use sqlx::PgPool; +use sqlx::Result; +use tracing::info; + +#[tokio::main] +async fn main() -> Result<()> { + dotenv::dotenv().expect("Unable to load environment variables from .env file"); + tracing_subscriber::fmt::init(); + + let db_url = std::env::var("DATABASE_URL").expect("Unable to read DATABASE_URL env var"); + let dry_mode: bool = std::env::var("DRY_RUN").is_ok(); + let days_old_str = std::env::var("DAYS_OLD").expect("Unable to read DAYS_OLD env var"); + let days_old: i32 = days_old_str + .parse() + .expect("Unable to parse DAYS_OLD as integer"); + + let pool = PgPool::connect(&db_url).await?; + + // Fetch the list of job IDs that match the criteria + let query = format!( + "SELECT b.id + FROM bulk_jobs b + JOIN ( + SELECT job_id, COUNT(*) as total_processed + FROM email_results + GROUP BY job_id + ) e ON b.id = e.job_id + WHERE b.total_records = e.total_processed + AND b.created_at <= current_date - interval '{} days'", + days_old + ); + + let job_ids_to_delete: Vec<(i32,)> = sqlx::query_as(&query).fetch_all(&pool).await?; + + match (dry_mode, job_ids_to_delete.is_empty()) { + (true, _) => info!("Job ids to delete {:?}", job_ids_to_delete), + (false, true) => info!("No jobs to delete"), + (false, false) => { + // Start a transaction + let tx = pool.begin().await?; + + // Before deleting from bulk_jobs, delete the corresponding records from email_results in a batch + let delete_email_results_query = + "DELETE FROM email_results WHERE job_id = ANY($1::int[])"; + + // Convert job_ids_to_delete to Vec before binding + let job_ids_to_delete_vec: Vec = + job_ids_to_delete.iter().map(|&(id,)| id).collect(); + + // Execute the delete query for email_results in a batch within the transaction + sqlx::query(delete_email_results_query) + .bind(&job_ids_to_delete_vec) + .execute(&pool) // Use execute on the query builder + .await?; + + info!( + "Email results for job IDs {:?} deleted successfully.", + job_ids_to_delete + ); + + // safely delete the records from bulk_jobs + let delete_bulk_jobs_query = "DELETE FROM bulk_jobs WHERE id = ANY($1::int[])"; + + // Execute the delete query for bulk_jobs in a batch within the transaction + sqlx::query(delete_bulk_jobs_query) + .bind(&job_ids_to_delete_vec) + .execute(&pool) // Use execute on the query builder + .await?; + + info!( + "Bulk jobs records with IDs {:?} deleted successfully.", + job_ids_to_delete + ); + + // Commit the transaction if both deletes are successful + tx.commit().await?; + } + } + + Ok(()) +} diff --git a/backend/src/http/mod.rs b/backend/src/http/mod.rs index 958369d12..97634f10b 100644 --- a/backend/src/http/mod.rs +++ b/backend/src/http/mod.rs @@ -21,6 +21,9 @@ use std::env; use std::net::IpAddr; use check_if_email_exists::LOG_TARGET; +use sqlx::postgres::PgPoolOptions; +use sqlx::{Pool, Postgres}; +use sqlxmq::JobRunnerHandle; use tracing::info; use warp::Filter; @@ -28,18 +31,27 @@ use super::errors; /// Creates the routes for the HTTP server. /// Making it public so that it can be used in tests/check_email.rs. -pub fn create_routes_without_bulk( +pub fn create_routes( + o: Option>, ) -> impl Filter + Clone { version::get::get_version() .or(v0::check_email::post::post_check_email()) + // The 3 following routes will 404 if o is None. + .or(v0::bulk::post::create_bulk_job(o.clone())) + .or(v0::bulk::get::get_bulk_job_status(o.clone())) + .or(v0::bulk::results::get_bulk_job_result(o)) .recover(errors::handle_rejection) } /// Runs the Warp server. /// /// This function starts the Warp server and listens for incoming requests. -/// It returns a `Result` indicating whether the server started successfully or encountered an error. -pub async fn run_warp_server() -> Result<(), Box> { +/// It returns a `Result` indicating whether the server started successfully or +/// encountered an error, as well as an optional `JobRunnerHandle` if the bulk +/// job listener is enabled. The handle can be used to stop the listener or to +/// keep it alive. +pub async fn run_warp_server( +) -> Result, Box> { let host = env::var("RCH_HTTP_HOST") .unwrap_or_else(|_| "127.0.0.1".into()) .parse::() @@ -51,10 +63,43 @@ pub async fn run_warp_server() -> Result<(), Box Result, sqlx::Error> { + let pg_conn = + env::var("DATABASE_URL").expect("Environment variable DATABASE_URL should be set"); + let pg_max_conn = env::var("RCH_DATABASE_MAX_CONNECTIONS").map_or(5, |var| { + var.parse::() + .expect("Environment variable RCH_DATABASE_MAX_CONNECTIONS should parse to u32") + }); + + // create connection pool with database + // connection pool internally the shared db connection + // with arc so it can safely be cloned and shared across threads + let pool = PgPoolOptions::new() + .max_connections(pg_max_conn) + .connect(pg_conn.as_str()) + .await?; + + sqlx::migrate!("./migrations").run(&pool).await?; + + Ok(pool) } diff --git a/backend/src/http/v0/bulk/db.rs b/backend/src/http/v0/bulk/db.rs new file mode 100644 index 000000000..9d995f046 --- /dev/null +++ b/backend/src/http/v0/bulk/db.rs @@ -0,0 +1,35 @@ +// Reacher - Email Verification +// Copyright (C) 2018-2023 Reacher + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use sqlx::{Pool, Postgres}; +use warp::Filter; + +/// Warp filter that extracts a Pg Pool if the option is Some, or else rejects +/// with a 404. +pub fn with_db( + o: Option>, +) -> impl Filter,), Error = warp::Rejection> + Clone { + warp::any().and_then(move || { + let o = o.clone(); // Still not 100% sure why I need to clone here... + async move { + if let Some(conn_pool) = o { + Ok(conn_pool) + } else { + Err(warp::reject::not_found()) + } + } + }) +} diff --git a/backend/src/http/v0/bulk/error.rs b/backend/src/http/v0/bulk/error.rs new file mode 100644 index 000000000..de5ddc9fd --- /dev/null +++ b/backend/src/http/v0/bulk/error.rs @@ -0,0 +1,44 @@ +// Reacher - Email Verification +// Copyright (C) 2018-2023 Reacher + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use warp::reject; + +#[derive(Debug)] +pub enum CsvError { + CsvLib(csv::Error), + CsvLibWriter(Box>>>), + Parse(&'static str), +} + +/// Catch all error struct for the bulk endpoints +#[derive(Debug)] +pub enum BulkError { + EmptyInput, + JobInProgress, + Db(sqlx::Error), + Csv(CsvError), + Json(serde_json::Error), +} + +// Defaults to Internal server error +impl reject::Reject for BulkError {} + +// wrap sql errors as db errors for reacher +impl From for BulkError { + fn from(e: sqlx::Error) -> Self { + BulkError::Db(e) + } +} diff --git a/backend/src/http/v0/bulk/get.rs b/backend/src/http/v0/bulk/get.rs new file mode 100644 index 000000000..004f591c1 --- /dev/null +++ b/backend/src/http/v0/bulk/get.rs @@ -0,0 +1,171 @@ +// Reacher - Email Verification +// Copyright (C) 2018-2023 Reacher + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! This file implements the `GET /bulk/{id}` endpoint. + +use check_if_email_exists::LOG_TARGET; +use serde::Serialize; +use sqlx::types::chrono::{DateTime, Utc}; +use sqlx::{Pool, Postgres}; +use tracing::error; +use warp::Filter; + +use super::{db::with_db, error::BulkError}; + +/// NOTE: Type conversions from postgres to rust types +/// are according to the table given by +/// [sqlx here](https://docs.rs/sqlx/latest/sqlx/postgres/types/index.html) +#[derive(Debug, Serialize, PartialEq, Eq)] +enum ValidStatus { + Running, + Completed, +} + +/// Job record stores the information about a submitted job +/// +/// `job_status` field is an update on read field. It's +/// status will be derived from counting number of +/// completed email verification tasks. It will be updated +/// with the most recent status of the job. +#[derive(sqlx::FromRow, Debug, Serialize)] +struct JobRecord { + id: i32, + created_at: DateTime, + total_records: i32, +} + +/// Summary of a bulk verification job status +#[derive(Debug, Serialize)] +struct JobStatusSummary { + total_safe: i32, + total_risky: i32, + total_invalid: i32, + total_unknown: i32, +} + +/// Complete information about a bulk verification job +#[derive(Debug, Serialize)] +struct JobStatusResponseBody { + job_id: i32, + created_at: DateTime, + finished_at: Option>, + total_records: i32, + total_processed: i32, + summary: JobStatusSummary, + job_status: ValidStatus, +} + +async fn job_status( + job_id: i32, + conn_pool: Pool, +) -> Result { + let job_rec = sqlx::query_as!( + JobRecord, + r#" + SELECT id, created_at, total_records FROM bulk_jobs + WHERE id = $1 + LIMIT 1 + "#, + job_id + ) + .fetch_one(&conn_pool) + .await + .map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to get job record for [job={}] with [error={}]", + job_id, e + ); + BulkError::from(e) + })?; + + let agg_info = sqlx::query!( + r#" + SELECT + COUNT(*) as total_processed, + COUNT(CASE WHEN result ->> 'is_reachable' LIKE 'safe' THEN 1 END) as safe_count, + COUNT(CASE WHEN result ->> 'is_reachable' LIKE 'risky' THEN 1 END) as risky_count, + COUNT(CASE WHEN result ->> 'is_reachable' LIKE 'invalid' THEN 1 END) as invalid_count, + COUNT(CASE WHEN result ->> 'is_reachable' LIKE 'unknown' THEN 1 END) as unknown_count, + (SELECT created_at FROM email_results WHERE job_id = $1 ORDER BY created_at DESC LIMIT 1) as finished_at + FROM email_results + WHERE job_id = $1 + "#, + job_id + ) + .fetch_one(&conn_pool) + .await + .map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to get aggregate info for [job={}] with [error={}]", + job_id, + e + ); + BulkError::from(e) + })?; + + let (job_status, finished_at) = if (agg_info + .total_processed + .expect("sql COUNT() returns an int. qed.") as i32) + < job_rec.total_records + { + (ValidStatus::Running, None) + } else { + ( + ValidStatus::Completed, + Some( + agg_info + .finished_at + .expect("always at least one task in the job. qed."), + ), + ) + }; + + Ok(warp::reply::json(&JobStatusResponseBody { + job_id: job_rec.id, + created_at: job_rec.created_at, + finished_at, + total_records: job_rec.total_records, + total_processed: agg_info + .total_processed + .expect("sql COUNT returns an int. qed.") as i32, + summary: JobStatusSummary { + total_safe: agg_info.safe_count.expect("sql COUNT returns an int. qed.") as i32, + total_risky: agg_info + .risky_count + .expect("sql COUNT returns an int. qed.") as i32, + total_invalid: agg_info + .invalid_count + .expect("sql COUNT returns an int. qed.") as i32, + total_unknown: agg_info + .unknown_count + .expect("sql COUNT returns an int. qed.") as i32, + }, + job_status, + })) +} + +pub fn get_bulk_job_status( + o: Option>, +) -> impl Filter + Clone { + warp::path!("v0" / "bulk" / i32) + .and(warp::get()) + .and(with_db(o)) + .and_then(job_status) + // View access logs by setting `RUST_LOG=reacher`. + .with(warp::log(LOG_TARGET)) +} diff --git a/backend/src/http/v0/bulk/mod.rs b/backend/src/http/v0/bulk/mod.rs new file mode 100644 index 000000000..061d27e3a --- /dev/null +++ b/backend/src/http/v0/bulk/mod.rs @@ -0,0 +1,65 @@ +// Reacher - Email Verification +// Copyright (C) 2018-2023 Reacher + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +mod db; +mod error; +pub mod get; +pub mod post; +pub mod results; +mod task; + +use std::env; + +use check_if_email_exists::LOG_TARGET; +use sqlx::{Pool, Postgres}; +use sqlxmq::{JobRegistry, JobRunnerHandle}; +use tracing::info; + +pub use task::email_verification_task; + +/// Create a job registry with one task: the email verification task. +pub async fn create_job_registry(pool: &Pool) -> Result { + let min_task_conc = env::var("RCH_MINIMUM_TASK_CONCURRENCY").map_or(10, |var| { + var.parse::() + .expect("Environment variable RCH_MINIMUM_TASK_CONCURRENCY should parse to usize") + }); + let max_conc_task_fetch = env::var("RCH_MAXIMUM_CONCURRENT_TASK_FETCH").map_or(20, |var| { + var.parse::() + .expect("Environment variable RCH_MAXIMUM_CONCURRENT_TASK_FETCH should parse to usize") + }); + + // registry needs to be given list of jobs it can accept + let registry = JobRegistry::new(&[email_verification_task]); + + // create runner for the message queue associated + // with this job registry + let registry = registry + // Create a job runner using the connection pool. + .runner(pool) + // Here is where you can configure the job runner + // Aim to keep 10-20 jobs running at a time. + .set_concurrency(min_task_conc, max_conc_task_fetch) + // Start the job runner in the background. + .run() + .await?; + + info!( + target: LOG_TARGET, + "Bulk endpoints enabled with concurrency min={min_task_conc} to max={max_conc_task_fetch}." + ); + + Ok(registry) +} diff --git a/backend/src/http/v0/bulk/post.rs b/backend/src/http/v0/bulk/post.rs new file mode 100644 index 000000000..5bc291324 --- /dev/null +++ b/backend/src/http/v0/bulk/post.rs @@ -0,0 +1,151 @@ +// Reacher - Email Verification +// Copyright (C) 2018-2023 Reacher + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! This file implements the `POST /v0/bulk` endpoint. + +use check_if_email_exists::CheckEmailInputProxy; +use check_if_email_exists::LOG_TARGET; +use serde::{Deserialize, Serialize}; +use sqlx::{Pool, Postgres}; +use tracing::{debug, error}; +use warp::Filter; + +use super::{ + db::with_db, + error::BulkError, + task::{submit_job, TaskInput}, +}; +use crate::check::check_header; + +/// Endpoint request body. +#[derive(Clone, Debug, Deserialize, Serialize)] +struct CreateBulkRequestBody { + input_type: String, + input: Vec, + proxy: Option, + hello_name: Option, + from_email: Option, + smtp_ports: Option>, +} + +struct CreateBulkRequestBodyIterator { + body: CreateBulkRequestBody, + index: usize, +} + +impl IntoIterator for CreateBulkRequestBody { + type Item = TaskInput; + type IntoIter = CreateBulkRequestBodyIterator; + + fn into_iter(self) -> Self::IntoIter { + CreateBulkRequestBodyIterator { + body: self, + index: 0, + } + } +} + +impl Iterator for CreateBulkRequestBodyIterator { + type Item = TaskInput; + + fn next(&mut self) -> Option { + if self.index < self.body.input.len() { + let to_email = &self.body.input[self.index]; + let item = TaskInput { + to_email: to_email.clone(), + smtp_ports: self.body.smtp_ports.clone().unwrap_or_else(|| vec![25]), + proxy: self.body.proxy.clone(), + hello_name: self.body.hello_name.clone(), + from_email: self.body.from_email.clone(), + }; + + self.index += 1; + Some(item) + } else { + None + } + } +} + +/// Endpoint response body. +#[derive(Clone, Debug, Deserialize, Serialize)] +struct CreateBulkResponseBody { + job_id: i32, +} + +/// handles input, creates db entry for job and tasks for verification +async fn create_bulk_request( + conn_pool: Pool, + body: CreateBulkRequestBody, +) -> Result { + if body.input.is_empty() { + return Err(BulkError::EmptyInput.into()); + } + + // create job entry + let rec = sqlx::query!( + r#" + INSERT INTO bulk_jobs (total_records) + VALUES ($1) + RETURNING id + "#, + body.input.len() as i32 + ) + .fetch_one(&conn_pool) + .await + .map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to create job record for [body={:?}] with [error={}]", + &body, e + ); + BulkError::from(e) + })?; + + for task_input in body.into_iter() { + let task_uuid = submit_job(&conn_pool, rec.id, task_input).await?; + + debug!( + target: LOG_TARGET, + "Submitted task to sqlxmq for [job={}] with [uuid={}]", + rec.id, task_uuid + ); + } + + Ok(warp::reply::json(&CreateBulkResponseBody { + job_id: rec.id, + })) +} + +/// Create the `POST /bulk` endpoint. +/// The endpoint accepts list of email address and creates +/// a new job to check them. +pub fn create_bulk_job( + o: Option>, +) -> impl Filter + Clone { + warp::path!("v0" / "bulk") + .and(warp::post()) + .and(check_header()) + .and(with_db(o)) + // When accepting a body, we want a JSON body (and to reject huge + // payloads)... + // TODO: Configure max size limit for a bulk job + .and(warp::body::content_length_limit(1024 * 16)) + .and(warp::body::json()) + .and_then(create_bulk_request) + // View access logs by setting `RUST_LOG=reacher_backend`. + .with(warp::log(LOG_TARGET)) +} diff --git a/backend/src/http/v0/bulk/results/csv_helper.rs b/backend/src/http/v0/bulk/results/csv_helper.rs new file mode 100644 index 000000000..121e86112 --- /dev/null +++ b/backend/src/http/v0/bulk/results/csv_helper.rs @@ -0,0 +1,205 @@ +// Reacher - Email Verification +// Copyright (C) 2018-2023 Reacher + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use serde::Serialize; +use std::convert::TryFrom; + +/// Wrapper for serde json value to convert +/// into a csv response +#[derive(Debug)] +pub struct CsvWrapper(pub serde_json::Value); + +/// Simplified output of `CheckEmailOutput` struct +/// for csv fields. +#[derive(Debug, Serialize)] +pub struct JobResultCsvResponse { + input: String, + is_reachable: String, + #[serde(rename = "misc.is_disposable")] + misc_is_disposable: bool, + #[serde(rename = "misc.is_role_account")] + misc_is_role_account: bool, + #[serde(rename = "misc.gravatar_url")] + misc_gravatar_url: Option, + #[serde(rename = "mx.accepts_mail")] + mx_accepts_mail: bool, + #[serde(rename = "smtp.can_connect")] + smtp_can_connect: bool, + #[serde(rename = "smtp.has_full_inbox")] + smtp_has_full_inbox: bool, + #[serde(rename = "smtp.is_catch_all")] + smtp_is_catch_all: bool, + #[serde(rename = "smtp.is_deliverable")] + smtp_is_deliverable: bool, + #[serde(rename = "smtp.is_disabled")] + smtp_is_disabled: bool, + #[serde(rename = "syntax.is_valid_syntax")] + syntax_is_valid_syntax: bool, + #[serde(rename = "syntax.domain")] + syntax_domain: String, + #[serde(rename = "syntax.username")] + syntax_username: String, + error: Option, +} + +/// Convert csv wrapper to csv response +/// Performs multiple allocations for string fields +/// throw error if field is missing +impl TryFrom for JobResultCsvResponse { + type Error = &'static str; + + fn try_from(value: CsvWrapper) -> Result { + let mut input: String = String::default(); + let mut is_reachable: String = String::default(); + let mut misc_is_disposable: bool = false; + let mut misc_is_role_account: bool = false; + let mut misc_gravatar_url: Option = None; + let mut mx_accepts_mail: bool = false; + let mut smtp_can_connect: bool = false; + let mut smtp_has_full_inbox: bool = false; + let mut smtp_is_catch_all: bool = false; + let mut smtp_is_deliverable: bool = false; + let mut smtp_is_disabled: bool = false; + let mut syntax_is_valid_syntax: bool = false; + let mut syntax_domain: String = String::default(); + let mut syntax_username: String = String::default(); + let mut error: Option = None; + + let top_level = value + .0 + .as_object() + .ok_or("Failed to find top level object")?; + for (key, val) in top_level.keys().zip(top_level.values()) { + match key.as_str() { + "input" => input = val.as_str().ok_or("input should be a string")?.to_string(), + "is_reachable" => { + is_reachable = val + .as_str() + .ok_or("is_reachable should be a string")? + .to_string() + } + "misc" => { + let misc_obj = val.as_object().ok_or("misc field should be an object")?; + for (key, val) in misc_obj.keys().zip(misc_obj.values()) { + match key.as_str() { + "error" => error = Some(val.to_string()), + "is_disposable" => { + misc_is_disposable = + val.as_bool().ok_or("is_disposable should be a boolean")? + } + "is_role_account" => { + misc_is_role_account = + val.as_bool().ok_or("is_role_account should be a boolean")? + } + "gravatar_url" => { + if Option::is_some(&val.as_str()) { + misc_gravatar_url = Some(val.to_string()) + } + } + _ => {} + } + } + } + "mx" => { + let mx_obj = val.as_object().ok_or("mx field should be an object")?; + for (key, val) in mx_obj.keys().zip(mx_obj.values()) { + match key.as_str() { + "error" => error = Some(val.to_string()), + "accepts_email" => { + mx_accepts_mail = + val.as_bool().ok_or("accepts_email should be a boolean")? + } + _ => {} + } + } + } + "smtp" => { + let smtp_obj = val.as_object().ok_or("mx field should be an object")?; + for (key, val) in smtp_obj.keys().zip(smtp_obj.values()) { + match key.as_str() { + "error" => error = Some(val.to_string()), + "can_connect_smtp" => { + smtp_can_connect = val + .as_bool() + .ok_or("can_connect_smtp should be a boolean")? + } + "has_full_inbox" => { + smtp_has_full_inbox = + val.as_bool().ok_or("has_full_inbox should be a boolean")? + } + "is_catch_all" => { + smtp_is_catch_all = + val.as_bool().ok_or("is_catch_all should be a boolean")? + } + "is_deliverable" => { + smtp_is_deliverable = + val.as_bool().ok_or("is_deliverable should be a boolean")? + } + "is_disabled" => { + smtp_is_disabled = + val.as_bool().ok_or("is_disabled should be a boolean")? + } + _ => {} + } + } + } + "syntax" => { + let syntax_obj = val.as_object().ok_or("syntax field should be an object")?; + for (key, val) in syntax_obj.keys().zip(syntax_obj.values()) { + match key.as_str() { + "error" => error = Some(val.to_string()), + "is_valid_syntax" => { + syntax_is_valid_syntax = + val.as_bool().ok_or("is_valid_syntax should be a boolean")? + } + "username" => { + syntax_username = val + .as_str() + .ok_or("username should be a string")? + .to_string() + } + "domain" => { + syntax_domain = + val.as_str().ok_or("domain should be a string")?.to_string() + } + _ => {} + } + } + } + // ignore unknown fields + _ => {} + } + } + + Ok(JobResultCsvResponse { + input, + is_reachable, + misc_is_disposable, + misc_is_role_account, + misc_gravatar_url, + mx_accepts_mail, + smtp_can_connect, + smtp_has_full_inbox, + smtp_is_catch_all, + smtp_is_deliverable, + smtp_is_disabled, + syntax_domain, + syntax_is_valid_syntax, + syntax_username, + error, + }) + } +} diff --git a/backend/src/http/v0/bulk/results/mod.rs b/backend/src/http/v0/bulk/results/mod.rs new file mode 100644 index 000000000..b19dbc9fa --- /dev/null +++ b/backend/src/http/v0/bulk/results/mod.rs @@ -0,0 +1,248 @@ +// Reacher - Email Verification +// Copyright (C) 2018-2023 Reacher + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! This file implements the /bulk/{id}/results endpoints. + +use check_if_email_exists::LOG_TARGET; +use csv::WriterBuilder; +use serde::{Deserialize, Serialize}; +use sqlx::{Executor, Pool, Postgres, Row}; +use std::convert::TryInto; +use std::iter::Iterator; +use tracing::error; +use warp::Filter; + +use super::{ + db::with_db, + error::{BulkError, CsvError}, +}; +use csv_helper::{CsvWrapper, JobResultCsvResponse}; + +mod csv_helper; + +/// Defines the download format, passed in as a query param. +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +enum JobResultResponseFormat { + Json, + Csv, +} + +// limit and offset are optional in the request +// If unspecified, offset will default to 0. +#[derive(Serialize, Deserialize)] +struct JobResultRequest { + format: Option, + limit: Option, + offset: Option, +} + +#[derive(Serialize, Deserialize)] +struct JobResultJsonResponse { + results: Vec, +} + +async fn job_result( + job_id: i32, + conn_pool: Pool, + req: JobResultRequest, +) -> Result { + // Throw an error if the job is still running. + // Is there a way to combine these 2 requests in one? + let total_records = sqlx::query!( + r#"SELECT total_records FROM bulk_jobs WHERE id = $1;"#, + job_id + ) + .fetch_one(&conn_pool) + .await + .map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to fetch total_records for [job={}] with [error={}]", + job_id, e + ); + BulkError::from(e) + })? + .total_records; + let total_processed = sqlx::query!( + r#"SELECT COUNT(*) FROM email_results WHERE job_id = $1;"#, + job_id + ) + .fetch_one(&conn_pool) + .await + .map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to get total_processed for [job={}] with [error={}]", + job_id, e + ); + BulkError::from(e) + })? + .count + .unwrap_or(0); + + if total_processed < total_records as i64 { + return Err(BulkError::JobInProgress.into()); + } + + let format = req.format.unwrap_or(JobResultResponseFormat::Json); + match format { + JobResultResponseFormat::Json => { + let data = + job_result_json(job_id, req.limit, req.offset.unwrap_or(0), conn_pool).await?; + + let reply = + serde_json::to_vec(&JobResultJsonResponse { results: data }).map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to convert json results to string for [job={}] with [error={}]", + job_id, e + ); + + BulkError::Json(e) + })?; + + Ok(warp::reply::with_header( + reply, + "Content-Type", + "application/json", + )) + } + JobResultResponseFormat::Csv => { + let data = + job_result_csv(job_id, req.limit, req.offset.unwrap_or(0), conn_pool).await?; + + Ok(warp::reply::with_header(data, "Content-Type", "text/csv")) + } + } +} + +async fn job_result_as_iter( + job_id: i32, + limit: Option, + offset: u64, + conn_pool: Pool, +) -> Result>, BulkError> { + let query = sqlx::query!( + r#" + SELECT result FROM email_results + WHERE job_id = $1 + ORDER BY id + LIMIT $2 OFFSET $3 + "#, + job_id, + limit.map(|l| l as i64), + offset as i64 + ); + + let rows = conn_pool.fetch_all(query).await.map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to get results for [job={}] [limit={}] [offset={}] with [error={}]", + job_id, + limit.map(|s| s.to_string()).unwrap_or_else(|| "n/a".into()), + offset, + e + ); + + BulkError::from(e) + })?; + + Ok(Box::new( + rows.into_iter() + .map(|row| row.get::("result")), + )) +} + +async fn job_result_json( + job_id: i32, + limit: Option, + offset: u64, + conn_pool: Pool, +) -> Result, warp::Rejection> { + // For JSON responses, we don't want ot return more than 50 results at a + // time, to avoid having a too big payload (unless client specifies a limit) + + Ok( + job_result_as_iter(job_id, limit.or(Some(50)), offset, conn_pool) + .await? + .collect(), + ) +} + +async fn job_result_csv( + job_id: i32, + limit: Option, + offset: u64, + conn_pool: Pool, +) -> Result, warp::Rejection> { + let rows = job_result_as_iter(job_id, limit, offset, conn_pool).await?; + let mut wtr = WriterBuilder::new().has_headers(true).from_writer(vec![]); + + for json_value in rows { + let result_csv: JobResultCsvResponse = CsvWrapper(json_value).try_into().map_err(|e: &'static str| { + error!( + target: LOG_TARGET, + "Failed to convert json to csv output struct for [job={}] [limit={}] [offset={}] to csv with [error={}]", + job_id, + limit.map(|s| s.to_string()).unwrap_or_else(|| "n/a".into()), + offset, + e + ); + + BulkError::Csv(CsvError::Parse(e)) + })?; + wtr.serialize(result_csv).map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to serialize result for [job={}] [limit={}] [offset={}] to csv with [error={}]", + job_id, + limit.map(|s| s.to_string()).unwrap_or_else(|| "n/a".into()), + offset, + e + ); + + BulkError::Csv(CsvError::CsvLib(e)) + })?; + } + + let data = wtr.into_inner().map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to convert results for [job={}] [limit={}] [offset={}] to csv with [error={}]", + job_id, + limit.map(|s| s.to_string()).unwrap_or_else(|| "n/a".into()), + offset, + e + ); + + BulkError::Csv(CsvError::CsvLibWriter(Box::new(e))) + })?; + + Ok(data) +} + +pub fn get_bulk_job_result( + o: Option>, +) -> impl Filter + Clone { + warp::path!("v0" / "bulk" / i32 / "results") + .and(warp::get()) + .and(with_db(o)) + .and(warp::query::()) + .and_then(job_result) + // View access logs by setting `RUST_LOG=reacher_backend`. + .with(warp::log(LOG_TARGET)) +} diff --git a/backend/src/http/v0/bulk/task.rs b/backend/src/http/v0/bulk/task.rs new file mode 100644 index 000000000..e4bc07ac3 --- /dev/null +++ b/backend/src/http/v0/bulk/task.rs @@ -0,0 +1,230 @@ +// Reacher - Email Verification +// Copyright (C) 2018-2023 Reacher + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published +// by the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//! This file implements the `POST /bulk` endpoint. + +use check_if_email_exists::LOG_TARGET; +use check_if_email_exists::{CheckEmailInput, CheckEmailInputProxy, CheckEmailOutput, Reachable}; +use serde::{Deserialize, Serialize}; +use sqlx::{Pool, Postgres}; +use sqlxmq::{job, CurrentJob}; +use std::error::Error; +use tracing::{debug, error}; +use uuid::Uuid; + +use super::error::BulkError; +use crate::check::check_email; + +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct TaskInput { + // fields for CheckEmailInput + pub to_email: String, // Email from request to verify. + pub smtp_ports: Vec, // Ports to try for each email, in given order. Defaults to [25]. + pub proxy: Option, + pub hello_name: Option, + pub from_email: Option, +} + +pub struct TaskInputIterator { + body: TaskInput, + index: usize, +} + +impl IntoIterator for TaskInput { + type Item = CheckEmailInput; + type IntoIter = TaskInputIterator; + + fn into_iter(self) -> Self::IntoIter { + TaskInputIterator { + body: self, + index: 0, + } + } +} + +/// Iterate through all the `smtp_ports`. +impl Iterator for TaskInputIterator { + type Item = CheckEmailInput; + + fn next(&mut self) -> Option { + if self.index < self.body.smtp_ports.len() { + let mut item = CheckEmailInput::new(self.body.to_email.clone()); + + if let Some(name) = &self.body.hello_name { + item.set_hello_name(name.clone()); + } + + if let Some(email) = &self.body.from_email { + item.set_from_email(email.clone()); + } + + item.set_smtp_port(self.body.smtp_ports[self.index]); + + if let Some(proxy) = &self.body.proxy { + item.set_proxy(proxy.clone()); + } + + self.index += 1; + Some(item) + } else { + None + } + } +} + +/// Struct that's serialized into the sqlxmq own `payload_json` table. +#[derive(Debug, Deserialize, Serialize)] +struct TaskPayload { + id: i32, + input: TaskInput, +} + +pub async fn submit_job( + conn_pool: &Pool, + job_id: i32, + task_input: TaskInput, +) -> Result { + let task_payload = TaskPayload { + id: job_id, + input: task_input, + }; + + let uuid = email_verification_task + .builder() + .set_json(&task_payload) + .map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to submit task with the following [input={:?}] with [error={}]", + task_payload.input, e + ); + + BulkError::Json(e) + })? + .spawn(conn_pool) + .await + .map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to submit task for [bulk_req={}] with [error={}]", + job_id, e + ); + + e + })?; + + Ok(uuid) +} + +/// Arguments to the `#[job]` attribute allow setting default task options. +/// This task tries to verify the given email and inserts the results +/// into the email verification db table +/// NOTE: if EMAIL_TASK_BATCH_SIZE is made greater than 1 this logic +/// will have to be changed to handle a vector outputs from `check_email`. +/// +/// Small note about namings: what sqlxmq calls a "job", we call it a "task". +/// We call a "job" a user bulk request, i.e. a list of "tasks". +/// Please be careful while reading code. +#[job] +pub async fn email_verification_task( + mut current_job: CurrentJob, + // Additional arguments are optional, but can be used to access context + // provided via [`JobRegistry::set_context`]. +) -> Result<(), Box> { + let task_payload: TaskPayload = current_job.json()?.ok_or("Got empty task.")?; + let job_id = task_payload.id; + + let mut final_response: Option = None; + + for check_email_input in task_payload.input { + debug!( + target: LOG_TARGET, + "Starting task [email={}] for [job={}] and [uuid={}]", + check_email_input.to_email, + task_payload.id, + current_job.id(), + ); + + let to_email = check_email_input.to_email.clone(); + let response = check_email(check_email_input).await; + + debug!( + target: LOG_TARGET, + "Got task result [email={}] for [job={}] and [uuid={}] with [is_reachable={:?}]", + to_email, + task_payload.id, + current_job.id(), + response.is_reachable, + ); + + let is_reachable = response.is_reachable == Reachable::Unknown; + final_response = Some(response); + // unsuccessful validation continue iteration with next possible smtp port + if is_reachable { + continue; + } + // successful validation attempt complete job break iteration + else { + break; + } + } + + // final response can only be empty if there + // were no validation attempts. This can can + // never occur currently + if let Some(response) = final_response { + // write results and terminate iteration + #[allow(unused_variables)] + let rec = sqlx::query!( + r#" + INSERT INTO email_results (job_id, result) + VALUES ($1, $2) + "#, + job_id, + serde_json::json!(response) + ) + // TODO: This is a simplified solution and will work when + // the job queue and email results tables are in the same + // database. Keeping them in separate database will require + // some custom logic on the job registry side + // https://github.com/Diggsey/sqlxmq/issues/4 + .fetch_optional(current_job.pool()) + .await + .map_err(|e| { + error!( + target: LOG_TARGET, + "Failed to write [email={}] result to db for [job={}] and [uuid={}] with [error={}]", + response.input, + job_id, + current_job.id(), + e + ); + + e + })?; + + debug!( + target: LOG_TARGET, + "Wrote result for [email={}] for [job={}] and [uuid={}]", + response.input, + job_id, + current_job.id(), + ); + } + + current_job.complete().await?; + Ok(()) +} diff --git a/backend/src/http/v0/mod.rs b/backend/src/http/v0/mod.rs index 02927d4ac..92bd84116 100644 --- a/backend/src/http/v0/mod.rs +++ b/backend/src/http/v0/mod.rs @@ -1 +1,2 @@ +pub mod bulk; pub mod check_email; diff --git a/backend/src/main.rs b/backend/src/main.rs index ffd7b1711..c068d8eae 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -35,8 +35,7 @@ async fn main() -> Result<(), Box> { // Setup sentry bug tracking. let _guard: sentry::ClientInitGuard = setup_sentry(); - let http_server = run_warp_server(); - http_server.await?; + let _bulk_job_runner = run_warp_server().await?; Ok(()) } diff --git a/backend/tests/check_email.rs b/backend/tests/check_email.rs index 7dcd832ff..dbe374994 100644 --- a/backend/tests/check_email.rs +++ b/backend/tests/check_email.rs @@ -18,7 +18,7 @@ use std::env; use check_if_email_exists::CheckEmailInput; use reacher_backend::check::REACHER_SECRET_HEADER; -use reacher_backend::http::create_routes_without_bulk; +use reacher_backend::http::create_routes; use warp::http::StatusCode; use warp::test::request; @@ -34,7 +34,7 @@ async fn test_input_foo_bar() { .method("POST") .header(REACHER_SECRET_HEADER, "foobar") .json(&serde_json::from_str::(r#"{"to_email": "foo@bar"}"#).unwrap()) - .reply(&create_routes_without_bulk()) + .reply(&create_routes(None)) .await; assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body()); @@ -50,7 +50,7 @@ async fn test_input_foo_bar_baz() { .method("POST") .header(REACHER_SECRET_HEADER, "foobar") .json(&serde_json::from_str::(r#"{"to_email": "foo@bar.baz"}"#).unwrap()) - .reply(&create_routes_without_bulk()) + .reply(&create_routes(None)) .await; assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body()); @@ -65,7 +65,7 @@ async fn test_reacher_secret_missing_header() { .path("/v0/check_email") .method("POST") .json(&serde_json::from_str::(r#"{"to_email": "foo@bar.baz"}"#).unwrap()) - .reply(&create_routes_without_bulk()) + .reply(&create_routes(None)) .await; assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{:?}", resp.body()); @@ -81,7 +81,7 @@ async fn test_reacher_secret_wrong_secret() { .method("POST") .header(REACHER_SECRET_HEADER, "barbaz") .json(&serde_json::from_str::(r#"{"to_email": "foo@bar.baz"}"#).unwrap()) - .reply(&create_routes_without_bulk()) + .reply(&create_routes(None)) .await; assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{:?}", resp.body()); @@ -97,7 +97,7 @@ async fn test_reacher_secret_correct_secret() { .method("POST") .header(REACHER_SECRET_HEADER, "foobar") .json(&serde_json::from_str::(r#"{"to_email": "foo@bar"}"#).unwrap()) - .reply(&create_routes_without_bulk()) + .reply(&create_routes(None)) .await; assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body()); @@ -113,7 +113,7 @@ async fn test_reacher_to_mail_empty() { .method("POST") .header(REACHER_SECRET_HEADER, "foobar") .json(&serde_json::from_str::(r#"{"to_email": ""}"#).unwrap()) - .reply(&create_routes_without_bulk()) + .reply(&create_routes(None)) .await; assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{:?}", resp.body()); @@ -129,7 +129,7 @@ async fn test_reacher_to_mail_missing() { .method("POST") .header(REACHER_SECRET_HEADER, "foobar") .json(&serde_json::from_str::(r#"{}"#).unwrap()) - .reply(&create_routes_without_bulk()) + .reply(&create_routes(None)) .await; assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{:?}", resp.body()); diff --git a/core/Cargo.toml b/core/Cargo.toml index 282a207e8..fbef9c1fd 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -16,7 +16,7 @@ repository = "https://github.com/reacherhq/check-if-email-exists" async-native-tls = { version = "0.4", default-features = false } async-recursion = "1.0.5" async-smtp = { version = "0.6.0", features = ["socks5"] } -chrono = "=0.4.31" +chrono = "0.4.31" fantoccini = { version = "0.19.3", optional = true } futures = { version = "0.3.30", optional = true } fast-socks5 = "0.9.2"