diff --git a/Cargo.lock b/Cargo.lock index 15b5624dc..806ab23fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,9 +52,9 @@ dependencies = [ [[package]] name = "android_system_properties" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7ed72e1635e121ca3e79420540282af22da58be50de153d36f81ddc6b83aa9e" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" dependencies = [ "libc", ] @@ -436,10 +436,12 @@ dependencies = [ "async-smtp", "async-std", "async-std-resolver", + "chrono", "fantoccini", "fast-socks5 0.9.1", "futures", "levenshtein", + "local-ip-address", "log", "mailchecker", "md5", @@ -479,7 +481,7 @@ dependencies = [ "num-integer", "num-traits", "serde", - "time 0.1.44", + "time 0.1.45", "wasm-bindgen", "winapi", ] @@ -1319,15 +1321,25 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.44" +version = "0.1.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "808cf7d67cf4a22adc5be66e75ebdf769b3f2ea032041437a7061f97a63dad4b" +checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20" dependencies = [ "android_system_properties", "core-foundation-sys", + "iana-time-zone-haiku", "js-sys", "wasm-bindgen", - "winapi", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", ] [[package]] @@ -1499,6 +1511,18 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f9f08d8963a6c613f4b1a78f4f4a4dbfadf8e6545b2d72861731e4858b8b47f" +[[package]] +name = "local-ip-address" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66357e687a569abca487dc399a9c9ac19beb3f13991ed49f00c144e02cbd42ab" +dependencies = [ + "libc", + "neli", + "thiserror", + "windows-sys 0.48.0", +] + [[package]] name = "lock_api" version = "0.4.7" @@ -1648,6 +1672,31 @@ dependencies = [ "tempfile", ] +[[package]] +name = "neli" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1100229e06604150b3becd61a4965d5c70f3be1759544ea7274166f4be41ef43" +dependencies = [ + "byteorder", + "libc", + "log", + "neli-proc-macros", +] + +[[package]] +name = "neli-proc-macros" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c168194d373b1e134786274020dae7fc5513d565ea2ebb9bc9ff17ffb69106d4" +dependencies = [ + "either", + "proc-macro2", + "quote", + "serde", + "syn 1.0.109", +] + [[package]] name = "nix" version = "0.27.1" @@ -2883,9 +2932,9 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "1b797afad3f312d1c66a56d11d0316f916356d11bd158fbc6ca6389ff6bf805a" dependencies = [ "libc", "wasi 0.10.0+wasi-snapshot-preview1", @@ -3451,6 +3500,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-core" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af6041b3f84485c21b57acdc0fee4f4f0c93f426053dc05fa5d6fc262537bbff" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.36.1" diff --git a/backend/openapi.json b/backend/openapi.json index 6566ed1d9..cadf1ebfe 100644 --- a/backend/openapi.json +++ b/backend/openapi.json @@ -57,7 +57,8 @@ "has_full_inbox": false, "is_catch_all": false, "is_deliverable": false, - "is_disabled": false + "is_disabled": false, + "method": "SmtpConnection" }, "syntax": { "domain": "gmail.com", @@ -180,15 +181,18 @@ }, "syntax": { "$ref": "#/components/schemas/SyntaxDetails" + }, + "debug": { + "$ref": "#/components/schemas/DebugDetails" } }, "required": [ "input", + "is_reachable", "misc", "mx", "smtp", - "syntax", - "is_reachable" + "syntax" ] }, "Error": { @@ -270,6 +274,18 @@ "is_disabled": { "type": "boolean", "description": "Has this email address been disabled by the email provider?" + }, + "method": { + "x-stoplight": { + "id": "axwtm6b7ao9n6" + }, + "enum": [ + "Headless", + "Api", + "Skipped", + "SmtpConnection" + ], + "description": "The method used to verify this email." } }, "required": [ @@ -277,7 +293,8 @@ "has_full_inbox", "is_catch_all", "is_deliverable", - "is_disabled" + "is_disabled", + "method" ] }, "SyntaxDetails": { @@ -314,12 +331,18 @@ }, "HotmailVerifyMethod": { "type": "string", + "x-stoplight": { + "id": "o6xocxf6tktur" + }, "title": "HotmailVerifyMethod", "enum": ["Api", "Headless", "Smtp"], "description": "An enum to describe how we verify Hotmail emails." }, "GmailVerifyMethod": { "type": "string", + "x-stoplight": { + "id": "jbq83vpkfcmth" + }, "title": "GmailVerifyMethod", "enum": ["Api", "Smtp"], "description": "An enum to describe how we verify Gmail emails.", @@ -433,6 +456,42 @@ } }, "required": ["secs", "nanos"] + }, + "DebugDetails": { + "title": "DebugDetails", + "x-stoplight": { + "id": "ndv3lpqeuypvn" + }, + "type": "object", + "properties": { + "start_time": { + "type": "string", + "x-stoplight": { + "id": "hxi446woaebm1" + } + }, + "end_time": { + "type": "string", + "x-stoplight": { + "id": "m4p986hhzuzbl" + } + }, + "duration": { + "$ref": "#/components/schemas/Duration" + }, + "server_ip": { + "type": "string", + "x-stoplight": { + "id": "ld3cp67qd2nx8" + } + }, + "server_name": { + "type": "string", + "x-stoplight": { + "id": "yy9pz1lxrr4si" + } + } + } } }, "securitySchemes": { diff --git a/backend/tests/check_email.rs b/backend/tests/check_email.rs index c0cd9d2c3..32f7ccf66 100644 --- a/backend/tests/check_email.rs +++ b/backend/tests/check_email.rs @@ -23,8 +23,8 @@ use reacher_backend::routes::create_routes; use warp::http::StatusCode; use warp::test::request; -const FOO_BAR_RESPONSE: &str = r#"{"input":"foo@bar","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}}"#; -const FOO_BAR_BAZ_RESPONSE: &str = r#"{"input":"foo@bar.baz","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false},"syntax":{"address":"foo@bar.baz","domain":"bar.baz","is_valid_syntax":true,"username":"foo","normalized_email":"foo@bar.baz","suggestion":null}}"#; +const FOO_BAR_RESPONSE: &str = r#"{"input":"foo@bar","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false,"method":"Skipped"},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}"#; +const FOO_BAR_BAZ_RESPONSE: &str = r#"{"input":"foo@bar.baz","is_reachable":"invalid","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"can_connect_smtp":false,"has_full_inbox":false,"is_catch_all":false,"is_deliverable":false,"is_disabled":false,"method":"Skipped"},"syntax":{"address":"foo@bar.baz","domain":"bar.baz","is_valid_syntax":true,"username":"foo","normalized_email":"foo@bar.baz","suggestion":null}"#; #[tokio::test] async fn test_input_foo_bar() { @@ -39,7 +39,8 @@ async fn test_input_foo_bar() { .await; assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body()); - assert_eq!(resp.body(), FOO_BAR_RESPONSE); + println!("{:?}", resp.body()); + assert!(resp.body().starts_with(FOO_BAR_RESPONSE.as_bytes())); } #[tokio::test] @@ -55,7 +56,7 @@ async fn test_input_foo_bar_baz() { .await; assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body()); - assert_eq!(resp.body(), FOO_BAR_BAZ_RESPONSE); + assert!(resp.body().starts_with(FOO_BAR_BAZ_RESPONSE.as_bytes())); } #[tokio::test] @@ -102,7 +103,7 @@ async fn test_reacher_secret_correct_secret() { .await; assert_eq!(resp.status(), StatusCode::OK, "{:?}", resp.body()); - assert_eq!(resp.body(), FOO_BAR_RESPONSE); + assert!(resp.body().starts_with(FOO_BAR_RESPONSE.as_bytes())); } #[tokio::test] diff --git a/core/Cargo.toml b/core/Cargo.toml index 4b838c320..8b579c313 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -18,10 +18,12 @@ async-recursion = "1.0.5" async-smtp = { version = "0.6.0", features = ["socks5"] } async-std = "1.12.0" async-std-resolver = "0.21.2" +chrono = "=0.4.22" fantoccini = { version = "0.19.3", optional = true } futures = { version = "0.3.29", optional = true } fast-socks5 = "0.9.1" levenshtein = "1.0.5" +local-ip-address = "0.5.6" log = "0.4.20" mailchecker = "6.0.1" md5 = "0.7.0" diff --git a/core/src/lib.rs b/core/src/lib.rs index ad6fbad4a..157cbed61 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -74,6 +74,7 @@ use misc::{check_misc, MiscDetails}; use mx::check_mx; use rand::Rng; use smtp::{check_smtp, SmtpDetails, SmtpError}; +use std::time::{Duration, SystemTime}; use syntax::{check_syntax, get_similar_mail_provider}; use trust_dns_proto::rr::rdata::MX; pub use util::constants::LOG_TARGET; @@ -111,6 +112,7 @@ fn calculate_reachable(misc: &MiscDetails, smtp: &Result /// Returns a `CheckEmailOutput` output, whose `is_reachable` field is one of /// `Safe`, `Invalid`, `Risky` or `Unknown`. pub async fn check_email(input: &CheckEmailInput) -> CheckEmailOutput { + let start_time = SystemTime::now(); let to_email = &input.to_email; log::debug!( @@ -233,6 +235,7 @@ pub async fn check_email(input: &CheckEmailInput) -> CheckEmailOutput { get_similar_mail_provider(&mut my_syntax); } + let end_time = SystemTime::now(); CheckEmailOutput { input: to_email.to_string(), is_reachable: calculate_reachable(&my_misc, &my_smtp), @@ -240,5 +243,13 @@ pub async fn check_email(input: &CheckEmailInput) -> CheckEmailOutput { mx: Ok(my_mx), smtp: my_smtp, syntax: my_syntax, + debug: DebugDetails { + start_time: start_time.into(), + end_time: end_time.into(), + duration: end_time + .duration_since(start_time) + .unwrap_or(Duration::from_secs(0)), + ..Default::default() + }, } } diff --git a/core/src/smtp/connect.rs b/core/src/smtp/connect.rs index ea35a9024..5c6ffb3e0 100644 --- a/core/src/smtp/connect.rs +++ b/core/src/smtp/connect.rs @@ -26,7 +26,7 @@ use std::iter; use std::str::FromStr; use std::time::Duration; -use super::parser; +use super::{parser, SmtpConnection, SmtpMethod}; use super::{SmtpDetails, SmtpError}; use crate::{ rules::{has_rule, Rule}, @@ -334,6 +334,9 @@ async fn check_smtp_without_retry( is_catch_all, is_deliverable: deliverability.is_deliverable, is_disabled: deliverability.is_disabled, + method: SmtpMethod::SmtpConnection(SmtpConnection { + host: host.to_string(), + }), }) } diff --git a/core/src/smtp/mod.rs b/core/src/smtp/mod.rs index 1a1764985..b183c608e 100644 --- a/core/src/smtp/mod.rs +++ b/core/src/smtp/mod.rs @@ -43,6 +43,20 @@ use self::{ yahoo::is_yahoo, }; +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct SmtpConnection { + pub host: String, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub enum SmtpMethod { + SmtpConnection(SmtpConnection), + Api, + Headless, + #[default] + Skipped, +} + /// Details that we gathered from connecting to this email via SMTP #[derive(Debug, Default, Deserialize, Serialize)] pub struct SmtpDetails { @@ -56,6 +70,7 @@ pub struct SmtpDetails { pub is_deliverable: bool, /// Is the email blocked or disabled by the provider? pub is_disabled: bool, + pub method: SmtpMethod, } /// Get all email details we can from one single `EmailAddress`, without diff --git a/core/src/smtp/outlook/headless.rs b/core/src/smtp/outlook/headless.rs index 46612cdb4..64690ecc4 100644 --- a/core/src/smtp/outlook/headless.rs +++ b/core/src/smtp/outlook/headless.rs @@ -23,7 +23,7 @@ use futures::TryFutureExt; use crate::{ smtp::{ headless::{create_headless_client, HeadlessError}, - SmtpDetails, + SmtpDetails, SmtpMethod, }, LOG_TARGET, }; @@ -102,6 +102,7 @@ pub async fn check_password_recovery( is_catch_all: false, is_deliverable, is_disabled: false, + method: SmtpMethod::Headless, }) } diff --git a/core/src/smtp/outlook/microsoft365.rs b/core/src/smtp/outlook/microsoft365.rs index 2948fee43..dc5fe265c 100644 --- a/core/src/smtp/outlook/microsoft365.rs +++ b/core/src/smtp/outlook/microsoft365.rs @@ -19,7 +19,7 @@ use reqwest::Error as ReqwestError; use serde::Serialize; use crate::{ - smtp::{http_api::create_client, SmtpDetails}, + smtp::{http_api::create_client, SmtpDetails, SmtpMethod}, util::ser_with_display::ser_with_display, CheckEmailInput, LOG_TARGET, }; @@ -85,6 +85,7 @@ pub async fn check_microsoft365_api( Ok(Some(SmtpDetails { can_connect_smtp: true, is_deliverable: true, + method: SmtpMethod::Api, ..Default::default() })) } else { diff --git a/core/src/smtp/yahoo/api.rs b/core/src/smtp/yahoo/api.rs index 9801b5133..c71b96d2c 100644 --- a/core/src/smtp/yahoo/api.rs +++ b/core/src/smtp/yahoo/api.rs @@ -16,7 +16,7 @@ use super::YahooError; use crate::{ - smtp::{http_api::create_client, SmtpDetails}, + smtp::{http_api::create_client, SmtpDetails, SmtpMethod}, util::{constants::LOG_TARGET, input_output::CheckEmailInput}, }; use regex::Regex; @@ -168,6 +168,7 @@ pub async fn check_api(to_email: &str, input: &CheckEmailInput) -> Result Result. +use std::env; +use std::net::IpAddr; use std::str::FromStr; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use async_smtp::{ClientSecurity, ClientTlsParameters}; +use chrono::{DateTime, Utc}; +use local_ip_address::{local_ip, Error as LocalIpError}; use serde::{ser::SerializeMap, Deserialize, Serialize, Serializer}; use crate::misc::{MiscDetails, MiscError}; @@ -424,6 +428,55 @@ pub enum Reachable { Unknown, } +/// Details about the email verification used for debugging. +#[derive(Debug)] +pub struct DebugDetails { + /// The name of the server that performed the email verification. + /// It's generally passed as an environment variable RCH_BACKEND_NAME. + pub server_name: String, + /// The IP address of the server that performed the email verification. + pub server_ip: Result, + /// The time when the email verification started. + pub start_time: DateTime, + /// The time when the email verification ended. + pub end_time: DateTime, + /// The duration of the email verification. + pub duration: Duration, +} + +impl Serialize for DebugDetails { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut map = serializer.serialize_map(Some(4))?; + map.serialize_entry("start_time", &self.start_time)?; + map.serialize_entry("end_time", &self.end_time)?; + map.serialize_entry("duration", &self.duration)?; + map.serialize_entry( + "server_ip", + &match &self.server_ip { + Ok(ip) => ip.to_string(), + Err(e) => e.to_string(), + }, + )?; + map.serialize_entry("server_name", &self.server_name)?; + map.end() + } +} + +impl Default for DebugDetails { + fn default() -> Self { + Self { + server_name: env::var("RCH_BACKEND_NAME").unwrap_or_else(|_| String::new()), + start_time: SystemTime::now().into(), + end_time: SystemTime::now().into(), + duration: Duration::default(), + server_ip: local_ip(), + } + } +} + /// The result of the [check_email](check_email) function. #[derive(Debug)] pub struct CheckEmailOutput { @@ -438,6 +491,8 @@ pub struct CheckEmailOutput { pub smtp: Result, /// Details about the email address. pub syntax: SyntaxDetails, + /// Details about the email verification used for debugging. + pub debug: DebugDetails, } impl Default for CheckEmailOutput { @@ -449,6 +504,7 @@ impl Default for CheckEmailOutput { mx: Ok(MxDetails::default()), smtp: Ok(SmtpDetails::default()), syntax: SyntaxDetails::default(), + debug: DebugDetails::default(), } } } @@ -503,13 +559,14 @@ impl Serialize for CheckEmailOutput { )?, } map.serialize_entry("syntax", &self.syntax)?; + map.serialize_entry("debug", &self.debug)?; map.end() } } #[cfg(test)] mod tests { - use super::CheckEmailOutput; + use super::{CheckEmailOutput, DebugDetails}; use async_smtp::smtp::response::{Category, Code, Detail, Response, Severity}; #[test] @@ -533,26 +590,27 @@ mod tests { mx: Ok(super::MxDetails::default()), syntax: super::SyntaxDetails::default(), smtp: Err(super::SmtpError::SmtpError(r.into())), + debug: DebugDetails::default(), } } let res = dummy_response_with_message("blacklist"); let actual = serde_json::to_string(&res).unwrap(); // Make sure the `description` is present with IpBlacklisted. - let expected = r#"{"input":"foo","is_reachable":"unknown","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"error":{"type":"SmtpError","message":"transient: blacklist"},"description":"IpBlacklisted"},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}}"#; - assert_eq!(expected, actual); + let expected = r#""smtp":{"error":{"type":"SmtpError","message":"transient: blacklist"},"description":"IpBlacklisted"}"#; + assert!(actual.contains(expected)); let res = dummy_response_with_message("Client host rejected: cannot find your reverse hostname"); let actual = serde_json::to_string(&res).unwrap(); // Make sure the `description` is present with NeedsRDNs. - let expected = r#"{"input":"foo","is_reachable":"unknown","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"error":{"type":"SmtpError","message":"transient: Client host rejected: cannot find your reverse hostname"},"description":"NeedsRDNS"},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}}"#; - assert_eq!(expected, actual); + let expected = r#"smtp":{"error":{"type":"SmtpError","message":"transient: Client host rejected: cannot find your reverse hostname"},"description":"NeedsRDNS"}"#; + assert!(actual.contains(expected)); let res = dummy_response_with_message("foobar"); let actual = serde_json::to_string(&res).unwrap(); // Make sure the `description` is NOT present. - let expected = r#"{"input":"foo","is_reachable":"unknown","misc":{"is_disposable":false,"is_role_account":false,"gravatar_url":null,"haveibeenpwned":null},"mx":{"accepts_mail":false,"records":[]},"smtp":{"error":{"type":"SmtpError","message":"transient: foobar"}},"syntax":{"address":null,"domain":"","is_valid_syntax":false,"username":"","normalized_email":null,"suggestion":null}}"#; - assert_eq!(expected, actual); + let expected = r#""smtp":{"error":{"type":"SmtpError","message":"transient: foobar"}}"#; + assert!(actual.contains(expected)); } }