From 2cb5b7c08bcac25898347f906f26284b6db82de8 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Sat, 17 Feb 2024 17:52:32 -0700 Subject: [PATCH 1/4] Added config option to enable the reqwest client adaptive window --- src/config/parser.rs | 4 ++++ src/results/aggregator.rs | 2 ++ src/server/routes/search.rs | 1 + websurfx/config.lua | 2 ++ 4 files changed, 9 insertions(+) diff --git a/src/config/parser.rs b/src/config/parser.rs index 63329c53..dcde182b 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -30,6 +30,8 @@ pub struct Config { pub logging: bool, /// It stores the option to whether enable or disable debug mode. pub debug: bool, + /// It toggles whether to use adaptive TCP windows + pub adaptive_window: bool, /// It stores all the engine names that were enabled by the user. pub upstream_search_engines: HashMap, /// It stores the time (secs) which controls the server request timeout. @@ -68,6 +70,7 @@ impl Config { let debug: bool = globals.get::<_, bool>("debug")?; let logging: bool = globals.get::<_, bool>("logging")?; + let adaptive_window: bool = globals.get::<_, bool>("adaptive_window")?; if !logging_initialized { set_logging_level(debug, logging); @@ -125,6 +128,7 @@ impl Config { }, logging, debug, + adaptive_window, upstream_search_engines: globals .get::<_, HashMap>("upstream_search_engines")?, request_timeout: globals.get::<_, u8>("request_timeout")?, diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index e7fc0903..2e64817e 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -71,6 +71,7 @@ pub async fn aggregate( upstream_search_engines: &[EngineHandler], request_timeout: u8, safe_search: u8, + adaptive_window: bool, ) -> Result> { let client = CLIENT.get_or_init(|| { ClientBuilder::new() @@ -78,6 +79,7 @@ pub async fn aggregate( .https_only(true) .gzip(true) .brotli(true) + .http2_adaptive_window(adaptive_window) .build() .unwrap() }); diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index 16cfa280..b26e5ac5 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -218,6 +218,7 @@ async fn results( .collect::>(), config.request_timeout, safe_search_level, + config.adaptive_window, ) .await? } diff --git a/websurfx/config.lua b/websurfx/config.lua index ce2d609c..989fcc05 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -14,6 +14,8 @@ rate_limiter = { number_of_requests = 20, -- The number of request that are allowed within a provided time limit. time_limit = 3, -- The time limit in which the quantity of requests that should be accepted. } +-- Set whether the server will use an adaptive/dynamic TCP window size +adaptive_window = true -- ### Search ### -- Filter results based on different levels. The levels provided are: From 6e2a5861d887c6a0b95ebd3b7aa987f181843d11 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Tue, 20 Feb 2024 12:11:33 -0700 Subject: [PATCH 2/4] Change adaptive window config name Co-authored-by: neon_arch --- websurfx/config.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/websurfx/config.lua b/websurfx/config.lua index 989fcc05..32b20daf 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -14,8 +14,8 @@ rate_limiter = { number_of_requests = 20, -- The number of request that are allowed within a provided time limit. time_limit = 3, -- The time limit in which the quantity of requests that should be accepted. } --- Set whether the server will use an adaptive/dynamic TCP window size -adaptive_window = true +-- Set whether the server will use an adaptive/dynamic HTTPS window size +https_adaptive_window_size = false -- ### Search ### -- Filter results based on different levels. The levels provided are: From 8431d8aab508e09344177903e860d300e167e586 Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Tue, 20 Feb 2024 20:05:44 -0700 Subject: [PATCH 3/4] Modified documentation --- src/config/parser.rs | 2 +- websurfx/config.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config/parser.rs b/src/config/parser.rs index dcde182b..4494db3f 100644 --- a/src/config/parser.rs +++ b/src/config/parser.rs @@ -30,7 +30,7 @@ pub struct Config { pub logging: bool, /// It stores the option to whether enable or disable debug mode. pub debug: bool, - /// It toggles whether to use adaptive TCP windows + /// It toggles whether to use adaptive HTTP windows pub adaptive_window: bool, /// It stores all the engine names that were enabled by the user. pub upstream_search_engines: HashMap, diff --git a/websurfx/config.lua b/websurfx/config.lua index 32b20daf..3b6c4ab9 100644 --- a/websurfx/config.lua +++ b/websurfx/config.lua @@ -14,7 +14,7 @@ rate_limiter = { number_of_requests = 20, -- The number of request that are allowed within a provided time limit. time_limit = 3, -- The time limit in which the quantity of requests that should be accepted. } --- Set whether the server will use an adaptive/dynamic HTTPS window size +-- Set whether the server will use an adaptive/dynamic HTTPS window size, see https://httpwg.org/specs/rfc9113.html#fc-principles https_adaptive_window_size = false -- ### Search ### From bb6dc68db6d95c9aeb84ab906de7aae2bc9d8dda Mon Sep 17 00:00:00 2001 From: ddotthomas Date: Sun, 25 Feb 2024 21:28:02 -0700 Subject: [PATCH 4/4] Trimmed down aggregate parameters --- src/results/aggregator.rs | 12 +++++------- src/server/routes/search.rs | 5 +---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/src/results/aggregator.rs b/src/results/aggregator.rs index 2e64817e..a95fb335 100644 --- a/src/results/aggregator.rs +++ b/src/results/aggregator.rs @@ -2,6 +2,7 @@ //! search engines and then removes duplicate results. use super::user_agent::random_user_agent; +use crate::config::parser::Config; use crate::handler::{file_path, FileType}; use crate::models::{ aggregation_models::{EngineErrorInfo, SearchResult, SearchResults}, @@ -66,20 +67,17 @@ type FutureVec = Vec, Report Result> { let client = CLIENT.get_or_init(|| { ClientBuilder::new() - .timeout(Duration::from_secs(request_timeout as u64)) // Add timeout to request to avoid DDOSing the server + .timeout(Duration::from_secs(config.request_timeout as u64)) // Add timeout to request to avoid DDOSing the server .https_only(true) .gzip(true) .brotli(true) - .http2_adaptive_window(adaptive_window) + .http2_adaptive_window(config.adaptive_window) .build() .unwrap() }); @@ -87,7 +85,7 @@ pub async fn aggregate( let user_agent: &str = random_user_agent(); // Add a random delay before making the request. - if random_delay || !debug { + if config.aggregator.random_delay || !config.debug { let nanos = SystemTime::now().duration_since(UNIX_EPOCH)?.subsec_nanos() as f32; let delay = ((nanos / 1_0000_0000 as f32).floor() as u64) + 1; tokio::time::sleep(Duration::from_secs(delay)).await; diff --git a/src/server/routes/search.rs b/src/server/routes/search.rs index b26e5ac5..a5b25586 100644 --- a/src/server/routes/search.rs +++ b/src/server/routes/search.rs @@ -209,16 +209,13 @@ async fn results( aggregate( query, page, - config.aggregator.random_delay, - config.debug, + config, &search_settings .engines .iter() .filter_map(|engine| EngineHandler::new(engine).ok()) .collect::>(), - config.request_timeout, safe_search_level, - config.adaptive_window, ) .await? }