diff --git a/src/cookies.rs b/src/cookies.rs index c2cc0bf1..85eeebbd 100644 --- a/src/cookies.rs +++ b/src/cookies.rs @@ -3,20 +3,25 @@ use url::Url; pub struct Cookie { pub domain: String, - pub tailmatch: bool, + pub include_subdomains: bool, pub path: String, - pub secure: bool, + pub https_only: bool, pub expires: u64, pub name: String, pub value: String, } +#[derive(Debug)] pub enum CookieFileContentsParseError { InvalidHeader, } impl Cookie { pub fn is_expired(&self) -> bool { + if self.expires == 0 { + return false; // Session, never expires + } + let start = SystemTime::now(); let since_the_epoch = start .duration_since(UNIX_EPOCH) @@ -28,29 +33,48 @@ impl Cookie { pub fn matches_url(&self, url: &str) -> bool { match Url::parse(&url) { Ok(url) => { + // Check protocol scheme match url.scheme() { "http" => { - if self.secure { + if self.https_only { return false; } - }, - "https" => { }, + } + "https" => {} _ => { + // Should never match URLs of protocols other than HTTP(S) return false; } } - if let Some(domain) = url.domain() { - if !domain.eq_ignore_ascii_case(&self.domain) { - return false; + // Check host + if let Some(url_host) = url.host_str() { + if self.domain.starts_with(".") && self.include_subdomains { + if !url_host.to_lowercase().ends_with(&self.domain) + && !url_host + .eq_ignore_ascii_case(&self.domain[1..self.domain.len() - 1]) + { + return false; + } + } else { + if !url_host.eq_ignore_ascii_case(&self.domain) { + return false; + } } + } else { + return false; } - // TODO: check path - }, + // Check path + if !url.path().eq_ignore_ascii_case(&self.path) + && !url.path().starts_with(&self.path) + { + return false; + } + } Err(_) => { return false; - }, + } } true @@ -64,18 +88,26 @@ pub fn parse_cookie_file_contents( for (i, line) in cookie_file_contents.lines().enumerate() { if i == 0 { - if !line.eq_ignore_ascii_case("# HTTP Cookie File") - && !line.eq_ignore_ascii_case("# Netscape HTTP Cookie File") - { + // Parsing first line + if !line.eq("# HTTP Cookie File") && !line.eq("# Netscape HTTP Cookie File") { return Err(CookieFileContentsParseError::InvalidHeader); } } else { + // Ignore comment lines + if line.starts_with("#") { + continue; + } + + // Attempt to parse values let mut fields = line.split("\t"); + if fields.clone().count() != 7 { + continue; + } cookies.push(Cookie { - domain: fields.next().unwrap().to_string(), - tailmatch: fields.next().unwrap().to_string() == "TRUE", + domain: fields.next().unwrap().to_string().to_lowercase(), + include_subdomains: fields.next().unwrap().to_string() == "TRUE", path: fields.next().unwrap().to_string(), - secure: fields.next().unwrap().to_string() == "TRUE", + https_only: fields.next().unwrap().to_string() == "TRUE", expires: fields.next().unwrap().parse::().unwrap(), name: fields.next().unwrap().to_string(), value: fields.next().unwrap().to_string(), diff --git a/src/main.rs b/src/main.rs index 365b7a22..fdd23599 100644 --- a/src/main.rs +++ b/src/main.rs @@ -65,7 +65,7 @@ pub fn read_stdin() -> Vec { } fn main() { - let options = Options::from_args(); + let mut options = Options::from_args(); // Check if target was provided if options.target.len() == 0 { @@ -141,17 +141,16 @@ fn main() { }; // Read and parse cookie file - if let Some(opt_cookies) = options.cookies.clone() { - match fs::read_to_string(opt_cookies) { + if let Some(opt_cookie_file) = options.cookie_file.clone() { + match fs::read_to_string(opt_cookie_file) { Ok(str) => match parse_cookie_file_contents(&str) { Ok(cookies) => { - for c in &cookies { - println!( - "{} {} {} {} {} {} {}", - c.domain, c.tailmatch, c.path, c.secure, c.expires, c.name, c.value - ); - println!("^ is expired: {}", c.is_expired()); - } + options.cookies = cookies; + // for c in &cookies { + // // if !cookie.is_expired() { + // // options.cookies.append(c); + // // } + // } } Err(_) => { eprintln!("Could not parse specified cookie file"); diff --git a/src/opts.rs b/src/opts.rs index a45f18b4..7a90694b 100644 --- a/src/opts.rs +++ b/src/opts.rs @@ -9,8 +9,8 @@ pub struct Options { pub base_url: Option, pub blacklist_domains: bool, pub no_css: bool, - pub cookies: Option, - pub _cookies: Vec, + pub cookie_file: Option, + pub cookies: Vec, pub domains: Option>, pub ignore_errors: bool, pub encoding: Option, @@ -108,6 +108,9 @@ impl Options { } options.blacklist_domains = app.is_present("blacklist-domains"); options.no_css = app.is_present("no-css"); + if let Some(cookie_file) = app.value_of("cookies") { + options.cookie_file = Some(cookie_file.to_string()); + } if let Some(encoding) = app.value_of("encoding") { options.encoding = Some(encoding.to_string()); } @@ -124,9 +127,6 @@ impl Options { options.insecure = app.is_present("insecure"); options.no_metadata = app.is_present("no-metadata"); options.output = app.value_of("output").unwrap_or("").to_string(); - if let Some(cookies) = app.value_of("cookies") { - options.cookies = Some(cookies.to_string()); - } options.silent = app.is_present("silent"); options.timeout = app .value_of("timeout") diff --git a/src/utils.rs b/src/utils.rs index 53497fa9..e786a917 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -306,7 +306,15 @@ pub fn retrieve_asset( // URL not in cache, we retrieve the file let mut headers = HeaderMap::new(); - headers.insert(COOKIE, HeaderValue::from_str("key=value").unwrap()); + if options.cookies.len() > 0 { + for cookie in &options.cookies { + if !cookie.is_expired() && cookie.matches_url(url.as_str()) { + let cookie_header_value: String = cookie.name.clone() + "=" + &cookie.value; + headers + .insert(COOKIE, HeaderValue::from_str(&cookie_header_value).unwrap()); + } + } + } match client.get(url.as_str()).headers(headers).send() { Ok(response) => { if !options.ignore_errors && response.status() != reqwest::StatusCode::OK { diff --git a/tests/cli/unusual_encodings.rs b/tests/cli/unusual_encodings.rs index 2a68c379..922922af 100644 --- a/tests/cli/unusual_encodings.rs +++ b/tests/cli/unusual_encodings.rs @@ -115,7 +115,7 @@ mod passing { let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME")).unwrap(); let out = cmd .arg("-M") - .arg("-C") + .arg("-E") .arg("utf8") .arg(format!( "tests{s}_data_{s}unusual_encodings{s}gb2312.html", @@ -158,7 +158,7 @@ mod passing { let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME")).unwrap(); let out = cmd .arg("-M") - .arg("-C") + .arg("-E") .arg("utf0") .arg(format!( "tests{s}_data_{s}unusual_encodings{s}gb2312.html", diff --git a/tests/cookies/cookie/is_expired.rs b/tests/cookies/cookie/is_expired.rs new file mode 100644 index 00000000..6bb479ca --- /dev/null +++ b/tests/cookies/cookie/is_expired.rs @@ -0,0 +1,68 @@ +// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗ +// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝ +// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗ +// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║ +// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝ +// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝ + +#[cfg(test)] +mod passing { + use monolith::cookies; + + #[test] + fn never_expires() { + let cookie = cookies::Cookie { + domain: String::from("127.0.0.1"), + include_subdomains: true, + path: String::from("/"), + https_only: false, + expires: 0, + name: String::from(""), + value: String::from(""), + }; + + assert!(!cookie.is_expired()); + } + + #[test] + fn expires_long_from_now() { + let cookie = cookies::Cookie { + domain: String::from("127.0.0.1"), + include_subdomains: true, + path: String::from("/"), + https_only: false, + expires: 9999999999, + name: String::from(""), + value: String::from(""), + }; + + assert!(!cookie.is_expired()); + } +} + +// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗ +// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝ +// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗ +// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║ +// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝ +// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝ + +#[cfg(test)] +mod failing { + use monolith::cookies; + + #[test] + fn expired() { + let cookie = cookies::Cookie { + domain: String::from("127.0.0.1"), + include_subdomains: true, + path: String::from("/"), + https_only: false, + expires: 1, + name: String::from(""), + value: String::from(""), + }; + + assert!(cookie.is_expired()); + } +} diff --git a/tests/cookies/cookie/matches_url.rs b/tests/cookies/cookie/matches_url.rs new file mode 100644 index 00000000..95dba63c --- /dev/null +++ b/tests/cookies/cookie/matches_url.rs @@ -0,0 +1,107 @@ +// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗ +// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝ +// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗ +// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║ +// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝ +// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝ + +#[cfg(test)] +mod passing { + use monolith::cookies; + + #[test] + fn secure_url() { + let cookie = cookies::Cookie { + domain: String::from("127.0.0.1"), + include_subdomains: true, + path: String::from("/"), + https_only: true, + expires: 0, + name: String::from(""), + value: String::from(""), + }; + assert!(cookie.matches_url("https://127.0.0.1/something")); + } + + #[test] + fn non_secure_url() { + let cookie = cookies::Cookie { + domain: String::from("127.0.0.1"), + include_subdomains: true, + path: String::from("/"), + https_only: false, + expires: 0, + name: String::from(""), + value: String::from(""), + }; + assert!(cookie.matches_url("http://127.0.0.1/something")); + } + + #[test] + fn subdomain() { + let cookie = cookies::Cookie { + domain: String::from(".somethingsomething.com"), + include_subdomains: true, + path: String::from("/"), + https_only: true, + expires: 0, + name: String::from(""), + value: String::from(""), + }; + assert!(cookie.matches_url("https://cdn.somethingsomething.com/something")); + } +} + +// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗ +// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝ +// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗ +// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║ +// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝ +// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝ + +#[cfg(test)] +mod failing { + use monolith::cookies; + + #[test] + fn empty_url() { + let cookie = cookies::Cookie { + domain: String::from("127.0.0.1"), + include_subdomains: true, + path: String::from("/"), + https_only: false, + expires: 0, + name: String::from(""), + value: String::from(""), + }; + assert!(!cookie.matches_url("")); + } + + #[test] + fn wrong_hostname() { + let cookie = cookies::Cookie { + domain: String::from("127.0.0.1"), + include_subdomains: true, + path: String::from("/"), + https_only: false, + expires: 0, + name: String::from(""), + value: String::from(""), + }; + assert!(!cookie.matches_url("http://0.0.0.0/")); + } + + #[test] + fn wrong_path() { + let cookie = cookies::Cookie { + domain: String::from("127.0.0.1"), + include_subdomains: false, + path: String::from("/"), + https_only: false, + expires: 0, + name: String::from(""), + value: String::from(""), + }; + assert!(!cookie.matches_url("http://0.0.0.0/path")); + } +} diff --git a/tests/cookies/cookie/mod.rs b/tests/cookies/cookie/mod.rs new file mode 100644 index 00000000..91b24578 --- /dev/null +++ b/tests/cookies/cookie/mod.rs @@ -0,0 +1,2 @@ +mod is_expired; +mod matches_url; diff --git a/tests/cookies/mod.rs b/tests/cookies/mod.rs new file mode 100644 index 00000000..973fa127 --- /dev/null +++ b/tests/cookies/mod.rs @@ -0,0 +1,2 @@ +mod cookie; +mod parse_cookie_file_contents; diff --git a/tests/cookies/parse_cookie_file_contents.rs b/tests/cookies/parse_cookie_file_contents.rs new file mode 100644 index 00000000..28857ded --- /dev/null +++ b/tests/cookies/parse_cookie_file_contents.rs @@ -0,0 +1,87 @@ +// ██████╗ █████╗ ███████╗███████╗██╗███╗ ██╗ ██████╗ +// ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗ ██║██╔════╝ +// ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║ ███╗ +// ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║ ██║ +// ██║ ██║ ██║███████║███████║██║██║ ╚████║╚██████╔╝ +// ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝ + +#[cfg(test)] +mod passing { + use monolith::cookies; + + #[test] + fn parse_file() { + let file_contents = + "# Netscape HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t0\tUSER_TOKEN\tin"; + let result = cookies::parse_cookie_file_contents(&file_contents).unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].domain, "127.0.0.1"); + assert_eq!(result[0].include_subdomains, false); + assert_eq!(result[0].path, "/"); + assert_eq!(result[0].https_only, false); + assert_eq!(result[0].expires, 0); + assert_eq!(result[0].name, "USER_TOKEN"); + assert_eq!(result[0].value, "in"); + } + + #[test] + fn parse_multiline_file() { + let file_contents = "# HTTP Cookie File\n127.0.0.1\tFALSE\t/\tFALSE\t0\tUSER_TOKEN\tin\n127.0.0.1\tTRUE\t/\tTRUE\t9\tUSER_TOKEN\tout\n\n"; + let result = cookies::parse_cookie_file_contents(&file_contents).unwrap(); + assert_eq!(result.len(), 2); + assert_eq!(result[0].domain, "127.0.0.1"); + assert_eq!(result[0].include_subdomains, false); + assert_eq!(result[0].path, "/"); + assert_eq!(result[0].https_only, false); + assert_eq!(result[0].expires, 0); + assert_eq!(result[0].name, "USER_TOKEN"); + assert_eq!(result[0].value, "in"); + assert_eq!(result[1].domain, "127.0.0.1"); + assert_eq!(result[1].include_subdomains, true); + assert_eq!(result[1].path, "/"); + assert_eq!(result[1].https_only, true); + assert_eq!(result[1].expires, 9); + assert_eq!(result[1].name, "USER_TOKEN"); + assert_eq!(result[1].value, "out"); + } +} + +// ███████╗ █████╗ ██╗██╗ ██╗███╗ ██╗ ██████╗ +// ██╔════╝██╔══██╗██║██║ ██║████╗ ██║██╔════╝ +// █████╗ ███████║██║██║ ██║██╔██╗ ██║██║ ███╗ +// ██╔══╝ ██╔══██║██║██║ ██║██║╚██╗██║██║ ██║ +// ██║ ██║ ██║██║███████╗██║██║ ╚████║╚██████╔╝ +// ╚═╝ ╚═╝ ╚═╝╚═╝╚══════╝╚═╝╚═╝ ╚═══╝ ╚═════╝ + +#[cfg(test)] +mod failing { + use monolith::cookies; + + #[test] + fn empty() { + let file_contents = ""; + let result = cookies::parse_cookie_file_contents(&file_contents).unwrap(); + assert_eq!(result.len(), 0); + } + + #[test] + fn no_header() { + let file_contents = "127.0.0.1 FALSE / FALSE 0 USER_TOKEN in"; + match cookies::parse_cookie_file_contents(&file_contents) { + Ok(_result) => { + assert!(false); + } + Err(_e) => { + assert!(true); + } + } + } + + #[test] + fn spaces_instead_of_tabs() { + let file_contents = + "# HTTP Cookie File\n127.0.0.1 FALSE / FALSE 0 USER_TOKEN in"; + let result = cookies::parse_cookie_file_contents(&file_contents).unwrap(); + assert_eq!(result.len(), 0); + } +} diff --git a/tests/mod.rs b/tests/mod.rs index 3ce28217..0f9928c0 100644 --- a/tests/mod.rs +++ b/tests/mod.rs @@ -1,4 +1,5 @@ mod cli; +mod cookies; mod css; mod html; mod js;