complete support for using cookie file

src/cookies.rs

@@ -3,20 +3,25 @@ use url::Url;
 
 pub struct Cookie {
     pub domain: String,
-    pub tailmatch: bool,
+    pub include_subdomains: bool,
     pub path: String,
-    pub secure: bool,
+    pub https_only: bool,
     pub expires: u64,
     pub name: String,
     pub value: String,
 }
 
+#[derive(Debug)]
 pub enum CookieFileContentsParseError {
     InvalidHeader,
 }
 
 impl Cookie {
     pub fn is_expired(&self) -> bool {
+        if self.expires == 0 {
+            return false; // Session, never expires
+        }
+
         let start = SystemTime::now();
         let since_the_epoch = start
             .duration_since(UNIX_EPOCH)
@@ -28,29 +33,48 @@ impl Cookie {
     pub fn matches_url(&self, url: &str) -> bool {
         match Url::parse(&url) {
             Ok(url) => {
+                // Check protocol scheme
                 match url.scheme() {
                     "http" => {
-                        if self.secure {
+                        if self.https_only {
                             return false;
                         }
-                    },
-                    "https" => { },
+                    }
+                    "https" => {}
                     _ => {
+                        // Should never match URLs of protocols other than HTTP(S)
                         return false;
                     }
                 }
 
-                if let Some(domain) = url.domain() {
-                    if !domain.eq_ignore_ascii_case(&self.domain) {
-                        return false;
+                // Check host
+                if let Some(url_host) = url.host_str() {
+                    if self.domain.starts_with(".") && self.include_subdomains {
+                        if !url_host.to_lowercase().ends_with(&self.domain)
+                            && !url_host
+                                .eq_ignore_ascii_case(&self.domain[1..self.domain.len() - 1])
+                        {
+                            return false;
+                        }
+                    } else {
+                        if !url_host.eq_ignore_ascii_case(&self.domain) {
+                            return false;
+                        }
                     }
+                } else {
+                    return false;
                 }
 
-                // TODO: check path
-            },
+                // Check path
+                if !url.path().eq_ignore_ascii_case(&self.path)
+                    && !url.path().starts_with(&self.path)
+                {
+                    return false;
+                }
+            }
             Err(_) => {
                 return false;
-            },
+            }
         }
 
         true
@@ -64,18 +88,26 @@ pub fn parse_cookie_file_contents(
 
     for (i, line) in cookie_file_contents.lines().enumerate() {
         if i == 0 {
-            if !line.eq_ignore_ascii_case("# HTTP Cookie File")
-                && !line.eq_ignore_ascii_case("# Netscape HTTP Cookie File")
-            {
+            // Parsing first line
+            if !line.eq("# HTTP Cookie File") && !line.eq("# Netscape HTTP Cookie File") {
                 return Err(CookieFileContentsParseError::InvalidHeader);
             }
         } else {
+            // Ignore comment lines
+            if line.starts_with("#") {
+                continue;
+            }
+
+            // Attempt to parse values
             let mut fields = line.split("\t");
+            if fields.clone().count() != 7 {
+                continue;
+            }
             cookies.push(Cookie {
-                domain: fields.next().unwrap().to_string(),
-                tailmatch: fields.next().unwrap().to_string() == "TRUE",
+                domain: fields.next().unwrap().to_string().to_lowercase(),
+                include_subdomains: fields.next().unwrap().to_string() == "TRUE",
                 path: fields.next().unwrap().to_string(),
-                secure: fields.next().unwrap().to_string() == "TRUE",
+                https_only: fields.next().unwrap().to_string() == "TRUE",
                 expires: fields.next().unwrap().parse::<u64>().unwrap(),
                 name: fields.next().unwrap().to_string(),
                 value: fields.next().unwrap().to_string(),

src/main.rs

@@ -65,7 +65,7 @@ pub fn read_stdin() -> Vec<u8> {
 }
 
 fn main() {
-    let options = Options::from_args();
+    let mut options = Options::from_args();
 
     // Check if target was provided
     if options.target.len() == 0 {
@@ -141,17 +141,16 @@ fn main() {
     };
 
     // Read and parse cookie file
-    if let Some(opt_cookies) = options.cookies.clone() {
-        match fs::read_to_string(opt_cookies) {
+    if let Some(opt_cookie_file) = options.cookie_file.clone() {
+        match fs::read_to_string(opt_cookie_file) {
             Ok(str) => match parse_cookie_file_contents(&str) {
                 Ok(cookies) => {
-                    for c in &cookies {
-                        println!(
-                            "{} {} {} {} {} {} {}",
-                            c.domain, c.tailmatch, c.path, c.secure, c.expires, c.name, c.value
-                        );
-                        println!("^ is expired: {}", c.is_expired());
-                    }
+                    options.cookies = cookies;
+                    // for c in &cookies {
+                    //     // if !cookie.is_expired() {
+                    //         // options.cookies.append(c);
+                    //     // }
+                    // }
                 }
                 Err(_) => {
                     eprintln!("Could not parse specified cookie file");

src/opts.rs

@@ -9,8 +9,8 @@ pub struct Options {
     pub base_url: Option<String>,
     pub blacklist_domains: bool,
     pub no_css: bool,
-    pub cookies: Option<String>,
-    pub _cookies: Vec<Cookie>,
+    pub cookie_file: Option<String>,
+    pub cookies: Vec<Cookie>,
     pub domains: Option<Vec<String>>,
     pub ignore_errors: bool,
     pub encoding: Option<String>,
@@ -108,6 +108,9 @@ impl Options {
         }
         options.blacklist_domains = app.is_present("blacklist-domains");
         options.no_css = app.is_present("no-css");
+        if let Some(cookie_file) = app.value_of("cookies") {
+            options.cookie_file = Some(cookie_file.to_string());
+        }
         if let Some(encoding) = app.value_of("encoding") {
             options.encoding = Some(encoding.to_string());
         }
@@ -124,9 +127,6 @@ impl Options {
         options.insecure = app.is_present("insecure");
         options.no_metadata = app.is_present("no-metadata");
         options.output = app.value_of("output").unwrap_or("").to_string();
-        if let Some(cookies) = app.value_of("cookies") {
-            options.cookies = Some(cookies.to_string());
-        }
         options.silent = app.is_present("silent");
         options.timeout = app
             .value_of("timeout")

src/utils.rs

@@ -306,7 +306,15 @@ pub fn retrieve_asset(
 
             // URL not in cache, we retrieve the file
             let mut headers = HeaderMap::new();
-            headers.insert(COOKIE, HeaderValue::from_str("key=value").unwrap());
+            if options.cookies.len() > 0 {
+                for cookie in &options.cookies {
+                    if !cookie.is_expired() && cookie.matches_url(url.as_str()) {
+                        let cookie_header_value: String = cookie.name.clone() + "=" + &cookie.value;
+                        headers
+                            .insert(COOKIE, HeaderValue::from_str(&cookie_header_value).unwrap());
+                    }
+                }
+            }
             match client.get(url.as_str()).headers(headers).send() {
                 Ok(response) => {
                     if !options.ignore_errors && response.status() != reqwest::StatusCode::OK {

tests/cli/unusual_encodings.rs

@@ -115,7 +115,7 @@ mod passing {
         let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME")).unwrap();
         let out = cmd
             .arg("-M")
-            .arg("-C")
+            .arg("-E")
             .arg("utf8")
             .arg(format!(
                 "tests{s}_data_{s}unusual_encodings{s}gb2312.html",
@@ -158,7 +158,7 @@ mod passing {
         let mut cmd = Command::cargo_bin(env!("CARGO_PKG_NAME")).unwrap();
         let out = cmd
             .arg("-M")
-            .arg("-C")
+            .arg("-E")
             .arg("utf0")
             .arg(format!(
                 "tests{s}_data_{s}unusual_encodings{s}gb2312.html",

tests/cookies/cookie/is_expired.rs

@@ -0,0 +1,68 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use monolith::cookies;
+
+    #[test]
+    fn never_expires() {
+        let cookie = cookies::Cookie {
+            domain: String::from("127.0.0.1"),
+            include_subdomains: true,
+            path: String::from("/"),
+            https_only: false,
+            expires: 0,
+            name: String::from(""),
+            value: String::from(""),
+        };
+
+        assert!(!cookie.is_expired());
+    }
+
+    #[test]
+    fn expires_long_from_now() {
+        let cookie = cookies::Cookie {
+            domain: String::from("127.0.0.1"),
+            include_subdomains: true,
+            path: String::from("/"),
+            https_only: false,
+            expires: 9999999999,
+            name: String::from(""),
+            value: String::from(""),
+        };
+
+        assert!(!cookie.is_expired());
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use monolith::cookies;
+
+    #[test]
+    fn expired() {
+        let cookie = cookies::Cookie {
+            domain: String::from("127.0.0.1"),
+            include_subdomains: true,
+            path: String::from("/"),
+            https_only: false,
+            expires: 1,
+            name: String::from(""),
+            value: String::from(""),
+        };
+
+        assert!(cookie.is_expired());
+    }
+}

tests/cookies/cookie/matches_url.rs

@@ -0,0 +1,107 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use monolith::cookies;
+
+    #[test]
+    fn secure_url() {
+        let cookie = cookies::Cookie {
+            domain: String::from("127.0.0.1"),
+            include_subdomains: true,
+            path: String::from("/"),
+            https_only: true,
+            expires: 0,
+            name: String::from(""),
+            value: String::from(""),
+        };
+        assert!(cookie.matches_url("https://127.0.0.1/something"));
+    }
+
+    #[test]
+    fn non_secure_url() {
+        let cookie = cookies::Cookie {
+            domain: String::from("127.0.0.1"),
+            include_subdomains: true,
+            path: String::from("/"),
+            https_only: false,
+            expires: 0,
+            name: String::from(""),
+            value: String::from(""),
+        };
+        assert!(cookie.matches_url("http://127.0.0.1/something"));
+    }
+
+    #[test]
+    fn subdomain() {
+        let cookie = cookies::Cookie {
+            domain: String::from(".somethingsomething.com"),
+            include_subdomains: true,
+            path: String::from("/"),
+            https_only: true,
+            expires: 0,
+            name: String::from(""),
+            value: String::from(""),
+        };
+        assert!(cookie.matches_url("https://cdn.somethingsomething.com/something"));
+    }
+}
+
+//  ███████╗ █████╗ ██╗██╗     ██╗███╗   ██╗ ██████╗
+//  ██╔════╝██╔══██╗██║██║     ██║████╗  ██║██╔════╝
+//  █████╗  ███████║██║██║     ██║██╔██╗ ██║██║  ███╗
+//  ██╔══╝  ██╔══██║██║██║     ██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║██║███████╗██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚═╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod failing {
+    use monolith::cookies;
+
+    #[test]
+    fn empty_url() {
+        let cookie = cookies::Cookie {
+            domain: String::from("127.0.0.1"),
+            include_subdomains: true,
+            path: String::from("/"),
+            https_only: false,
+            expires: 0,
+            name: String::from(""),
+            value: String::from(""),
+        };
+        assert!(!cookie.matches_url(""));
+    }
+
+    #[test]
+    fn wrong_hostname() {
+        let cookie = cookies::Cookie {
+            domain: String::from("127.0.0.1"),
+            include_subdomains: true,
+            path: String::from("/"),
+            https_only: false,
+            expires: 0,
+            name: String::from(""),
+            value: String::from(""),
+        };
+        assert!(!cookie.matches_url("http://0.0.0.0/"));
+    }
+
+    #[test]
+    fn wrong_path() {
+        let cookie = cookies::Cookie {
+            domain: String::from("127.0.0.1"),
+            include_subdomains: false,
+            path: String::from("/"),
+            https_only: false,
+            expires: 0,
+            name: String::from(""),
+            value: String::from(""),
+        };
+        assert!(!cookie.matches_url("http://0.0.0.0/path"));
+    }
+}

tests/cookies/cookie/mod.rs

@@ -0,0 +1,2 @@
+mod is_expired;
+mod matches_url;