add HTTP referer header for resource requests

Cargo.toml

@@ -7,6 +7,7 @@ authors = [
     "Emmanuel Delaborde <th3rac25@gmail.com>",
     "Emi Simpson <emi@alchemi.dev>",
     "rhysd <lin90162@yahoo.co.jp>",
+    "Andriy Rakhnin <a@rakhnin.com>",
 ]
 edition = "2021"
 description = "CLI tool for saving web pages as a single HTML file"

src/url.rs

@@ -79,6 +79,17 @@ pub fn parse_data_url(url: &Url) -> (String, String, Vec<u8>) {
     (media_type, charset, blob)
 }
 
+pub fn get_referer_url(url: Url) -> Url {
+    let mut url = url.clone();
+    // Spec: https://httpwg.org/specs/rfc9110.html#field.referer
+    // Must not include the fragment and userinfo components of the URI
+    url.set_fragment(None);
+    url.set_username(&"").unwrap();
+    url.set_password(None).unwrap();
+
+    url
+}
+
 pub fn resolve_url(from: &Url, to: &str) -> Url {
     match Url::parse(&to) {
         Ok(parsed_url) => parsed_url,

src/utils.rs

@@ -1,12 +1,12 @@
 use reqwest::blocking::Client;
-use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE, COOKIE};
+use reqwest::header::{HeaderMap, HeaderValue, CONTENT_TYPE, COOKIE, REFERER};
 use std::collections::HashMap;
 use std::fs;
 use std::path::{Path, PathBuf};
 use url::Url;
 
 use crate::opts::Options;
-use crate::url::{clean_url, parse_data_url};
+use crate::url::{clean_url, get_referer_url, parse_data_url};
 
 const ANSI_COLOR_RED: &'static str = "\x1b[31m";
 const ANSI_COLOR_RESET: &'static str = "\x1b[0m";
@@ -298,6 +298,13 @@ pub fn retrieve_asset(
                     }
                 }
             }
+            // Add referer header for page resource requests
+            if parent_url != url {
+                headers.insert(
+                    REFERER,
+                    HeaderValue::from_str(get_referer_url(parent_url.clone()).as_str()).unwrap(),
+                );
+            }
             match client.get(url.as_str()).headers(headers).send() {
                 Ok(response) => {
                     if !options.ignore_errors && response.status() != reqwest::StatusCode::OK {

tests/url/clean_url.rs

@@ -46,7 +46,7 @@ mod passing {
     }
 
     #[test]
-    fn removesempty_fragment_and_keeps_empty_query() {
+    fn removes_empty_fragment_and_keeps_query() {
         assert_eq!(
             url::clean_url(Url::parse("https://somewhere.com/font.eot?a=b&#").unwrap()).as_str(),
             "https://somewhere.com/font.eot?a=b&"

tests/url/get_referer_url.rs

@@ -0,0 +1,91 @@
+//  ██████╗  █████╗ ███████╗███████╗██╗███╗   ██╗ ██████╗
+//  ██╔══██╗██╔══██╗██╔════╝██╔════╝██║████╗  ██║██╔════╝
+//  ██████╔╝███████║███████╗███████╗██║██╔██╗ ██║██║  ███╗
+//  ██╔═══╝ ██╔══██║╚════██║╚════██║██║██║╚██╗██║██║   ██║
+//  ██║     ██║  ██║███████║███████║██║██║ ╚████║╚██████╔╝
+//  ╚═╝     ╚═╝  ╚═╝╚══════╝╚══════╝╚═╝╚═╝  ╚═══╝ ╚═════╝
+
+#[cfg(test)]
+mod passing {
+    use reqwest::Url;
+
+    use monolith::url;
+
+    #[test]
+    fn preserve_original() {
+        let original_url: Url = Url::parse("https://somewhere.com/font.eot#iefix").unwrap();
+        let referer_url: Url = url::get_referer_url(original_url.clone());
+        assert_eq!(referer_url.as_str(), "https://somewhere.com/font.eot");
+        assert_eq!(
+            original_url.as_str(),
+            "https://somewhere.com/font.eot#iefix"
+        );
+    }
+
+    #[test]
+    fn removes_fragment() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://somewhere.com/font.eot#iefix").unwrap())
+                .as_str(),
+            "https://somewhere.com/font.eot"
+        );
+    }
+
+    #[test]
+    fn removes_empty_fragment() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://somewhere.com/font.eot#").unwrap()).as_str(),
+            "https://somewhere.com/font.eot"
+        );
+    }
+
+    #[test]
+    fn removes_empty_fragment_and_keeps_empty_query() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://somewhere.com/font.eot?#").unwrap()).as_str(),
+            "https://somewhere.com/font.eot?"
+        );
+    }
+
+    #[test]
+    fn removes_empty_fragment_and_keeps_query() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://somewhere.com/font.eot?a=b&#").unwrap())
+                .as_str(),
+            "https://somewhere.com/font.eot?a=b&"
+        );
+    }
+
+    #[test]
+    fn removes_credentials() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://cookie:monster@gibson.lan/path").unwrap())
+                .as_str(),
+            "https://gibson.lan/path"
+        );
+    }
+
+    #[test]
+    fn removes_empty_credentials() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://@gibson.lan/path").unwrap()).as_str(),
+            "https://gibson.lan/path"
+        );
+    }
+
+    #[test]
+    fn removes_empty_username_credentials() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://:monster@gibson.lan/path").unwrap()).as_str(),
+            "https://gibson.lan/path"
+        );
+    }
+
+    #[test]
+    fn removes_empty_password_credentials() {
+        assert_eq!(
+            url::get_referer_url(Url::parse("https://cookie@gibson.lan/path").unwrap()).as_str(),
+            "https://gibson.lan/path"
+        );
+    }
+}

tests/url/mod.rs

@@ -1,5 +1,6 @@
 mod clean_url;
 mod create_data_url;
+mod get_referer_url;
 mod is_url_and_has_protocol;
 mod parse_data_url;
 mod resolve_url;