Add support for Text fragment feature (#1545)

README.md

@@ -469,6 +469,9 @@ Options:
       --include-fragments
           Enable the checking of fragments in links
 
+      --include-text-fragments
+          Enable the checking of Text Fragments in links
+
   -t, --timeout <TIMEOUT>
           Website timeout in seconds from connect to response finished
 

lychee-bin/src/client.rs

@@ -77,6 +77,7 @@ pub(crate) fn create(cfg: &Config, cookie_jar: Option<&Arc<CookieStoreMutex>>) -
         .require_https(cfg.require_https)
         .cookie_jar(cookie_jar.cloned())
         .include_fragments(cfg.include_fragments)
+        .include_text_fragments(cfg.include_text_fragments)
         .fallback_extensions(cfg.fallback_extensions.clone())
         .build()
         .client()

lychee-bin/src/options.rs

@@ -423,6 +423,11 @@ separated list of accepted status codes. This example will accept 200, 201,
     #[serde(default)]
     pub(crate) include_fragments: bool,
 
+    /// Enable the checking of Text Fragments in links
+    #[arg(long)]
+    #[serde(default)]
+    pub(crate) include_text_fragments: bool,
+
     /// Website timeout in seconds from connect to response finished
     #[arg(short, long, default_value = &TIMEOUT_STR)]
     #[serde(default = "timeout")]
@@ -568,6 +573,7 @@ impl Config {
             require_https: false;
             cookie_jar: None;
             include_fragments: false;
+            include_text_fragments: false;
             accept: StatusCodeSelector::default();
         }
 

lychee-lib/Cargo.toml

@@ -18,6 +18,7 @@ cached = "0.54.0"
 check-if-email-exists = { version = "0.9.1", optional = true }
 cookie_store = "0.21.1"
 email_address = "0.2.9"
+fancy-regex = "0.14.0"
 futures = "0.3.31"
 glob = "0.3.2"
 headers = "0.4.0"

lychee-lib/src/checker/website.rs

@@ -3,10 +3,12 @@ use crate::{
     quirks::Quirks,
     retry::RetryExt,
     types::uri::github::GithubUri,
+    utils::url::UrlExt,
     BasicAuthCredentials, ErrorKind, Status, Uri,
 };
 use async_trait::async_trait;
 use http::StatusCode;
+use log::info;
 use octocrab::Octocrab;
 use reqwest::Request;
 use std::{collections::HashSet, time::Duration};
@@ -41,6 +43,9 @@ pub(crate) struct WebsiteChecker {
     ///
     /// This would treat unencrypted links as errors when HTTPS is available.
     require_https: bool,
+
+    /// Verify Text Fragments for a website
+    validate_text_fragments: bool,
 }
 
 impl WebsiteChecker {
@@ -53,6 +58,7 @@ impl WebsiteChecker {
         accepted: Option<HashSet<StatusCode>>,
         github_client: Option<Octocrab>,
         require_https: bool,
+        validate_text_fragments: bool,
         plugin_request_chain: RequestChain,
     ) -> Self {
         Self {
@@ -64,6 +70,7 @@ impl WebsiteChecker {
             retry_wait_time,
             accepted,
             require_https,
+            validate_text_fragments,
         }
     }
 
@@ -86,9 +93,37 @@ impl WebsiteChecker {
     }
 
     /// Check a URI using [reqwest](https://github.com/seanmonstar/reqwest).
+    ///
+    /// If Fragment Directive check is enabled and the URL has fragment directive,
+    /// Fragment Directive checker is run to validate the response against directives given
+    ///
+    /// # Errors
+    /// - if the fragment directive check fails, return one of the `TextFragmentStatus` error code
     async fn check_default(&self, request: Request) -> Status {
+        let req_url = request.url().clone();
+        let has_fragment_directive = req_url.has_fragment_directive();
+
         match self.reqwest_client.execute(request).await {
-            Ok(ref response) => Status::new(response, self.accepted.clone()),
+            Ok(response) => {
+                let mut status = Status::new(&response, self.accepted.clone());
+                if self.validate_text_fragments && has_fragment_directive {
+                    if let Ok(res) = response.text().await {
+                        info!("checking fragment directive...");
+                        if let Some(fd) = req_url.fragment_directive() {
+                            info!("directive: {:?}", fd.text_directives);
+                            match fd.check(&res) {
+                                Ok(stat) => {
+                                    status = stat;
+                                }
+                                Err(e) => {
+                                    return e.into();
+                                }
+                            }
+                        }
+                    }
+                }
+                status
+            }
             Err(e) => e.into(),
         }
     }

lychee-lib/src/client.rs

@@ -278,6 +278,9 @@ pub struct ClientBuilder {
     /// Enable the checking of fragments in links.
     include_fragments: bool,
 
+    /// Enable the checking of text fragment in a website
+    include_text_fragments: bool,
+
     /// Requests run through this chain where each item in the chain
     /// can modify the request. A chained item can also decide to exit
     /// early and return a status, so that subsequent chain items are
@@ -391,6 +394,7 @@ impl ClientBuilder {
             self.accepted,
             github_client,
             self.require_https,
+            self.include_text_fragments,
             self.plugin_request_chain,
         );
 
@@ -665,6 +669,7 @@ mod tests {
         // Same, but ignore certificate error
         let res = ClientBuilder::builder()
             .allow_insecure(true)
+            .include_text_fragments(true)
             .build()
             .client()
             .unwrap()
@@ -841,6 +846,65 @@ mod tests {
         assert!(res.status().is_unsupported());
     }
 
+    #[tokio::test]
+    async fn test_fragment_directive() {
+        let client = ClientBuilder::builder()
+            .include_text_fragments(true)
+            .build()
+            .client()
+            .unwrap();
+
+        // Start only
+        println!("testing START only directive...");
+        let res = client.check("https://developer.mozilla.org/en-US/docs/Web/URI/Fragment/Text_fragments#:~:text=without%20relying%20on%20the%20presence%20of%20IDs").await.unwrap();
+        assert!(res.status().is_success());
+
+        // Start and End
+        println!("\ntesting START and END directive...");
+        let res = client.check("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/a#:~:text=linked%20URL,defining%20a%20value").await.unwrap();
+        assert!(res.status().is_success());
+
+        // Prefix and start
+        println!("\ntesting Prefix with START...");
+        let res = client
+            .check("https://example.com/#:~:text=asking-,for")
+            .await
+            .unwrap();
+        assert!(res.status().is_success());
+
+        // start with suffix
+        println!("\ntesting start with suffix...");
+        let res = client.check("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/a#:~:text=linked%20URL\'s,-format").await.unwrap();
+        assert!(res.status().is_success());
+
+        let res = client.check("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/a#:~:text=downgrade:-,The%20Referer,be%20sent,-to%20origins").await.unwrap();
+        assert!(res.status().is_success());
+    }
+
+    #[tokio::test]
+    async fn test_multiple_directives() {
+        let client = ClientBuilder::builder()
+            .include_text_fragments(true)
+            .build()
+            .client()
+            .unwrap();
+
+        let res = client.check("https://developer.mozilla.org/en-US/docs/Web/HTML/Element/a#:~:text=causes&text=linked").await.unwrap();
+        assert!(res.status().is_success());
+    }
+
+    #[tokio::test]
+    async fn fail_fragment_directive_test() {
+        let client = ClientBuilder::builder()
+            .include_text_fragments(true)
+            .build()
+            .client()
+            .unwrap();
+
+        let res = client.check("https://developer.mozilla.org/en-US/docs/Web/URI/Fragment/Text_fragments#:~:text=without%20relying%20on%20the%20presence%20of%20DIs").await.unwrap();
+        assert!(res.status().is_error());
+    }
+
     #[tokio::test]
     async fn test_max_redirects() {
         let mock_server = wiremock::MockServer::start().await;