Add Web Discovery content scraper, payload generator and privacy guard

brave · Aug 1, 2024 · 4b7529f · 4b7529f
1 parent 4f11211
commit 4b7529f
Show file tree

Hide file tree

Showing 38 changed files with 4,900 additions and 1 deletion.
diff --git a/browser/brave_tab_helpers.cc b/browser/brave_tab_helpers.cc
@@ -32,6 +32,7 @@
 #include "brave/components/request_otr/common/buildflags/buildflags.h"
 #include "brave/components/speedreader/common/buildflags/buildflags.h"
 #include "brave/components/tor/buildflags/buildflags.h"
+#include "brave/components/web_discovery/common/buildflags/buildflags.h"
 #include "build/build_config.h"
 #include "chrome/browser/browser_process.h"
 #include "chrome/browser/profiles/profile.h"
@@ -69,6 +70,12 @@
 #include "brave/components/ai_chat/core/browser/utils.h"
 #endif
 
+#if BUILDFLAG(ENABLE_WEB_DISCOVERY_NATIVE)
+#include "brave/browser/web_discovery/web_discovery_service_factory.h"
+#include "brave/components/web_discovery/browser/web_discovery_tab_helper.h"
+#include "brave/components/web_discovery/common/features.h"
+#endif
+
 #if BUILDFLAG(ENABLE_WIDEVINE)
 #include "brave/browser/brave_drm_tab_helper.h"
 #endif
@@ -218,6 +225,19 @@ void AttachTabHelpers(content::WebContents* web_contents) {
     }
   }
 #endif  // BUILDFLAG(ENABLE_PLAYLIST)
+
+#if BUILDFLAG(ENABLE_WEB_DISCOVERY_NATIVE)
+  if (base::FeatureList::IsEnabled(
+          web_discovery::features::kWebDiscoveryNative)) {
+    auto* web_discovery_service =
+        web_discovery::WebDiscoveryServiceFactory::GetForBrowserContext(
+            web_contents->GetBrowserContext());
+    if (web_discovery_service) {
+      web_discovery::WebDiscoveryTabHelper::CreateForWebContents(
+          web_contents, web_discovery_service);
+    }
+  }
+#endif
 }
 
 }  // namespace brave
diff --git a/chromium_src/chrome/renderer/chrome_content_renderer_client.cc b/chromium_src/chrome/renderer/chrome_content_renderer_client.cc
@@ -7,6 +7,7 @@
 #include "brave/components/ai_chat/core/common/buildflags/buildflags.h"
 #include "brave/components/ai_rewriter/common/buildflags/buildflags.h"
 #include "brave/components/content_settings/renderer/brave_content_settings_agent_impl.h"
+#include "brave/components/web_discovery/common/buildflags/buildflags.h"
 #include "chrome/common/chrome_isolated_world_ids.h"
 #include "chrome/renderer/chrome_render_thread_observer.h"
 #include "components/dom_distiller/content/renderer/distillability_agent.h"
@@ -23,6 +24,11 @@
 #include "brave/components/ai_rewriter/renderer/ai_rewriter_agent.h"
 #endif
 
+#if BUILDFLAG(ENABLE_WEB_DISCOVERY_NATIVE)
+#include "brave/components/web_discovery/common/features.h"
+#include "brave/components/web_discovery/renderer/blink_document_extractor.h"
+#endif
+
 namespace {
 
 void RenderFrameWithBinderRegistryCreated(
@@ -43,6 +49,13 @@ void RenderFrameWithBinderRegistryCreated(
     new ai_rewriter::AIRewriterAgent(render_frame, registry);
   }
 #endif
+
+#if BUILDFLAG(ENABLE_WEB_DISCOVERY_NATIVE)
+  if (base::FeatureList::IsEnabled(
+          web_discovery::features::kWebDiscoveryNative)) {
+    new web_discovery::BlinkDocumentExtractor(render_frame, registry);
+  }
+#endif
 }
 
 }  // namespace

diff --git a/components/web_discovery/browser/BUILD.gn b/components/web_discovery/browser/BUILD.gn
@@ -9,13 +9,23 @@ assert(enable_web_discovery_native)
 
 static_library("browser") {
   sources = [
+    "content_scraper.cc",
+    "content_scraper.h",
     "credential_manager.cc",
     "credential_manager.h",
     "credential_signer.cc",
     "credential_signer.h",
+    "hash_detection.cc",
+    "hash_detection.h",
     "patterns.cc",
     "patterns.h",
+    "payload_generator.cc",
+    "payload_generator.h",
     "pref_names.h",
+    "privacy_guard.cc",
+    "privacy_guard.h",
+    "regex_util.cc",
+    "regex_util.h",
     "rsa.cc",
     "rsa.h",
     "server_config_loader.cc",
@@ -24,13 +34,17 @@ static_library("browser") {
     "util.h",
     "web_discovery_service.cc",
     "web_discovery_service.h",
+    "web_discovery_tab_helper.cc",
+    "web_discovery_tab_helper.h",
   ]
   deps = [
     "anonymous_credentials/rs/cxx:rust_lib",
+    "document_extractor/rs:rust_lib",
     "//base",
     "//brave/brave_domains",
     "//brave/components/constants",
     "//brave/components/web_discovery/common",
+    "//brave/components/web_discovery/common:mojom",
     "//components/keyed_service/core",
     "//components/prefs",
     "//content/public/browser",
@@ -49,7 +63,10 @@ source_set("unit_tests") {
   testonly = true
   sources = [
     "credential_manager_unittest.cc",
+    "hash_detection_unittest.cc",
     "patterns_unittest.cc",
+    "payload_generator_unittest.cc",
+    "privacy_guard_unittest.cc",
     "server_config_loader_unittest.cc",
   ]
   deps = [
@@ -63,3 +80,21 @@ source_set("unit_tests") {
     "//testing/gtest",
   ]
 }
+
+source_set("browser_tests") {
+  testonly = true
+  sources = [ "content_scraper_browsertest.cc" ]
+  deps = [
+    ":browser",
+    "//base/test:test_support",
+    "//brave/components/constants",
+    "//brave/components/web_discovery/common",
+    "//brave/components/web_discovery/common:mojom",
+    "//chrome/test:test_support",
+    "//content/test:test_support",
+    "//net:test_support",
+    "//services/service_manager/public/cpp",
+    "//testing/gtest",
+  ]
+  defines = [ "HAS_OUT_OF_PROC_TEST_RUNNER" ]
+}
diff --git a/components/web_discovery/browser/DEPS b/components/web_discovery/browser/DEPS
@@ -1,7 +1,9 @@
 include_rules = [
   "+services/network/public",
+  "+content/public/browser",
   "+extensions/buildflags/buildflags.h",
   "+services/service_manager/public/cpp",
+  "+third_party/boringssl/src/include",
   "+third_party/re2",
   "+third_party/zlib",
 ]