Add Web Discovery content scraper, payload generator and privacy guard

brave · Oct 28, 2024 · 9ed101d · 9ed101d
1 parent b0de36a
commit 9ed101d
Show file tree

Hide file tree

Showing 40 changed files with 4,931 additions and 2 deletions.
diff --git a/browser/brave_tab_helpers.cc b/browser/brave_tab_helpers.cc
@@ -71,6 +71,12 @@
 #include "brave/components/ai_chat/content/browser/ai_chat_tab_helper.h"
 #endif  // BUILDFLAG(ENABLE_AI_CHAT)
 
+#if BUILDFLAG(ENABLE_WEB_DISCOVERY_NATIVE)
+#include "brave/browser/web_discovery/web_discovery_service_factory.h"
+#include "brave/components/web_discovery/browser/web_discovery_tab_helper.h"
+#include "brave/components/web_discovery/common/features.h"
+#endif
+
 #if BUILDFLAG(ENABLE_WIDEVINE)
 #include "brave/browser/brave_drm_tab_helper.h"
 #endif
@@ -213,6 +219,19 @@ void AttachTabHelpers(content::WebContents* web_contents) {
     }
   }
 #endif  // BUILDFLAG(ENABLE_PLAYLIST)
+
+#if BUILDFLAG(ENABLE_WEB_DISCOVERY_NATIVE)
+  if (base::FeatureList::IsEnabled(
+          web_discovery::features::kBraveWebDiscoveryNative)) {
+    auto* web_discovery_service =
+        web_discovery::WebDiscoveryServiceFactory::GetForBrowserContext(
+            web_contents->GetBrowserContext());
+    if (web_discovery_service) {
+      web_discovery::WebDiscoveryTabHelper::CreateForWebContents(
+          web_contents, web_discovery_service);
+    }
+  }
+#endif
 }
 
 }  // namespace brave
diff --git a/browser/sources.gni b/browser/sources.gni
@@ -364,6 +364,7 @@ if (enable_web_discovery_native) {
   brave_chrome_browser_deps += [
     "//brave/browser/web_discovery",
     "//brave/components/web_discovery/browser",
+    "//brave/components/web_discovery/browser:tab_helper",
     "//brave/components/web_discovery/common",
   ]
 }

diff --git a/chromium_src/chrome/renderer/chrome_content_renderer_client.cc b/chromium_src/chrome/renderer/chrome_content_renderer_client.cc
@@ -7,6 +7,7 @@
 #include "brave/components/ai_chat/core/common/buildflags/buildflags.h"
 #include "brave/components/ai_rewriter/common/buildflags/buildflags.h"
 #include "brave/components/content_settings/renderer/brave_content_settings_agent_impl.h"
+#include "brave/components/web_discovery/common/buildflags/buildflags.h"
 #include "chrome/common/chrome_isolated_world_ids.h"
 #include "chrome/renderer/process_state.h"
 #include "components/dom_distiller/content/renderer/distillability_agent.h"
@@ -23,6 +24,11 @@
 #include "brave/components/ai_rewriter/renderer/ai_rewriter_agent.h"
 #endif
 
+#if BUILDFLAG(ENABLE_WEB_DISCOVERY_NATIVE)
+#include "brave/components/web_discovery/common/features.h"
+#include "brave/components/web_discovery/renderer/blink_document_extractor.h"
+#endif
+
 namespace {
 
 void RenderFrameWithBinderRegistryCreated(
@@ -42,6 +48,13 @@ void RenderFrameWithBinderRegistryCreated(
     new ai_rewriter::AIRewriterAgent(render_frame, registry);
   }
 #endif
+
+#if BUILDFLAG(ENABLE_WEB_DISCOVERY_NATIVE)
+  if (base::FeatureList::IsEnabled(
+          web_discovery::features::kBraveWebDiscoveryNative)) {
+    new web_discovery::BlinkDocumentExtractor(render_frame, registry);
+  }
+#endif
 }
 
 }  // namespace

diff --git a/components/web_discovery/browser/BUILD.gn b/components/web_discovery/browser/BUILD.gn
@@ -12,12 +12,22 @@ component("browser") {
   sources = [
     "background_credential_helper.cc",
     "background_credential_helper.h",
+    "content_scraper.cc",
+    "content_scraper.h",
     "credential_manager.cc",
     "credential_manager.h",
     "credential_signer.h",
+    "hash_detection.cc",
+    "hash_detection.h",
     "patterns.cc",
     "patterns.h",
+    "payload_generator.cc",
+    "payload_generator.h",
     "pref_names.h",
+    "privacy_guard.cc",
+    "privacy_guard.h",
+    "regex_util.cc",
+    "regex_util.h",
     "rsa.cc",
     "rsa.h",
     "server_config_loader.cc",
@@ -29,10 +39,12 @@ component("browser") {
   ]
   deps = [
     "anonymous_credentials/rust:rust_lib",
+    "document_extractor/rust:rust_lib",
     "//base",
     "//brave/brave_domains",
     "//brave/components/constants",
     "//brave/components/web_discovery/common",
+    "//brave/components/web_discovery/common:mojom",
     "//components/keyed_service/core",
     "//components/prefs",
     "//crypto",
@@ -46,11 +58,28 @@ component("browser") {
   ]
 }
 
+component("tab_helper") {
+  output_name = "web_discovery_tab_helper"
+  sources = [
+    "web_discovery_tab_helper.cc",
+    "web_discovery_tab_helper.h",
+  ]
+  deps = [
+    "//base",
+    "//brave/components/web_discovery/browser",
+    "//content/public/browser",
+    "//services/service_manager/public/cpp",
+  ]
+}
+
 source_set("unit_tests") {
   testonly = true
   sources = [
     "credential_manager_unittest.cc",
+    "hash_detection_unittest.cc",
     "patterns_unittest.cc",
+    "payload_generator_unittest.cc",
+    "privacy_guard_unittest.cc",
     "server_config_loader_unittest.cc",
   ]
   deps = [
@@ -64,3 +93,21 @@ source_set("unit_tests") {
     "//testing/gtest",
   ]
 }
+
+source_set("browser_tests") {
+  testonly = true
+  sources = [ "content_scraper_browsertest.cc" ]
+  deps = [
+    ":browser",
+    "//base/test:test_support",
+    "//brave/components/constants",
+    "//brave/components/web_discovery/common",
+    "//brave/components/web_discovery/common:mojom",
+    "//chrome/test:test_support",
+    "//content/test:test_support",
+    "//net:test_support",
+    "//services/service_manager/public/cpp",
+    "//testing/gtest",
+  ]
+  defines = [ "HAS_OUT_OF_PROC_TEST_RUNNER" ]
+}
diff --git a/components/web_discovery/browser/DEPS b/components/web_discovery/browser/DEPS
@@ -1,9 +1,11 @@
 include_rules = [
   "-content",
   "+services/network/public",
+  "+content/public/browser",
   "+extensions/buildflags/buildflags.h",
   "+services/network/test",
   "+services/service_manager/public/cpp",
+  "+third_party/boringssl/src/include",
   "+third_party/re2",
   "+third_party/zlib",
 ]