Merge branch 'jbarker/blacklist_malicious_url' into 'main'

Blacklist malicious url in openwebtext See merge request ADLR/megatron-lm!669
aklife97 · Jul 11, 2023 · 7809781 · 7809781
2 parents be0e123 + 7316f73
commit 7809781
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/tools/openwebtext/blacklist_urls.py b/tools/openwebtext/blacklist_urls.py
@@ -1,4 +1,6 @@
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2023, NVIDIA CORPORATION. All rights reserved.
+
+# WARNING! This file contains a blacklist of known malicious sites and thus contains some NSFW language.
 
 
 import glob
@@ -47,6 +49,7 @@
  'google',
  'gunprime',
  'gyazo',
+ 'horsefucker',
  'hotdealstar',
  'imagefap',
  'imageshack',