From 51fb95e06c517ec6fbe72835974f929634ced9ab Mon Sep 17 00:00:00 2001 From: Marien Fressinaud Date: Fri, 18 Oct 2024 10:46:22 +0200 Subject: [PATCH] tec: Update the ClearUrls rules --- .../lib/ClearUrlsRules/data.min.json | 414 ++++++++++++++++-- 1 file changed, 377 insertions(+), 37 deletions(-) diff --git a/lib/SpiderBits/lib/ClearUrlsRules/data.min.json b/lib/SpiderBits/lib/ClearUrlsRules/data.min.json index 2aaab0b2..8c5704b9 100644 --- a/lib/SpiderBits/lib/ClearUrlsRules/data.min.json +++ b/lib/SpiderBits/lib/ClearUrlsRules/data.min.json @@ -34,7 +34,10 @@ "dchild", "camp", "creative", - "s" + "s", + "content-id", + "dib", + "dib_tag" ], "referralMarketing": [ "tag", @@ -50,6 +53,19 @@ "redirections": [], "forceRedirection": false }, + "msn": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?msn\\.com", + "completeProvider": false, + "rules": [ + "cvid", + "ocid" + ], + "rawRules": [], + "referralMarketing": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, "amazon search": { "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?amazon(?:\\.[a-z]{2,}){1,}\\/s\\?", "completeProvider": false, @@ -123,7 +139,6 @@ "gws_[a-z]*", "atyp", "vet", - "zx", "_u", "je", "dcr", @@ -141,7 +156,8 @@ "sxsrf", "rlz", "i-would-rather-use-firefox", - "pcampaignid" + "pcampaignid", + "sca_esv" ], "referralMarketing": [ "referrer" @@ -149,15 +165,16 @@ "rawRules": [], "exceptions": [ "^https?:\\/\\/mail\\.google\\.com\\/mail\\/u\\/", + "^https?:\\/\\/accounts\\.google\\.com\\/o\\/oauth2\\/", + "^https?:\\/\\/accounts\\.google\\.com\\/signin\\/oauth\\/", "^https?:\\/\\/(?:docs|accounts)\\.google(?:\\.[a-z]{2,}){1,}", - "^https?:\\/\\/([a-z0-9-\\.])*drive\\.google\\.com\\/videoplayback", + "^https?:\\/\\/([a-z0-9-\\.])*(chat|drive)\\.google\\.com\\/videoplayback", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}(?:\\/upload)?\\/drive\\/", "^https?:\\/\\/news\\.google\\.com.*\\?hl=.", - "^https?:\\/\\/hangouts\\.google\\.com\\/webchat.*?zx=.", - "^https?:\\/\\/client-channel\\.google\\.com\\/client-channel.*?zx=.", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/s\\?tbm=map.*?gs_[a-z]*=.", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/(?:complete\\/search|setprefs|searchbyimage)", - "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/(?:appsactivity|aclk\\?)" + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/(?:appsactivity|aclk\\?)", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/safe[-]?browsing\\/([^&]+)" ], "redirections": [ "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?google(?:\\.[a-z]{2,}){1,}\\/url\\?.*?(?:url|q)=(https?[^&]+)", @@ -228,8 +245,6 @@ "(?:%3F)?gs_l", "(?:%3F)?mkt_tok", "(?:%3F)?hmb_(?:campaign|medium|source)", - "(?:%3F)?ref_?", - "(?:%3F)?referrer", "(?:%3F)?gclid", "(?:%3F)?otm_[a-z_]*", "(?:%3F)?cmpid", @@ -245,14 +260,37 @@ "(?:%3F)?spm", "(?:%3F)?vn(?:_[a-z]*)+", "(?:%3F)?tracking_source", - "(?:%3F)?ceneo_spo" + "(?:%3F)?ceneo_spo", + "(?:%3F)?itm_(?:campaign|medium|source)", + "(?:%3F)?__hsfp", + "(?:%3F)?__hssc", + "(?:%3F)?__hstc", + "(?:%3F)?_hsenc", + "(?:%3F)?__s", + "(?:%3F)?hsCtaTracking", + "(?:%3F)?mc_(?:eid|cid|tc)", + "(?:%3F)?ml_subscriber", + "(?:%3F)?ml_subscriber_hash", + "(?:%3F)?msclkid", + "(?:%3F)?oly_anon_id", + "(?:%3F)?oly_enc_id", + "(?:%3F)?rb_clickid", + "(?:%3F)?s_cid", + "(?:%3F)?vero_conv", + "(?:%3F)?vero_id", + "(?:%3F)?wickedid", + "(?:%3F)?twclid" + ], + "referralMarketing": [ + "(?:%3F)?ref_?", + "(?:%3F)?referrer" ], - "referralMarketing": [], "rawRules": [], "exceptions": [ "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?matrix\\.org\\/_matrix\\/", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?(?:cloudflare\\.com|prismic\\.io|tangerine\\.ca|gitlab\\.com)", "^https?:\\/\\/myaccount.google(?:\\.[a-z]{2,}){1,}", + "^https?:\\/\\/accounts.google(?:\\.[a-z]{2,}){1,}", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?gcsip\\.(?:com|nl)[^?]*\\?.*?&?ref_?=.", "^https?:\\/\\/[^/]+/[^/]+/[^/]+\\/-\\/refs\\/switch[^?]*\\?.*?&?ref_?=.", "^https?:\\/\\/bugtracker\\.[^/]*\\/[^?]+\\?.*?&?ref_?=[^/?&]*", @@ -284,11 +322,14 @@ "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?onet\\.pl\\/[^?]*\\?.*?utm_campaign=.", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?stripe\\.com\\/[^?]+.*?&?referrer=[^/?&]*", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lichess\\.org\\/login.*?&?referrer=.*?", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?microsoft\\.com\\/.*?research\\/redirect", "^https?:\\/\\/like.co\\/api\\/like\\/likebutton\\/[^?]+.*?&?referrer=[^/?&]*", "^https?:\\/\\/button.like.co\\/in\\/.*?&?referrer=[^/?&]*", "^https?:\\/\\/www\\.mma\\.go\\.kr", "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?github\\.com", - "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?billiger\\.de\\/.*?mc=" + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?billiger\\.de\\/.*?mc=", + "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?\\.youtrack\\.cloud", + "^https?:\\/\\/cu\\.bankid\\.com" ], "redirections": [], "forceRedirection": false @@ -348,12 +389,13 @@ "forceRedirection": false }, "youtube": { - "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?youtube\\.com", + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?(youtube\\.com|youtu\\.be)", "completeProvider": false, "rules": [ "feature", "gclid", - "kw" + "kw" , + "si" ], "referralMarketing": [], "rawRules": [], @@ -405,7 +447,17 @@ "action_history", "tracking", "referral_code", - "referral_story_type" + "referral_story_type", + "eav", + "sfnsn", + "idorvanity", + "wtsid", + "rdc", + "rdr", + "paipv", + "_nc_x", + "_rdr", + "mibextid" ], "referralMarketing": [], "rawRules": [], @@ -440,6 +492,24 @@ "redirections": [], "forceRedirection": false }, + "x": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?x.com", + "completeProvider": false, + "rules": [ + "(?:ref_?)?src", + "s", + "cn", + "ref_url", + "t" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [ + "^https?:\\/\\/x.com\\/i\\/redirect" + ], + "redirections": [], + "forceRedirection": false + }, "reddit": { "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?reddit.com", "completeProvider": false, @@ -450,6 +520,7 @@ "ref_campaign", "ref_source", "%243p", + "rdt", "\\$3p", "%24original_url", "\\$original_url", @@ -1059,7 +1130,8 @@ "rules": [ "refId", "trk", - "li[a-z]{2}" + "li[a-z]{2}", + "trackingId" ], "referralMarketing": [], "rawRules": [], @@ -1119,7 +1191,8 @@ "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?instagram\\.com", "completeProvider": false, "rules": [ - "igshid" + "igshid", + "igsh" ], "referralMarketing": [], "rawRules": [], @@ -1129,24 +1202,6 @@ ], "forceRedirection": false }, - "lazada.com.my": { - "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lazada\\.com\\.my", - "completeProvider": false, - "rules": [ - "ad_src", - "did", - "pa", - "mp", - "impsrc", - "cid", - "pos" - ], - "referralMarketing": [], - "rawRules": [], - "exceptions": [], - "redirections": [], - "forceRedirection": false - }, "imgsrc.ru": { "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?imgsrc\\.ru", "completeProvider": false, @@ -1726,25 +1781,45 @@ "from_source", "from", "seid", + "mid", "share_source", "msource", "refer_from", + "share_from", "share_medium", "share_source", "share_plat", "share_tag", "share_session_id", "timestamp", - "unique_k" + "unique_k", + "vd_source", + "plat_id", + "buvid", + "is_story_h5", + "up_id" ], "referralMarketing": [], "rawRules": [], "exceptions": [ - "^https?:\\/\\/api\\.bilibili\\.com" + "^https?:\\/\\/api\\.bilibili\\.com", + "^https?:\\/\\/space\\.bilibili\\.com" ], "redirections": [], "forceRedirection": false }, + "space.bilibili.com": { + "urlPattern": "^https?:\\/\\/space\\.bilibili\\.com", + "completeProvider": false, + "rules": [ + "spm_id_from" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, "m.bilibili.com": { "urlPattern": "^https?:\\/\\/m\\.bilibili\\.com", "completeProvider": false, @@ -2697,6 +2772,271 @@ "exceptions": [], "redirections": [], "forceRedirection": false + }, + "duckduckgo": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?duckduckgo\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/duckduckgo\\.com\\/l\\/.*?uddg=([^&]+)" + ], + "forceRedirection": false + }, + "mercadolibre": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?mercadolibre\\.com", + "completeProvider": false, + "rules": [ + "DEAL_ID", + "L", + "S", + "T", + "V", + "pdp_filters", + "position", + "search_layout", + "tracking_id", + "type", + "c_[_a-zA-Z]+", + "me\\.[_a-zA-Z]+", + "reco_[_a-zA-Z]+" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "quizlet": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?quizlet\\.com", + "completeProvider": false, + "rules": [ + "funnelUUID" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "bbc": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?bbc\\.com", + "completeProvider": false, + "rules": [ + "xtor", + "at_[a-z_]+" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "airbnb": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?airbnb\\.(com|ae|ca|co\\.in|co\\.nz|co\\.uk|co\\.za|com\\.au|com\\.mt|com\\.sg|de|gy|ie)", + "completeProvider": false, + "rules": [ + "federated_search_id", + "search_type", + "source", + "source_impression_id" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "partner-ads.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?partner-ads\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/.*?partner-ads\\.com\\/.*?htmlurl=([^&]+)" + ], + "forceRedirection": false + }, + "kahoot.it": { + "urlPattern": "^https?://(?:[a-z0-9-]+\\.)*?kahoot\\.it", + "completeProvider": false, + "rules": [ + "refer_method" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "href.li": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?href\\.li", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/href\\.li\\/\\?(http.+)" + ], + "forceRedirection": false + }, + "adform.net": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?adform\\.net", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/track\\.adform\\.net\\/C\\/.*?ckurl=([^&]+)" + ], + "forceRedirection": false + }, + "artefact.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?artefact\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/.*?artefact\\.com\\/trck\\/.*?deeplinkurl=([^&]+)" + ], + "forceRedirection": false + }, + "awin1.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?awin1\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/.*?awin1\\.com\\/.*?ued=([^&]+)" + ], + "forceRedirection": false + }, + "telekom.de": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?telekom\\.de", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/aaa\\.telekom\\.de\\/trck\\/.*?deeplinkurl=([^&]+)" + ], + "forceRedirection": false + }, + "cc.loginfra.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?loginfra\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/cc\\.loginfra\\.com\\/.*?u=([^&]+)" + ], + "forceRedirection": false + }, + "t.umblr.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?umblr\\.com", + "completeProvider": false, + "rules": [], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [ + "^https?:\\/\\/t\\.umblr\\.com\\/redirect\\?z=([^&]+)" + ], + "forceRedirection": false + }, + "goodreads.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?goodreads\\.com", + "completeProvider": false, + "rules": [ + "from_search", + "from_srp", + "qid", + "rank", + "ac" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "sohu": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?sohu\\.com", + "completeProvider": false, + "rules": [ + "pvid", + "scm" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "shopee": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?shopee\\.(com|co\\.th)", + "completeProvider": false, + "rules": [ + "publish_id", + "sp_atk", + "xptdk" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "lazada": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?lazada\\.(com|co\\.th|co\\.id|com\\.my|com\\.ph|sg|vn)", + "completeProvider": false, + "rules": [ + "clickTrackInfo", + "abid", + "pvid", + "ad_src", + "spm", + "src", + "from", + "scm", + "pa", + "pid_pvid", + "did", + "mp", + "cid", + "impsrc", + "pos" + ], + "referralMarketing": [], + "rawRules": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false + }, + "pantip.com": { + "urlPattern": "^https?:\\/\\/(?:[a-z0-9-]+\\.)*?pantip\\.com", + "completeProvider": false, + "rules": [], + "rawRules": [ + "#lead.*" + ], + "referralMarketing": [], + "exceptions": [], + "redirections": [], + "forceRedirection": false } } }