added several site and removed duplicates

proxyScraper.py

@@ -108,6 +108,38 @@ async def handle(self, response):
         return "\n".join(proxies)
 
 
+# For websites using div in html
+class GeneralDivScraper(Scraper):
+
+    async def handle(self, response):
+        soup = BeautifulSoup(response.text, "html.parser")
+        proxies = set()
+        table = soup.find("div", attrs={"class": "list"})
+        for row in table.findAll("div"):
+            count = 0
+            proxy = ""
+            for cell in row.findAll("div", attrs={"class": "td"}):
+                if count == 2:
+                    break
+                proxy += cell.text+":"
+                count += 1
+            proxy = proxy.rstrip(":")
+            proxies.add(proxy)
+        return "\n".join(proxies)
+
+# For scraping live proxylist from github
+class GitHubScraper(Scraper):
+
+    async def handle(self, response):
+        tempproxies = response.text.split("\n")
+        proxies = set()
+        for prxy in tempproxies:
+            if self.method in prxy:
+                proxies.add(prxy.split("//")[-1])
+
+        return "\n".join(proxies)
+
+
 scrapers = [
     SpysMeScraper("http"),
     SpysMeScraper("socks"),
@@ -123,6 +155,16 @@ async def handle(self, response):
     GeneralTableScraper("http", "http://free-proxy-list.net"),
     GeneralTableScraper("http", "http://us-proxy.org"),
     GeneralTableScraper("socks", "http://socks-proxy.net"),
+    GeneralDivScraper("http", "https://freeproxy.lunaproxy.com/"),
+    GitHubScraper("http", "https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/all/data.txt"),
+    GitHubScraper("socks4", "https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/all/data.txt"),
+    GitHubScraper("socks5", "https://raw.githubusercontent.com/proxifly/free-proxy-list/main/proxies/all/data.txt"),
+    GitHubScraper("http", "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/all.txt"),
+    GitHubScraper("socks", "https://raw.githubusercontent.com/monosans/proxy-list/main/proxies/all.txt"),
+    GitHubScraper("https", "https://raw.githubusercontent.com/zloi-user/hideip.me/main/https.txt"),
+    GitHubScraper("http", "https://raw.githubusercontent.com/zloi-user/hideip.me/main/http.txt"),
+    GitHubScraper("socks4", "https://raw.githubusercontent.com/zloi-user/hideip.me/main/socks4.txt"),
+    GitHubScraper("socks5", "https://raw.githubusercontent.com/zloi-user/hideip.me/main/socks5.txt"),
 ]
 
 
@@ -158,6 +200,7 @@ async def scrape_scraper(scraper):
     await asyncio.gather(*tasks)
     await client.aclose()
 
+    proxies = set(proxies)
     verbose_print(verbose, f"Writing {len(proxies)} proxies to file...")
     with open(output, "w") as f:
         f.write("\n".join(proxies))