111

proxypool/crawlers/public/free_proxy_list.py

@@ -0,0 +1,43 @@
+import re
+
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+
+
+class FreeProxyListCrawler(BaseCrawler):
+
+    urls  = [
+            'https://free-proxy-list.net',
+        ]
+
+    @logger.catch
+    def crawl(self):
+        """
+        crawl main method
+        """
+        for url in self.urls:
+            logger.info(f'fetching {url}')
+            html = self.fetch(url)
+            for proxy in self.parse(html):
+                logger.info(f'fetched proxy {proxy.string()} from {url}')
+                yield proxy
+
+    def parse(self, html):
+        """
+        parse html file to get proxies
+        :return:
+        """
+        print(html)
+        proxies = re.findall(
+            r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)',
+            html)
+        for proxy in proxies:
+            yield Proxy(host=proxy[0], port=proxy[1])
+
+
+if __name__ == '__main__':
+    crawler = FreeProxyListCrawler()
+    for proxy in crawler.crawl():
+        print(proxy)