add xici and data5u

proxypool/crawlers/base.py

@@ -23,6 +23,7 @@ class BaseCrawler(object):
         for url in self.urls:
             logger.info(f'fetching {url}')
             html = self.fetch(url)
+            print('html', html)
             for proxy in self.parse(html):
                 logger.info(f'fetched proxy {proxy.string()} from {url}')
                 yield proxy

proxypool/crawlers/public/data5u.py

@@ -0,0 +1,47 @@
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+
+BASE_URL = 'http://www.data5u.com'
+
+
+class Data5UCrawler(BaseCrawler):
+    """
+    data5u crawler, http://www.data5u.com
+    """
+    urls = [BASE_URL]
+    
+    headers = {
+        'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
+    }
+
+    @logger.catch
+    def crawl(self):
+        """
+        crawl main method
+        """
+        for url in self.urls:
+            logger.info(f'fetching {url}')
+            html = self.fetch(url, headers=self.headers)
+            for proxy in self.parse(html):
+                logger.info(f'fetched proxy {proxy.string()} from {url}')
+                yield proxy
+    
+    def parse(self, html):
+        """
+        parse html file to get proxies
+        :return:
+        """
+        doc = pq(html)
+        items = doc('.wlist ul.l2').items()
+        for item in items:
+            host = item.find('span:first-child').text()
+            port = int(item.find('span:nth-child(2)').text())
+            yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+    crawler = Data5UCrawler()
+    for proxy in crawler.crawl():
+        print(proxy)

proxypool/crawlers/public/xicidaili.py

@@ -0,0 +1,50 @@
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+
+BASE_URL = 'https://www.xicidaili.com/'
+
+
+class XicidailiCrawler(BaseCrawler):
+    """
+    xididaili crawler, https://www.xicidaili.com/
+    """
+    urls = [BASE_URL]
+    
+    headers = {
+        'User-Agent': 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36'
+    }
+
+    @logger.catch
+    def crawl(self):
+        """
+        crawl main method
+        """
+        for url in self.urls:
+            logger.info(f'fetching {url}')
+            html = self.fetch(url, headers=self.headers)
+            for proxy in self.parse(html):
+                logger.info(f'fetched proxy {proxy.string()} from {url}')
+                yield proxy
+    
+    def parse(self, html):
+        """
+        parse html file to get proxies
+        :return:
+        """
+        doc = pq(html)
+        items = doc('#ip_list tr:contains(高匿)').items()
+        for item in items:
+            country = item.find('td.country').text()
+            if not country or country.strip() != '高匿':
+                continue
+            host = item.find('td:nth-child(2)').text()
+            port = int(item.find('td:nth-child(3)').text())
+            yield Proxy(host=host, port=port)
+
+
+if __name__ == '__main__':
+    crawler = XicidailiCrawler()
+    for proxy in crawler.crawl():
+        print(proxy)