111

proxypool/crawlers/base.py

@@ -1,3 +1,5 @@
+import random
+
 from retrying import retry
 import requests
 from loguru import logger
@@ -6,13 +8,42 @@ from proxypool.setting import GET_TIMEOUT
 
 class BaseCrawler(object):
     urls = []
-    
+
+    @property
+    def user_agent(self):
+        """
+        return an User-Agent at random
+        :return:
+        """
+        ua_list = [
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.101',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.71',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.71',
+            'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; QQDownload 732; .NET4.0C; .NET4.0E)',
+            'Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/2.0.0 Opera 9.50',
+            'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
+        ]
+        return random.choice(ua_list)
+
+    @property
+    def header(self):
+        """
+        basic header
+        :return:
+        """
+        return {'User-Agent': self.user_agent,
+                'Accept': '*/*',
+                'Connection': 'keep-alive',
+                'Accept-Language': 'zh-CN,zh;q=0.8'}
+
     @retry(stop_max_attempt_number=3, retry_on_result=lambda x: x is None, wait_fixed=2000)
     def fetch(self, url, **kwargs):
         try:
             kwargs.setdefault('timeout', GET_TIMEOUT)
             kwargs.setdefault('verify', False)
-            response = requests.get(url, **kwargs)
+            response = requests.get(url,headers=self.header, **kwargs)
             if response.status_code == 200:
                 response.encoding = 'utf-8'
                 return response.text

proxypool/crawlers/public/hidemy.py

@@ -0,0 +1,46 @@
+import re
+
+from pyquery import PyQuery as pq
+from proxypool.schemas.proxy import Proxy
+from proxypool.crawlers.base import BaseCrawler
+from loguru import logger
+
+
+class HidemyCrawler(BaseCrawler):
+
+    urls  = [
+            'https://hidemy.name/en/proxy-list/?type=h&anon=4#list',
+            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=64#list',
+            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=128#list',
+            'https://hidemy.name/en/proxy-list/?type=h&anon=4&start=192#list',
+        ]
+
+    @logger.catch
+    def crawl(self):
+        """
+        crawl main method
+        """
+        for url in self.urls:
+            logger.info(f'fetching {url}')
+            html = self.fetch(url)
+            for proxy in self.parse(html):
+                logger.info(f'fetched proxy {proxy.string()} from {url}')
+                yield proxy
+
+    def parse(self, html):
+        """
+        parse html file to get proxies
+        :return:
+        """
+        print(html)
+        proxies = re.findall(
+            r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td><td>(\d+)</td>',
+            html)
+        for proxy in proxies:
+            yield Proxy(host=proxy[0], port=proxy[1])
+
+
+if __name__ == '__main__':
+    crawler = HidemyCrawler()
+    for proxy in crawler.crawl():
+        print(proxy)