Merge pull request #2 from okakaino/ted

Ted

proxy provider.txt

@@ -0,0 +1,7 @@
+代理：
+https://proxy.mimvp.com/free.php?proxy=in_hp
+http://www.coobobo.com/free-http-proxy
+http://ip.zdaye.com/
+http://www.mayidaili.com/free/anonymous/%E9%AB%98%E5%8C%BF
+http://http.taiyangruanjian.com/
+http://http.zhimaruanjian.com/

proxypool/crawler.py

@@ -24,13 +24,13 @@ class Crawler(object, metaclass=ProxyMetaclass):
             proxies.append(proxy)
         return proxies
         
-    def crawl_daxiang(self):
-        url = 'http://vtp.daxiangdaili.com/ip/?tid=559363191592228&num=50&filter=on'
-        html = get_page(url)
-        if html:
-            urls = html.split('\n')
-            for url in urls:
-                yield url
+    # def crawl_daxiang(self):
+    #     url = 'http://vtp.daxiangdaili.com/ip/?tid=559363191592228&num=50&filter=on'
+    #     html = get_page(url)
+    #     if html:
+    #         urls = html.split('\n')
+    #         for url in urls:
+    #             yield url
           
     def crawl_daili66(self, page_count=4):
         """
@@ -84,11 +84,11 @@ class Crawler(object, metaclass=ProxyMetaclass):
     def crawl_ip181(self):
         start_url = 'http://www.ip181.com/'
         html = get_page(start_url)
-        ip_adress = re.compile('<tr.*?>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
+        ip_address = re.compile('<tr.*?>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
         # \s* 匹配空格，起到换行作用
-        re_ip_adress = ip_adress.findall(html)
-        for adress,port in re_ip_adress:
-            result = adress + ':' + port
+        re_ip_address = ip_address.findall(html)
+        for address,port in re_ip_address:
+            result = address + ':' + port
             yield result.replace(' ', '')
 
 
@@ -96,34 +96,23 @@ class Crawler(object, metaclass=ProxyMetaclass):
         for page in range(1, 4):
             start_url = 'http://www.ip3366.net/free/?stype=1&page={}'.format(page)
             html = get_page(start_url)
-            ip_adress = re.compile('<tr>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
+            ip_address = re.compile('<tr>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
             # \s * 匹配空格，起到换行作用
-            re_ip_adress = ip_adress.findall(html)
-            for adress, port in re_ip_adress:
-                result = adress+':'+ port
+            re_ip_address = ip_address.findall(html)
+            for address, port in re_ip_address:
+                result = address+':'+ port
                 yield result.replace(' ', '')
 
 
-    def crawl_data5u(self):
-        for i in ['gngn', 'gnpt']:
-            start_url = 'http://www.data5u.com/free/{}/index.shtml'.format(i)
-            html = get_page(start_url)
-            ip_adress = re.compile(' <ul class="l2">\s*<span><li>(.*?)</li></span>\s*<span style="width: 100px;"><li class=".*">(.*?)</li></span>')
-            # \s * 匹配空格，起到换行作用
-            re_ip_adress = ip_adress.findall(html)
-            for adress, port in re_ip_adress:
-                result = adress+':'+port
-                yield result.replace(' ','')
-
     def crawl_kxdaili(self):
-        for i in range(1, 4):
+        for i in range(1, 11):
             start_url = 'http://www.kxdaili.com/ipList/{}.html#ip'.format(i)
             html = get_page(start_url)
-            ip_adress = re.compile('<tr.*?>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
+            ip_address = re.compile('<tr.*?>\s*<td>(.*?)</td>\s*<td>(.*?)</td>')
             # \s* 匹配空格，起到换行作用
-            re_ip_adress = ip_adress.findall(html)
-            for adress, port in re_ip_adress:
-                result = adress + ':' + port
+            re_ip_address = ip_address.findall(html)
+            for address, port in re_ip_address:
+                result = address + ':' + port
                 yield result.replace(' ', '')
 
 
@@ -132,21 +121,121 @@ class Crawler(object, metaclass=ProxyMetaclass):
             start_url = 'https://premproxy.com/proxy-by-country/{}.htm'.format(i)
             html = get_page(start_url)
             if html:
-                ip_adress = re.compile('<td data-label="IP:port ">(.*?)</td>') 
-                re_ip_adress = ip_adress.findall(html)
-                for adress_port in re_ip_adress:
-                    yield adress_port.replace(' ','')
+                ip_address = re.compile('<td data-label="IP:port ">(.*?)</td>') 
+                re_ip_address = ip_address.findall(html)
+                for address_port in re_ip_address:
+                    yield address_port.replace(' ','')
 
     def crawl_xroxy(self):
         for i in ['CN','TW']:
             start_url = 'http://www.xroxy.com/proxylist.php?country={}'.format(i)
             html = get_page(start_url)
             if html:
-                ip_adress1 = re.compile("title='View this Proxy details'>\s*(.*).*")
-                re_ip_adress1 = ip_adress1.findall(html)
-                ip_adress2 = re.compile("title='Select proxies with port number .*'>(.*)</a>") 
-                re_ip_adress2 = ip_adress2.findall(html)
-                for adress,port in zip(re_ip_adress1,re_ip_adress2):
-                    adress_port = adress+':'+port
-                    yield adress_port.replace(' ','')
+                ip_address1 = re.compile("title='View this Proxy details'>\s*(.*).*")
+                re_ip_address1 = ip_address1.findall(html)
+                ip_address2 = re.compile("title='Select proxies with port number .*'>(.*)</a>") 
+                re_ip_address2 = ip_address2.findall(html)
+                for address,port in zip(re_ip_address1,re_ip_address2):
+                    address_port = address+':'+port
+                    yield address_port.replace(' ','')
+    
+    def crawl_kuaidaili(self):
+        for i in range(1, 4):
+            start_url = 'http://www.kuaidaili.com/free/inha/{}/'.format(i)
+            html = get_page(start_url)
+            if html:
+                ip_address = re.compile('<td data-title="IP">(.*?)</td>') 
+                re_ip_address = ip_address.findall(html)
+                port = re.compile('<td data-title="PORT">(.*?)</td>')
+                re_port = port.findall(html)
+                for address,port in zip(re_ip_address, re_port):
+                    address_port = address+':'+port
+                    yield address_port.replace(' ','')
+
+    def crawl_xicidaili(self):
+        for i in range(1, 3):
+            start_url = 'http://www.xicidaili.com/nn/{}'.format(i)
+            headers = {
+                'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+                'Cookie':'_free_proxy_session=BAh7B0kiD3Nlc3Npb25faWQGOgZFVEkiJWRjYzc5MmM1MTBiMDMzYTUzNTZjNzA4NjBhNWRjZjliBjsAVEkiEF9jc3JmX3Rva2VuBjsARkkiMUp6S2tXT3g5a0FCT01ndzlmWWZqRVJNek1WanRuUDBCbTJUN21GMTBKd3M9BjsARg%3D%3D--2a69429cb2115c6a0cc9a86e0ebe2800c0d471b3',
+                'Host':'www.xicidaili.com',
+                'Referer':'http://www.xicidaili.com/nn/3',
+                'Upgrade-Insecure-Requests':'1',
+            }
+            html = get_page(start_url, options=headers)
+            if html:
+                find_trs = re.compile('<tr class.*?>(.*?)</tr>', re.S)
+                trs = find_trs.findall(html)
+                for tr in trs:
+                    find_ip = re.compile('<td>(\d+\.\d+\.\d+\.\d+)</td>') 
+                    re_ip_address = find_ip.findall(tr)
+                    find_port = re.compile('<td>(\d+)</td>')
+                    re_port = find_port.findall(tr)
+                    for address,port in zip(re_ip_address, re_port):
+                        address_port = address+':'+port
+                        yield address_port.replace(' ','')
+    
+    def crawl_ip3366(self):
+        for i in range(1, 4):
+            start_url = 'http://www.ip3366.net/?stype=1&page={}'.format(i)
+            html = get_page(start_url)
+            if html:
+                find_tr = re.compile('<tr>(.*?)</tr>', re.S)
+                trs = find_tr.findall(html)
+                for s in range(1, len(trs)):
+                    find_ip = re.compile('<td>(\d+\.\d+\.\d+\.\d+)</td>')
+                    re_ip_address = find_ip.findall(trs[s])
+                    find_port = re.compile('<td>(\d+)</td>')
+                    re_port = find_port.findall(trs[s])
+                    for address,port in zip(re_ip_address, re_port):
+                        address_port = address+':'+port
+                        yield address_port.replace(' ','')
+    
+    def crawl_iphai(self):
+        start_url = 'http://www.iphai.com/'
+        html = get_page(start_url)
+        if html:
+            find_tr = re.compile('<tr>(.*?)</tr>', re.S)
+            trs = find_tr.findall(html)
+            for s in range(1, len(trs)):
+                find_ip = re.compile('<td>\s+(\d+\.\d+\.\d+\.\d+)\s+</td>', re.S)
+                re_ip_address = find_ip.findall(trs[s])
+                find_port = re.compile('<td>\s+(\d+)\s+</td>', re.S)
+                re_port = find_port.findall(trs[s])
+                for address,port in zip(re_ip_address, re_port):
+                    address_port = address+':'+port
+                    yield address_port.replace(' ','')
+
+    def crawl_89ip(self):
+        start_url = 'http://www.89ip.cn/apijk/?&tqsl=1000&sxa=&sxb=&tta=&ports=&ktip=&cf=1'
+        html = get_page(start_url)
+        if html:
+            find_ips = re.compile('(\d+\.\d+\.\d+\.\d+:\d+)', re.S)
+            ip_ports = find_ips.findall(html)
+            for address_port in ip_ports:
+                yield address_port
+
+    def crawl_data5u(self):
+        start_url = 'http://www.data5u.com/free/gngn/index.shtml'
+        headers = {
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Accept-Encoding': 'gzip, deflate',
+            'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
+            'Cache-Control': 'max-age=0',
+            'Connection': 'keep-alive',
+            'Cookie': 'JSESSIONID=47AA0C887112A2D83EE040405F837A86',
+            'Host': 'www.data5u.com',
+            'Referer': 'http://www.data5u.com/free/index.shtml',
+            'Upgrade-Insecure-Requests': '1',
+            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36',
+        }
+        html = get_page(start_url, options=headers)
+        if html:
+            ip_address = re.compile('<span><li>(\d+\.\d+\.\d+\.\d+)</li>.*?<li class=\"port.*?>(\d+)</li>', re.S)
+            re_ip_address = ip_address.findall(html)
+            for address, port in re_ip_address:
+                result = address + ':' + port
+                yield result.replace(' ', '')
+
 
+            

proxypool/setting.py

@@ -1,11 +1,11 @@
 # Redis数据库地址
-REDIS_HOST = 'DataCrawl-Pool.redis.cache.chinacloudapi.cn'
+REDIS_HOST = '127.0.0.1'
 
 # Redis端口
 REDIS_PORT = 6379
 
 # Redis密码，如无填None
-REDIS_PASSWORD = 'A0y1VJ6t9B7R5d6id1/2Rk/aDvZXjdwAR/tWylORuwA='
+REDIS_PASSWORD = None
 
 REDIS_KEY = 'proxies'
 

proxypool/utils.py

@@ -4,7 +4,7 @@ from requests.exceptions import ConnectionError
 base_headers = {
     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36',
     'Accept-Encoding': 'gzip, deflate, sdch',
-    'Accept-Language': 'zh-CN,zh;q=0.8'
+    'Accept-Language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7'
 }