Merge branch 'weixin'

proxypool/crawler.py

@@ -49,7 +49,7 @@ class Crawler(object, metaclass=ProxyMetaclass):
             proxies = result.get('data').get('proxy_list')
             for proxy in proxies:
                 yield proxy
-
+                
     def crawl_daili66(self, page_count=4):
         """
         获取代理66

proxypool/importer.py

@@ -1,5 +1,3 @@
-import requests
-
 from proxypool.db import RedisClient
 
 conn = RedisClient()
@@ -14,10 +12,10 @@ def set(proxy):
 def scan():
     print('请输入代理, 输入exit退出读入')
     while True:
-        account = input()
-        if account == 'exit':
+        proxy = input()
+        if proxy == 'exit':
             break
-        set(account)
+        set(proxy)
 
 
 if __name__ == '__main__':

proxypool/setting.py

@@ -25,7 +25,7 @@ TESTER_CYCLE = 20
 GETTER_CYCLE = 20
 
 # 测试API，建议抓哪个网站测哪个
-TEST_URL = 'https://m.weibo.cn/api/container/getIndex?type=uid&value=2145291155&containerid=1076032145291155&page=14'
+TEST_URL = 'http://weixin.sogou.com/weixin?type=2&query=nba'
 
 # API配置
 API_HOST = '0.0.0.0'
@@ -35,3 +35,6 @@ API_PORT = 5555
 TESTER_ENABLED = True
 GETTER_ENABLED = True
 API_ENABLED = True
+
+# 最大批测试量
+BATCH_TEST_SIZE = 100

proxypool/tester.py

@@ -1,5 +1,6 @@
 import asyncio
 import aiohttp
+import time
 
 try:
     from aiohttp import ClientError
@@ -12,7 +13,7 @@ from proxypool.setting import *
 class Tester(object):
     def __init__(self):
         self.redis = RedisClient()
-
+    
     async def test_single_proxy(self, proxy):
         """
         测试单个代理
@@ -26,17 +27,17 @@ class Tester(object):
                     proxy = proxy.decode('utf-8')
                 real_proxy = 'http://' + proxy
                 print('正在测试', proxy)
-                async with session.get(TEST_URL, proxy=real_proxy, timeout=15) as response:
+                async with session.get(TEST_URL, proxy=real_proxy, timeout=15, allow_redirects=False) as response:
                     if response.status in VALID_STATUS_CODES:
                         self.redis.max(proxy)
                         print('代理可用', proxy)
                     else:
                         self.redis.decrease(proxy)
-                        print('请求响应码不合法，IP', proxy)
+                        print('请求响应码不合法 ', response.status, 'IP', proxy)
             except (ClientError, aiohttp.client_exceptions.ClientConnectorError, asyncio.TimeoutError, AttributeError):
                 self.redis.decrease(proxy)
                 print('代理请求失败', proxy)
-
+    
     def run(self):
         """
         测试主函数
@@ -46,7 +47,10 @@ class Tester(object):
         try:
             proxies = self.redis.all()
             loop = asyncio.get_event_loop()
-            tasks = [self.test_single_proxy(proxy) for proxy in proxies]
-            loop.run_until_complete(asyncio.wait(tasks))
+            for i in range(0, len(proxies), BATCH_TEST_SIZE):
+                test_proxies = proxies[i:i + BATCH_TEST_SIZE]
+                tasks = [self.test_single_proxy(proxy) for proxy in test_proxies]
+                loop.run_until_complete(asyncio.wait(tasks))
+                time.sleep(5)
         except Exception as e:
             print('测试器发生错误', e.args)